"""An exact Riemann solver for the Euler equations with a gamma-law
gas.  The left and right states are stored as State objects.  We then
create a RiemannProblem object with the left and right state:

> rp = RiemannProblem(left_state, right_state)

Next we solve for the star state:

> rp.find_star_state()

Finally, we sample the solution to find the interface state, which
is returned as a State object:

> q_int = rp.sample_solution()
"""

import numpy as np
import scipy.optimize as optimize

class State(object):
    """ a simple object to hold a primitive variable state """

    def __init__(self, p=1.0, u=0.0, rho=1.0):
        self.p = p
        self.u = u
        self.rho = rho

    def __str__(self):
        return "rho: {}; u: {}; p: {}".format(self.rho, self.u, self.p)

class RiemannProblem(object):
    """ a class to define a Riemann problem.  It takes a left
        and right state.  Note: we assume a constant gamma """

    def __init__(self, left_state, right_state, gamma=1.4):
        self.left = left_state
        self.right = right_state
        self.gamma = gamma

        self.ustar = None
        self.pstar = None

    def u_hugoniot(self, p, side):
        """define the Hugoniot curve, u(p)."""

        if side == "left":
            state = self.left
            s = 1.0
        elif side == "right":
            state = self.right
            s = -1.0

        c = np.sqrt(self.gamma*state.p/state.rho)

        if p < state.p:
            # rarefaction
            u = state.u + s*(2.0*c/(self.gamma-1.0))* \
                (1.0 - (p/state.p)**((self.gamma-1.0)/(2.0*self.gamma)))
        else:
            # shock
            beta = (self.gamma+1.0)/(self.gamma-1.0)
            u = state.u + s*(2.0*c/np.sqrt(2.0*self.gamma*(self.gamma-1.0)))* \
                (1.0 - p/state.p)/np.sqrt(1.0 + beta*p/state.p)

        return u

    def find_star_state(self, p_min=0.001, p_max=1000.0):
        """ root find the Hugoniot curve to find ustar, pstar """

        # we need to root-find on
        self.pstar = optimize.brentq(
            lambda p: self.u_hugoniot(p, "left") - self.u_hugoniot(p, "right"),
            p_min, p_max)
        self.ustar = self.u_hugoniot(self.pstar, "left")

    def shock_solution(self, sgn, state):
        """return the interface solution considering a shock"""

        p_ratio = self.pstar/state.p
        c = np.sqrt(self.gamma*state.p/state.rho)

        # Toro, eq. 4.52 / 4.59
        S = state.u + sgn*c*np.sqrt(0.5*(self.gamma + 1.0)/self.gamma*p_ratio +
                                    0.5*(self.gamma - 1.0)/self.gamma)

        # are we to the left or right of the shock?
        if (self.ustar < 0 and S < 0) or (self.ustar > 0 and S > 0):
            # R/L region
            solution = state
        else:
            # * region -- get rhostar from Toro, eq. 4.50 / 4.57
            gam_fac = (self.gamma - 1.0)/(self.gamma + 1.0)
            rhostar = state.rho * (p_ratio + gam_fac)/(gam_fac * p_ratio + 1.0)
            solution = State(rho=rhostar, u=self.ustar, p=self.pstar)

        return solution

    def rarefaction_solution(self, sgn, state):
        """return the interface solution considering a rarefaction wave"""

        # find the speed of the head and tail of the rarefaction fan

        # isentropic (Toro eq. 4.54 / 4.61)
        p_ratio = self.pstar/state.p
        c = np.sqrt(self.gamma*state.p/state.rho)
        cstar = c*p_ratio**((self.gamma-1.0)/(2*self.gamma))

        lambda_head = state.u + sgn*c
        lambda_tail = self.ustar + sgn*cstar

        gam_fac = (self.gamma - 1.0)/(self.gamma + 1.0)

        if (sgn > 0 and lambda_head < 0) or (sgn < 0 and lambda_head > 0):
            # R/L region
            solution = state

        elif (sgn > 0 and lambda_tail > 0) or (sgn < 0 and lambda_tail < 0):
            # * region, we use the isentropic density (Toro 4.53 / 4.60)
            solution = State(rho = state.rho*p_ratio**(1.0/self.gamma),
                             u = self.ustar, p = self.pstar)

        else:
            # we are in the fan -- Toro 4.56 / 4.63
            rho = state.rho * (2/(self.gamma + 1.0) -
                               sgn*gam_fac*state.u/c)**(2.0/(self.gamma-1.0))
            u = 2.0/(self.gamma + 1.0) * ( -sgn*c + 0.5*(self.gamma - 1.0)*state.u)
            p = state.p * (2/(self.gamma + 1.0) -
                           sgn*gam_fac*state.u/c)**(2.0*self.gamma/(self.gamma-1.0))
            solution = State(rho=rho, u=u, p=p)

        return solution

    def sample_solution(self):
        """given the star state (ustar, pstar), find the state on the interface"""

        if self.ustar < 0:
            # we are in the R* or R region
            state = self.right
            sgn = 1.0
        else:
            # we are in the L* or L region
            state = self.left
            sgn = -1.0

        # is the non-contact wave a shock or rarefaction?
        if self.pstar > state.p:
            # compression! we are a shock
            solution = self.shock_solution(sgn, state)

        else:
            # rarefaction
            solution = self.rarefaction_solution(sgn, state)

        return solution

def cons_flux(state, v):
    """ given an interface state, return the conservative flux"""
    flux = np.zeros((v.nvar), dtype=np.float64)

    flux[v.urho] = state.rho * state.u
    flux[v.umx] = flux[v.urho] * state.u + state.p
    flux[v.uener] = (0.5 * state.rho * state.u**2 +
                     state.p/(v.gamma - 1.0) + state.p) * state.u
    return flux

if __name__ == "__main__":

    q_l = State(rho=1.0, u=0.0, p=1.0)
    q_r = State(rho=0.125, u=0.0, p=0.1)

    rp = RiemannProblem(q_l, q_r, gamma=1.4)

    rp.find_star_state()
    q_int = rp.sample_solution()
    print(q_int)


#run with python 3
from primepriv import isprime #importing my isprime function from another file

import math
#Testing if input is valid or not
while True:

	try:

		lower = int(input("lower range: "))

		upper = int(input("upper range: "))
		#if the user puts in valid numbers
		if upper > 0 and lower > 0 and upper > lower:
			#only continue if the user puts in valid input
			break
		else:
			print("Please put in positive numbers and make sure that the upper range is larger than the lower range")

	except:
		#Throw an exception if the users does not put in an input
		print("Please put in numbers")
#semi prime function
def semiprime (upper, lower, primes):

    #array which will hold the semi primes
    semiprimes = []
    #squarert is the square root of the upper value given
    squarert = math.sqrt(upper)
    #iterate from 0 to the amount of primes that we have calculated
    for x in range(0, len(primes)):
    	#if we are at a primes that is larger than the square root of the upper, we can stop calculating (rest of them will be above the given range)
        if (primes[x] > squarert):

            break
        #find the upper range of primes that we need for this particular number and round down so we are inclusive
        upperprimerange = upper // primes[x]
        #find the lower range of primes that we need for this number (add 1 because we need to round up to)
        lowerprimerange = lower // primes[x] + 1
        #iterate through from the current prime to the rest of them
        for i in range(x, len (primes)):
            #if this prime is above the range, we can continue to the next number
            if primes[i] > upperprimerange:

                break
            #if the prime is in the range, calculate the semi prime and add it to the list
            if primes[i] >= lowerprimerange:
                #calculate the semiprime
                product = primes[x] * primes[i]
                semiprimes.append(product)

    return semiprimes
#largestprime is the largest possible prime that we need for this calculation (half of the upper range)
largestprime = math.floor(upper/2)
#primes is a list of the primes that we need in the calculation
primes = isprime(largestprime)#isprime function takes a parameter of the upper range of primes
#call semiprimes function
semiprimes = semiprime(upper, lower, primes)
#sort the semiprimes in ascending order
semiprimes.sort()
#print the semi primes
print("Number of semi primes in range: " + str(len(semiprimes)))
print ("Semi primes in range are: " + str(semiprimes))


from flask import Flask, json, jsonify
import logging
import datetime
##the timestamp
# log format -> "{{TIMESTAMP}}, {{ ENDPOINT_NAME}} endpoint was reached"
current_time = datetime.datetime.now()
TimeStamp=current_time.strftime("%I:%M %p at %d-%B-%Y ")
print(TimeStamp)
## creating logs
logging.basicConfig(filename='app.log',encoding='utf-8', level=logging.DEBUG
                    ,format='%(levelname)s: %(asctime)s ,%(message)s endpoint was reached')
##Multiple route example routes= enpoints
app1 = Flask(__name__)

## data
STATUS={"user": "admin","result": "OK - healthy"}
DATA={"user": "admin","data": {"UserCount": 140, "UserCountActive": 23}}

## creating routes
@app1.route("/") # main route
def hello():
    #message= TimeStamp +","+ "main route "+ "endpoint was reached"
    logging.debug("main route ")
    return "hello world"

@app1.route('/status')
def status():
    message= TimeStamp +","+ "status route "+ "endpoint was reached"
    logging.debug("status route ")
    return jsonify(STATUS)

@app1.route('/status1')
def status1():
    return jsonify(STATUS)

@app1.route('/metrics')
def metrics():
    response=app1.response_class(response=json.dumps(DATA),
                                status=200,
                                mimetype='application/json')
    #message= TimeStamp +","+ "metrics route  "+ "endpoint was reached"
    logging.debug("metrics route  ")
    return response

if __name__ == "__main__":
    app1.run(host='0.0.0.0')
    app1.debug=True ## to make it run once


from plenum.test.cli.helper import createClientAndConnect

import pytest
from stp_core.common.log import Logger

@pytest.fixture(scope="function")
def patch_msg_len(tconf):
    old_value = tconf.MSG_LEN_LIMIT
    tconf.MSG_LEN_LIMIT = 128 * 1024
    yield tconf.MSG_LEN_LIMIT
    print(old_value)
    tconf.MSG_LEN_LIMIT = old_value

def test_error_if_long_message(
        patch_msg_len, cli, tconf, createAllNodes, validNodeNames, set_info_log_level):
    operation = '{{"Hello": "{}"}}'.format("T" * tconf.MSG_LEN_LIMIT)
    createClientAndConnect(cli, validNodeNames, "Alice")

    cli.enterCmd('client {} send {}'.format("Alice", operation))
    assert "Message will be discarded due to InvalidMessageExceedingSizeException" in cli.lastCmdOutput


# -*- coding: utf-8 -*-
# Copyright (c) 2017, Frappe and contributors
# For license information, please see license.txt

from __future__ import unicode_literals
import frappe
from frappe.model.document import Document
from subprocess import check_output

class SiteBackup(Document):
	def autoname(self):
		if self.site_name == None:
			return
		self.name = self.site_name +' '+ self.date +' '+ self.time

	def validate(self):
		if self.get("__islocal"):
			if self.developer_flag == 0:
				frappe.throw("If you want to create a backup, then goto Sites")
			self.developer_flag = 0

	def on_trash(self):
		if self.developer_flag == 0:
			check_output('rm ' + self.file_path + '_database.sql*',
				shell=True, cwd='..')
			if self.public_file_backup:
				check_output('rm ' + self.file_path + '_files.tar',
					shell=True, cwd='..')
			if self.private_file_backup:
				check_output('rm ' + self.file_path + '_private_files.tar',
					shell=True, cwd='..')
			frappe.msgprint('Backup deleted !')
		else:
			pass
			# frappe.msgprint('Deleting the entry but not the Backup')

@frappe.whitelist()
def get_restore_options(doctype, docname):
	return [x['name'] for x in frappe.get_all('Site')]

@frappe.whitelist()
def restore_backup(doctype, docname, on_a_new_site, existing_site, new_site_name, key):
	backup = frappe.get_doc('Site Backup', docname)
	if on_a_new_site == '1':
		exec_str_list = ["bench new-site "+new_site_name]
		str_to_exec = 'bench --site ' + existing_site + ' --force restore ' + backup.file_path + '_database.sql*'
		if backup.public_file_backup:
			str_to_exec += ' --with-public-files ../' + backup.file_path + '_files.tar'
		if backup.private_file_backup:
			str_to_exec += ' --with-private-files ../' + backup.file_path + '_private_files.tar'
		exec_str_list.append(str_to_exec)
		frappe.enqueue('bench_manager.bench_manager.utils.run_command', exec_str_list=exec_str_list, cwd='..', doctype=doctype, key=key, docname=docname, shell=True)
	else:
		str_to_exec = 'bench --site ' + existing_site + ' --force restore ' + backup.file_path + '_database.sql*'
		if backup.public_file_backup:
			str_to_exec += ' --with-public-files ../' + backup.file_path + '_files.tar'
		if backup.private_file_backup:
			str_to_exec += ' --with-private-files ../' + backup.file_path + '_private_files.tar'
		frappe.enqueue('bench_manager.bench_manager.utils.run_command', exec_str_list=[str_to_exec], cwd='..', doctype=doctype, key=key, docname=docname, shell=True)


import argparse
import sys
import time

import matplotlib.cm
import networkx as nx
import numpy as np
from PIL import Image

# TODO:
# Create more efficient graph (less nodes) for scalability. The test is already created

def entrance_and_exit(maze):
    """
    Looks for the single entry and exit positions of the maze.

    Parameters
    ----------
    array : array_like
        The maze as a boolean array, True for walls and False for clear

    Returns
    -------
    entrance : entry position in the array of the maze
    exit : exit position in the array of the maze

    Raises
    ------
    ValueError if the positions do not exist or they are not unique
    """
    entry_and_exit = []
    rows, cols = maze.shape
    top = maze[0, :]
    right = maze[:, -1]
    bottom = maze[-1, :]
    left = maze[:, 0]
    if not np.all(top):
        entry_and_exit.append((0, np.where(top == False)[0][0]))
    if not np.all(right):
        entry_and_exit.append((np.where(right == False)[0][0], rows-1))
    if not np.all(bottom):
        entry_and_exit.append((cols-1, np.where(bottom == False)[0][0]))
    if not np.all(left):
        entry_and_exit.append((np.where(left == False), 0))

    if len(entry_and_exit) != 2:
        raise ValueError("Entrance and exit positions of the maze not found or they are not unique")
    else:
        return entry_and_exit[0], entry_and_exit[1]

def create_graph(maze, entrance_position, exit_position):
    """
    Create a graph by adding nodes in clear cells and establishing edges
    with their direct neighbours (i-1, j), (i+1, j), (i, j-1), (i, j+1).

    Parameters
    ----------
    maze : array_like
        The maze represented as a boolean array, True for wall False for clear
    entrance_position : tuple
        (i, j) position of the entrance to the maze array
    exit_position : tuple
        (i, j) position of the exit to the maze array

    Returns
    -------
    G : networkx Graph
        Graph of the maze
    """
    rows, cols = maze.shape

    # Construct the graph
    G = nx.Graph()
    G.add_node(entrance_position)
    G.add_node(exit_position)

    for row in range(1, rows):
        for col in range(1, cols):
            # Check that we are in a clear cell
            position = (row, col)
            if maze[row, col] == False:
                G.add_node(position)
                # If we have neighbours in the graph, then add edge
                possible_neighbours_in_graph = [(row-1, col), (row, col-1)]
                for u in possible_neighbours_in_graph:
                    if u in G.nodes():
                        G.add_edge(u, position)
    return G

def colour_exit_path(image, path, path_colour):
    """
    Returns the exit path coloured on the image

    Parameters
    ----------
    image : PIL image
        The black and white converted to RGB image
    path : list
        List containing the vertices (nodes) of the path
    path_color : tuple
        Colour for the path

    Raises
    ------
    IndexError if two contiguous vertices in the path correspond to
        different rows and columns (the rows or the columns of a contiguous
        pair must be the same, as it can only move straight)
    """
    image_pixels = image.load()
    for it, vertex in enumerate(path[:-1]):
        row_current, col_current = vertex
        row_forward, col_forward = path[it+1]
        if row_forward - row_current == 0:
            # Advances along columns
            for col in range(col_current, col_forward, 1 if col_current < col_forward else -1):
                # PIL accesses images in Cartesian co-ordinates, so it is Image[columns, rows]
                image_pixels[col, int(row_current)] = path_colour
        elif col_forward - col_current == 0:
            # Advances along rows
            for row in range(row_current, row_forward, 1 if row_current < row_forward else -1):
                # PIL accesses images in Cartesian co-ordinates, so it is Image[columns, rows]
                image_pixels[int(col_current), row] = path_colour
        else:
            raise IndexError("Non contiguous indices: %s %s" % (vertex, path[it+1],))
    row_end, col_end = path[-1]
    image_pixels[int(col_end), int(row_end)] = path_colour
    return image

def main(args):
    time_start = time.time()
    print("Opening the image", "| %f s" % (time.time() - time_start))
    maze_im = Image.open(args.input_file)
    maze_array = np.invert(np.array(maze_im).astype(bool))
    print("Finding the entrance and exit positions", "| %f s" % (time.time() - time_start))
    try:
        node_in, node_out = entrance_and_exit(maze_array)
    except ValueError as e:
        print(e)
        sys.exit(1)
    print("Creating the graph", "| %f s" % (time.time() - time_start))
    G = create_graph(maze_array, node_in, node_out)
    print("Graph created", "| %f s" % (time.time() - time_start))
    print("Finding the solution(s)", "| %f s" % (time.time() - time_start))
    path_quantity = 0
    for path in nx.all_shortest_paths(G, node_in, node_out):
        path_quantity += 1
    print("Creating solution image", "| %f s" % (time.time() - time_start))
    maze_im = maze_im.convert("RGB")
    cmap = matplotlib.cm.get_cmap("jet")
    for path_number, path in enumerate(nx.all_shortest_paths(G, node_in, node_out)):
        r, g, b, _ = cmap(path_number/path_quantity)
        try:
            maze_im = colour_exit_path(maze_im, path, (int(r*255), int(g*255), int(b*255)))
        except IndexError as e:
            print(e)
            sys.exit(1)
    # Save the solution
    maze_im.save(args.output_file)
    print("Image created", "| %f s" % (time.time() - time_start))

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description='Find the exit path in the maze.')
    parser.add_argument("input_file", type=str, help='the maze image')
    parser.add_argument("output_file", type=str, help='the maze coloured output file')
    args = parser.parse_args()
    main(args)


# ------------------------------------------
# NOTEARS (linear)
# ------------------------------------------
#
#
# code shamelessly taken from:
# https://github.com/xunzheng/notears
#

import numpy as np
import torch
import torch.nn as nn
from scipy.special import expit as sigmoid

class LinearNOTEARS:
    def __init__(self, lambda1, loss_type, max_iter=100, h_tol=1e-8, rho_max=1e+16, w_threshold=0.3):
        """
        Args:
            lambda1 (float): l1 penalty parameter
            loss_type (str): l2, logistic, poisson
            max_iter (int): max num of dual ascent steps
            h_tol (float): exit if |h(w_est)| <= htol
            rho_max (float): exit if rho >= rho_max
            w_threshold (float): drop edge if |weight| < threshold
        """
        self.lambda1 = lambda1
        self.loss_type = loss_type
        self.max_iter = max_iter
        self.h_tol = h_tol
        self.rho_max = rho_max
        self.w_threshold = w_threshold
        self.d = None
        self.W_est = None

    def _h(self, W):
        """Evaluate value and gradient of acyclicity constraint."""
        #     E = slin.expm(W * W)  # (Zheng et al. 2018)
        #     h = np.trace(E) - d
        M = np.eye(self.d) + W * W / self.d  # (Yu et al. 2019)
        E = np.linalg.matrix_power(M, self.d - 1)
        h = (E.T * M).sum() - self.d
        G_h = E.T * W * 2
        return h, G_h

    def _loss(self, W, X):
        """Evaluate value and gradient of loss."""
        M = X @ W
        if self.loss_type == 'l2':
            R = X - M
            loss = 0.5 / X.shape[0] * (R ** 2).sum()
            G_loss = - 1.0 / X.shape[0] * X.T @ R
        elif self.loss_type == 'logistic':
            loss = 1.0 / X.shape[0] * (np.logaddexp(0, M) - X * M).sum()
            G_loss = 1.0 / X.shape[0] * X.T @ (sigmoid(M) - X)
        elif self.loss_type == 'poisson':
            S = np.exp(M)
            loss = 1.0 / X.shape[0] * (S - X * M).sum()
            G_loss = 1.0 / X.shape[0] * X.T @ (S - X)
        else:
            raise ValueError('unknown loss type')
        return loss, G_loss

    def _adj(self, w):
        """Convert doubled variables ([2 d^2] array) back to original variables ([d, d] matrix)."""
        return (w[:self.d * self.d] - w[self.d * self.d:]).reshape([self.d, self.d])

    def _func(self, w, X, rho, alpha):
        """Evaluate value and gradient of augmented Lagrangian for doubled variables ([2 d^2] array)."""
        W = self._adj(w)
        loss, G_loss = self._loss(W, X)
        h, G_h = self._h(W)
        obj = loss + 0.5 * rho * h * h + alpha * h + self.lambda1 * w.sum()
        G_smooth = G_loss + (rho * h + alpha) * G_h
        g_obj = np.concatenate((G_smooth + self.lambda1, - G_smooth + self.lambda1), axis=None)
        return obj, g_obj

    def predict_proba(self, data):
        """
        infer the causal direction based on linear implementation of the NOTEARS
        algorithm presented by Zheng et al (2018, NuerIPS).

        Final changes (lines 112 onwards) apply to bivariate measures of causal direciton!

        Solve min_W L(W; data) + lambda1 ‖W‖_1 s.t. h(W) = 0 using augmented Lagrangian.
        """

        n, d = data.shape
        self.d = d
        w_est, rho, alpha, h = np.zeros(2 * d * d), 1.0, 0.0, np.inf  # double w_est into (w_pos, w_neg)
        bnds = [(0, 0) if i == j else (0, None) for _ in range(2) for i in range(d) for j in range(d)]
        for _ in range(self.max_iter):
            w_new, h_new = None, None
            while rho < self.rho_max:
                sol = sopt.minimize(self._func, x0=w_est, args=(data, rho, alpha),
                                    method='L-BFGS-B', jac=True, bounds=bnds)
                w_new = sol.x
                h_new, _ = self._h(self._adj(w_new))
                if h_new > 0.25 * h:
                    rho *= 10
                else:
                    break
            w_est, h = w_new, h_new
            alpha += rho * h
            if h <= self.h_tol or rho >= self.rho_max:
                break
        W_est = self._adj(w_est)
        W_est[np.abs(W_est) < self.w_threshold] = 0
        self.W_est = W_est

        # get measure of causal direction
        R = np.abs(W_est[1, 0]) - np.abs(W_est[0, 1])
        causal_dir = 'x->y' if R < 0 else 'y->x'

        return R, causal_dir

def linear_notears_dir(x, y, lambda1, loss_type, max_iter=100, h_tol=1e-8, rho_max=1e+16, w_threshold=0.3):
    """
    infer the causal direction based on linear implementation of the NOTEARS
    algorithm presented by Zheng et al (2018, NuerIPS).

    Final changes (lines 97 onwards) apply to bivariate measures of causal direciton!

    Solve min_W L(W; X) + lambda1 ‖W‖_1 s.t. h(W) = 0 using augmented Lagrangian.

    Args:
        x (numpy.ndarray): Variable seen as cause
        y (numpy.ndarray): Variable seen as effect
        lambda1 (float): l1 penalty parameter
        loss_type (str): l2, logistic, poisson
        max_iter (int): max num of dual ascent steps
        h_tol (float): exit if |h(w_est)| <= htol
        rho_max (float): exit if rho >= rho_max
        w_threshold (float): drop edge if |weight| < threshold

    Returns:
        W_est (np.ndarray): [d, d] estimated DAG
    """

    # stack observations together
    X = np.vstack((x, y)).T

    def _loss(W):
        """Evaluate value and gradient of loss."""
        M = X @ W
        if loss_type == 'l2':
            R = X - M
            loss = 0.5 / X.shape[0] * (R ** 2).sum()
            G_loss = - 1.0 / X.shape[0] * X.T @ R
        elif loss_type == 'logistic':
            loss = 1.0 / X.shape[0] * (np.logaddexp(0, M) - X * M).sum()
            G_loss = 1.0 / X.shape[0] * X.T @ (sigmoid(M) - X)
        elif loss_type == 'poisson':
            S = np.exp(M)
            loss = 1.0 / X.shape[0] * (S - X * M).sum()
            G_loss = 1.0 / X.shape[0] * X.T @ (S - X)
        else:
            raise ValueError('unknown loss type')
        return loss, G_loss

    def _h(W):
        """Evaluate value and gradient of acyclicity constraint."""
        #     E = slin.expm(W * W)  # (Zheng et al. 2018)
        #     h = np.trace(E) - d
        M = np.eye(d) + W * W / d  # (Yu et al. 2019)
        E = np.linalg.matrix_power(M, d - 1)
        h = (E.T * M).sum() - d
        G_h = E.T * W * 2
        return h, G_h

    def _adj(w):
        """Convert doubled variables ([2 d^2] array) back to original variables ([d, d] matrix)."""
        return (w[:d * d] - w[d * d:]).reshape([d, d])

    def _func(w):
        """Evaluate value and gradient of augmented Lagrangian for doubled variables ([2 d^2] array)."""
        W = _adj(w)
        loss, G_loss = _loss(W)
        h, G_h = _h(W)
        obj = loss + 0.5 * rho * h * h + alpha * h + lambda1 * w.sum()
        G_smooth = G_loss + (rho * h + alpha) * G_h
        g_obj = np.concatenate((G_smooth + lambda1, - G_smooth + lambda1), axis=None)
        return obj, g_obj

    n, d = X.shape
    w_est, rho, alpha, h = np.zeros(2 * d * d), 1.0, 0.0, np.inf  # double w_est into (w_pos, w_neg)
    bnds = [(0, 0) if i == j else (0, None) for _ in range(2) for i in range(d) for j in range(d)]
    for _ in range(max_iter):
        w_new, h_new = None, None
        while rho < rho_max:
            sol = sopt.minimize(_func, w_est, method='L-BFGS-B', jac=True, bounds=bnds)
            w_new = sol.x
            h_new, _ = _h(_adj(w_new))
            if h_new > 0.25 * h:
                rho *= 10
            else:
                break
        w_est, h = w_new, h_new
        alpha += rho * h
        if h <= h_tol or rho >= rho_max:
            break
    W_est = _adj(w_est)
    W_est[np.abs(W_est) < w_threshold] = 0

    # get measure of causal direction
    R = np.abs(W_est[1, 0]) - np.abs(W_est[0, 1])
    predictCausalDir = 'x->y' if R < 0 else 'y->x'

    return R, predictCausalDir, W_est

# ------------------------------------------
# NOTEARS (nonlinear - implemented with MLP)
# ------------------------------------------
#
#
# code shamelessly taken from:
# https://github.com/xunzheng/notears
#

class NotearsMLP(nn.Module):
    def __init__(self, dims, bias=True):
        super(NotearsMLP, self).__init__()
        assert len(dims) >= 2
        assert dims[-1] == 1
        d = dims[0]
        self.dims = dims
        # fc1: variable splitting for l1
        self.fc1_pos = nn.Linear(d, d * dims[1], bias=bias)
        self.fc1_neg = nn.Linear(d, d * dims[1], bias=bias)
        self.fc1_pos.weight.bounds = self._bounds()
        self.fc1_neg.weight.bounds = self._bounds()
        # fc2: local linear layers
        layers = []
        for l in range(len(dims) - 2):
            layers.append(LocallyConnected(d, dims[l + 1], dims[l + 2], bias=bias))
        self.fc2 = nn.ModuleList(layers)

    def _bounds(self):
        d = self.dims[0]
        bounds = []
        for j in range(d):
            for m in range(self.dims[1]):
                for i in range(d):
                    if i == j:
                        bound = (0, 0)
                    else:
                        bound = (0, None)
                    bounds.append(bound)
        return bounds

    def forward(self, x):  # [n, d] -> [n, d]
        x = self.fc1_pos(x) - self.fc1_neg(x)  # [n, d * m1]
        x = x.view(-1, self.dims[0], self.dims[1])  # [n, d, m1]
        for fc in self.fc2:
            x = torch.sigmoid(x)  # [n, d, m1]
            x = fc(x)  # [n, d, m2]
        x = x.squeeze(dim=2)  # [n, d]
        return x

    def h_func(self):
        """Constrain 2-norm-squared of fc1 weights along m1 dim to be a DAG"""
        d = self.dims[0]
        fc1_weight = self.fc1_pos.weight - self.fc1_neg.weight  # [j * m1, i]
        fc1_weight = fc1_weight.view(d, -1, d)  # [j, m1, i]
        A = torch.sum(fc1_weight * fc1_weight, dim=1).t()  # [i, j]
        # h = trace_expm(A) - d  # (Zheng et al. 2018)
        M = torch.eye(d) + A / d  # (Yu et al. 2019)
        E = torch.matrix_power(M, d - 1)
        h = (E.t() * M).sum() - d
        return h

    def l2_reg(self):
        """Take 2-norm-squared of all parameters"""
        reg = 0.
        fc1_weight = self.fc1_pos.weight - self.fc1_neg.weight  # [j * m1, i]
        reg += torch.sum(fc1_weight ** 2)
        for fc in self.fc2:
            reg += torch.sum(fc.weight ** 2)
        return reg

    def fc1_l1_reg(self):
        """Take l1 norm of fc1 weight"""
        reg = torch.sum(self.fc1_pos.weight + self.fc1_neg.weight)
        return reg

    @torch.no_grad()
    def fc1_to_adj(self) -> np.ndarray:  # [j * m1, i] -> [i, j]
        """Get W from fc1 weights, take 2-norm over m1 dim"""
        d = self.dims[0]
        fc1_weight = self.fc1_pos.weight - self.fc1_neg.weight  # [j * m1, i]
        fc1_weight = fc1_weight.view(d, -1, d)  # [j, m1, i]
        A = torch.sum(fc1_weight * fc1_weight, dim=1).t()  # [i, j]
        W = torch.sqrt(A)  # [i, j]
        W = W.cpu().detach().numpy()  # [i, j]
        return W

def squared_loss(output, target):
    n = target.shape[0]
    loss = 0.5 / n * torch.sum((output - target) ** 2)
    return loss

def dual_ascent_step(model, X, lambda1, lambda2, rho, alpha, h, rho_max):
    """Perform one step of dual ascent in augmented Lagrangian."""
    h_new = None
    optimizer = LBFGSBScipy(model.parameters())
    X_torch = torch.from_numpy(X)
    while rho < rho_max:
        def closure():
            optimizer.zero_grad()
            X_hat = model(X_torch)
            loss = squared_loss(X_hat, X_torch)
            h_val = model.h_func()
            penalty = 0.5 * rho * h_val * h_val + alpha * h_val
            l2_reg = 0.5 * lambda2 * model.l2_reg()
            l1_reg = lambda1 * model.fc1_l1_reg()
            primal_obj = loss + penalty + l2_reg + l1_reg
            primal_obj.backward()
            return primal_obj

        optimizer.step(closure)  # NOTE: updates model in-place
        with torch.no_grad():
            h_new = model.h_func().item()
        if h_new > 0.25 * h:
            rho *= 10
        else:
            break
    alpha += rho * h_new
    return rho, alpha, h_new

def notears_nonlinear(model: nn.Module,
                      X: np.ndarray,
                      lambda1: float = 0.,
                      lambda2: float = 0.,
                      max_iter: int = 100,
                      h_tol: float = 1e-8,
                      rho_max: float = 1e+16,
                      w_threshold: float = 0.3):
    rho, alpha, h = 1.0, 0.0, np.inf
    for _ in range(max_iter):
        rho, alpha, h = dual_ascent_step(model, X, lambda1, lambda2,
                                         rho, alpha, h, rho_max)
        if h <= h_tol or rho >= rho_max:
            break
    W_est = model.fc1_to_adj()
    W_est[np.abs(W_est) < w_threshold] = 0
    return W_est

def nonlinear_notears_dir(X):
    """"""
    model = NotearsMLP(dims=[2, 2, 1], bias=True)
    W_est = notears_nonlinear(model, X, lambda1=0.01, lambda2=0.01)
    return W_est

## ----------------------------------------
# HELPERS
## ----------------------------------------
import torch
import scipy.optimize as sopt
import math
import torch.nn as nn

class LBFGSBScipy(torch.optim.Optimizer):
    """Wrap L-BFGS-B algorithm, using scipy routines.

    Courtesy: Arthur Mensch's gist
    https://gist.github.com/arthurmensch/c55ac413868550f89225a0b9212aa4cd
    """

    def __init__(self, params):
        defaults = dict()
        super(LBFGSBScipy, self).__init__(params, defaults)

        if len(self.param_groups) != 1:
            raise ValueError("LBFGSBScipy doesn't support per-parameter options"
                             " (parameter groups)")

        self._params = self.param_groups[0]['params']
        self._numel = sum([p.numel() for p in self._params])

    def _gather_flat_grad(self):
        views = []
        for p in self._params:
            if p.grad is None:
                view = p.data.new(p.data.numel()).zero_()
            elif p.grad.data.is_sparse:
                view = p.grad.data.to_dense().view(-1)
            else:
                view = p.grad.data.view(-1)
            views.append(view)
        return torch.cat(views, 0)

    def _gather_flat_bounds(self):
        bounds = []
        for p in self._params:
            if hasattr(p, 'bounds'):
                b = p.bounds
            else:
                b = [(None, None)] * p.numel()
            bounds += b
        return bounds

    def _gather_flat_params(self):
        views = []
        for p in self._params:
            if p.data.is_sparse:
                view = p.data.to_dense().view(-1)
            else:
                view = p.data.view(-1)
            views.append(view)
        return torch.cat(views, 0)

    def _distribute_flat_params(self, params):
        offset = 0
        for p in self._params:
            numel = p.numel()
            # view as to avoid deprecated pointwise semantics
            p.data = params[offset:offset + numel].view_as(p.data)
            offset += numel
        assert offset == self._numel

    def step(self, closure):
        """Performs a single optimization step.
        Arguments:
            closure (callable): A closure that reevaluates the model
                and returns the loss.
        """
        assert len(self.param_groups) == 1

        def wrapped_closure(flat_params):
            """closure must call zero_grad() and backward()"""
            flat_params = torch.from_numpy(flat_params)
            flat_params = flat_params.to(torch.get_default_dtype())
            self._distribute_flat_params(flat_params)
            loss = closure()
            loss = loss.item()
            flat_grad = self._gather_flat_grad().cpu().detach().numpy()
            return loss, flat_grad.astype('float64')

        initial_params = self._gather_flat_params()
        initial_params = initial_params.cpu().detach().numpy()

        bounds = self._gather_flat_bounds()

        # Magic
        sol = sopt.minimize(wrapped_closure,
                            initial_params,
                            method='L-BFGS-B',
                            jac=True,
                            bounds=bounds)

        final_params = torch.from_numpy(sol.x)
        final_params = final_params.to(torch.get_default_dtype())
        self._distribute_flat_params(final_params)

class LocallyConnected(nn.Module):
    """Local linear layer, i.e. Conv1dLocal() with filter size 1.
    Args:
        num_linear: num of local linear layers, i.e.
        in_features: m1
        out_features: m2
        bias: whether to include bias or not
    Shape:
        - Input: [n, d, m1]
        - Output: [n, d, m2]
    Attributes:
        weight: [d, m1, m2]
        bias: [d, m2]
    """

    def __init__(self, num_linear, input_features, output_features, bias=True):
        super(LocallyConnected, self).__init__()
        self.num_linear = num_linear
        self.input_features = input_features
        self.output_features = output_features

        self.weight = nn.Parameter(torch.Tensor(num_linear,
                                                input_features,
                                                output_features))
        if bias:
            self.bias = nn.Parameter(torch.Tensor(num_linear, output_features))
        else:
            # You should always register all possible parameters, but the
            # optional ones can be None if you want.
            self.register_parameter('bias', None)

        self.reset_parameters()

    @torch.no_grad()
    def reset_parameters(self):
        k = 1.0 / self.input_features
        bound = math.sqrt(k)
        nn.init.uniform_(self.weight, -bound, bound)
        if self.bias is not None:
            nn.init.uniform_(self.bias, -bound, bound)

    def forward(self, input: torch.Tensor):
        # [n, d, 1, m2] = [n, d, 1, m1] @ [1, d, m1, m2]
        out = torch.matmul(input.unsqueeze(dim=2), self.weight.unsqueeze(dim=0))
        out = out.squeeze(dim=2)
        if self.bias is not None:
            # [n, d, m2] += [d, m2]
            out += self.bias
        return out

    def extra_repr(self):
        # (Optional)Set the extra information about this module. You can test
        # it by printing an object of this class.
        return 'num_linear={}, in_features={}, out_features={}, bias={}'.format(
            self.num_linear, self.in_features, self.out_features,
            self.bias is not None
        )


import curses
import time

from life import GameOfLife
from ui import UI

class Console(UI):

    def __init__(self, life: GameOfLife) -> None:
        super().__init__(life)

    def draw_borders(self, screen) -> None:
        """ Отобразить рамку. """
        screen.addch(0, 0, '+')
        screen.addch(self.life.rows + 1, self.life.cols + 1, '+')
        screen.addch(0, self.life.cols + 1, '+')
        screen.addch(self.life.rows + 1, 0, '+')

        for y in range(self.life.rows):
            screen.addch(y + 1, 0, '|')
            screen.addch(y + 1, self.life.cols + 1, '|')

        for x in range(self.life.cols):
            screen.addch(0, x + 1, '-')
            screen.addch(self.life.rows + 1, x + 1, '-')

    def draw_grid(self, screen) -> None:
        """ Отобразить состояние клеток. """
        for y in range(self.life.rows):
            for x in range(self.life.cols):
                if self.life.curr_generation[y][x] == 0:
                    screen.addch(y + 1, x + 1, ' ')
                else:
                    screen.addch(y + 1, x + 1, '*')

    def run(self) -> None:
        screen = curses.initscr()
        self.draw_borders(screen)
        screen.refresh()
        while self.life.is_max_generations_exceeded and self.life.is_changing:
            self.draw_grid(screen)
            screen.refresh()
            self.life.step()
            time.sleep(0.01)
        curses.endwin()


# -*- coding: utf8 -*-

from datetime import datetime
from scrapy.http import Request
from alascrapy.spiders.base_spiders.ala_spider import AlaSpider
import alascrapy.lib.dao.incremental_scraping as incremental_utils
from alascrapy.items import ProductItem, ReviewItem, ProductIdItem
import dateparser

class LesmobilesSpider(AlaSpider):
    name = 'lesmobiles'
    allowed_domains = ['lesmobiles.com']
    page_number = 1
    base_url = 'https://www.lesmobiles.com/tests/telephones-mobiles.html?p={}'
    start_urls = [base_url.format(page_number)]

    def __init__(self, *args, **kwargs):
        super(LesmobilesSpider, self).__init__(self, *args, **kwargs)
        self.stored_last_date = incremental_utils.get_latest_pro_review_date(
            self.mysql_manager, self.spider_conf["source_id"])

        # In order to test another stored_last_date
        self.stored_last_date = datetime(2017, 2, 8)

    def parse(self, response):
        # print " 	...PARSE: " + response.url
        # print " --self.stored_last_date: " + str(self.stored_last_date)

        review_articles_xpath = '//article[@class="clearfix"]'
        review_articles = response.xpath(review_articles_xpath)

        date = None
        for r_art in review_articles:
            date_xpath = './/time//text()'
            date = r_art.xpath(date_xpath).get()
            # date looks like: '19 décembre 2018 - 09h00'
            date = date.split('-')[0]
            date = date.strip()
            date = dateparser.parse(date, date_formats=['%d %B %Y'])

            if date > self.stored_last_date:
                review_url_xpath = './a/@href'
                review_url = r_art.xpath(review_url_xpath).get()

                yield response.follow(url=review_url,
                                      callback=self.parse_product_review,
                                      meta={'date': date.strftime("%Y-%m-%d")})

        # Checking whether we should scrape the next page
        if date and date > self.stored_last_date:
            self.page_number += 1
            yield Request(url=self.base_url.format(self.page_number),
                          callback=self.parse)

    def get_product_name_based_on_title(self, title):
        ''' This function will 'clean' the title of the review
            in order to get the product name '''
        p_name = title

        # Spliting title
        get_first_piece = [u':']

        for g_f in get_first_piece:
            if g_f in p_name:
                p_name = p_name.split(g_f)[0]

        # Removing certain words
        words_to_remove = ['Test du', u'Test de l’', u"Test de l'", 'Test de',
                           'Prise en main du']
        for w in words_to_remove:
            if w in title:
                p_name = p_name.replace(w, '')

        # Removing unnecessary spaces:
        p_name = p_name.replace('  ', ' ')
        p_name = p_name.strip()

        return p_name

    def parse_product_review(self, response):
        # print "     ...PARSE_PRODUCT_REVIEW: " + response.url

        # REVIEW ITEM -----------------------------------------------------
        review_xpaths = {
            'TestTitle': '//meta[@property="og:title"]/@content',
            'TestSummary': '//meta[@property="og:description"]/@content',
        }

        # Create the review
        review = self.init_item_by_xpaths(response, "review", review_xpaths)

        # 'ProductName'
        title = review['TestTitle']
        review['ProductName'] = self.get_product_name_based_on_title(title)

        # 'Author'
        author_xpath = '//p[@itemprop="author"]//text()'
        author = response.xpath(author_xpath).get()
        # author looks like: Par Samir Azzemou
        if 'Par' in author:
            author = author.replace('Par', '')
            author = author.strip()
        review['Author'] = author

        # 'TestDateText'
        review['TestDateText'] = response.meta.get('date')

        # 'SourceTestScale' and 'SourceTestRating'
        rating_xpath = '//span[@itemprop="ratingValue"]//text()'
        rating = response.xpath(rating_xpath).get()
        scale_xpath = '//span[@itemprop="bestRating"]//text()'
        scale = response.xpath(scale_xpath).get()
        if rating and scale:
            review['SourceTestRating'] = rating
            review['SourceTestScale'] = scale

        # 'DBaseCategoryName'
        review['DBaseCategoryName'] = 'PRO'

        # 'source_internal_id'
        sid = response.url
        # sid looks like: https://www.lesmobiles.com/test/apple-iphone-xs-64go
        sid = sid.split('/')[-1]
        review['source_internal_id'] = sid
        # -----------------------------------------------------------------

        # PRODUCT ITEM ----------------------------------------------------
        product = ProductItem()
        product['source_internal_id'] = review['source_internal_id']
        product['OriginalCategoryName'] = 'Smartphone'
        product['ProductName'] = review['ProductName']

        pic_url_xpath = '//meta[@property="og:image"]/@content'
        pic_url = response.xpath(pic_url_xpath).get()
        product['PicURL'] = pic_url

        product['TestUrl'] = response.url
        # -----------------------------------------------------------------

        # PRICE ITEM ------------------------------------------------------
        price_xpath = '//p[@class="price"]//text()'
        price = response.xpath(price_xpath).get()
        if price:
                price = price.strip()
                yield ProductIdItem.from_product(product,
                                                 kind='price',
                                                 value=price
                                                 )
        # -----------------------------------------------------------------

        yield review
        yield product


import math
import numpy as np
import matplotlib.pyplot as plt

''' this func. takes INPUT: location
            and give OUTPUT: joints' angles
the calculations based on the inverse kinematics, three angles for three joints respectively '''

def joints_angles(x, y, z):
    # the func gets point and return the angles of the joints

    l0 = 40
    l1 = 44
    l2 = 108
    l3 = 60
    l4 = 31.5
    z2 = z - (l0 + l1)
    d = math.sqrt(z2 ** 2 + y ** 2 + x ** 2)

    # Need to a add cases for x = 0**
    # teta1 = math.atan(y / x)
    teta1 = math.atan2(y, x) + math.pi/2
    arg = ((d ** 2 - (l3 ** 2 + l2 ** 2)) / (2 * l2 * l3))
    '''if arg > 1:
        teta3 = math.acos(0.99)
    elif arg < -1:
        teta3 = math.acos(-0.99)
    else:'''
    teta3 = math.acos((d ** 2 - (l3 ** 2 + l2 ** 2)) / (2 * l2 * l3))
    teta2 = math.atan2(math.sqrt(y ** 2 + x ** 2), z2) - math.asin((l3 * math.sin(teta3)) / d)

    teta1_deg = teta1 * 180 / math.pi
    teta2_deg = teta2 * 180 / math.pi
    teta3_deg = teta3 * 180 / math.pi
    '''print(teta1_deg)
    print(teta2_deg)
    print(teta3_deg)'''

    """teta1 = math.pi/10 + math.pi/2
    teta2 = math.pi/10
    teta3 = math.pi/10"""

    '''teta1 = 104.03/180.0*math.pi + math.pi/2.0
    teta2 = (89.56)/180.0*math.pi
    teta3 = (95.20)/180.0*math.pi'''

    return teta1, teta2, teta3

# joints_angles(-25, 40, -25)


# -*- coding: utf-8 -*-
"""
Created on Mon Feb 20 18:51:24 2017

@author: Rodion
"""
import logging

logging.basicConfig(level=logging.ERROR)

def cross(A, B):
    "Cross product of elements in A and elements in B."
    return [a+b for a in A for b in B]

assignments = []
rows = 'ABCDEFGHI'
cols = '123456789'
boxes = cross(rows, cols)
row_units = [cross(r, cols) for r in rows]
column_units = [cross(rows, c) for c in cols]
square_units = [cross(rs, cs) for rs in ('ABC','DEF','GHI') for cs in ('123','456','789')]
diagonal_units = [[r+c for r,c in zip(rows,cols)], [r+c for r,c in zip(rows,cols[::-1])]]
unitlist = row_units + column_units + square_units + diagonal_units
units = dict((s, [u for u in unitlist if s in u]) for s in boxes)
peers = dict((s, set(sum(units[s],[]))-set([s])) for s in boxes)


import csv
import pyrankvote as pv
import re

def ranked_vote(table, places):
    themes = table[0]
    themes = [re.search(r'\[(.*)\]$', theme)[1] for theme in themes]
    themes = [pv.Candidate(theme) for theme in themes]

    print('Themes:')
    for theme in themes: print(f'\t{theme.name}')
    print()

    choices_text = ['1st Choice', '2nd Choice', '3rd Choice'] + [str(i) + 'th Choice' for i in range(4,len(themes)+1)]

    raw_votes = [line[1:-1] for line in lines[1:]]
    votes = []
    for raw_vote in raw_votes:
        vote = []
        for ct in choices_text:
            try:
                i = raw_vote.index(ct)
                vote.append(themes[i])
            except ValueError: pass
        votes.append(vote)

    print('Votes:')
    for i, vote in enumerate(votes):
        print(f'\t{i+1}.', end='')
        print(', '.join([choice.name for choice in vote]))
    print()

    ballots = [pv.Ballot(ranked_candidates=vote) for vote in votes]

    election_result = pv.single_transferable_vote(themes, ballots, places)

    print(election_result)

with open('results.csv') as f:
    lines = list(csv.reader(f))

Q1 = [line[1:-3] for line in lines]

ranked_vote(Q1, 4)


# -*- coding: utf-8 -*-
from django.conf.urls import patterns, url
from django.contrib.auth import views

urlpatterns = patterns(
    '',
    url(r'^$', 'apps.hello.views.view_contact', name='index'),
    url(r'^requests/', 'apps.hello.views.priority_requests', name='requests'),
    url(r'^requests_count/', 'apps.hello.views.requests_count',
        name='requests_count'),
    url(r'^priority_requests/', 'apps.hello.views.priority_requests',
        name='priority_requests'),
    url(r'^change_priority/', 'apps.hello.views.change_priority',
        name='change_priority'),
    url(r'^login/', views.login, name='login'),
    url(r'^logout/', views.logout, {'next_page': '/'}, name='logout'),
    url(r'^edit_person/', 'apps.hello.views.edit_person', name='edit_person'),
)


# 49. Group Anagrams

# Given an array of strings, group anagrams together.

# Example:

# Input: ["eat", "tea", "tan", "ate", "nat", "bat"],
# Output:
# [
#   ["ate","eat","tea"],
#   ["nat","tan"],
#   ["bat"]
# ]
# Note:

# All inputs will be in lowercase.
# The order of your output does not matter.
from typing import List, Tuple
from string import ascii_lowercase

def string2count(raw: str) -> Tuple[int]:
    """
    convert string to tuple Int
    """
    count = [0 for _ in range(26)]
    for s in raw:
        count[ord(s) - ord('a')] += 1
    return tuple(count)

class Solution:
    def groupAnagrams(self, strs: List[str]) -> List[List[str]]:
        result = []
        keys = []
        for s in strs:
            key = string2count(s)
            if key in keys:
                result[keys.index(key)].append(s)
            else:
                keys.append(key)
                result.append([s])
        return result

if __name__ == '__main__':
    from util import Test
    t = Test(Solution().groupAnagrams)
    t.equal([], ["eat", "tea", "tan", "ate", "nat", "bat"])


#!/usr/bin/env /data/mta/Script/Python3.8/envs/ska3-shiny/bin/python

#################################################################################
#                                                                               #
#       clean_table.py: sort and cleaned data tables in <data_dir>/Results      #
#                                                                               #
#           author: t. isobe(tisobe@cfa.harvard.edu)                            #
#                                                                               #
#           Last Update:    Mar 02, 2021                                        #
#                                                                               #
#################################################################################

import os
import sys
import re
import string
import random
import operator
import time
#
#--- reading directory list
#
path = '/data/mta/Script/ACIS/CTI/house_keeping/dir_list_py'

with open(path, 'r') as f:
    data = [line.strip() for line in f.readlines()]

for ent in data:
    atemp = re.split(':', ent)
    var   = atemp[1].strip()
    line  = atemp[0].strip()
    exec("%s = %s" %(var, line))
#
#--- append  pathes to private folders to a python directory
#
sys.path.append(bin_dir)
sys.path.append(mta_dir)
#
#--- import several functions
#
import mta_common_functions as mcf   #---- contains other functions commonly used in MTA scripts
#
#--- temp writing file name
#
rtail  = int(time.time() * random.random())
zspace = '/tmp/zspace' + str(rtail)

#---------------------------------------------------------------------------
#-- clean_table: sort and clean the table data in <data_dir>/Results      --
#---------------------------------------------------------------------------

def clean_table():
    """
    sort and clean the table data in <data_dir>/Results
    Input:  none but read from <data_dir>/Results/<elm>_ccd<ccd#>
    Output: cleaned up <data_dir>/Results/<elm>_ccd<ccd#>
    """
#
#--- make a backup
#
    out  = time.strftime('%m%d%y', time.gmtime())
    cout = data_dir + '/Results/Save_' + out

    cmd  = 'rm -rf ' + cout
    os.system(cmd)
    cmd  = 'mkdir -p '  + cout
    os.system(cmd)
    cmd  = 'cp -f  ' + data_dir + '/Results/*_ccd* ' + cout + '/.'
    os.system(cmd)
#
#--- now clean up the data
#
    cmd = 'ls ' + data_dir + '/Results/*_ccd* > ' + zspace
    os.system(cmd)

    flist = mcf.read_data_file(zspace, remove=1)

    for ifile in flist:
        data = mcf.read_data_file(ifile)
        data.sort()

        prev    = data[0]
        cleaned = [prev]

        for ent in data:
            if ent == prev:
                prev = ent
            else:
                cleaned.append(ent)
                prev = ent

        with  open(ifile, 'w') as fo:
            for ent in cleaned:
                line = ent + '\n'
                fo.write(line)

#--------------------------------------------------------------------

if __name__ == '__main__':

    clean_table()


import re
import numpy as np
import pandas as pd
import support
from sklearn.model_selection import KFold, cross_validate
from sklearn.svm import SVC, SVR
from sklearn.gaussian_process import GaussianProcessClassifier, GaussianProcessRegressor
from sklearn.neighbors import KNeighborsClassifier, KNeighborsRegressor
from sklearn.neural_network import MLPClassifier, MLPRegressor

"""Training Dataset as:
https://archive.ics.uci.edu/ml/datasets/iris
https://archive.ics.uci.edu/ml/datasets/Connectionist+Bench+%28Sonar%2C+Mines+vs.+Rocks%29
https://archive.ics.uci.edu/ml/datasets/Glass+Identification
https://archive.ics.uci.edu/ml/datasets/Airfoil+Self-Noise
https://archive.ics.uci.edu/ml/datasets/Wine+Quality
"""

if __name__ == '__main__':

    # ベンチマークとなるアルゴリズムと、アルゴリズムを実装したモデルの一覧
    models = [
        ('SVM', SVC(random_state=1), SVR()),
        ('GaussianProcess', GaussianProcessClassifier(random_state=1),
         GaussianProcessRegressor(normalize_y=True, alpha=1, random_state=1)),
        ('KNeighbors', KNeighborsClassifier(), KNeighborsRegressor()),
        ('MLP', MLPClassifier(random_state=1),
         MLPRegressor(hidden_layer_sizes=(5),solver='lbfgs', random_state=1))
    ]

    # 検証用データセットのファイルと、ファイルの区切り文字、ヘッダーとなる行の位置、インデックスとなる列の位置のリスト
    classifier_files = ['./data/iris.data', './data/sonar.all-data', './data/glass.data']
    classifier_params = [(',', None, None), (',', None, None), (',', None, 0)]
    regressor_files = ['./data/airfoil_self_noise.dat', './data/winequality-red.csv', './data/winequality-white.csv']
    regressor_params = [(r'\t', None, None), (';', 0, None), (';', 0, None)]

    # 評価スコアを検証用データセットのファイル、アルゴリズムごとに保存する表
    result = pd.DataFrame(columns = ['target', 'function'] + [m[0] for m in models],
                          index = range(len(classifier_files+regressor_files) * 2))

    # 最初にクラス分類アルゴリズムを評価
    ncol = 0

    for i, (c,p) in enumerate(zip(classifier_files, classifier_params)):
        df = pd.read_csv(c, sep=p[0], header=p[1], index_col=p[2])
        x = df[df.columns[:-1]].values
        y, clz = support.clz_to_prob(df[df.columns[:-1]])

        result.loc[ncol, 'target'] = re.split(r'[._]', re.split('/', c)[-1])[0]
        result.loc[ncol+1, 'target'] = ''
        result.loc[ncol, 'function'] = 'F1Score'
        result.loc[ncol+1, 'function'] = 'Accuracy'

        # アルゴリズムの評価
        for l, c_m, r_m in models:
            kf = KFold(n_splits=5, random_state=1, shuffle=True)
            s = cross_validate(c_m, x, y.argmax(axis=1), cv=kf, scoring = ('f1_weighted', 'accuracy'))
            result.loc[ncol, l] = np.mean(s['test_f1_weighted'])
            result.loc[ncol+1, l] = np.mean(s['test_accuracy'])

        ncol += 2

    # 次に回帰アルゴリズムを評価
    for i, (c,p) in enumerate(zip(regressor_files, regressor_params)):
        df = pd.read_csv(c, sep=p[0], header=p[1], index_col=p[2])
        x = df[df.columns[:-1]].values
        y = df[df.columns[-1]].values.reshape((-1,))

        result.loc[ncol, 'target'] = re.split(r'[._]', re.split('/', c)[-1])[0]
        result.loc[ncol + 1, 'target'] = ''
        result.loc[ncol, 'function'] = 'R2Score'
        result.loc[ncol + 1, 'function'] = 'MeanSquared'

        # アルゴリズムの評価
        for l, c_m, r_m in models:
            kf = KFold(n_splits=5, random_state=1, shuffle=True)
            s = cross_validate(r_m, x, y, cv=kf, scoring=('r2', 'neg_mean_squared_error'))
            result.loc[ncol, l] = np.mean(s['test_r2'])
            result.loc[ncol + 1, l] = -np.mean(s['test_neg_mean_squared_error'])

        ncol += 2

    print(result)
    result.to_csv('baseline.csv', index=None)


# 先定位到想要點擊的地方，再用ActionChains模擬點擊

from selenium import webdriver
from selenium.webdriver.common.action_chains import ActionChains
import time

url = "https://www.python.org/"

driver = webdriver.Chrome()
driver.implicitly_wait(10)
driver.get(url)

menu = driver.find_element_by_css_selector("#about > a")
item = driver.find_element_by_css_selector("#about > ul > li.tier-2.element-1 > a")

actions = ActionChains(driver)
actions.move_to_element(menu)
actions.click(item)
actions.perform()
time.sleep(5)

driver.quit()


# -*- coding: utf-8 -*-
from __future__ import division

p=input('digite p:')
q=input('digite q:')

cont=0
a=p
while p>0:
    p=p//10
    cont=cont+1
p=a
sub=0
while q>0:
    ulti=q%(10**cont)
    if ulti==p:
        sub=bub+1
        break
    else:
        q=q//10

    if sub==0:
        print('N')
    else:
        print('S')


from django.conf.urls import url
from . import views
urlpatterns = [
	url(r'^$', views.deck_choice, name='deck_choice'),
	url(r'^(?P<deck_slug>[\w-]+)/$', views.deck_details, name='deck_details'),
	url(r'^(?P<deck_slug>[\w-]+)/add_question/$', views.add_question, name='add_question'),
	url(r'^(?P<deck_slug>[\w-]+)/study/$', views.question_in_deck, name='question_in_deck'),
	url(r'^(?P<deck_slug>[\w-]+)/question/remember$', views.remember, name='remember'),
	url(r'^create-deck$', views.create_deck, name='create_deck'),
	url(r'^deck/question/edit$', views.edit_question, name="edit_question"),

]


import sys
import re
import pickle
import pandas as pd
from nltk import ngrams
import argparse
import datefinder
from tqdm import tqdm

"""
FEATURES:

raw_text:               The raw text

processed_text:         The raw text of the last word in the N-gram

text_pattern:           The raw text, after replacing uppercase characters with X,
                        lowercase with x, numbers with 0, repeating whitespace with
                        single whitespace and the rest with ?

bottom_margin:          Vertical coordinate of the bottom margin of the
                        N-gram normalized to the page height

top_margin:             Vertical coordinate of the top margin of the
                        N-gram normalized to the page height

right_margin:           Horizontal coordinate of the right margin of the
                        N-gram normalized to the page width

left_margin:            Horizontal coordinate of the left margin of the
                        N-gram normalized to the page width

has_digits:             Whether there are any digits 0-9 in the N-gram

length:                 Number of characters in the N-gram

position_on_line:       Count of words to the left of this N-gram normalized
                        to the count of total words on this line

line_size:              The number of words on this line

page_height:            The height of the page of this N-gram

page_width:             The width of the page of this N-gram

parses_as_amount:       Whether the N-gram parses as a fractional amount

parses_as_date:         Whether the N-gram parses as a date

parses_as_number:       Whether the N-gram parses as an integer
"""

def ngrammer(tokens, length=4):
    """
    Generates n-grams from the given tokens
    :param tokens: list of tokens in the text
    :param length: n-grams of up to this length
    :return: n-grams as tuples
    """
    for n in range(1, min(len(tokens) + 1, length+1)):
        for gram in ngrams(tokens, n):
            yield gram

def extract_features(path):
    """
    Loads a pickled dataframe from the given path, creates n-grams and extracts features
    :param path: path to pickled dataframe
    :return: dataframe containing n-grams and corresponding features
    """

    with open(path, 'rb') as pklfile:
        df = pickle.load(pklfile)

    files = {}

    print("\nExtracting features...\n")

    # Filters the data into individual files and finds out the minimum and maximum
    # x and y coordinates to estimate the width and height of each file.
    # Also estimates the x coordinate for each token in each line for every file.
    for i, row in df.iterrows():
        if row['files'] not in files:
            files[row['files']] = {'lines': {'words': [], 'labels': [], 'ymin': [], 'ymax': []},
                                   'xmin': sys.maxsize, 'ymin': sys.maxsize, 'xmax': 0, 'ymax': 0}
        tokens = row['words'].strip().split(' ')
        char_length = (row['coords'][2] - row['coords'][0]) / len(row['words'].strip())
        token_coords = [{'xmin': row['coords'][0],
                         'xmax': row['coords'][0] + (char_length * len(tokens[0]))}]
        for idx in range(1, len(tokens)):
            token_coords.append({'xmin': token_coords[-1]['xmax'] + char_length,
                                 'xmax': token_coords[-1]['xmax'] + (char_length * (len(tokens[idx])+1))})
        files[row['files']]['lines']['words'].append({'tokens': tokens, 'coords': token_coords})
        files[row['files']]['lines']['labels'].append(row['labels'])
        files[row['files']]['lines']['ymin'].append(row['coords'][1])
        files[row['files']]['lines']['ymax'].append(row['coords'][3])
        files[row['files']]['xmin'] = min(files[row['files']]['xmin'], row['coords'][0])
        files[row['files']]['ymin'] = min(files[row['files']]['ymin'], row['coords'][1])
        files[row['files']]['xmax'] = max(files[row['files']]['xmax'], row['coords'][2])
        files[row['files']]['ymax'] = max(files[row['files']]['ymax'], row['coords'][3])

    del df

    grams = {'raw_text': [],
             'processed_text': [],
             'text_pattern': [],
             'length': [],
             'line_size': [],
             'position_on_line': [],
             'has_digits': [],
             'bottom_margin': [],
             'top_margin': [],
             'left_margin': [],
             'right_margin': [],
             'page_width': [],
             'page_height': [],
             'parses_as_amount': [],
             'parses_as_date': [],
             'parses_as_number': [],
             'label': [],
             'closest_ngrams': []
             }

    label_dict = {0: 0, 1: 1, 2: 2, 18: 3}

    # Calculates N-grams of lengths ranging from 1-4 for each line in each
    # file and calculates 17 features for each N-gram.
    with tqdm(total=len(files)) as pbar:
        for key, value in files.items():
            num_ngrams = len(grams['raw_text'])
            page_height = value['ymax'] - value['ymin']
            page_width = value['xmax'] - value['xmin']
            for i in range(len(value['lines']['words'])):
                tokens = value['lines']['words'][i]['tokens']
                token_coords = value['lines']['words'][i]['coords']
                for ngram in ngrammer(tokens):
                    grams['parses_as_date'].append(0.0)
                    grams['parses_as_amount'].append(0.0)
                    grams['parses_as_number'].append(0.0)
                    processed_text = []
                    for word in ngram:
                        if bool(list(datefinder.find_dates(word))):
                            processed_text.append('date')
                            grams['parses_as_date'][-1] = 1.0
                        elif bool(re.search(r'\d\.\d', word)) or '$' in word:
                            processed_text.append('amount')
                            grams['parses_as_amount'][-1] = 1.0
                        elif word.isnumeric():
                            processed_text.append('number')
                            grams['parses_as_number'][-1] = 1.0
                        else:
                            processed_text.append(word.lower())
                    raw_text = ' '.join(ngram)
                    grams['raw_text'].append(raw_text)
                    grams['processed_text'].append(' '.join(processed_text))
                    grams['text_pattern'].append(re.sub('[a-z]', 'x', re.sub('[A-Z]', 'X', re.sub('\d', '0', re.sub(
                        '[^a-zA-Z\d\ ]', '?', raw_text)))))
                    grams['length'].append(len(' '.join(ngram)))
                    grams['line_size'].append(len(tokens))
                    grams['position_on_line'].append(tokens.index(ngram[0])/len(tokens))
                    grams['has_digits'].append(1.0 if bool(re.search(r'\d', raw_text)) else 0.0)
                    grams['left_margin'].append((token_coords[tokens.index(ngram[0])]['xmin'] - value['xmin']) / page_width)
                    grams['top_margin'].append((value['lines']['ymin'][i] - value['ymin']) / page_height)
                    grams['right_margin'].append((token_coords[tokens.index(ngram[-1])]['xmax'] - value['xmin']) / page_width)
                    grams['bottom_margin'].append((value['lines']['ymax'][i] - value['ymin']) / page_height)
                    grams['page_width'].append(page_width)
                    grams['page_height'].append(page_height)
                    grams['label'].append(label_dict[value['lines']['labels'][i]])

            # Finds the closest N-grams on all 4 sides for each N-gram
            for i in range(num_ngrams, len(grams['raw_text'])):
                grams['closest_ngrams'].append([-1] * 4)
                distance = [sys.maxsize] * 6
                for j in range(num_ngrams, len(grams['raw_text'])):
                    d = [grams['top_margin'][i] - grams['bottom_margin'][j],
                         grams['top_margin'][j] - grams['bottom_margin'][i],
                         grams['left_margin'][i] - grams['right_margin'][j],
                         grams['left_margin'][j] - grams['right_margin'][i],
                         abs(grams['left_margin'][i] - grams['left_margin'][j])]
                    if i == j:
                        continue
                    # If in the same line, check for closest ngram to left and right
                    if d[0] == d[1]:
                        if distance[2] > d[2] > 0:
                            distance[2] = d[2]
                            grams['closest_ngrams'][i][2] = j
                        if distance[3] > d[3] > 0:
                            distance[3] = d[3]
                            grams['closest_ngrams'][i][3] = j
                    # If this ngram is above current ngram
                    elif distance[0] > d[0] >= 0 and distance[4] > d[4]:
                        distance[0] = d[0]
                        distance[4] = d[4]
                        grams['closest_ngrams'][i][0] = j
                    # If this ngram is below current ngram
                    elif distance[1] > d[1] >= 0 and distance[5] > d[4]:
                        distance[1] = d[1]
                        distance[5] = d[4]
                        grams['closest_ngrams'][i][1] = j

            pbar.update(1)

    return pd.DataFrame(data=grams)

def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--data", default="data/dftrain.pk", help="path to training data")
    ap.add_argument("--save_as", default="data/features.pk", help="save extracted features with this name")
    args = ap.parse_args()
    features = extract_features(args.data)
    features.to_pickle(args.save_as, protocol=3)
    print("\nSaved features as {}".format(args.save_as))

if __name__ == '__main__':
    main()


"""Classes and functions for Word analysis (text analysis of abstract texts)."""

import datetime
from bs4 import BeautifulSoup
import nltk
from nltk.corpus import stopwords

# Import custom code
from erpsc.base import Base
from erpsc.erp_data import ERPData
from erpsc.core.urls import URLS
from erpsc.core.utils import CatchNone, CatchNone2, comb_terms, extract

#################################################################################################
#################################### ERPSC - WORDS - Classes ####################################
#################################################################################################

class Words(Base):
    """Class for searching through words in the abstracts of specified papers.

    Attributes
    ----------
    result_keys : list of str
        Keys for each result data attached to object.
    results : list of ERPData() objects
        Results for each ERP, stored in custom Words object.
    """

    def __init__(self):
        """Initialize ERP-SCANR Words() object."""

        # Inherit from ERPSC Base Class
        Base.__init__(self)

        # Initialize a list to store results for all the erps
        self.result_keys = list()
        self.results = list()

    def __getitem__(self, key):
        """Index into Words object with ERP result key."""

        # Give up if object is empty
        if len(self.result_keys) == 0:
            raise IndexError('Object is empty - cannot index.')

        # Check if requested key is available
        try:
            ind = self.result_keys.index(key)
        except ValueError:
            raise IndexError('Requested key not available in object.')

        return self.results[ind]

    def add_results(self, new_result):
        """Add a new ERPData results object.

        Parameters
        ----------
        new_result : ERPData() object
            Object with information about current ERP term.
        """

        self.result_keys.append(new_result.label)
        self.results.append(new_result)

    @staticmethod
    def extract_add_info(cur_erp, new_id, art):
        """Extract information from article web page and add to

        Parameters
        ----------
        cur_erp : ERPData() object
            Object to store information for the current ERP term.
        new_id : int
            Paper ID of the new paper.
        art : bs4.element.Tag() object
            Extracted pubmed article.

        NOTES
        -----
        - Data extraction is all in try/except statements in order to
        deal with missing data, since fields may be missing.
        """

        # Add ID of current article
        cur_erp.add_id(new_id)
        cur_erp.add_title(extract(art, 'ArticleTitle', 'str'))
        cur_erp.add_authors(_process_authors(extract(art, 'AuthorList', 'raw')))
        cur_erp.add_journal(extract(art, 'Title', 'str'), extract(art, 'ISOAbbreviation', 'str'))
        cur_erp.add_words(_process_words(extract(art, 'AbstractText', 'str')))
        cur_erp.add_kws(_process_kws(extract(art, 'Keyword', 'all')))
        cur_erp.add_pub_date(_process_pub_date(extract(art, 'PubDate', 'raw')))
        cur_erp.add_doi(_process_ids(extract(art, 'ArticleId', 'all'), 'doi'))

        # Increment number of articles included in ERPData
        cur_erp.increment_n_articles()

        return cur_erp

    def scrape_data(self, db=None, retmax=None, use_hist=False, verbose=False):
        """Search through pubmed for all abstracts referring to a given ERP.

        The scraping does an exact word search for the ERP term given.
        It then loops through all the artciles found about that data.
        For each article, pulls title, year and word data.

        Notes
        -----
        - Pulls data using the hierarchical tag structure that organize the articles.
        - Initially, the procedure was to pull all tags of a certain type.
            For example: extract all 'DateCreated' tags.
            This procedure fails (or badly organizes data) when an articles is
                missing a particular tag.
            Now: take advantage of the hierarchy, loop through each article tag.
                From here, pull out the data, if available.
                This way, can deal with cases of missing data.
        """

        # Set date of when data was collected
        self.date = datetime.datetime.now().strftime("%Y-%m-%d_%H:%M:%S")

        # Get e-utils URLS object
        if use_hist: hist_val = 'y'
        else: hist_val = 'n'
        urls = URLS(db=db, usehistory=hist_val, retmax=retmax, retmode='xml', field='TIAB', auto_gen=False)
        urls.build_info(['db'])
        urls.build_search(['db', 'usehistory', 'retmax', 'retmode', 'field'])
        urls.build_fetch(['db', 'retmode'])

        # Get current information about database being used
        self.get_db_info(urls.info)

        # Loop through all the erps
        #for ind, erp in enumerate(self.erps):
        for ind, lab in enumerate(self.labels):

            # Print out status
            print('Scraping words for: ', lab)

            # Initiliaze object to store data for current erp papers
            cur_erp = ERPData(lab, self.erps[ind])

            # Set up search terms - add exclusions, if there are any
            if self.exclusions[ind][0]:
                #term_arg = '"' + erp[0] + '"' + 'NOT' + '"' + self.exclusions[ind][0] + '"'
                term_arg = comb_terms(self.erps[ind], 'or') + comb_terms(self.exclusions[ind], 'not')
            else:
                #term_arg = '"' + erp[0] + '"'
                term_arg = comb_terms(self.erps[ind], 'or')

            # Create the url for the erp search term
            url = urls.search + term_arg

            # Get page and parse
            page = self.req.get_url(url)
            page_soup = BeautifulSoup(page.content, 'lxml')

            # Using history
            if use_hist:

                #
                ret_start = 0
                ret_max = 100

                #
                count = int(page_soup.find('count').text)
                web_env = page_soup.find('webenv').text
                query_key = page_soup.find('querykey').text

                # Update History
                cur_erp.update_history('Start Scrape')

                #
                while ret_start < count:

                    #
                    art_url = urls.fetch + '&WebEnv=' + web_env + '&query_key=' + query_key + \
                              '&retstart=' + str(ret_start) + '&retmax=' + str(ret_max)
                    art_page = self.req.get_url(art_url)
                    art_page_soup = BeautifulSoup(art_page.content, "xml")

                    # Pull out articles
                    articles = art_page_soup.findAll('PubmedArticle')

                    # Loop through each article, extracting relevant information
                    for ind, art in enumerate(articles):

                        # Get ID of current article
                        new_id = _process_ids(extract(art, 'ArticleId', 'all'), 'pubmed')
                        #new_id = int(ids[ind].text)

                        # Extract and add all relevant info from current articles to ERPData object
                        cur_erp = self.extract_add_info(cur_erp, new_id, art)

                    #
                    ret_start += ret_max

            # Without using history
            else:

                # Get all ids
                ids = page_soup.find_all('id')

                # Convert ids to string
                ids_str = _ids_to_str(ids)

                # Get article page
                art_url = urls.fetch + '&id=' + ids_str
                art_page = self.req.get_url(art_url)
                art_page_soup = BeautifulSoup(art_page.content, "xml")

                # Pull out articles
                articles = art_page_soup.findAll('PubmedArticle')

                # Update History
                cur_erp.update_history('Start Scrape')

                # Loop through each article, extracting relevant information
                for ind, art in enumerate(articles):

                    # Get ID of current article
                    new_id = int(ids[ind].text)

                    # Extract and add all relevant info from current articles to ERPData object
                    cur_erp = self.extract_add_info(cur_erp, new_id, art)

            # Check consistency of extracted results
            cur_erp.check_results()
            cur_erp.update_history('End Scrape')

            # Save out and clear data
            cur_erp.save_n_clear()

            # Add the object with current erp data to results list
            self.add_results(cur_erp)

        # Set Requester object as finished being used
        self.req.close()

#######################################################################################################
################################# ERPSC - WORDS - FUNCTIONS (PRIVATE) #################################
#######################################################################################################

def _ids_to_str(ids):
    """Takes a list of pubmed ids, returns a str of the ids separated by commas.

    Parameters
    ----------
    ids : bs4.element.ResultSet
        List of pubmed ids.

    Returns
    -------
    ids_str : str
        A string of all concatenated ids.
    """

    # Check how many ids in list
    n_ids = len(ids)

    # Initialize string with first id
    ids_str = str(ids[0].text)

    # Loop through rest of the id's, appending to end of id_str
    for i in range(1, n_ids):
        ids_str = ids_str + ',' + str(ids[i].text)

    # Return string of ids
    return ids_str

@CatchNone
def _process_words(text):
    """Processes abstract text - sets to lower case, and removes stopwords and punctuation.

    Parameters
    ----------
    text : str
        Text as one long string.

    Returns
    -------
    words_cleaned : list of str
        List of words, after processing.
    """

    # Tokenize input text
    words = nltk.word_tokenize(text)

    # Remove stop words, and non-alphabetical tokens (punctuation). Return the result.
    return [word.lower() for word in words if ((not word.lower() in stopwords.words('english'))
                                               & word.isalnum())]

@CatchNone
def _process_kws(keywords):
    """Processes keywords - extract the keywords from tags and converts to strings.

    Parameters
    ----------
    kws : bs4.element.ResultSet
        List of all the keyword tags.

    Returns
    -------
    list of str
        List of all the keywords.
    """

    # NOTE: UPDATE WITH MOVE TO PY35
    return [kw.text.lower() for kw in keywords]
    #return [kw.text.encode('ascii', 'ignore') for kw in keywords]

@CatchNone
def _process_authors(author_list):
    """

    Parameters
    ----------
    author_list : bs4.element.Tag
        AuthorList tag, which contains tags related to author data.

    Returns
    -------
    out : list of tuple of (str, str, str, str)
        List of authors, each as (LastName, FirstName, Initials, Affiliation).
    """

    # Pull out all author tags from the input
    authors = extract(author_list, 'Author', 'all')

    # Initialize list to return
    out = []

    # Extract data for each author
    for author in authors:
        out.append((extract(author, 'LastName', 'str'), extract(author, 'ForeName', 'str'),
                    extract(author, 'Initials', 'str'), extract(author, 'Affiliation', 'str')))

    return out

@CatchNone2
def _process_pub_date(pub_date):
    """

    Parameters
    ----------
    pub_date : bs4.element.Tag
        PubDate tag, which contains tags with publication date information.

    Returns
    -------
    year : int
        xx
    month : str
        xx
    """

    # Extract year, convert to int if not None
    year = extract(pub_date, 'Year', 'str')
    if year: year = int(year)

    # Extract month
    month = extract(pub_date, 'Month', 'str')

    return year, month

@CatchNone
def _process_ids(ids, id_type):
    """

    Parameters
    ----------
    ids : bs4.element.ResultSet
        All the ArticleId tags, with all IDs for the article.

    Returns
    -------
    str or None
        The DOI if available, otherwise None.
    """

    lst = [str(i.contents[0]) for i in ids if i.attrs == {'IdType' : id_type}]

    if lst == []: return None
    else: return lst[0]


"""The tests for the uptime sensor platform."""
import pytest

from homeassistant.components.sensor import SensorDeviceClass
from homeassistant.components.uptime.const import DOMAIN
from homeassistant.const import ATTR_DEVICE_CLASS
from homeassistant.core import HomeAssistant
from homeassistant.helpers import device_registry as dr, entity_registry as er

from tests.common import MockConfigEntry

@pytest.mark.freeze_time("2022-03-01 00:00:00+00:00")
async def test_uptime_sensor(
    hass: HomeAssistant,
    init_integration: MockConfigEntry,
) -> None:
    """Test Uptime sensor."""
    state = hass.states.get("sensor.uptime")
    assert state
    assert state.state == "2022-03-01T00:00:00+00:00"
    assert state.attributes["friendly_name"] == "Uptime"
    assert state.attributes[ATTR_DEVICE_CLASS] == SensorDeviceClass.TIMESTAMP

    entity_registry = er.async_get(hass)
    entry = entity_registry.async_get("sensor.uptime")
    assert entry
    assert entry.unique_id == init_integration.entry_id

    device_registry = dr.async_get(hass)
    assert entry.device_id
    device_entry = device_registry.async_get(entry.device_id)
    assert device_entry
    assert device_entry.identifiers == {(DOMAIN, init_integration.entry_id)}
    assert device_entry.name == init_integration.title
    assert device_entry.entry_type == dr.DeviceEntryType.SERVICE


# Copyright (c) 2015 Uber Technologies, Inc.
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.

"""Python client for the Uber API.

This client is designed to make calls to the Uber API.
An UberRidesClient is instantiated with a Session which holds either
your server token or OAuth 2.0 credentials. Your usage of this
module might look like:

    client = UberRidesClient(session)
    products = client.get_products(latitude, longitude)
    profile = client.get_user_profile()
    ride_details = client.get_ride_details(ride_id)
"""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

from requests import codes

from uber_rides.auth import refresh_access_token
from uber_rides.auth import revoke_access_token
from uber_rides.errors import ClientError
from uber_rides.errors import UberIllegalState
from uber_rides.request import Request
from uber_rides.utils import auth

VALID_PRODUCT_STATUS = frozenset([
    'processing',
    'accepted',
    'arriving',
    'in_progress',
    'driver_canceled',
    'completed',
])

PRODUCTION_HOST = 'api.uber.com.cn'
SANDBOX_HOST = 'sandbox-api.uber.com.cn'

class UberRidesClient(object):
    """Class to make calls to the Uber API."""

    def __init__(self, session, sandbox_mode=False):
        """Initialize an UberRidesClient.

        Parameters
            session (Session)
                The Session object containing access credentials.
            sandbox_mode (bool)
                Default (False) is not using sandbox mode.
        """
        self.session = session
        self.api_host = SANDBOX_HOST if sandbox_mode else PRODUCTION_HOST

    def _api_call(self, method, target, args=None):
        """Create a Request object and execute the call to the API Server.

        Parameters
            method (str)
                HTTP request (e.g. 'POST').
            target (str)
                The target URL with leading slash (e.g. '/v1/products').
            args (dict)
                Optional dictionary of arguments to attach to the request.

        Returns
            (Response)
                The server's response to an HTTP request.
        """
        self.refresh_oauth_credential()
        handlers = [surge_handler]
        request = Request(
            auth_session=self.session,
            api_host=self.api_host,
            method=method,
            path=target,
            handlers=handlers,
            args=args,
        )
        return request.execute()

    def get_products(self, latitude, longitude):
        """Get information about the Uber products offered at a given location.

        Parameters
            latitude (float)
                The latitude component of a location.
            longitude (float)
                The longitude component of a location.

        Returns
            (Response)
                A Response object containing available products information.
        """
        args = {
            'latitude': latitude,
            'longitude': longitude,
        }

        return self._api_call('GET', 'v1/products', args=args)

    def get_product(self, product_id):
        """Get information about a specific Uber product.

        Parameters
            product_id (str)
                Unique identifier representing a specific product for a
                given location.

        Returns
            (Response)
                A Response containing information about a specific product.
        """
        endpoint = 'v1/products/{}'.format(product_id)
        return self._api_call('GET', endpoint)

    def get_price_estimates(
        self,
        start_latitude,
        start_longitude,
        end_latitude,
        end_longitude,
    ):
        """Get price estimates for products at a given location.

        Parameters
            start_latitude (float)
                The latitude component of a start location.
            start_longitude (float)
                The longitude component of a start location.
            end_latitude (float)
                The latitude component of a end location.
            end_longitude (float)
                The longitude component of a end location.

        Returns
            (Response)
                A Response object containing each product's price estimates.
        """
        args = {
            'start_latitude': start_latitude,
            'start_longitude': start_longitude,
            'end_latitude': end_latitude,
            'end_longitude': end_longitude,
        }

        return self._api_call('GET', 'v1/estimates/price', args=args)

    def get_pickup_time_estimates(self, start_latitude, start_longitude):
        """Get pickup time estimates for products at a given location.

        Parameters
            start_latitude (float)
                The latitude component of a start location.
            start_longitude (float)
                The longitude component of a start location.

        Returns
            (Response)
                A Response containing each product's pickup time estimates.
        """
        args = {
            'start_latitude': start_latitude,
            'start_longitude': start_longitude,
        }

        return self._api_call('GET', 'v1/estimates/time', args=args)

    def get_promotions(
        self,
        start_latitude,
        start_longitude,
        end_latitude,
        end_longitude,
    ):
        """Get information about the promotions available to a user.

        Parameters
            start_latitude (float)
                The latitude component of a start location.
            start_longitude (float)
                The longitude component of a start location.
            end_latitude (float)
                The latitude component of a end location.
            end_longitude (float)
                The longitude component of a end location.

        Returns
            (Response)
                A Response object containing available promotions.
        """
        args = {
            'start_latitude': start_latitude,
            'start_longitude': start_longitude,
            'end_latitude': end_latitude,
            'end_longitude': end_longitude,
        }

        return self._api_call('GET', 'v1/promotions', args=args)

    def get_user_activity(self, offset=None, limit=None):
        """Get activity about the user's lifetime activity with Uber.

        Parameters
            offset (int)
                The integer offset for activity results. Default is 0.
            limit (int)
                Integer amount of results to return. Maximum is 50.
                Default is 5.

        Returns
            (Response)
                A Response object containing ride history.
        """
        args = {
            'offset': offset,
            'limit': limit,
        }

        return self._api_call('GET', 'v1.2/history', args=args)

    def get_user_profile(self):
        """Get information about the authorized Uber user.

        Returns
            (Response)
                A Response object containing account information.
        """
        return self._api_call('GET', 'v1/me')

    def estimate_ride(
        self,
        product_id,
        start_latitude,
        start_longitude,
        end_latitude=None,
        end_longitude=None,
    ):
        """Estimate ride details given a product, start, and end location.

        Only pickup time estimates and surge pricing information provided
        if no end location provided.

        Parameters
            product_id (str)
                Unique identifier representing a specific product for a
                given location.
            start_latitude (float)
                The latitude component of a start location.
            start_longitude (float)
                The longitude component of a start location.
            end_latitude (float)
                Optional latitude component of a end location.
            end_longitude (float)
                Optional longitude component of a end location.

        Returns
            (Response)
                A Response object containing time, price, and distance
                estimates for a ride.
        """
        args = {
            'product_id': product_id,
            'start_latitude': start_latitude,
            'start_longitude': start_longitude,
            'end_latitude': end_latitude,
            'end_longitude': end_longitude,
        }

        return self._api_call('POST', 'v1/requests/estimate', args=args)

    def request_ride(
        self,
        product_id,
        start_latitude,
        start_longitude,
        end_latitude,
        end_longitude,
        surge_confirmation_id=None,
    ):
        """Request a ride on behalf of an Uber user.

        Parameters
            product_id (str)
                Unique identifier representing a specific product for a
                given location.
            start_latitude (float)
                The latitude component of a start location.
            start_longitude (float)
                The longitude component of a start location.
            end_latitude (float)
                The latitude component of a end location.
            end_longitude (float)
                The longitude component of a end location.
            surge_confirmation_id (str)
                Optional unique identifier of the surge session for a user.

        Returns
            (Response)
                A Response object containing the ride request ID and other
                details about the requested ride.

        Raises
            SurgeError (ClientError)
                Thrown when the requested product is currently surging.
                User must confirm surge price through surge_confirmation_href.
        """
        args = {
            'product_id': product_id,
            'start_latitude': start_latitude,
            'start_longitude': start_longitude,
            'end_latitude': end_latitude,
            'end_longitude': end_longitude,
            'surge_confirmation_id': surge_confirmation_id,
        }

        return self._api_call('POST', 'v1/requests', args=args)

    def get_ride_details(self, ride_id):
        """Get status details about an ongoing or past ride.

        Params
            ride_id (str)
                The unique ID of the Ride Request.

        Returns
            (Response)
                A Response object containing the ride's
                status, location, driver, and other details.
        """
        endpoint = 'v1/requests/{}'.format(ride_id)
        return self._api_call('GET', endpoint)

    def cancel_ride(self, ride_id):
        """Cancel an ongoing ride on behalf of a user.

        Params
            ride_id (str)
                The unique ID of the Ride Request.

        Returns
            (Response)
                A Response object with successful status_code
                if ride was canceled.
        """
        endpoint = 'v1/requests/{}'.format(ride_id)
        return self._api_call('DELETE', endpoint)

    def get_ride_map(self, ride_id):
        """Get a map with a visual representation of a Request.

        Params
            ride_id (str)
                The unique ID of the Ride Request.

        Returns
            (Response)
                A Response object with a link to a map.
        """
        endpoint = 'v1/requests/{}/map'.format(ride_id)
        return self._api_call('GET', endpoint)

    def get_ride_receipt(self, ride_id):
        """Get receipt information from a completed ride.

        Params
            ride_id (str)
                The unique ID of the Ride Request.

        Returns
            (Response)
                A Response object containing the charges for
                the given ride.
        """
        endpoint = 'v1/requests/{}/receipt'.format(ride_id)
        return self._api_call('GET', endpoint)

    def update_sandbox_ride(self, ride_id, new_status):
        """Update the status of an ongoing sandbox request.

        Params
            ride_id (str)
                The unique ID of the Ride Request.
            new_status (str)
                Status from VALID_PRODUCT_STATUS.

        Returns
            (Response)
                A Response object with successful status_code
                if ride status was updated.
        """
        if new_status not in VALID_PRODUCT_STATUS:
            message = '{} is not a valid product status.'
            raise UberIllegalState(message.format(new_status))

        args = {'status': new_status}
        endpoint = 'v1/sandbox/requests/{}'.format(ride_id)
        return self._api_call('PUT', endpoint, args=args)

    def update_sandbox_product(
        self,
        product_id,
        surge_multiplier=None,
        drivers_available=None,
    ):
        """Update sandbox product availability.

        Params
            product_id (str)
                Unique identifier representing a specific product for a
                given location.
            surge_multiplier (float)
                Optional surge multiplier to manipulate pricing of product.
            drivers_available (bool)
                Optional boolean to manipulate availability of product.

        Returns
            (Response)
                The Response with successful status_code
                if product status was updated.
        """
        args = {
            'surge_multiplier': surge_multiplier,
            'drivers_available': drivers_available,
        }

        endpoint = 'v1/sandbox/products/{}'.format(product_id)
        return self._api_call('PUT', endpoint, args=args)

    def refresh_oauth_credential(self):
        """Refresh session's OAuth 2.0 credentials if they are stale."""
        if self.session.token_type == auth.SERVER_TOKEN_TYPE:
            return

        credential = self.session.oauth2credential
        if credential.is_stale():
            refresh_session = refresh_access_token(credential)
            self.session = refresh_session

    def revoke_oauth_credential(self):
        """Revoke the session's OAuth 2.0 credentials."""
        if self.session.token_type == auth.SERVER_TOKEN_TYPE:
            return

        credential = self.session.oauth2credential
        revoke_access_token(credential)

def surge_handler(response, **kwargs):
    """Error Handler to surface 409 Surge Conflict errors.

    Attached as a callback hook on the Request object.

    Parameters
        response (requests.Response)
            The HTTP response from an API request.
        **kwargs
            Arbitrary keyword arguments.
    """
    if response.status_code == codes.conflict:
        json = response.json()
        error = json['errors'][0]

        if error['code'] == 'surge':
            raise SurgeError(response)

    return response

class SurgeError(ClientError):
    """Raise for 409 Surge Conflicts."""

    def __init__(self, response, message=None):
        """
        Parameters
            response (requests.Response)
                The HTTP response from an API request.
            message (str)
                An error message string. If one is not provided, the
                default message is used.
        """
        if not message:
            message = (
                'Surge pricing is currently in effect for this product. '
                'User must confirm surge by visiting the confirmation url.'
            )

        super(SurgeError, self).__init__(
            response=response,
            message=message,
        )

        surge_href, surge_id = self.adapt_meta(self.meta)
        self.surge_confirmation_href = surge_href
        self.surge_confirmation_id = surge_id

    def adapt_meta(self, meta):
        """Convert meta from error response to href and surge_id attributes."""

        surge = meta.get('surge_confirmation')
        href = surge.get('href')
        surge_id = surge.get('surge_confirmation_id')

        return href, surge_id