import matplotlib.pyplot as plt
import numpy as np
plt.ion()
plt.rcParams['axes.labelsize'] = 18
plt.rcParams['axes.titlesize'] = 20
plt.rcParams['font.size'] = 16
plt.rcParams['lines.linewidth'] = 2.0
plt.rcParams['lines.markersize'] = 8
plt.rcParams['legend.fontsize'] = 14

class Simulator:
    eps = 1e-16

    def __init__(self, num_agents = 15, max_iterations = 1000, step_size = None, \
                convergence_tol = 0.001, x_bounds = (0,1), y_bounds = (0, 1)):

        # convergence_tol : % of dimensions of the room
        self.convergence_tol = convergence_tol

        # Dimensions of the room
        self.x_bounds = x_bounds
        self.y_bounds = y_bounds

        self.step_size = step_size
        self.num_agents = num_agents
        self.max_iterations = max_iterations

        self.iteration = 0
        self.converged_at_iteration = None
        self.mean_step = []

        self.__initialize_positions()

        self.__choose_attractors()

    def __find_third_vertex(self, first_vertex, second_vertex):

        """ Returns both possible options for the third vertex that makes an
        equilateral triangle with two given points"""

        # Midpoint:
        mid_x, mid_y = 0.5*(first_vertex[0] + second_vertex[0]), 0.5*(first_vertex[1] + second_vertex[1])

        # Squared length of side of equilateral triangle:
        D2 = (first_vertex[0] - second_vertex[0])**2 + (first_vertex[1] - second_vertex[1])**2

        y_diff = first_vertex[1] - second_vertex[1]

        if y_diff < Simulator.eps:
            # avoid division by zero
            y_diff += Simulator.eps

        # Negative Reciprocal slope of line joining first and second vertex:
        slope = -(first_vertex[0] - second_vertex[0]) / y_diff

        # Intercept of perpendicular bisector line between first and second vertex:
        intercept = mid_y - slope * mid_x

        # For the quadratic formula:
        A = 1
        B = -2 * mid_x
        C = mid_x**2 - (3/4) * D2 /(slope**2 + 1)

        Rx = np.roots([A, B, C])
        Ry = slope*Rx + intercept

        vertex_options = (Rx, Ry)

        return vertex_options

    def __find_projections(self, target_location_x, target_location_y, current_x, current_y):
        R_vect = np.array([target_location_x - current_x, target_location_y - current_y])
        Rx_vect = np.array([target_location_x - current_x, 0])
        Ry_vect = np.array([0, target_location_y - current_y])

        # Make the distance travelled a proportion of R_vect
        x_projection = self.step_size * np.dot(Rx_vect, R_vect) / (np.linalg.norm(Rx_vect) + Simulator.eps)
        y_projection = self.step_size * np.dot(Ry_vect, R_vect) / (np.linalg.norm(Ry_vect) + Simulator.eps)

        signed_projection = np.sign(R_vect) * np.array([x_projection, y_projection])
        return (signed_projection[0], signed_projection[1])

    def __initialize_positions(self):

        # Container for the whole simulation:
        self.X = np.zeros((self.num_agents, self.max_iterations + 1))
        self.Y = np.zeros((self.num_agents, self.max_iterations + 1))

        # Initialize first positions:
        self.X[:,0] = np.random.rand(self.num_agents,)
        self.Y[:,0] = np.random.rand(self.num_agents,)

    def __choose_attractors(self):
        if self.num_agents < 3:
            raise Exception('The number of agents must be at least 3')

        # Populate the options for each agent to follow, anyone but herself
        options = np.arange(self.num_agents)
        options = np.tile(options,(len(options),1))
        options = options[~np.eye(options.shape[0],dtype=bool)].reshape(options.shape[0],-1)

        # Pick two random indices to options for two people to follow
        # (scale the random number by the range and round.)

        # Actually will need to loop here for the second agent because have to make sure not
        # choosing same two people:

        # Initialize
        follows = np.zeros((self.num_agents, 2))

        # First attractor:
        follows[:, 0, np.newaxis] = np.round( (options.shape[1] - 1) * np.random.rand(self.num_agents, 1) ).astype(int)

        # Second attractor:
        for agent in range(self.num_agents):
            firstDraw = follows[agent,0]

            # Initialize:
            secondDraw = firstDraw
            while secondDraw == firstDraw:
                # Want a different random draw from the options
                secondDraw = np.round( (options.shape[1] - 1) * np.random.rand() ).astype(int)
            follows[agent,1] = secondDraw

        follows=follows.astype(int)

        self.first_attractor = options[np.arange(options.shape[0]), follows[:,0], np.newaxis]
        self.second_attractor = options[np.arange(options.shape[0]), follows[:,1], np.newaxis]

    def _update_positions(self):
        """
        This allows each agent to jump directly to the third vertex that would create an equilateral triangle
        with the agent and the agent's two targets. However, everyone is jumping at the same time so these
        triangles are not likely to be formed until later in the simulation (if ever)
        """
        if self.step_size is not None:
            if self.step_size > 1:
                raise Exception('The step size should be less than 1')

        for agent in range(self.num_agents):

            # Find the points where you want to go to complete the triangle
            first_vertex = (self.X.item((self.first_attractor.item(agent), self.iteration)), \
               self.Y.item(self.first_attractor.item(agent), self.iteration))
            second_vertex = (self.X.item((self.second_attractor.item(agent), self.iteration)), \
               self.Y.item(self.second_attractor.item(agent), self.iteration))

            options_x, options_y = self.__find_third_vertex(first_vertex, second_vertex)

            # Find the closest of the two vertices to your current position, or the one that is inside the room:
            # For now, just don't update position if both are out of bounds

            out_of_bounds = (options_x > self.x_bounds[1]) | (options_x < self.x_bounds[0]) | \
                            (options_y > self.y_bounds[1]) | (options_y < self.y_bounds[0])

            options_x = options_x[~out_of_bounds]
            options_y = options_y[~out_of_bounds]

            current_x = self.X[agent, self.iteration]
            current_y = self.Y[agent, self.iteration]

            # Update the next position
            if len(options_x) > 1:
                # Distance to first & second options:
                D1 = ( (options_x[0] - current_x)**2 + (options_y[0] - current_y)**2 )**0.5
                D2 = ( (options_x[1] - current_x)**2 + (options_y[1] - current_y)**2 )**0.5
                closest_ind = np.argmin([D1, D2])

                if self.step_size is not None:
                    x_projection, y_projection = self.__find_projections(options_x.item(closest_ind), \
                                            options_y.item(closest_ind), current_x, current_y)
                    self.X[agent, self.iteration + 1] = current_x + x_projection
                    self.Y[agent, self.iteration + 1] = current_y + y_projection
                else:
                    self.X[agent, self.iteration + 1] = options_x[closest_ind]
                    self.Y[agent, self.iteration + 1] = options_y[closest_ind]

            elif len(options_x) == 1:
                if self.step_size is not None:
                    x_projection, y_projection = self.__find_projections(options_x.item(0), \
                                            options_y.item(0), current_x, current_y)
                    self.X[agent, self.iteration + 1] = current_x + x_projection
                    self.Y[agent, self.iteration + 1] = current_y + y_projection
                else:
                    self.X[agent, self.iteration + 1] = options_x
                    self.Y[agent, self.iteration + 1] = options_y

            else: # Don't change position
                self.X[agent, self.iteration + 1] = current_x
                self.Y[agent, self.iteration + 1] = current_y

    def plot_positions(self, initialize_plot, plot_sides = False, zoom = False):

        if initialize_plot:
            # Setting the x and y data explictly for dynamic plot update only works for plot, not scatter:
            # Going to follow the first attractor with a different color
            self.ax1.plot(self.X[0, self.iteration], self.Y[0, self.iteration], 'r.')
            self.ax1.plot(self.X[self.first_attractor.item(0), self.iteration], \
                         self.Y[self.first_attractor.item(0), self.iteration],'r+')
            self.ax1.plot(self.X[self.second_attractor.item(0), self.iteration], \
                         self.Y[self.second_attractor.item(0), self.iteration],'r+')
            self.ax1.plot(self.X[1:, self.iteration], self.Y[1:, self.iteration],'b.')

            self.ax1.set_aspect('equal')
            self.ax1.set_xlim(self.x_bounds[0], self.x_bounds[1])
            self.ax1.set_ylim(self.y_bounds[0], self.y_bounds[1])
            self.ax1.set_ylabel("Y")
            self.ax1.set_xlabel("X")
            self.ax1.set_title("Position of Agents")
        else:
            # Plot the new position
            self.ax1.set_title("Iteration = {}".format(self.iteration))
            for lin_num, line in enumerate(self.ax1.lines):
                if lin_num==0:
                    line.set_xdata(self.X[0, self.iteration])
                    line.set_ydata(self.Y[0, self.iteration])
                elif lin_num==1:
                    line.set_xdata(self.X[self.first_attractor.item(0), self.iteration - 1])
                    line.set_ydata(self.Y[self.first_attractor.item(0), self.iteration - 1])
                elif lin_num==2:
                    line.set_xdata(self.X[self.second_attractor.item(0), self.iteration - 1])
                    line.set_ydata(self.Y[self.second_attractor.item(0), self.iteration - 1])
                else:
                    line.set_xdata(self.X[1:, self.iteration])
                    line.set_ydata(self.Y[1:, self.iteration])

        self.fig.canvas.draw()
        # This is crucial for viewing the plots from the command line:
        try:
            plt.pause(0.5)
        except Exception:
            pass

        if plot_sides:
            for agent in range(self.num_agents):
                # Grab the positions for the attractors of each agent & plot the triangle in green at the end
                X_triangle = np.hstack((self.X[agent, self.iteration], \
                                        self.X[self.first_attractor.item(agent), self.iteration], \
                                        self.X[self.second_attractor.item(agent), self.iteration], \
                                        self.X[agent, self.iteration]))
                Y_triangle = np.hstack((self.Y[agent, self.iteration], \
                                        self.Y[self.first_attractor.item(agent), self.iteration], \
                                        self.Y[self.second_attractor.item(agent), self.iteration], \
                                        self.Y[agent, self.iteration]))
                self.ax1.plot(X_triangle, Y_triangle, '-g')
        if zoom:
            # Zoom In on the final positions
            self.ax1.set_xlim(0.9 * min(self.X[:, self.iteration]), 1.1 * max(self.X[:, self.iteration]))
            self.ax1.set_ylim(0.9 * min(self.Y[:, self.iteration]), 1.1 * max(self.Y[:, self.iteration]))
            self.ax1.set_aspect('equal')

    def run(self, plot_trajectories = True, plot_convergence = True):

        if plot_trajectories:
            self.fig, self.ax1 = plt.subplots(nrows = 1, ncols = 1, figsize=(8, 8)) # two axes on figure
            self.plot_positions(initialize_plot = True)

        while self.iteration < self.max_iterations:

            # Check for convergence using mean step size for all agents:
            self.mean_step.append(np.mean( ( (self.X[:, self.iteration, np.newaxis] \
                                            - self.X[:, self.iteration - 1, np.newaxis] )**2 \
                                            + (self.Y[:, self.iteration, np.newaxis] \
                                            - self.Y[:, self.iteration - 1, np.newaxis] )**2 )**0.5 ) )

            # Define convergence as once the mean step size has dropped below the threshold for 100 iterations
            # Stop the simulation once converged.
            if self.iteration > 100: # Don't bother with convergence rules unless dealing with a significant simulation
                if all( ms <= self.convergence_tol for ms in self.mean_step[self.iteration - 100: self.iteration + 1] ):
                    self.converged_at_iteration = self.iteration
                    self.X = self.X[:, np.arange(self.iteration+1)]
                    self.Y = self.Y[:, np.arange(self.iteration+1)]
                    break

            self._update_positions()

            # Update
            self.iteration += 1

            if plot_trajectories:
                self.plot_positions(initialize_plot = False)

        if plot_convergence:
            # Plot the end positions of the agents, even if we weren't plotting
            # their trajectories throughout, along with the sides of the
            # triangles and the convergence graph
            plot_sides = True

            if not plot_trajectories:
                self.fig, self.ax1  = plt.subplots(nrows = 1, ncols = 1, figsize=(8, 8))
                initialize = True
            else:
                initialize = False

            #if self.step_size is not None:
            #    zoom = True
            #else:
            #    zoom = False

            self.plot_positions(initialize, plot_sides)

            self.fig2, self.ax2  = plt.subplots(nrows = 1, ncols = 1, figsize=(8, 4))
            self.ax2.plot(self.mean_step)
            self.ax2.set_ylabel("Mean Step Size")
            self.ax2.set_xlabel("Iteration")
            self.ax2.set_title("Motion of Agents")


# coding:utf-8

from flask import request
from flask import Flask,  render_template
from controller import search, getPage, get_suggestion
import sys
import json
# reload(sys)
# sys.setdefaultencoding('utf-8')

app = Flask(__name__)

@app.route('/')
def index():
    return "the server is running!"
   # return render_template('/result.html',name = 'zhangsan',)

@app.route('/search', methods=['GET'])
def do_search():
    print(request.args)
    params = {
        'query': request.args.get('query'),
        'method': request.args.get('method')
    }
    res = search(params)
    return json.dumps({
        'status': 1,
        'result': res['result'],
        'time': res['time']
    }, ensure_ascii=False)

@app.route('/page', methods=['GET'])
def page():
    docId = request.args.get('id')
    res = getPage(docId)
    return json.dumps({
        'status': 1,
        'page': res
    }, ensure_ascii=False)

@app.route('/suggestion', methods=['GET'])
def suggestion():
    query =request.args.get('query')
    res = get_suggestion(query)
    return json.dumps({
        'status':1,
        'queries':res
    }, ensure_ascii=False)

if __name__ == '__main__':
    app.run(host="0.0.0.0", port=5000, debug=False)


# find GCD/HCF of two numbers

def fun(a, b):
    res = min(a, b)

    ans = None
    for i in range(1, res + 1):
        if a % res == 0 and b % res == 0:
            ans = i
    return ans

# using recursion
def funrecursive(a, b):
    if b == 0:
        return a
    return fun(b, a % b)

if __name__ == '__main__':
    print(fun(30, 60))
    print(funrecursive(30, 60))


from __future__ import print_function, division
import time

from floor_plans.visualize import View
from floor_plans.floorplan import FloorPlan
from floor_plans import polygon
from floor_plans.math_util import dist
from floor_plans import floorplan_statistics as fps

names = [
    'SPEECH', 'ST', 'COMP LAB', 'PK',
    'SUPER', 'FINANCE', 'ASST', 'SPED', 'BOILER',
    'ART', 'PK', 'PRINCIPLE', 'AP ASSISTANT PRINCIPLE', 'CONFERENCE',
    'G', 'G', 'WORK', 'DIS', 'VOL', 'STORAGE',
    'TOILET', 'CONF', 'CURR', 'WORK', 'SEC',
    'RECEPT', 'TOILET', 'hallway', 'STORAGE', 'STAGE',
    'KITCHEN', 'AUTISM', 'STORAGE', 'LIFE SKILLS', 'TOILET',
    'FACULTY', 'hallway', 'PROJ', 'CONF', 'LIT',
    'MATH', 'K', 'K', 'hallway', '2',
    '2', '2', '2', '1', '2',
    '1', '1', '1', '1', 'hallway',
    'hallway', 'TEAM', 'K', 'TITLE 1', 'TEST',
    'hallway', 'K', 'K', 'hallway', 'JANITORIAL',
    'TOILET', 'RR', 'hallway', 'STORAGE', 'NONE',
    'hallway', 'hallway', 'hallway', 'RESOURCE',
    'hallway', 'WORK', 'hallway', 'hallway', 'TOILET',
    'NURSE', 'WAIT', 'hallway', 'PK', 'LIBRARY',
    'EXAM', 'hallway', 'hallway', 'WORK', 'RECORDS',
    'EVAL', 'EVAL', 'TOILET', 'hallway', 'LOUNGE',
    'GYM STORAGE', 'hallway',
    'hallway',# 'entrance',
    'KILN', 'ART STORAGE',
    'MUSIC STORAGE', 'OT/PT', 'NONE', 'ELECTRICAL', 'MUSIC',
    'hallway', 'TOILET', 'GYM', 'hallway', 'CAFETERIA',
    'hallway', 'TOILET', 'hallway', 'TOILET','CONFERENCE3',
    'RECEPT2', 'CUSTODIAL', 'RECYCLE', 'hallway', 'hallway',
    'hallway', 'hallway', 'BLDG/EQUIP STORAGE', 'hallway', 'hallway',
    'hallway'
]
names = [n.lower() for n in names]

doors = [
    (43, 44, .5),
    (44, 45, .5),
    (45, 46, .5),
    (46, 47, .5),
    (47, 48, .5),
    (48, 49, .1),
    (19, 20, .5),
    (20, 21, .5),
    (21, 22, .5),
    (22, 23, .5),
    (23, 24, .5),
    (29, 30, .5),
    (65, 78, .5),
    (66, 65, .5),
    (67, 66, .5),
    (51, 44, .1),
    (20, 13, .1),
    (33, 29, .5),
    (35, 33, .1),
    (60, 64, .5),
    (59, 60, .5),
    (57, 59, .5),
    (39, 57, .5),
    (24, 69, .5),
    (248, 250, .5),
    (250, 251, .5),
    (49, 251, .5),
    (68, 77, .5),
    (64, 63, .5),
    (246, 247, .5),
    (70, 65, .1),
    (71, 66, 1),
    (29, 31, .1),
    (63, 37, .5),
    (76, 245, .5),
    (76, 246, .5),
    (25, 235, .5),
    (27, 236, .5),
    (74, 2, .9),
    (2, 3, .5),
    (2, 0, .5),
    (26, 28, .5),
    (26, 18, .5),
    (8, 10, .5),
    (10, 1, .5),
    (4, 9, .5),
    (23, 16, .5),
    (4, 253, .5),
    (28, 142, .5),
    (238, 240, .5),
    (243, 242, .5),
    (242, 84, .5),
    (208, 148, .9),
    (155, 213, .9),
    (207, 155, .5),
    (138, 223, .5),
    (214, 216, .5),
    (201, 138, .5),
    (216, 218, .5),
    (220, 203, .5),
    (221, 203, .5),
    (221, 255, .5),
    (139, 137, .1),
    (139, 137, .9),
    (152, 93, .5),
    (89, 91, .1),
    (89, 87, .5),
    (234, 232, .1),
    (234, 232, .9),
    (142, 144, .5),
    (53, 46, .1),
    (54, 47, .5),
    (21, 14, .5),
    (154, 145, .5),
    (153, 149, .5),
    (146, 139, .5),
    (150, 98, .5),
    (161, 163, .5),
    (161, 162, .5),
    (136, 137, .5),
    (178, 182, .5),
    (107, 106, .5),
    (157, 133, .5), # Entrance.
]

entrances = [352, 311, 299, 283, 376, 333, 365]

obj_path = 'school/floor1.obj'
bounds = (750, 750)
view = View(*bounds, scale=2)

source_upf = (39.645008 - 32.433640) / 24. # units per foot
# source_ppm = ppf / 0.3048 # pixels per meter

target_upf = 1.
scale =  target_upf / source_upf

floor = FloorPlan.from_obj(obj_path, names, doors, scale, entrances)

def merge_rooms(name, room_ids, verts):
    room_ids = set(room_ids)
    new_room_id = max(floor.rooms.keys())+1

    # Delete interior room nodes.
    for rid in room_ids:
        if floor.room_centers[rid] in floor:\
        # , floor.room_names[rid]
            floor.remove_node(floor.room_centers[rid])
        for vid in floor.rooms[rid]:
            if vid not in verts and vid in floor:
                floor.remove_node(vid)

    for ID in room_ids:
        del floor.rooms[ID]
        del floor.room_names[ID]
        del floor.room_centers[ID]
        for vid, rooms in floor.vert_to_room.items():
            if len(rooms.intersection(room_ids)):
                rooms -= room_ids
                rooms.add(new_room_id)

    poly = [floor.vertices[vi] for vi in verts]
    center = polygon.center(poly)
    center_i = len(floor.vertices)
    floor.vertices.append(center)

    floor.rooms[new_room_id] = verts
    floor.room_names[new_room_id] = name
    floor.room_centers[new_room_id] = center_i
    floor.room_sizes[new_room_id] = polygon.area(poly)

    return new_room_id

""" Merge administration room clusters together.
"""
# admin1_id = len(floor.rooms) # create new room ID.
# # IDs of rooms to merge
admin1_rooms = [113, 87, 88, 89, 90, 93, 92, 117, 114, 112, 4, 5, 6, 7, 22, 21, 23]
admin1_verts = [156, 170, 172, 174, 176, 178, 179, 183, 182, 180, 157, 168,
                167, 187, 229, 159, 185, 200, 198, 196, 194, 192, 190, 158,
                204, 254, 166, 169]

admin2_rooms = [25, 16, 17, 18, 19, 20, 80, 79, 84, 15, 14, 13, 12, 11, 24, 86]
admin2_verts = [133, 128, 129, 130, 165, 225, 131, 121, 107, 227, 226, 104,
                108, 122, 114, 113, 112, 111, 110, 109, 134, 230, 135, 132]
admin1_id = merge_rooms('administration', admin1_rooms, admin1_verts)
admin2_id = merge_rooms('administration', admin2_rooms, admin2_verts)

floor.add_edge(floor.room_centers[admin1_id], floor.room_centers[119], outside=False, inner=True, width=10)
floor.add_edge(floor.room_centers[admin2_id], floor.room_centers[81], outside=False, inner=True, width=10)

# try:
# print(fps.calculate_all(floor))
for k, v in fps.calculate_all(floor).items():
    print(k, v)
# except Exception as e:
#     print(e)

view.draw_floorplan(floor)
view.hold()


class MyError(Exception):
    def __init__(self, stri):
        self.stri = stri

    def process(self):
        if len(self.stri) < 5:
            print("字符串长度小于5")
        else:
            print("咕咕咕")

try:
    MEr = MyError("waaaa")
    MEr.process()
except MyError as error:
    print(error)

定义类继承Exception，判断输入字符串的长度是否小于5.py

定义类继承Exception，判断输入字符串的长度是否小于5.py


from telegram.ext import Updater, MessageHandler,Filters
from Adafruit_IO import Client
import os

aio = Client('adeebsheriff', os.getenv('adeebsheriff'))

def demo1(bot,update):
  chat_id = bot.message.chat_id
  path = 'https://cdn3.vectorstock.com/i/1000x1000/87/22/i-am-fine-lettering-typography-calligraphy-overlay-vector-15208722.jpg'
  bot.message.reply_text('I am fine')
  update.bot.sendPhoto(chat_id=chat_id,photo=path)
def demo2(bot,update):
  chat_id = bot.message.chat_id
  path = 'https://static.scientificamerican.com/sciam/cache/file/2B38DE31-C1D3-4339-8808D61972976EE4.jpg'
  bot.message.reply_text('Light is turned ON')
  aio.send('bedroom-light', 1)
  data1 = aio.receive('bedroom-light')
  print(f'Received value: {data1.value}')
  update.bot.sendPhoto(chat_id=chat_id,photo=path)
def demo3(bot,update):
  chat_id = bot.message.chat_id
  path = 'https://image.shutterstock.com/image-photo/light-bulb-turned-off-over-260nw-320485652.jpg'
  bot.message.reply_text('Light is turned OFF')
  aio.send('bedroom-light', 0)
  data1 = aio.receive('bedroom-light')
  print(f'Received value: {data1.value}')
  update.bot.sendPhoto(chat_id=chat_id,photo=path)
def demo4(bot,update):
  chat_id = bot.message.chat_id
  path = 'https://cdn.frontdoorhome.com/ahs/blog/prod/static/cs/ahs/image/running-fan.jpg'
  bot.message.reply_text('Fan is turned ON')
  aio.send('bedroom-fan', 1)
  data2 = aio.receive('bedroom-fan')
  print(f'Received value: {data2.value}')
  update.bot.sendPhoto(chat_id=chat_id,photo=path)
def demo5(bot,update):
  chat_id = bot.message.chat_id
  path = 'https://www.destinationlighting.com/fliptheswitch/wp-content/uploads/sites/2/2018/05/zudio-casablanca.jpg'
  bot.message.reply_text('Fan is turned OFF')
  aio.send('bedroom-fan', 0)
  data2 = aio.receive('bedroom-fan')
  print(f'Received value: {data2.value}')
  update.bot.sendPhoto(chat_id=chat_id,photo=path)
def main(bot,update):
  a = bot.message.text.lower()
  print(a)

  if a == "how are you?":
    demo1(bot,update)
  elif a =="light on" or a=="turn on light":
    demo2(bot,update)
  elif a =="light off" or a=="turn off light":
    demo3(bot,update)
  elif a =="switch on fan" or a=="turn on fan":
    demo4(bot,update)
  elif a =="switch off fan" or a=="turn off fan":
    demo5(bot,update)
  else:
    bot.message.reply_text('Invalid Text')
BOT_TOKEN = os.getenv('BOT_TOKEN')
u = Updater(BOT_TOKEN,use_context=True)
dp = u.dispatcher
dp.add_handler(MessageHandler(Filters.text,main))
u.start_polling()
u.idle()


from multiprocessing import Pool
import math
from functools import partial
import numpy as np
from pyfaidx import Fasta, Faidx
import subprocess
import pysam
from liftoff import aligned_seg, liftoff_utils
from os import path

def align_features_to_target(ref_chroms, target_chroms, args, feature_hierarchy, liftover_type, unmapped_features):
    if args.subcommand == "polish":
        sam_files = [args.dir + "/polish.sam"]
    else:
        target_fasta_dict = split_target_sequence(target_chroms, args.target, args.dir)
        genome_size = get_genome_size(target_fasta_dict)
        threads_per_alignment = max(1, math.floor(int(args.p) / len(ref_chroms)))
        sam_files = []
        pool = Pool(int(args.p))
        print("aligning features")
        func = partial(align_single_chroms, ref_chroms, target_chroms, threads_per_alignment, args, genome_size,
                       liftover_type)
        for result in pool.imap_unordered(func, np.arange(0, len(target_chroms))):
            sam_files.append(result)
        pool.close()
        pool.join()
    return parse_all_sam_files(feature_hierarchy, unmapped_features, liftover_type, sam_files)

def split_target_sequence(target_chroms, target_fasta_name, inter_files):
    Faidx(target_fasta_name)
    target_fasta_dict = Fasta(target_fasta_name, key_function=lambda x: x.split()[0])
    for chrm in target_chroms:
        if chrm != target_fasta_name:
            out = open(inter_files + "/" + chrm + ".fa", 'w')
            out.write(">" + chrm + "\n" + str(target_fasta_dict[chrm]))
    return target_fasta_dict

def get_genome_size(target_fasta_dict):
    genome_size = 0
    for value in target_fasta_dict.values():
        genome_size += len(value)
    return genome_size

def align_single_chroms(ref_chroms, target_chroms, threads, args, genome_size, liftover_type, index):
    max_single_index_size = 4000000000
    features_file, features_name = get_features_file(ref_chroms, args, liftover_type, index)
    target_file, output_file = get_target_file_and_output_file(liftover_type, target_chroms, index, features_name, args)
    threads_arg = str(threads)
    minimap2_path = get_minimap_path(args)
    target_prefix = get_target_prefix_name(target_chroms, index, args, liftover_type)
    if genome_size > max_single_index_size:
        split_prefix = args.dir + "/" + features_name + "_to_" + target_prefix + "_split"
        command = [minimap2_path, '-o', output_file, target_file, features_file] + args.mm2_options.split(" ") + [
            "--split-prefix", split_prefix, '-t', threads_arg]
        subprocess.run(command)
    else:
        minimap2_index = build_minimap2_index(target_file, args, threads_arg, minimap2_path)
        command = [minimap2_path, '-o', output_file, minimap2_index, features_file] + args.mm2_options.split(" ") + [
            '-t', threads_arg]
        subprocess.run(command)
    return output_file

def get_features_file(ref_chroms, args, liftover_type, index):
    if ref_chroms[index] == args.reference and (liftover_type == "chrm_by_chrm" or liftover_type == "copies"):
        features_name = 'reference_all'
    elif liftover_type == "unmapped":
        features_name = "unmapped_to_expected_chrom"
    elif liftover_type == "unplaced":
        features_name = "unplaced"
    else:
        features_name = ref_chroms[index]
    return args.dir + "/" + features_name + "_genes.fa", features_name

def get_target_file_and_output_file(liftover_type, target_chroms, index, features_name, args):
    if liftover_type != "chrm_by_chrm" or target_chroms[0] == args.target:
        target_file = args.target
        out_file_target = "target_all"
    else:
        target_file = args.dir + "/" + target_chroms[index] + ".fa"
        out_file_target = target_chroms[index]
    output_file = args.dir + "/" + features_name + "_to_" + out_file_target + ".sam"
    return target_file, output_file

def get_minimap_path(args):
    if args.m is None:
        minimap2 = "minimap2"
    else:
        minimap2 = args.m
    return minimap2

def get_target_prefix_name(target_chroms, index, args, liftover_type):
    if liftover_type != "chrm_by_chrm" or target_chroms[0] == args.target:
        prefix = "target_all"
    else:
        prefix = target_chroms[index]
    return prefix

def build_minimap2_index(target_file, args, threads, minimap2_path):
    if path.exists(target_file + ".mmi") is False:
        subprocess.run(
            [minimap2_path, '-d', target_file + ".mmi", target_file] + args.mm2_options.split(" ") + ['-t',
             threads ])
    return target_file + ".mmi"

def parse_all_sam_files(feature_hierarchy, unmapped_features, liftover_type, sam_files):
    aligned_segments_dict = {}
    for file in sam_files:
        aligned_segments = parse_alignment(file, feature_hierarchy, unmapped_features, liftover_type)
        aligned_segments_dict.update(aligned_segments)
    return aligned_segments_dict

def parse_alignment(file, feature_hierarchy, unmapped_features, search_type):
    all_aligned_blocks = {}
    sam_file = pysam.AlignmentFile(file, 'r', check_sq=False, check_header=False)
    sam_file_iter = sam_file.fetch()
    aln_id = 0
    name_dict = {}
    align_count_dict = {}
    for ref_seq in sam_file_iter:
        if ref_seq.is_unmapped is False:
            aln_id = add_alignment(ref_seq,  align_count_dict, search_type, name_dict,aln_id, feature_hierarchy,
                                   all_aligned_blocks)
        else:
            unmapped_features.append(feature_hierarchy.parents[ref_seq.query_name])
    remove_alignments_without_children(all_aligned_blocks, unmapped_features, feature_hierarchy)
    return all_aligned_blocks

def add_alignment(ref_seq, align_count_dict, search_type, name_dict, aln_id, feature_hierarchy,
                  all_aligned_blocks):
    ref_seq.query_name = edit_name(search_type, ref_seq, name_dict)
    aln_id += 1
    if ref_seq.query_name in align_count_dict:
        align_count = align_count_dict[ref_seq.query_name] + 1
    else:
        align_count = 0
    align_count_dict[ref_seq.query_name] = align_count
    aligned_blocks = get_aligned_blocks(ref_seq, aln_id, feature_hierarchy, search_type)
    if ref_seq.query_name in all_aligned_blocks:
        all_aligned_blocks[ref_seq.query_name].extend(aligned_blocks)
    else:
        all_aligned_blocks[ref_seq.query_name] = aligned_blocks
    return aln_id

def edit_name(search_type, ref_seq, name_dict):
    if search_type != "copies":
        return ref_seq.query_name + "_0"
    else:
        if ref_seq.query_name not in name_dict:
            name_dict[ref_seq.query_name] = 0
        name_dict[ref_seq.query_name] += 1
        return ref_seq.query_name + "_" + str(name_dict[ref_seq.query_name])

def get_aligned_blocks(alignment, aln_id, feature_hierarchy, search_type):
    cigar_operations = get_cigar_operations()
    cigar = alignment.cigar
    parent = feature_hierarchy.parents[liftoff_utils.convert_id_to_original(alignment.query_name)]
    query_start, query_end = get_query_start_and_end(alignment, cigar, cigar_operations)
    children = feature_hierarchy.children[liftoff_utils.convert_id_to_original(alignment.query_name)]
    end_to_end = is_end_to_end_alignment(parent, query_start, query_end)
    if search_type == "copies" and end_to_end is False:
        return []
    reference_block_start, reference_block_pos = alignment.reference_start, alignment.reference_start
    query_block_start, query_block_pos = query_start, query_start
    new_blocks, mismatches = [], []
    merged_children_coords = liftoff_utils.merge_children_intervals(children)
    for operation, length in cigar:
        if base_is_aligned(operation, cigar_operations):
            query_block_pos, reference_block_pos = add_aligned_base(operation, query_block_pos, reference_block_pos,
                                                                    length, cigar_operations, mismatches)
            if query_block_pos == query_end:
                add_block(query_block_pos, reference_block_pos, aln_id, alignment, query_block_start,
                          reference_block_start, mismatches, new_blocks, merged_children_coords, parent)
                break
        elif is_alignment_gap(operation, cigar_operations):
            add_block(query_block_pos, reference_block_pos, aln_id, alignment, query_block_start, reference_block_start,
                      mismatches, new_blocks, merged_children_coords, parent)
            mismatches, query_block_start, reference_block_start, query_block_pos, reference_block_pos = \
                end_block_at_gap(
                    operation, query_block_pos, reference_block_pos, length, cigar_operations)
    return new_blocks

def get_cigar_operations():
    return {"insertion": 1, "deletion": 2, "hard_clip": 5, "match": 7, "mismatch": 8}

def get_query_start_and_end(alignment, cigar, cigar_operations):
    query_start = alignment.query_alignment_start
    query_end = alignment.query_alignment_end
    if cigar[0][0] == cigar_operations["hard_clip"]:
        query_start += cigar[0][1]
        query_end += cigar[0][1]
    return query_start, query_end

def is_end_to_end_alignment(parent, query_start, query_end):
    return parent.end - parent.start + 1 == query_end - query_start

def base_is_aligned(operation, cigar_operations):
    return operation == cigar_operations["match"] or operation == cigar_operations["mismatch"]

def add_aligned_base(operation, query_block_pos, reference_block_pos, length, cigar_operations, mismatches):
    if operation == cigar_operations["mismatch"]:
        for i in range(query_block_pos, query_block_pos + length):
            mismatches.append(i)
    query_block_pos, reference_block_pos = adjust_position(operation, query_block_pos, reference_block_pos,
                                                           length, cigar_operations)
    return query_block_pos, reference_block_pos

def adjust_position(operation, query_block_pos, reference_block_pos, length, cigar_operations):
    if operation == cigar_operations["match"] or operation == cigar_operations["mismatch"] or operation == \
            cigar_operations["insertion"]:
        query_block_pos += length
    if operation == cigar_operations["match"] or operation == cigar_operations["mismatch"] or operation == \
            cigar_operations["deletion"]:
        reference_block_pos += length
    return query_block_pos, reference_block_pos

def add_block(query_block_pos, reference_block_pos, aln_id, alignment, query_block_start, reference_block_start,
              mismatches, new_blocks, merged_children_coords, parent):
    query_block_end = query_block_pos - 1
    reference_block_end = reference_block_pos - 1
    new_block = aligned_seg.aligned_seg(aln_id, alignment.query_name, alignment.reference_name, query_block_start,
                                        query_block_end,
                                        reference_block_start, reference_block_end, alignment.is_reverse,
                                        np.array(mismatches).astype(int))
    overlapping_children = find_overlapping_children(new_block, merged_children_coords, parent)
    if overlapping_children != []:

        new_blocks.append(new_block)

def find_overlapping_children(aln, children_coords, parent):
    overlapping_children = []
    for child_interval in children_coords:
        relative_start = liftoff_utils.get_relative_child_coord(parent, child_interval[0], aln.is_reverse)
        relative_end = liftoff_utils.get_relative_child_coord(parent, child_interval[1], aln.is_reverse)
        child_start, child_end = min(relative_start, relative_end), max(relative_start, relative_end)
        overlap = liftoff_utils.count_overlap(child_start, child_end, aln.query_block_start, aln.query_block_end)
        if overlap > 0:
            overlapping_children.append(child_start)
            overlapping_children.append(child_end)
    return overlapping_children

def is_alignment_gap(operation, cigar_operations):
    return operation == cigar_operations["insertion"] or operation == cigar_operations["deletion"]

def end_block_at_gap(operation, query_block_pos, reference_block_pos, length, cigar_operations):
    mismatches = []
    query_block_pos, reference_block_pos = adjust_position(operation, query_block_pos, reference_block_pos,
                                                           length, cigar_operations)
    query_block_start = query_block_pos
    reference_block_start = reference_block_pos
    return mismatches, query_block_start, reference_block_start, query_block_pos, reference_block_pos

def remove_alignments_without_children(all_aligned_blocks, unmapped_features, feature_hierarchy):
    features_to_remove = []
    for seq in all_aligned_blocks:
        if all_aligned_blocks[seq] == []:
            features_to_remove.append(seq)
            unmapped_features.append(feature_hierarchy.parents[liftoff_utils.convert_id_to_original(seq)])
    for feature in features_to_remove:
        del all_aligned_blocks[feature]
    return all_aligned_blocks


#정렬 : 데이터를 특정한 기준에 따라 순서대로 나열하는것

#1.선택정렬 : 처리되지 않은 데이터 중에서 가장 작은 데이터를 선택해 맨 앞에 있는 데이터와 바꾸는 것을 반복 - O(N^2)
array = [7, 5, 9, 0, 3, 1, 6, 2, 4, 8]
print(array)
for i in range(len(array)):
    min_idx = i
    for j in range(i + 1, len(array)):
        if array[min_idx] > array[j]:
            min_idx = j

    array[i], array[min_idx] = array[min_idx], array[i]
print(array)
print()

#2.삽입정렬 : 처리되지 않은 데이터를 하나씩 골라 적절한 위치에 삽입 - O(N^2) 이지만 정렬정도에 따라 O(N) 까지 달라짐
array = [7, 5, 9, 0, 3, 1, 6, 2, 4, 8]
print(array)
for i in range(1, len(array)):
    for j in range(i, 0, -1):
        if array[j] < array[j - 1]:
            array[j], array[j - 1] = array[j - 1], array[j]
        else:
            break
print(array)
print()

#3.퀵정렬 : 기준 데이터를 설정하고 그 기준보다 큰 데이터와 작은 데이터의 위치를 바꾸는 방법
array = [5, 7, 9, 0, 3, 1, 6, 2, 4, 8]
print(array)
def quick_sort(array):
    if len(array) <= 1:
        return array
    pivot = array[0]
    tail = array[1:]

    left_side = [x for x in tail if x <= pivot]
    right_side = [x for x in tail if x > pivot]

    return quick_sort(left_side) + [pivot] + quick_sort(right_side)
print(quick_sort(array))

#4.계수정렬 : 특정한 조건이 부합할 때만 사용할 수 있지만 매우 빠르게 동작하는 정렬 알고리즘 O(N + K)
#           데이터의 크기 범위가 제한되어 정수 형태로 표현할 수 있을 때 사용 가능
#           동일한 값을 가지는 데이터가 여러개 등장할 때 효과적
array = [7, 5, 9, 0, 3, 1, 6, 2, 9, 1, 4, 8, 0, 5, 2]
count = [0] * (max(array) + 1)
for i in range(len(array)):
    count[array[i]] += 1

for i in range(len(count)):
    for j in range(count[i]):
        print(i, end=' ')


from spynnaker.pyNN import exceptions
from spynnaker.pyNN.models.neural_projections.connectors.abstract_connector \
    import AbstractConnector
from spynnaker.pyNN.models.neural_properties.synaptic_list import SynapticList
from spynnaker.pyNN.models.neural_properties.synapse_row_info \
    import SynapseRowInfo
import logging
import numpy

logger = logging.getLogger(__name__)

class FromListConnector(AbstractConnector):
    """
    Make connections according to a list.

    :param `list` conn_list:
        a list of tuples, one tuple for each connection. Each
        tuple should contain::

         (pre_idx, post_idx, weight, delay)

        where pre_idx is the index (i.e. order in the Population,
        not the ID) of the presynaptic neuron, and post_idx is
        the index of the postsynaptic neuron.
    """

    def __init__(self, conn_list=None, safe=True, verbose=False):
        """
        Creates a new FromListConnector.
        """
        if not safe:
            logger.warn("the modification of the safe parameter will be "
                        "ignored")
        if verbose:
            logger.warn("the modification of the verbose parameter will be "
                        "ignored")
        if conn_list is None:
            conn_list = []
        self._conn_list = conn_list

    def generate_synapse_list(
            self, presynaptic_population, postsynaptic_population, delay_scale,
            weight_scale, synapse_type):

        prevertex = presynaptic_population._get_vertex
        postvertex = postsynaptic_population._get_vertex

        # Convert connection list into numpy record array
        conn_list_numpy = numpy.array(
            self._conn_list, dtype=[("source", "uint32"), ("target", "uint32"),
                                    ("weight", "float"), ("delay", "float")])
        if (conn_list_numpy["target"] >= postvertex.n_atoms).any():
            raise exceptions.ConfigurationException("Target atom out of range")

        # Sort by pre-synaptic neuron
        conn_list_numpy = numpy.sort(conn_list_numpy, order="source")

        # Apply weight and delay scaling
        conn_list_numpy["weight"] *= weight_scale
        conn_list_numpy["delay"] *= delay_scale

        # Count number of connections per pre-synaptic neuron
        pre_counts = numpy.histogram(
            conn_list_numpy["source"], numpy.arange(prevertex.n_atoms + 1))[0]

        # Take cumulative sum of these counts to get start and end indices of
        # the blocks of connections coming from each pre-synaptic neuron
        pre_end_idxs = numpy.cumsum(pre_counts)
        pre_start_idxs = numpy.append(0, pre_end_idxs[:-1])

        # Loop through slices of connections
        synaptic_rows = []
        for _, (start, end) in enumerate(zip(pre_start_idxs, pre_end_idxs)):

            # Get slice
            pre_conns = conn_list_numpy[start:end]

            # Repeat synapse type correct number of times
            synapse_type_row = numpy.empty(len(pre_conns), dtype="uint32")
            synapse_type_row.fill(synapse_type)

            # Combine post-synaptic neuron ids, weights, delays
            # and synapse types together into synaptic row
            synaptic_rows.append(
                SynapseRowInfo(pre_conns["target"],
                               pre_conns["weight"],
                               pre_conns["delay"],
                               synapse_type_row))

        # Return full synaptic list
        return SynapticList(synaptic_rows)


from django.contrib.auth.models import User
from django.http import JsonResponse, Http404
from django.shortcuts import redirect, get_object_or_404
from django.urls import reverse
from django.utils.decorators import method_decorator
from django.views.generic import DetailView, CreateView, View, TemplateView, DeleteView

from blog.decorators import authorized_only
from blog.forms import PostForm
from blog.models import Post, Subscription, PostRead

@method_decorator(authorized_only, name='dispatch')
class SubscribeView(View):

    def post(self, request):
        try:
            user_id = request.POST.get('pk')
            user = User.objects.get(pk=user_id)
        except User.DoesNotExist:
            return JsonResponse({'status': 'error', 'error': 'Пользователя не существует'})

        sub, created = Subscription.objects.get_or_create(user=user)

        # если пользователь уже в подписках, то удаляем его оттуда, иначе добавляем
        if request.user in sub.subscribers.all():
            sub.subscribers.remove(request.user.id)
            subscribed = False
            PostRead.objects.filter(user_id=request.user.id, post__user_id=user_id).delete()
        else:
            sub.subscribers.add(request.user.id)
            subscribed = True
        sub.save()

        return JsonResponse({'status': 'ok', 'subscribed': subscribed})

class BlogView(DetailView):
    model = User
    template_name = 'blog/user.html'

    def get_context_data(self, **kwargs):
        context = super(BlogView, self).get_context_data(**kwargs)
        context['posts'] = Post.objects.filter(user_id=self.object.id).prefetch_related('user')
        return context

class PostCreateView(CreateView):
    model = Post
    form_class = PostForm
    template_name = 'blog/create.html'

    def form_valid(self, form):
        post = form.save(commit=False)
        post.user = self.request.user
        post.save()
        return redirect(reverse('blog:detail', args=(post.user_id, post.id)))

class PostDetailView(DetailView):
    model = Post
    template_name = 'blog/detail.html'

class HomeView(TemplateView):
    template_name = 'home.html'

    def get_context_data(self, **kwargs):
        context = super(HomeView, self).get_context_data(**kwargs)

        # 2 запроса для получения всех постов
        # Получение списка id пользователей на которых подписан
        # Получение всех постов с этими id
        subscription = list(self.request.user.subscription.values_list('user_id', flat=True))
        subscription.append(self.request.user.id)
        context['posts'] = Post.objects.filter(user_id__in=subscription).prefetch_related('user').distinct()
        return context

class PostReadView(View):
    def post(self, request):
        try:
            post = Post.objects.exclude(user_id=request.user.id).get(pk=request.POST.get('post_id'))
        except Post.DoesNotExist:
            return JsonResponse({'status': 'error', 'error': 'Пост не найден'})

        PostRead.objects.get_or_create(user=request.user, post=post)
        return JsonResponse({'status': 'ok'})

class PostDeleteView(DeleteView):
    model = Post

    def get_object(self, queryset=None):
        return get_object_or_404(Post, user_id=self.request.user.id, pk=self.kwargs.get('pk'))

    def get_success_url(self):
        return reverse('blog:home')


__all__ = ["save", "load", "load_state_dict", "arange", "cat", "cos", "clamp", "Device", "from_numpy", "flatten",
           "LongTensor", "matmul", "mm", "normal", "ones", "x2ms_pow", "sin", "tanh", "x2ms_tensor", "Tensor",
           "split", 'as_tensor', 'argmax', 'Generator', 'sigmoid', 'rand', 'floor', 'bernoulli', 'equal', 'var_mean',
           'randperm', 'sqrt', 'stack', 'log', 'exp', 'typename', 'is_tensor', 'randn', 'FloatTensor', 'x2ms_max',
           'x2ms_min', 'bmm', 'x2ms_abs', 'square', 'squeeze', 'unsqueeze', 'transpose', 'repeat_interleave', 'div',
           'ones_like', 'where', 'tensordot', 'meshgrid', 'roll', 'linspace', 'full', 'empty', 'x2ms_sum',
           'multinomial', 'gather', 'sort', 'topk', 'x2ms_all', 'cumsum', 'einsum', 'full_like', 'masked_select',
           'x2ms_mean', 'mul', 'isfinite', 'diag', 'acos', 'add', 'argsort', 'asin', 'atan2', 'bincount',
           'broadcast_tensors', 'chunk', 'conj', 'cosh', 'cross', 'cumprod', 'diagflat', 'x2ms_diagonal', 'eq',
           'zeros_like', 'atan', 'unique', 'triu', 'nonzero', 'log2', 'cdist', 'erf', 'softmax', 'eye', 'prod', 'norm',
           'zeros', 'lt', 'ge', 'ne', 'le', 'reshape', 'reminder', 'result_type', 'real', 'reciprocal', 'neg', 'isinf',
           'isnan', 'argmin', 'floor_divide', 'fmod', 'empty_like', 'erfc', 'erfinv', 'expm1', 'flip', 'gt',
           'bitwise_and', 'bitwise_or', 'bitwise_xor', 'bartlett_window', 'blackman_window', 'hamming_window', 'histc',
           'imag', 'ceil', 'lerp', 'log1p', 'logical_and', 'logical_not', 'logical_or', 'logical_xor', 'var', 'unbind',
           'trunc', 'true_divide', 'triu_indices', 'triu', 'tril', 'trapz', 'trapezoid', 'trace', 'tan', 'take', 'lt',
           'ge', 'ne', 'le', 'reshape', 'reminder', 'result_type', 'real', 'reciprocal', 'neg',
           'minimum', 'hann_window', 'dot', 'scatter', 'ger', 'addmm', 'BoolTensor', 'finfo', 'IntTensor',
           'get_rng_state', 'set_rng_state', 'randint', 'randn_like', 'ByteTensor', 'index_select', 'allclose', 't',
           'vstack', 'rsqrt', 'x2ms_round', 'acosh', 'addcmul', 'addcdiv', 'asinh', 'atanh', 'amax', 'amin',
           'cummax', 'cummin', 'logsumexp', 'renorm', 'xlogy', 'sign', 'sinh', 'less', 'narrow', 'tensor_zeros_like',
           'Size', 'DoubleTensor', 'cosine_similarity', 'block_diag', 'cholesky_solve', 'lu_solve', 'x2ms_any',
           'greater', 'greater_equal', 'less_equal', 'not_equal', 'multiply', 'logspace', 'tril_indices', 'vander',
           'atleast_1d', 'atleast_2d', 'atleast_3d', 'column_stack', 'rad2deg', 'outer', 'negative', 'log10',
           'count_nonzero', 'signbit', 'isposinf', 'isin', 'isneginf', 'copysign', 'deg2rad', 'diff', 'gcd',
           'heaviside']

import numbers
import numpy as np
import mindspore
import mindspore.default_config
import mindspore.nn
import mindspore.numpy
import mindspore.dataset.transforms
import mindspore.ops.functional as F
from mindspore.nn import DistributedGradReducer
from mindspore.ops import composite
from mindspore.parallel._utils import (_get_device_num, _get_gradients_mean, _get_parallel_mode)
from mindspore.common.parameter import ParameterTuple
from mindspore.context import ParallelMode
from .core.decorator import x2ms_func_decorator
from .torch_api.save_load import save, load, load_state_dict
from .torch_api import tensor_api
from .torch_api import autograd
from .third_party_adapter import math_api
from .torch_api.nn_api import nn_cell
from .torch_api.nn_api import nn_init
from .torch_api.nn_api import nn_functional
from .core.context import x2ms_context
from .utils.util_api import logger
from .torch_api.torch_base_api import arange, cat, cos, clamp, Device, from_numpy, flatten, \
    LongTensor, matmul, mm, normal, ones, x2ms_pow, sin, tanh, x2ms_tensor, Tensor, zeros, split, as_tensor, dot, \
    x2ms_sum, argmax, Generator, sigmoid, rand, floor, bernoulli, equal, randperm, var_mean, sqrt, stack, log, exp, \
    typename, is_tensor, randn, FloatTensor, x2ms_max, x2ms_min, bmm, x2ms_abs, square, squeeze, unsqueeze, \
    transpose, repeat_interleave, div, ones_like, where, tensordot, meshgrid, roll, linspace, full, empty, \
    multinomial, gather, sort, topk, x2ms_all, cumsum, einsum, full_like, masked_select, x2ms_mean, mul, isfinite, \
    diag, acos, add, argsort, asin, atan2, bincount, broadcast_tensors, chunk, conj, cosh, cross, cumprod, \
    diagflat, x2ms_diagonal, zeros_like, atan, unique, nonzero, log2, cdist, erf, softmax, eye, prod, norm, \
    lt, ge, eq, ne, le, reshape, reminder, result_type, real, reciprocal, neg, isinf, isnan, argmin, floor_divide, \
    fmod, empty_like, erfc, erfinv, expm1, flip, gt, bitwise_and, bitwise_or, bitwise_xor, bartlett_window, \
    blackman_window, hamming_window, histc, imag, ceil, lerp, log1p, logical_and, logical_not, logical_or, \
    logical_xor, var, unbind, trunc, true_divide, triu_indices, triu, tril, trapz, trapezoid, trace, tan, take, \
    minimum, hann_window, scatter, ger, addmm, BoolTensor, finfo, IntTensor, get_rng_state, set_rng_state, randint, \
    randn_like, ByteTensor, index_select, allclose, t, vstack, rsqrt, x2ms_round, acosh, addcmul, addcdiv, asinh, \
    atanh, amax, amin, cummax, cummin, logsumexp, renorm, xlogy, sign, sinh, less, narrow, tensor_zeros_like, Size, \
    DoubleTensor, cosine_similarity, block_diag, cholesky_solve, lu_solve, x2ms_any, greater, greater_equal, \
    less_equal, not_equal, multiply, logspace, tril_indices, vander, atleast_1d, atleast_2d, atleast_3d, \
    column_stack, rad2deg, outer, negative, log10, count_nonzero, signbit, isposinf, isin, isneginf, copysign, \
    deg2rad, diff, gcd, heaviside

# overwrite Magic methods
mindspore.Tensor.__and__ = tensor_api.tensor_and
mindspore.Tensor.__or__ = tensor_api.tensor_or
mindspore.Tensor.__format__ = tensor_api.tensor_format
mindspore.Tensor.__getitem__ = tensor_api.tensor_getitem
mindspore.Tensor.__matmul__ = tensor_api.matmul
mindspore.Tensor.__setitem__ = tensor_api.tensor_setitem
mindspore.Tensor.T = tensor_api.transpose_

mindspore.Tensor.__float__ = lambda t: float(t.asnumpy())
mindspore.Tensor.__int__ = lambda t: int(t.asnumpy())

mindspore.Parameter.__iadd__ = tensor_api.parameter_iadd
mindspore.Parameter.__isub__ = tensor_api.parameter_isub
mindspore.Parameter.__imul__ = tensor_api.parameter_imul
mindspore.Parameter.__idiv__ = tensor_api.parameter_idiv

# overwrite properties
mindspore.Tensor.is_cuda = tensor_api.is_cuda
mindspore.Tensor.data = tensor_api.property_data
mindspore.Tensor.device = tensor_api.property_device
mindspore.Parameter.grad = tensor_api.grad
mindspore.Parameter.grad = tensor_api.set_grad

@property
def parameter_data(self):
    return self

@parameter_data.setter
def set_data(self, new_data):
    self.set_data(new_data)

mindspore.Parameter.data = parameter_data
mindspore.Parameter.data = set_data

def _get_calculate_shape(obj, other):
    if not isinstance(other, mindspore.Tensor):
        return obj.shape
    return np.broadcast_shapes(obj.shape, other.shape)

def _replace_tensor_calculate_func(origin_func_name, output_type=None):
    origin_func = getattr(mindspore.Tensor, origin_func_name)

    def new_func(obj, other):
        if obj.dtype == mindspore.float64:
            obj = obj.astype(mindspore.float32)
        if isinstance(other, np.ndarray):
            other = mindspore.Tensor(other, obj.dtype)
        if obj.size == 0 or (isinstance(other, mindspore.Tensor) and other.size == 0):
            if output_type is None:
                return mindspore.ops.Zeros()(_get_calculate_shape(obj, other), obj.dtype)
            else:
                return mindspore.ops.Zeros()(_get_calculate_shape(obj, other), output_type)
        return origin_func(obj, other)

    setattr(mindspore.Tensor, origin_func_name, new_func)

for func_name in ("__add__", "__sub__", "__mul__", "__truediv__", "__mod__", "__pow__"):
    _replace_tensor_calculate_func(func_name)

for func_name in ("__lt__", "__gt__", "__le__", "__ge__", "__eq__", "__ne__"):
    _replace_tensor_calculate_func(func_name, mindspore.bool_)

class GraphTrainStep(mindspore.nn.TrainOneStepCell):
    def __init__(self, network, optimizer):
        super(GraphTrainStep, self).__init__(network, optimizer)

    def call_construct(self, *inputs):
        new_input = list(mindspore.Tensor(value, dtype=mindspore.float32)
                         if not isinstance(value, mindspore.Tensor)
                         else value
                         for value in inputs)
        return self.__call__(*new_input)

    def construct(self, *inputs):
        output = self.network(*inputs)
        loss = output[0]
        model_output = output[1:]
        sens = (F.fill(loss.dtype, loss.shape, self.sens),)
        for output_value in model_output:
            sens += self._get_sens(output_value)
        grads = self.grad(self.network, self.weights)(*inputs, sens)
        grads = self.grad_reducer(grads)
        F.depend(loss, self.optimizer(grads))
        return output

    @staticmethod
    def _get_sens(value):
        if isinstance(value, mindspore.Tensor):
            return (F.fill(value.dtype, value.shape, 0),)
        if isinstance(value, list):
            sens = []
            for tensor in value:
                sens.append(F.fill(tensor.dtype, tensor.shape, 0))
            return (sens,)
        if isinstance(value, tuple):
            sens = ()
            for tensor in value:
                sens += (F.fill(tensor.dtype, tensor.shape, 0),)
            return (sens,)
        return (0,)

def add_module(obj, name, module):
    setattr(obj, name, module)

classic_cell_init = mindspore.nn.Cell.__init__

def new_cell_init(self, auto_prefix=True, flags=None):
    classic_cell_init(self, auto_prefix, flags)
    self.training = True

# same name and inherit subclass api
mindspore.nn.Cell.add_module = add_module
mindspore.nn.Cell.__init__ = new_cell_init
mindspore.nn.Cell._modules = nn_cell._modules

@property
def is_floating_point(self):
    return self in (mindspore.float16, mindspore.float32, mindspore.float64)

mindspore.dtype.typing.Number.is_floating_point = is_floating_point

def cuda_set_device(device):
    pass

def is_cuda_available():
    """
       Stub function for torch.cuda.is_available.
       get the info from default_config.
    """
    return True

def memory_cached():
    return 0.0

def memory_reserved():
    return 0.0

def max_memory_reserved(device=None):
    return 0.0

def max_memory_allocated(device=None):
    return 0.0

def memory_allocated(device=None):
    return 0.0

def get_device():
    return mindspore.context.get_context('device_target')

@x2ms_func_decorator(mindspore.nn.Cell)
def parameters(obj, *args, **kwargs):
    return get_cell_params(obj, *args, **kwargs)

def get_cell_params(cell, recurse=True):
    return iter(cell.trainable_params(recurse) + cell.untrainable_params(recurse))

@x2ms_func_decorator(mindspore.nn.Cell)
def named_parameters(model, prefix='', recurse=True):
    return list(param for param in model.parameters_and_names(prefix, recurse))

@x2ms_func_decorator(mindspore.nn.Cell)
def named_modules(model, prefix=''):
    return model.cells_and_names(prefix)

@x2ms_func_decorator(mindspore.nn.Cell)
def graph_forward(obj, *args, **kwargs):
    return obj(*args, **kwargs)

@x2ms_func_decorator(mindspore.nn.Cell)
def forward(obj, *args, **kwargs):
    return obj.construct(*args, **kwargs)

@x2ms_func_decorator(mindspore.nn.Cell)
def x2ms_train(obj, *args, **kwargs):
    if len(obj.trainable_params()) > 0:
        if obj not in x2ms_context.amp_model:
            x2ms_context.amp_model.append(obj)
    return obj.set_train(*args, **kwargs)

@x2ms_func_decorator(mindspore.nn.Cell)
def x2ms_eval(obj, *args, **kwargs):
    return obj.set_train(False)

class TrainCellWithoutOptimizer(mindspore.nn.Cell):
    def __init__(self, network, sens=1.0):
        super(TrainCellWithoutOptimizer, self).__init__(auto_prefix=False)
        self.network = network
        self.network.set_grad()
        self.model_weights = ParameterTuple(parameters(network))
        self.grad = composite.GradOperation(get_by_list=True, sens_param=True)
        self.sens = sens
        self.parallel_flag = False
        self.grad_reducer = F.identity
        self.parallel_mode = _get_parallel_mode()
        self.parallel_flag = self.parallel_mode in (ParallelMode.DATA_PARALLEL, ParallelMode.HYBRID_PARALLEL)
        if self.parallel_flag:
            self.grads_mean = _get_gradients_mean()
            self.device_num = _get_device_num()
            self.grad_reducer = DistributedGradReducer(self.model_weights, self.grads_mean, self.device_num)

    def construct(self, *inputs):
        train_loss = self.network(*inputs)
        train_sens = F.fill(train_loss.dtype, train_loss.shape, self.sens)
        grads = self.grad(self.network, self.model_weights)(*inputs, train_sens)
        grads = self.grad_reducer(grads)
        for i, parameter in enumerate(self.model_weights):
            parameter.grad = grads[i]
        return train_loss

_wrapped_model_dict = {}

def train_one_step_cell(model, optimizer=None):
    key = id(model)
    if key in _wrapped_model_dict.keys():
        return _wrapped_model_dict.get(key)
    if x2ms_context.amp_opt_level is None or x2ms_context.amp_model is None:
        if optimizer is None:
            wrapped_model = TrainCellWithoutOptimizer(model)
        else:
            wrapped_model = mindspore.nn.TrainOneStepCell(model, optimizer)
    else:
        if isinstance(x2ms_context.loss_scale, numbers.Number) and x2ms_context.amp_opt_level != "O2":
            wrapped_model = mindspore.amp.build_train_network(model, optimizer, level=x2ms_context.amp_opt_level,
                                                              loss_scale_manager=mindspore.FixedLossScaleManager(
                                                                  x2ms_context.loss_scale))
        else:
            wrapped_model = mindspore.amp.build_train_network(model, optimizer, level=x2ms_context.amp_opt_level)
    _wrapped_model_dict[key] = wrapped_model
    return wrapped_model

def graph_train_one_step_cell(model, optimizer):
    key = id(model)
    if key in _wrapped_model_dict.keys():
        return _wrapped_model_dict.get(key)
    if x2ms_context.amp_opt_level is None or x2ms_context.amp_model is None:
        wrapped_model = GraphTrainStep(model, optimizer)
    else:
        raise NotImplementedError("Graph mode does not currently support Mixed precision")
    _wrapped_model_dict[key] = wrapped_model
    return wrapped_model

def load_state_dict_from_url(url, model_dir=None, map_location=None, progress=True, check_hash=False):
    """
    Current not support 'model_dir', 'map_location', 'progress', 'check_hash' parameter.
    """
    logger.warning("Not support load_state_dict_from_url now")
    return {}

def to(obj, *args, **kwargs):
    if isinstance(obj, mindspore.nn.Cell):
        return _cell_to(obj, *args, **kwargs)
    elif isinstance(obj, mindspore.Tensor):
        return _tensor_to(obj, *args, **kwargs)
    else:
        return obj.to(*args, **kwargs)

def _cell_to(obj, *args, **kwargs):
    if args:
        param = args[0]
        if isinstance(param, mindspore.Type) and param in (mindspore.float16, mindspore.float32):
            return obj.to_float(dst_type=param)
        if isinstance(param, mindspore.Tensor) and param.dtype in (mindspore.float16, mindspore.float32):
            return obj.to_float(dst_type=param.dtype)
    if len(args) > 1:
        param = args[1]
        if param in (mindspore.float16, mindspore.float32):
            return obj.to_float(dst_type=param)
    if 'dtype' in kwargs.keys() and kwargs['dtype'] in (mindspore.float16, mindspore.float32):
        return obj.to_float(dst_type=kwargs['dtype'])
    if 'other' in kwargs.keys() and kwargs['other'].dtype in (mindspore.float16, mindspore.float32):
        return obj.to_float(dst_type=kwargs['other'])
    return obj

def _tensor_to(obj, *args, **kwargs):
    if args:
        param = args[0]
        if isinstance(param, mindspore.common.Type):
            return obj.astype(dtype=param)
        if isinstance(param, mindspore.Tensor):
            return obj.astype(dtype=param.dtype)
    if len(args) > 1:
        return obj.astype(dtype=args[1])
    if 'dtype' in kwargs.keys():
        return obj.astype(dtype=kwargs['dtype'])
    if 'other' in kwargs.keys():
        return obj.astype(dtype=kwargs['other'].dtype)
    return obj

def get_device_properties(device):
    return CUDADeviceProperty()

def convert_sync_batchnorm(module, process_group=None):
    return module

class CUDADeviceProperty:
    def __init__(self):
        device_target = mindspore.context.get_context('device_target')
        device_id = mindspore.context.get_context('device_id')
        self.name = f'{device_target}:{device_id}'
        self.total_memory = 0


# Chapter 1
spam_amount = 0
print(spam_amount)

spam_amount += 4

if spam_amount > 0 :
	print("But I don't want Any spam!")

viking_song = "spam " * spam_amount # spam spam spam spam
#viking_song = "spam " + spam_amount
print(viking_song)

print(spam_amount*4) # 16
print(float(str(spam_amount)*4)*4) #17776.0

print(type(spam_amount))

print(5/2) # 2.5
print(5//2) # 2
print(5%2) # 1
print(5**2) # 25

print(min(1,2,3,4,5)) #1
print(max(1,2,3,4,5)) #5

print(abs(-32)) #32

a = [1,2,3]
b = [3,2,1]

temp=a; a=b; b=temp
print(a)

a,b = b,a #미쳤다
print(a)

#Chapter 2 : help & function
help(round)
help(round(-2.01)) #help(int)
help(print)

def least_difference(a, b, c) :
	""" Return the smallest diferrence between any two numbers
	among a, b, and c.

	>>> least_difference(1,5,-5)
	4
	"""
	diff1 = abs(a-b);
	diff2 = abs(b-c);
	diff3 = abs(a-c);
	return min(diff1, diff2, diff3)

print(
	least_difference(1,10,100),
	least_difference(1,10,10),
	least_difference(5,6,7),
	)

help(least_difference)

print(1,2,3, sep=' < ') # default sep = ' '

def greet(who="Colin") :
	print("Hello, ", who)

greet() #default who="Colin"
greet(who="Kaggle")
greet("world")

def multy_by_five(x) :
	return 5 * x

def call(fn, arg) :
	"""Call fn on arg"""
	return fn(arg)

def squared_call(fn,arg) :
	return fn(fn(arg))

print (
	call(multy_by_five, 1),
	squared_call(multy_by_five, 1),
	sep='\n',
)

def mod_5(x) :
	"""Return the remainder of x after dividing by 5"""
	return x % 5

print (
	max(100,51,14), # 100
	max(100,51,14, key=mod_5), # 각 val의 mod_5한 값 중 max인 val 14
	sep='\n',
)

print(round(511123,-2)) # 100미만 절사

from time import time
t=time()
print(t,"seconds since the Epoch")

from time import sleep
duration =5
print("Getting sleepy. See you in",duration, "seconds")
sleep(duration)
print("I'm back")


import multiprocessing
from multiprocessing import Process
from cleanup import TwitterCleanuper
from preprocessing import TwitterData
from word2vec import Word2VecProvider
import pandas as pd

def preprocess(results, data_path, is_testing, data_name, min_occurrences=5, cache_output=None):
    twitter_data = TwitterData()
    twitter_data.initialize(data_path, is_testing)
    twitter_data.build_features()
    twitter_data.cleanup(TwitterCleanuper())
    twitter_data.tokenize()
    twitter_data.build_wordlist(min_occurrences=min_occurrences)
    twitter_data.build_data_model()
    # twitter_data.build_ngrams()
    # twitter_data.build_ngram_model()
    # twitter_data.build_data_model(with_ngram=2)
    word2vec = Word2VecProvider()
    word2vec.load("/home/mike/Desktop/glove.twitter.27B.200d.txt")
    twitter_data.build_word2vec_model(word2vec)
    if cache_output is not None:
        twitter_data.data_model.to_csv(cache_output, index_label="idx", float_format="%.6f")
    results[data_name] = twitter_data.data_model

def preprare_data(min_occurrences):
    import os
    training_data = None
    testing_data = None
    print("Loading data...")
    test_data_file_name = "data/processed_test_word2vec_bow_" + str(min_occurrences) + ".csv"
    train_data_file_name = "data/processed_train_word2vec_bow_" + str(min_occurrences) + ".csv"
    use_cache = os.path.isfile(train_data_file_name) and os.path.isfile(
        test_data_file_name)
    if use_cache:
        training_data = TwitterData()
        training_data.initialize(None, from_cached=train_data_file_name)
        training_data = training_data.data_model

        testing_data = TwitterData()
        testing_data.initialize(None, from_cached=test_data_file_name)
        testing_data = testing_data.data_model
        print("Loaded from cached files...")
    else:
        print("Preprocessing data...")
        with multiprocessing.Manager() as manager:

            results = manager.dict()

            preprocess_training = Process(target=preprocess, args=(
                results, "data/train.csv", False, "train", min_occurrences, train_data_file_name,))

            preprocess_testing = Process(target=preprocess, args=(
                results, "data/train.csv", True, "test", min_occurrences, test_data_file_name,))

            preprocess_training.start()
            preprocess_testing.start()
            print("Multiple processes started...")

            preprocess_testing.join()
            print("Preprocessed testing data...")

            preprocess_training.join()
            print("Preprocessed training data...")

            training_data = results["train"]
            testing_data = results["test"]

            print("Data preprocessed & cached...")

    return training_data, testing_data

class TwitterData( TwitterData_ExtraFeatures ):

    def build_final_model (self, word2vec_provider, stopwords=nltk.corpus.stopwords.words( "english" )):
        whitelist = self.whitelist
        stopwords = list( filter( lambda sw: sw not in whitelist, stopwords ) )
        extra_columns = [col for col in self.processed_data.columns if col.startswith( "number_of" )]
        similarity_columns = ["bad_similarity", "good_similarity", "information_similarity"]
        label_column = []
        if not self.is_testing:
            label_column = ["label"]

        columns = label_column + ["original_id"] + extra_columns + similarity_columns + list(
            map( lambda i: "word2vec_{0}".format( i ), range( 0, word2vec_provider.dimensions ) ) ) + list(
            map( lambda w: w + "_bow", self.wordlist ) )
        labels = []
        rows = []
        for idx in self.processed_data.index:
            current_row = []

            if not self.is_testing:
                # add label
                current_label = self.processed_data.loc[idx, "emotion"]
                labels.append( current_label )
                current_row.append( current_label )

            current_row.append( self.processed_data.loc[idx, "id"] )

            for _, col in enumerate( extra_columns ):
                current_row.append( self.processed_data.loc[idx, col] )

            # average similarities with words
            tokens = self.processed_data.loc[idx, "tokenized_text"]
            for main_word in map( lambda w: w.split( "_" )[0], similarity_columns ):
                current_similarities = [abs( sim ) for sim in
                                        map( lambda word: word2vec_provider.get_similarity( main_word, word.lower() ),
                                             tokens ) if
                                        sim is not None]
                if len( current_similarities ) <= 1:
                    current_row.append( 0 if len( current_similarities ) == 0 else current_similarities[0] )
                    continue
                max_sim = max( current_similarities )
                min_sim = min( current_similarities )
                current_similarities = [((sim - min_sim)/(max_sim - min_sim)) for sim in
                                        current_similarities]  # normalize to <0;1>
                current_row.append( np.array( current_similarities ).mean() )

            # add word2vec vector
            tokens = self.processed_data.loc[idx, "tokenized_text"]
            current_word2vec = []
            for _, word in enumerate( tokens ):
                vec = word2vec_provider.get_vector( word.lower() )
                if vec is not None:
                    current_word2vec.append( vec )

            averaged_word2vec = list( np.array( current_word2vec ).mean( axis=0 ) )
            current_row += averaged_word2vec

            # add bag-of-words
            tokens = set( self.processed_data.loc[idx, "text"] )
            for _, word in enumerate( self.wordlist ):
                current_row.append( 1 if word in tokens else 0 )

            rows.append( current_row )

        self.data_model = pd.DataFrame( rows, columns=columns )
        self.data_labels = pd.Series( labels )
        return self.data_model, self.data_labels

def log(text):
    print(text)
    with open("log.txt", "a") as log_file:
        log_file.write(str(text) + "\n")

if __name__ == "__main__":

    def main():

        for m in range( 3, 4 ):
            print("Preparing data with min_occurrences=" + str( m ))
            training_data, testing_data = preprare_data( m )
            log( "********************************************************" )
            log( "Validating for {0} min_occurrences:".format( m ) )
            # drop idx & id columns
            # if training_data.columns[0] == "idx":
            #     training_data = training_data.iloc[:, 1:]
            #
            # if testing_data.columns[0] == "idx":
            #     testing_data = testing_data.iloc[:, 1:]
            #
            # if "original_id" in training_data.columns:
            #     training_data.drop( "original_id", axis=1, inplace=True )
            #
            # if "original_id" in testing_data.columns:
            #     testing_data.drop( "original_id", axis=1, inplace=True )

            td = TwitterData()
            td.initialize( "data\\train.csv" )
            td.build_features()
            td.cleanup( TwitterCleanuper() )
            td.tokenize()
            td.stem()
            td.build_wordlist()
            td.build_final_model( word2vec )

            td.data_model.head( 5 )

        print("Done!")

    main()


'''
다음은 ISLR패키지의 Carseats 데이터 세트이다.
매출(Sales)의 이상값을 제외한 데이를 훈련 데이터로 선정할 때
Age의 표준편차를 구하시오.
(이상 값은 평균보다 1.5표준편차이하이거나 이상인 값으로 선정한다.
'''
import pandas as pd
data = 'Carseats.csv'
df = pd.read_csv(data)
#print(df)
#print(df.info())
#print(df.describe())

chk = df.Sales.std()
#print(chk)
train = df[(df.Sales < (df.Sales.mean() + chk*1.5)) & (df.Sales > (df.Sales.mean() - chk*1.5))]
ans = train.Age.std()
print(ans)

'''
다음은 MASS 패키지의 Cars93 데이터 세트이다.
Luggage.room의 결측값을 중앙값으로 변환한 후
변환 전, 후 평균의 차이를 구하시오.
'''
import pandas as pd
data = 'Cars93.csv'
df = pd.read_csv(data)
#print(df)
#print(df.info())
#print(df.describe())

before = df['Luggage.room'].mean()
df.loc[df['Luggage.room'].isnull(), 'Luggage.room'] = df['Luggage.room'].fillna(df['Luggage.room'].median())
after = df['Luggage.room'].mean()
ans = abs(before - after)
print(ans)

'''
다음은 Covid19의 TimeAge데이터 세트이다.
연령(age)이 20대(20s)인 확진자(confirmed)의 평균과
50대(50s)인 확진자(confirmed) 평균의 차이를 구하시오.
'''
import pandas as pd
data = 'TimeAge.csv'
df = pd.read_csv(data)
#print(df)
#print(df.info())
#print(df.describe())

chk = df.groupby('age').mean().reset_index()
#print(chk)
ans = chk[chk.age == '20s'].confirmed.values[0] - chk[chk.age == '50s'].confirmed.values[0]
print(ans)


from django.conf import settings
from django.urls import NoReverseMatch, reverse

def get_slug_or_pk(object, slug_field=None):
    res = dict()
    field = slug_field if hasattr(object, slug_field) else "pk"
    if object:
        param = "slug" if hasattr(object, slug_field) else "pk"
        res.update({param: getattr(object, field)})
    return res

def get_urls_of_site(site, object=None, user=None):
    urls = {}
    app = site.model._meta.app_label
    model = site.model._meta.model_name
    kwargs = get_slug_or_pk(object, slug_field=site.slug_field)
    for action, perm in (("list", "view"), ("create", "add")):
        try:
            url_name = site.get_url_name(action)
            if not user:
                urls.update({action: reverse(url_name)})
            elif user.has_perm(f"{app}.{perm}_{model}"):
                urls.update({action: reverse(url_name)})
        except NoReverseMatch:
            if settings.DEBUG:
                print("DEBUG: Url not found: %s" % url_name)
    if not kwargs:
        return urls
    for action, perm in (
        ("update", "change"),
        ("detail", "view"),
        ("delete", "delete"),
    ):
        try:
            url_name = site.get_url_name(action)
            if not user:
                urls.update({action: reverse(url_name, kwargs=kwargs)})
            elif user.has_perm(f"{app}.{perm}_{model}"):
                urls.update({action: reverse(url_name, kwargs=kwargs)})
        except NoReverseMatch:
            if settings.DEBUG:
                print("DEBUG: Url not found: %s" % url_name)
    return urls


"""
Name : portfolio_optimizer.py
Author : Yinsen Miao
 Contact : yinsenm@gmail.com
Time    : 7/21/2021
Desc: Solve mean-variance optimization
"""

import numpy as np
import pandas as pd
from scipy.optimize import minimize
from gerber import gerber_cov_stat1, gerber_cov_stat2
from ledoit import ledoit

def set_eps_wgt_to_zeros(in_array, eps=1e-4):
    # set small weights to 0 and return a list
    out_array = np.array(in_array)
    out_array[np.abs(in_array) < eps] = 0
    out_array = np.array(out_array) / np.sum(out_array)
    return out_array

class portfolio_optimizer:
    def __init__(self, min_weight: float = 0., max_weight: float = 1.0,
                 cov_function: str = "HC",
                 freq: str = "monthly",
                 gs_threshold: float = 0.5):
        """
        :param min_weight:
        :param max_weight:
        :param cov_function: can be one of the HC (historical covariance matrix), GS1 (Gerber Stat1), GS2 (Gerber Stat2)
        :param freq: frequency of the returns series either daily or monthly
        :param gs_threshold: threshold of Gerber statistics between 0 and 1
        """
        # check arguments
        assert cov_function in ['HC', 'GS1', 'GS2', 'SM'], "The covariance function must be one from HC, SM, GS1, and GS2"
        assert freq in ['daily', 'monthly'], "The return series can only be either daily or monthly"
        assert 1 > min_weight >= 0, "The minimal weight shall be in [0, 1)"
        assert 1 >= max_weight > 0, "The maximum weight shall be in (0, 1]"
        assert 1 >= gs_threshold > 0, "The Gerber shrinkage threshold shall be in (0, 1]"

        self.min_weight = min_weight
        self.max_weight = max_weight

        self.factor = 252 if freq == "daily" else 12  # annual converter
        self.cov_function = cov_function  # covariance function can be one of HC, GS1, GS2
        self.freq = freq  # freq of return series can be either daily or monthly
        self.init_weights = None  # initial portfolio weights
        self.covariance = None
        self.returns_df = None
        self.negative_returns_df = None
        self.covariance_neg = None  # covariance matrix of only negative returns for sortino ratio
        self.obj_function = None
        self.by_risk = None
        self.gs_threshold = gs_threshold

    def set_returns(self, returns_df: pd.DataFrame):
        """
        pass the return series to the class
        :param returns_df: pd.DataFrame of historical daily or monthly returns
        """
        self.returns_df = returns_df.copy(deep=True)
        self.negative_returns_df = returns_df[returns_df < 0].fillna(0)  # keep only the negative returns

    def optimize(self, obj_function: str,
                 target_std: float = None,
                 target_return: float = None,
                 prev_weights: np.array = None,
                 init_weights: np.array = None,
                 cost: float = None) -> np.array:
        """
        Perform portfolio optimization given a series of returns
        :param obj_function:
        :param target_std: targeted annaulized portfolio standard deviation (std)
        :param target_return: targeted annaulized portfolio return deviation
        :param prev_weights: previous weights
        :param prices: current price level when we rebalance our portfolio
        :param cost: cost of transaction fee and slippage in bps or 0.01%
        :return: an array of portfolio weights p x 1
        """
        n, p = self.returns_df.shape  # n is number of observations, p is number of assets

        if init_weights is None:
            self.init_weights = np.array(p * [1. / p])  # initialize weights: equal weighting
        else:
            self.init_weights = init_weights  # otherwise use the nearby weights as hot start for MVO

        self.obj_function = obj_function

        # get covariance matrix
        if self.cov_function == "HC":
            self.covariance = self.returns_df.cov().to_numpy()  # convert to numpy
            self.covariance_neg = self.negative_returns_df.cov().to_numpy()  # convert to numpy
        elif self.cov_function == "SM":
            self.covariance, _ = ledoit(self.returns_df.values)
            self.covariance_neg, _ = ledoit(self.negative_returns_df.values)
        elif self.cov_function == "GS1":
            self.covariance, _ = gerber_cov_stat1(self.returns_df.values, threshold=self.gs_threshold)
            self.covariance_neg, _ = gerber_cov_stat1(self.negative_returns_df.values, threshold=self.gs_threshold)
        elif self.cov_function == "GS2":
            self.covariance, _ = gerber_cov_stat2(self.returns_df.values, threshold=self.gs_threshold)
            self.covariance_neg, _ = gerber_cov_stat2(self.negative_returns_df.values, threshold=self.gs_threshold)

        # set objective function
        if obj_function == "equalWeighting":
            self.init_weights = np.array(p * [1. / p])  # initialize weights: equal weighting
            return self.init_weights

        # set the bounds of each asset holding from 0 to 1
        bounds = tuple((self.min_weight, self.max_weight) for k in range(p))
        constraints = [{'type': 'eq', 'fun': lambda x: np.sum(x) - 1.0}]  # fully invest

        if obj_function == 'meanVariance':
            if target_std is not None:
                self.by_risk = True
                # optimize under risk constraint
                constraints.append({'type': 'eq', 'fun': lambda weights: \
                    self.calc_annualized_portfolio_std(weights) - target_std})
            else:
                # optimize under return constraint
                self.by_risk = False
                constraints.append({'type': 'eq', 'fun': lambda weights: \
                    self.calc_annualized_portfolio_return(weights) - target_return})

        if prev_weights is not None and cost is not None:
            # cost function with transaction fee
            cost_fun = lambda weights: self.object_function(weights) +\
                                       np.abs(weights - prev_weights).sum() * cost / 10000.
        else:
            # cost function without any transaction fee
            cost_fun = lambda weights: self.object_function(weights)

        # trust-constr, SLSQP, L-BFGS-B
        try:
            opt = minimize(cost_fun, x0=self.init_weights, bounds=bounds, constraints=constraints, method="SLSQP")
        except:
            # if SLSQP fails then switch to trust-constr
            opt = minimize(cost_fun, x0=self.init_weights, bounds=bounds, constraints=constraints, method="trust-constr")

        return set_eps_wgt_to_zeros(opt['x'])   # pull small values to zeros

    def object_function(self, weights: np.array) -> float:
        """
        :param weights: current weights to be optimized
        """

        if self.obj_function == "maxReturn":
            f = self.calc_annualized_portfolio_return(weights)
            return -f
        elif self.obj_function == "minVariance":
            f = self.calc_annualized_portfolio_std(weights)
            return f
        elif self.obj_function == "meanVariance" and self.by_risk:
            f = self.calc_annualized_portfolio_return(weights)  # maximize target return level
            return -f
        elif self.obj_function == "meanVariance" and not self.by_risk:
            f = self.calc_annualized_portfolio_std(weights)  # minimize target risk or std level
            return f
        elif self.obj_function == "maxSharpe":
            f = self.calc_annualized_portfolio_sharpe_ratio(weights)
            return -f
        elif self.obj_function == "maxSortino":
            f = self.calc_annualized_sortino_ratio(weights)
            return -f
        elif self.obj_function == 'riskParity':
            f = self.calc_risk_parity_func(weights)
            return f
        else:
            raise ValueError("Object function shall be one of the equalWeighting, maxReturn, minVariance, " +
                             "meanVariance, maxSharpe, maxSortino or riskParity")

    def calc_annualized_portfolio_return(self, weights: np.array) -> float:
        # calculate the annualized standard returns
        annualized_portfolio_return = float(np.sum(self.returns_df.mean() * self.factor * weights))
        #float(np.sum(((1 + self.returns_df.mean()) ** self.factor - 1) * weights))
        return annualized_portfolio_return

    def calc_annualized_portfolio_std(self, weights: np.array) -> float:
        if self.obj_function == "equalWeighting":
            # if equal weight then set the off diagonal of covariance matrix to zero
            annualized_portfolio_std = np.sqrt(np.dot(weights.T, np.dot(np.diag(self.covariance.diagonal()) * self.factor, weights)))
        else:
            temp = np.dot(weights.T, np.dot(self.covariance * self.factor, weights))
            if temp <= 0:
                temp = 1e-20  # set std to a tiny number
            annualized_portfolio_std = np.sqrt(temp)
        if annualized_portfolio_std <= 0:
            raise ValueError('annualized_portfolio_std cannot be zero. Weights: {weights}')
        return annualized_portfolio_std

    def calc_annualized_portfolio_neg_std(self, weights: np.array) -> float:
        if self.obj_function == "equalWeighting":
            # if equal weight then set the off diagonal of covariance matrix to zero
            annualized_portfolio_neg_std = np.sqrt(np.dot(weights.T, np.dot(np.diag(self.covariance_neg.diagonal()) * self.factor, weights)))
        else:
            annualized_portfolio_neg_std = np.sqrt(np.dot(weights.T, np.dot(self.covariance_neg * self.factor, weights)))
        if annualized_portfolio_neg_std == 0:
            raise ValueError('annualized_portfolio_std cannot be zero. Weights: {weights}')
        return annualized_portfolio_neg_std

    def calc_annualized_portfolio_moments(self, weights: np.array) -> tuple:
        # calculate the annualized portfolio returns as well as its standard deviation
        return self.calc_annualized_portfolio_return(weights), self.calc_annualized_portfolio_std(weights)

    def calc_annualized_portfolio_sharpe_ratio(self, weights: np.array) -> float:
        # calculate the annualized Sharpe Ratio
        return self.calc_annualized_portfolio_return(weights) / self.calc_annualized_portfolio_std(weights)

    def calc_annualized_sortino_ratio(self, weights: np.array) -> float:
        # calculate the annualized Sortino Ratio
        return self.calc_annualized_portfolio_return(weights) / self.calc_annualized_portfolio_neg_std(weights)

    def calc_risk_parity_func(self, weights):
        # Spinu formulation of risk parity portfolio
        assets_risk_budget = self.init_weights
        portfolio_volatility = self.calc_annualized_portfolio_std(weights)

        x = weights / portfolio_volatility
        risk_parity = (np.dot(x.T, np.dot(self.covariance * self.factor, x)) / 2.) - np.dot(assets_risk_budget.T, np.log(x + 1e-10))
        return risk_parity

    def calc_relative_risk_contributions(self, weights):
        # calculate the relative risk contributions for each asset given returns and weights
        rrc = weights * np.dot(weights.T, self.covariance) / np.dot(weights.T, np.dot(self.covariance, weights))
        return rrc

# unitest the code
if __name__ == "__main__":
    bgn_date = "2016-01-01"
    end_date = "2020-01-01"
    file_path = "../data/prcs.csv"
    rets_df = pd.read_csv(file_path, parse_dates=['Date'], index_col=["Date"]).pct_change()[bgn_date: end_date]
    rets = rets_df.values

    # test objective function list
    obj_function_list = ['equalWeighting', 'minVariance', 'maxReturn', 'maxSharpe', 'maxSortino', 'riskParity']
    cov_function_list = ["HC", "SM", "GS1", "GS2"]

    for cov_fun in cov_function_list:
        print("MVO based on %s covariance function ..." % cov_fun)
        port_opt = portfolio_optimizer(min_weight=0, max_weight=1, cov_function=cov_fun, freq="monthly")
        port_opt.set_returns(returns_df=rets_df)
        # run MVO under various optimization goals

        for obj_fun_str in obj_function_list:
            weights = port_opt.optimize(obj_fun_str)
            ret, std = port_opt.calc_annualized_portfolio_moments(weights=weights)
            sharpe = port_opt.calc_annualized_portfolio_sharpe_ratio(weights=weights)
            sortino = port_opt.calc_annualized_sortino_ratio(weights=weights)
            print("%20s: ret %.3f, std %.3f, Sharpe %.3f, Sortino %.3f" % (obj_fun_str, ret, std, sharpe, sortino))
        obj_fun_str = "meanVariance"
        # optimize for target std levels
        target_stds = [3, 6, 9, 12, 15]
        for target_std in target_stds:
            weights = port_opt.optimize(obj_fun_str, target_std / 100.)
            # print(weights)
            ret, std = port_opt.calc_annualized_portfolio_moments(weights=weights)
            sharpe = port_opt.calc_annualized_portfolio_sharpe_ratio(weights=weights)
            sortino = port_opt.calc_annualized_sortino_ratio(weights=weights)
            print("%20s (%02d%%): ret %.3f, std %.3f, Sharpe %.3f, Sortino %.3f" % (
                obj_fun_str, target_std, ret, std, sharpe, sortino))


# Importing modules
import numpy as np
import pandas as pd
from sklearn.model_selection import train_test_split
import joblib
import findspark
findspark.init()
from pyspark.sql import SparkSession
from pyspark.ml import PipelineModel
from pyspark.sql.functions import *

# Configure spark session
spark = SparkSession\
    .builder\
    .master('local[2]')\
    .appName('quake_etl')\
    .config('spark.jars.package', 'org.mongodb.spark:mongo-spark-connector 2.12:2.4.1')\
    .getOrCreate()
from bokeh.io import output_notebook, output_file
from bokeh.plotting import figure, show, ColumnDataSource
from bokeh.models.tools import HoverTool
import math
from math import pi
from bokeh.palettes import Category20c
from bokeh.transform import cumsum
from bokeh.tile_providers import CARTODBPOSITRON, get_provider, Vendors
from bokeh.themes import built_in_themes
from bokeh.io import curdoc
import warnings
warnings.filterwarnings('ignore')
from pyspark.sql.functions import desc

df=joblib.load('./joblibs/df.joblib')
data=spark.createDataFrame(df)
df_quake_freq=joblib.load('./visualization_files/df_quake_freq.joblib')
df_pred=joblib.load('./visualization_files/rffpred.joblib')
def svm(a):
    clf=joblib.load('./joblibs/svmModel.joblib')
    a=np.array([a])
    y_pred_svm=clf.predict(a)
    return y_pred_svm[0]/10
def dt(a):
    a=np.array([a])
    dc=joblib.load('./joblibs/dtModel.joblib')
    y_pred_dc=dc.predict(a)
    return y_pred_dc[0]
def rf(a):
    pipe=PipelineModel.load("./joblibs/rfmodel.model_v0")
    ip = pd.DataFrame(np.array([a]))
    ip.columns=['Latitude','Longitude','Depth']
    dip=spark.createDataFrame(ip)
    pred_results_RF = pipe.transform(dip)
    return pred_results_RF.collect()[0][4]
def knn(a):
    from sklearn.neighbors import KNeighborsRegressor
    #Seperating X and y
    X=df[['Year','Latitude','Longitude','Depth']]
    y=df[['Magnitude']]
    kneigh=KNeighborsRegressor(n_neighbors = 5)
    kneigh.fit(X, y.values.ravel())
    a=np.array([a])
    y_pred_knn=kneigh.predict(a)
    return y_pred_knn[0]
def style(p):

        p.title.align='center'
        p.title.text_font_size = '20pt'
        p.title.text_font = 'serif'

        p.xaxis.axis_label_text_font_size = '14pt'
        p.xaxis.axis_label_text_font_style= 'bold'
        p.yaxis.axis_label_text_font_size = '14pt'
        p.yaxis.axis_label_text_font_style= 'bold'

        p.xaxis.major_label_text_font_size = '12pt'
        p.yaxis.major_label_text_font_size = '12pt'

        p.legend.location = 'top_left'
        return p
def showmap():

        # Earthquakein Map Representation
    df_quakes_2016 = data[data['Year']==2016]

    df_quakes_2016=df_quakes_2016.toPandas()

    def plotMap():
        lat = df_quakes_2016['Latitude'].values.tolist()
        lon = df_quakes_2016['Longitude'].values.tolist()

        pred_lat = df_pred['Latitude'].values.tolist()
        pred_lon = df_pred['Longitude'].values.tolist()

        lst_lat = []
        lst_lon = []
        lst_pred_lat = []
        lst_pred_lon = []

        i=0
        j=0

        for i in range (len(lon)):
            r_major = 6378137.000
            x = r_major * math.radians(lon[i])
            scale = x/lon[i]
            y = 180.0/math.pi * math.log(math.tan(math.pi/4.0 +
            lat[i] * (math.pi/180.0)/2.0)) * scale

            lst_lon.append(x)
            lst_lat.append(y)
            i += 1

        for j in range (len(pred_lon)):
            r_major = 6378137.000
            x = r_major * math.radians(pred_lon[j])
            scale = x/pred_lon[j]
            y = 180.0/math.pi * math.log(math.tan(math.pi/4.0 +
            pred_lat[j] * (math.pi/180.0)/2.0)) * scale

            lst_pred_lon.append(x)
            lst_pred_lat.append(y)
            j += 1

        df_quakes_2016['coords_x'] = lst_lat
        df_quakes_2016['coords_y'] = lst_lon
        df_pred['coords_x'] = lst_pred_lat
        df_pred['coords_y'] = lst_pred_lon

        df_quakes_2016['Mag_Size'] = df_quakes_2016['Magnitude'] * 4
        df_pred['Mag_Size'] = df_pred['Pred_Magnitude'] * 4

        lats = df_quakes_2016['coords_x'].tolist()
        longs = df_quakes_2016['coords_y'].tolist()
        mags = df_quakes_2016['Magnitude'].tolist()
        years = df_quakes_2016['Year'].tolist()
        mag_size = df_quakes_2016['Mag_Size'].tolist()

        pred_lats = df_pred['coords_x'].tolist()
        pred_longs = df_pred['coords_y'].tolist()
        pred_mags = df_pred['Pred_Magnitude'].tolist()
        pred_year = df_pred['Year'].tolist()
        pred_mag_size = df_pred['Mag_Size'].tolist()

        cds = ColumnDataSource(
        data=dict(
        lat=lats,
        lon=longs,
        mag=mags,
        year=years,
        mag_s=mag_size
        )
        )

        pred_cds = ColumnDataSource(
        data=dict(
        pred_lat=pred_lats,
        pred_long=pred_longs,
        pred_mag=pred_mags,
        year=pred_year,
        pred_mag_s=pred_mag_size
        )
        )

        TOOLTIPS = [( "Year", "@year"), ("Magnitude", "@mag"),("Predicted Magnitude", "@pred_mag")
        ]

        p = figure(title = 'Earthquake Map',
        plot_width=2300, plot_height=450,
        x_range=(-2000000, 6000000),
        y_range=(-1000000, 7000000),
        tooltips=TOOLTIPS)

        p.circle(x='lon', y='lat', size='mag_s', fill_color='#cc0000', fill_alpha=0.7,
        source=cds, legend='Quakes 2016')

        p.circle(x='pred_long', y='pred_lat', size='pred_mag_s', fill_color='#ccff33', fill_alpha=7.0,
        source=pred_cds, legend='Predicted Quakes 2017')

        tile_provider = get_provider(Vendors.CARTODBPOSITRON)
        p.add_tile(tile_provider)

        p.title.align='center'
        p.title.text_font_size='20pt'
        p.title.text_font='serif'

        p.legend.location='bottom_right'
        p.legend.background_fill_color='black'
        p.legend.background_fill_alpha=0.8
        p.legend.click_policy='hide'
        p.legend.label_text_color='white'
        p.xaxis.visible=False
        p.yaxis.visible=False
        p.axis.axis_label=None
        p.axis.visible=False
        p.grid.grid_line_color=None
        show(p)
    df_quakes_2016['Magnitude']
    plotMap()

def freqgraph():
    #Frequency of Earthquake By Year
    def plotBar():

        cds = ColumnDataSource(data=dict(
            yrs= df_quake_freq[ 'Year'].values.tolist(),
            numQuakes = df_quake_freq['Counts'].values.tolist()))

        TOOLTIPS =[ ('Number of earthquakes','@numQuakes'),('Year','@yrs')]

        barChart = figure(title='Frequency of Earthquakes by Year',
                    plot_height=400,
                    plot_width=1150,
                    x_axis_label='Years',
                    y_axis_label='Number of Occurances',
                    x_minor_ticks=2,
                    y_range=(0, df_quake_freq['Counts'].max() +100),
                    toolbar_location=None,
                    tooltips=TOOLTIPS)

        print(cds)
        barChart.vbar (x='yrs', bottom=0, top='numQuakes',
                color='#cc0000', width=0.75,
                legend='Year', source=cds)

        barChart = style(barChart)
        show(barChart)
        return barChart
    plotBar()

def maggraph():
    def plotMagnitude():

        cds= ColumnDataSource(data=dict(
        yrs = df_quake_freq[ 'Year'].sort_values().values.tolist(),
        avg_mag = df_quake_freq['avg(Magnitude)'].round(1).values.tolist(),
        max_mag= df_quake_freq [ 'max(Magnitude)'].values.tolist()))

        TOOLTIPS = [('Year', '@yrs'),('avg(Magnitude)', '@avg_mag'),('max(Magnitude)','@max_mag')]

        mp = figure(title='Maximum and Average Magnitude by Year',
                    plot_width=1150,
                    plot_height=400,
        x_axis_label='Years',
        y_axis_label='Magnitude', y_range=(5, df_quake_freq[ 'max(Magnitude)'].max() + 1),
        x_minor_ticks=2,
        toolbar_location=None,
        tooltips= TOOLTIPS)

        mp.line(x='yrs', y='max_mag', color='#cc0000', line_width=2, legend= 'Max Magnitude', source=cds)
        mp.circle(x='yrs', y='max_mag', color='#cc0000', size=8, fill_color='#cc0000', source=cds)

        mp.line(x='yrs', y='avg_mag', color='yellow', line_width=2, legend = 'Avg Magnitude', source=cds)
        mp.circle(x='yrs', y='avg_mag', color='yellow', size=8, fill_color='yellow', source=cds)
        mp =style(mp)
        show(mp)
        return mp

    plotMagnitude()