import sympy as sp
import numpy as np
from IPython.display import display, Latex

# Asymmetric loss ASL
class AsymmetricLossObjective(object):
    def __init__(self, der1, der2, gamma_minus=.2, gamma_plus=.6):
        self.g_minus = gamma_minus
        self.g_plus = gamma_plus
        self.der1 = der1
        self.der2 = der2

    def calc_ders_range(self, x, y, weights):
        assert len(x) == len(y)
        if weights is not None:
            assert len(weights) == len(x)

        e = np.exp(x)
        p = e / (1 + e)
        log_p = np.log(p)
        log_1minus_p = np.log(1-p)

        der1 = self.der1(p=p, y=y, g_minus=self.g_minus, g_plus=self.g_plus, log_p=log_p, log_1minus_p=log_1minus_p)
        der2 = self.der2(p=p, y=y, g_minus=self.g_minus, g_plus=self.g_plus, log_p=log_p, log_1minus_p=log_1minus_p)

        # der1 = self.der1(p=p, y=y, g_minus=self.g_minus, g_plus=self.g_plus)
        # der2 = self.der2(p=p, y=y, g_minus=self.g_minus, g_plus=self.g_plus)

        if weights is not None:
            der1 *= weights
            der2 *= weights

        result = list(zip(der1, der2))

        return result

class LoglossObjective_loop(object):
    def calc_ders_range(self, approxes, targets, weights):
        assert len(approxes) == len(targets)
        if weights is not None:
            assert len(weights) == len(approxes)

        result = []
        for index in range(len(targets)):
            e = np.exp(approxes[index])
            p = e / (1 + e)
            der1 = targets[index] - p
            der2 = -p * (1 - p)

            if weights is not None:
                der1 *= weights[index]
                der2 *= weights[index]

            result.append((der1, der2))
        return result

#%%

class LoglossObjective_np(object):
    def calc_ders_range(self, approxes, targets, weights):
        assert len(approxes) == len(targets)
        if weights is not None:
            assert len(weights) == len(approxes)

        e = np.exp(approxes)
        p = e / (1 + e)
        der1 = targets - p
        der2 = -p * (1 - p)

        if weights is not None:
            der1 *= weights
            der2 *= weights

        result = list(zip(der1, der2))
        return result

def FocalLossFormulas():
    x, y, gamma = sp.symbols('x y gamma')

    p = sp.exp(x) / (1 + sp.exp(x))
    display(sp.Eq(sp.S('p'), p))

    L_plus = (1 - p)**gamma * sp.log(p)
    L_minus = p**gamma * sp.log(1 - p)

    display(sp.Eq(sp.S('L_plus'), L_plus))
    display(sp.Eq(sp.S('L_minus'), L_minus))

    L = -y*L_plus - (1 - y)*L_minus
    display(sp.Eq(sp.S('L'), L))

    der1 = sp.diff(L, x)
    display(sp.Eq(sp.S('L__der1'), der1))

    der2 = sp.diff(der1, x)
    display(sp.Eq(sp.S('L__der2'), der2))
    return der1, der2

# der1, der2 = FocalLossFormulas()

def AsymmetricLossFormulas():
    x, y, g_minus, g_plus = sp.symbols('x y g_minus g_plus')

    p = sp.exp(x) / (1 + sp.exp(x))
    display(sp.Eq(sp.S('p'), p))

    L_plus = (1 - p)**g_plus * sp.log(p)
    L_minus = p**g_minus * sp.log(1 - p)

    display(sp.Eq(sp.S('L_plus'), L_plus))
    display(sp.Eq(sp.S('L_minus'), L_minus))

    L = -y*L_plus - (1 - y)*L_minus
    display(sp.Eq(sp.S('L'), L))

    der1 = sp.diff(L, x)
    display(sp.Eq(sp.S('L__der1'), der1))

    der2 = sp.diff(der1, x)
    display(sp.Eq(sp.S('L__der2'), der2))
    return der1, der2

def get_simplified_derivative(derivative, verbose=False, der_name='ALS_der'):
    """
    Упрощает выражение для производной методами библиотеки sympy

    :param derivative: функция производной, полученная методом sympy.diff()
    :param verbose: Вывести промежуточные формулы на экран, могут быть длинными
    :return: Функция производной, оптимизированная для numpy
    """

    def print_der(der, message):
        if verbose:
            display(message)
            display(sp.Eq(sp.S(der_name), der))

    p, x, y = sp.symbols('p x y')
    g_minus, g_plus = sp.symbols('g_minus g_plus')
    log_p, log_1minus_p = sp.symbols('log_p log_1minus_p')

    # log_p, log_1minus_p = sp.symbols(log_p, log_1minus_p)
    der = derivative

    print_der(derivative, 'Входящая функция производной')

    der = der.subs(sp.exp(x) / (1 + sp.exp(x)), p)
    print_der(der, '*********** der.subs(sp.exp(x) / (1 + sp.exp(x)), p) *************')

    der = der.subs(sp.exp(x) / (sp.exp(x) + 1), p)
    print_der(der, '*********** der.subs(sp.exp(x) / (1 + sp.exp(x)), p) *************')

    der = der.subs((1 + sp.exp(x)) ** 2 * sp.exp(x) ** (-2), p ** (-2))
    print_der(der, '**********    der.subs((1 + sp.exp(x))**2 * sp.exp(x)**(-2), p**(-2))  *****************')

    der = der.subs(sp.exp(x), p / (1 - p))
    print_der(der, '**********    der.subs(sp.exp(x), p / (1-p))  *****************')

    der = der.subs(sp.log(p), log_p)
    print_der(der, '**********    der.subs(sp.log(p), log_p)  *****************')

    der = der.subs(sp.log(1-p), log_1minus_p)
    print_der(der, '**********    der.subs(sp.log(1-p), log_1minus_p)  *****************')

    der = sp.simplify(der)
    print_der(der, '*********** sp.symplify(der) *************')

    der = sp.factor(der)
    print_der(der, '*********** sp.factor(der) *************')

    der_lambdify = sp.lambdify([p, y, g_minus, g_plus, log_p, log_1minus_p], der, 'numpy')

    return der_lambdify

# der_l2 = get_simplified_derivative(ALS_der2, verbose=True, der_name='ALS_der2')


import numpy as np
import torch
import torch.nn as nn
from torch.nn import init
import torch.optim as optim
import math
import random
import os
from pathlib import Path
import time
from tqdm import tqdm
from data_loader import fetch_data
from gensim.models import Word2Vec

unk = '<UNK>'

class RNN(nn.Module):
    def __init__(self, input_dim, hidden_dim, n_layers): # Add relevant parameters
        super(RNN, self).__init__()
        # Fill in relevant parameters
        self.hidden_dim = hidden_dim
        self.n_layers = n_layers
        self.rnn = nn.RNN(input_dim, hidden_dim, n_layers)
        self.fc = nn.Linear(hidden_dim, 5)
        # Ensure parameters are initialized to small values, see PyTorch documentation for guidance
        self.softmax = nn.LogSoftmax()
        self.loss = nn.NLLLoss()

    def compute_Loss(self, predicted_vector, gold_label):
        return self.loss(predicted_vector, gold_label)

    def forward(self, inputs):
        #begin code
        hidden = torch.zeros(self.n_layers, self.hidden_dim)
        out, hidden = self.rnn(inputs, hidden)
        out = out.contiguous().view(-1, self.hidden_dim)
        out = self.fc(out)
        predicted_vector = self.softmax(out) # Remember to include the predicted unnormalized scores which should be normalized into a (log) probability distribution
        #end code
        return predicted_vector

# You may find the functions make_vocab() and make_indices from ffnn.py useful; you are free to copy them directly (or call those functions from this file)

# Returns:
# vocab = A set of strings corresponding to the vocabulary
def make_vocab(data):
    vocab = set()
    for document, _ in data:
        for word in document:
            vocab.add(word)
    return vocab

# Returns:
# vocab = A set of strings corresponding to the vocabulary including <UNK>
# word2index = A dictionary mapping word/token to its index (a number in 0, ..., V - 1)
# index2word = A dictionary inverting the mapping of word2index
def make_indices(vocab):
    vocab_list = sorted(vocab)
    vocab_list.append(unk)
    word2index = {}
    index2word = {}
    for index, word in enumerate(vocab_list):
        word2index[word] = index
        index2word[index] = word
    vocab.add(unk)
    return vocab, word2index, index2word

# Returns:
# vectorized_data = A list of pairs (vector representation of input, y)
def convert_to_vector_representation(data):
    sentences = []
    vectorized_data = []
    for document, _ in data:
        sentences.append(document)
    model = Word2Vec(sentences, min_count=1)
    for document, y in data:
        vector = []
        for word in document:
            vector.append(model[word])
        vectorized_data.append((vector, y))
    return vectorized_data

def main(hidden_dim, n_layers, number_of_epochs): # Add relevant parameters
    print("Fetching data")
    train_data, valid_data = fetch_data() # X_data is a list of pairs (document, y); y in {0,1,2,3,4}
    vocab = make_vocab(train_data)
    vocab, word2index, index2word = make_indices(vocab)
    print("Fetched and indexed data")
    train_data = convert_to_vector_representation(train_data)
    valid_data = convert_to_vector_representation(valid_data)
    print("Vectorized data")
    # Think about the type of function that an RNN describes. To apply it, you will need to convert the text data into vector representations.
    # Further, think about where the vectors will come from. There are 3 reasonable choices:
    # 1) Randomly assign the input to vectors and learn better embeddings during training; see the PyTorch documentation for guidance
    # 2) Assign the input to vectors using pretrained word embeddings. We recommend any of {Word2Vec, GloVe, FastText}. Then, you do not train/update these embeddings.
    # 3) You do the same as 2) but you train (this is called fine-tuning) the pretrained embeddings further.
    # Option 3 will be the most time consuming, so we do not recommend starting with this

    model = RNN(len(vocab), hidden_dim, n_layers) # Fill in parameters
    optimizer = optim.SGD(model.parameters(), lr=0.01, momentum=0.8)
    print("Training for {} epochs".format(number_of_epochs))
    for epoch in range(number_of_epochs): # How will you decide to stop training and why
        # You will need further code to operationalize training, ffnn.py may be helpful
        model.train()
        optimizer.zero_grad()
        loss = None
        correct = 0
        total = 0
        start_time = time.time()
        print("Training started for epoch {}".format(epoch + 1))
        random.shuffle(train_data) # Good practice to shuffle order of training data
        minibatch_size = 16
        N = len(train_data)
        for minibatch_index in tqdm(range(N // minibatch_size)):
            optimizer.zero_grad()
            loss = None
            for example_index in range(minibatch_size):
                input_vector, gold_label = train_data[minibatch_index * minibatch_size + example_index]
                predicted_vector = model.forward(input_vector)
                predicted_label = torch.argmax(predicted_vector)
                correct += int(predicted_label == gold_label)
                total += 1
                example_loss = model.compute_Loss(predicted_vector.view(1,-1), torch.tensor([gold_label]))
                if loss is None:
                    loss = example_loss
                else:
                    loss += example_loss
            loss = loss / minibatch_size
            loss.backward()
            optimizer.step()
        print("Training completed for epoch {}".format(epoch + 1))
        print("Training accuracy for epoch {}: {}".format(epoch + 1, correct / total))
        print("Training time for this epoch: {}".format(time.time() - start_time))
        loss = None
        correct = 0
        total = 0
        start_time = time.time()
        print("Validation started for epoch {}".format(epoch + 1))
        random.shuffle(valid_data) # Good practice to shuffle order of validation data
        minibatch_size = 16
        N = len(valid_data)
        for minibatch_index in tqdm(range(N // minibatch_size)):
            #optimizer.zero_grad()
            loss = None
            for example_index in range(minibatch_size):
                input_vector, gold_label = valid_data[minibatch_index * minibatch_size + example_index]
                predicted_vector = model.forward(input_vector)
                predicted_label = torch.argmax(predicted_vector)
                correct += int(predicted_label == gold_label)
                total += 1
                example_loss = model.compute_Loss(predicted_vector.view(1,-1), torch.tensor([gold_label]))
                if loss is None:
                    loss = example_loss
                else:
                    loss += example_loss
            loss = loss / minibatch_size
            #loss.backward()
            #optimizer.step()
        print("Validation completed for epoch {}".format(epoch + 1))
        print("Validation accuracy for epoch {}: {}".format(epoch + 1, correct / total))
        print("Validation time for this epoch: {}".format(time.time() - start_time))
        # You may find it beneficial to keep track of training accuracy or training loss;

        # Think about how to update the model and what this entails. Consider ffnn.py and the PyTorch documentation for guidance

        # You will need to validate your model. All results for Part 3 should be reported on the validation set.
        # Consider ffnn.py; making changes to validation if you find them necessary


import math

from envs.csb.unit import Unit
from envs.csb.point import Point
from envs.csb.util import MAX_THRUST, TIMEOUT

class Pod(Unit):

    __slots__ = ('angle', 'ncpid', 'timeout', 'shield', 'lap', 'world', 'boost_available')

    def __init__(self, id, x, y, world):
        super().__init__(id, x, y, 400, 0, 0)
        self.angle = 0
        self.ncpid = 1
        self.timeout = TIMEOUT
        self.shield = 0
        self.lap = 0
        self.world = world
        self.boost_available = True

    def incrcpid(self):
        self.ncpid += 1
        if self.ncpid == 1:
            self.lap += 1
        if self.ncpid >= self.world.circuit.nbcp():
            self.ncpid = 0

    def nbChecked(self):
        lastCP = self.ncpid-1
        if lastCP == -1:
            lastCP = self.world.circuit.nbcp() - 1
        return self.lap * self.world.circuit.nbcp() + lastCP

    def score(self, use_cp_dist_score=False):
        if not use_cp_dist_score:
            return self.nbChecked()

        current_cp = self.world.circuit.cp((self.ncpid - 1) % self.world.circuit.nbcp())
        next_cp = self.world.circuit.cp(self.ncpid)
        distance_cp_to_ncp = current_cp.distance(next_cp)
        cp_dist_score = (distance_cp_to_ncp - self.distance(next_cp)) / distance_cp_to_ncp
        return self.nbChecked() + cp_dist_score

    def block_score(self, opp_run_pod):
        return -self.distance(opp_run_pod.next_checkpoint()) / 5000.0

    def diffAngle(self, p):
        a = math.atan2(p.y - self.y, p.x - self.x) * 180.0 / math.pi
        right = a - self.angle if self.angle <= a else 360.0 - self.angle + a
        left = self.angle - a if self.angle >= a else self.angle + 360.0 - a
        return right if right < left else -left

    def rotate(self, p):
        a = self.diffAngle(p)
        if a > 18.0:
            a = 18.0
        elif a < -18.0:
            a = -18.0

        self.angle += a
        if self.angle >= 360.0:
            self.angle -= 360.0
        elif self.angle < 0.0:
            self.angle += 360.0

    def boost(self, thrust):
        # assert self.shield == 0
        ra = self.angle * math.pi / 180.0
        self.vx += math.cos(ra) * thrust
        self.vy += math.sin(ra) * thrust

    def move(self, t):
        self.x += self.vx*t
        self.y += self.vy*t

    def end(self):
        self.x = int(self.x + 0.5)
        self.y = int(self.y + 0.5)
        self.vx = int(self.vx * 0.85)
        self.vy = int(self.vy * 0.85)
        self.timeout -= 1

    def bounce(self, unit):
        tm = 10.0 if self.shield == 4 else 1.0
        em = 10.0 if unit.shield == 4 else 1.0
        mcoeff = (tm + em) / (tm * em)
        nx = self.x - unit.x
        ny = self.y - unit.y
        nxnysquare = nx*nx + ny*ny
        dvx = self.vx - unit.vx
        dvy = self.vy - unit.vy
        product = nx*dvx + ny*dvy
        fx = (nx * product) / (nxnysquare * mcoeff)  # Can raise ZeroDivisionError: float division by zero
        fy = (ny * product) / (nxnysquare * mcoeff)

        self.vx -= fx / tm
        self.vy -= fy / tm
        unit.vx += fx / em
        unit.vy += fy / em

        impulse = math.sqrt(fx*fx + fy*fy)
        if impulse < 120.0:
            fx = fx * 120.0 / impulse
            fy = fy * 120.0 / impulse

        self.vx -= fx / tm
        self.vy -= fy / tm
        unit.vx += fx / em
        unit.vy += fy / em

    def next_checkpoint(self, number_next=0):
        return self.world.circuit.cp((self.ncpid + number_next) % self.world.circuit.nbcp())

    def get_new_angle(self, gene):
        res = self.angle + gene * 36.0 - 18.0
        if res >= 360.0:
            res -= 360.0
        elif res < 0.0:
            res += 360.0
        return res

    def get_new_power(self, gene):
        return gene * MAX_THRUST

    def apply_move(self, move):
        self.angle = self.get_new_angle(move[0])
        if move[2] <= 0.2 and self.boost_available:
            if self.shield == 0:
                self.boost_available = False
                self.boost(650.0)
        elif move[2] >= 0.8:
            self.shield = 4
        elif self.shield == 0:
            self.boost(self.get_new_power(move[1]))

    def to_dummy_move(self, speed):
        return [
            (self.diffAngle(self.world.circuit.cp(self.ncpid)) + 18.0) / 36.0,
            speed / MAX_THRUST,
            0.5,
        ]

    def genes_from_vincent_command(self, command):
        return [
            (self.diffAngle(Point(command.target.x, command.target.y)) + 18.0) / 36.0,
            command.thrust / MAX_THRUST,
            1.0 if command.shield else 0.5,
        ]


from captcha.image import ImageCaptcha
from random import sample
from tkinter import messagebox, Label, Entry, Tk, PhotoImage

liist = "abcdefghijklmnopqrstuvwxzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"

def main():

	image = ImageCaptcha(width = 250, height = 200)

	captchatext = "".join(sample(liist, 5))

	data = image.generate(captchatext)

	image.write(captchatext, "captcha.png")

                        #################################################
	win = Tk()

	ws = win.winfo_screenwidth()
	hs = win.winfo_screenheight()

	xx = 250
	yy = 230

	x = ws/2 - xx/2
	y = hs/2 - yy/2

	win.geometry("%dx%d+%d+%d" % (xx, yy, x, y))

	win.resizable(0, 0)

	win.title("captcha")

	capochino = PhotoImage(file='captcha.png')

	Label(win, image = capochino).place(x = 0, y = 0)

	def message():
		global msg
		msg = messagebox.askquestion(title = "captcha", message = "Your captcha session has expired !\nDo you want to retry ?")
		if msg == "no":
			win.destroy()
		else:
			win.destroy()
			main()

	win.after(30000, message)

	entry = Entry(width = 10, bd = 3, justify = "center")

	def getinput(click):
		e = entry.get()
		if e == captchatext :
			win.destroy()

			# here goes the followed program after captcha verification !

			# here ends

		else :
			mssg = messagebox.askquestion(title = "captcha", message = "Your captcha verification is wrong !\nDo you want to retry ?")
			if mssg == "yes":
				win.destroy()
				main()
			else :
				win.quit()

	win.bind("<Return>", getinput)

	entry.place(x = 80, y = 203)

	win.mainloop()

main()


import os
import warnings

import joblib
# from dotenv import load_dotenv
from elasticsearch import Elasticsearch

import utils

warnings.filterwarnings("ignore")
# load_dotenv(override=True)

# create map dictionaries
# gender_map = {"Male": 0, "Female": 1}
# family_history_map = {"yes": 1, "no": 0}
# favc_map = {"yes": 1, "no": 0}
# caec_map = {"Always": 1.00, "Frequently": 0.67, "Sometimes": 0.33, "no": 0.00 }
# smoke_map = {"yes": 1, "no": 0}
# scc_map = {"yes": 1, "no": 0}
# calc_map = {"Always": 1.00, "Frequently": 0.67, "Sometimes": 0.33, "no": 0.00 }
# mtrans_map = {"Automobile": 1.00, "Motorbike": 0.75, "Bike": 0.50, "Public_Transportation": 0.25, "Walking": 0.00}
# nobesity_map = {"Obesity_Type_III": 0, "Obesity_Type_II": 1, "Obesity_Type_I": 2, "Overweight_Level_II": 3, "Overweight_Level_I": 4, "Normal_Weight": 5, "Insufficient_Weight": 6}
# feature_col_names = ['Gender', 'Age', 'Height', 'Weight', 'family_history_with_overweight', 'FAVC', 'FCVC', 'NCP', 'CAEC', 'SMOKE', 'CH2O', 'SCC', 'FAF', 'TUE', 'CALC', 'MTRANS']

def predict_obesity_risk(attributes):
    current_dir = os.path.dirname(os.path.abspath(__file__))
    file_path = os.path.join(current_dir, 'rf_model.joblib')
    rf = joblib.load(file_path)
    prediction = rf.predict(attributes)
    return prediction

def main():
    try:
        client_local = Elasticsearch("http://localhost:9200")
    except ConnectionError as err:
        print("Unable to connect to local Elasticsearch db: ", err)
        client_local = None
    if client_local != None:
        utils.read_data()
        prediction = predict_obesity_risk([[1, 27, 1.7018, 68, 0, 1, 2.0, 2.0, 0.67, 0, 1.0, 0, 4.0, 4.0, 0.33, 0]])
        print('prediction: ', prediction)

if __name__ == "__main__":
    main()


import elasticsearch
import time

es = elasticsearch.Elasticsearch()
index_name = "bookstore"
type_name = "books"
try:
    es.indices.delete(index_name)
    es.cluster.health(wait_for_status="yellow")
    es.indices.delete_mapping(index_name, type_name)
    es.cluster.health(wait_for_status="yellow")
except Exception as ex:
    print("index not exists")

mapping_body = {
    "books": {
        "_source": {
            "enabled": True
        },
        "_all": {
            "enabled": True,
            "index_analyzer": "ik",
            "search_analyzer": "ik",
            "term_vector": "no",
            "store": "false"
        },
        "index_analyzer": "ik",
        "search_analyzer": "ik",
        "dynamic": True,
        "dynamic_templates": [
            {
                "string_analyze_template": {
                    "match": "*",
                    "match_mapping_type": "string",
                    "mapping": {
                        "type": "multi_field",
                        "fields": {
                            "{name}": {"type": "{dynamic_type}", "index": "analyzed"},
                            "Raw": {"type": "{dynamic_type}", "index": "not_analyzed"}
                        }
                    }
                }
            }
        ]
    }
}
es.indices.create(index_name)
es.cluster.health(wait_for_status="yellow")
r = es.indices.put_mapping(index=index_name, doc_type=type_name, body=mapping_body)
es.cluster.health(wait_for_status="yellow")

time.sleep(0.5)
print(r)
print("set mapping bookstore/books in ES !")


import json
import re
import argparse

def find_words(source_file):
    allfind = []
    with open(source_file ,encoding='UTF-8') as f:
        for line in f:
            result = re.findall('"count": (\d+), "word": "(.*?)"', line)#这里不知道如何匹配中文
            allfind.extend(result)
    return allfind

def checkCh(item):
    flag = True
    for char in item[1]:
        if u'\u4e00' <= char <= u'\u9fff':
            continue
        else:
            flag = False
    return flag

def make_dic(allfind):
    words = dict()
    for item in allfind:
        if checkCh(item):
            if item[1] in words.keys():
                words[item[1]] += int(item[0])
            else:
                words[item[1]] = int(item[0])
    return words

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('--source', default="log.2018-05-31")
    parser.add_argument('--target', default="words.json" )
    args = parser.parse_args()
    source_file = args.source
    target_file = args.target
    findall = find_words(source_file)
    data = make_dic(findall)
    data_sorted = sorted(data.items(), key=lambda v: v[1],reverse=True)
    with open(target_file,'w',encoding="UTF-8") as f:
        f.write(json.dumps(data_sorted,ensure_ascii=False))


from django.urls import path

from . import views

urlpatterns = [
    path('bairros', views.bairros, name='bairros'), #GET ONLY

    path('locais', views.locais, name='locais'), #GET E POST

    path('pacientes', views.pacientes, name='pacientes'), #GET E POST
    path('pacientes/<int:paciente_id>', views.paciente, name='paciente'), #GET PUT E DELETE

    path('sintomas', views.sintomas, name='sintomas'), #GET ONLY

    path('casos', views.casos, name='bairros'), #GET E POST
    path('casos/<int:caso_id>', views.caso, name='caso'), #GET PUT E DELETE
    path('casos/bairro/<int:bairro_id>', views.casosPorBairro, name='casosporbairro'), #GET
    path('casos/local/<int:local_id>', views.casosPorLocalVisitado, name='casosporlocalvisitado'), #GET

    path('formascontagio', views.formasContagio, name='formascontagio') #GET ONLY
]


from flask import render_template, flash, redirect, url_for, request, json, make_response, jsonify
from flask_login import login_user, logout_user, current_user, login_required
from werkzeug.urls import url_parse
from app import app, db
from app.forms import LoginForm, RegistrationForm, DateForm, CancelForm, BookForm, AddDoctorForm, AddHospitalForm, RegistrationDepartmentForm, BookInternalForm, AddWorkingTimeForm, AddLocationForm, BookPatientForm
from app.models import User, Appointment, Doctors, Patient, Hospital, Department, DoctorDate, TimeSlots, Location
from datetime import datetime, date, timedelta
import pandas as pd
import os
from twilio.rest import Client

try:
    account_sid = os.environ['TWILIO_ACCOUNT_SID']
    auth_token = os.environ['TWILIO_AUTH_TOKEN']
    client = Client(account_sid, auth_token)
    send_message = True
except:
    send_message = False

@app.route('/')
def home():
    return render_template("home.html")

@app.route('/index')
@login_required
def index():
    return render_template('index.html', title='Home')

@app.route('/login', methods=['GET', 'POST'])
def login():
    # redirect already logged in user to their home screen
    if current_user.is_authenticated:
        return redirect(url_for('index'))
    # load the form
    form = LoginForm()
    if form.validate_on_submit():
        # add user to database
        user = User.query.filter_by(username=form.username.data).first()
        if user is None or not user.check_password(form.password.data):
            flash('Invalid username or password')
            return redirect(url_for('login'))
        # log user in
        login_user(user, remember=form.remember_me.data)
        next_page = request.args.get('next')
        if not next_page or url_parse(next_page).netloc != '':
            next_page = url_for('index')
        return redirect(next_page)
    return render_template('login.html', title='Sign In', form=form)

@app.route('/logout')
def logout():
    logout_user()
    return redirect(url_for('index'))

@app.route('/register')
def register():
    if current_user.is_authenticated:
        return redirect(url_for('index'))
    return render_template('register.html', title='Register')

# Patient view
@app.route('/register_patient', methods=['GET', 'POST'])
def register_patient():
    if current_user.is_authenticated:
        return redirect(url_for('index'))
    form = RegistrationForm()
    if form.validate_on_submit():

        if Patient.query.filter_by(phone=form.phone.data).first() is None:
            patient = Patient(phone=form.phone.data)
            db.session.add(patient)

        patient = Patient.query.filter_by(phone=form.phone.data).first()
        user = User(username=form.username.data, email=form.email.data, role="patient", patient = patient)
        user.set_password(form.password.data)
        db.session.add(user)
        db.session.commit()
        flash('Congratulations, you are now a registered patient!')
        return redirect(url_for('login'))
    return render_template('register_patient.html', title='Register', form=form)

@app.route('/myappointments', methods=['GET', 'POST'])
@login_required
def myappointments():
    appointments = db.session.query(Appointment).outerjoin(Doctors).outerjoin(Department).filter(Appointment.patient_id ==  current_user.patient_id).all()
    table_header = ["#", "Date", "Time", "Doctor"]
    appoint = []
    for appoin in appointments:
        appoint.append([appoin.id, appoin.apdate, appoin.aptime, appoin.doctor.name])

    return render_template('myappointments.html', title='Appointment', table_header=table_header, appointments= appoint)

@app.route('/book_appointment_patient', methods=['GET', 'POST'])
@login_required
def book_appointment_patient():
    form = BookPatientForm()
    city_choices = Hospital.query.with_entities(Hospital.city).distinct().all()
    city_choices = [h[0] for h in city_choices]
    form.city.choices = city_choices
    department_choices = Department.query.with_entities(Department.name).distinct().all()
    department_choices = [d[0] for d in department_choices]
    form.department.choices = department_choices
    hospital_choices = Hospital.query.with_entities(Hospital.id).distinct().all()
    hospital_choices = [h[0] for h in hospital_choices]
    form.hospital.choices = hospital_choices
    times = (pd.DataFrame(columns=['NULL'],
    index=pd.date_range('2016-09-02T00:00:00Z', '2016-09-02T23:55:00Z',
    freq='15T')).between_time("00:00","23:55").index.strftime('%H:%M').tolist())
    form.aptime.choices = times

    if form.validate_on_submit():
        patient = Patient.query.filter_by(id=int(current_user.patient_id)).first()
        doc_match = DoctorDate.query.outerjoin(TimeSlots).outerjoin(Doctors).outerjoin(Department).outerjoin(Hospital).filter(Hospital.id ==  form.hospital.data, Department.name == form.department.data, DoctorDate.date == form.apdate.data, TimeSlots.slot == form.aptime.data).all()
        doc_match = [d.doctor_id for d in doc_match]
        Aptimes = Appointment.query.outerjoin(Doctors).outerjoin(Department).outerjoin(Hospital).filter(Hospital.id ==  form.hospital.data, Department.name == form.department.data, DoctorDate.date == form.apdate.data, Appointment.aptime == form.aptime.data).all()
        if not Aptimes is None:
            Aptimes = [d.doctor_id for d in Aptimes]
            doc_match = [x for x in doc_match if x not in Aptimes]
        doc_match = doc_match[0]
        doc = Doctors.query.filter_by(id=doc_match).first()
        appoint = Appointment(apdate=str(form.apdate.data), aptime=str(form.aptime.data), patient=patient, doctor=doc)
        db.session.add(appoint)
        db.session.commit()
        update_timeslot_status = TimeSlots.query.outerjoin(DoctorDate).outerjoin(Doctors).filter(Doctors.id == doc.id, DoctorDate.date == str(form.apdate.data), TimeSlots.slot == form.aptime.data).first()
        update_timeslot_status.free = "False"
        db.session.commit()

        if send_message:
            location=Location.query.filter_by(id = doc.department_id).first()
            text_start = "Thank you for your appointment with " + doc.name + " at " + str(form.apdate.data) + " " + form.aptime.data + ". "
            text_adress = " Come to " + location.adress + " to the room " + location.room + " by following the color " + location.colortape + ". You can find the map " + location.link + ". "
            text_process = "Bring your isurance card."
            text_body = text_start + text_adress + text_process
            message = client.messages.create(
            to=patient.phone,
            from_='HSPTLEYES',
            body=text_body)
        return redirect(url_for('myappointments'))

    return render_template('book_appointment_patient.html', form=form)

@app.route('/_update_hospital', methods=['GET', 'POST'])
def update_hospital():
    city = request.args.get('city')
    department = request.args.get('department')
    today = date.today().strftime("%Y-%m-%d")
    future_30_days = datetime.now() + timedelta(days=30)
    future_30_days = future_30_days.strftime("%Y-%m-%d")
    hospitals = db.session.query(Hospital, db.func.count(TimeSlots.id)).outerjoin(Department).outerjoin(Doctors).outerjoin(DoctorDate).outerjoin(TimeSlots).filter(Hospital.city ==  city, Department.name == department, TimeSlots.free == "True").filter(DoctorDate.date >= today).filter(DoctorDate.date <= future_30_days).group_by(Hospital.name, Hospital.id).all()
    if hospitals is None:
        hospitals = []
    else:
        hospitals = [(t[0].id, t[0].name + " Available Slots: " + str(t[1])) for t in hospitals]

    response = make_response(json.dumps(hospitals))
    response.content_type = 'application/json'
    return response

@app.route('/_update_timeslots_patient', methods=['GET', 'POST'])
def update_timeslots_patient():
    department = request.args.get('department')
    hospital = request.args.get('hospital')
    date = request.args.get('date')
    timeslots = db.session.query(TimeSlots).outerjoin(DoctorDate).outerjoin(Doctors).outerjoin(Department).outerjoin(Hospital).filter(Hospital.id ==  hospital, Department.name == department, DoctorDate.date == date).with_entities(TimeSlots.slot, DoctorDate.doctor_id).all()
    if timeslots is None:
        timeslots = []
    else:
        Aptimes = db.session.query(Appointment).outerjoin(Doctors).outerjoin(Department).outerjoin(Hospital).filter(Hospital.id ==  hospital, Department.name == department, DoctorDate.date == date).with_entities(Appointment.aptime, Appointment.doctor_id).all()
        if not Aptimes is None:
            timeslots = [x for x in timeslots if x not in Aptimes]
        timeslots = [t[0] for t in timeslots]
        timeslots = list(set(timeslots))
        timeslots.sort()
    response = make_response(json.dumps(timeslots))
    response.content_type = 'application/json'
    return response

# Hospital view

@app.route('/register_department', methods=['GET', 'POST'])
def register_department():
    if current_user.is_authenticated:
        return redirect(url_for('index'))
    form = RegistrationDepartmentForm()
    # only allow for already added hospitals
    form.hospital.choices = [(g.id, g.name) for g in Hospital.query.order_by('name')]
    # if department has not been registered add to database
    if form.validate_on_submit():
        if Department.query.filter_by(name=form.name.data, hospital_id=form.hospital.data).first() is None:
            hospital_dep = Hospital.query.filter_by(id=form.hospital.data).first()
            department = Department(name=form.name.data, hospital=hospital_dep)
            db.session.add(department)
        # connect user to department
        department = Department.query.filter_by(name=form.name.data, hospital_id=form.hospital.data).first()
        user = User(username=form.username.data, email=form.email.data, role="hospital", department = department)
        user.set_password(form.password.data)
        db.session.add(user)
        db.session.commit()
        flash('Congratulations, you are now a registered department!')
        return redirect(url_for('login'))
    return render_template('register_department.html', title='Register', form=form)

@app.route('/add_hospital', methods=['GET', 'POST'])
def add_hospital():
    form = AddHospitalForm()
    if form.validate_on_submit():
        hospital = Hospital(name=form.name.data, city=form.city.data)
        db.session.add(hospital)
        db.session.commit()
        flash('The hospital has been added!')
    return render_template('add_hospital.html', form=form)

@app.route('/adddoctor', methods=['GET', 'POST'])
def add_doctor():
    if current_user.is_authenticated:
        if current_user.role == "hospital":
            form = AddDoctorForm()
            if form.validate_on_submit():
                dep = Department.query.filter_by(id = current_user.department_id).first()
                doc = Doctors(name=form.name.data, department=dep)
                db.session.add(doc)
                db.session.commit()
                flash('New Doctor added!')
                return redirect(url_for('index'))
            return render_template('add_doctor.html', title='Add doctor', form=form)
        else:
            return redirect(url_for('index'))
    else:
        return redirect(url_for('index'))

@app.route('/Cancel',methods=['GET','POST'])
def cancel_appointmetn():

    form = CancelForm()

    if form.validate_on_submit():

        id = form.id.data
        ap = Appointment.query.det(id)
        db.session.delete(ap)
        db.session.commmit()

        return redirect(url_for('list.html'))

@app.route('/book_internal_appointment', methods=['GET', 'POST'])
def book_internal_appointment():
    if current_user.is_authenticated:
        if current_user.role == "hospital":
            times = (pd.DataFrame(columns=['NULL'],
            index=pd.date_range('2016-09-02T07:00:00Z', '2016-09-02T18:00:00Z',
            freq='15T'))
            .index.strftime('%H:%M')
            .tolist())

            form = BookInternalForm()
            form.doctor.choices = [
                (g.id, g.name) for g in Doctors.query.filter_by(department_id = current_user.department_id).order_by('name')]
            form.aptime.choices = [(g, g) for g in times]
            locations = Location.query.filter_by(department_id = current_user.department_id).order_by('name')
            if locations.first() is None:
                form.location.choices = []
            else:
                form.location.choices = [(g.id, g.name) for g in locations]

            if form.validate_on_submit():
                if Patient.query.filter_by(phone=form.phone.data).first() is None:
                    patient = Patient(phone=form.phone.data)
                    db.session.add(patient)

                patient = Patient.query.filter_by(phone=form.phone.data).first()
                doc = Doctors.query.filter_by(id=form.doctor.data).first()
                appoint = Appointment(apdate=str(form.apdate.data), aptime=str(form.aptime.data), patient=patient, doctor=doc)
                db.session.add(appoint)
                db.session.commit()
                update_timeslot_status = TimeSlots.query.outerjoin(DoctorDate).outerjoin(Doctors).filter(Doctors.id == doc.id, DoctorDate.date == str(form.apdate.data), TimeSlots.slot == form.aptime.data).first()
                update_timeslot_status.free = "False"
                db.session.commit()

                if send_message:
                    location=Location.query.filter_by(id = form.location.data).first()
                    text_start = "Thank you for your appointment with " + doc.name + " at " + str(form.apdate.data) + " " + form.aptime.data + ". "
                    text_adress = " Come to " + location.adress + " to the room " + location.room + " by following the color " + location.colortape + ". You can find the map " + location.link + ". "
                    text_process = "Bring your isurance card."
                    text_body = text_start + text_adress + text_process
                    message = client.messages.create(
                    to=form.phone.data,
                    from_='HSPTLEYES',
                    body=text_body)

                flash('The appointment has been made')
                return redirect(url_for('book_internal_appointment'))
        else:
            return redirect(url_for('index'))
    else:
        return redirect(url_for('index'))
    return render_template('book_internal_appointment.html', title='Appointment', form=form)

@app.route('/_update_timeslots', methods=['GET', 'POST'])
def update_timeslots():
    times = []
    startdate = request.args.get('date')
    doctor = request.args.get('doc')
    doctor_id = doctor
    date = DoctorDate.query.filter_by(date=startdate, doctor_id=doctor_id).first()
    if date is None:
        times = []
    else:
        times = TimeSlots.query.filter_by(doctor_date_id = date.id).all()
        times = [t.slot for t in times]
    Aptimes = Appointment.query.filter_by(apdate=startdate, doctor_id = doctor_id).all()
    if not Aptimes is None:
        Time = [aptime.aptime for aptime in Aptimes]
        times = [x for x in times if x not in Time]

    response = make_response(json.dumps(times))
    response.content_type = 'application/json'
    return response

@app.route('/add_workingtime', methods=['GET', 'POST'])
@login_required
def add_workingtime():
    form = AddWorkingTimeForm()
    form.doctor.choices = [
    (g.id, g.name) for g in Doctors.query.filter_by(department_id = current_user.department_id).order_by('name')]
    if form.validate_on_submit():
        daterange = pd.date_range(start=form.startdate.data, end=form.enddate.data)
        daterange = [d.strftime('%Y-%m-%d') for d in daterange]
        doctor = Doctors.query.filter_by(id=form.doctor.data).first()
        times = (pd.DataFrame(columns=['NULL'],
        index=pd.date_range('2016-09-02T07:00:00Z', '2016-09-02T18:00:00Z',
        freq='15T'))
        .between_time(form.starttime.data,form.endtime.data)
        .index.strftime('%H:%M')
        .tolist())
        for i in daterange:
            date = DoctorDate(date=i, doctor=doctor)
            db.session.add(date)
            db.session.commit()
            date = DoctorDate.query.filter_by(date=i, doctor_id=doctor.id).first()
            for j in times:
                time = TimeSlots(slot = j, date = date, free = 'True')
                db.session.add(time)
                db.session.commit()

    return render_template('add_workingtime.html', title='Appointment', form=form)

@app.route('/add_location', methods=['GET', 'POST'])
@login_required
def add_location():
    form = AddLocationForm()
    if form.validate_on_submit():
        department = Department.query.filter_by(id = current_user.department_id).first()
        location = Location(name=form.name.data, adress = form.adress.data, room = form.room.data, colortape=form.colortape.data, link=form.link.data, department = department)
        db.session.add(location)
        db.session.commit()
        flash('The location has been added!')
    return render_template('add_location.html', form=form)


import math

import pandas as pd
from matplotlib import pyplot as plt

from chester.zero_break.problem_specification import DataInfo

class CategoricalStats:
    def __init__(self, data_info: DataInfo, max_print=None):
        self.data_info = data_info
        self.max_print = max_print
        self.cols = list(set(self.data_info.feature_types_val["categorical"]))
        self.data = self.data_info.data[self.cols]
        if self.data.columns.duplicated().any():
            self.data = self.data.loc[:, ~ self.data.columns.duplicated()]
        self.data = self.data.sample(min(10000, len(self.data)))
        self.cols_sorted = self.sort_by_cardinality()

    def any_categorical(self):
        return True if len(self.cols) > 0 else False

    def sort_by_cardinality(self):
        if not self.any_categorical():
            return []
        cardinalities = []
        for col in self.cols:
            data = self.data[col]
            cardinalities.append((col, data.nunique()))
        sorted_cardinalities = sorted(cardinalities, key=lambda x: x[1], reverse=True)
        return [x[0] for x in sorted_cardinalities]

    def sample_top_features(self, n):
        top_features = self.cols_sorted[:3 * n]
        return top_features

    def plot_value_counts(self, n=25, plot=True):
        if not self.any_categorical():
            return None
        if not plot:
            return None
        top_n = self.cols_sorted[:min(len(self.cols_sorted), n)]
        num_plots = len(top_n)
        if num_plots == 1:
            col = top_n[0]
            fig, ax1 = plt.subplots(1, 1, figsize=(10, 5))
            cat_col = self.data[col].apply(lambda x: "cat " + str(x))
            count_data = cat_col.value_counts()
            percent_data = count_data / count_data.sum() * 100
            ax1.bar(count_data.index[:5], count_data.values[:5], color='gray')
            ax2 = ax1.twinx()
            ax2.plot(percent_data.index[:5], percent_data.values[:5], color='red', marker='o')
            ax1.set_ylabel('Counts', color='gray')
            ax2.set_ylabel('Percentages', color='red')
            ax1.set_xlabel(None)
            ax1.set_title(f"{col}")
            return None
        else:
            dim = max(math.floor(math.sqrt(len(top_n))), 2)
            fig, ax = plt.subplots(dim, dim, figsize=(18, 3 + 3 * dim))
            fig.tight_layout()
            fig.suptitle("Top 5 Value Counts and Percentages for Each Feature")
            for i, col in enumerate(top_n):
                if i >= dim * dim:
                    break
                count_data = pd.DataFrame(self.data[col].value_counts().head(5)).reset_index(drop=False)
                total_count = self.data[col].count()
                percent_data = count_data.copy()
                percent_data[col] = percent_data[col] / total_count * 100
                percent_data.rename(columns={col: 'percentage', 'index': col}, inplace=True)

                plot_title = f"{col}"
                ax_i = ax[i // dim, i % dim]
                ax1_i = ax_i
                ax1_i.bar(count_data.iloc[:, 0], count_data.iloc[:, 1].to_list(), color='gray')
                ax2_i = ax1_i.twinx()
                ax2_i.plot(percent_data.iloc[:, 0], percent_data.iloc[:, 1].to_list(), marker='o', color='red')
                ax1_i.set_ylabel('Counts', color='gray')
                ax2_i.set_ylabel('Percentages', color='red')
                ax1_i.set_xlabel(None)
                ax1_i.set_title(plot_title)
            plt.show()
            plt.close()

    def calculate_stats(self, is_print=True):
        from chester.util import ReportCollector, REPORT_PATH
        rc = ReportCollector(REPORT_PATH)
        if not self.any_categorical():
            return None
        result_dicts = []
        for col in self.cols:
            data = self.data[col]
            unique_values = data.nunique()
            missing_values = data.isnull().sum()
            value_counts = data.value_counts().rename("count").reset_index()
            value_counts["percentage"] = 100 * value_counts["count"] / value_counts["count"].sum()
            value_counts = value_counts.sort_values("count", ascending=False)
            dist_str = ', '.join([f"{row['index']}: {row['percentage']:.0f}%" for _, row in value_counts.iterrows()])
            result_dicts.append(
                {'col': col, '# unique': unique_values, '# missing': missing_values, 'Distribution': dist_str})
            data_unique_values = data.dropna().drop_duplicates()
            col_len = len(data_unique_values)
            values_to_sample = 3
            if col_len < values_to_sample:
                values_to_sample = col_len
            sample_values = [str(value) for value in data_unique_values.sample(min(col_len, values_to_sample)).values]
            result_dicts[-1]['Sample'] = ', '.join(sample_values)

            # add more columns
            # 1. % from all that covers the top 5 values
            top_5 = 100 * value_counts.head(5)["count"].sum() / value_counts["count"].sum()
            result_dicts[-1][f'Top 5 values coverage'] = f"{top_5:.0f}%"

        results_df = pd.DataFrame(result_dicts)

        if is_print:
            if self.max_print is not None:
                formatted_df = format_df(df=results_df,
                                         max_value_width=self.max_print,
                                         )
                print(formatted_df)
            else:
                formatted_df = format_df(results_df)
                print(formatted_df)
            len_df = len(formatted_df)
            rc.save_object(obj=formatted_df.sample(min(len_df, 10)), text="Feature stats:")
        return results_df

    def run(self, plot=True):
        self.calculate_stats()
        self.plot_value_counts(plot=plot)
        return None

def format_df(df, max_value_width=30):
    pd.options.display.max_columns = None

    def trim_value(val):
        if len(str(val)) > max_value_width:
            return str(val)[:max_value_width] + "..."
        return str(val)

    df = df.applymap(trim_value)

    return df


import requests
from bs4 import BeautifulSoup
import pandas as pd
import numpy as np
import re

class BellRezepte():
    """ This class will output the recipe details. Recipes have the
    following properties:
    Attributes:
        url: The url of the recipe on the Bell recipes site.
    Methods:
        Serves: How many people the recipe serves.
        Cooking time: What it says on the tin.
        Difficulty: Recipe difficulty.
        Ingredients: Recipe ingredients.
        Directions: Recipe directions.
        Preferences: Food preferences.
        Image: Image of the recipe.
    """
    def __init__(self, url):
        self.url = url
        self.soup = BeautifulSoup(requests.get(url).content, 'html.parser')

        first_div = self.soup.find('div', {'class': 'recipe-meta-item'})
        self.all_divs = [first_div] + first_div.find_next_siblings('div', {'class': 'recipe-meta-item'})

    def recipe_name(self):
        """ Locates the recipe title """
        try:
            return self.soup.find('h1').text.strip()
        except:
            return np.nan

    def serves(self):
        """ Locates the number of people the meal serves """
        try:
            return self.soup.find('span', attrs={'class': 'recipe-servings-size'}).text.split()[0]
        except:
            return np.nan

    def cooking_time(self):
        """ Locates the cooking time (in mins or hours and mins) """
        try:
            return ' '.join(self.all_divs[0].text.split())
        except:
            return np.nan

    def difficulty(self):
        """ Locates the cooking difficulty """
        try:
            return ''.join(self.all_divs[1].text.split())
        except:
            return np.nan

    def ingredients(self):
        """ Locates the ingredients of the recipe """
        try:
            quantity, name = [], []
            for i in range(0, len(self.soup.find_all('td')), 2):
                ingred = ' '.join(self.soup.find_all('td')[i].text.split())
                quantity.append(ingred)
            for i in range(1, len(self.soup.find_all('td')), 2):
                ingred = ' '.join(self.soup.find_all('td')[i].text.split())
                name.append(ingred)
            ingredients = [{"quantity": q, "name": n} for q, n in zip(quantity, name)]
            return ingredients
        except:
            return np.nan

    def directions(self):
        """ Locates the directions for cooking the recipe """
        try:
            directions = []
            for i in self.soup.find_all('div', {'class': 'col-md-8'}):
                d = ' '.join(i.text.split())
                d = d.replace('Zubereitung', '')
                directions.append(d)
            return directions[0]
        except:
            return np.nan

    def preferences(self):
        """" Locates the recipe preferences for the user """
        try:
            if len(self.all_divs) > 2:
                preferences = []
                for i in self.all_divs[2:]:
                    p = ' '.join(i.text.split())
                    preferences.append(p)
            return preferences
        except:
            return np.nan

    def image(self):
        """" Locates the image of the recipe meal """
        try:
            return self.soup.picture.img['src']
        except:
            return np.nan


from django.urls import path
from instruction.views import (
    CreateMedicalInsturctionView,
    DoctorListHealthRecordMedicalInstructionView,
    SupervisorListHealthRecordMedicalInstructionView,
    UpdateMedicalInstructionView,
    ListInstCategoryView,
)

urlpatterns = [
    path('doctor/instruction/', CreateMedicalInsturctionView.as_view(), name='create-medical_instruction'),
    path('doctor/instruction/<int:healthrecord>/', DoctorListHealthRecordMedicalInstructionView.as_view(), name = 'doctor-list-medical_instruction'),
    path('doctor/action/instruction/', UpdateMedicalInstructionView.as_view(), name = 'doctor-update-medical_instruction'),
    path('supervisor/instruction/', SupervisorListHealthRecordMedicalInstructionView.as_view(),name='supervisor-list-medical_instruction'),

    # API Get all instruction categories,
    path('instructions-categories/', ListInstCategoryView.as_view(),name='list-medical-instr-categories'),
]


import sys

from PyQt5.QtCore import pyqtSignal
from PyQt5.QtWidgets import QApplication, QMainWindow
from python_forms.main_window_GUI import Ui_MainWindow
from src.Managers.employees_manager import EmployeesManager
from system_properties import SystemProperties
from src.Managers.clients_manager import ClientsManager
from src.Managers.brokers_manager import BrokersManager
from src.Managers.company_manager import CompaniesManager
from src.Managers.leads_manager import LeadsManager
from src.Managers.invoices_manager import InvoicesManager
from dashboard import Dashboard

class AppMainWindow(QMainWindow):
    reload_dashboard = pyqtSignal()

    def __init__(self, parent=None):
        super(AppMainWindow, self).__init__(parent=parent)
        self.ui = Ui_MainWindow()
        self.ui.setupUi(self)
        self.invoices_manager = InvoicesManager()
        self.dashboard = Dashboard()
        self.widgets = dict()
        self.initialize_pages()
        self.connect_signals_slots()
        self.ui.window_content.setCurrentIndex(0)

    def connect_signals_slots(self):
        self.ui.dashboard_btn.clicked.connect(
            lambda: self.change_widget(self.widgets["dashboard"]["index"]))
        self.ui.employees_btn.clicked.connect(
            lambda: self.change_widget(self.widgets["employees_manager"]["index"]))
        self.ui.system_properties_btn.clicked.connect(
            lambda: self.change_widget(self.widgets["system_properties"]["index"]))
        self.ui.clients_btn.clicked.connect(
            lambda: self.change_widget(self.widgets["clients_manager"]["index"]))
        self.ui.companies_btn.clicked.connect(
            lambda: self.change_widget(self.widgets["companies_manager"]["index"])
        )
        self.ui.leads_btn.clicked.connect(
            lambda: self.change_widget(self.widgets["leads_manager"]["index"]))
        self.ui.brokers_btn.clicked.connect(
            lambda: self.change_widget(self.widgets["brokers_manager"]["index"]))
        self.ui.invoices_btn.clicked.connect(
            lambda: self.change_widget(self.widgets["invoices_manager"]["index"]))
        self.invoices_manager.reload_dashboard.connect(self.dashboard.refresh_insights)

    def initialize_pages(self):
        self.widgets = {
            "dashboard":
                {"index": 0, "widget": self.dashboard},
            "system_properties":
                {"index": 1, "widget": SystemProperties()},
            "employees_manager":
                {"index": 2, "widget": EmployeesManager()},
            "clients_manager":
                {"index": 3, "widget": ClientsManager()},
            'companies_manager':
                {"index": 4, "widget": CompaniesManager()},
            "leads_manager":
                {"index": 5, "widget": LeadsManager()},
            "brokers_manager":
                {"index": 6, "widget": BrokersManager()},
            "invoices_manager":
                {"index": 7, "widget": self.invoices_manager}
        }
        [self.ui.window_content.insertWidget(self.widgets[widget]["index"], self.widgets[widget]["widget"]) for widget in self.widgets]

    def change_widget(self, index):
        """
        Change the stacked widget to the widget at index
        :param index:Integer represents the widget's index in the stacked widget
        :return: None
        """
        self.ui.window_content.setCurrentIndex(index)

if __name__ == '__main__':
    app = QApplication(sys.argv)
    window = AppMainWindow()
    window.show()
    sys.exit(app.exec_())


import time
import pickle
import progressbar
import Matrix2tree
import Greedy_CM
from utils import Test_IF

N = 1000
p = progressbar.ProgressBar()
for n in p(range(3, N)):
    data_input = open('../../data/Matrix1/'+str(n)+'.txt', 'rb')
    matrices=pickle.load(data_input)
    data_input.close()
    I = matrices[0]
    F = matrices[1]
# I, F=Test_IF()
# n=12
    start = time.time()

    A=Matrix2tree.Matrix2tree(I,'Initial')
    A.matrix2tree(printer=False)
    A.CNL_cal()
    A.tree_render(printer=False, draw=False)
    A.Pointer()
    Asigs=A.Pointer_int(printer=False)

    B=Matrix2tree.Matrix2tree(F, 'Final')
    B.matrix2tree(printer=False)
    B.CNL_cal()
    B.tree_render(printer=False, draw=False)
    B.Pointer()
    Bsigs=B.Pointer_int(printer=False)

    test_cm=Greedy_CM.CM(A, B)

    test_cm.UV(printer=False)
    D, error_cm=test_cm.Difference(printer=False)

    time_cm = time.time() - start

    result=[error_cm, time_cm]

    data_output = open('../../data/results/Greedy_CM/'+str(n)+'.txt', 'wb')
    pickle.dump(result, data_output)
    data_output.close()


#!/usr/bin/env python
# -*- coding: utf-8 -*-

from .base import InfoScreen
import pygame, logging, locale
from datetime import datetime
from .helper.fontsurface import FontSurface
from lxml import etree
from .helper.textrect import render_textrect, TextRectException

class EventScreen(InfoScreen):
	"""Info screen that shows upcoming events."""
	def __init__(self, screen, config):
		"""Creates the event screen.

		Keyword arguments:
		screen -- screen to display on"""
		super(EventScreen, self).__init__(screen)

		self.config = config
		# seconds to display this info screen
		self.displayTime = 15

	def getEvents(self):
		"""Parses event feed and returns events as date-title dict."""
		events = []
		parser = etree.XMLParser(ns_clean=True)
		try:
			# xml parsing and xpath magic
			tree = etree.parse(self.config["feed"], parser)
			ns = {"ns": "http://www.w3.org/2005/Atom"}
			entries = tree.xpath("//ns:entry", namespaces=ns)
			for entry in entries:
				try:
					title = entry.xpath("ns:title/text()", namespaces=ns)[0]
					date = entry.xpath("ns:summary/text()", namespaces=ns)[0]
					events.append((date, title))
				except IndexError:
					self.log(logging.ERROR, "XPath did not return anything.")
					continue
		except (IOError, etree.XMLSyntaxError) as e:
			self.log(logging.ERROR, e)
			return []
		return events

	def show(self):
		"""Shows upcoming events."""
		# fonts
		headlineFont = pygame.font.Font("fonts/blue_highway_bd.ttf",
			self.relH(.1))
		stdFont = pygame.font.Font("fonts/DejaVuSansMono.ttf",
			self.relH(.06))

		# colors used
		black = (0, 0, 0)
		white = (255, 255, 255)

		# black screen
		self.screen.fill(black)

		# event loop
		currentY = self.relH(.2)
		events = self.getEvents()
		for date, title in events:
			# parse date
			locale.setlocale(locale.LC_TIME, "de_DE.UTF-8")
			try:
				date = datetime.strptime(date.encode("utf-8"), "%H:%M, %d. %b. %Y")
			except ValueError:
				date = datetime.strptime(date.encode("utf-8"), "%H:%M, %d. %b %Y")
			locale.setlocale(locale.LC_TIME, "")
			# render one event at a time with a bullet point
			eventStr = u"\u00BB %s: %s" % (date.strftime("%d.%m. %H:%M"), title)

			# this only acts as a container here
			event = FontSurface(self.screen, "", stdFont)
			scrRect = self.screen.get_rect()
			surface = render_textrect(eventStr, stdFont, scrRect, white, black)
			event.surface = surface
			event.pos.x = self.relW(.007)
			event.pos.y = currentY
			event.blit()
			currentY += event.pos.height * 2

		# no events available
		if len(events) == 0:
			self.log(logging.DEBUG, "No events found. Skipping.")
			return
			"""
			msgStr = "Momentan sind keine Termine geplant."
			msg = FontSurface(self.screen, msgStr, stdFont)
			msg.centerX()
			msg.centerY()
			msg.blit()
			"""

		# headline
		headline = FontSurface(self.screen, u"Nächste Termine", headlineFont)
		headline.centerX()
		headline.pos.y = 0
		headline.blit()

		# show it
		pygame.display.flip()
		# nothing else to do
		self.wait(self.displayTime)


from extract_image_data import ImageExtractor
from unet import unet_model
from metrics import bce_dice_loss
from metrics import dice_coef
import random
import numpy as np
import pickle

from keras.preprocessing.image import ImageDataGenerator
from sklearn.model_selection import train_test_split
from keras.callbacks import EarlyStopping, ModelCheckpoint

# Runtime data augmentation
def get_train_test_augmented(X_data, Y_data, validation_split=0.25, batch_size=32, seed=42):
    X_train, X_test, Y_train, Y_test = train_test_split(X_data,
                                                        Y_data,
                                                        train_size=1 - validation_split,
                                                        test_size=validation_split,
                                                        random_state=seed)
    # Image data generator distortion options
    data_gen_args = dict(rotation_range=45.,
                         width_shift_range=0.1,
                         height_shift_range=0.1,
                         shear_range=0.2,
                         zoom_range=0.2,
                         horizontal_flip=True,
                         vertical_flip=True,
                         fill_mode='reflect')

    # Train data, provide the same seed and keyword arguments to the fit and flow methods
    X_datagen = ImageDataGenerator(**data_gen_args)
    Y_datagen = ImageDataGenerator(**data_gen_args)
    X_datagen.fit(X_train, augment=True, seed=seed)
    Y_datagen.fit(Y_train, augment=True, seed=seed)
    X_train_augmented = X_datagen.flow(X_train, batch_size=batch_size, shuffle=True, seed=seed)
    Y_train_augmented = Y_datagen.flow(Y_train, batch_size=batch_size, shuffle=True, seed=seed)

    # Test data, no data augmentation, but we create a generator anyway
    X_datagen_val = ImageDataGenerator()
    Y_datagen_val = ImageDataGenerator()
    X_datagen_val.fit(X_test, augment=True, seed=seed)
    Y_datagen_val.fit(Y_test, augment=True, seed=seed)
    X_test_augmented = X_datagen_val.flow(X_test, batch_size=batch_size, shuffle=True, seed=seed)
    Y_test_augmented = Y_datagen_val.flow(Y_test, batch_size=batch_size, shuffle=True, seed=seed)

    # combine generators into one which yields image and masks
    train_generator = zip(X_train_augmented, Y_train_augmented)
    test_generator = zip(X_test_augmented, Y_test_augmented)

    return train_generator, test_generator

if __name__ == "__main__":

  seed = 42
  random.seed = seed
  np.random.seed(seed=seed)

  IMG_WIDTH = 256
  IMG_HEIGHT = 256
  filepath = "./results/"

  ig = ImageExtractor(img_width=IMG_WIDTH, img_height=IMG_HEIGHT,  img_channels=3, train_path='./data/stage1_train/',
                  test_path='./data/stage1_test/')

  ig.extract_data()

  X_train, Y_train = ig.get_train_data()

  with open(filepath+'ig.pickle', 'wb') as f:
       pickle.dump(ig, f)

  model = unet_model(img_width=IMG_WIDTH, img_height=IMG_HEIGHT, n_ch_exps = [4, 5, 6, 7, 8, 9])
  model.summary()

  model.compile(optimizer='adam', loss=bce_dice_loss, metrics=[dice_coef])

  X_tr, X_test, y_tr, y_test = train_test_split(
      X_train, Y_train, test_size=0.05, random_state=seed)

  earlystopper = EarlyStopping(patience=5, verbose=1)
  checkpointer = ModelCheckpoint('model-dsbowl2018-1.h5', verbose=1, save_best_only=True)

  batch_size = 16

  train_generator, test_generator = get_train_test_augmented(X_data=X_tr, Y_data=y_tr, validation_split=0.15,
                                                             batch_size=batch_size, seed=seed
                                                            )
  model.fit_generator(train_generator, validation_data=test_generator, validation_steps=batch_size / 2,
                      steps_per_epoch=len(X_train) / (batch_size * 2), epochs=30,
                      callbacks=[earlystopper, checkpointer])

  results = model.evaluate(X_test, y_test, batch_size=batch_size)

  print("Dice coef:" + str(results[1]))

  model.save(filepath+'model-dsbowl2018-1.h5')


import pandas as pd
import seaborn as sn
import matplotlib.pyplot as plt
import scipy.stats as st
import numpy as np
import statsmodels.api as sm
import statsmodels.formula.api as smf
from sklearn import preprocessing
plt.rc("font", size=14)
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import train_test_split
sn.set(style="white")
sn.set(style="whitegrid", color_codes=True)

data = pd.read_csv("final_cleaned_spotify.csv")
df = pd.DataFrame(data)

#correlation
correlation_matrix = df.corr()
correlation_matrix = correlation_matrix.drop(correlation_matrix.columns[[range(15)]], axis=1)
print (correlation_matrix)

min_diff = st.norm.ppf(.975)

matrix_shape = correlation_matrix.shape
height = matrix_shape[0]
width = matrix_shape[1]

significance_top_100 = 0
significance_pop_artist = 0
total_features = height

sn.heatmap(correlation_matrix, annot=True)
plt.show()

#logistic regression
top_7_features = df.drop(df.columns[[0, 1, 4, 6, 8, 10, 11, 12, 14, 16, 17, 18, 19]], axis=1)
top100 = df[['Top100']]
logreg = sm.Logit(top100, top_7_features).fit()
print(logreg.summary())

acousticness = df['acousticness']
liveness = df['liveness']
first_interaction = smf.ols(formula='top100 ~ acousticness * liveness', data=df).fit()
print(first_interaction.summary())

top_7_and_pop = df.drop(df.columns[[0, 1, 4, 6, 8, 10, 11, 12, 14, 16, 17, 18]], axis=1)
logreg = sm.Logit(top100, top_7_and_pop).fit()
print(logreg.summary())


from copy import deepcopy
import pygame as pg
import moviepy.editor as mpe

BAR_H = 20
BAR_COLOR = pg.Color(100, 100, 100, 255)
CLIP_COLOR = pg.Color(200, 130, 0, 255)
CURSOR_COLOR = pg.Color(255, 255, 255, 255)

PAUSE = 0
PLAY = 1
SCRUB_LEFT = 2
SCRUB_RIGHT = 3
SCRUB_MAX_SPEED = 4.0

ED_ACCEPT = 0
ED_REJECT = 1
ED_CONTINUE = 2

# keyboard driven editor for speed.
class Editor(object):

	def __init__(self, imgseq, fps, clip, screen):
		self.screen = screen
		self.imgseq = imgseq
		self.domain = deepcopy(clip)
		self.clip = deepcopy(clip)
		self.fps = fps
		self.cursor = clip[0]

		w, h = screen.get_size()
		vw, vh, _ = imgseq[0].shape
		self.bar_top = h - BAR_H
		x_scale = w / float(clip[1] - clip[0])
		self.t2x = lambda t : x_scale * (t - clip[0])
		self.winsz = (w, h)

		self.state = PAUSE

	def cursor_clip(self):
		if self.cursor < self.domain[0]:
			self.cursor = self.domain[0]
		elif self.cursor >= self.domain[1]:
			self.cursor = self.domain[1] - 1

	def step(self, dt):
		frames = dt * self.fps
		if self.state == PAUSE:
			pass
		if self.state == PLAY:
			self.cursor += frames
		if self.state in [SCRUB_LEFT, SCRUB_RIGHT]:
			self.scrub_speed = min(self.scrub_speed * 1.05, SCRUB_MAX_SPEED)
			factor = self.scrub_speed * (-1 if self.state == SCRUB_LEFT else 1)
			self.cursor += factor * frames
		self.cursor_clip()

	def draw(self):
		frame = self.imgseq[int(self.cursor)]
		surface = pg.surfarray.make_surface(frame)
		r = pg.transform.rotate(surface, -90)
		f = pg.transform.flip(r, True, False)
		s = pg.transform.scale(f, self.winsz)
		self.screen.blit(s, (0, 0))
		self.screen.fill(BAR_COLOR, self.rect(*self.domain))
		self.screen.fill(CLIP_COLOR, self.rect(*self.clip))
		self.screen.fill(CURSOR_COLOR, self.cursor_rect())

	def cursor_rect(self):
		x = self.t2x(self.cursor)
		return pg.Rect(x - 2, self.bar_top, 4, BAR_H)

	def rect(self, t0, t1):
		x = self.t2x(t0)
		w = self.t2x(t1) - x
		return pg.Rect(x, self.bar_top, w, BAR_H)

	def keyscan(self):
		keys = pg.key.get_pressed()
		if keys[pg.K_LEFT] and keys[pg.K_RIGHT]:
			self.state = PAUSE
		elif keys[pg.K_LEFT]:
			self.state = SCRUB_LEFT
			self.scrub_speed = 1.0
		elif keys[pg.K_RIGHT]:
			self.state = SCRUB_RIGHT
			self.scrub_speed = 1.0
		else:
			self.state = PAUSE

	def keyevent(self, key):
		if key == pg.K_RIGHTBRACKET:
			self.clip[1] = int(self.cursor + 1)
		if key == pg.K_LEFTBRACKET:
			self.clip[0] = int(self.cursor)
		if key == pg.K_COMMA:
			self.cursor = int(self.cursor) - 1
			self.cursor_clip()
		if key == pg.K_PERIOD:
			self.cursor = int(self.cursor) + 1
			self.cursor_clip()
		if key == pg.K_b:
			self.cursor = self.clip[0]
		if key == pg.K_e:
			self.cursor = self.clip[1]
		if key == pg.K_SPACE:
			self.state = PLAY

		return ED_CONTINUE

class EditorTree(object):
	def __init__(self, editor):
		self.ed = editor
		self.left = self.right = None
		self.active = self.ed

	def step(self, dt):
		self.active.step(dt)

	def draw(self):
		self.active.draw()

	def keyscan(self):
		self.active.keyscan()

	def get_clips(self, arr):
		if self.left is not None:
			print("recursing left")
			self.left.get_clips(arr)
		if self.right is not None:
			print("recursing right")
			self.right.get_clips(arr)
		if (self.left, self.right) == (None, None):
			arr.append(self.ed.clip)

	def keyevent(self, key):
		ret = self.active.keyevent(key)

		if self.active is self.left:
			if ret == ED_REJECT:
				self.left = None
			if ret in [ED_ACCEPT, ED_REJECT]:
				self.active = self.right

		elif self.active is self.right:
			if ret == ED_REJECT:
				self.right = None
				self.active = None
				return ED_ACCEPT if self.left is not None else ED_REJECT
			if ret == ED_ACCEPT:
				self.active = None
				return ED_ACCEPT

		else:
			assert self.active == self.ed
			# split
			if key == pg.K_s:
				clip0 = [self.ed.clip[0], self.ed.cursor]
				clip1 = [self.ed.cursor, self.ed.clip[1]]
				self.left = EditorTree(Editor(self.ed.imgseq, self.ed.fps, clip0, self.ed.screen))
				self.right = EditorTree(Editor(self.ed.imgseq, self.ed.fps, clip1, self.ed.screen))
				self.active = self.left
				return ED_CONTINUE

			if key == pg.K_RETURN:
				return ED_ACCEPT

			if key == pg.K_BACKSPACE:
				return ED_REJECT

		return ED_CONTINUE

def interactive_editor(video, clip):
	ratio = float(video.size[0]) / video.size[1]
	h = 600
	w = int(ratio * h)
	screen = pg.display.set_mode((w, h),
		pg.DOUBLEBUF | pg.HWSURFACE | pg.FULLSCREEN)

	imgseq = list(video.subclip(*clip).iter_frames())
	ed = Editor(imgseq, video.fps, [0, len(imgseq)], screen)
	root = EditorTree(ed)

	fps = video.fps/2
	running = True
	clock = pg.time.Clock()
	clips = []

	while running:
		dt = clock.tick(fps) / 1000.0
		root.keyscan()
		root.step(dt)
		root.draw()
		pg.display.flip()

		for event in pg.event.get():
			if event.type == pg.QUIT:
				running = False
			if event.type == pg.KEYDOWN:
				ret = root.keyevent(event.key)
				if ret == ED_ACCEPT:
					root.get_clips(clips)
					running = False
				elif ret == ED_REJECT:
					running = False

	frame2time = lambda f: clip[0] + (1.0 / video.fps) * f
	clip2time = lambda c: (frame2time(c[0]), frame2time(c[1]))
	print("real end:", clip[1])
	print("approx end:", frame2time(len(imgseq)))
	return [clip2time(c) for c in clips]


from django.urls import path, include
from rest_framework.routers import DefaultRouter
from .views import ViewBrand, ViewCategory, ViewProducts, ViewProvider

router = DefaultRouter()

router.register(r'', ViewProducts, basename='products')
router.register(r'provider', ViewProvider, basename='provider')
router.register(r'category', ViewCategory, basename='category')
router.register(r'brand', ViewBrand, basename='brand')

urlpatterns = [
    path('', include(router.urls))
]


import json
import os
from aws_lambda_powertools import Logger
from datalake_library.configuration import DynamoConfiguration, SQSConfiguration, StateMachineConfiguration, \
    S3Configuration
from datalake_library.interfaces import DynamoInterface
from datalake_library.interfaces import SQSInterface
from datalake_library.interfaces import StatesInterface
from aws_solutions.core.helpers import get_service_client
from cloudwatch_metrics import metrics

logger = Logger(service="SDLF pipeline stage B", level="INFO", utc=True)

resource_prefix = os.environ["RESOURCE_PREFIX"]
STACK_NAME = os.environ['STACK_NAME']
METRICS_NAMESPACE = os.environ['METRICS_NAMESPACE']

def lambda_handler(event, _):
    """Checks if any items need processing and triggers state machine
    Arguments:
        event {dict} -- Dictionary with no relevant details
        context {dict} -- Dictionary with details on Lambda context
    """
    # record Lambda invocation to CloudWatch metric
    metrics.Metrics(METRICS_NAMESPACE, STACK_NAME, logger).put_metrics_count_value_1(metric_name="SdlfHeavyTransformRouting")

    try:
        team = event['team']
        pipeline = event['pipeline']
        stage = event['pipeline_stage']
        dataset = event['dataset']
        env = event['env']
        stage_bucket = S3Configuration(resource_prefix).stage_bucket
        dynamo_config = DynamoConfiguration(resource_prefix)
        dynamo_interface = DynamoInterface(dynamo_config)
        transform_info = dynamo_interface.get_transform_table_item(
            '{}-{}'.format(team, dataset))
        MIN_ITEMS_TO_PROCESS = int(
            transform_info['min_items_process']['stage_{}'.format(stage[-1].lower())])
        MAX_ITEMS_TO_PROCESS = int(
            transform_info['max_items_process']['stage_{}'.format(stage[-1].lower())])
        sqs_config = SQSConfiguration(resource_prefix, team, dataset, stage)
        queue_interface = SQSInterface(sqs_config.get_stage_queue_name)
        keys_to_process = []

        logger.info(
            'Querying {}-{} objects waiting for processing'.format(team, dataset))
        keys_to_process = queue_interface.receive_min_max_messages(
            MIN_ITEMS_TO_PROCESS, MAX_ITEMS_TO_PROCESS)
        # If no keys to process, break
        if not keys_to_process:
            return

        logger.info('{} Objects ready for processing'.format(
            len(keys_to_process)))
        keys_to_process = list(set(keys_to_process))

        response = {
            'statusCode': 200,
            'body': {
                "bucket": stage_bucket,
                "keysToProcess": keys_to_process,
                "team": team,
                "pipeline": pipeline,
                "pipeline_stage": stage,
                "dataset": dataset,
                "env": env
            }
        }
        logger.info('Starting State Machine Execution')
        state_config = StateMachineConfiguration(resource_prefix, team, pipeline, stage)
        StatesInterface().run_state_machine(
            state_config.get_stage_state_machine_arn, response)
        # record State Machine invocation to CloudWatch metric
        metrics.Metrics(METRICS_NAMESPACE, STACK_NAME, logger).put_metrics_count_value_1(metric_name="SdlfHeavyTransformRoutingSM")
    except Exception as e:
        # If failure send to DLQ
        if keys_to_process:
            dlq_interface = SQSInterface(sqs_config.get_stage_dlq_name)
            dlq_interface.send_message_to_fifo_queue(
                json.dumps(response), 'failed')
        logger.error("Fatal error", exc_info=True)
        raise e


#!/usr/bin/env python
# -*- encoding: utf-8 -*-
# Created on 2015-08-30 11:32:41
# Project: common

from pyspider.libs.base_handler import *
from pyspider.libs.url import _build_url
from six.moves.urllib.parse import urlparse
import urllib
import requests
import time
from datetime import datetime

keywords = [u'电力', u'电网|尼玛']

def get_proxy_list():
    proxy_list = ['localhost:1080']
    return proxy_list

class ProxyManager(object):
    def __init__(self, proxies=[]):
        self.proxies = list(proxies)
        self.record = {}
        self._cur = 0
        self._total = len(self.proxies)

    def update(self):
        pass

    def pick_one(self):

        proxy = self.proxies[self._cur] if self._total > 0 else None

        if self._cur < self._total -1 :
            self._cur +=1
        else:
            self._cur = 0

        return proxy

class CommonSiteHandler(BaseHandler):
    crawl_config = {
        'headers': {
            'User-Agent':'Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/43.0.2357.81 Safari/537.36',
        }
    }

    UPDATE_SETTINGS_INTERVAL = 3600

    def __init__(self):
        self.last_update_settings = time.time()
        self.keywords = [u'电力', u'电网|尼玛']
        self.proxy_on = False
        self.max_recurrence = 3
        self.proxy_manager = {}
        self.target_urls = [
            'http://www.cepca.org.cn/',
            'http://www.chng.com.cn',
            'http://www.chder.com/',
            'http://www.bj.xinhuanet.com/power',
            'http://gu.qq.com/sh600795',
            'http://www.epae.cn/',
            'http://www.ceppea.net/',
            'http://ecp.sgcc.com.cn/',
            'http://www.shiep.edu.cn/',
            'http://www.cgdc.com.cn/',
            'http://www.cpnn.com.cn/cpnn_zt/csdjh/',
            'http://www.dlks.com.cn',
            'http://www.csg.cn/',
            'http://www.fj.95598.cn',
            'http://www.chinapowerbid.com/',
            'http://www.clypg.com.cn/',
            'http://www.cpicorp.com.cn/',
            'http://www.cpeinet.com.cn',
            'http://www.cecm.net.cn',
            'http://www.dl-maxonic.com/',
        ]

    def _need_update(self):
        return time.time() - self.last_update_settings > self.UPDATE_SETTINGS_INTERVAL

    def _update_settings(self):

        ''' get from server  '''

        self.proxy_on = True
        self.need_parse = True
        self.max_recurrence = 3
        self.keywords = [u'电力', u'电网|尼玛']
        # self.target_urls = []
        if self.proxy_on:
            proxy_list = get_proxy_list()
            self.proxy_manager = ProxyManager(proxy_list)
        self.last_update_settings = time.time()

    def crawl(self, url, **kwargs):
        if self._need_update():
            self._update_settings()

        if self.proxy_on:
            kwargs['proxy'] = self.proxy_manager.pick_one()

        return super(CommonSiteHandler, self).crawl(url, **kwargs)

    def on_start(self):
        self._update_settings()

        for url in self.target_urls:
            parsed = urlparse(url)
            context = {
                'hostname': parsed.hostname,
                'cur_recurrence':self.max_recurrence
            }
            self.crawl(url, callback=self.index_page, save=context, force_update=True)

    def index_page(self, response):
        cur_recurrence = response.save.get('cur_recurrence', 0)
        response.save['cur_recurrence'] = cur_recurrence -1

        if cur_recurrence:
            response.save['cur_recurrence'] = cur_recurrence -1
            for each in response.doc('a[href^="http"]').items():
                title = each.text().strip(' \n\r\t')
                if title and any(title.find(kw) > -1 for kw in keywords):
                    self.crawl(each.attr.href, callback=self.index_page, save=response.save)
        else:
            return {
                "url": response.url,
                "title": response.doc('title').text(),
                "text": response.text,
            }


import os
import re
import sys
from bitarray import bitarray
import time

CHROM, POS, ID, REF, ALT, QUAL, FILTER, INFO, FORMAT = range(9)
R, A1, A2 = range(3)
UNPHASED, PHASED = '/', '|'

# Original Code: Terry Yu
# Reformat and Optimize using bcftools: John Luo

class VCF_Reader:
    def __init__(self, vcf_file: str, samples_file: str, outdir: str) -> None:
        print("Extracting sample ids from " + samples_file)
        t1s = time.time()

        sfd = open(samples_file, "r")
        samples = list(set(sid.strip() for sid in sfd.readlines()))
        sfd.close()

        succ_samples = []

        print("Extract samples: {0} from vcf file".format(samples))

        bcf_file = outdir + "data.bt.bcf"
        if os.path.exists(bcf_file):
            print("BCF file exists")
        else:
            print("Convert VCF file to filtered compressed BCF file..")
            os.system("bcftools view --force-samples -s {0} -O b {1} > {2}".format(','.join(samples), vcf_file, bcf_file))

        pra_file = outdir + "pra.txt"
        if os.path.exists(pra_file):
            print("POS+REF+ALT file exists")
        else:
            print("Obtain POS+REF+ALT information..")
            os.system("bcftools query -f '%POS\t%REF\t%ALT\n' {0} -o {1}".format(bcf_file, pra_file))

        rcode = 0
        for sample in samples:
            sample_file = "{0}{1}.txt".format(outdir, sample)
            print("Extracting information for sample: " + sample)
            rcode = os.system("bcftools view -O b -s {0} {1} | bcftools query -f '[%GT]\n' -o {2}".format(
                sample, bcf_file, sample_file
            ))
            if rcode != 0:
                print("Fail to extract sample: " + sample)
            else:
                print("Success to extract sample: " + sample)
                succ_samples.append(sample)

                rfd = open(sample_file, "r")
                # assume alt index < 10
                pfilter = bitarray(map(lambda x: x[0] != '.' and x[2] != '.', rfd.readlines()))
                rfd.close()

                pfilter_file = "{0}{1}_pfilter.txt".format(outdir, sample)
                os.system("echo "" > " + pfilter_file)

                pfd = open(pfilter_file, "w")
                pfd.write("{0}\n".format(pfilter.to01()))
                pfd.close()

        self.succ_samples = succ_samples
        self.outdir = outdir
        self.step1time = round(time.time() - t1s, 2)
        self.step2time = 0

        print("VCF reader initialized")
        print("All the successful extracted samples: ", succ_samples)
        return

    def split_samples(self):
        tss = time.time()
        pra_list = []
        pra_file = self.outdir + "pra.txt"
        max_variation = 0
        with open(pra_file, "r") as pfd:
            for line in pfd:
                pos, ref, alts = line.strip().split("\t")
                alt_list = alts.split(",")

                max_variation = max(max_variation, len(alt_list))

                alt_list.extend([ref, '.'])
                pra_list.append((pos, ref, alt_list))
            pfd.close()

        # for fast lookup instead of type conversion
        roundup_idx = {}
        for i in range(1, max(2, max_variation)):
            roundup_idx[str(i)] = i - 1
        # special case for ref
        roundup_idx['0'] = -2
        # special case for missing
        roundup_idx['.'] = -1

        for sample in self.succ_samples:
            ts = time.time()
            sample_file = "{0}{1}.txt".format(self.outdir, sample)
            afile0 = "{0}{1}_0.txt".format(self.outdir, sample)
            afile1 = "{0}{1}_1.txt".format(self.outdir, sample)
            os.system("echo "" > {0}; echo "" > {1}".format(afile0, afile1))
            afd1, afd2 = open(afile0, "w"), open(afile1, "w")
            sfd = open(sample_file, "r")
            for (pos, ref, alts), line in zip(pra_list, sfd.readlines()):
                try:
                    a1, a2 = re.split("[\|]", line.strip())
                    afd1.write("{0}:{1}:{2}\n".format(pos, ref, alts[roundup_idx[a1]]))
                    afd2.write("{0}:{1}:{2}\n".format(pos, ref, alts[roundup_idx[a2]]))
                except:
                    print(roundup_idx)
                    sys.exit(1)

            afd1.close()
            afd2.close()
            sfd.close()
            te = time.time()
            print("Sample: {0} finished, time elapsed: {1}".format(sample, round(te-ts, 2)))

        self.step2time = round(time.time() - tss, 2)
        return 0


import matplotlib.pyplot as plt

import numpy as np

import timeit

class LogisticRightHandSide:
    def __init__(self, h, b_l, b_r, kappa):
        self.h, self.kappa = h, kappa
        num_points = int(1 / h) + 1
        self.A = np.zeros((num_points, num_points))
        flat_a = self.A.ravel()
        flat_a[1::num_points + 1] = 1
        flat_a[num_points::num_points + 1] = 1
        flat_a[::num_points + 1] = -2
        self.F = np.zeros(num_points)
        self.F[0] = b_l
        self.F[-1] = b_r

    def __call__(self, u):
        return self.kappa / self.h ** 2 * (np.dot(self.A, u) + self.F)

class ComputationalMethod:
    def __init__(self, f, h, tau, t_start, t_end):
        self.f, self.h_dim = f, int(1 / h) + 1
        self.t_dim = int((t_end - t_start) / tau) + 1
        self.dt = tau
        self.solutionArray = np.zeros((self.t_dim, self.h_dim))
        self.timeArray = np.linspace(t_start, t_end, self.t_dim)
        self.spaceArray = np.linspace(0, 1, self.h_dim)

        self.tStart, self.tEnd = float(t_start), float(t_end)

        print('Объект класса ' + self.__class__.__name__ + ' создан.')

    def Solve(self):
        print('Начало рассчёта методом %s' % self.__class__.__name__)
        start = timeit.default_timer()
        u0 = np.zeros(self.h_dim)
        for i in range(self.h_dim):
            if (self.spaceArray[i] >= 0.4) and (self.spaceArray[i] <= 0.6):
                u0[i] = 1.
        self.solutionArray[0] = u0

        for t in range(self.t_dim - 1):
            u_old = self.solutionArray[t]
            self.solutionArray[t + 1] = self.GetNextStep(u_old)

            if (t + 1) % ((self.t_dim - 1) // 10) == 0:
                t = self.timeArray[t + 1]
                print('%.2f%% вычислений завершено' % (100. * float(t) / (self.tEnd - self.tStart)))

        elapsed_time = float(timeit.default_timer() - start)
        print('Время расчета: %.2e сек.' % elapsed_time)

    def GetNextStep(self, u_old):
        raise NotImplementedError

    def SetPlot(self):
        print(self.solutionArray[-1])
        plt.plot(self.spaceArray, self.solutionArray[-1], '-o',
                 label=str(self.__class__.__name__))

class ExplicitEuler(ComputationalMethod):
    def GetNextStep(self, u_old):
        return u_old + self.dt * self.f(u_old)

class Heun(ComputationalMethod):
    def GetNextStep(self, u_old):
        f = self.f
        prev = u_old + self.dt * f(u_old)
        return u_old + self.dt / 2. * (f(u_old) + f(prev))

class RK4(ComputationalMethod):
    def GetNextStep(self, u_old):
        f, dt = self.f, self.dt
        k1 = f(u_old)
        k2 = f(u_old + dt / 2. * k1)
        k3 = f(u_old + dt / 2. * k2)
        k4 = f(u_old + dt * k3)
        return u_old + dt / 6. * (k1 + 2. * k2 + 2. * k3 + k4)

b_l = 0.
b_r = 0.
kappa = 0.1
h = 1. / 25
tau = h ** 2 / (200 * kappa)
tStart = 0.
tEnd = 0.04

func = LogisticRightHandSide(h, b_l, b_r, kappa)

cmp_mtds = [ExplicitEuler, Heun, RK4]

plt.figure(figsize=(11, 7))
plt.rc('font', size=20)

for m in cmp_mtds:
    method = m(func, h, tau, tStart, tEnd)
    method.Solve()
    method.SetPlot()

plt.grid(False)
plt.title('Динамика популяции')
plt.xlabel('Пространство')
plt.ylabel('Популяция')
plt.legend()
plt.show()


# -*- coding: utf-8 -*-
"""
Created on Wed Feb 27 21:40:44 2019

@author: Administrator
"""

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt

import warnings
warnings.filterwarnings('ignore')

from bokeh.plotting import figure,show,output_file
from bokeh.models import ColumnDataSource

'''
(1) 加载数据
'''

import os
os.chdir(r'C:\Users\Administrator\Desktop\spyder\城市餐饮店铺选址分析')
df2 =pd.read_excel('空间统计.xlsx',sheetname=0)
df2.fillna(0,inplace = True)
df2.columns = ['人口密度','道路长度','餐饮计数','素菜餐饮计数','lng','lat']

'''
(2) 指标统计
'''

df2['rkmd_norm'] = (df2['人口密度']-df2['人口密度'].min())/(df2['人口密度'].max()-df2['人口密度'].min()) # 人口密度指标标准化
df2['cyrd_norm'] = (df2['餐饮计数']-df2['餐饮计数'].min())/(df2['餐饮计数'].max()-df2['餐饮计数'].min()) # 餐饮热度指标标准化
df2['tljp_norm'] = (df2['素菜餐饮计数'].max()-df2['素菜餐饮计数'])/(df2['素菜餐饮计数'].max()-df2['素菜餐饮计数'].min()) # 同类竞品指标标准化
df2['dlmi_norm'] = (df2['道路长度']-df2['道路长度'].min())/(df2['道路长度'].max()-df2['道路长度'].min()) # 道路密度指标标准化
# 指标标准化

df2['final_score'] = df2['rkmd_norm']*0.4 + df2['cyrd_norm']*0.3 + df2['tljp_norm']*0.1 + df2['dlmi_norm']*0.2
data_final_q2 = df2.sort_values(by = 'final_score',ascending=False).reset_index()
data_final_q2[:10]
# 计算综合评分并查看TOP10的网格ID

'''
(3) 制作空间散点图
'''
from bokeh.models import HoverTool

output_file('project07_h2.html')

data_final_q2['size'] = data_final_q2['final_score'] * 15
data_final_q2['color'] = 'green'
data_final_q2['color'].iloc[:10] = 'red'
# 添加size字段

source = ColumnDataSource(data_final_q2)
# 创建ColumnDataSource数据

hover = HoverTool(tooltips=[("经度", "@lng"),
                            ("纬度", "@lat"),
                            ("最终得分", "@final_score"),
                           ])  # 设置标签显示内容
p = figure(plot_width=800, plot_height=800,
                title="空间散点图" ,
                tools=[hover,'box_select,reset,wheel_zoom,pan,crosshair'])
# 构建绘图空间

p.square(x = 'lng',y = 'lat',source = source,
         line_color = 'black',fill_alpha = 0.5,
        size = 'size',color = 'color')
p.ygrid.grid_line_dash = [6, 4]
p.xgrid.grid_line_dash = [6, 4]
# 散点图
show(p)

print('finashed')


import matplotlib
import matplotlib.pyplot as plt
import seaborn as sns
import numpy as np
from utils import tools
from sklearn import preprocessing
import scanpy as sc
import umap

sns.set(style='white', rc={'figure.figsize': (8, 6), 'figure.dpi': 150})
matplotlib.use('TkAgg')
bar = ['c', 'violet', 'darkorange', 'darkseagreen', 'r', 'silver']

def plot_gradient_marker_gene(x, adata, gene_names, max_list, xy_label, figsize, sub_range, save_path, font_size=14, cmap='Blues'):
    all_gene = adata.var

    index_list, gene_value, color_list, x_list, xy_label_list = [], [], [], [], []
    for name in gene_names:
        index_list.append(tools.find_gene_pos(all_gene, name))
        x_list.append(x)
        xy_label_list.append(xy_label)
    for i in index_list:
        gene_value.append(tools.cacu_color(adata.X.A, i))

    fig = plt.figure(figsize=figsize)
    for i in range(len(x_list)):
        para_sub = sub_range * 10 + i + 1
        ax = fig.add_subplot(para_sub)
        sc = ax.scatter(x_list[i][:, 0], x_list[i][:, 1], c=gene_value[i], vmin=0, vmax=max_list[i], cmap=cmap, s=1, linewidth=0)
        plt.xlabel(xy_label[i] + '1')
        plt.ylabel(xy_label[i] + '2')
        plt.title(gene_names[i], fontsize=font_size)
        cbar = fig.colorbar(sc)
    plt.savefig(save_path)

def plot_mark_gene(x, adata, gene_names, thresholds, xy_label, figsize, sub_range, save_path, font_size=14):
    all_gene = adata.var
    index_list, gene_value, color_list, x_list, xy_label_list = [], [], [], [], []
    for name in gene_names:
        index_list.append(tools.find_gene_pos(all_gene, name))
        x_list.append(x)
        xy_label_list.append(xy_label)
    for i in index_list:
        gene_value.append(preprocessing.minmax_scale(tools.cacu_color(adata.X.A, i)))
    for k in range(len(gene_value)):
        para_color = []
        for i in range(len(gene_value[k])):
            if gene_value[k][i] < thresholds[k]:
                para_color.append(bar[5])
            else:
                para_color.append(bar[4])
        color_list.append(para_color)
    plot_multi(x_list, color_list, xy_label_list, gene_names, figsize, sub_range, 1, save_path, font_size=font_size)

def plot_multi(x, color, xy_label, title, fig_size, sub_range, size, sava_path, c_map='Set2', font_size=14):
    fig = plt.figure(figsize=fig_size)

    for i in range(len(x)):
        para_sub = sub_range*10+i+1
        ax = fig.add_subplot(para_sub)
        ax.scatter(x[i][:, 0], x[i][:, 1], c=color[i], cmap=c_map, s=size, linewidth=0, alpha=0.7)
        plt.xlabel(xy_label[i] + '1')
        plt.ylabel(xy_label[i] + '2')
        plt.title(title[i], fontsize=font_size)
    plt.savefig(sava_path)

# given expression matrix, gene_list(marker_gene) , plot a heat map
def plot_expression_matrix(x, gene_index_list, title, save_path, normalize='minmax', cmap = 'Set2'):
    expression = []
    for i in range(len(gene_index_list)):
        expression.append(x[:, gene_index_list[i]])
    expression = np.array(expression)
    if normalize == 'minmax':
        expression = preprocessing.scale(expression)
    elif normalize == 'zscore':
        expression = preprocessing.minmax_scale(expression)
    plt.figure()
    plt.title(title)
    plt.imshow(expression, interpolation='nearest', cmap=cmap, origin='lower')
    plt.savefig(save_path)

def plot_integration(datasets, title, out_path, xy_label='umap'):
    fig = plt.figure(figsize=(10, 6))
    ax = fig.add_subplot(111)
    plt.title(title)
    for i in range(len(datasets)):
        ax.scatter(datasets[i][:, 0], datasets[i][:, 1], c=bar[i], s=2.5, linewidth=0)
    plt.xlabel(xy_label+'1')
    plt.ylabel(xy_label+'2')
    plt.savefig(out_path)

def plot_cluster(x, label, save_path, cmap='Set1', xy_label='tsne'):
    fig = plt.figure(figsize=(10, 6))
    ax = fig.add_subplot(111)
    ax.scatter(x[:, 0], x[:, 1], c=label, cmap=cmap, s=2, linewidth=0)
    plt.xlabel(xy_label + '1')
    plt.ylabel(xy_label + '2')
    plt.savefig(save_path)

def plot_integration_with_cluster(datasets, label, out_path, c_map='Set1', xy_label='tsne', title='Original'):
    fig = plt.figure(figsize=(18, 8))
    data = np.concatenate(datasets)
    ax = fig.add_subplot(121)
    plt.title(title)

    scatter = ax.scatter(datasets[0][:, 0], datasets[0][:, 1], c=bar[0], s=2.5, linewidth=0, label='DNA')
    scatter = ax.scatter(datasets[1][:, 0], datasets[1][:, 1], c=bar[1], s=2.5, linewidth=0, label='Chromatin')
    plt.xlabel(xy_label + '1')
    plt.ylabel(xy_label + '2')
    plt.legend(loc="upper right", title="Batch")

    ax = fig.add_subplot(122)
    plt.title(title)
    plt.xlabel(xy_label + '1')
    plt.ylabel(xy_label + '2')
    scatter = ax.scatter(data[:, 0], data[:, 1], c=label, cmap=c_map, s=3, linewidth=0)
    handles, labels = scatter.legend_elements(num=19, alpha=0.6)
    celltype = ['Ex23_Cux1', 'Ex6_Tle4', 'Ex345_Foxp1', 'IP_Eomes','RG','Ex4_Tenm3','Ex5_Crmp1','Ex45_Galntl6','In_1','In_2',
                'IP_Hmgn2','Ex56_Epha6','Ex23_Cntn2','IP_Gadd45g','CR','Endo','Peri','OPC','Mic']

    legend2 = plt.legend(handles, celltype, loc="upper center", title="Cell Type", ncol=4)
    plt.savefig(out_path)

def plot_davae_result_labeled(adata, title='DAVAE', cmap_batch='Set2', cmap_label='Set3', save=None):
    fig = plt.figure(figsize=(18, 8))
    ax = fig.add_subplot(121)
    plt.title(title)
    batch = adata.obs['batch'].values
    batch = np.array(batch, dtype=int)
    label = adata.obs['label']
    data_emb = adata.obsm['umap']
    xy_label = 'umap'
    ax.scatter(data_emb[:, 0], data_emb[:, 1], cmap=cmap_batch, c=batch, s=1, linewidth=0)
    plt.xlabel(xy_label + '1')
    plt.ylabel(xy_label + '2')
    ax = fig.add_subplot(122)
    plt.title(title)
    plt.xlabel(xy_label + '1')
    plt.ylabel(xy_label + '2')
    ax.scatter(data_emb[:, 0], data_emb[:, 1], c=label, cmap=cmap_label, s=1, linewidth=0)
    plt.show()
    if save != None:
        plt.savefig(save)

def plot_davae_result(adata, cmap='Set2'):
    title = 'DAVAE'
    fig = plt.figure(figsize=(12, 8))
    ax = fig.add_subplot(111)
    plt.title(title)
    batch = adata.obs['batch']
    data_emb = adata.obsm['umap']
    xy_label = 'umap'
    ax.scatter(data_emb[:, 0], data_emb[:, 1], cmap=cmap, c=batch, s=1, linewidth=0)
    plt.xlabel(xy_label + '1')
    plt.ylabel(xy_label + '2')
    plt.show()


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models, migrations

class Migration(migrations.Migration):

    dependencies = [
    ]

    operations = [
        migrations.CreateModel(
            name='Content',
            fields=[
                ('id', models.AutoField(serialize=False, auto_created=True, primary_key=True, verbose_name='ID')),
                ('description', models.CharField(max_length=255)),
                ('content', models.TextField()),
                ('created', models.DateTimeField(auto_now_add=True)),
                ('last_edited', models.DateTimeField(auto_now_add=True)),
            ],
        ),
        migrations.CreateModel(
            name='Node',
            fields=[
                ('id', models.AutoField(serialize=False, auto_created=True, primary_key=True, verbose_name='ID')),
                ('name', models.CharField(max_length=200)),
                ('oneormany', models.BooleanField(default=False)),
                ('parent', models.ManyToManyField(related_name='parent_rel_+', blank=True, to='resolvespec.Node')),
            ],
        ),
        migrations.AddField(
            model_name='content',
            name='node',
            field=models.ForeignKey(to='resolvespec.Node'),
        ),
    ]