import time
import pandas as pd
import numpy as np
from sklearn.model_selection import train_test_split
from sklearn.metrics import accuracy_score

class Tree(object):
    def __init__(self, node_type, label = None, feature_index = None):
        self.node_type = node_type
        self.label = label
        self.feature_index = feature_index

        self.dict={}    #key->value

    def add_node(self, key ,value):
        self.dict[key] = value

    def predict(self, features):
        if self.node_type == 'leaf':
            return self.label

        tree = self.dict[features[self.feature_index]]
        return tree.predict(features)

class DecisionTree(object):

    def compute_ent(self, x):
        #随机变量分布
        x_values = set(x[i] for i in range(x.shape[0]))   ##行数
        ent = 0.0
        for x_value in x_values:
            p = float(x[x == x_value].shape[0]) / x.shape[0]
            log_p = np.log2(p)
            ent -= p * log_p
        return ent

    def compute_condition_ent(self, x, y):
        #x:feature y:train_labels
        x_values = set(x[i] for i in range(x.shape[0]))   ##行数
        condition_ent = 0.0
        # 根据特征x划分n个子集 y_sub_1, y_sub_2, y_sub_3,...
        for x_value in x_values:
            #子集
            y_sub = y[x == x_value]
            #子集样本个数/训练集样本个数
            y_p = float(y_sub.shape[0]) / y.shape[0]
            #子集熵
            y_ent = self.compute_ent(y_sub)
            condition_ent += y_p * y_ent
        return condition_ent

    def compute_info_gain(self, x, y):

        info_gain = self.compute_ent(y) - self.compute_condition_ent(x, y)
        return info_gain

    def train(self,train_features, train_labels, features_index, epsilon):

        global total_label
        LEAF = 'leaf'
        INTERNAL = 'internal'

        labels_set = set(train_labels)

        #step1:所有实例都属于C_k
        if len(labels_set) == 1:
            return Tree(LEAF, label=labels_set.pop())

        #def g(x): return x == i
        #key :处理参数
        #(max_label, max_len) = max([(i,len(list(filter(lambda x:x==i,train_labels)))) for i in range(total_labels)], key = lambda x:x[1])
        max_label =max(train_labels)

        #step2:特征集为空集
        if len(labels_set) == 0:
            return Tree(LEAF, label = max_label)

        #step3:
        max_feature_index = 0
        max_info_gain = 0

        train_ent = self.compute_ent(train_labels)

        for feature_index in features_index:

            feature = np.array(train_features[:, feature_index].flat)  #f=[1,0,2,1,0,...]
            feature_condition_ent = self.compute_condition_ent(feature, train_labels)
            feature_info_gain = train_ent - feature_condition_ent

            if feature_info_gain > max_info_gain:
                max_feature_index, max_info_gain = feature_index, feature_info_gain

        #step4:
        if max_info_gain < epsilon:
            return Tree(LEAF, label = max_label)

        #step5:
        sub_features_index = list(filter(lambda x: x != max_feature_index, features_index))
        tree = Tree(INTERNAL, feature_index=max_feature_index)

        feature_col = np.array(train_features[:, max_feature_index].flat)
        feature_values = set([feature_col[i] for i in range(feature_col.shape[0])])
        #以各子集为特征集 分别递归
        for feature_value in feature_values:
            index = []
            for i in range(len(train_labels)):
                if train_features[i][max_feature_index] == feature_value:
                    index.append(i)

            sub_train_feature = train_features[index]
            sub_train_label = train_labels[index]

            sub_tree = self.train(sub_train_feature, sub_train_label, sub_features_index, epsilon)
            tree.add_node(feature_value, sub_tree)

        return tree

    def predict(self, test_features, tree):
        result = []
        for f in test_features:
            f_predict = tree.predict(f)
            result.append(f_predict)
        return np.array(result)

if __name__ == '__main__':
    print('Start loading data...')
    time_1 = time.time()
    raw_data = pd.read_csv('../data/loan_application.csv', header=0)
    data = raw_data.values

    features = data[0::, 1::]
    labels = data[::, 0]

    total_labels = 2

    # 选取 2/3 数据作为训练集， 1/3 数据作为测试集
    train_features, test_features, train_labels, test_labels = train_test_split(features, labels, test_size=0.33,
                                                                                random_state=23323)
    time_2 = time.time()
    print('loading data cost ', time_2 - time_1,' seconds', '\n')
    print('Start training...')
    time_3 =time.time()
    dt = DecisionTree()
    tree = dt.train(train_features, train_labels, [i for i in range(len(train_features[0]))], 0.1)
    print('training cost ', time_3 - time_2, ' seconds', '\n')

    print('Start predicting...')
    test_predict = dt.predict(test_features, tree)
    time_4 = time.time()
    print('predicting cost ', time_4 - time_3, ' seconds', '\n')

    score = accuracy_score(test_labels, test_predict)

    print('The accruacy socre is ', score)


#!/usr/bin/env python3
# -*- coding:utf-8 -*-
# Author: lionel

import tensorflow as tf
from tensorflow.contrib.layers import xavier_initializer
import numpy as np

class ner(object):
    def __init__(self, config, iterator):
        self.config = config
        self.input, self.target = iterator.get_next()

        self.seq_lengths = np.full(self.config['batch_size'], self.config['max_length'])

        with tf.device('/cpu:0'), tf.name_scope('embedding'):
            embedding = tf.get_variable(name='embedding',
                                        shape=[self.config['vocab_size'], self.config['embedding_size']],
                                        dtype=tf.float32,
                                        initializer=xavier_initializer())
            embed = tf.nn.embedding_lookup(embedding, self.input)

        fw_cell = tf.nn.rnn_cell.BasicLSTMCell(self.config['units'])
        bw_cell = tf.nn.rnn_cell.BasicLSTMCell(self.config['units'])

        outputs, final_states = tf.nn.bidirectional_dynamic_rnn(fw_cell, bw_cell, embed, dtype=tf.float32)

        # output = tf.concat(outputs, axis=2)

        output_fw, output_bw = outputs
        output = tf.concat([output_fw, output_bw], axis=-1)

        # 在这里设置一个无偏置的线性层
        self.W = tf.get_variable('W', shape=[2 * self.config['units'], self.config['num_tags']], dtype=tf.float32)
        self.b = tf.get_variable('b', shape=[self.config['num_tags']], dtype=tf.float32)
        matricized_output = tf.reshape(output, [-1, 2 * self.config['units']])
        matricized_unary_scores = tf.matmul(matricized_output, self.W) + self.b
        self.scores = tf.reshape(matricized_unary_scores,
                                 [-1, self.config['max_length'], self.config['num_tags']])

        ## softmax
        # self.prediction = tf.cast(tf.argmax(self.scores, axis=-1), tf.int32)
        # losses = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=self.scores, labels=self.target)
        # mask = tf.sequence_mask(self.seq_lengths)
        # losses = tf.boolean_mask(losses, mask)
        # self.loss = tf.reduce_mean(losses)
        # self.correct_num = tf.cast(tf.equal(self.target, self.prediction), tf.float32)

        ## crf
        # 计算log-likelihood并获得transition_params
        self.log_likelihood, transition_params = tf.contrib.crf.crf_log_likelihood(self.scores, self.target,
                                                                                self.seq_lengths)
        # 进行解码（维特比算法），获得解码之后的序列viterbi_sequence和分数viterbi_score
        self.viterbi_sequence, viterbi_score = tf.contrib.crf.crf_decode(
            self.scores, transition_params, self.seq_lengths)

        self.loss = tf.reduce_mean(-self.log_likelihood)

        self.train_op = tf.train.AdamOptimizer(0.01).minimize(self.loss)

        self.correct_num = tf.cast(tf.equal(self.target, self.viterbi_sequence), tf.float32)
        self.accuracy = tf.reduce_mean(self.correct_num)


import os
os.system('cls');

def GetMessageInfo():
    first = input("What is your first name?  ")
    last = input(first + "'s last name is: ")
    greeting = "Greetings " + first + " " + last + "!"
    return greeting;

#myGreeting = GetMessageInfo()
#print(myGreeting)

print(GetMessageInfo())

#print(greeting)   #private to function

first = input("What is your first name outside of function?  ")
last = input(first + "'s last name is outside of function: ")

if len(first) < 5:
    print("The name " + first + " has a length of " + str(len(first)))
    username = first.ljust(5, '*')
    print(username)
elif len(first) >= 10:
    print("The 10th character is " + first[9])
    print("The new name is " + first[2 : 10])
elif len(first) == 6:
    print(first.upper().ljust(9, 'a').rjust(12, '$'))
else:
    print("The name of " + first + " " + last + " is perfect already")

#print(username)

friends = []

for x in range(0, 5):
    myFriend = input("Name of friend number " + str(x) + "?  ")
    friends.append(myFriend)

for fr in friends:
    print("My friend is " + fr)

fileOfFriends = input("\nName of file that will hold friends:  ")

with open(fileOfFriends, "w") as myFile:  #create object named myFile - file open for write
    myFile.write("Here is my list of friends:\n")
    for f in friends:
        myFile.write(f)
        myFile.write("\n")

input("Press Enter to Read " + fileOfFriends)

with open(fileOfFriends, "r+") as myFile:  #create object named myFile - file open for read
    print(myFile.read())


import logging

log_filepath = f"{__file__}.log"
with open(log_filepath, mode='w', encoding='utf-8'):
  logging.basicConfig(filename=log_filepath, level=logging.DEBUG)

# logging.basicConfig(filename='example.log', encoding='utf-8', level=logging.DEBUG)  # after Python3.9
logging.debug('This message should go to the log file')
logging.info('So should this')
logging.warning('And this, too')
logging.error('And non-ASCII stuff, too, like Øresund and Malmö')


"""
@ Version : ??
@ Author  : liujinjia
@ File    : message.py
@ Project : base_test
@ Create Time: 2017-05-19 16:52
"""

CASE_IS_NOT_DEFINE = "你所查询的测试用例不存在， 请确认查询信息是否正确？"
CASE_CREATE_OVER = "测试用例已全部生成"
CASE_ALREADY_EXISTS = "测试用例信息已存在"


# -*- coding: utf-8 -*-
"""
Created on Thu May 10 18:23:02 2018

@author: xinger
"""
import os
import sqlite3
import pandas as pd
from datetime import datetime
import warnings

warnings.filterwarnings("ignore")
today = int(datetime.strftime(datetime.today(), '%Y%m%d'))

class sqlite_db(object):
    def __init__(self, fp):
        path = os.path.join(fp, 'data.sqlite')
        self.fp = fp
        self.path = path
        self.conn = sqlite3.connect(path)

    @property
    def all_table_names(self):
        sql = '''SELECT name FROM sqlite_master WHERE type ='table' ORDER BY name;'''
        c = self.conn.cursor()
        c.execute(sql)
        return [i[0] for i in c.fetchall()]

    def execute(self, sql):
        c = self.conn.cursor()
        c.execute(sql)
        self.conn.commit()

    def set_attr(self, view, attrs):
        if 'attrs' not in self.all_table_names:
            sql = '''CREATE TABLE attrs
                    (
                    view CHAR(20) NOT NULL,
                    updated_date INT NOT NULL
                    );'''
            self.execute(sql)

        df = pd.read_sql('select * from attrs;', self.conn)
        if df.shape == (0, 0):
            sql = '''INSERT INTO attrs(view,updated_date) VALUES("%s",%s);''' % (view, attrs)
            self.execute(sql)

        elif view not in df['view'].values:
            sql = '''INSERT INTO attrs(view,updated_date) VALUES("%s",%s);''' % (view, attrs)
            self.execute(sql)

        else:
            sql = '''UPDATE attrs SET updated_date = %s  WHERE view = "%s";''' % (attrs, view)
            self.execute(sql)

    def update_attr(self):
        c = self.conn.cursor()
        for view in self.all_table_names:
            datename = None
            c.execute('''PRAGMA table_info("%s");''' % (view, ))
            fields = [i[1] for i in c.fetchall()]
            if 'trade_date' in fields:
                datename = 'trade_date'
            elif 'ann_date' in fields:
                datename = 'ann_date'

            if datename:
                sql = '''select max(%s) from "%s";''' % (datename, view)
                c.execute(sql)
                last_date = c.fetchall()[0][0]
                self.set_attr(view, last_date)
            else:
                self.set_attr(view, today)

    def get_update_info(self, view):
        sql = '''select updated_date from "attrs" WHERE view = "%s";''' % (view,)
        c = self.conn.cursor()
        try:
            c.execute(sql)
            date = c.fetchall()[0][0]
        except:
            date = None
        return date

    def update_table(self, view, data, if_exists='append', on='index'):
        if on == 'index':
            return self.update_on_index(view, data, if_exists=if_exists)
        if on == 'columns':
            return self.update_on_index(view, data, if_exists=if_exists)

    def update_on_index(self, view, data, if_exists='append'):
        data.to_sql(view, self.conn, if_exists=if_exists, index=False)
        print(view, 'ok!')

    def update_on_columns(self):
        pass

if __name__ == '__main__':
    fp = r'C:\Users\xinger\Sync\data'
    sqldb = sqlite_db(fp)
    sqldb.update_attr()


import random
from datetime import timedelta
from django.core.management.base import BaseCommand
from django_seed import Seed
from users import models as user_models
from rooms import models as room_models
from reservations import models as reservation_models

NAME = "Reservations"

class Command(BaseCommand):
    help = f"Seed {NAME}"

    def add_arguments(self, parser):
        parser.add_argument(
            "--number",
            default=1,
            type=int,
            help=f"How many times do you want to repeat to create {NAME}?",
        )

    def handle(self, *args, **options):
        number = options.get("number")
        seeder = Seed.seeder()

        all_users = user_models.User.objects.all()
        all_rooms = room_models.Room.objects.all()

        seeder.add_entity(
            reservation_models.Reservation,
            number,
            {
                "guest": lambda x: random.choice(all_users),
                "room": lambda x: random.choice(all_rooms),
                "check_in": lambda x: seeder.faker.date_between(
                    start_date="-60d", end_date="-59d"
                ),
                "check_out": lambda x: seeder.faker.date_between(
                    start_date="-57d", end_date="-56d"
                )
                + timedelta(days=random.randint(1, 31)),
            },
        )
        seeder.execute()

        self.stdout.write(
            self.style.NOTICE(f"{number} {NAME} are created successfully!")
        )


import contextlib
import io
import random
import shutil
import string
import tempfile
from pathlib import Path

from .utils.fileobj_wrapper import FileobjWrapper

__all__ = ['get_public_temppath', 'MemoryTempfile']

@contextlib.contextmanager
def get_public_temppath(dir_=None, suffix='.tmp', delete=True):
    """
    Returns a context manager that yields a path to publicly accessible file. The file will be deleted when the context manager is closed.

    :param dir_:
    :param suffix:
    :param delete:
    :return:
    """
    if dir_ is None:
        dir_ = tempfile.gettempdir()

    dir_ = Path(dir_)

    def random_filename():
        return ''.join(random.choice(string.ascii_lowercase) for _ in range(10))

    while True:
        path = dir_ / (random_filename() + suffix)
        try:
            path.touch(0o777, exist_ok=False)
            break
        except FileExistsError:
            continue

    try:
        yield path
    finally:
        if delete:
            path.unlink()

class FakeFileIO(io.FileIO):
    def __init__(self, *args, **kwargs):
        # skip the io.FileIO constructor because we don't have a filename that we could pass to it
        super(io.FileIO, self).__init__(*args, **kwargs)

class MemoryTempfile(FileobjWrapper, FakeFileIO):
    # because FileobjWrapper may or may not use a __new__ method to return fully
    # functional wrapper objects, we have to implement our own __new__ that
    # creates a file object and calls super().__new__(fileobj). We then have to
    # manually call FileobjWrapper.__init__(fileobj) because that file object
    # won't be automatically passed as an argument to __init__.
    def __new__(cls, mode='w+b', *args, **kwargs):
        fileobj = cls._make_fileobj(mode)
        self = super().__new__(cls, fileobj)
        FileobjWrapper.__init__(self, fileobj, forward_context=True)
        return self

    def __init__(self, mode=None, maxsize=20*1024**2, tempfile_factory=tempfile.NamedTemporaryFile, delete=True):
        self._exists_on_disk = False
        self._tempfile_factory = tempfile_factory

        self._mode = mode
        self._remaining_size = maxsize
        self._delete = delete

    @staticmethod
    def _make_fileobj(mode):
        if mode is None:
            return None
        elif 'b' in mode:
            return io.BytesIO()
        else:
            return io.StringIO()

    def _set_fileobj(self, mode):
        self._mode = mode
        self.wrapped_fileobj = self._make_fileobj(mode)

    def _switch_to_disk(self):
        if self._exists_on_disk:
            return

        self._internal_fileobj = self._tempfile_factory(self._mode, delete=self._delete)
        self._internal_fileobj.__enter__()

        self.wrapped_fileobj.seek(0)
        shutil.copyfileobj(self.wrapped_fileobj, self._internal_fileobj)
        self.wrapped_fileobj = self._internal_fileobj

        self._exists_on_disk = True

    def write(self, data):
        if self._mode is None:
            self._set_fileobj('w+' if isinstance(data, str) else 'w+b')

        if not self._exists_on_disk:
            self._remaining_size -= len(data)

            if self._remaining_size < 0:
                self._switch_to_disk()

        self.wrapped_fileobj.write(data)

    @property
    def name(self):
        self._switch_to_disk()
        return self.wrapped_fileobj.name

    def fileno(self):
        self._switch_to_disk()
        return self.wrapped_fileobj.fileno()

    def __exit__(self, *args, **kwargs):
        # if we've created a temporary file on disk, make sure it's
        # closed even if FileobjWrapper won't do it for us
        if self._exists_on_disk and not self.forward_context:
            self.wrapped_fileobj.__exit__(*args, **kwargs)

        return super().__exit__(*args, **kwargs)

    def __repr__(self):
        return object.__repr__(self)


import sys

sys.path.append('..')
from common.core import *
from common.gfxutil import *

from kivy.core.window import Window
from kivy.graphics.instructions import InstructionGroup
from kivy.graphics import Color, Ellipse, Rectangle, Line
from kivy.graphics import Translate

from level import *
from player import *
from music.midi_controller import MidiController
import globals
from const import *
from block import *

import pdb

Window.size = WINDOW_SIZE

class MainWidget(BaseWidget):
    def __init__(self):
        super(MainWidget, self).__init__()

        import sys
        try:
            start_spot = int(sys.argv[1])
        except:
            start_spot = 0
        if start_spot != 0:
            player_start_mult = 2
        else:
            player_start_mult = 1

        self.info = topleft_label()
        self.add_widget(self.info)

        background = Image(source='../../data/background/city2.png').texture
        self.background_color = BACKGROUND
        self.canvas.add(self.background_color)
        self.canvas.add(Rectangle(pos=(0, 0),
                        size=Window.size,
                        texture=background))

        self.background_texture = None

        self.audio = MidiController("music/grieg_mountain_king_with_levels.mid", self.level_on_update, start_spot=start_spot)
        self.level = Level(self.audio.platform_messages)
        self.player = Player((PLAYER_X+1024*start_spot, PLAYER_Y*player_start_mult), self.level)

        self.canvas.add(self.level)

        # create reverse translator to have player in the right place
        self.reverse_translator = Translate()
        self.canvas.add(self.reverse_translator)
        self.canvas.add(self.player)

        Window.bind(on_joy_axis=self.on_joy_axis)
        Window.bind(on_joy_button_down=self.on_joy_button_down)
        Window.bind(on_joy_hat=self.on_joy_hat)

        self.desaturate = False
        self.time = 0

        #Hack to setup correct initiall color
        self.set_color(0)

    def level_on_update(self, *args, **kwargs):
        return self.level.on_update(*args, **kwargs)

    def on_key_down(self, keycode, modifiers):

        if keycode[1] == 'up':
            self.up()

        if keycode[1] == 'down':
            self.down()

        color_idx = lookup(keycode[1], 'asd', (RED_IDX, GREEN_IDX, BLUE_IDX))
        if color_idx != None:
            self.set_color(color_idx)

        if keycode[1] == 'left':
            self.left()

        if keycode[1] == 'right':
            self.right()

        if keycode[1] == 'p':
            self.start()

        if keycode[1] == 'r':
            self.reset()

    def on_key_up(self, keycode):
        if self.level.alive:
            if keycode[1] == 'down':
                self.neutral()

    def on_joy_button_down(self, window, null, button):
        if button == START:
            self.start()

        if button == SELECT:
            self.reset()

        mapping = {0: GREEN_IDX, 1: RED_IDX, 2: BLUE_IDX}
        if button in mapping:
            color = mapping[button]
            self.set_color(color)

    def on_joy_axis(self, win, stickid, axisid, value):

        if axisid == X_AXIS and value < -JOYSTICK_THRESH:
            self.left()
        if axisid == X_AXIS and value > JOYSTICK_THRESH:
            self.right()
        if axisid == Y_AXIS and value < -JOYSTICK_THRESH:
            self.up()
        if axisid == Y_AXIS and value > JOYSTICK_THRESH:
            self.down()
        if axisid == Y_AXIS and -JOYSTICK_THRESH < value < JOYSTICK_THRESH:
            self.neutral()

    def on_joy_hat(self, window, null1, null2, coords):
        x, y = coords
        if y == 1:
            self.up()
        if y == -1:
            self.down()
        if coords == (0, 0):
            self.neutral()
        if x == 1:
            self.right()
        if x == -1:
            self.left()

    def up(self):
        if self.level.alive:
            self.player.jump()

    def down(self):
        if self.level.alive:
            self.player.duck()

    def neutral(self):
        if self.level.alive:
            self.player.un_duck()

    def left(self):
        if self.level.alive and self.level.direction == 1:
            self.level.set_next_barline()

            def complete_reverse(*args, **kwargs):
                self.level.reverse(*args, **kwargs)
                self.player.left()

            self.audio.reverse(complete_reverse)

    def right(self):
        if self.level.alive and self.level.direction == -1:
            self.level.set_previous_barline()

            def complete_reverse(*args, **kwargs):
                self.level.forward(*args, **kwargs)
                self.player.right()

            self.audio.reverse(complete_reverse)

    def start(self):
        if self.level.alive:
            self.level.start()
            self.audio.start()

    def reset(self):
        self.player.un_duck()
        y_pos, color_idx, y_vel, x_pos = self.level.reset()
        self.player.reset(y_pos, color_idx, y_vel)
        self.audio.reset(lost=False, tick=x_pos)

        self.desaturate = False
        self.set_color(color_idx)

    def set_color(self, color_idx):
        if self.level.alive:
            self.player.set_color(color_idx)

        if color_idx == 0:
            RED.rgb = COLOR_VALS[0]
            GREEN.rgb = DESAT_VALS[1]
            BLUE.rgb = DESAT_VALS[2]
            BACKGROUND.rgb = BACKGROUND_VALS[0]
        elif color_idx == 1:
            RED.rgb = DESAT_VALS[0]
            GREEN.rgb = COLOR_VALS[1]
            BACKGROUND.rgb = BACKGROUND_VALS[1]
        else:
            RED.rgb = DESAT_VALS[0]
            GREEN.rgb = DESAT_VALS[1]
            BLUE.rgb = COLOR_VALS[2]
            BACKGROUND.rgb = BACKGROUND_VALS[2]

    def lose(self):
        self.desaturate = True
        self.level.lose()
        self.audio.reset(lost=True)

    # checks for loss conditions and returns ground level if player is on ground, otherwise
    def check_block_collision_and_death(self):
        # did you fall off?
        if self.player.pos[1] < 0:
            self.lose()

        current_blocks = self.level.get_current_blocks()
        # fall if no blocks
        if not current_blocks:
            return 0

        for b in current_blocks:
            ground = b.on_ground(self.player)
            # check if on the ground and return ground level if so
            if ground:
                return ground
            # check if a collision has occured
            possible_loss = b.check_game_loss(self.player)
            if possible_loss:
                self.lose()

        return 0

    def on_update(self):
        prev_time = self.time
        new_time = self.audio.sched.get_tick() * (512. / 480.)
        dt = int((new_time * 1. - prev_time) / 16)
        self.time = prev_time + dt * 16

        self.info.text = 'fps:%d' % kivyClock.get_fps()
        ground = self.check_block_collision_and_death()
        self.player.on_update(dt, self.level.alive, ground)
        self.level.check_checkpoint(self.player.pos[1], self.player.color_idx, self.player.y_vel)
        self.audio.on_update()

        if self.desaturate:
            for color in CYCLE_COLORS:
                if color.s > .05:
                    color.s -= .05

        # update reverse translator
        self.reverse_translator.x = -self.level.translator.x

run(MainWidget)


'''
Functions that are used while a Generation is being Evaluated
'''
import os
import random
import multiprocessing
from rdkit import Chem
import numpy as np
from random import randrange
import discriminator as D
import evolution_functions as evo
from SAS_calculator.sascorer import calculateScore
manager = multiprocessing.Manager()
lock = multiprocessing.Lock()

#Added to deal with Similarity:                                         #!# ----------------------------------------
from rdkit.Chem import AllChem
from rdkit.Chem.rdMolDescriptors import GetUSRScore, GetUSRCAT

def calc_prop_USR(unseen_smile_ls, property_name, props_collect):
    '''Calculate Similarity for each molecule in unseen_smile_ls, and record results
       in locked dictionary props_collect
    '''

    #To provide reference molecule:
    reference_smile = 'C1(=NC(=NC2=C1N=C[N]2[H])N(C3=CC=C(C=C3)[S](=O)(=O)N([H])[H])[H])C4=CC(=CC=C4)C5=CC=CC=C5'
    ref_mol = Chem.MolFromSmiles(reference_smile)
    AllChem.EmbedMolecule(ref_mol, useRandomCoords = True, enforceChirality = False)
    ref_embed_usrcat = GetUSRCAT(ref_mol)

    for smile in unseen_smile_ls:
        mol, smi_canon, did_convert = evo.sanitize_smiles(smile) #esure valid smile
        if did_convert:
            try:
                mol_test = Chem.MolFromSmiles(smile)
                mol_test = Chem.AddHs(mol_test)
                AllChem.EmbedMolecule(mol_test, useRandomCoords = True, enforceChirality = False)
                mol_test = Chem.RemoveHs(mol_test)
                UsrcatMol = GetUSRCAT(mol_test)
            except ValueError:
                SimScore = 0
            else:
                SimScore = GetUSRScore(ref_embed_usrcat, UsrcatMol)
            props_collect[property_name][smile] = SimScore
        else:
            raise Exception('Invalid smile encountered while atempting to calculate Similarity') #!# ----------------

#Added to deal with Tanimoto
from rdkit import DataStructs

def calc_prop_Tanimoto(unseen_smile_ls, property_name, props_collect):                  #-------------------------
    '''Calculate Tanimoto Coeff. for each molecule in unseen_smile_ls, and record results
       in locked dictionary props_collect
    '''

    #To provide reference molecule:
    reference_smile = 'C1(=NC(=NC2=C1N=C[N]2[H])N(C3=CC=C(C=C3)[S](=O)(=O)N([H])[H])[H])C4=CC(=CC=C4)C5=CC=CC=C5'
    Tani_ref_mol = Chem.MolFromSmiles(reference_smile)
    Tani_ref_FP = Chem.RDKFingerprint(Tani_ref_mol)

    for smile in unseen_smile_ls:
        mol, smi_canon, did_convert = evo.sanitize_smiles(smile) #esure valid smile
        if did_convert:
            mol_Tani_test = Chem.MolFromSmiles(smile)
            Tani_mol_FP = Chem.RDKFingerprint(mol_Tani_test)
            TaniScore = DataStructs.FingerprintSimilarity(Tani_mol_FP, Tani_ref_FP)
            props_collect[property_name][smile] = TaniScore
        else:
            raise Exception('Invalid smile encountered while atempting to calculate Tanimoto') #----------------

def calc_prop_logP(unseen_smile_ls, property_name, props_collect):
    '''Calculate logP for each molecule in unseen_smile_ls, and record results
       in locked dictionary props_collect
    '''
    for smile in unseen_smile_ls:
        mol, smi_canon, did_convert = evo.sanitize_smiles(smile)
        if did_convert:                                          # ensure valid smile
            props_collect[property_name][smile] = evo.get_logP(mol) # Add calculation
        else:
            raise Exception('Invalid smile encountered while atempting to calculate logP')

def calc_prop_SAS(unseen_smile_ls, property_name, props_collect):
    '''Calculate synthetic accesibility score for each molecule in unseen_smile_ls,
       results are recorded in locked dictionary props_collect
    '''
    for smile in unseen_smile_ls:
        mol, smi_canon, did_convert = evo.sanitize_smiles(smile)
        if did_convert:                                         # ensure valid smile
            props_collect[property_name][smile] = calculateScore(mol)
        else:
            raise Exception('Invalid smile encountered while atempting to calculate SAS ', smile)

def calc_prop_RingP(unseen_smile_ls, property_name, props_collect):
    '''Calculate Ring penalty for each molecule in unseen_smile_ls,
       results are recorded in locked dictionary props_collect
    '''
    for smi in unseen_smile_ls:
        mol, smi_canon, did_convert = evo.sanitize_smiles(smi)
        if did_convert:
            cycle_list = mol.GetRingInfo().AtomRings()
            if len(cycle_list) == 0:
                cycle_length = 0
            else:
                cycle_length = max([ len(j) for j in cycle_list ])
            if cycle_length <= 6:
                cycle_length = 0
            else:
                cycle_length = cycle_length - 6
            props_collect[property_name][smi] = cycle_length
        else:
            raise Exception('Invalid smile encountered while atempting to calculate Ring penalty ', smi)

def calc_prop_SIMIL(starting_smile, unseen_smile_ls, property_name, props_collect):
    '''Calculate logP for each molecule in unseen_smile_ls, and record results
       in locked dictionary props_collect
    '''
    target, _, _ = evo.sanitize_smiles(starting_smile)

    for smile in unseen_smile_ls:
        mol, smi_canon, did_convert = evo.sanitize_smiles(smile)
        if did_convert:                                                                # ensure valid smile
            props_collect[property_name][smile] = evo.molecule_similarity(mol, target) # Add calculation
        else:
            raise Exception('Invalid smile encountered while atempting to calculate SIMILARITY: ', smile)

def create_parr_process(chunks, property_name, starting_smile):
    ''' Create parallel processes for calculation of properties
    '''
    # Assign data to each process
    process_collector    = []
    collect_dictionaries = []

    for item in chunks:
        props_collect  = manager.dict(lock=True)
        smiles_map_    = manager.dict(lock=True)
        props_collect[property_name] = smiles_map_
        collect_dictionaries.append(props_collect)

        if property_name == 'logP':
            process_collector.append(multiprocessing.Process(target=calc_prop_logP, args=(item, property_name, props_collect, )))

        if property_name == 'SAS':
           process_collector.append(multiprocessing.Process(target=calc_prop_SAS, args=(item, property_name, props_collect, )))

        if property_name == 'RingP':
            process_collector.append(multiprocessing.Process(target=calc_prop_RingP, args=(item, property_name, props_collect, )))

        if property_name == 'SIMILR':
            process_collector.append(multiprocessing.Process(target=calc_prop_SIMIL, args=(starting_smile, item, property_name, props_collect, )))

        if property_name == 'USRSim':               #!#
            process_collector.append(multiprocessing.Process(target=calc_prop_USR, args=(item, property_name, props_collect, )))

        if property_name == 'TaniSim':               #!#
            process_collector.append(multiprocessing.Process(target=calc_prop_Tanimoto, args=(item, property_name, props_collect, )))

    for item in process_collector:
        item.start()

    for item in process_collector: # wait for all parallel processes to finish
        item.join()

    combined_dict = {}             # collect results from multiple processess
    for i,item in enumerate(collect_dictionaries):
        combined_dict.update(item[property_name])

    return combined_dict

def fitness(molecules_here,    properties_calc_ls,
            discriminator,     disc_enc_type,   generation_index,
            max_molecules_len, device,          num_processors,    writer, beta, data_dir, starting_smile, desired_delta, save_curve):
    ''' Calculate fitness fo a generation in the GA

    All properties are standardized based on the mean & stddev of the zinc dataset

    Parameters:
    molecules_here    (list)         : List of a string of molecules
    properties_calc_ls               : List of propertoes to calculate
    discriminator     (torch.Model)  : Pytorch classifier
    disc_enc_type     (string)       : Indicated type of encoding shown to discriminator
    generation_index  (int)          : Which generation indicator
    max_molecules_len (int)          : Largest mol length
    device            (string)       : Device of discrimnator

    Returns:
    fitness                   (np.array) : A lin comb of properties and
                                           discriminator predictions
    discriminator_predictions (np.array) : The predictions made by the discrimantor

    '''
    if properties_calc_ls == None:
        raise Exception('Fail discrm trying to be invoked')
        fitness = discriminator_predictions

    else:

        molecules_here_unique = list(set(molecules_here))

        ratio            = len(molecules_here_unique) / num_processors
        chunks           = evo.get_chunks(molecules_here_unique, num_processors, ratio)
        chunks           = [item for item in chunks if len(item) >= 1]
        # Parallelize the calculation of logPs
        if 'logP' in properties_calc_ls:
            logP_results = create_parr_process(chunks, 'logP', starting_smile)

        # Parallelize the calculation of SAS
        if 'SAS' in properties_calc_ls:
            SAS_results = create_parr_process(chunks, 'SAS', starting_smile)

        # Parallize the calculation of Ring Penalty
        if 'RingP' in properties_calc_ls:
            ringP_results = create_parr_process(chunks, 'RingP', starting_smile)

        # Parallelize the calculation of SIMILR
        if 'SIMILR' in properties_calc_ls:
            similar_results = create_parr_process(chunks, 'SIMILR', starting_smile)

        # Parallize the calculation of USRCAT Sim                               #!#
        if 'USRSim' in properties_calc_ls:
            USRSim_results = create_parr_process(chunks, 'USRSim', starting_smile)

        # Parallize the calculation of Tanimoto                                 #!#
        if 'TaniSim' in properties_calc_ls:
            TaniSim_results = create_parr_process(chunks, 'TaniSim', starting_smile)

        logP_calculated, SAS_calculated, RingP_calculated, logP_norm, SAS_norm, RingP_norm, Similarity_calculated, USRSim_calculated, USRSim_norm, TaniSim_calculated, TaniSim_norm = obtained_standardized_properties(molecules_here, logP_results, SAS_results, ringP_results, similar_results, USRSim_results, TaniSim_results)

        # Add Objectives
        fitness = (USRSim_norm)

        # Similarity Based Fitness _________
        writer.add_scalar('Mean Similarty',        Similarity_calculated.mean(), generation_index) # Mean similarity
        writer.add_scalar('Max  Similarty',        max(Similarity_calculated),   generation_index) # Max similarity

        Similarity_calculated = np.array([0 if x > desired_delta else -10**6 for x in Similarity_calculated])
        Similarity_calculated = Similarity_calculated.reshape((fitness.shape[0], 1))

        fitness = fitness + Similarity_calculated

        # Plot fitness without discriminator
        writer.add_scalar('max fitness without discr',  max(fitness),     generation_index)
        save_curve.append(max(fitness))
        writer.add_scalar('avg fitness without discr',  fitness.mean(),   generation_index)

        # max fitness without discriminator
        f = open('{}/max_fitness_no_discr.txt'.format(data_dir), 'a+')
        f.write(str(max(fitness)[0]) + '\n')
        f.close()
        # avg fitness without discriminator
        f = open('{}/avg_fitness_no_discr.txt'.format(data_dir), 'a+')
        f.write(str(fitness.mean()) + '\n')
        f.close()

#        fitness = (beta * discriminator_predictions) + fitness

        # Plot fitness with discriminator
        writer.add_scalar('max fitness with discrm',  max(fitness),     generation_index)
        writer.add_scalar('avg fitness with discrm',  fitness.mean(),   generation_index)

        # max fitness with discriminator
        f = open('{}/max_fitness_discr.txt'.format(data_dir), 'a+')
        f.write(str(max(fitness)[0]) + '\n')
        f.close()
        # avg fitness with discriminator
        f = open('{}/avg_fitness_discr.txt'.format(data_dir), 'a+')
        f.write(str(fitness.mean()) + '\n')
        f.close()

        # Plot properties
        writer.add_scalar('non standr max logp',   max(logP_calculated),         generation_index) # logP plots
        writer.add_scalar('non standr mean logp',  logP_calculated.mean(),       generation_index)
        writer.add_scalar('non standr min sas',    min(SAS_calculated),          generation_index) # SAS plots
        writer.add_scalar('non standr mean sas',   SAS_calculated.mean(),        generation_index)
        writer.add_scalar('non standr min ringp',  min(RingP_calculated),        generation_index) # RingP plots
        writer.add_scalar('non standr mean ringp', RingP_calculated.mean(),      generation_index)
        writer.add_scalar('non standr max USRCAT Similarity', max(USRSim_calculated), generation_index) # USRCAT Similarity plots             #!#
        writer.add_scalar('non standr mean USRCAT Similarity', USRSim_calculated.mean(), generation_index)
        writer.add_scalar('non standr max Tanimoto Similarity', max(TaniSim_calculated), generation_index) # Tanimoto Similarity plots        #!#
        writer.add_scalar('non standr mean Tanimoto Similarity', TaniSim_calculated.mean(), generation_index)

        # max logP - non standardized
        f = open('{}/max_logp.txt'.format(data_dir), 'a+')
        f.write(str(max(logP_calculated)) + '\n')
        f.close()
        # mean logP - non standardized
        f = open('{}/avg_logp.txt'.format(data_dir), 'a+')
        f.write(str(logP_calculated.mean()) + '\n')
        f.close()
        # min SAS - non standardized
        f = open('{}/min_SAS.txt'.format(data_dir), 'a+')
        f.write(str(min(SAS_calculated)) + '\n')
        f.close()
        # mean SAS - non standardized
        f = open('{}/avg_SAS.txt'.format(data_dir), 'a+')
        f.write(str(SAS_calculated.mean()) + '\n')
        f.close()
        # min RingP - non standardized
        f = open('{}/min_RingP.txt'.format(data_dir), 'a+')
        f.write(str(min(RingP_calculated)) + '\n')
        f.close()
        # mean RingP - non standardized
        f = open('{}/avg_RingP.txt'.format(data_dir), 'a+')
        f.write(str(RingP_calculated.mean()) + '\n')
        f.close()
        # max USRCAT Similarity - non standardised                             #!#
        f = open('{}/max_Similarity.txt'.format(data_dir), 'a+')
        f.write(str(max(USRSim_calculated)) + '\n')
        f.close()
        # mean USRCAT Similarity - non standardized                            #!#
        f = open('{}/avg_Similarity.txt'.format(data_dir), 'a+')
        f.write(str(USRSim_calculated.mean()) + '\n')
        f.close()
        # max Tanimoto Similarity - non standardised                           #!#
        f = open('{}/max_Tanimoto.txt'.format(data_dir), 'a+')
        f.write(str(max(TaniSim_calculated)) + '\n')
        f.close()
        # mean Tanimoto Similarity - non standardized                          #!#
        f = open('{}/avg_Tanimoto.txt'.format(data_dir), 'a+')
        f.write(str(TaniSim_calculated.mean()) + '\n')
        f.close()

    return fitness, logP_calculated, SAS_calculated, RingP_calculated, USRSim_calculated, TaniSim_calculated

def obtained_standardized_properties(molecules_here,  logP_results, SAS_results, ringP_results, similar_results, USRSim_results, TaniSim_results):     #!#
    ''' Obtain calculated properties of molecules in molecules_here, and standardize
    values base on properties of the Zinc Data set.
    '''
    logP_calculated       = []
    SAS_calculated        = []
    RingP_calculated      = []
    Similarity_calculated = []
    USRSim_calculated     = [] #!#
    TaniSim_calculated    = [] #!#

    for smi in molecules_here:
        logP_calculated.append(logP_results[smi])
        SAS_calculated.append(SAS_results[smi])
        RingP_calculated.append(ringP_results[smi])
        Similarity_calculated.append(similar_results[smi])
        USRSim_calculated.append(USRSim_results[smi])               #!#
        TaniSim_calculated.append(TaniSim_results[smi])             #!#
    logP_calculated  = np.array(logP_calculated)
    SAS_calculated   = np.array(SAS_calculated)
    RingP_calculated = np.array(RingP_calculated)
    Similarity_calculated = np.array(Similarity_calculated)
    USRSim_calculated = np.array(USRSim_calculated)         #!#
    TaniSim_calculated = np.array(TaniSim_calculated)       #!#

    # Standardize logP based on zinc logP (mean: 2.4729421499641497 & std : 1.4157879815362406)
    logP_norm = (logP_calculated - 2.4729421499641497) / 1.4157879815362406
    logP_norm = logP_norm.reshape((logP_calculated.shape[0], 1))

    # Standardize SAS based on zinc SAS(mean: 3.0470797085649894    & std: 0.830643172314514)
    SAS_norm = (SAS_calculated - 3.0470797085649894) / 0.830643172314514
    SAS_norm = SAS_norm.reshape((SAS_calculated.shape[0], 1))

    # Standardiize RingP based on zinc RingP(mean: 0.038131530820234766 & std: 0.2240274735210179)
    RingP_norm = (RingP_calculated - 0.038131530820234766) / 0.2240274735210179
    RingP_norm = RingP_norm.reshape((RingP_calculated.shape[0], 1))

    # Standardize USRCAT Similarity based on zinc ChemBL Similarity(mean: 3.053230897406870 & std: 0.834794987448313)                #!#
    USRSim_norm = (USRSim_calculated - 0.186428542) / 0.035664915
    USRSim_norm = USRSim_norm.reshape((USRSim_calculated.shape[0], 1))                                                               #!#

    # Standardize Tanimoto Similarity based on ChemBL Tanimoto Similarity(mean: 0.350252265668509 & std: 0.0681108949632873)         #!#
    TaniSim_norm = (TaniSim_calculated - 0.350252265668509) / 0.0681108949632873
    TaniSim_norm = TaniSim_norm.reshape((TaniSim_calculated.shape[0], 1))                                                            #!#

    return logP_calculated, SAS_calculated, RingP_calculated, logP_norm, SAS_norm, RingP_norm, Similarity_calculated, USRSim_calculated, USRSim_norm, TaniSim_calculated, TaniSim_norm

def obtain_fitness(disc_enc_type, smiles_here, selfies_here, properties_calc_ls,
                   discriminator, generation_index, max_molecules_len, device, generation_size, num_processors, writer, beta, image_dir, data_dir, starting_smile, desired_delta, save_curve):
    ''' Obtain fitness of generation based on choices of disc_enc_type.
        Essentially just calls 'fitness'
    '''
    # ANALYSE THE GENERATION                        #!#
    if disc_enc_type == 'smiles' or disc_enc_type == 'properties_rdkit':
        fitness_here, logP_calculated, SAS_calculated, RingP_calculated, USRSim_calculated, TaniSim_calculated = fitness(smiles_here,   properties_calc_ls ,   discriminator,
                                                                           disc_enc_type, generation_index,   max_molecules_len, device, num_processors, writer, beta, data_dir, starting_smile, desired_delta, save_curve)
    elif disc_enc_type == 'selfies':
        fitness_here, logP_calculated, SAS_calculated, RingP_calculated, USRSim_calculated, TaniSim_calculated = fitness(selfies_here,  properties_calc_ls ,   discriminator,
                                                                           disc_enc_type, generation_index,   max_molecules_len, device, num_processors, writer, beta, data_dir, starting_smile, desired_delta, save_curve)

    fitness_here = fitness_here.reshape((generation_size, ))
    order, fitness_ordered, smiles_ordered, selfies_ordered = order_based_on_fitness(fitness_here, smiles_here, selfies_here)

    # Order molecules based on ordering of 'smiles_ordered'
    logP_calculated  = [logP_calculated[idx] for idx in order]
    SAS_calculated   = [SAS_calculated[idx] for idx in order]
    RingP_calculated = [RingP_calculated[idx] for idx in order]
    USRSim_calculated = [USRSim_calculated[idx] for idx in order]                       #!#
    TaniSim_calculated = [TaniSim_calculated[idx] for idx in order]                     #!#

    os.makedirs('{}/{}'.format(data_dir, generation_index))
    #  Write ordered smiles in a text file
    f = open('{}/{}/smiles_ordered.txt'.format(data_dir, generation_index), 'a+')
    f.writelines(["%s\n" % item  for item in smiles_ordered])
    f.close()
    #  Write logP of ordered smiles in a text file
    f = open('{}/{}/logP_ordered.txt'.format(data_dir, generation_index), 'a+')
    f.writelines(["%s\n" % item  for item in logP_calculated])
    f.close()
    #  Write sas of ordered smiles in a text file
    f = open('{}/{}/sas_ordered.txt'.format(data_dir, generation_index), 'a+')
    f.writelines(["%s\n" % item  for item in SAS_calculated])
    f.close()
    #  Write ringP of ordered smiles in a text file
    f = open('{}/{}/ringP_ordered.txt'.format(data_dir, generation_index), 'a+')
    f.writelines(["%s\n" % item  for item in RingP_calculated])
    f.close()
    #  Write USRCAT Similarity of ordered smiles in a text file                                        #!#
    f = open('{}/{}/USRCATSimilarity_ordered.txt'.format(data_dir, generation_index), 'a+')
    f.writelines(["%s\n" % item  for item in USRSim_calculated])
    f.close()
    #  Write Tanimoto Similarity of ordered smiles in a text file                                      #!#
    f = open('{}/{}/Tanimoto_ordered.txt'.format(data_dir, generation_index), 'a+')
    f.writelines(["%s\n" % item  for item in TaniSim_calculated])
    f.close()

    #print statement for the best molecule in the generation
    print('Best best molecule in generation ', generation_index)
    print('    smile  : ', smiles_ordered[0])
    print('    fitness: ', fitness_ordered[0])
    print('    logP   : ', logP_calculated[0])
    print('    sas    : ', SAS_calculated[0])
    print('    ringP  : ', RingP_calculated[0])
    print('    USRCAT : ', USRSim_calculated[0])                               #!#
    print('    Tanimoto : ', TaniSim_calculated[0])                            #!#                                                                                #!#  --> Another Similarity change to the right and two lines down |
                                                                                #!#                                                                V

    f = open('{}/best_in_generations.txt'.format(data_dir), 'a+')
    best_gen_str = 'index: {},  smile: {}, fitness: {}, logP: {}, sas: {}, ringP: {}, USRCAT: {}, Tanimoto: {}'.format(generation_index, smiles_ordered[0], fitness_ordered[0], logP_calculated[0], SAS_calculated[0], RingP_calculated[0], USRSim_calculated[0], TaniSim_calculated[0])
    f.write(best_gen_str + '\n')
    f.close()
                                                                                #!# -->
    show_generation_image(generation_index, image_dir, smiles_ordered, fitness_ordered, logP_calculated, SAS_calculated, RingP_calculated, USRSim_calculated, TaniSim_calculated)

    return fitness_here, order, fitness_ordered, smiles_ordered, selfies_ordered

def show_generation_image(generation_index, image_dir, smiles_ordered, fitness, logP, SAS, RingCount, USRSim, TaniSim):  #!#
    ''' Plot 100 molecules with the best fitness in in a generation
        Called after at the end of each generation. Image in each generation
        is stored with name 'generation_index.png'

    Images are stored in diretory './images'
    '''
    if generation_index > 1:
        A = list(smiles_ordered)
        A = A[:100]
        if len(A) < 100 : return #raise Exception('Not enough molecules provided for plotting ', len(A))
        A = [Chem.MolFromSmiles(x) for x in A]

        evo.create_100_mol_image(A, "./{}/{}_ga.png".format(image_dir, generation_index), fitness, logP, SAS, RingCount, USRSim, TaniSim)    #!#

def obtain_previous_gen_mol(starting_smiles,  starting_selfies, generation_size,
                            generation_index, selfies_all,      smiles_all):
    '''Obtain molecules from one generation prior.
       If generation_index is 1, only the the starting molecules are returned

     Parameters:

     Returns:

    '''
    # Obtain molecules from the previous generation

    if generation_index == 1:

        randomized_smiles  = []
        randomized_selfies = []
        for i in range(generation_size): # nothing to obtain from previous gen
                                         # So, choose random moleclues from the starting list
            index = randrange(len(starting_smiles))
            randomized_smiles.append(starting_smiles[index])
            randomized_selfies.append(starting_selfies[index])

        return randomized_smiles, randomized_selfies
    else:
        return smiles_all[generation_index-2], selfies_all[generation_index-2]

def order_based_on_fitness(fitness_here, smiles_here, selfies_here):
    '''Order elements of a lists (args) based om Decreasing fitness
    '''
    order = np.argsort(fitness_here)[::-1] # Decreasing order of indices, based on fitness
    fitness_ordered = [fitness_here[idx] for idx in order]
    smiles_ordered = [smiles_here[idx] for idx in order]
    selfies_ordered = [selfies_here[idx] for idx in order]

    return order, fitness_ordered, smiles_ordered, selfies_ordered

def apply_generation_cutoff(order, generation_size):
    ''' Return of a list of indices of molecules that are kept (high fitness)
        and a list of indices of molecules that are replaced   (low fitness)

    The cut-off is imposed using a Fermi-Function

    Parameters:
    order (list)          : list of molecule indices arranged in Decreasing order of fitness
    generation_size (int) : number of molecules in a generation

    Returns:
    to_replace (list): indices of molecules that will be replaced by random mutations of
                       molecules in list 'to_keep'
    to_keep    (list): indices of molecules that will be kept for the following generations
    '''
    # Get the probabilities that a molecule with a given fitness will be replaced
    # a fermi function is used to smoothen the transition
    positions     = np.array(range(0, len(order))) - 0.2*float(len(order))
    probabilities = 1.0 / (1.0 + np.exp(-0.02 * generation_size * positions / float(len(order))))

    to_replace = [] # all molecules that are replaced
    to_keep    = [] # all molecules that are kept
    for idx in range(0,len(order)):
        if np.random.rand(1) < probabilities[idx]:
            to_replace.append(idx)
        else:
            to_keep.append(idx)

    return to_replace, to_keep

def obtain_next_gen_molecules(order,           to_replace,     to_keep,
                              selfies_ordered, smiles_ordered, max_molecules_len):
    ''' Obtain the next generation of molecules. Bad molecules are replaced by
    mutations of good molecules

    Parameters:
    order (list)            : list of molecule indices arranged in Decreasing order of fitness
    to_replace (list)       : list of indices of molecules to be replaced by random mutations of better molecules
    to_keep (list)          : list of indices of molecules to be kept in following generation
    selfies_ordered (list)  : list of SELFIE molecules, ordered by fitness
    smiles_ordered (list)   : list of SMILE molecules, ordered by fitness
    max_molecules_len (int) : length of largest molecule

    Returns:
    smiles_mutated (list): next generation of mutated molecules as SMILES
    selfies_mutated(list): next generation of mutated molecules as SELFIES
    '''
    smiles_mutated = []
    selfies_mutated = []
    for idx in range(0,len(order)):
        if idx in to_replace: # smiles to replace (by better molecules)
            random_index=np.random.choice(to_keep, size=1, replace=True, p=None)[0]                             # select a random molecule that survived
            grin_new, smiles_new = evo.mutations_random_grin(selfies_ordered[random_index], max_molecules_len)  # do the mutation

            # add mutated molecule to the population
            smiles_mutated.append(smiles_new)
            selfies_mutated.append(grin_new)
        else: # smiles to keep
            smiles_mutated.append(smiles_ordered[idx])
            selfies_mutated.append(selfies_ordered[idx])

    return smiles_mutated, selfies_mutated

def update_gen_res(smiles_all, smiles_mutated, selfies_all, selfies_mutated, smiles_all_counter):
    '''Collect results that will be shared with global variables outside generations
    '''
    smiles_all.append(smiles_mutated)
    selfies_all.append(selfies_mutated)

    for smi in smiles_mutated:
        if smi in smiles_all_counter:
            smiles_all_counter[smi] += 1
        else:
            smiles_all_counter[smi] = 1

    return smiles_all, selfies_all, smiles_all_counter

Exp 4 4 with USRCAT and Tanimoto/generation_props.py


'''
    Stats tuplet with (teamNo, intake, score, defend, descore, rely, rock, mini, s, p, m)
'''

teams = {}

def register_teams(dict):
    for team in teams:
        workingTeam = Team()

        team.setup(team[0], team[1], team[2], team[3], team[4], team[5], team[6],\
        team[7], team[8], team[9], team[10])

        dict[team[0]] = workingTeam


from .models import Post, Category, Tag, SocialFeed, PostSlider

def show_op_context(request):

    sb_recent_posts = Post.objects.all().order_by('-publish')[:3]
    sb_social_embed = SocialFeed.objects.all()[:3]

    home_slider = PostSlider.objects.all()

    return {
        'sb_recent_posts': sb_recent_posts,
        'sb_social_embed': sb_social_embed,
        'home_slider': home_slider,

    }


# -*- coding: utf-8 -*-

import glob

import tensorflow as tf
import tensorflow.keras as keras

from tasks.tianchi_news_rec.constant import *
from zimu_ml_sys.core.feature_columns import SparseFeat, VarLenSparseFeat
from zimu_ml_sys.core.ctr_models.din import DIN
from zimu_ml_sys.utils.data_util import read_encoding_mapping_data
from zimu_ml_sys.utils.tf_record_util import get_tf_record_beans, tf_record_to_dataset

gpus = tf.config.experimental.list_physical_devices(device_type='GPU')
for gpu in gpus:
    tf.config.experimental.set_memory_growth(gpu, True)

class DinTask(object):
    embedding_size = 16

    batch_size = 512

    epochs = 10

    label_name = CTR_FIELDS.pop(-1)

    def get_feature_columns(self):

        sparse_features = [field for field in CTR_FIELDS if not field.startswith('history')]

        encoding_fields_mapping = read_encoding_mapping_data(TASK_NAME)

        dnn_feature_colums = [SparseFeat(name=field,
                                         vocabulary_size=len(encoding_fields_mapping[field][0]) + 1,
                                         embedding_dim=self.embedding_size) for field in sparse_features]

        dnn_feature_colums += [VarLenSparseFeat(SparseFeat(name='history_click_articles',
                                                           vocabulary_size=len(
                                                               encoding_fields_mapping['article_id'][0]) + 1,
                                                           embedding_dim=self.embedding_size,
                                                           embedding_name='article_id'),
                                                maxlen=20, is_hist_mean_pooling=False),
                               VarLenSparseFeat(SparseFeat(name='history_click_categories',
                                                           vocabulary_size=len(
                                                               encoding_fields_mapping['category_id'][0]) + 1,
                                                           embedding_dim=self.embedding_size,
                                                           embedding_name='category_id'),
                                                maxlen=20, is_hist_mean_pooling=False)
                               ]

        return dnn_feature_colums

    def get_dataset(self, feature_columns, is_train=True):
        """
        获取dataset 数据集
        :param feature_columns:
        :param is_train:
        :return:
        """

        tf_record_beans = get_tf_record_beans(feature_columns, label_name=self.label_name)

        if is_train:
            path = CTR_TF_RECORD_DATA_PATH + 'train_data/' + "part*"
        else:
            path = CTR_TF_RECORD_DATA_PATH + 'test_data/' + "part*"

        files_list = glob.glob(path)

        dataset = tf_record_to_dataset(files_list,
                                       tf_record_beans=tf_record_beans,
                                       is_train=is_train,
                                       batch_size=self.batch_size,
                                       is_shuffle=True,
                                       epochs=self.epochs)

        return dataset

    def train(self):
        feature_columns = self.get_feature_columns()

        train_dataset = self.get_dataset(feature_columns, is_train=True)
        test_dataset = self.get_dataset(feature_columns, is_train=False)

        model = DIN(feature_columns)

        model.summary()

        model.compile(optimizer='adam', loss=keras.losses.binary_crossentropy, metrics=['accuracy'])

        model.fit(train_dataset, epochs=self.epochs, steps_per_epoch=13358,
                  validation_data=test_dataset,
                  validation_steps=489)

if __name__ == '__main__':
    DinTask().train()


import pytest

from pyppl import Proc, Diot, ProcSet
from pyppl.proctree import ProcNode, ProcTree
from pyppl.exception import ProcTreeProcExists, ProcTreeParseError#, ProcHideError

@pytest.fixture(autouse = True)
def resetNodes():
	ProcTree.NODES = {}
	yield

@pytest.fixture
def set1():
	p15 = Proc()
	p16 = Proc()
	p17 = Proc()
	p18 = Proc()
	p19 = Proc()
	p20 = Proc()
	#        hide
	# p15 -> p16  ->  p17 -> 19
	#         \_ p18_/  \_ p20
	#p16.hide = True
	p20.depends = p17
	p19.depends = p17
	p17.depends = p16, p18
	p18.depends = p16
	p16.depends = p15
	ProcTree.register(p15)
	return Diot(p15 = p15, p16 = p16, p17 = p17, p18 = p18, p19 = p19, p20 = p20)

@pytest.fixture(scope = 'function')
def set2():
	p14 = Proc()
	p15 = Proc()
	p16 = Proc()
	p17 = Proc()
	p18 = Proc()
	p19 = Proc()
	p20 = Proc()
	# p15 -> p16  ->  p17 -> 19
	# p14 _/  \_ p18_/  \_ p20
	#           hide
	#p18.hide = True
	p20.depends = p17
	p19.depends = p17
	p17.depends = p16, p18
	p18.depends = p16
	p16.depends = p14, p15
	ProcTree.register(p14, p15)
	return Diot(p15 = p15, p16 = p16, p17 = p17, p18 = p18, p19 = p19, p20 = p20, p14 = p14)

def test_procnode():
	# init
	p1 = Proc()
	pn = ProcNode(p1)
	assert pn.proc is p1
	assert pn.prev == []
	assert pn.next == []
	assert pn.ran == False
	assert pn.start == False

	p2 = Proc(id = 'p1')
	assert pn.sameIdTag(p2)

	assert repr(pn).startswith('<ProcNode(<Proc(id=p1,tag=notag) @')

def test_proctree_init():
	p21 = Proc()
	p22 = Proc()
	p23 = Proc()
	p24 = Proc()
	p25 = Proc()
	p23.depends = p21, p22
	#p23.hide    = True
	p24.depends = p23
	p25.depends = p23
	ProcTree.register(p21)
	ProcTree.register(p22)

	pt = ProcTree()
	pt.init()
	# with pytest.raises(ProcHideError):
	# 	pt.init()
	assert p21 in ProcTree.NODES
	assert p22 in ProcTree.NODES
	assert p23 in ProcTree.NODES
	assert p24 in ProcTree.NODES
	assert p25 in ProcTree.NODES
	assert ProcTree.NODES[p21].prev == []
	assert ProcTree.NODES[p22].prev == []
	assert ProcTree.NODES[p23].prev == [ProcTree.NODES[p21], ProcTree.NODES[p22]]
	assert ProcTree.NODES[p24].prev == [ProcTree.NODES[p23]]
	assert ProcTree.NODES[p25].prev == [ProcTree.NODES[p23]]
	assert ProcTree.NODES[p21].next == [ProcTree.NODES[p23]]
	assert ProcTree.NODES[p22].next == [ProcTree.NODES[p23]]
	assert ProcTree.NODES[p23].next == [ProcTree.NODES[p24], ProcTree.NODES[p25]]
	assert ProcTree.NODES[p24].next == []
	assert ProcTree.NODES[p25].next == []

def test_proctree_register():
	p3 = Proc()
	ProcTree.register(p3)
	del ProcTree.NODES[p3]
	ProcTree.register(p3)
	assert ProcTree.NODES[p3].proc is p3

def test_proctree_check():
	p4 = Proc()
	p5 = Proc(id = 'p4')
	ProcTree.register(p4)
	ProcTree.register(p5)
	with pytest.raises(ProcTreeProcExists):
		ProcTree.check(p4)

def test_proctree_getprevstr():
	p6 = Proc()
	p7 = Proc()
	p7.depends = p6
	ProcTree.register(p6)
	ProcTree().init()
	assert ProcTree.getPrevStr(p6) == 'START'
	assert ProcTree.getPrevStr(p7) == '[p6]'

def test_proctree_getnextstr():
	p8 = Proc()
	p9 = Proc()
	p9.depends = p8
	ProcTree.register(p8)
	ProcTree().init()
	assert ProcTree.getNextStr(p8) == '[p9]'
	assert ProcTree.getNextStr(p9) == 'END'

def test_proctree_getnext_and_reset():
	p10 = Proc()
	p11 = Proc()
	p11.depends = p10
	ProcTree.register(p10)
	ProcTree().init()
	assert ProcTree.getNext(p10) == [p11]
	assert ProcTree.getNext(p11) == []

	ProcTree.reset()
	ProcTree.NODES[p10].prev == []
	ProcTree.NODES[p10].next == []
	ProcTree.NODES[p11].prev == []
	ProcTree.NODES[p11].next == []
	ProcTree.NODES[p10].ran == False
	ProcTree.NODES[p10].start == False
	ProcTree.NODES[p11].ran == False
	ProcTree.NODES[p11].start == False

def test_proctree_setgetstarts():
	p12 = Proc()
	p13 = Proc()
	p14 = Proc()
	#p14.hide = True
	p12.depends = p13
	ProcTree.register(p13)
	ProcTree.register(p14)
	pt = ProcTree()
	pt.init()
	# with pytest.raises(ProcHideError):
	# 	pt.setStarts([p14])
	pt.ends = [1,2,3]
	pt.setStarts([p13])
	assert ProcTree.NODES[p12].start == False
	assert ProcTree.NODES[p13].start == True
	assert ProcTree.NODES[p14].start == False
	assert pt.starts == [p13]
	assert pt.ends == []

	pt.starts = []
	assert pt.getStarts() == [p13]
	assert pt.starts == [p13]
	assert pt.getStarts() == [p13]

def test_proctree_getpaths(set1):
	#        hide
	# p15 -> p16  ->  p17 -> 19
	#         \_ p18_/  \_ p20
	pt = ProcTree()
	pt.init()
	assert pt.getPaths(set1.p15) == []
	assert pt.getPaths(set1.p19) == [
		[set1.p17, set1.p16, set1.p15], [set1.p17, set1.p18, set1.p16, set1.p15]]
	assert pt.getPaths(set1.p20) == [
		[set1.p17, set1.p16, set1.p15], [set1.p17, set1.p18, set1.p16, set1.p15]]
	# assert pt.getPaths(set1.p19, check_hide = True)  == [
	# 	[set1.p17, set1.p15], [set1.p17, set1.p18, set1.p15]]
	# assert pt.getPaths(set1.p20, check_hide = True)  == [
	# 	[set1.p17, set1.p15], [set1.p17, set1.p18, set1.p15]]

	# circulic dependence
	p21 = Proc()
	p22 = Proc()
	p23 = Proc()
	# p21 -> p22 -> p23 -> p21
	p21.depends = p23
	p23.depends = p22
	p22.depends = p21
	pt = ProcTree()
	pt.init()
	with pytest.raises(ProcTreeParseError):
		pt.getPaths(p23)

def test_proctree_getpathstostarts(set2):
	# p15 -> p16  ->  p17 -> 19
	# p14 _/  \_ p18_/  \_ p20
	#           hide
	pt = ProcTree()
	pt.init()
	pt.setStarts([set2.p15])
	assert pt.getPathsToStarts(set2.p15) == []
	assert pt.getPathsToStarts(set2.p19) == [[set2.p17, set2.p16, set2.p15], [set2.p17, set2.p18, set2.p16, set2.p15]]
	assert pt.getPathsToStarts(set2.p20) == [[set2.p17, set2.p16, set2.p15], [set2.p17, set2.p18, set2.p16, set2.p15]]
	# assert pt.getPathsToStarts(set2.p19, check_hide = True) == [[set2.p17, set2.p16, set2.p15]]
	# assert pt.getPathsToStarts(set2.p20, check_hide = True) == [[set2.p17, set2.p16, set2.p15]]

	pt.setStarts([set2.p14, set2.p15])
	assert pt.getPathsToStarts(set2.p14) == []
	assert pt.getPathsToStarts(set2.p15) == []
	assert pt.getPathsToStarts(set2.p16) == [[set2.p14], [set2.p15]]
	assert pt.getPathsToStarts(set2.p18) == [[set2.p16, set2.p14], [set2.p16, set2.p15]]
	assert pt.getPathsToStarts(set2.p17) == [
		[set2.p16, set2.p14],
		[set2.p16, set2.p15],
		[set2.p18, set2.p16, set2.p14],
		[set2.p18, set2.p16, set2.p15],
	]
	assert pt.getPathsToStarts(set2.p19) == [
		[set2.p17, set2.p16, set2.p14],
		[set2.p17, set2.p16, set2.p15],
		[set2.p17, set2.p18, set2.p16, set2.p14],
		[set2.p17, set2.p18, set2.p16, set2.p15],
	]
	assert pt.getPathsToStarts(set2.p20) == [
		[set2.p17, set2.p16, set2.p14],
		[set2.p17, set2.p16, set2.p15],
		[set2.p17, set2.p18, set2.p16, set2.p14],
		[set2.p17, set2.p18, set2.p16, set2.p15],
	]
	# assert pt.getPathsToStarts(set2.p17, check_hide = True) == [[set2.p16, set2.p14], [set2.p16, set2.p15]]
	# assert pt.getPathsToStarts(set2.p19, check_hide = True) == [[set2.p17, set2.p16, set2.p14], [set2.p17, set2.p16, set2.p15]]
	# assert pt.getPathsToStarts(set2.p20, check_hide = True) == [[set2.p17, set2.p16, set2.p14], [set2.p17, set2.p16, set2.p15]]

def test_proctree_chechpath(set2):
	# p15 -> p16  ->  p17 -> 19
	# p14 _/  \_ p18_/  \_ p20
	#           hide
	pt = ProcTree()
	pt.init()
	assert pt.checkPath(set2.p16) == [set2.p14]
	pt.setStarts([set2.p14])
	assert pt.checkPath(set2.p16) == [set2.p15]
	pt.setStarts([set2.p14, set2.p15])
	assert pt.checkPath(set2.p16) is True

def test_proctree_getends(set2):

	# p15 -> p16  ->  p17 -> 19
	# p14 _/  \_ p18_/  \_ p20
	#           hide
	pt = ProcTree()
	pt.init()
	pt.setStarts([set2.p14, set2.p15])
	assert set(pt.getEnds()) == {set2.p19, set2.p20}
	assert set(pt.ends) == {set2.p19, set2.p20}
	assert set(pt.getEnds()) == {set2.p19, set2.p20}

	#set2.p19.hide = True
	pt.ends = []
	# with pytest.raises(ProcHideError):
	# 	pt.getEnds()

def test_proctree_getends_failed():
	p1 = Proc()
	p2 = Proc()
	p2.depends = p1
	ProcTree.register(p1)
	pt = ProcTree()
	pt.init()
	with pytest.raises(ProcTreeParseError):
		#Failed to determine end processes by start processes
		pt.getEnds()

	p3 = Proc()
	p3.depends = p2
	pt = ProcTree()
	pt.init()
	pt.setStarts([p3])
	with pytest.raises(ProcTreeParseError):
		# Failed to determine end processes, one of the paths cannot go through: 'p3 <- p2 <- p1'
		pt.getEnds()

def test_proctree_getallpaths(set2):
	# p15 -> p16  ->  p17 -> 19
	# p14 _/  \_ p18_/  \_ p20
	#         ##hide moved to plugin
	pt = ProcTree()
	pt.init()
	pt.setStarts([set2.p14, set2.p15])
	allpath = list(pt.getAllPaths())
	assert len(allpath) == 8
	assert [set2.p19, set2.p17, set2.p16, set2.p14] in allpath
	assert [set2.p19, set2.p17, set2.p16, set2.p15] in allpath
	assert [set2.p20, set2.p17, set2.p16, set2.p14] in allpath
	assert [set2.p20, set2.p17, set2.p16, set2.p15] in allpath
	assert [set2.p19, set2.p17, set2.p18, set2.p16, set2.p14] in allpath
	assert [set2.p19, set2.p17, set2.p18, set2.p16, set2.p15] in allpath
	assert [set2.p20, set2.p17, set2.p18, set2.p16, set2.p14] in allpath
	assert [set2.p20, set2.p17, set2.p18, set2.p16, set2.p15] in allpath

def test_proctree_getallpaths_single():
	p1 = Proc()
	ProcTree.register(p1)
	pt = ProcTree()

	pt.setStarts([p1])
	assert list(pt.getAllPaths()) == [[p1]]

def test_proctree_getnexttorun(set2):
	# p15 -> p16  ->  p17 -> 19
	# p14 _/  \_ p18_/  \_ p20
	#           hide
	p1 = Proc()
	#ProcTree.register(p1)
	p2 = Proc()
	p2.depends = p1
	pt = ProcTree()
	pt.init()
	pt.setStarts([set2.p14, set2.p15])
	pt.NODES[set2.p14].ran = True
	assert pt.getNextToRun() is set2.p15
	pt.NODES[set2.p15].ran = True
	assert pt.getNextToRun() is set2.p16
	pt.NODES[set2.p16].ran = True
	assert pt.getNextToRun() is set2.p18
	pt.NODES[set2.p18].ran = True
	assert pt.getNextToRun() is set2.p17
	pt.NODES[set2.p17].ran = True
	assert pt.getNextToRun() is set2.p20
	pt.NODES[set2.p20].ran = True
	assert pt.getNextToRun() is set2.p19
	pt.NODES[set2.p19].ran = True
	assert pt.getNextToRun() is None


from typing import Dict

from aiohttp import web
from aiohttp_swagger import swagger_path
from webargs.aiohttpparser import use_args

from app.config import settings
from app.models.exchange_rate import ExchangeRateModel
from app.schemas.exchange_rate_schema import ExchangeRateRequestSchema
from app.services.mongo import ExchangeRateService

@swagger_path(f"{settings.SWAGGER_PATH}/exchange_rate.yaml")
class ExchangeRate(web.View):
    async def get(self) -> web.Response:
        service: ExchangeRateService = self.request.app["exchange_rate_service"]
        exchange_rates = await service.find()

        return web.json_response(text=exchange_rates.json())

    @use_args(ExchangeRateRequestSchema, location='json')
    async def post(self, args: Dict) -> web.Response:
        service: ExchangeRateService = self.request.app["exchange_rate_service"]
        exchange_rate = await service.find_one(args['exchange_from'], args['exchange_to'])

        if exchange_rate:
            raise web.HTTPConflict(
                reason=f"Already exists a exchange rate with current data {args['exchange_from']} > {args['exchange_to']}"
            )

        exchange_rate = ExchangeRateModel(**args)
        await service.insert(exchange_rate)
        response_data = {
            'message': "Exchange rate created",
        }

        return web.json_response(response_data, status=web.HTTPCreated.status_code)

    @use_args(ExchangeRateRequestSchema, location='json')
    async def put(self, args: Dict) -> web.Response:
        service: ExchangeRateService = self.request.app["exchange_rate_service"]
        exchange_rate = await service.find_one(args['exchange_from'], args['exchange_to'])

        if not exchange_rate:
            raise web.HTTPNotFound(
                reason="Could not find the requested exchange rate to update"
            )

        exchange_rate = ExchangeRateModel(**args)
        await service.update(exchange_rate)
        response_data = {
            'message': "Exchange rate updated",
        }

        return web.json_response(response_data, status=web.HTTPAccepted.status_code)


import pandas as pd
import numpy as np
import math
import matplotlib.pyplot as plt
from statistics import mean

class Kmeans:
    def __init__(self):
        self.k = 0
        self.centroids = []
        self.centroids_groups = []
        self.flag = True
        self.data = []
        self.data_1=[]

    def fit(self, dataframe, k):
        total_records = len(dataframe.index)
        self.k = k
        for i in range(k):
            self.centroids_groups.append([])

        for i in range(k):
            random_point = int(np.random.randint(low=0, high=total_records, size=1))
            self.centroids.append(random_point)

        for row_index in range(0, total_records):
            some_data_point = dataframe.iloc[row_index, :].tolist()

            for centroid_index in self.centroids:
                centroid = df.iloc[centroid_index, :].tolist()
                distance_1 = self.find_distance(centroid, some_data_point)
                centroid = df.iloc[self.centroids[1], :].tolist()
                distance_2 = self.find_distance(centroid, some_data_point)

                if distance_1 < distance_2:
                    self.centroids_groups[0].append(row_index)
                else:
                    self.centroids_groups[1].append(row_index)
                break
        x = []
        y = []
        for index in self.centroids_groups[0]:
            centroid = df.iloc[index, :].tolist()
            x.append(centroid[0])
            y.append(centroid[1])
        plt.scatter(x, y, c='red')
        X = []
        Y = []
        for index in self.centroids_groups[1]:
            centroid = df.iloc[index, :].tolist()
            X.append(centroid[0])
            Y.append(centroid[1])
        plt.scatter(X, Y, c='green')

        plt.show()
        for x_,y_ in zip(x,y):
            self.data.append([x_, y_])
        for x_,y_ in zip(X,Y):
            self.data.append(x_, y_)

    def find_mean(self, d_list):
        return mean(d_list)

        print(self.centroids_groups[0])
        print('************************')
        print(self.centroids_groups[1])

    def find_distance(self, centroid, point):
        try:
            distance = math.sqrt(math.pow(centroid[0] - point[0], 2) + math.pow(centroid[1] - point[1], 2))
            return distance
        except:
            print("Some Error")

df = pd.read_csv('train.csv')
obj = Kmeans()
obj.fit(df, 2)


import matplotlib.pyplot as plt
import numpy as np # != p
from parameters import COLORS
import parameters

# Pie chart, where the slices will be ordered and plotted counter-clockwise:
labels = 'Installation cost', 'Service cost', 'Parts cost'
sizes = [30000, 1590, 3216]

def plot_elevators(elevators, global_downtime):
    '''
    global_downtime: [1,1,1,0,0,0,1,1]
    '''

    global_downtime_journal = []
    _start = 0
    _end = 0

    for i in range(len(global_downtime)-1):
        if global_downtime[i] != global_downtime[i+1]:
            if global_downtime[i+1] == 1:
                _end = i+2
                global_downtime_journal.append([_start, _end])
            else:
                _start = i+2

    if _start > _end:
        global_downtime_journal.append([_start, parameters.LIFETIME_OF_ELEVATOR])

    y = 0
    plt.figure(num=None, figsize=[15, 5])

    for e in elevators:
        y+=1

        plt.text(-70, y, f'Elevator {y}', fontsize=15)

        for data in e.journal:
            plt.fill_between(data, y1=y-0.2, y2=y+0.2, color=parameters.COLORS[y])

    y+=1

    plt.fill_between([0, parameters.LIFETIME_OF_ELEVATOR], y1=y-0.2, y2=y+0.2, color='white')

    plt.text(-70, y, f'Downtime', fontsize=15)

    for data in global_downtime_journal:
        plt.fill_between(data, y1=y-0.2, y2=y+0.2, color='red')

    #plt.axes().get_yaxis().set_visible(False)
    plt.axes().get_yaxis().set_ticks([])

    plt.xlabel('Time [days]')
    plt.ylabel('Uptime of a given elevator')
    plt.show()

def save_or_show(plt, save_to):
    if save_to:
        plt.savefig(save_to, dpi=220)
        print(f'saved {save_to}')
    else:
        plt.show()

def make_pie_plot(labels, sizes, save_to=None):
    plt.pie(sizes, labels=labels, autopct='%1.1f%%',
            shadow=False, startangle=90, colors=COLORS)
    plt.axis('equal')  # Equal aspect ratio ensures that pie is drawn as a circle.

    save_or_show(plt, save_to)

def make_line_plot(x, y, save_to=None):
    if x:
        plt.plot(x, y)
    else:
        plt.plot(y)

    save_or_show(plt, save_to)

# EXAMPLE
'''
make_line_plot(
    [1,2,3], [[3,2,1],
              [3.2,1.9,1.5],
              [3.1,2.2,1.2]]
)
'''

def make_stack_plot(x, y_series, labels, save_to=None):
    fig, ax = plt.subplots()
    plt.stackplot(x, y_series, labels=labels)
    ax.legend(loc='upper left')
    save_or_show(plt, save_to)

# EXAMPLE
'''
make_stack_plot(
    [1,2,3,4,5,6,7],
    [
        [3,1,6,3,7,8,2],
        [7,5,9,3,1,4,7],
        [9,8,7,6,5,4,3]
    ],
    ['Deficyt Bączka', 'Deficyt Marka', 'Deficyt Żurawia']
)

'''

# TODO
def make_table_plot():

    data = [[ 30000, 60000,  50000, 10000],
            [ 4000, 8000,  2000,  4000],
            [8000, 16000, 3000, 6000]]

    columns = ('1 cheap elevator', '2 cheap elevators', '1 pro elevator', '2 pro elevators')
    rows = ['Installation', 'Service costs', 'Repair costs']

    values = np.arange(0, 2500, 500)
    value_increment = 10

    # Get some pastel shades for the colors
    colors = COLORS #plt.cm.BuPu(np.linspace(0, 0.5, len(rows)))
    n_rows = len(data)

    index = np.arange(len(columns)) + 0.3
    bar_width = 0.4

    # Initialize the vertical-offset for the stacked bar chart.
    y_offset = np.zeros(len(columns))

    # Plot bars and create text labels for the table
    cell_text = []
    for row in range(n_rows):
        plt.bar(index, data[row], bar_width, bottom=y_offset, color=colors[row])
        y_offset = y_offset + data[row]
        cell_text.append(['%1.1f' % (x / 1000.0) for x in y_offset])
    # Reverse colors and text labels to display the last value at the top.
    colors = colors[::-1]
    cell_text.reverse()

    # Add a table at the bottom of the axes
    the_table = plt.table(cellText=cell_text,
                        rowLabels=rows,
                        rowColours=colors,
                        colLabels=columns,
                        loc='bottom')

    # Adjust layout to make room for the table:
    plt.subplots_adjust(left=0.2, bottom=0.2)

    plt.ylabel("Loss in ${0}'s".format(value_increment))
    plt.yticks(values * value_increment, ['%d' % val for val in values])
    plt.xticks([])
    plt.title('Loss by Disaster')

    plt.show()

if __name__ == '__main__':

    fig = plt.figure()
    x = np.arange(10)
    y = 2.5 * np.sin(x / 20 * np.pi)
    yerr = np.linspace(0.05, 0.2, 10)

    plt.errorbar(x, y + 3, yerr=yerr, label='both limits (default)')

    plt.errorbar(x, y + 2, yerr=yerr, uplims=True, label='uplims=True')

    plt.errorbar(x, y + 1, yerr=yerr, uplims=True, lolims=True,
                label='uplims=True, lolims=True')

    upperlimits = [True, False] * 5
    lowerlimits = [False, True] * 5
    plt.errorbar(x, y, yerr=yerr, uplims=upperlimits, lolims=lowerlimits,
                label='subsets of uplims and lolims')

    plt.legend(loc='lower right')


import matplotlib.pyplot as plt

# x-coordinates of left sides of bars
left = [1, 2, 3, 4, 5]

height = [10, 24, 60, 40, 5]

tick_label = ['model1', 'model2', 'model3', 'model4', 'model5']

plt.bar(left, height, tick_label=tick_label,
        width=0.8, color=['red', 'red', 'blue'])

plt.xlabel('types of models')
plt.ylabel('accuracy in %')
plt.title('Rehoboam Analysis')

# function to show the plot
plt.show()


# general imports
import tkinter as tk
import matplotlib.pyplot as plt
import numpy as np

# GUI has to have
class UI(object):
    def __init__(self):
        self.window = tk.Tk()
        self.width = int(self.window.winfo_screenwidth() / 2)
        self.height = int(self.window.winfo_screenheight() / 2)
        self.UI_loc = (self.height/2,self.width /2)
        self.main()

    def main(self):
        self.add_info()
        self._draw_cirle()
        self.window.geometry(f'{self.width}x{self.height}')
        self.window.mainloop()

    def add_info(self):
        self._info_b = tk.Button(self.window, text = 'Info', command = self.info)
        # self._info_b.place(x = int(self.width * 0.75), y = int(self.height/4))
        self._info_b.pack(side = 'left', padx = int(self.width * 0.5 * 0.7), pady = int(self.height/4), expand = 1)

    def info(self):
        info = tk.Tk()
        info.geometry(f'{int(self.width/2)}x{int(self.height/2)}')
        label = tk.Label(info, text = "cycle time").grid(row = 0)
        label1 = tk.Label(info, text = "rest time").grid(row = 1)
        label2 = tk.Label(info, text = "number of squats").grid(row = 2)
        # order
        # 1. cycle time
        # 2. rest time
        # 3. number of squat
        self.e1 = tk.Entry(info)
        self.e2 = tk.Entry(info)
        self.e3 = tk.Entry(info)
        self.e1.grid(row = 0, column = 1)
        self.e2.grid(row = 1, column = 1)
        self.e3.grid(row = 2, column = 1)

        tk.Button(info, text = "Enter", command = self._get_value).grid(row = 5,
                                                                       column= 0,
                                                                       sticky = tk.W,
                                                                       pady = 4)
        tk.Button(info, text = "Quit", command = info.destroy).grid(row = 5,
                                                                 column = 1,
                                                                 sticky = tk.W,
                                                                 pady = 4)
        self.info_window = info

    # code for the getting the value
    def _get_value(self):
        self.cycle_time = self.e1.get()
        self.rest_time = self.e2.get()
        self.no_squat = self.e3.get()
        print(self.cycle_time, self.rest_time, self.no_squat)
        self.info_window.destroy()

    def _draw_cirle(self):
        self.canvas = tk.Canvas(self.window)
        bbox = 0, 0, 100, 100
        print(bbox)
        self.canvas.create_oval(bbox,fill = 'black')
        self.canvas.place()

if __name__ == '__main__':
    UI()


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Sat Apr 15 22:01:58 2017

@author: wangtianpei
"""

dir_path = "/Users/wangtianpei/Desktop/Random_Week/"
import PIL
from PIL import Image
for i in range(671):
    image = Image.open(dir_path+'flow_{}.png'.format(i))
    img = image.resize((2100,1290), PIL.Image.ANTIALIAS)
    img.save(dir_path+'flow_test_{}.png'.format(i))


import numpy as np
import multiprocessing as mp
from datetime import datetime
from collections import defaultdict

embeddings = None
filter_vocab = None
n_neighbors = None

def _mp_compute_snd(args):

    return compute_snd(args)

def compute_snd(w):

    distances_df = embeddings.all_distances([w])
    filtered_distances_df = distances_df.loc[:, distances_df.columns.isin(filter_vocab)]
    sorted_distances = filtered_distances_df.loc[w].sort_values(ascending=True)
    snd = np.mean(sorted_distances[1:n_neighbors + 1])  # index 0 is identical to target word, so skip it

    return w, snd

def neighborhood_density(sem_space, target_words, reference_vocab, n=20, threads=32):

    """
    :param sem_space:       a SemanticSpace object.
    :param target_words:    list, target words for which to compute snd.
    :param reference_vocab: list, reference vocabulary listing words to consider as valid neighbors.
    :param n:               int, number of neighbours to consider. Default to 20.
    :param threads:         int, the number of cores to use for parallel processing.
    :return:                dict, mapping words to their respective SND values. Higher values indicate sparser semantic
                            neighborhoods
    """

    global embeddings
    global filter_vocab
    global n_neighbors
    embeddings = sem_space
    filter_vocab = reference_vocab
    n_neighbors = n

    w2snd = defaultdict(float)

    print(datetime.now().strftime("%d/%m/%Y %H:%M:%S: Started computing semantic neighborhood density..."))

    with mp.Pool(threads) as pool:
        outputs = pool.imap(_mp_compute_snd, ((word) for word in target_words))
        for output in outputs:
            w, snd = output
            w2snd[w] = snd
    print(datetime.now().strftime("%d/%m/%Y %H:%M:%S: Done."))

    embeddings, filter_vocab, n_neighbors = None, None, None

    return w2snd


"""
Michael duPont - michael@mdupont.com
avwx_api.api - Functional API endpoints separate from static views
"""

# stdlib
import asyncio as aio
# library
import yaml
from dicttoxml import dicttoxml as fxml
from quart import Response, abort, jsonify, request
from quart_openapi import Resource
from quart_openapi.cors import crossdomain
from voluptuous import Invalid, MultipleInvalid
# module
from avwx_api import handle, structs, token, validators

async def validate_token() -> (str, int):
    """
    Aborts request if a header token is not valid
    """
    if not token.PSQL_URI:
        return
    auth_token = request.headers.get('Authorization')
    if not auth_token or len(auth_token) < 10:
        abort(401)
    # Remove 'Token ' from token value
    if not await token.validate_token(auth_token.strip()[7:]):
        abort(403)

class Base(Resource):
    """
    Base report endpoint
    """

    validator = validators.report
    struct = structs.FetchParams
    report_type: str = None
    note: str = None

    # Replace the key's name in the final response
    _key_repl: dict = None
    # Remove the following keys from the final response
    _key_remv: [str] = None

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self._key_repl = {}
        self._key_remv = []

    def validate(self, **kwargs) -> structs.Params:
        """
        Returns all validated request parameters or an error response dict
        """
        try:
            params = {'report_type': self.report_type, **request.headers, **request.args, **kwargs}
            # Unpack param lists. Ex: options: ['info,speech'] -> options: 'info,speech'
            for k, v in params.items():
                if isinstance(v, list):
                    params[k] = v[0]
            return self.struct(**self.validator(params))
        except (Invalid, MultipleInvalid) as exc:
            key = exc.path[0]
            return {
                'error': str(exc.msg),
                'param': key,
                'help': validators.HELP.get(key)
            }

    def format_dict(self, output: dict) -> dict:
        """
        Formats a dict by recursively replacing and removing key

        Returns the item as-is if not a dict
        """
        if not isinstance(output, dict):
            return output
        resp = {}
        for k, v in output.items():
            if k in self._key_remv:
                continue
            elif k in self._key_repl:
                k = self._key_repl[k]
            if isinstance(v, dict):
                v = self.format_dict(v)
            elif isinstance(v, list):
                v = [self.format_dict(item) for item in v]
            resp[k] = v
        return resp

    def format_response(self, output: dict, format: str, meta: str = 'meta') -> Response:
        """
        Returns the output string based on format param
        """
        output = self.format_dict(output)
        if self.note:
            if meta not in output:
                output[meta] = {}
            output[meta]['note'] = self.note
        if format == 'xml':
            return Response(fxml(output, custom_root=self.report_type.upper()), mimetype='text/xml')
        elif format == 'yaml':
            return Response(yaml.dump(output, default_flow_style=False), mimetype='text/x-yaml')
        return jsonify(output)

class Report(Base):
    """
    """

    @crossdomain(origin='*')
    async def get(self, station: str) -> Response:
        """
        GET handler returning reports
        """
        params = self.validate(station=station)
        if isinstance(params, dict):
            resp = jsonify(params)
            resp.status_code = 400
        else:
            nofail = params.onfail == 'cache'
            handler = getattr(handle, self.report_type).handle_report
            data, code = await handler(params.station, params.options, nofail)
            resp = self.format_response(data, params.format)
            resp.status_code = code
        resp.headers['X-Robots-Tag'] = 'noindex'
        return resp

class LegacyReport(Report):
    """
    Legacy report endpoint to return data in pre-Sept2018 format

    Will eventually be phased out
    """

    key_repl = {
        'clouds': 'Cloud-List',
        'dewpoint_decimal': 'Dew-Decimal',
        'icing': 'Icing-List',
        'other': 'Other-List',
        'raw': 'Raw-Report',
        'runway_visibility': 'Runway-Vis-List',
        'temperature_decimal': 'Temp-Decimal',
        'turbulance': 'Turb-List',
        'wind_variable_direction': 'Wind-Variable-Dir',
    }

    note = ("The /api/<report-type> endpoint will switch to the "
            "/api/preview/<report-type> format on April 1, 2019. "
            "If you need more time, use /api/legacy/<report-type>")

    def revert_value(self, value: object) -> object:
        """
        Reverts a value based on content type
        """
        if isinstance(value, dict):
            # Revert cloud layer
            if 'modifier' in value:
                temp = [value['type'], str(value['base']).zfill(3)]
                if value['modifier'] is not None:
                    temp.append(value['modifier'])
                return temp
            # Revert number or timestamp
            elif 'repr' in value:
                return value['repr']
            # Else recursive call on embedded dict
            else:
                return self.format_dict(value)
        elif isinstance(value, list):
            return [self.revert_value(item) for item in value]
        elif value is None:
            return ''
        return value

    def format_dict(self, data: dict) -> dict:
        """
        Reverts a dict's keys and values to the legacy format
        """
        resp = {}
        for k, v in data.items():
            # Special case for TAF line
            if k == 'raw' and 'wind_shear' in data:
                k = 'Raw-Line'
            elif k in self.key_repl:
                k = self.key_repl[k]
            # 'flight_rules' -> 'Flight-Rules'
            else:
                k = k.replace('_', '-').title()
            resp[k] = self.revert_value(v)
        return resp

    @crossdomain(origin='*')
    async def get(self, station: str) -> Response:
        """
        GET handler returning METAR and TAF reports in the legacy format
        """
        params = self.validate(station=station)
        if isinstance(params, dict):
            resp = jsonify(params)
            resp.status_code = 400
        else:
            nofail = params.onfail == 'cache'
            handler = getattr(handle, self.report_type).handle_report
            data, code = await handler(params.station, params.options, nofail)
            resp = self.format_response(data, params.format, meta='Meta')
            resp.status_code = code
        resp.headers['X-Robots-Tag'] = 'noindex'
        return resp

class Parse(Base):
    """
    Given report endpoint
    """

    validator = validators.given
    struct = structs.GivenParams

    @crossdomain(origin='*')
    async def post(self) -> Response:
        """
        POST handler to parse given METAR and TAF reports
        """
        data = await request.data
        params = self.validate(report=data.decode() or None)
        if isinstance(params, dict):
            resp = jsonify(params)
            resp.status_code = 400
        else:
            handler = getattr(handle, self.report_type).parse_given
            data, code = handler(params.report, params.options)
            resp = self.format_response(data, params.format)
            resp.status_code = code
        resp.headers['X-Robots-Tag'] = 'noindex'
        return resp

class MultiReport(Base):
    """
    Multiple METAR and TAF reports in one endpoint
    """

    validator = validators.multi_report

    @crossdomain(origin='*')
    async def get(self, stations: str) -> Response:
        """
        GET handler returning multiple METAR and TAF reports
        """
        await validate_token()
        params = self.validate(station=stations)
        if isinstance(params, dict):
            resp = jsonify(params)
            resp.status_code = 400
        else:
            nofail = params.onfail == 'cache'
            handler = getattr(handle, self.report_type).handle_report
            results = await aio.gather(*[handler(
                [station],
                params.options,
                nofail
            ) for station in params.station])
            results = dict(zip(params.station, [r[0] for r in results]))
            resp = self.format_response(results, params.format)
        resp.headers['X-Robots-Tag'] = 'noindex'
        return resp

from avwx_api.api import metar, pirep, taf


import os, tempfile
from django.conf import settings
from ecs.utils.testcases import EcsTestCase
from ecs.utils.gpgutils import encrypt_sign, decrypt_verify

class Gpgutilstest(EcsTestCase):
    '''Tests for the gpgutils module

    Tests for data encryption, decryption and signature verification.
    '''

    def setUp(self):
        super().setUp()
        self.gpghome = settings.STORAGE_VAULT['gpghome']
        self.encryption_uid = settings.STORAGE_VAULT['encryption_uid']
        self.signature_uid = settings.STORAGE_VAULT['signature_uid']

    def testConsistency(self):
        '''Tests if data can be encrypted and signed and then if it can be decrypted and verified via gpg and that it matches the previously encrypted test data.'''

        self.testdata=b"im very happy to be testdata"

        osdescriptor, encryptedfilename = tempfile.mkstemp()
        os.close(osdescriptor)

        try:
            with tempfile.TemporaryFile() as inputfile:
                inputfile.write(self.testdata)
                inputfile.seek(0)
                encrypt_sign(inputfile, encryptedfilename, self.gpghome,
                    self.encryption_uid, self.signature_uid)

            decryptedfile = decrypt_verify(encryptedfilename,
                self.gpghome, self.encryption_uid, self.signature_uid)

            self.assertNotEqual(self.testdata, open(encryptedfilename, 'rb').read())
            self.assertEqual(self.testdata, decryptedfile.read())
        finally:
            if os.path.exists(encryptedfilename):
                os.remove(encryptedfilename)


import numpy as np
import numpy.linalg as npl
import matplotlib.pyplot as plt

def array_color(phase, mag):
    a = 0.6 # saturation min
    b = 0.4 # value      min
    x = np.zeros([*phase.shape] + [3], np.uint8)
    for j in range(phase.shape[0]):
        for i in range(phase.shape[1]):
            x[j,i] = hsv_rgb(phase[j,i], a+(1.0-a)*mag[j,i], b+(1.0-b)*mag[j,i])
    return x

def hsv_rgb(h,S,V):
    while h <   0: h += 360
    while h > 360: h -= 360

    C = V * S
    X = C * (1.0 - abs(((h//60) % 2) - 1))
    M = V - C

    u = int((C+M)*255)
    v = int((X+M)*255)
    w = int((0+M)*255)

    if   0 <= h and h <=  60: return [u, v, w]
    if  60 <= h and h <= 120: return [v, u, w]
    if 120 <= h and h <= 180: return [w, u, v]
    if 180 <= h and h <= 240: return [w, v, u]
    if 240 <= h and h <= 300: return [v, w, u]
    if 300 <= h and h <= 360: return [u, w, v]

def plot_derivative(rmin, rmax, n, f, fx, fy):
    # take central differences
    h = (rmax - rmin) / n
    recip = 1.0 / (h*h)

    X = np.linspace(rmin, rmax, n)
    Y = np.linspace(rmin, rmax, n)
    x,y = np.meshgrid(X,Y)

    z  = f(x,y)
    zx = fx(x,y)
    zy = fy(x,y)

    z_bounds = np.zeros([n+2, n+2])
    z_bounds[1:-1,1:-1] = z

    z2 = 2 * z_bounds[1:-1, 1:-1]
    grad = np.zeros([n, n, 2])
    grad[:,:,0] = z_bounds[1:-1, 2:] + z_bounds[1:-1,  :-2] - z2  # f_xx
    grad[:,:,1] = z_bounds[2:, 1:-1] + z_bounds[:-2,  1:-1] - z2  # f_yy
    grad *= recip

    # plotting
    fig, ax = plt.subplots(2,3,figsize=(10,6.5))

    img1 = np.stack([zx,zy], -1)
    img2 = grad

    # -- magnitude
    img1_mag = npl.norm(img1, axis=-1)
    img2_mag = npl.norm(img2, axis=-1)
    mag_diff = abs(img2_mag - img1_mag)

    m = img1_mag.max()

    img1_mag = img1_mag / m
    img2_mag = img2_mag / m
    img2_mag[img2_mag > 1.0] = 1.0
    mag_diff = mag_diff / mag_diff.max()

    ax[0][0].set_title("magnitude $\\nabla_2 f$")
    ax[0][0].imshow(img1_mag)

    ax[0][1].set_title("magnitude $\\Delta_2 f$")
    ax[0][1].imshow(img2_mag)

    ax[0][2].set_title("difference")
    ax[0][2].imshow(mag_diff)

    # -- phase
    img1_phase = np.degrees(np.arctan2(img1[:,:,1], img1[:,:,0]))
    img2_phase = np.degrees(np.arctan2(img2[:,:,1], img2[:,:,0]))
    phase_diff = abs(img2_phase - img1_phase)

    img1_phase = array_color(img1_phase, img1_mag)
    img2_phase = array_color(img2_phase, img2_mag)
    phase_diff = phase_diff / phase_diff.max()

    ax[1][0].set_title("phase $\\nabla_2 f$")
    ax[1][0].imshow(img1_phase)

    ax[1][1].set_title("phase $\\Delta_2 f$")
    ax[1][1].imshow(img2_phase)

    ax[1][2].set_title("difference")
    ax[1][2].imshow(phase_diff)

    plt.show()

rmin = 0     # range min
rmax = 4     # range max
n    = 32    # number of intervals

# Sinusoidal Example
f  = lambda x,y: np.sin(x*y)
fx = lambda x,y: -y*y*np.sin(x*y)
fy = lambda x,y: -x*x*np.sin(x*y)
plot_derivative(rmin, rmax, n, f, fx, fy)