import numpy as np
import scipy as sp
import scipy.optimize
import matplotlib as mpl
import matplotlib.pyplot as plt

R = 1#1.3/2
n = 1.35
shift = 0#-.75 - R
last_x = 5
old = False

def intersect_circle(alpha, x0, y0, R):
    a = 1 + np.tan(alpha)**2
    b = 2*np.tan(alpha)*(y0 - x0*np.tan(alpha))
    c = x0**2 * np.tan(alpha)**2 - 2*x0*y0*np.tan(alpha) + y0**2 - R**2

    discr = b**2 - 4*a*c
    if discr >= 0:
        xa = (-b + np.sqrt(discr))/(2 * a)
        xb = (-b - np.sqrt(discr))/(2 * a)
        return [xa, xb]
    else:
        return None

def refract(alpha, x0, y0, ni, nt):
    ray_inc = np.array([1, np.tan(alpha)])
    ray_inc /= np.linalg.norm(ray_inc)
    normal = np.array([x0, y0])
    normal /= np.linalg.norm(normal)
    print(ray_inc)
    print(normal)

    if old:
        theta_i = np.arccos(np.dot(ray_inc, normal))
        if theta_i > np.pi/2:
            theta_i = np.pi - theta_i
        print("theta_i {0}".format(theta_i/np.pi*180))
        theta_t = np.arcsin(ni * np.sin(theta_i) / nt)
        print("theta_t {0}".format(theta_t/np.pi*180))
        print("alpha {0}".format(alpha/np.pi*180))
        vec0 = ni * np.cross(ray_inc, normal)
        print("ray_inc x normal {0}".format(vec0))
        alpha_new1 = theta_t - theta_i + alpha
        vec1a = nt * np.cross([1, np.tan(alpha_new1)]/(1 + np.tan(alpha_new1)**2), normal)
        alpha_new2 = alpha - (theta_t - theta_i)
        vec1b = nt * np.cross([1, np.tan(alpha_new2)]/(1 + np.tan(alpha_new2)**2), normal)
        print("ray_trans x normal {0}, {1}".format(vec1a, vec1b))
        if np.abs(vec1a - vec0) < np.abs(vec1b - vec0):
            alpha_new = alpha_new1
        else:
            alpha_new = alpha_new2
        if np.amin([np.abs(vec1a - vec0),  np.abs(vec1b - vec0)]) > .01:
            print("warning raytracing")

    else:
        ray_inc_3 = np.array([ray_inc[0], ray_inc[1], 0])
        normal_3 = np.array([normal[0], normal[1], 0])
        if np.dot(ray_inc, normal) > 0:
            normal_3 = -normal_3
        kt = ni/nt*np.cross(normal_3, np.cross(-1*normal_3, ray_inc_3)) - normal_3 * np.sqrt(1 - (ni/nt)**2 * np.linalg.norm(np.cross(normal_3, ray_inc_3))**2)
        print("{0} {1} {2}".format(kt, np.arctan2(kt[1], kt[0]), nt*np.cross([kt[0], kt[1]], normal)))
        alpha_new = np.arctan2(kt[1], kt[0])
        print("ni * ki x normal {0}".format(ni * np.cross(ray_inc, normal)))
        print("nt * kt x normal {0}".format(nt*np.cross([kt[0], kt[1]], normal)))
    return (alpha_new, x0, y0)

def plot_segment(x0, y0, x1, y2, alpha, color='b-'):
    beta = np.linspace(0, 1)
    x = beta * x0 + (1 - beta) * x1
    y = np.tan(alpha) * (x - x0) + y0
    plt.plot(x - shift, y, color)

def raytrace(alpha, x0, y0, color='b-'):
    #plot ray 1
    x1 = np.amin(intersect_circle(alpha, x0, y0, R))
    if x1 is None:
        x1 = last_x
        y1 = np.tan(alpha) * (x1 - x0) + y0
        plot_segment(x0, y0, x1, y1, alpha, color)
        return
    y1 = np.tan(alpha) * (x1 - x0) + y0
    plot_segment(x0, y0, x1, y1, alpha, color)

    #plot ray 2
    alpha1, *c = refract(alpha, x1, y1, 1, n)
    print(alpha1)
    x2 = np.amax(intersect_circle(alpha1, x1, y1, R))
    y2 = np.tan(alpha1) * (x2 - x1) + y1
    plot_segment(x1, y1, x2, y2, alpha1, color)

    #plot ray 3
    alpha2, *c = refract(alpha1, x2, y2, n, 1)
    print(alpha2)
    x3 = last_x
    y3 = np.tan(alpha2) * (x3 - x2) + y2
    plot_segment(x2, y2, x3, y3, alpha2, color)

def raytrace_angles(angles, x0, y0, color='b-'):
    for angle in angles:
        raytrace(angle/180*np.pi, x0, y0, color)

x0 = -2 - R
y0 = 0

"""
# Chromatic Aberration
n = 1.33141
raytrace_angles(range(-18, 19, 4), x0, y0, 'r-')
n = 1.34235
raytrace_angles(range(-18, 19, 4), x0, y0, 'b-')
"""

"""Spherical Aberration
raytrace_angles(range(-18, 19, 2), x0, y0, 'b-')
"""

raytrace_angles(range(-4,17,2), x0, y0, 'b-')
#raytrace_angles(range(-10,11,2), x0, y0, 'r-')

"""
old = True
raytrace(3/180*np.pi, x0, y0, 'r-')
raytrace(5/180*np.pi, x0, y0, 'r-')
raytrace(10/180*np.pi, x0, y0, 'r-')
raytrace(18/180*np.pi, x0, y0, 'r-')
"""

y0 = .2
"""
raytrace(-.05, x0, y0, 'g-')
raytrace(-.01, x0, y0, 'g-')
raytrace(-.1, x0, y0, 'g-')
raytrace(-.03, x0, y0, 'g-')
raytrace(.1, x0, y0, 'g-')
raytrace(.05, x0, y0, 'g-')
raytrace(.01, x0, y0, 'g-')
raytrace(.03, x0, y0, 'g-')
raytrace(0, x0, y0, 'g-')
"""

"""
alpha = .03
raytrace(alpha, x0, 0, 'k-')
raytrace(alpha, x0, .1, 'k-')
raytrace(alpha, x0, .2, 'k-')
raytrace(alpha, x0, .3, 'k-')
raytrace(alpha, x0, .4, 'k-')
"""

"""
alpha = 0
raytrace(alpha, x0, 0, 'r-')
raytrace(alpha, x0, .1, 'r-')
raytrace(alpha, x0, .2, 'r-')
raytrace(alpha, x0, .3, 'r-')
raytrace(alpha, x0, .4, 'r-')
raytrace(alpha, x0, -.1, 'r-')
raytrace(alpha, x0, -.2, 'r-')
raytrace(alpha, x0, -.3, 'r-')
raytrace(alpha, x0, -.4, 'r-')
"""

plt.axis([-3,5,-1,1])
#plot lens
phi = np.linspace(0, 2 * np.pi)
plt.plot(R*np.cos(phi) - shift, R*np.sin(phi), 'r-')

plt.grid(b=True, which='major', color='k', linestyle='--')
plt.show()


"""empty message

Revision ID: d8d772eb526f
Revises: dd61148aa0eb
Create Date: 2023-03-31 17:58:32.322215

"""
from alembic import op
import sqlalchemy as sa

# revision identifiers, used by Alembic.
revision = 'd8d772eb526f'
down_revision = 'dd61148aa0eb'
branch_labels = None
depends_on = None

def upgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.create_table('posts',
    sa.Column('title', sa.String(length=255), nullable=False),
    sa.Column('slug', sa.String(length=255), nullable=False),
    sa.Column('content', sa.Text(), nullable=False),
    sa.Column('date', sa.DateTime(), nullable=True),
    sa.Column('is_published', sa.Boolean(), nullable=True),
    sa.Column('users_id', sa.Integer(), nullable=False),
    sa.Column('id', sa.Integer(), nullable=False),
    sa.Column('created_at', sa.DateTime(), nullable=True),
    sa.Column('updated_at', sa.DateTime(), nullable=True),
    sa.ForeignKeyConstraint(['users_id'], ['users.id'], ondelete='CASCADE'),
    sa.PrimaryKeyConstraint('id'),
    sa.UniqueConstraint('slug'),
    sa.UniqueConstraint('title')
    )
    # ### end Alembic commands ###

def downgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.drop_table('posts')
    # ### end Alembic commands ###


from termcolor import colored

print(colored('{txt:*^119}'.format(txt='*'), 'red'))
print(colored("""
                                                         |--__
                                                         |
                                                         X
                                                |-___   / \       |--__
                                                |      =====      |
                                                X      | .:|      X
                                               / \     | O |     / \\
                                              =====   |:  . |   =====
                                              |.: |__| .   : |__| :.|
                                              |  :|. :  ...   : |.  |
                                      __   __W| .    .  ||| .      :|W__  --
                                    -- __  W  WWWW______"_"______WWWW   W -----  --
                                -  -     ___  ---    ____     ____----       --__  -
                                    --__    --    --__     -___        __-   _
""", 'red'))
print(colored('{txt:*^119}'.format(txt='*'), 'red'))
print(colored('{txt:*^119}'.format(txt='*'), 'blue'))
print(colored("""
                                                     Привет!

                    Это приложение по созданию листа персонажа для игры Dungeons & Dragons.
                    Ниже будут появляться инструкции, следуй им и у тебя будет самый топовый персонаж
""", 'blue'))
print(colored('{txt:*^119}'.format(txt='*'), 'blue'))
print('{txt:*^119}\n'.format(txt='*'))
name_player = input('Шаг 1: Для начала давай познакомимся! Введите ваше имя: ')  # имя игрока
print()

def input_gender():
    print('Шаг 2: За персонажа какого пола вы будете играть: ')
    gender = {
        1: ['Мужчина (Man)'],
        2: ['Женщина (Woman)']
    }
    for number, gender_name in gender.items():
        print(' {num:^5} |    {name} '.format(num=number, name=gender_name[0]))
    user_input_gender = int(input('Введите номер пола: '))

    while user_input_gender not in gender:
        print('Такого пола нет')
        user_input_gender = int(input("Введите номер пола: "))
    else:
        print('Вы выбрали пол', gender[user_input_gender][0], '\n')
        name_gender = gender[user_input_gender][0]
        return name_gender

def input_level():
    print('Шаг 3: За персонажа какого уровня вы будете играть: ')
    user_input_level = int(input('Введите номер уровня от 1 до 20: '))
    lvl = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20}

    while user_input_level not in lvl:
        print('Такого уровня нет')
        user_input_level = int(input("Введите номер уровня от 1 до 20: "))
    else:
        print('Вы выбрали уровень', user_input_level, '\n')
        name_level = user_input_level
        return name_level

name = ''  # имя персонажа
game_gender = input_gender()  # пол персонажа
age = ''  # возраст персонажа
height = ''  # рост персонажа
weight = ''  # вес персонажа

race = ''  # раса
game_class = ''  # класс
background = ''  # предыстория
alignment = ''  # мировоззрение

like_mechanics = ''  # призвание/гильдия/судьбоносное происшествие
personality_traits = ''  # черты характера
ideals = ''  # идеалы
bonds = ''  # привязанности
flaws = ''  # слабости

level = input_level()  # уровень
experience_point = 0  # опыт
proficiency_bonus = 0  # бонус мастерства

if level == 1:
    experience_point = 0
    proficiency_bonus = 2
elif level == 2:
    experience_point = 300
    proficiency_bonus = 2
elif level == 3:
    experience_point = 900
    proficiency_bonus = 2
elif level == 4:
    experience_point = 2700
    proficiency_bonus = 2
elif level == 5:
    experience_point = 6500
    proficiency_bonus = 3
elif level == 6:
    experience_point = 14000
    proficiency_bonus = 3
elif level == 7:
    experience_point = 23000
    proficiency_bonus = 3
elif level == 8:
    experience_point = 34000
    proficiency_bonus = 3
elif level == 9:
    experience_point = 48000
    proficiency_bonus = 4
elif level == 10:
    experience_point = 64000
    proficiency_bonus = 4
elif level == 11:
    experience_point = 85000
    proficiency_bonus = 4
elif level == 12:
    experience_point = 100000
    proficiency_bonus = 4
elif level == 13:
    experience_point = 120000
    proficiency_bonus = 5
elif level == 14:
    experience_point = 140000
    proficiency_bonus = 5
elif level == 15:
    experience_point = 165000
    proficiency_bonus = 5
elif level == 16:
    experience_point = 195000
    proficiency_bonus = 5
elif level == 17:
    experience_point = 225000
    proficiency_bonus = 6
elif level == 18:
    experience_point = 265000
    proficiency_bonus = 6
elif level == 19:
    experience_point = 305000
    proficiency_bonus = 6
elif level == 20:
    experience_point = 355000
    proficiency_bonus = 6

strength = 0  # сила
dexterity = 0  # ловкость
constitution = 0  # телосложение
intelligence = 0  # интеллект
wisdom = 0  # мудрость
charisma = 0  # харизма

mod_strength = 0  # модификатор сила
mod_dexterity = 0  # модификатор ловкость
mod_constitution = 0  # модификатор телосложение
mod_intelligence = 0  # модификатор интеллект
mod_wisdom = 0  # модификатор мудрость
mod_charisma = 0  # модификатор харизма

check_saving_throws_strength = False  # знание спас бросок сила
check_saving_throws_dexterity = False  # знание  спас бросок ловкость
check_saving_throws_constitution = False  # знание  спас бросок телосложение
check_saving_throws_intelligence = False  # знание  спас бросок интеллект
check_saving_throws_wisdom = False  # знание  спас бросок мудрость
check_saving_throws_charisma = False  # знание  спас бросок харизма

saving_throws_strength = mod_strength  # спас бросок сила
saving_throws_dexterity = mod_dexterity  # спас бросок ловкость
saving_throws_constitution = mod_constitution  # спас бросок телосложение
saving_throws_intelligence = mod_intelligence  # спас бросок интеллект
saving_throws_wisdom = mod_wisdom  # спас бросок мудрость
saving_throws_charisma = mod_charisma  # спас бросок харизма

check_acrobatics = False  # знание акробатика
check_perception = False  # знание анализ
check_athletics = False  # знание атлетика
check_investigation = False  # знание внимательность
check_survival = False  # знание выживание
check_performance = False  # знание выступление
check_intimidation = False  # знание запугивание
check_history = False  # знание история
check_sleight_of_hand = False  # знание ловкость рук
check_arcana = False  # знание магия
check_medicine = False  # знание медицина
check_deception = False  # знание обман
check_nature = False  # знание природа
check_insight = False  # знание проницательность
check_religion = False  # знание религия
check_stealth = False  # знание скрытность
check_persuasion = False  # знание убеждение
check_animal_handling = False  # знание уход за животными

acrobatics = mod_dexterity  # акробатика
perception = mod_intelligence  # анализ
athletics = mod_strength  # атлетика
investigation = mod_wisdom  # внимательность
survival = mod_wisdom  # выживание
performance = mod_charisma  # выступление
intimidation = mod_charisma  # запугивание
history = mod_intelligence  # история
sleight_of_hand = mod_dexterity  # ловкость рук
arcana = mod_intelligence  # магия
medicine = mod_wisdom  # медицина
deception = mod_charisma  # обман
nature = mod_intelligence  # природа
insight = mod_wisdom  # проницательность
religion = mod_intelligence  # религия
stealth = mod_dexterity  # скрытность
persuasion = mod_charisma  # убеждение
animal_handling = mod_wisdom  # уход за животными

passive_wisdom = 10 + mod_wisdom  # пассивная мудрость (внимательность)

armor_class = 0  # КД (класс доспеха)
initiative = dexterity  # инициатива
speed = 0  # скорость

hit_point_maximum = 0  # максимальное кол-во очков жизни
hit_dise = ''  # кубик жизни

dark_vision = ''  # темное зрение
language = ''  # языки
other = ''  # другие умения
skills = ''  # навыки
equipment = ''  # снаряжение


import autograd.numpy as np
from autograd.scipy.special import logsumexp
import autograd.numpy.random as npr
from autograd import grad
from autograd.misc.optimizers import adam
import os
import sys
import matplotlib.pyplot as plt

from sklearn.metrics import confusion_matrix, ConfusionMatrixDisplay

# path to data
sys.path.append(os.path.join(os.getcwd(), "../../../data"))
from data.data_utils import load_dataset

# from typing import list, Tuple

def init_randn(m, n, rs=npr.RandomState(0)):
    """ init mxn matrix using small random normal
    """
    return 0.1 * rs.randn(m, n)

def init_xavier(m, n, rs=npr.RandomState(0)):
    return rs.randn(m, n) / np.sqrt(m)

def init_net_params(layer_sizes, init_fcn, rs=npr.RandomState(0)):
    """ inits a (weights, biases) tuples for all layers using the intialize function (init_fcn)"""
    return [
        (init_fcn(m, n), np.zeros(n))  # weight matrix  # bias vector
        for m, n in zip(layer_sizes[:-1], layer_sizes[1:])
    ]

def relu(x):
    return np.maximum(0, x)

def neural_net_predict(params, inputs):
    for W, b in params:
        outputs = np.dot(inputs, W) + b # general formula for transformation of a layer in neural net
        inputs = relu(outputs)
    return outputs - logsumexp(outputs, axis=1, keepdims=True)

def mean_log_like(params, inputs, targets):
    # raise NotImplementedError("mean_log_like function not completed (see Q3).")
    # print('computing mean-log-like')
    return np.mean(neural_net_predict(params, inputs) * targets) * 10

def mean_log_like2(params, inputs, targets):
    N = len(inputs)
    for W, b in params:
        outputs = np.dot(inputs, W) + b # general formula for transformation of a layer in neural net
        inputs = relu(outputs)
    # outputs = relu(outputs - logsumexp(outputs, axis=1, keepdims=True))
    log_p = 0
    for i, f_hat in enumerate(outputs):
        for j, val in enumerate(f_hat):
            if targets[i][j]:
                if (val > 0):
                    continue
                else:
                    outputs[i][j] = 0
    return log_p / N

def accuracy(params, inputs, targets):
    """ return the accuracy of the neural network defined by params
    """
    target_class = np.argmax(targets, axis=1)
    predicted_class = np.argmax(neural_net_predict(params, inputs), axis=1)
    return np.mean(predicted_class == target_class)

if __name__ == "__main__":
    ''' best so far: roughly 57% with 100 0.0004 10000 [784, 512, 10] (peaks at 50 iterations)
    some ideas:
        - two hidden layers? how many units in each?
        - probably best to tune the layer count first, then worry about getting good grad desc params
    '''
    # ----------------------------------------------------------------------------------
    # EXAMPLE OF HOW TO USE STARTER CODE BELOW
    # ----------------------------------------------------------------------------------

    # loading data
    x_train, x_valid, x_test, y_train, y_valid, y_test = load_dataset("mnist_small")
    # print(len(x_train))
    # initializing parameters
    layer_sizes = [784, 392,10]
    params = init_net_params(layer_sizes, init_randn)

    # setting up training parameters
    num_epochs = 30
    learning_rate = 2e-3
    batch_size = 1000 # total of 10000 datapoints

    # Constants for batching
    num_batches = int(np.ceil(len(x_train) / batch_size))
    rind = np.arange(len(x_train))
    npr.shuffle(rind)

    def batch_indices(iter):
        idx = iter % num_batches
        return rind[slice(idx * batch_size, (idx + 1) * batch_size)]

    # Define training objective
    def objective(params, iter):
        # get indices of data in batch
        idx = batch_indices(iter)
        # print(x_train[idx][0])
        return -mean_log_like(params, x_train[idx], y_train[idx])

    # Get gradient of objective using autograd.
    objective_grad = grad(objective)
    print(num_epochs, learning_rate, batch_size, layer_sizes)
    print("     Epoch     |    Train accuracy  |       Test accuracy  ")

    # Dictionary to store train/val history
    opt_history = {
        "train_nll": [],
        "val_nll": [],
    }
    global g_params
    g_params = None

    def callback(params, iter, gradient):
        if iter % num_batches == 0:
            # record training & val. accuracy every epoch
            opt_history["train_nll"].append(-mean_log_like(params, x_train, y_train))
            opt_history["val_nll"].append(-mean_log_like(params, x_valid, y_valid))
            train_acc = accuracy(params, x_train, y_train)
            val_acc = accuracy(params, x_valid, y_valid)
            print("{:15}|{:20}|{:20}".format(iter // num_batches, train_acc, val_acc))
            global g_params
            g_params = params

    # We will optimize using Adam (a variant of SGD that makes better use of
    # gradient information).
    opt_params = adam(
        objective_grad,
        params,
        step_size=learning_rate,
        num_iters=num_epochs * num_batches,
        callback=callback,
    )
    # Plotting the train and validation negative log-likelihood
    plt.plot(opt_history["train_nll"], label='training')
    plt.plot(opt_history["val_nll"], label = 'validation')
    plt.xlabel('Epoch')
    plt.ylabel('Loss')
    plt.legend()
    # plt.show()
    plt.cla()
    target_class = np.argmax(y_test, axis=1)
    predicted_class = np.argmax(neural_net_predict(g_params, x_test), axis=1)
    cm = confusion_matrix(predicted_class, target_class)
    disp = ConfusionMatrixDisplay(cm)
    disp.plot()
    plt.show()


"""GraphLoader.py"""

import sys
import csv
import pandas as pd
import numpy as np
from scipy.sparse import coo_matrix
import networkx as nx
from .graph import Graph

def custom_read_edgelist(path, comments='#', delimiter=' ', \
                         create_using=None, \
                         nodetype=np.int64, data=True, \
                         edgetype=np.int64, encoding='utf-8'):
    """
    DESC: Customized read_edgelist() to construct graph adjacency matrix
          in the form of scipy csr matrix directly from input file.
    PARAMS:    Same as in networkx.read_edgelist().
               nodetype, data, edgetype, encoding are not used.
    """
    # checking number of columns in input file
    fstr = open(path, 'r')
    reader = csv.reader(fstr, delimiter=delimiter)
    sample = next(reader)
    while sample[0][0] == comments: #skipping comments
        sample = next(reader)
    fstr.close()

    ncol = len(sample)
    if ncol == 2:
        names = ['src', 'dst']
    elif ncol == 3:
        names = ['src', 'dst', 'wgt']
    else:
        msg = "read_edgelist: Expected 2 or 3 columns in input file!\n"
        msg = msg + str(ncol) + " column detected in first row: " + str(sample)
        msg = msg + "\nPlease ensure if the specified delimiter '"
        msg = msg + delimiter + "' is correct!"
        raise ValueError(msg)

    # loading data by excluding duplicate rows
    df = pd.read_csv(path, sep=delimiter, comment=comments, \
                     names = names, dtype = edgetype)

    # converting to numpy array
    mat = df.drop_duplicates().values
    num_edges = mat.shape[0]

    # checking whether data is 0-based or 1-based
    tarr = mat[:, :2].flatten()
    if sys.version_info[0] < 3:
        min_id = long(tarr.min()) #'long' should be replaced with 'int',
        max_id = long(tarr.max()) #'int' gives same behaviour as 'long' in py3.
    else:
        min_id = int(tarr.min())
        max_id = int(tarr.max())

    if min_id == 0:
        rowid = mat[:, 0]
        colid = mat[:, 1]
        num_vertices = max_id + 1
    else:
        rowid = mat[:, 0] - 1
        colid = mat[:, 1] - 1
        num_vertices = max_id

    # extracting edge weight information (if available)
    if ncol == 3:
        data = mat[:, 2]
    else:
        data = np.ones(num_edges)

    # constructing sparse matrix structure
    data = np.asarray(data, dtype = edgetype)
    rowid = np.asarray(rowid, dtype = nodetype)
    colid = np.asarray(colid, dtype = nodetype)
    shape = (num_vertices, num_vertices)
    if isinstance(create_using, nx.classes.digraph.DiGraph):
        coo = coo_matrix((data, (rowid, colid)), shape=shape)
    else:
        data_ = np.concatenate((data, data))
        rowid_ = np.concatenate((rowid, colid))
        colid_ = np.concatenate((colid, rowid))
        coo = coo_matrix((data_, (rowid_, colid_)), shape=shape)
    return coo.tocsr()

def read_edgelist(path, comments='#', delimiter=' ', \
                  create_using=None,\
                  nodetype=np.int64, data=True, \
                  edgetype=np.int64, encoding='utf-8'):
    """
    DESC: Reads edgelist data from persistent storage.
    PARAMS:    path : file or string
                      File or filename to read. If a file is
                      provided, it must be opened in 'rb' mode. Filenames
                      ending in .gz or .bz2 will be uncompressed.
               comments : string, optional
                      The character used to indicate the start of a comment.
               delimiter : string, optional
                      The string used to separate values.  The default is
                      whitespace.
               create_using : NetworkX graph constructor, optional
                              (default=nx.Graph)
                      Graph type to create. If graph instance, then cleared
                      before populated.
               nodetype : int, float, str, Python type, optional
                      Convert node data from strings to specified type
               data : bool or list of (label,type) tuples
                      Tuples specifying dictionary key names and types for
                      edge data
               edgetype : int, float, str, Python type, optional OBSOLETE
                      Convert edge data from strings to specified type and
                      use as 'weight'
               encoding: string, optional
                      Specify which encoding to use when reading file.
    """
    #nx_graph = nx.read_edgelist(path, comments, delimiter, create_using, \
    #                            nodetype, data, edgetype, encoding)
    #return Graph(nx_graph=nx_graph)
    smat = custom_read_edgelist(path, comments, delimiter, create_using, \
                                nodetype, data, edgetype, encoding)
    return Graph(nx_graph=smat)


import aioredis

from gloop.channels import Channel

class RedisChannel(Channel):

    def __init__(self, topic: str, address: str):
        super(RedisChannel, self).__init__()
        self._topic: str = topic
        self._address: str = address
        self._redis: aioredis.Redis = None

    async def connect(self):
        self._redis = await aioredis.create_redis(self._address)

    async def send(self, message: str):
        # XADD self._topic * message {message}
        if self._redis is None:
            await self.connect()
        await self._redis.xadd(
            self._topic,
            dict(
                message=message
            )
        )

    async def receive(self) -> str:
        if self._redis is None:
            await self.connect()
        messages = await self._redis.xread([self._topic])
        message = messages[0]
        topic, message_hash, body = message
        content = list(body.values())[0]
        return content.decode()

    async def close(self):
        self._redis.close()
        await self._redis.wait_closed()


"""Time the 'dtool copy' command."""

import time

try:
    from urlparse import urlparse
except ImportError:
    from urllib.parse import urlparse

import click
import dtoolcore

@click.command()
@click.argument("input_uri")
@click.argument("output_prefix")
@click.argument("output_storage", default="file")
def main(input_uri, output_prefix, output_storage):
    StorageBroker = dtoolcore._get_storage_broker(input_uri, None)
    parsed_uri = urlparse(input_uri)

    for uri in StorageBroker.list_dataset_uris(parsed_uri.path, None):
        dataset = dtoolcore.DataSet.from_uri(uri)
        name = dataset.name

        start = time.time()
        dtoolcore.copy(uri, output_prefix, output_storage)
        elapsed = time.time() - start
        print("Copying {} took: {}s".format(name, elapsed))

if __name__ == "__main__":
    main()


from selenium import webdriver

class WebDriverFactory():

    def __init__(self, browser, private=False, baseURL="https://courses.letskodeit.com"):
        self.browser = browser
        self.private = private
        self.baseURL = baseURL

    def getWebDriverInstance(self):

        if self.browser == "firefox":
            profile = webdriver.FirefoxProfile()
            if self.private:
                profile.set_preference("browser.privatebrowsing.autostart", True)

            driver = webdriver.Firefox(firefox_profile=profile)

        elif self.browser == "chrome":
            chrome_options = webdriver.ChromeOptions()
            chrome_options.add_experimental_option("useAutomationExtension", False)
            chrome_options.add_experimental_option("excludeSwitches", ["enable-automation"])
            if self.private:
                chrome_options.add_argument("--incognito")

            driver = webdriver.Chrome(options=chrome_options)
        else:
            driver = webdriver.Firefox()
        # Setting Driver Implicit Time out for An Element
        driver.implicitly_wait(3)
        # Maximize the window
        driver.maximize_window()
        # Loading browser with App URL
        driver.get(self.baseURL)
        return driver

    def gotoBaseURL(self, your_driver):
        your_driver.get(self.baseURL)


"""
Strokes gained analyses.
"""
from pga_analysis.putting import *
from pga_analysis.par_5_tee_shots import *
import warnings
from scipy.stats import wilcoxon
import logging

warnings.simplefilter(action='ignore', category=FutureWarning)

def prepare_tournament_data(tournament, pga_data_added):
    """
    Returns pga_data for a given tournament, for players that made the cut (played 4 rounds).
    """
    tournament_data = pga_data_added[pga_data_added.tournament == tournament]

    players_made_the_cut = tournament_data.groupby('player')['round_number'].max() == 4
    players_made_the_cut = pd.Series(players_made_the_cut[players_made_the_cut].index)
    tournament_data = tournament_data.merge(players_made_the_cut, how='inner', on='player')

    return tournament_data

def compute_total_putting_strokes_gained_by_player(tournament_data, putting_data):
    """
    Computes total putting strokes gained for each player in the data.
    """
    _, first_shot_on_green = prepare_putting_data(tournament_data)
    putting_strokes_gained = compute_putting_strokes_gained(putting_data, first_shot_on_green, by_player=True,
                                                            aggregation='sum')
    return putting_strokes_gained

def compute_total_par_5_strokes_gained_by_player(tournament_data, strokes_model, par_5_tee_shot_features, pga_data_added):
    """
    Computes total strokes gained from the tee shot on a Par 5 for each player in the data.
    """
    par_5_analysis_data = get_par_5_analysis_data(tournament_data, common_shot_end_locations(pga_data_added))
    par_5_model_features = par_5_analysis_data.drop(['player', 'strokes_from_here', 'strokes_to_finish'], axis=1)

    # Gets columns in the same order as when the model was trained
    orig_model_columns = \
        par_5_tee_shot_features.drop(['player', 'strokes_to_finish', 'strokes_from_here'], axis=1).columns
    for column in orig_model_columns:
        if column not in par_5_analysis_data.columns:
            par_5_model_features[column] = [0] * len(par_5_model_features)
    par_5_model_features = par_5_model_features[orig_model_columns]

    par_5_strokes_gained = strokes_model.predict(par_5_model_features)
    strokes_left = pd.DataFrame({'player': par_5_analysis_data.player,
                                 'strokes_to_finish': par_5_analysis_data.strokes_to_finish,
                                 'predicted_strokes_to_finish': par_5_strokes_gained})
    strokes_left['strokes_gained'] = strokes_left.predicted_strokes_to_finish - strokes_left.strokes_to_finish
    cum_strokes_gained = strokes_left.groupby('player')['strokes_gained'].sum()

    return cum_strokes_gained

def plot_strokes_gained_driving_vs_putting_by_winnings(putting_data, strokes_model,
                                                       par_5_tee_shot_features, pga_data_added,
                                                       winnings_rank_min=0, winnings_rank_max=5,
                                                       over_all_tournaments=False):
    """
    Strokes gained violin plot for driving (left half) vs putting (right half).

    winnings_rank_{min / max}: Min/max rank of tournament maximum winnings to plot.
    """
    most_winnings_tournaments = \
        pga_data_added.groupby('tournament').winnings.max().sort_values(ascending=False)[winnings_rank_min:winnings_rank_max].index

    if not over_all_tournaments:
        fig, ax = plt.subplots(figsize=(17, 6))

        putting_strokes_gained_list = []
        tee_shot_strokes_gained_list = []

        for tournament in most_winnings_tournaments:
            tournament_data = prepare_tournament_data(tournament, pga_data_added)
            putting_strokes_gained = compute_total_putting_strokes_gained_by_player(tournament_data, putting_data).iloc[:, 0]
            tee_shot_strokes_gained = compute_total_par_5_strokes_gained_by_player(tournament_data, strokes_model,
                                                                                par_5_tee_shot_features, pga_data_added)

            putting_strokes_gained_list.append(putting_strokes_gained)
            tee_shot_strokes_gained_list.append(tee_shot_strokes_gained)

    else:
        fig, ax = plt.subplots(figsize=(7, 6))

        putting_strokes_gained_list = [compute_total_putting_strokes_gained_by_player(pga_data_added, putting_data).iloc[:, 0]]
        tee_shot_strokes_gained_list = [compute_total_par_5_strokes_gained_by_player(pga_data_added, strokes_model,
                                                                                par_5_tee_shot_features, pga_data_added)]

    def violin_plot(data, half, width_factor=1.0):
        violin_plot = ax.violinplot(data, positions=np.arange(0, len(data)),
                                    showmeans=False, showextrema=False, showmedians=False, widths=0.5 * width_factor)
        for b in violin_plot['bodies']:
            m = np.mean(b.get_paths()[0].vertices[:, 0])
            if half == 'left':
                b.get_paths()[0].vertices[:, 0] = np.clip(b.get_paths()[0].vertices[:, 0], -np.inf, m)
                b.set_color('b')
            elif half == 'right':
                b.get_paths()[0].vertices[:, 0] = np.clip(b.get_paths()[0].vertices[:, 0], m, np.inf)
                b.set_color('r')
        return violin_plot

    violin_left = violin_plot(tee_shot_strokes_gained_list, half='left')
    width_factor = 0.7 if not over_all_tournaments else 0.85
    violin_right = violin_plot(putting_strokes_gained_list, half='right', width_factor=width_factor)

    ax.legend([violin_left['bodies'][0],violin_right['bodies'][0]],['DRIVING', 'PUTTING'])
    if not over_all_tournaments:
        ax.set_xticklabels(
            ['', 'THE PLAYERS CHAMPIONSHIP 2021\n$2.7M', 'PGA CHAMPIONSHIP 2020\n$2M', 'WGC CONCESSION 2021\n$1.8M'
                , 'WGC FEDEX ST JUDE INVITATIONAL 2020\n$1.8M', 'BMW CHAMPIONSHIP 2020\n$1.7M']
        )
    else:
        ax.set_xticks([])
    plt.ylabel('Strokes Gained')
    plt.title('Total Strokes Gained (Player)')

def plot_putting_strokes_gained_distribution(first_shot_to_green):
    """
    Plot distribution of average putting strokes gained per round, by player
    """
    putting_strokes_gained_by_player = first_shot_to_green.groupby(['player', 'tournament', 'round_number']) \
        ['strokes_gained'].sum().reset_index().groupby(['player']).strokes_gained.mean()
    grid = sns.kdeplot(putting_strokes_gained_by_player, shade=True)
    plt.xlim([-3, 3])
    plt.ylabel('')
    grid.set(yticks=[])
    set_size(10, 2)
    plt.title('Putting: Average Strokes Gained per Round, by Player')
    plt.xlabel('Strokes Gained')

def plot_strokes_gained_above_or_below_15_feet(putting_data, first_shot_to_green):
    """
    Scatter plot of each player's strokes gained for putts below or above 15 feet.
    """
    sg_below_15_feet_by_player = compute_putting_strokes_gained(putting_data, first_shot_to_green[
        first_shot_to_green.shot_start_distance_feet < 15], by_player=True)
    sg_above_15_feet_by_player = compute_putting_strokes_gained(putting_data, first_shot_to_green[
        first_shot_to_green.shot_start_distance_feet >= 15], by_player=True)
    sg_15_feet_putts_threshold = sg_below_15_feet_by_player.merge(sg_above_15_feet_by_player, on='player', how='inner')
    sg_15_feet_putts_threshold.columns = ['below_15_feet', 'above_15_feet']
    grid = sns.lmplot(x='below_15_feet', y='above_15_feet', data=sg_15_feet_putts_threshold)
    plt.xlabel('Avg SG BELOW 15-feet')
    plt.ylabel('Avg SG ABOVE 15-feet')
    plt.title('Small correlation between good SHORT vs LONG putters\n')
    plt.xlim([-0.2, 0.2])
    plt.ylim([-0.2, 0.2])
    set_size(5,4)
    grid.set(xticks=[-0.2, 0, 0.2], yticks=[-0.2, 0, 0.2])
    plt.tight_layout()

def drive_or_putt_for_dough(pga_data_added, putting_data, strokes_model, par_5_tee_shot_features):
    """
    Wilcoxon Signed Rank test to test whether players gained more strokes from putting vs driving.
    Unused due to insignificant result.
    """
    putting_sg = compute_total_putting_strokes_gained_by_player(pga_data_added, putting_data).iloc[:,0]
    driving_sg = compute_total_par_5_strokes_gained_by_player(pga_data_added, strokes_model, par_5_tee_shot_features, pga_data_added)

    statistic, p_value = wilcoxon(x=driving_sg, y=putting_sg, alternative='greater')

    logger = logging.getLogger()
    logger.info(f'For each player, PUTTING contributed more SG than PAR 5 TEE SHOTS.\n'
                f'p-value = {round(p_value,3)}')


import easyocr
import time
import pandas as pd
from PIL import Image
import cv2
import numpy as np
import shutil
import os
import tensorflow as tf
from PIL import Image
from matplotlib.pyplot import imshow
import matplotlib.pyplot as plt
import time
import keras_ocr
start_time = time.time()
assert tf.test.is_gpu_available(), 'No GPU is available.'

#####################################################################################################
# FRAME CREATION AND THEIR EHANCEMENTS

def create_frames(path):
    """Creates frames from the given input video
    Args:path of the video"""
    frame_counter = 0
    reduced_frames = 0
    capture = cv2.VideoCapture(path)
    name = path.split("/")[-1]
    directory = f'./images/{name}'
    if (os.path.isdir(directory)):
        shutil.rmtree(directory)

    os.makedirs(directory)
    while True:
        success, frame = capture.read()
        if success:
            # Deleting low resolution frame
            if np.mean(frame) < 15:
                print("Removed the frame")
                continue
            if frame_counter % 400 == 0:
                resized_frame = cv2.resize(frame, (960, 540))
                cv2.imwrite(
                    f'./images/{name}/frame_{frame_counter}.jpg', resized_frame)
                reduced_frames += 1
        else:
            break
        frame_counter += 1
    print(f'There were {frame_counter} frames in this video')
    print(f'After reducing frames there are {reduced_frames} in this video. ')
    capture.release()
    return

def enhance_powerlaw(path, file_name, gamma):
    image = cv2.imread(path)
    normalized_image = image / 255.0
    result = cv2.pow(normalized_image, gamma)
    frame_normed = 255 * (result - result.min()) / \
        (result.max() - result.min())
    frame_normed = np.array(frame_normed, np.int64)

    cv2.imwrite(
        f'./enhanced_images/{dataset}/{file_name}.jpg', frame_normed)
    return

def enhance_inverse(path, file_name, file):
    image = Image.open(path)
    image_array = np.asarray(image)
    image_inverse = 255 - image_array
    fm = './enhanced_images/inverse_law/'+dataset+'/'+file_name
    # print(fm)
    file.write(fm+'\n')
    cv2.imwrite(fm, image_inverse)
    return

def enhance_frames(path, opt, file):
    dirs = os.listdir(path)
    # print("runnnnel")
    if (opt == 0):
        # Applying Power Law
        directory = f'./enhanced_images/power_law/'+dataset
        if (os.path.isdir(directory)):
            shutil.rmtree(directory)

        os.makedirs(directory)
        for file_name in dirs:
            enhance_powerlaw(path + '/' + file_name, file_name, 1.6)
    else:
        # Applying Inverse law
        directory = f'./enhanced_images/inverse_law/'+dataset
        if (os.path.isdir(directory)):
            shutil.rmtree(directory)

        os.makedirs(directory)
        for file_name in dirs:
            enhance_inverse(path + '/' + file_name, file_name, file)

#####################################################################################################
# OPTICAL CHARACTER RECOGNITION (OCR) ON ROI
d = {}
reader = easyocr.Reader(['en'], recog_network='iter_30000', gpu=True)

def OCR(imgs, mode):
    if mode == 2:
        for img in imgs:
            image = img[1]
            result = reader.readtext(image)
            if img[0] not in d:
                d[img[0]] = []
                if (len(result) > 0 and len(result[0]) >= 2):
                    d[img[0]].append(result[0][-2])
                else:
                    d[img[0]].append(None)
            else:
                if (len(result) > 0 and len(result[0]) >= 2):
                    d[img[0]].append(result[0][-2])
                else:
                    d[img[0]].append(None)

#####################################################################################################
####################################################################################################
# for kocr_detection_orientation_check

def order_points(pts):
    # initialzie a list of coordinates that will be ordered
    # such that the first entry in the list is the top-left,
    # the second entry is the top-right, the third is the
    # bottom-right, and the fourth is the bottom-left
    rect = np.zeros((4, 2), dtype="float32")
    # the top-left point will have the smallest sum, whereas
    # the bottom-right point will have the largest sum
    s = pts.sum(axis=1)
    rect[0] = pts[np.argmin(s)]
    rect[2] = pts[np.argmax(s)]
    # now, compute the difference between the points, the
    # top-right point will have the smallest difference,
    # whereas the bottom-left will have the largest difference
    diff = np.diff(pts, axis=1)
    rect[1] = pts[np.argmin(diff)]
    rect[3] = pts[np.argmax(diff)]
    # return the ordered coordinates
    return rect

def four_point_transform(image, pts):
    # obtain a consistent order of the points and unpack them
    # individually
    rect = order_points(pts)
    (tl, tr, br, bl) = rect
    # compute the width of the new image, which will be the
    # maximum distance between bottom-right and bottom-left
    # x-coordiates or the top-right and top-left x-coordinates
    widthA = np.sqrt(((br[0] - bl[0]) ** 2) + ((br[1] - bl[1]) ** 2))
    widthB = np.sqrt(((tr[0] - tl[0]) ** 2) + ((tr[1] - tl[1]) ** 2))
    maxWidth = max(int(widthA), int(widthB))
    # compute the height of the new image, which will be the
    # maximum distance between the top-right and bottom-right
    # y-coordinates or the top-left and bottom-left y-coordinates
    heightA = np.sqrt(((tr[0] - br[0]) ** 2) + ((tr[1] - br[1]) ** 2))
    heightB = np.sqrt(((tl[0] - bl[0]) ** 2) + ((tl[1] - bl[1]) ** 2))
    maxHeight = max(int(heightA), int(heightB))
    # now that we have the dimensions of the new image, construct
    # the set of destination points to obtain a "birds eye view",
    # (i.e. top-down view) of the image, again specifying points
    # in the top-left, top-right, bottom-right, and bottom-left
    # order
    dst = np.array([
        [0, 0],
        [maxWidth - 1, 0],
        [maxWidth - 1, maxHeight - 1],
        [0, maxHeight - 1]], dtype="float32")
    # compute the perspective transform matrix and then apply it
    M = cv2.getPerspectiveTransform(rect, dst)
    warped = cv2.warpPerspective(image, M, (maxWidth, maxHeight))
    # return the warped image
    return warped

#####################################################################################################

def showROIS(ROIs, img_raw):
    # counter to save image with different name
    crop_number = 0
    imgs = []
    # loop over every bounding box save in array "ROIs"
    for rect in ROIs:
        if rect[1].shape == (4,):
            x1 = rect[1][0]
            y1 = rect[1][1]
            x2 = rect[1][2]
            y2 = rect[1][3]
            # crop roi from original image
            img_crop = img_raw[y1:y1+y2, x1:x1+x2]
            # save cropped image
            imgs.append((rect[0], "crop"+str(crop_number)+".jpeg"))
            cv2.imwrite("crop"+str(crop_number)+".jpeg", img_crop)
            crop_number += 1
            # hold window
            cv2.waitKey(0)
        else:
            warped = four_point_transform(img_raw, rect[1])
            imgs.append((rect[0], "crop"+str(crop_number)+".jpeg"))
            cv2.imwrite("crop"+str(crop_number)+".jpeg", warped)
            crop_number += 1

    OCR(imgs, 2)
    # closing all open windows
    cv2.destroyAllWindows()
####################################################################################################
# REGION OF INTEREST (ROI) SELECTION

det = keras_ocr.detection.Detector()
det.model.load_weights('./detector_icdar2013.h5')

def createROIS(image_folder, mode):
    files = os.listdir(image_folder)
    print(files)
    lst = []
    for file in files:
        lst.append(image_folder + "/" + file)
    print(lst)

    if mode == 1:
        for path in image_folder:
            img_raw = cv2.imread(image_folder + "/" + path)
            # select ROIs function
            ROIs = cv2.selectROIs(
                "Select ROIs and press Enter to move further, Press Esc to exit, Press c to clear selection", img_raw)
            # print rectangle points of selected roi
            print(ROIs)
            showROIS(ROIs, img_raw)
    elif mode == 2:
        path = files[0]
        p = image_folder + "/" + path
        img_raw = cv2.imread(p)
        image = keras_ocr.tools.read(p)
        img_det = det.detect([image])
        img_tup = []
        for i in range(len(img_det[0])):
            st = 'roi'+str(i+1)
            img_tup.append((st, img_det[0][i]))
        img = keras_ocr.tools.drawBoxes(image, img_det[0], color=(
            36, 255, 12), thickness=1, boxes_format='boxes')
        for i in img_tup:
            x = int(i[1][0][0])
            y = int(i[1][0][1])
            cv2.putText(img, i[0], (x, y-5),
                        cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 20, 147), 2)

        cv2.imshow('window_name', img)
        cv2.waitKey(0)
        r = input(
            "Please select the ROIs on which you want OCR to be done by entering them with comma as a separator:")
        rl = r.split(",")
        rm = list(map(lambda x: "roi"+x, rl))
        rf = tuple(filter(lambda x: x[0] in rm, img_tup))
        print(rf)
        ROIs = cv2.selectROIs(
            "Select ROIs and press Enter to move further, Press Esc to exit, Press c to clear selection", img)
        # print rectangle points of selected roi
        print(ROIs)
        ROI_tup = []
        for i in ROIs:
            name = input(
                'Please enter the attribute name for the selected ROIS:')
            ROI_tup.append((name, i))
        ROI_tup.extend(rf)
        for name, coords in ROI_tup:
            print(f'{name} --- {coords}  {type(coords)} {coords.shape}')
        showROIS(ROI_tup, img_raw)
        del files[0]
        for path in files:
            img_raw = cv2.imread(image_folder + "/" + path)
            showROIS(ROI_tup, img_raw)
    else:
        for path in files:
            print(image_folder + "/" + path)

            abcd = reader.readtext(image_folder + "/" + path)
            for line in abcd:
                print(f"{line[0]}      {line[1]}")
            print("*****************************************************************")

#####################################################################################################
# MAIN DRIVER FUNCTIONS

file = open('images.txt', 'w')
dataset = "Stable_Video_1.mp4"
create_frames("./dataset/"+dataset)
enhance_frames("./images/"+dataset, 1, file)
createROIS("./enhanced_images/inverse_law/"+dataset, mode=2)
formated_output = pd.DataFrame(d)
print(formated_output)
file.close()
print("Execution time: ", time.time() - start_time)


import numpy as np
from fastestimator.summary.logs import parse_log_file
from matplotlib import pyplot as plt

def visualize_image_classification(
        file_path="../../../logs/supplementary/lr_search/image_classification.txt",
        file_extension=".txt"):
    summary = parse_log_file(file_path, file_extension)
    acc_steps = [x[0] for x in summary.history["eval"]["accuracy"].items()]
    acc_values = [x[1] for x in summary.history["eval"]["accuracy"].items()]
    best_step, best_acc = max(summary.history["eval"]["accuracy"].items(),
                              key=lambda k: k[1])
    max_lr = summary.history["train"]["model_lr"][best_step]
    lr_values = [summary.history["train"]["model_lr"][x] for x in acc_steps]
    assert len(lr_values) == len(acc_values)
    plt.plot(lr_values, acc_values)
    plt.plot(max_lr,
             best_acc,
             'o',
             color='r',
             label="Best Acc={}, LR={}".format(best_acc, max_lr))
    plt.xlabel("Learning Rate")
    plt.ylabel("Evaluation Accuracy")
    plt.legend(loc='upper left', frameon=False)
    plt.title("Image Classification Learning Rate Search")
    # plt.show()
    plt.savefig("lr_search_ic.png", dpi=300)

def visualize_instance_detection(
        file_path="../../../logs/supplementary/lr_search/instance_detection.txt",
        file_extension=".txt"):
    summary = parse_log_file(file_path, file_extension)
    acc_steps = [x[0] for x in summary.history["eval"]["total_loss"].items()]
    loss_values = [x[1] for x in summary.history["eval"]["total_loss"].items()]
    best_step, best_acc = min(summary.history["eval"]["total_loss"].items(),
                              key=lambda k: k[1])
    max_lr = summary.history["train"]["model_lr"][best_step]
    lr_values = [summary.history["train"]["model_lr"][x] for x in acc_steps]
    assert len(lr_values) == len(loss_values)
    plt.plot(lr_values, loss_values)
    plt.plot(max_lr,
             best_acc,
             'o',
             color='r',
             label="Best Loss={}, LR={}".format(best_acc, max_lr))
    plt.xlim(0.0, 0.022)
    plt.ylim(1.7, 2.5)
    plt.xticks(np.arange(0.0, 0.022, 0.005))
    plt.xlabel("Learning Rate")
    plt.ylabel("Evaluation Loss")
    plt.legend(frameon=False)
    plt.title("Instance Detection Learning Rate Search")
    # plt.show()
    plt.savefig("lr_search_id.png", dpi=300)

def visualize_language_modeling(
        file_path="../../../logs/supplementary/lr_search/language_modeling.txt",
        file_extension=".txt"):
    summary = parse_log_file(file_path, file_extension)
    acc_steps = [x[0] for x in summary.history["eval"]["perplexity"].items()]
    perplexity_values = [
        x[1] for x in summary.history["eval"]["perplexity"].items()
    ]
    best_step, best_acc = min(summary.history["eval"]["perplexity"].items(),
                              key=lambda k: k[1])
    max_lr = summary.history["train"]["model_lr"][best_step]
    lr_values = [summary.history["train"]["model_lr"][x] for x in acc_steps]
    assert len(lr_values) == len(perplexity_values)
    plt.plot(lr_values, perplexity_values)
    plt.plot(max_lr,
             best_acc,
             'o',
             color='r',
             label="Best Perplexity={}, LR={}".format(best_acc, max_lr))
    plt.xlim(0.0, 46)
    plt.ylim(50, 1000)
    plt.xlabel("Learning Rate")
    plt.ylabel("Evaluation Perplexity")
    plt.legend(frameon=False)
    plt.title("Language Modeling Learning Rate Search")
    # plt.show()
    plt.savefig("lr_search_lm.png", dpi=300)

if __name__ == "__main__":
    visualize_language_modeling()


# from sklearn import svm as sksvm
from models.libsvm import svmutil as libsvm, _svm as libsvm_low

import os
import numpy as np
from datetime import datetime
from pprint import pformat

try:
    import cPickle as pickle
except:
    import pickle

from models.model_base import ModelBase
from utilities import linux_path, profile

class SVM(ModelBase):
    """
    :type _params: SVM.Params
    :type _weights: libsvm.svm_train
    :type _problem: libsvm.svm_problem
    :type _libsvm_params: libsvm.svm_parameter
    :type _train_features: np.ndarray
    :type _train_labels: np.ndarray
    :type labels: np.ndarray
    :type probabilities: np.ndarray
    """

    class Params(ModelBase.Params):
        """
        :ivar implementation: SVM implementation to use: 0: libsvm (high level)
        1: libsvm (low level) 2: sklearn.svm 3: lasvm

        :ivar verbose: Show detailed diagnostic messages, if any, from the underlying
        library

        """

        def __init__(self):
            ModelBase.Params.__init__(self)

            self.implementation = 0
            self.verbose = 0

    def __init__(self, params, logger, feature_shape, name):
        """
        :type params: SVM.Params
        :type logger: logging.RootLogger
        :rtype: None
        """
        ModelBase.__init__(self, params, logger, feature_shape, name, 'SVM', n_classes=2)

        self._params = params

        self._weights = None
        self._problem = None
        self._libsvm_params = None

        self._train_opts = '-c 1 -g 1 -b 1 -h 0'
        self._predict_opts = '-b 1'

        if not self._params.verbose:
            # self._train_opts += ' -q'
            self._predict_opts += ' -q'

        self._labels = None
        # self.accuracies = None
        self._probabilities = None
        self.accuracies = None

        if self._params.implementation == 0:
            self._logger.info('Using high level implementation')
            self._problem_fn = libsvm.svm_problem
        elif self._params.implementation == 1:
            self._logger.info('Using low level implementation')
            self._problem_fn = libsvm_low.svm_problem
        else:
            raise IOError('Invalid implementation type: {}'.format(self._params.implementation))

        if not self._params.accumulative:
            self._logger.warning('accumulative training is disabled')

        self._params.min_samples = int(self._params.min_samples)

        if self._params.min_samples > 0:
            self._logger.info('skipping training with less than {:d} samples'.format(self._params.min_samples))

        if self._params.min_samples_ratio > 0:
            self._logger.info(
                'skipping training with ratio of new samples less than {:.3f}'.format(self._params.min_samples_ratio))

        self._params.max_samples = int(self._params.max_samples)

        if self._params.max_samples > 0:
            self._logger.info('limiting the maximum number of training samples to {:d}'.format(self._params.max_samples))

    def train(self, labels, features):
        """
        :type labels: np.ndarray
        :type features: np.ndarray
        :rtype: None
        """

        if not self._check_train(labels, features):
            return

        n_samples = self._train_labels.size

        _train_labels, _train_labels = self._train_labels, self._train_labels

        max_samples = int(self._params.max_samples)

        if max_samples > 0 and max_samples < n_samples:
            self._logger.info('randomly selecting {} / {} training samples'.format(
                max_samples, n_samples))

            all_idx = list(range(n_samples))
            random_idx = list(np.random.permutation(all_idx))
            random_sampled_idx = random_idx[:max_samples]

            _train_features = self._train_features[random_sampled_idx, ...]
            _train_labels = self._train_labels[random_sampled_idx]

            n_samples = max_samples
        else:
            _train_features, _train_labels = self._train_features, self._train_labels

        features_list = list(map(list, _train_features))
        labels_list = list(_train_labels)

        # params = libsvm.svm_parameter('-c 1 -q')
        # prob = libsvm.svm_problem([1,-1], [{1:1, 3:1}, {1:-1,3:-1}])

        start_time = datetime.now().strftime("%Y/%m/%d %H:%M:%S")
        self._logger.info(f'{start_time} :: training on {n_samples} samples')

        times = {}
        with profile('svm_train', times, show=1):
            self._problem = self._problem_fn(labels_list, features_list)
            self._libsvm_params = libsvm.svm_parameter(self._train_opts)
            self._weights = libsvm.svm_train(self._problem, self._libsvm_params)

        self._logger.info('time taken: {}'.format(times['svm_train']))

        self.is_trained = True
        self.is_loaded = False

        # self.logger.debug('Done')

    def predict(self, features, gt_labels=None, vis=None):
        """
        :type features: np.ndarray
        :type gt_labels: np.ndarray | None
        :type vis bool | int | None
        :rtype: None
        """
        n_samples = features.shape[0]
        if self._params.verbose:
            self._logger.info('predicting for {} samples'.format(n_samples))

        features_list = list(map(list, features))
        labels = -np.ones((n_samples,))
        labels_list = list(labels)

        _labels, _accuracies, _probabilities = libsvm.svm_predict(
            labels_list, features_list, self._weights, self._predict_opts)

        self._labels = np.array(_labels)
        self._probabilities = np.array(_probabilities)
        self.accuracies = np.array(_accuracies)
        return self._labels, self._probabilities

    def batch_train(self):
        """

        :return: str
        """
        _batch_params = self._params.batch  # type: ModelBase.Params.BatchTrain

        self._load_samples_recursive(_batch_params.db_path, non_negative_labels=False)

        self._split_samples(_batch_params)

        save_path = _batch_params.save_dir
        load_path = _batch_params.load_dir
        if not load_path:
            load_path = save_path

        if _batch_params.load_weights:
            self.load(load_path)

        # self.train(self._all_labels, self._all_features)

        n_samples = self._all_labels.size
        start_time = datetime.now().strftime("%Y/%m/%d %H:%M:%S")
        self._logger.info(f'{start_time} :: batch training on {n_samples} samples')

        times = {}
        with profile('svm_train', times, show=1):
            features_list = list(map(list, self._all_features))
            labels_list = list(self._all_labels)

            self._problem = self._problem_fn(labels_list, features_list)
            self._libsvm_params = libsvm.svm_parameter(self._train_opts)
            self._weights = libsvm.svm_train(self._problem, self._libsvm_params)

        self._logger.info('time taken: {}'.format(times['svm_train']))

        self.is_trained = True
        self.is_loaded = False

        if not save_path:
            save_root_dir = os.path.dirname(self._db_path[0])
            timestamp = datetime.now().strftime("%y%m%d_%H%M%S")
            save_path = linux_path(save_root_dir, f'svm_batch_{timestamp}')

        os.makedirs(save_path, exist_ok=True)

        model_save_path = self.save(save_path, save_samples=False)

        model_dict = {
            'model_save_path': model_save_path,
            'n_samples': n_samples,
            'timestamp': datetime.now().strftime("%Y/%m/%d %H:%M:%S"),
        }
        pickle.dump(model_dict, open('{:s}/model_info.bin'.format(save_path), "wb"))

        return model_save_path

    def save(self, save_dir, save_samples=True):
        """
        :type save_path: str
        :type save_samples: bool | int
        :rtype: None
        """
        save_dir = self._check_save(save_dir)
        if save_dir is None:
            return

        model_save_path = linux_path(save_dir, 'model.bin')

        self._logger.info(f'Saving weights to {model_save_path}')
        libsvm.svm_save_model(model_save_path, self._weights)

        assert os.path.isfile(model_save_path), "model saving failed"

        if save_samples and self._params.accumulative:
            samples_save_dir = linux_path(save_dir, 'train_samples')
            self._save_train_samples(samples_save_dir, reset=not self._params.accumulative)

        return model_save_path

    def load(self, load_dir):
        """
        :type load_dir: str
        :type load_samples: bool | int
        :rtype: None
        """

        if self._params.batch.load_samples:
            self._load_train_samples(load_dir)
            return True
        """
        hack to allow custom load_path for the model independently of the target
        """
        if self._params.batch.load_dir:
            load_dir = self._params.batch.load_dir

        if os.path.isdir(load_dir):
            model_load_path = linux_path(load_dir, 'model.bin')
        else:
            model_load_path = load_dir

        if not os.path.exists(model_load_path):
            msg = 'weights file does not exist: {}'.format(model_load_path)
            raise IOError(msg)

        self._logger.info('Loading weights from: {}'.format(model_load_path))
        self._weights = libsvm.svm_load_model(model_load_path)

        assert self._weights is not None, "Model loading failed"

        model_info_path = linux_path(load_dir, 'model_info.bin')

        if os.path.isfile(model_info_path):
            model_info = pickle.load(open(model_info_path, "rb"))
            self._logger.info('Loaded model with train info: {}'.format(pformat(model_info)))

        # if load_samples and self._params.accumulative:
        #     samples_load_path = linux_path(load_dir, 'train_samples')
        #     self._load_train_samples(samples_load_path)

        self.is_trained = True
        self.is_loaded = True

        return True


from django import template
from githubpage.models import GitHubRepo, GitHubRepoPage
from datetime import datetime, timezone
import util
register = template.Library()

@register.simple_tag(takes_context=True)
def reload_github_repo(context, user="", repo=""):
    """Refreshes/created data for a GitHubRepoPage using the GitHub API."""

    fetch_new_data = False  # if we need to fetch new data

    _repos = GitHubRepo.objects.filter(owner_name__iexact=user, name__iexact=repo)
    if _repos.count() > 0:
        old_repo = _repos[0]
        if (datetime.now(timezone.utc) - old_repo.object_last_updated).days >= 1:  # if at least 1 day passed since last update
            fetch_new_data = True  # fetch new data, because the one we have is outdated
    else:  # if there is no repo matching the given owner and repo name
        fetch_new_data = True  # ...we definitely ned to fetch teh data from the GitHub API
        old_repo = None

    if fetch_new_data:
        ghdata = util.retrieve_github_repo(user, repo)
        if ghdata['status'] == '200 OK':
            repo_status = util.convert_github_private_status(ghdata['private'])
            if old_repo is None:
                new_github_data = GitHubRepo(
                    name=ghdata['name'],
                    owner_name=ghdata['owner']['login'],
                    owner_id=ghdata['owner']['id'],
                    access_status=repo_status,
                    description=ghdata['description'],
                    created_on=ghdata['created_at'],
                    last_updated=ghdata['updated_at'],
                    size=ghdata['size'],
                    language=ghdata['language'],
                    issues=ghdata['open_issues'],
                    forks=ghdata['forks_count'],
                    stargazers=ghdata['stargazers_count'],
                    subscribers=ghdata['subscribers_count'],
                    object_last_updated=datetime.now(),
                    repo_url=ghdata['html_url'],
                )
            else:
                new_github_data = GitHubRepo(
                    id=old_repo.id,
                    name=ghdata['name'],
                    owner_name=ghdata['owner']['login'],
                    owner_id=ghdata['owner']['id'],
                    access_status=repo_status,
                    description=ghdata['description'],
                    created_on=ghdata['created_at'],
                    last_updated=ghdata['updated_at'],
                    size=ghdata['size'],
                    language=ghdata['language'],
                    issues=ghdata['open_issues'],
                    forks=ghdata['forks_count'],
                    stargazers=ghdata['stargazers_count'],
                    subscribers=ghdata['subscribers_count'],
                    object_last_updated=datetime.now(),
                    repo_url=ghdata['html_url'],
                    )
            new_github_data.save()
            curr_page = GitHubRepoPage.objects.filter(id=context["curr_page_id"])[0]
            curr_page.repo = new_github_data
            curr_page.save()
            context['update_successful'] = True  # tell site context that we successfully updated the repo
        else:  # if we didn't recieve a 200 OK response
            context['update_successful'] = False  # tell site context that the update was not successful
    else:
        context['update_successful'] = True  # tell site that no update was needed, which is a success in itself
    return ''  # return an empty string, because we cannot return nothing


import numpy as np
import pandas as pd
import math
import matplotlib.pyplot as plt

def calculate_garman_klass_volatility(open_price, close_price, high_price, low_price, num_trading_days):

    try:
        ln_ratio_open_close = math.log(close_price / open_price)
        ln_ratio_high_low = math.log(high_price / low_price)

        first_component = (0.5 * ln_ratio_open_close) ** 2
        second_component = (2 * ln_ratio_high_low - ln_ratio_open_close) ** 2

        average_daily_volatility = (first_component - second_component) / (num_trading_days * math.log(2))
        #print(f"average_daily_volatility: {average_daily_volatility}")
        realized_volatility = math.sqrt(abs(average_daily_volatility))

        return realized_volatility
    except:
        pass

df = pd.read_csv("optionsdata.csv")
#print(df.head())
#print(df.info())
#print(df.iloc[1])
#print(df.get(["OPEN","CLOSE","HIGH","LOW"]))
test = pd.DataFrame(df.get(["OPEN","CLOSE","HIGH","LOW"]))
l = test.values.tolist()
#l = test.iloc[1:3].values.tolist()
#print(l)
gks = []

for i in l:
    #print(f"i: {i}")
    garm = calculate_garman_klass_volatility(i[0],i[1],i[2],i[3],20)
    if garm != None:
        gks.append(garm)
    #print("Garman-Klass Vol: ", garm)

#time = np.arange(0, len(gks)*200, 200)

# Calculate the average of every 200 values
agks = [np.mean(gks[i:i+110]) for i in range(0, len(gks), 110)]

# Plotting the data
plt.plot(agks)

# Adding labels and title
plt.xlabel('Time')
plt.ylabel('Data')
plt.title('Data vs. Time')

# Display the plot
plt.show()

#Fourier Analysis on the signal

from numpy.fft import rfft, irfft, rfftfreq
from scipy import pi, signal, fftpack

gks_ft = np.abs(rfft(agks))

gks_freq = rfftfreq(len(agks))

plt.plot(gks_freq, gks_ft)

plt.xlabel('frequency (1/day)')
plt.show()

# Apply FFT
fft_values = np.fft.fft(agks)

# Calculate magnitude spectrum
magnitude_spectrum = np.abs(fft_values)

# Generate frequency axis
N = len(agks)
T = 1  # Assuming unit time interval between data points
frequencies = np.fft.fftfreq(N, T)

# Define the threshold value
threshold = 30  # Adjust as needed

# Filter out high-frequency components
fft_values_filtered = fft_values.copy()
fft_values_filtered[magnitude_spectrum < threshold] = 0
denoised_data = np.fft.ifft(fft_values_filtered).real
plt.plot(range(N), agks, label='Original Data')
plt.plot(range(N), denoised_data, label='Denoised Data')
plt.xlabel('Time')
plt.ylabel('Value')
plt.title('Original Data vs Denoised Data')
plt.legend()

#ARIMA
#
#import pandas as pd
#import matplotlib.pyplot as plt
#from statsmodels.tsa.arima.model import ARIMA
#
## Example time series data
#
#data = agks
## Create the ARIMA model
#model = ARIMA(agks, order=(1, 0, 10))
#
## Fit the ARIMA model
#model_fit = model.fit()
#
## Generate forecasts
#forecast_values = model_fit.forecast(steps=5)
#
#
## Plot the original data and the forecasts
#plt.plot(range(len(data)), data, label='Original Data')
#plt.plot(range(len(data), len(data) + len(forecast_values)), forecast_values, label='Forecasted Data')
#plt.xlabel('Time')
#plt.ylabel('Value')
#plt.title('ARIMA Forecast')
#plt.legend()
#plt.show()


import input
from box import PriceBox
from yahoofinancials import YahooFinancials

df = []
tickers = ['TSLA','GME','AMC']
start_date = '2015-11-27'
end_date = '2015-11-27'
financials = YahooFinancials(tickers)
# for ticker in (tickers):
#     df.append(input.get_stock_price_data_withPD(financials,ticker,start_date,end_date,'daily'))

box1 = PriceBox(start_date, end_date,financials)
print(box1)


"""Proyecto2 URL Configuration

The `urlpatterns` list routes URLs to views. For more information please see:
    https://docs.djangoproject.com/en/4.1/topics/http/urls/
Examples:
Function views
    1. Add an import:  from my_app import views
    2. Add a URL to urlpatterns:  path('', views.home, name='home')
Class-based views
    1. Add an import:  from other_app.views import Home
    2. Add a URL to urlpatterns:  path('', Home.as_view(), name='home')
Including another URLconf
    1. Import the include() function: from django.urls import include, path
    2. Add a URL to urlpatterns:  path('blog/', include('blog.urls'))
"""
from django.urls import path
from AppAvanci.views import (
    mostrar_inicio,
    mostrar_cursos,
    mostrar_entregables,
    mostrar_estudiantes,
    mostrar_profesores,
    procesar_formulario,
    procesar_formulario_2,
    busqueda,
    buscar,
    busqueda_2,
    buscar_2,
)

urlpatterns = [
    path("inicio/", mostrar_inicio, name="inicio"),
    path("cursos/", mostrar_cursos, name="cursos"),
    path("profesores/", mostrar_profesores, name="profesores"),
    path("estudiantes/", mostrar_estudiantes, name="estudiantes"),
    path("entregables/", mostrar_entregables, name="entregables"),
    path("formulario/", procesar_formulario, name="formulario"),
    path("formulario-2/", procesar_formulario_2, name="formulario_2"),
    path("busqueda", busqueda, name="busqueda"),
    path("buscar/", buscar),
    path("busqueda-2/", busqueda_2, name="busqueda-2"),
    path("buscar-2/", buscar_2)
]


import itertools
import pickle
from collections import Counter

import matplotlib.pyplot as plt
import numpy as np
from keras.utils import to_categorical
from sklearn.metrics import confusion_matrix
from sklearn.model_selection import train_test_split

def load_data(path):
    f = open(path, 'rb')
    images, labels = pickle.load(f)
    f.close()
    return images, labels

def preprocess_data(raw_images, raw_labels):
    images = raw_images
    labels = raw_labels.flatten()

    images = images[labels != 30]
    labels = labels[labels != 30]

    return images, labels

def make_train_test_split(images, labels, test_size):
    images_tr, images_tst, labels_tr, labels_tst = train_test_split(images, labels, test_size=test_size,
                                                                    random_state=2018)

    test_sample = (images_tst, labels_tst)
    with open('test_sample.pkl', 'wb') as f:
        pickle.dump(test_sample, f)

    train_sample = (images_tr, labels_tr)
    with open('train_sample.pkl', 'wb') as f:
        pickle.dump(train_sample, f)

def plot_img(img, label):
    plt.imshow(np.reshape(img, (56, 56)), cmap='binary')
    plt.suptitle(label)
    plt.show()

def plot_img_for_each_class(images, labels):
    for label in set(labels):
        images_of_label = images[labels==label]
        for img in images_of_label[:3]:
            plot_img(img, label)

def calculate_class_weight(labels):
    labels_dict = Counter(labels)
    m = max(labels_dict.values())
    class_weights = dict()

    for label in labels_dict.keys():
        if labels_dict[label]:
            score = m / labels_dict[label]
        else:
            score = 1
        class_weights[label] = score
    return class_weights

def plot_confusion_matrix(labels_true, labels_predicted):
    classes = ['6', 'P', 'O', 'V', 'W', '3', 'A', '8', 'T', '1', '0', '9', 'H', 'R', 'N', '7', 'K', 'L', 'G', '4', 'Y',
               'C', 'E', 'J', '5', 'I', 'S', '2', 'F', 'Z', '-', 'Q', 'M', 'B', 'D', 'U']

    cm = confusion_matrix(labels_true, labels_predicted, labels=range(36))

    plt.imshow(cm, interpolation='nearest', cmap=plt.cm.Blues)
    plt.title('Confusion matrix')
    plt.colorbar()

    tick_marks = np.arange(len(classes))
    plt.xticks(tick_marks, classes)
    plt.yticks(tick_marks, classes)

    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):
        plt.text(j, i, cm[i, j],
                 horizontalalignment="center",
                 color="white" if cm[i, j] > cm.max() / 2 else "black")

    plt.tight_layout()
    plt.ylabel('True label')
    plt.xlabel('Predicted label')

    plt.show()

if __name__ == '__main__':
    from predict import predict

    images_tst, labels_tst = load_data('test_sample.pkl')
    images_tst = images_tst.reshape(-1, 56, 56, 1)
    labels_tst = to_categorical(labels_tst, num_classes=36)

    labels_predicted = predict(images_tst)
    labels_true = np.argmax(labels_tst, axis=1)

    plot_confusion_matrix(labels_true, labels_predicted)


from django.db import models
from django.contrib.auth.models import User
from django.utils import timezone
from DjangoUeditor.models import UEditorField

# 商品标签
class CommodityTag(models.Model):
	tag = models.CharField(max_length = 300)

	def __str__(self):
		return self.tag

# 商品货源
class CommoditySource(models.Model):
	source = models.CharField(max_length = 300)

	def __str__(self):
		return self.source

# 商品
class Commodity(models.Model):
	title = models.CharField(max_length = 300)
	owner = models.ForeignKey(User, related_name = "commodity", on_delete = models.CASCADE)
	commodity_tag = models.ForeignKey(CommodityTag, related_name = "commodity_tag", on_delete = models.CASCADE)
	commodity_source = models.ForeignKey(CommoditySource, related_name = "commodity_source", on_delete = models.CASCADE)
	body = UEditorField(verbose_name = "commodity_descriprion",imagePath="ueditorImages/",
			width=700, height=600, filePath='ueditorFiles/', toolbars="full",
			upload_settings={"imageMaxSize":1204000}, blank = True)
	price = models.DecimalField(max_digits=7, decimal_places=2, default = 0.00)
	image = models.ImageField(blank=True)
	for_sale = models.BooleanField(default = True)
	is_verified = models.BooleanField(default = True)
	created = models.DateTimeField(default = timezone.now)
	updated = models.DateTimeField(auto_now = True)

	class Meta:
		ordering = ('-updated',)

	def __str__(self):
		return self.title


from PIL import Image, ImageDraw
import pandas as pd
from typing import Union, Optional
from GoogleDriveClass import GoogleClient
import os
from tqdm import tqdm
import glob

COLOR_DICT = {
    0: (0, 191, 255),
    1: (255, 235, 153),
    2: (255, 26, 26),
    3: (77, 184, 255),
    4: (191, 128, 64),
    5: (204, 204, 255),
    6: (204, 163, 0),
    7: (20, 31, 31),
    8: (153, 153, 153),
    9: (255, 204, 255),
}

def draw_circle(x: float, y: float, radius: int, color: int,  draw_object: ImageDraw.Draw) -> None:
    """

    Args:
        x: x coordinate of center
        y: y coordinate of center
        color: a number from 1 to 10 to represent the color of the circle.
        radius: the radius

    Returns:

    """
    draw_object.ellipse((x - radius, y - radius, x + radius, y + radius), fill=COLOR_DICT[color])

def get_concat_h(im1: Image, im2: Image) -> Image:
    dst = Image.new('RGB', (im1.width + im2.width, im1.height))
    dst.paste(im1, (0, 0))
    dst.paste(im2, (im1.width, 0))
    return dst

def nuclei_by_group(image_path: str, csv_path: str, k: int, circle_radius: int=4,
                       output_path: Optional[str]=None, show: bool=True):
    original_image = Image.open(image_path)
    changed_image = Image.open(image_path)
    df = pd.read_csv(csv_path)
    draw = ImageDraw.Draw(changed_image)

    for index, row in df.iterrows():
        draw_circle(row['Centroid X px'], row['Centroid Y px'], circle_radius, row[f'K={k}'], draw)
    concatenated_image = get_concat_h(original_image, changed_image)
    if show:
        concatenated_image.show()
    if output_path:
        concatenated_image.save(output_path)
        print("Saved output_path")

def download_and_color_whole_slide(slide_name: str, k: int, output_folder: Optional[str]=None,
                                   images_folder: str='Normalized Images', data_folder: str='Kmean Mini Batch (Standard Scaler)',
                                   from_drive: bool = True):
    """

    Args:
        slide_name: The name of the slide i.e MB16CC002
        k: the number of clusters
        output_folder: The folder to output all the files to. If left empty prints all results
        images_folder: The images folder in Google Drive
        data_folder: The data folder in Google Drive
        from_drive: whether to download the files from Google Drive

    Returns:

    """
    def file_id_by_name(cluster_data_list: list, file_name: str):
        """
        searches cluster_data_list for a file that starts with file_name.

        Args:
            cluster_data_list: a list of file dictionaries in the format {'id':XXXX, 'name':XXXX}
            file_name: the name of the file

        Returns:
            the file's id from cluster data_list
        """
        for file_dict in cluster_data_list:
            if file_name in file_dict.get('name'):
                return file_dict.get('id')
        return None

    if output_folder and not os.path.exists(output_folder):
        os.makedirs(output_folder)

    if from_drive:
        client = GoogleClient()
        slides = client.get_all_files(folder_name=images_folder, substring=slide_name)
        kmeans_data = client.get_all_files(folder_name=data_folder, substring=slide_name)

        for file_dict in tqdm(slides):
            ## If we want to download a lot, could be done in threads.
            pass

    else:
        slide = glob.glob(f'{images_folder}/*{slide_name}*')
        kmeans_data = glob.glob(f'{data_folder}/*{slide_name}*')

        ### to implement

download_and_color_whole_slide('MB16CC003', k=7, output_folder='Outputs')


#!/usr/bin/env python
# coding: utf-8

# In[1]:

from problem_statement import  *
import tqdm
import time
Vmax = 200

# In[2]:

def sigmoid(x):
    return (1 / (1 + np.exp(-x)))

# In[3]:

def tabu_search(Y0, maxiter = 2000):

    tabu_len = int(nbI1**2*0.01)

    tabu_set = []

    Y_current = Y0
    _, current_val = cost_fja(Y_current)

    best_val = current_val
    Y_best = Y_current

    k = 0
    i = 0
    while i < maxiter:

        if k > maxiter*0.2:
            Y_new = new_feasible_slower(Y_current,k = max(int(p/4),1))
        else:
            Y_new, code = new_feasible(Y_current)

        if code not in tabu_set:

            _, new_value = cost_fja(Y_new)
            i+=1
            if new_value > current_val:

                current_val = new_value
                Y_current = Y_new
                k=0
                if new_value> best_val:
                    tabu_set = []
                    best_val = new_value
                    Y_best = Y_new

                    #print(best_val)
            else:
                  k+=1
        else:

            if len(tabu_set) < tabu_len:
                tabu_set.append(code)
            else:
                tabu_set.pop(0)
                tabu_set.append(code)

    return   Y_best

# In[4]:

class Particle():

    #initiliaze particle
    def __init__(self, Y0):
        #individual parameters
        self.position_Y_i = [] #particle position // matrix Y

        self.velocity_i = [] #particle velosity
        self.err_i= -1       #error individual

        #global parameters
        self.pos_best_Y = [] #best position individual Y

        self.err_best = -1   #best error individual

        global tabu_set
        tabu_set = []

        for i in range(num_dimensions):
            #add velocity vector
            self.velocity_i.append(random.uniform(-Vmax,Vmax))

        #initialize position vectors
        self.position_Y_i = Y0.copy()

    # evaluate current fitness for a particle
    def evaluate(self,costFunction):
        _,self.err_i = costFunction(self.position_Y_i)

        if self.err_i > self.err_best or self.err_best ==-1:

            #pos_Y_ts, err_ts = tabu_search_(self.position_Y_i, tabu_set, maxiter = 3)

            #if err_ts > self.err_i:
            #   self.pos_best_Y = pos_Y_ts
            #   self.err_best = err_ts

            self.pos_best_Y = self.position_Y_i

            self.err_best = self.err_i

        #if len(tabu_set)<(100):

        #   tabu_set.append(list(self.position_Y_i))

    def update_velocity(self, pos_best_g, w = 2, c1 = 10, c2 = 10):
        # w -> constant inertia weight (how much to weigh the previous velocity)
        # c1 -> cognative constant
        # c2 -> social constant

        for i in range(num_dimensions):
            r1=random.uniform(0,1)
            r2=random.uniform(0,1)

            #cognitive influence
            vel_cognitive=c1*r1*(self.pos_best_Y[i]-self.position_Y_i[i])
            #social influence
            vel_social=c2*r2*(pos_best_g[i]-self.position_Y_i[i])
            self.velocity_i[i]=w*self.velocity_i[i] + vel_cognitive + vel_social

        # limit the velocity to be in (-Vmax,Vmax)

            self.velocity_i = np.divide(self.velocity_i, np.linalg.norm(self.velocity_i))*Vmax

    def update_position(self):

        for i in range(nbI1):

            sig_velocity = sigmoid(self.velocity_i[i])
            rand_val = random.random()

            self.position_Y_i[i] = rand_val < sig_velocity

        self.position_Y_i = make_Y_feasible(self.position_Y_i)

# In[5]:

def PSO(costFunc = cost_fja,num_particles = 30, maxiter = 100):

    global num_dimensions

    num_dimensions= nbI1 + nbI2
    err_best_g=-1                   # best error for group
    pos_best_g=[]                   # best position for group
    time_start = time.time()

    # establish the swarm
    swarm=[]
    for i in range(10):
        swarm.append(Particle(tabu_search(initialize(), maxiter = 2 * maxiter)))

    for i in range(10,num_particles):
        swarm.append(Particle(initialize()))

    # begin optimization loop

    for i in tqdm.tqdm(range(maxiter)):

        for j in range(num_particles):

            swarm[j].evaluate(costFunc)

            # determine if current particle is the best (globally)
            if swarm[j].err_i > err_best_g or err_best_g == -1:

                pos_best_g=list(swarm[j].position_Y_i)
                err_best_g=float(swarm[j].err_i)

                #err_best_g, pos_best_g = tabu_search(swarm[j].position_Y_i)

                time_best = time.time() - time_start

        # cycle through swarm and update velocities and position
        for j in range(0,num_particles):
            swarm[j].update_velocity(pos_best_g)
            swarm[j].update_position()
    print(err_best_g)
    time_end = time.time() - time_start

    return err_best_g, time_best, time_end

# In[6]:

#err_best_g, time_best, time_end = PSO(maxiter = 1000)

# In[ ]:

# In[7]:

#err_best_g

# In[8]:

#time_best

# In[9]:

#time_end

# In[ ]:


import json
import os

# Library
os.system('wget -nc -O lib/browser-polyfill.js https://unpkg.com/webextension-polyfill@0.8.0/dist/browser-polyfill.js')

# Preprocess
os.system('wget -nc https://raw.githubusercontent.com/CanCLID/ToJyutping/74f8e9c/preprocess.py')
os.system("sed -i 's/src\/ToJyutping\/jyut6ping3.simple.dict.yaml/jyut6ping3.simple.dict.yaml/' preprocess.py")
os.system('python preprocess.py')

l = []

with open('jyut6ping3.simple.dict.yaml') as f:
    for line in f:
        k, v = line.rstrip('\n').split('\t')
        l.append((k, v))

# *.json.txt: See mozilla/addons-linter#1700
with open('background_scripts/dictionary.json.txt', 'w') as f:
    f.write(json.dumps(l, ensure_ascii=False).replace('], [', '],\n['))
    f.write('\n')  # Add line break at the end of file


import pygame
import sys
from settings import *
from plain import Plain

class Game:
    def __init__(self):
        pygame.init()
        pygame.display.set_caption(TITLE)
        # screen settings
        self.screen = pygame.display.set_mode((WIDTH, HEIGHT))
        self.background = pygame.Surface((WIDTH, HEIGHT))
        self.background.fill(pygame.Color('#000000'))
        self.clock = pygame.time.Clock()
        # plain initialization
        self.plain = Plain(self.clock)

    def run(self):
        while True:
            for event in pygame.event.get():
                # quit game
                if event.type == pygame.QUIT:
                    pygame.quit()
                    sys.exit()

            self.clock.tick(FPS)
            self.background.fill(pygame.Color('#000000'))
            self.plain.update()
            pygame.display.update()
            self.screen.blit(self.background, (0, 0))

if __name__ == '__main__':
    game = Game()
    game.run()


import torch
import torch.nn as nn
import torch.nn.functional as F

device = torch.device("cuda" if torch.cuda.is_available() else "cpu")

class FirstVertex(nn.Module):
    def __init__(self, opts, feats_dim, feats_channels):
        super(FirstVertex, self).__init__()
        self.grid_size = feats_dim
        self.opts = opts

        self.edge_conv = nn.Conv2d(
            in_channels = feats_channels,
            out_channels = 16,
            kernel_size = 3,
            padding = 1
        )

        self.edge_fc = nn.Linear(
            in_features = feats_dim**2 * 16,
            out_features = feats_dim**2
        )

        self.vertex_conv = nn.Conv2d(
            in_channels = feats_channels,
            out_channels = 16,
            kernel_size = 3,
            padding = 1
        )

        self.vertex_fc = nn.Linear(
            in_features = feats_dim**2 * 16,
            out_features = feats_dim**2
        )

    def forward(self, feats, temperature=0.0, beam_size=1):
        """
        if temperature < 0.01, use greedy
        else, use temperature
        """
        batch_size = feats.size(0)
        conv_edge = self.edge_conv(feats)
        conv_edge = F.relu(conv_edge, inplace=True)
        edge_logits = self.edge_fc(conv_edge.view(batch_size, -1))

        # Different from before, this used to take conv_edge as input before
        conv_vertex = self.vertex_conv(feats)
        conv_vertex = F.relu(conv_vertex)
        vertex_logits = self.vertex_fc(conv_vertex.view(batch_size, -1))
        logprobs = F.log_softmax(vertex_logits, -1)

        # Sample a first vertex
        if temperature < 0.01:
            logprob, pred_first = torch.topk(logprobs, beam_size, dim=-1)

        else:
            probs = torch.exp(logprobs/temperature)
            pred_first = torch.multinomial(probs, beam_size)

            # Get logprob of the sampled vertex
            logprob = logprobs.gather(1, pred_first)

        # Remove the last dimension if it is 1
        pred_first = torch.squeeze(pred_first, dim=-1)
        logprob = torch.squeeze(logprob, dim=-1)

        return edge_logits, vertex_logits, logprob, pred_first

    def first_point_correction(self, first_v, poly_class, thresh):
        first_v = first_v.unsqueeze(-1)
        poly_class = poly_class.to(torch.int64)

        x = first_v % self.grid_size
        y = first_v / self.grid_size
        # Each of shape batch_size, 1

        x_gt = poly_class % self.grid_size
        y_gt = poly_class / self.grid_size
        invalid = torch.eq(poly_class, self.grid_size**2)
        valid = torch.ne(poly_class, self.grid_size**2)
        # Each of shape batch_size, max_time_steps

        lengths = torch.sum(valid, dim=-1)

        dist = torch.abs(x_gt - x) + torch.abs(y_gt - y)
        # Manhattan Distance

        dist.masked_fill_(invalid, 10*self.grid_size)
        val, idx = torch.min(dist, dim=-1)

        new_poly_class = torch.zeros_like(poly_class, device=device)
        new_poly_class += self.grid_size**2
        # Fill with EOS

        for b in range(first_v.size(0)):
            # Across batches
            pos = idx[b]
            last_idx = lengths[b]
            if last_idx != self.opts['max_poly_len']:
                last_idx = last_idx - 1
                # Because we synthetically add the first
                # point at the end unless it's of max length

            new_poly_class[b, :last_idx] =\
            torch.cat((poly_class[b, pos:last_idx], poly_class[b, :pos]))

            if val[b] <= thresh:
                # No correction for first_v
                new_poly_class[b, 0] = first_v[b]

            if last_idx != self.opts['max_poly_len']:
                # Add first point to the end
                new_poly_class[b, last_idx] = new_poly_class[b, 0]

        return new_poly_class


import pygame, threading
from MainEngine import EngineUtils
from MainEngine import Types
import Run
import time

class PregameSettings():
    def __init__(self, engine):
        self.engine = engine
    def SetScreenDimensions(self, dimensions: pygame.math.Vector2):
        self.engine._Globals.screen = pygame.display.set_mode(dimensions)
        self.engine._Globals.display = dimensions
    def SetMaxFramerate(self, rate):
        self.engine._Globals.maxFramerate = rate
class Engine():
    def __init__(self, initialStart=True):
        if initialStart is True:
            pygame.init()
            pygame.mixer.init()
        self._Globals: self.Globals = self.Globals() #A local "private" instantiation of Globals class

    @property
    def timeScale(self):
        return self._Globals.timeScale
    @timeScale.setter
    def timeScale(self, value):
        if (type(value) == float) or (type(value) == int):
            self._Globals.timeScale = value
        else:
            print("Can only set timeScale to a numeric value.")

    def Start(self, main: Run.Main):
        self._Globals.clock = pygame.time.Clock()
        self.Input = EngineUtils.InputHandler(self)
        self.Collisions = EngineUtils.Collision(self)
        main._POSTSTART()
        self.mainReference = main
        while True:
            self.FrameEvents()

    def LoadMusic(self, music): #Use this class to load in music from a path
        pygame.mixer.music.load(music)

    def UnloadMusic(self): #Unloads the current loaded music
        pygame.mixer.music.unload()

    def ChangeMusicVolume(self, volume):
        pygame.mixer.music.set_volume(volume)

    def PlayMusic(self, volume=1, interrupt=True): #Only one music track can be playing at a time
        if self.GetUniversal("music_enabled") is False:
            volume = 0
        if interrupt is False:
            if pygame.mixer.music.get_busy() is True:
                return
        pygame.mixer.music.set_volume(volume)
        pygame.mixer.music.play()

    def PauseMusic(self):
        if pygame.mixer.music.get_busy() is True:
            pygame.mixer.music.pause()

    def ResumeMusic(self):
        pygame.mixer.music.unpause()

    def StopMusic(self):
        if pygame.mixer.music.get_busy() is True:
            pygame.mixer.music.stop()

    def PlaySound(self, sound): #Plays a sound. Multiple sounds can be played at once whereas music can only play one track concurrently
        if self.GetUniversal("sfx_enabled") == False:
            return
        pygame.mixer.Sound(sound).play()

    def SetUniversal(self, key: str, value):
        self._Globals.Universals[key] = value

    def GetUniversal(self, key: str):
        return self._Globals.Universals[key]

    def FindObject(self, name: str):
        for subscriber in self._Globals.sceneObjectsArray:
            if subscriber.gameObject.name == name:
                return subscriber
        return None

    def FrameEvents(self): #Runs events that are scheduled to run on this frame
        timeStamp1 = time.time()
        self._Globals.clock.tick(self._Globals.maxFramerate)
        self._Globals.lastRunTime = self._Globals.currentRunTime
        self._Globals.currentRunTime = pygame.time.get_ticks()

        self._Globals._totalTime += self.GetDeltaTime()
        self._PostEventsToInput()

        self._UpdateSubscribers() #Tell every GameObject to call their Update function.
        if (self.Input.TestFor.QUIT()):
            self.Quit()
        self.Input.clearEvents()
        self.Render() #Call a render update
        timeStamp2 = time.time()
        if (1/(timeStamp2 - timeStamp1)) > self._Globals.maxFramerate:
            pygame.time.delay(round((1/self._Globals.maxFramerate) - (timeStamp2 - timeStamp1)))

    def CreateNewObject(self, _object): #Adds an instantiated object to the scene to be rendered. Intended to be called from inside behaviors.
        try:
            self._Globals.sceneObjectsArray.append(_object)
            _object.obj.Start()
        except AttributeError:
            pass

    def _PostEventsToInput(self):
        self.Input.postEvents(pygame.event.get())

    def _UpdateSubscribers(self):
        for subscriber in self._Globals.sceneObjectsArray:
            try:
                subscriber.obj.Update()
            except AttributeError:
                pass

    def GetDeltaTime(self): #Returns the seconds from the last frame adjusted to the timeScale.
        return (self.GetDeltaTimeRAW() * self._Globals.timeScale)

    def GetDeltaTimeRAW(self): #Returns the raw seconds from the last frame.
        timeRaw = ((self._Globals.currentRunTime - self._Globals.lastRunTime)) / 1000
        timeRaw = timeRaw if timeRaw > 1/self._Globals.maxFramerate else 1/self._Globals.maxFramerate
        return timeRaw

    def GetTotalTime(self): #Returns the total amount of time since the engine was created.
        return self._Globals._totalTime

    def CalculateRenderOrder(self):
        array = []
        linkedObjArray = []
        sOA_copy = self._Globals.sceneObjectsArray.copy()
        if (len(sOA_copy) != 0):
            for obj in sOA_copy:
                if (obj.gameObject.renderEnabled):
                    array.append(obj.gameObject.position.z) #Add z-component of position to sorting array
                    linkedObjArray.append(obj) #Add behavior to sorting array to be sorted by z-layer
            n = len(array)
            EngineUtils.MathTools.QuickSort.LinkedObject.QuickSort(array, linkedObjArray, 0, n-1) #Use our linked object quicksort algorithm
            return (linkedObjArray, array)
        del array, linkedObjArray, sOA_copy
        return None

    def Quit(self): #Used to exit the program at the engine level
        quit()

    def Reload(self): #Restarts the program without actually closing and re-opening.
        copyOfSceneObjs = self._Globals.sceneObjectsArray.copy()
        for item in copyOfSceneObjs:
            try:
                item.gameObject.sprite.kill()
            except Exception:
                pass

            try:
                self._Globals.sceneObjectsArray.remove(item)
                item.obj.Destroy()
            except Exception:
                pass
            del item
        del self.Input
        del self.Collisions
        del copyOfSceneObjs
        del self._Globals
        self.mainReference.Reload()

    def Render(self): #PLEASE NOTE: EVERYTHING will be rendered unless specified otherwise by the behaviors GameObject. Even if it's not actually on the screen.
        self._Globals._screen.fill((0,0,0)) #Background, can remove.
        renderOrderReturnVal = self.CalculateRenderOrder()
        if (renderOrderReturnVal == None):
            return
        array = renderOrderReturnVal[0]
        for i in array:
            self._Globals.screen.blit(i.gameObject.sprite.image, (i.gameObject.position.x + i.gameObject._offset.x, i.gameObject.position.y + i.gameObject._offset.y))
        pygame.display.flip()

    def AddImageAsset(self, key: str, value: str or list, transparency=True): #Loads an image into memory and makes it easier to access images just by their specified key.
        if type(value) == str:
            self._Globals.Assets[key] = pygame.image.load(value).convert_alpha() if transparency else pygame.image.load(value)
        elif type(value) == list:
            self._Globals.Assets[key] = [(img.convert_alpha() if transparency else img) for img in value]
        elif type(value) == pygame.Surface:
            self._Globals.Assets[key] = value.convert_alpha() if transparency else value
        else:
            print("Can only import lists of paths or paths.")

    def GetImageAsset(self, key: str or pygame.Surface) -> pygame.Surface: #Grabs a loaded image from memory using the specified key.
        try:
            return self._Globals.Assets[key]
        except KeyError:
            return None

    def AddAnimation(self, key: str, sequence: list, framerate: float, loop=True): #Same as AddImageAsset but for animations
        self._Globals.Animations[key] = Types.Animation(sequence, framerate, loop)

    def GetAnimation(self, key: str) -> Types.Animation: #Same as GetImageAsset but for animations
        try:
            return self._Globals.Animations[key]
        except KeyError:
            return None

    def SetCaption(self, value: str): #Sets the caption of the window
        pygame.display.set_caption(value)

    def SetIcon(self, value: str): #Sets the icon of the window. NOTE: icon must be the path of the image.
        pygame.display.set_icon(pygame.image.load(value))

    class Globals():
        _totalTime = 0
        Assets = {}
        Animations = {}
        sceneObjectsArray = []
        _display = (512, 512)
        clock = None
        lastRunTime = pygame.time.get_ticks()
        currentRunTime = pygame.time.get_ticks()
        Universals = {}
        maxFramerate = 60

        _screen = pygame.display.set_mode(_display)
        timeScale = 1
        @property
        def display(self):
            return self._display
        @display.setter
        def display(self, value):
            self._display = value
            self._screen = pygame.display.set_mode(self._display)


import numpy as np
import pandas as pd
import pickle
import matplotlib.pyplot as plt
from PIL import Image
import sys
import cv2

from keras import  backend as K
from keras.datasets import mnist
from keras.models import Sequential
from keras.layers import Dense, Activation, Dropout
from tensorflow.keras.utils import to_categorical

(x_train, y_train),(x_test, y_test) = mnist.load_data()
unique, counts = np.unique(y_train, return_counts=True)
print("Train labels: ", dict(zip(unique, counts)))

unique, counts = np.unique(y_test, return_counts=True)
print("\nTest labels: ", dict(zip(unique, counts)))

num_labels = len(np.unique(y_train))
y_train = to_categorical(y_train)
y_test = to_categorical(y_test)

image_size = x_train.shape[1]
input_size = image_size * image_size

x_train = np.reshape(x_train, [-1, input_size])
x_train = x_train.astype('float32') / 255
x_test = np.reshape(x_test, [-1, input_size])
x_test = x_test.astype('float32') / 255

batch_size = 128
hidden_units = 256
dropout = 0.45

model = Sequential()
model.add(Dense(hidden_units, input_dim=input_size))
model.add(Activation('relu'))
model.add(Dropout(dropout))
model.add(Dense(hidden_units))
model.add(Activation('relu'))
model.add(Dropout(dropout))
model.add(Dense(num_labels))
model.add(Activation('softmax'))

#model.summary()

model.compile(loss='categorical_crossentropy',
              optimizer='adam',
              metrics=['accuracy'])

model.fit(x_train, y_train, epochs=20, batch_size=batch_size)

loss, acc = model.evaluate(x_test, y_test, batch_size=batch_size)
print("\nTest accuracy: %.1f%%" % (100.0 * acc))
filename = 'finalized_model'
model.save(filename)
#pickle.dump(model, open(filename, 'wb'))