import numpy as np
import torch
import librosa
import librosa.display
import pandas as pd
from IPython.display import Audio
import seaborn as sns
import os
import librosa
import librosa.display

from scipy.signal import convolve, fftconvolve

from collections import defaultdict

import tqdm
import matplotlib.pyplot as plt
from sklearn.metrics import label_ranking_average_precision_score
from sklearn.model_selection import train_test_split

min_db = -100
PREEMPHASIS = 0.97
ref_db = 20

# алгоритм Гриффин-Лим

def griffin_lim(S, num_iter=20):
    np.random.seed(42)
    # реализуйте алгоритм для добавления к вещественной матрице S фазы
    # на выходе нужно вернуть комплексную матрицу
    # ...

    phi = 2 * np.pi * np.random.rand()
    H = istft(S * np.exp(1j * phi))

    for iteration in range(num_iter):
        phi = np.angle(stft(H))
        H = istft(S * np.exp(1j * phi))

    return stft(H)
#     return H

def load(config, path):
    signal, _ = librosa.load(path, sr=config.sampling_rate)
    return signal

def show(audio):
    return Audio(audio, rate=sample_rate)

def mel_to_linear(M):
    return np.dot(inv_mel_basis, M)

def inv_normalize(S):
    return S * -min_db + min_db

def db_to_amp(x):
    return 10**(x/20)

def de_emphasis(x):
    # Это эквивалентно scipy.signal.lfilter([1], [1, PREEMPHASIS], x)
    return scipy.signal.lfilter([1, PREEMPHASIS], [1, 0, -PREEMPHASIS**2], x)# ...

def inv_melspectrogram(M):
    M = inv_normalize(M)
    M = mel_to_linear(db_to_amp(M + ref_db))
    M = istft(griffin_lim(M))
    x = de_emphasis(M)
    return x

def read_audio(conf, pathname, trim_long_data):
    y, sr = librosa.load(pathname, sr=conf.sampling_rate)
    # trim silence
    if 0 < len(y): # workaround: 0 length causes error
        y, _ = librosa.effects.trim(y) # trim, top_db=default(60)

    # return y
    # make it unified length to conf.samples
    if len(y) > conf.samples: # long enough
        if trim_long_data:
#             y = y[0:0+123121523]
            # print(conf.samples, len(y))
            y = y[0:0+conf.samples]
    else: # pad blank
        padding = conf.samples - len(y)    # add padding at both ends
        offset = padding // 2
        y = np.pad(y, (offset, conf.samples - len(y) - offset), 'constant')
    return y

def trim_and_mel(conf, audio):
    if len(audio) > conf.samples:
        audio = audio[:conf.samples]

    result = audio_to_melspectrogram(conf, audio)
    return result

def audio_to_melspectrogram(conf, audio):
    spectrogram = librosa.feature.melspectrogram(audio,
                                                 sr=conf.sampling_rate,
                                                 n_mels=conf.n_mels,
                                                 hop_length=conf.hop_length,
                                                 n_fft=conf.n_fft,
                                                 fmin=conf.fmin,
                                                 fmax=conf.fmax)
    spectrogram = librosa.power_to_db(spectrogram)
    spectrogram = spectrogram.astype(np.float32)
    return spectrogram

def show_melspectrogram(conf, mels, title='Log-frequency power spectrogram'):
    librosa.display.specshow(mels, x_axis='time', y_axis='mel',
                             sr=conf.sampling_rate, hop_length=conf.hop_length,
                            fmin=conf.fmin, fmax=conf.fmax)
    plt.colorbar(format='%+2.0f dB')
    plt.title(title)
    plt.show()

def read_as_melspectrogram(conf, pathname, trim_long_data, debug_display=False):
    x = read_audio(conf, pathname, trim_long_data)
    mels = audio_to_melspectrogram(conf, x)
    if debug_display:
        IPython.display.display(IPython.display.Audio(x, rate=conf.sampling_rate))
        show_melspectrogram(conf, mels)
    return mels

def read_columns():
    columns = pd.read_csv("./data/sample_submission.csv").columns[1:]
    column_encoder = {c:id for id, c in enumerate(columns)}

    return columns, column_encoder

def read_dataset(path, config, transform):
    meta = pd.read_csv(path + ".csv")
    all_data = []
    all_labels = []

    max_len = 0

    for id, (f, label) in tqdm.tqdm(enumerate(zip(meta.fname, meta.labels)), total=len(meta)):
        all_labels.append(label)
        all_data.append(transform(read_audio(config, os.path.join(path, f), trim_long_data=config.trim_long_data)))
        max_len = max(all_data[-1].shape[-1], max_len)

    y = np.zeros((len(all_data), len(config.columns)))
    for id, label in enumerate(all_labels):
        labels = label.split(",")
        for l in labels:
            y[id, config.column_encoder[l]] = 1.0

    return all_data, y, max_len

def read_dataset_split(metapath, path, config, transform):
    labels = pd.read_csv(metapath)
    labels = labels.set_index("fname")
    meta = list(os.listdir(path))
    meta.sort()
    all_data = []
    all_labels = []

    max_len = 0

    for id, f in tqdm.tqdm(enumerate(meta), total=len(meta)):
        label = labels.loc[f]['labels']
        all_labels.append(label)
        all_data.append(transform(read_audio(config, os.path.join(path, f), trim_long_data=config.trim_long_data)))
        max_len = max(all_data[-1].shape[-1], max_len)

    y = np.zeros((len(all_data), len(config.columns)))
    for id, label in enumerate(all_labels):
        labels = label.split(",")
        for l in labels:
            y[id, config.column_encoder[l]] = 1.0

    return all_data, y, max_len

def read_test(path, config, transform):
    all_data = []
    max_len = 0
    meta = list(os.listdir(path))
    meta.sort()

    for id, f in tqdm.tqdm(enumerate(meta), total=len(meta)):
        all_data.append(transform(read_audio(config, os.path.join(path, f), trim_long_data=config.trim_long_data)))
        max_len = max(all_data[-1].shape[-1], max_len)

    y = np.zeros((len(all_data), len(columns)))

    return meta, all_data, y, max_len

def read_filters(path, config, transform):
    all_data = []
    max_len = 0
    meta = list(os.listdir(path))
    meta.sort()

    for id, f in tqdm.tqdm(enumerate(meta), total=len(meta)):
        pathname = os.path.join(path, f)
        audio, sr = librosa.load(pathname, sr=config.sampling_rate)
        all_data.append(transform(audio))

    return meta, all_data

def apply_filter(sound, filter):
    modified_sound = fftconvolve(in2=filter, in1=sound) #/ np.linalg.norm(sound)
    return modified_sound

def predict(trainer, meta, loader):
    prediction = trainer.predict(loader)
    result = {col:prediction[:, id] for id, col in enumerate(columns)}
    result['fname'] = meta
    result = pd.DataFrame(result)
    result = result.set_index('fname').reset_index()

    return result

sampling_rate = 44100 # от оргов контеста
duration = 2 # сколько секунд от записи хотим взять
n_samples = sampling_rate * duration
n_mels=128 # дефолтный параметр в самой либросе

def audio2mel(audio):
    #audio, _ = librosa.effects.trim(audio) # убирает участки тишины из audio, почти не влияет, да и хз, может вредит
    melspectr = librosa.feature.melspectrogram(
        y=audio,
        sr=sampling_rate,
        n_mels=n_mels, # количество мел-признаков, дефолтное значение для librosa
        n_fft=n_mels * 20, # не знаю, почему так, но это ширина окна для FFT (размер базиса, получается?)
        hop_length=346 * duration, # длина фрейма, n_samples // 128, чтобы получить 128 фреймов (или что-то такое)
        # ось времени таки есть!
        fmax=sampling_rate // 2, # максимальная частота сигнала, дефолтное значение для librosa,
        fmin = 0., # дефолтная наименьшая частота сигнала
    )
    melspectr = librosa.power_to_db(melspectr).astype(np.float32) # перевод в дб
    return melspectr

def read_dataset_kirill(path, config, transform):
    meta = pd.read_csv(path + ".csv")
    all_data = []
    all_labels = []

    max_len = 0

    for id, (f, label) in tqdm.tqdm(enumerate(zip(meta.fname, meta.labels)), total=len(meta)):
        all_labels.append(label)
        fp = os.path.join(path, f)
        y, _ = librosa.load(fp, sr=sampling_rate) # загрузка аудиофайла
        # в кернелах они делают паддинг нулями с двух сторон, я делаю паддинг нулями в начале записи
        y = np.concatenate([[0] * (n_samples - len(y)), y[:n_samples]]) # паддинг
        mel = audio2mel(y)
        all_data.append(mel)
        max_len = max(all_data[-1].shape[-1], max_len)

    y = np.zeros((len(all_data), len(config.columns)))
    for id, label in enumerate(all_labels):
        labels = label.split(",")
        for l in labels:
            y[id, config.column_encoder[l]] = 1.0

    return all_data, y, max_len

def read_dataset_kirill_split(metapath, path, config, transform):
    labels = pd.read_csv(metapath)
    labels = labels.set_index("fname")
    meta = list(os.listdir(path))
    meta.sort()
    all_data = []
    all_labels = []

    max_len = 0

    for id, f in tqdm.tqdm(enumerate(meta), total=len(meta)):
        label = labels.loc[f]['labels']
        all_labels.append(label)
        fp = os.path.join(path, f)
        y, _ = librosa.load(fp, sr=sampling_rate) # загрузка аудиофайла
        # в кернелах они делают паддинг нулями с двух сторон, я делаю паддинг нулями в начале записи
        y = np.concatenate([[0] * (n_samples - len(y)), y[:n_samples]]) # паддинг
        mel = audio2mel(y)
        all_data.append(mel)
        max_len = max(all_data[-1].shape[-1], max_len)

    y = np.zeros((len(all_data), len(config.columns)))
    for id, label in enumerate(all_labels):
        labels = label.split(",")
        for l in labels:
            y[id, config.column_encoder[l]] = 1.0

    return all_data, y, max_len


import unittest

import numpy as np
import sympy
from sympy import symbols, Matrix, Eq, sqrt, FiniteSet, Derivative

from scipy.optimize import root
from scipy.stats import poisson, entropy

from epipack import (
            SymbolicEpiModel,
            SymbolicODEModel,
            SymbolicSISModel,
            SymbolicSIRModel,
            SymbolicSIRSModel,
            SymbolicSIModel,
            get_temporal_interpolation,
            MatrixSISModel,
            MatrixEpiModel,
            SymbolicMatrixEpiModel,
        )

class SymbolicEpiTest(unittest.TestCase):

    def test_compartments(self):
        comps = sympy.symbols("S E I R")
        epi = SymbolicEpiModel(comps)
        assert(all([ i == epi.get_compartment_id(C) for i, C in enumerate(comps) ]))

    def test_linear_rates(self):

        S, E, I, R = sympy.symbols("S E I R")
        epi = SymbolicEpiModel([S,E,I,R])
        epi.add_transition_processes([
                (E, 1, I),
                (I, 1, R),
                ])
        linear_rates = [ E, I ]
        linear_events = [ Matrix([[0,-1,+1,0]]), Matrix([[0,0,-1,+1]]) ]
        for r0, r1 in zip(linear_rates, epi.linear_rate_functions):
            assert(r0 == r1)
        for e0, e1 in zip(linear_events, epi.linear_event_updates):
            assert(all([_e0==_e1 for _e0, _e1 in zip(e0, e1)]))

    def test_adding_linear_rates(self):

        S, E, I, R = sympy.symbols("S E I R")
        epi = SymbolicEpiModel([S,E,I,R])

        epi.set_processes([
                (E, 1, I),
                ])

        epi.add_transition_processes([
                (I, 1, R),
                ])
        linear_rates = [ E, I ]
        linear_events = [ Matrix([[0,-1,+1,0]]), Matrix([[0,0,-1,+1]]) ]
        for r0, r1 in zip(linear_rates, epi.linear_rate_functions):
            assert(r0 == r1)
        for e0, e1 in zip(linear_events, epi.linear_event_updates):
            assert(all([_e0==_e1 for _e0, _e1 in zip(e0, e1)]))

    def test_quadratic_processes(self):

        S, E, I, R = sympy.symbols("S E I R")
        epi = SymbolicEpiModel([S,E,I,R])
        quadratic_rates = [ I*S ]
        quadratic_events = [ Matrix([[-1,+1,0,0]]) ]
        epi.add_transmission_processes([
                (S, I, 1, I, E),
            ])
        for r0, r1 in zip(quadratic_rates, epi.quadratic_rate_functions):
            assert(r0 == r1)
        for e0, e1 in zip(quadratic_events, epi.quadratic_event_updates):
            assert(all([_e0==_e1 for _e0, _e1 in zip(e0, e1)]))

    def test_adding_quadratic_processes(self):

        S, E, I, A, R, rho = sympy.symbols("S E I A R rho")
        epi = SymbolicEpiModel([S,E,I,A,R])

        epi.set_processes([
                (S, I, rho, I, E),
                ])
        epi.add_transmission_processes([
                (S, A, rho, A, E),
                ])
        quadratic_rates = [ I*S*rho, S*A*rho ]
        quadratic_events = [ Matrix([[-1,+1,0,0,0]]), Matrix([[-1,+1,0,0,0]]) ]
        for r0, r1 in zip(quadratic_rates, epi.quadratic_rate_functions):
            assert(r0 == r1)
        for e0, e1 in zip(quadratic_events, epi.quadratic_event_updates):
            assert(all([_e0==_e1 for _e0, _e1 in zip(e0, e1)]))

    def test_SIS_with_simulation_restart_and_euler(self):
        S, I = sympy.symbols("S I")
        N = 100
        epi = SymbolicSISModel(infection_rate=2,recovery_rate=1,initial_population_size=N)
        epi.set_initial_conditions({S:0.99*N,I:0.01*N})
        tt = np.linspace(0,100,2)
        result = epi.integrate(tt,[S])
        assert(np.isclose(result[S][-1],N/2))

        tt = np.linspace(0,100,1000)
        result = epi.integrate_and_return_by_index(tt,[S],integrator='euler')
        assert(np.isclose(result[0,-1],N/2))

    def test_basic_analytics(self):
        S, I, R, eta, rho, omega, t = symbols("S I R eta rho omega, t")

        SIRS = SymbolicEpiModel([S,I,R])

        SIRS.set_processes([
            #### transmission process ####
            # S + I (eta)-> I + I
            (S, I, eta, I, I),

            #### transition processes ####
            # I (rho)-> R
            # R (omega)-> S
            (I, rho, R),
            (R, omega, S),

        ])

        odes = SIRS.ODEs()

        expected = [Eq(Derivative(S, t), -I*S*eta + R*omega), Eq(Derivative(I, t), I*(S*eta - rho)), Eq(Derivative(R, t), I*rho - R*omega)]

        assert(all([got==exp for got, exp in zip(odes,expected) ]))

        fixed_points = SIRS.find_fixed_points()
        expected = FiniteSet((S, 0, 0), (rho/eta, R*omega/rho, R))

        assert(all([got==exp for got, exp in zip(fixed_points,expected) ]))

        J = SIRS.jacobian()
        expected = Matrix([[-I*eta, -S*eta, omega], [I*eta, S*eta - rho, 0], [0, rho, -omega]])
        N = SIRS.N_comp
        assert(all([J[i,j]==expected[i,j] for i in range(N) for j in range(N)]))

        eig = SIRS.get_eigenvalues_at_disease_free_state()
        expected = {-omega: 1, eta - rho: 1, 0: 1}

        assert(all([v == expected[k] for k, v in eig.items()]))

    def test_numerical(self):

        S, I, eta, rho = sympy.symbols("S I eta rho")

        epi = SymbolicSISModel(eta, rho)

        epi.set_initial_conditions({S: 1-0.01, I:0.01 })
        epi.set_parameter_values({eta:2})

        self.assertRaises(ValueError, epi.integrate,
                    [0,1]
                    )

        epi = SymbolicSISModel(eta, rho)

        epi.set_initial_conditions({S: 1-0.01, I:0.01 })
        epi.set_parameter_values({eta:2,rho:1})

        tt = np.linspace(0,10,1000)
        result = epi.integrate(tt)

        epi2 = MatrixSISModel(2, 1)

        epi2.set_initial_conditions({"S": 1-0.01, "I":0.01 })

        tt = np.linspace(0,10,1000)
        result2 = epi2.integrate(tt)

        for c0, res0 in result.items():
            assert(np.allclose(res0, result2[str(c0)]))

    def test_time_dependent_rates(self):

        B, t = sympy.symbols("B t")
        epi = SymbolicEpiModel([B])
        epi.add_fission_processes([
                (B, t, B, B),
            ])
        epi.set_initial_conditions({B:1})
        result = epi.integrate([0,3],adopt_final_state=True)
        assert(np.isclose(epi.y0[0], np.exp(3**2/2)))
        epi.set_initial_conditions({B:1})
        result = epi.integrate(np.linspace(0,3,10000),integrator='euler',adopt_final_state=True)
        eul = epi.y0[0]
        real = np.exp(3**2/2)
        assert(np.abs(1-eul/real)<1e-2)

    def test_exceptions(self):

        B, mu, t = sympy.symbols("B mu t")
        epi = SymbolicEpiModel([B])
        epi.add_fission_processes([
                (B, mu, B, B),
            ])
        epi.set_initial_conditions({B:1})

        self.assertRaises(ValueError,epi.integrate,[0,1])

        self.assertRaises(ValueError,SymbolicEpiModel,[t])

        self.assertRaises(ValueError,epi.get_eigenvalues_at_disease_free_state)

    def test_custom_models(self):
        eta = sympy.symbols("eta")
        epi = SymbolicSIModel(eta)
        epi.set_parameter_values({eta:1})
        epi.set_initial_conditions({epi.compartments[0]:0.99, epi.compartments[1]:0.01})
        epi.integrate([0,1000],adopt_final_state=True)
        assert(np.isclose(epi.y0[0],0))

        S, I, R, eta, rho = sympy.symbols("S I R eta rho")
        epi = SymbolicSIRModel(eta,rho)
        epi.set_parameter_values({eta:2,rho:1})
        S0 = 0.99
        epi.set_initial_conditions({S:S0, I:1-S0})
        R0 = 2
        Rinf = lambda x: 1-x-S0*np.exp(-x*R0)
        res = epi.integrate([0,1000])

        theory = root(Rinf,0.5)
        assert(np.isclose(res[R][-1],theory.x[0]))

        epi = SymbolicSISModel(eta, rho, initial_population_size=100)

        epi.set_initial_conditions({S: 99, I:1 })
        epi.set_parameter_values({eta:2,rho:1})

        tt = np.linspace(0,1000,2)
        result = epi.integrate(tt)
        assert(np.isclose(result[S][-1],50))

        S, I, R, eta, rho, omega = sympy.symbols("S I R eta rho omega")
        epi = SymbolicSIRSModel(eta, rho, omega)
        _eta = 2
        _rho = 1
        _omega = 1

        epi.set_initial_conditions({S: 0.99, I:0.01 })
        epi.set_parameter_values({eta:_eta, rho:_rho, omega:_omega})

        tt = np.linspace(0,1000,2)
        result = epi.integrate(tt)
        assert(np.isclose(result[R][-1],(1-_rho/_eta)/(1+_omega/_rho)))

    def test_ODEs_jupyter(self):

        eta = sympy.symbols("eta")
        epi = SymbolicSIModel(eta)
        epi.ODEs_jupyter()

    def test_temporal_interpolation(self):

        self.assertRaises(ValueError, get_temporal_interpolation, [0,1],[0,1],interpolation_degree=0)

        t = sympy.symbols("t")
        p = sympy.Piecewise((1, (0 <= t) & (t < 1)), (2,(1<=t) & (t<2)))
        p2 = get_temporal_interpolation([0,1,2],[1,2],0)
        assert(p.subs(t,0.5)==p2.subs(t,0.5))
        assert(p.subs(t,1.5)==p2.subs(t,1.5))
        assert(str(p) == str(p2))

        p = sympy.interpolating_spline(1,t,[0,1,2],[1,2,3])
        p2 = get_temporal_interpolation([0,1,2],[1,2,3],1)
        assert(p.subs(t,0.5)==p2.subs(t,0.5))
        assert(p.subs(t,1.5)==p2.subs(t,1.5))
        assert(str(p) == str(p2))

    def test_changing_population_size(self):

        A, B, C, t = sympy.symbols("A B C t")
        epi = SymbolicEpiModel([A,B,C],10,correct_for_dynamical_population_size=True)
        epi.set_initial_conditions({A:5, B:5})
        epi.set_processes([
                (A, B, 1, C),
            ],allow_nonzero_column_sums=True)

        dydt = epi.dydt()
        assert(dydt[0] == -1*A*B/(A + B + C))

        _, res = epi.simulate(1e9)
        assert(res[C][-1] == 5)

        epi.set_processes([
                (None, 1+sympy.log(1+t), A),
                (A, 1+sympy.log(1+t), B),
                (B, 1+sympy.log(1+t), None),
            ],allow_nonzero_column_sums=True)

        rates, comp_changes = epi.get_numerical_event_and_rate_functions()
        _, res = epi.simulate(200,sampling_dt=0.05)

        vals = np.concatenate([res[A][_>10], res[B][_>10]])
        rv = poisson(vals.mean())
        measured, bins = np.histogram(vals,bins=np.arange(10)-0.5,density=True)
        theory = [ rv.pmf(i) for i in range(0,len(bins)-1) if measured[i] > 0]
        experi = [ measured[i] for i in range(0,len(bins)-1) if measured[i] > 0]
        # make sure the kullback-leibler divergence is below some threshold
        assert(entropy(theory, experi) < 2e-3)
        assert(np.median(res[A]) == 1)

    def test_stochastic_well_mixed(self):

        S, E, I, R = sympy.symbols("S E I R")

        N = 75000
        tmax = 100
        model = SymbolicEpiModel([S,E,I,R],N)
        model.set_processes([
                ( S, I, 2, E, I ),
                ( I, 1, R),
                ( E, 1, I),
            ])
        model.set_initial_conditions({S: N-100, I: 100})

        tt = np.linspace(0,tmax,10)
        result_int = model.integrate(tt)

        t, result_sim = model.simulate(tmax,sampling_dt=1,return_compartments=[S, R])

        for c, res in result_sim.items():
            #print(c, np.abs(1-res[-1]/result_int[c][-1]))
            #print(c, np.abs(1-res[-1]/result_sim[c][-1]))
            assert(np.abs(1-res[-1]/result_int[c][-1]) < 0.05)

    def test_ODE_model(self):

        eta, rho = sympy.symbols("eta rho")

        orig_model = SymbolicSISModel(eta, rho)

        ode_model = SymbolicODEModel(orig_model.ODEs())

        assert(all([eq0 == eq1 for eq0, eq1 in zip(orig_model.ODEs(), ode_model.ODEs())]))

        self.assertRaises(AttributeError, ode_model.simulate)
        self.assertRaises(AttributeError, ode_model.set_linear_events)
        self.assertRaises(AttributeError, ode_model.set_quadratic_events)
        self.assertRaises(AttributeError, ode_model.set_processes)
        self.assertRaises(AttributeError, ode_model.add_linear_events)
        self.assertRaises(AttributeError, ode_model.add_quadratic_events)
        self.assertRaises(AttributeError, ode_model.add_transition_processes)
        self.assertRaises(AttributeError, ode_model.add_fission_processes)
        self.assertRaises(AttributeError, ode_model.add_fusion_processes)
        self.assertRaises(AttributeError, ode_model.add_transmission_processes)

if __name__ == "__main__":

    T = SymbolicEpiTest()
    T.test_ODE_model()
    sys.exit(0)
    T.test_changing_population_size()
    T.test_stochastic_well_mixed()
    T.test_SIS_with_simulation_restart_and_euler()
    T.test_compartments()
    T.test_linear_rates()
    T.test_adding_linear_rates()
    T.test_quadratic_processes()
    T.test_adding_quadratic_processes()
    T.test_basic_analytics()
    T.test_numerical()
    T.test_time_dependent_rates()
    T.test_exceptions()
    T.test_custom_models()
    T.test_ODEs_jupyter()
    T.test_temporal_interpolation()


from apiclient.discovery import build
from httplib2 import Http
from oauth2client import file, client, tools
import os

"""
Convenience class wrapper for Google Drive OAuth.

This requires that the user download credentials.json
from Google Cloud Console (API section).

The user should set client_secret_file to that JSON file,
probably named client_secret.json, that contains the
application's public and private authentication token.
These tokens are APPLICATION specific.

The authentication process uses these credentials to ask for
an OAuth token on behalf of the user. This step requires the
user to log in, and when they do it creates credentials.json.
These contain the credentials to perform actions as the user
that just logged in/granted permission to your app.
"""

class GDrive(object):
    def __init__(self, gdrive_token_path, config):
        """
        Set up the Google Drive API instance.
        Create the API instance here, hand it over in get_service().
        """
        # Setup the Drive v3 API
        self.SCOPES = 'https://www.googleapis.com/auth/drive.metadata.readonly'
        self.store = file.Storage(gdrive_token_path)

    def get_service(self):
        """
        Return an instance of the Google Drive API service.
        """
        creds = self.store.get()
        if not creds or creds.invalid:
            raise Exception("Error: invalid or missing Google Drive API credentials")

        service = build('drive', 'v3', http=creds.authorize(Http()))
        return service

if __name__=="__main__":
    g = GDrive()
    s = g.get_service()


from pathlib import Path
import os

def extract_video_id(url):
    url_split = url.split('?')
    if len(url_split) < 2:
        return null
    for param in url_split[1].split('&'):
        key, value = param.split("=")
        if key == "v":
            return value
    return null

def create_download_directory():
    """Tries to create download directory at startup."""

    path = Path(os.getcwd()).joinpath('download')
    try:
        Path(path).mkdir(parents=True, exist_ok=True)
    except (OSError,FileExistsError,FileNotFoundError) as error:
        print("Creation of downlaod directory failed")
        print(error)


import random
import torch
import torch.nn as nn
from torchtext.data import Field, TabularDataset

device = torch.device("cuda" if torch.cuda.is_available() else "cpu")

def tokin(text):
    # This is tokinizer
    text = text.split(" ")
    m = []
    for i in text:
        k = i
        for j in range(len(i)):
            if i[j].isalpha():
                break
            k = k.replace(i[j], "")
        for j in range(1, len(i)):
            if i[-j].isalpha():
                break
            k = k.replace(i[-j], "")
        if k != "":
            m.append(k)
    return m

# Create a field
rus = Field( tokenize=tokin, lower = True, init_token = "<sos>", eos_token = "<eos>")
train_data, validation_data, test_data = TabularDataset.splits(
                                        path="Final",
                                        train="train.json",
                                        validation="validation.json",
                                        test="test.json",
                                        format="json",
                                        fields = {"src" : ("src", rus), "trg" : ("trg",rus)})
# Create vocabulary
rus.build_vocab(train_data, max_size = 16384, min_freq=4)

class Encoder(nn.Module):

    def __init__(self, input_size, embedding_size, hidden_size, num_layers, p):
        super(Encoder, self).__init__()
        self.hidden_size = hidden_size
        self.num_layers = num_layers

        self.embedding = nn.Embedding(input_size, embedding_size)
        self.rnn = nn.LSTM(embedding_size, hidden_size, num_layers, bidirectional=True)

        self.fc_hidden = nn.Linear(hidden_size * 2, hidden_size)
        self.fc_cell = nn.Linear(hidden_size * 2, hidden_size)
        self.dropout = nn.Dropout(p)

    def forward(self, x):
        # x: (seq_length, N) where N is batch size

        embedding = self.dropout(self.embedding(x))
        # embedding shape: (seq_length, N, embedding_size)

        encoder_states, (hidden, cell) = self.rnn(embedding)
        # outputs shape: (seq_length, N, hidden_size)

        # Use forward, backward cells and hidden through a linear layer
        # so that it can be input to the decoder which is not bidirectional
        # Also using index slicing ([idx:idx+1]) to keep the dimension
        hidden = self.fc_hidden(torch.cat((hidden[0:1], hidden[1:2]), dim=2))
        cell = self.fc_cell(torch.cat((cell[0:1], cell[1:2]), dim=2))

        return encoder_states, hidden, cell

class Decoder(nn.Module):
    def __init__(self, input_size, embedding_size, hidden_size,
                 output_size, num_layers, p):
        super(Decoder, self).__init__()
        self.hidden_size = hidden_size
        self.num_layers = num_layers

        self.embedding = nn.Embedding(input_size, embedding_size)
        self.rnn = nn.LSTM(hidden_size * 2 + embedding_size, hidden_size, num_layers)

        self.energy = nn.Linear(hidden_size * 3, 1)
        self.fc = nn.Linear(hidden_size, output_size)
        self.dropout = nn.Dropout(p)
        self.softmax = nn.Softmax(dim=0)
        self.relu = nn.ReLU()

    def forward(self, x, encoder_states, hidden, cell):
        x = x.unsqueeze(0)
        embedding = self.dropout(self.embedding(x))
        sequence_length = encoder_states.shape[0]
        h_reshaped = hidden.repeat(sequence_length, 1, 1)
        energy = self.relu(self.energy(torch.cat((h_reshaped, encoder_states), dim=2)))
        attention = self.softmax(energy)
        context_vector = torch.einsum("snk,snl->knl", attention, encoder_states)
        rnn_input = torch.cat((context_vector, embedding), dim=2)
        outputs, (hidden, cell) = self.rnn(rnn_input, (hidden, cell))
        # outputs shape: (1, N, hidden_size)

        predictions = self.fc(outputs).squeeze(0)
        # predictions: (N, hidden_size)

        return predictions, hidden, cell

class Seq2Seq(nn.Module):
    def __init__(self, encoder, decoder):
        super(Seq2Seq, self).__init__()
        self.encoder = encoder
        self.decoder = decoder

    def forward(self, source, target, teacher_force_ratio=0.5):
        batch_size = source.shape[1]
        target_len = target.shape[0]
        target_vocab_size = len(rus.vocab)

        outputs = torch.zeros(target_len, batch_size, target_vocab_size).to(device)
        encoder_states, hidden, cell = self.encoder(source)

        # First input will be <SOS> token
        x = target[0]

        for t in range(1, target_len):

            output, hidden, cell = self.decoder(x, encoder_states, hidden, cell)
            outputs[t] = output

            best_guess = output.argmax(1)
            x = target[t] if random.random() < teacher_force_ratio else best_guess

        return outputs


import configparser
import collections

import re
import env
from file_handler.struct_ini import StructINIConfig

#from plc DataType to opcua DataType
#datatypes identifiers
#https://python-opcua.readthedocs.io/en/latest/opcua.ua.html#opcua.ua.object_ids.ObjectIds.Float
plc2ua = {
    'int'  : 'Int16'   , #datatype=4
    'word' : 'Int16'   , #datatype=4
    'uint' : 'UInt16'  , #datatype=5
    'dint' : 'Int32'   , #datatype=6
    'udint': 'UInt32'  , #datatype=7
    'usint': 'UInt32'  , #datatype=7
    'real' : 'Float'   , #datatype=10
    'bool' : 'Boolean' , #datatype=1
    'dtl'  : 'DateTime', #datatype=13
    'dword': 'Int32'   , #datatype=6
    'lreal': 'Double'  , #datatype=11
}

namespace_info = {
   #index : uri/name
    'PNR OPC UA Server'   : '2'
}

def convert_type_plc_to_opcua(config):
    for key, value in config.items():
        ua_type = ''
        value = value.lower()

        ua_type = plc2ua[ value ]

        config[ key ] = ua_type

class HandlerININodes():

    def __init__(self, config_path, log_info):
        self.log_info    = log_info
        self.config_path = config_path
        self.logger      = log_info.init_class_logger( self.__class__.__name__ )
        self.sections    = collections.OrderedDict()

        fh = configparser.ConfigParser()
        fh.optionxform = str #set 'key' (node id's string identifier ) case sensitive
        fh.read( config_path )

        for section_name in fh:
            if section_name == 'DEFAULT':
                continue
            ini_nodes = StructINIConfig( self.config_path, self.log_info )
            for key, value in fh[section_name].items():
                ini_nodes.set_field(key, value)
            self.sections[ section_name ] = ini_nodes

    def init_section(self, section_name, config, namespace_uri, namespace_id, db_format):
        if isinstance( config, collections.OrderedDict ):

            ini_nodes = StructINIConfig( self.config_path, self.log_info )
            self.sections[ section_name ] = ini_nodes

            convert_type_plc_to_opcua( config )
            ini_nodes.set_field('db_format', db_format )

            for key, value in config.items():
                ini_nodes.set_field( key, value )

            ini_nodes.save_to_config_path( section_name )

        else:
            self.logger.warn(f"Incorrect type of config, should be '{collections.OrderedDict}'")

    def get_struct(self, section_name):
        config = self.sections[ section_name ].get_copied_config()
        rv = list()
        for key, value in config.items():
            rv.append([key, value])
        return rv


import bpy
from bpy.props import *
from ... events import executionCodeChanged
from ... sockets.info import toBaseDataType, isBase
from ... base_types import AnimationNode, ListTypeSelectorSocket

class SetListElementNode(AnimationNode, bpy.types.Node):
    bl_idname = "an_SetListElementNode"
    bl_label = "Set List Element"
    errorHandlingType = "MESSAGE"

    assignedType: ListTypeSelectorSocket.newProperty(default = "Float")

    clampIndex: BoolProperty(name = "Clamp Index", default = False,
        description = "Clamp the index between the lowest and highest possible index",
        update = executionCodeChanged)

    allowNegativeIndex: BoolProperty(name = "Allow Negative Index",
        description = "-2 means the second last list element",
        update = executionCodeChanged, default = False)

    def create(self):
        prop = ("assignedType", "BASE")
        self.newInput(ListTypeSelectorSocket(
            "List", "list", "LIST", prop, dataIsModified = True))
        self.newInput(ListTypeSelectorSocket(
            "Element", "element", "BASE", prop))
        self.newInput("Integer", "Index", "index")

        self.newOutput(ListTypeSelectorSocket(
            "List", "list", "LIST", prop))

    def drawAdvanced(self, layout):
        layout.prop(self, "clampIndex")
        layout.prop(self, "allowNegativeIndex")
        self.invokeSelector(layout, "DATA_TYPE", "assignListDataType",
            dataTypes = "LIST", text = "Change Type", icon = "TRIA_RIGHT")

    def getExecutionCode(self, required):
        if self.allowNegativeIndex:
            if self.clampIndex:
                yield "if len(list) != 0: list[min(max(index, -len(list)), len(list) - 1)] = element"
            else:
                yield "if -len(list) <= index < len(list): list[index] = element"
        else:
            if self.clampIndex:
                yield "if len(list) != 0: list[min(max(index, 0), len(list) - 1)] = element"
            else:
                yield "if 0 <= index < len(list): list[index] = element"
        yield "else: self.setErrorMessage('Index out of range')"

    def assignListDataType(self, listDataType):
        self.assignType(toBaseDataType(listDataType))

    def assignType(self, baseDataType):
        if not isBase(baseDataType): return
        if baseDataType == self.assignedType: return
        self.assignedType = baseDataType
        self.refresh()


#!/usr/bin/python3
# -*- coding: utf-8 -*-

import os, sys
from utils import measure

def execute(day):
  main = __import__(f'{day}.main').main.main
  os.chdir(day)
  measure(main)

def list_available_days():
  dirs = sorted([dir for dir in os.listdir(base_dir) if os.path.isdir(dir) and dir.isnumeric()])
  print('Available days:')
  i = 0
  while len(dirs) > i:
    print('\t'.join(dirs[i:i+5]))
    i += 5

if __name__ == '__main__':
  base_dir = os.path.realpath(os.path.dirname(__file__))
  os.chdir(base_dir)

  if len(sys.argv) >= 2 and sys.argv[-2] == '--day':
    execute(sys.argv[-1])
  else:
    list_available_days()
    day = input('> ')
    execute(day)


from datetime import datetime, timedelta

def check_time_restriction(start_time, end_time):
    st = start_time.split(':')
    et = end_time.split(':')

    message_time = datetime.now()
    start_time = message_time.replace(hour=int(st[0]), minute=int(st[1]), second=int(st[2]), microsecond=0)
    end_time = message_time.replace(hour=int(et[0]), minute=int(et[1]), second=int(et[2]), microsecond=0)

    # Tweak the date if the time range spans midnight
    if end_time.hour < start_time.hour:
        # If check time is after midnight, move start time back to the previous day
        if message_time.hour < end_time.hour:
            start_time -= timedelta(days=1)
        # .. and if before midnight, move end time to the next day.
        else:
            end_time += timedelta(days=1)

    if start_time <= message_time <= end_time:
        return True

    return False


import requests
from decouple import config
import os
#Getting configs
url = config('url')
min_value = int(config('min_value'))
max_value = int(config('max_value'))
file_format = config('file_format')
num_pad = int(config('num_pad'))

#create downloads folder
if not os.path.isdir('downloads'):
    os.mkdir('downloads')

for current_value in range(min_value, max_value+1):

    if(num_pad):
        current_value = str(current_value).zfill(num_pad)
    r = requests.get(url.format(current_value))
    open('downloads/{}.{}'.format(current_value, file_format), 'wb').write(r.content)
    print("downloaded {}.{}".format(current_value, file_format))


from pathlib import Path
import pandas as pd

from prefect import flow,task
from prefect_gcp.cloud_storage import GcsBucket
from prefect_gcp import GcpCredentials

@task(log_prints=True,retries=2)
def extract_from_gcs(playlist_name:str) -> Path:
    """Download trip data from GCS"""

    gcs_path = f"data/DTC_{playlist_name}_playlist_snapshot.parquet"
    gcs_block = GcsBucket.load("zoom-gcs")
    gcs_block.get_directory(from_path=gcs_path, local_path=f"./data/")
    print(f"path: ./data/{gcs_path}")

    return Path(f"./data/{gcs_path}")

@task(log_prints=True)
def transform(path: Path) -> pd.DataFrame:
    """Data cleaning for bq"""
    print(path)
    df = pd.read_parquet(path)
    print(f"number of rows: {len(df)}")

    #df[''].fillna(df[''].mean(),inplace=True)

    return df

@task(log_prints=True,retries=2)
def write_bq(df: pd.DataFrame,playlist_name:str) -> None:
    """Write DataFrame to BigQuery"""
    gcp_credentials_block = GcpCredentials.load("gcp-credentials")
    df.to_gbq(destination_table=f"youtube_playlist.youtube_playlist_{playlist_name}_info",
        project_id="my-rides-antonis",
        credentials= gcp_credentials_block.get_credentials_from_service_account(),
        chunksize=500000,
        if_exists="replace"
    )
    return

@flow(name="ETL-TO-BIG-QUERY")
def etl_gcs_to_bq(playlist_name:str) -> None:
    """Main ETL flow to load data into Big Query"""

    path = extract_from_gcs(playlist_name)
    df = transform(path)
    write_bq(df,playlist_name)

@flow(name="Parent-ETL-TO-BIG-QUERY")
def etl_parent_gcs_to_bq(playlist_name_list:list[str]) -> None:
    """Parent of ETL flow to load data into Big Query"""

    #for every playlist create a different table
    for playlist_name in playlist_name_list:
        etl_gcs_to_bq(playlist_name)

if __name__ == '__main__':
    playlist_name_list = ['Data Engineering Zoomcamp 2023','Machine Learning Zoomcamp 2022','MLOps Zoomcamp 2022']
    etl_parent_gcs_to_bq(playlist_name_list)


from django.db import models
from base.softdelete import SoftDeletionManager
from base.utils import get_guid
from django_userforeignkey.models.fields import UserForeignKey

class Base(models.Model):
    id = models.CharField(primary_key=True, blank=True, max_length=40, verbose_name="ID")
    deleted = models.BooleanField(default=False)
    create = models.DateTimeField(auto_now_add=True)
    update = models.DateTimeField(auto_now=True)
    user_create = UserForeignKey(auto_user_add=True, related_name='+')
    user_update = UserForeignKey(auto_user=True, related_name='+')

    objects = SoftDeletionManager()
    all_objects = SoftDeletionManager(alive_only=False)

    class Meta:
        abstract = True

    def save(self, force_insert=False, force_update=False, update_fields=None, using=None, request=None):
        is_new = False
        if not self.id:
            is_new = True
            self.id = get_guid()
        super(Base, self).save(force_insert, force_update, using, update_fields)


from time import sleep
from selenium.webdriver.edge.options import Options
from selenium import webdriver
from selenium.webdriver.common.action_chains import ActionChains
from selenium.webdriver.common.by import By
import pandas as pd
import re

options = Options()
# options.add_argument("headless")
# options.add_experimental_option('excludeSwitches', ['enable-automation'])
options.add_argument('--disable-blink-features=AutomationControlled')
driver = webdriver.Edge(options=options)
actions = ActionChains(driver)

def clickpre() -> bool:
    try:
        elem = driver.find_element(By.CSS_SELECTOR, ".pagination:nth-child(1) > .page-item:nth-child(2)")
    except:
        return False
    if "disabled" in elem.get_attribute("class"):
        return False
    elem.click()
    sleep(2)
    return True
links = ["0x4a66ad0bca2d700f11e1f2fc2c106f7d3264504c"
, "0x4b319cE02BdC8977Fb39fa70C49258a1a7C27d9B"
, "0x4b71ad9c1a84b9b643aa54fdd66e2dec96e8b152"
, "0x4D391B4350cA1C690eBaf00d35bfcDd5721470F8"
, "0x4dAA9DC438a77Bd59E8A43C6D46cbfE84cD04255"
, "0x4e73b32ed6c35f570686b89848e5f39f20ecc106"
, "0x4Ea0Df261BA584572CDED3F2E35a0E63375Ac4f1"
, "0x50ca2DE80e803bF4b00f188545BCA959540C5582"
, "0x524960d55174d912768678d8c606b4d50b79d7b1"
, "0x524A1407223446e3BAb8819ABb3fb8348C29afEc"
, "0x52d2e0f9b01101a59b38a3d05c80b7618aeed984"
, "0x531917942B6Bb866031aC22BB00E7838613753AC"
, "0x561eac93c92360949ab1f1403323e6db345cbf31"
, "0x584a0EC8A7A3fcfEd27Db5C234A9062863883778"
, "0x5aa88d2901c68fda244f1d0584400368d2c8e739"
, "0x5aD6Dc0A267693c8A14AC9fF2A29c7D63A3d96c2"
, "0x610495793564aed0f9c7fc48dc4c7c9151d34fd6"
, "0x61dc161B06088b9cbaaD53391134929A83D1EB7A"
, "0x627fa62ccbb1c1b04ffaecd72a53e37fc0e17839"
, "0x63e634330A20150DbB61B15648bC73855d6CCF07"
]

# links = ["0xc7d020d8c92d099b3ade17321310b4815ef20a90"]

try:
    for i in range(len(links)):
        driver.get("https://etherscan.io/address/"+links[i])
        sleep(1.2)
        # get all the transactions in web
        # driver.get_screenshot_as_file("1.png")
        driver.set_window_size(1080,840)
        try:
            driver.find_element(By.CSS_SELECTOR, "#btnCookie").click()
        except:
            pass
        try:
            elem = driver.find_element(By.CSS_SELECTOR, ".mr-2 > a:nth-child(2)")
            actions.move_to_element(elem).click().perform()
            sleep(1)
        except:
            pass
        try:
            elem= driver.find_element(By.CSS_SELECTOR, ".pagination:nth-child(1) > .page-item:nth-child(5) > .page-link")
            actions.move_to_element(elem).click().perform()
            sleep(1)
        except:
            pass

        # num = 1250
        # for j in range(num//50):
        #     clicknext()
        output = []
        table = []
        times = 0
        while True:
            count = len(driver.find_elements(By.CSS_SELECTOR, "tr .u-label--info"))
            for j in range(count, 0, -1):
                # crawl_from
                detail = {}
                elem = driver.find_element(By.CSS_SELECTOR, "tr:nth-child("+str(j)+") > td:nth-child(7) a")
                actions.move_to_element(elem).perform()
                address = elem.get_attribute("data-original-title")
                if address is None:
                    address = elem.get_attribute("textContent")
                table.append(address)
                # print("{0},{1}".format(address,j))
                # get method name (temp)
                elem = driver.find_element(By.CSS_SELECTOR, "tr:nth-child("+str(j)+") .u-label--info")
                actions.move_to_element(elem).perform()
                method = elem.get_attribute("data-original-title")
                # click into transaction details
                elem = driver.find_element(By.CSS_SELECTOR, "tr:nth-child("+str(j)+") .myFnExpandBox_searchVal")
                actions.move_to_element(elem).click().perform()
                sleep(1.2)
                # click to open the arguments details
                driver.find_element(By.CSS_SELECTOR,"#collapsedLink > span").click()
                sleep(1.2)
                # get limit
                text = driver.find_element(By.CSS_SELECTOR, "#ContentPlaceHolder1_spanGasLimit").get_attribute("textContent")
                text = re.sub("[\n,]","",text)
                detail["GasLimit"] = text
                # get usage
                text = driver.find_element(By.CSS_SELECTOR, "#ContentPlaceHolder1_spanGasUsedByTxn").get_attribute("textContent")
                text = re.sub("[\n,]","",text)
                detail["GasUsage"] = text
                # get value
                text = driver.find_element(By.CSS_SELECTOR, "#ContentPlaceHolder1_spanValue").get_attribute("textContent")
                detail["Value"] = text
                # get method name
                text = driver.find_element(By.CSS_SELECTOR, "#inputdata").get_attribute("textContent")
                if "Function" in text:
                    try:
                        method = re.search("Function: (.*)\(",text).group(1)
                    except:
                        pass
                elif method == "0x60606040":
                    method = "constructor"
                print(method,j)
                detail["Method"] = method
                # use actionchains to click something that "Element is not clickable at point" , scroll the screen to make the buttom clickable
                # click to format the arguments
                try:
                    elem = driver.find_element(By.CSS_SELECTOR,"#ContentPlaceHolder1_btnDecodetab")
                    actions.scroll_to_element(elem).move_to_element(elem).click().perform()
                    sleep(1.3)
                except:
                    pass
                # markdown the arguments
                elem1s = driver.find_elements(By.CSS_SELECTOR,"tr > td:nth-child(3)")
                elem2s = driver.find_elements(By.CSS_SELECTOR,"tr > td:nth-child(4)")
                arguments = []
                for elem1,elem2 in zip(elem1s,elem2s):
                    text1 = elem1.get_attribute("textContent")
                    text2 = elem2.get_attribute("innerHTML")
                    text2 = re.sub(r'<br>',"\n",text2)
                    text2 = re.sub(r'<.*?>',"",text2)
                    arguments.append([text1,text2])
                detail["Args"] = arguments
                # print(arguments)
                output.append(detail)
                driver.back()
                sleep(1)
            sleep(2)
            times += 1
            #next page or quit
            if clickpre()==False or times == 7:
                break

        output = pd.DataFrame(data=output)
        output.to_csv(links[i]+"_result.csv",mode='a')
        address = pd.DataFrame(data=table)
        address.to_csv(links[i]+"_from.csv",header=False,mode='a')
    driver.quit()
except:
    output = pd.DataFrame(data=output)
    output.to_csv(links[i]+"_result.csv",header=False,mode='a')
    address = pd.DataFrame(data=table)
    address.to_csv(links[i]+"_from.csv",header=False,mode='a')


# -*- coding: utf-8 -*-
from datetime import datetime

from sqlalchemy import Column, Integer, Text, String

from app.lib.orm import db
from app.model.base import Base

class Project(Base):
    id = Column(Integer, primary_key=True)
    name = Column(String(64), nullable=True, unique=True)
    mocks = db.relationship('Mock', backref=db.backref('project', lazy="joined"), lazy='joined')
    desc = Column(Text)

    # GET
    @staticmethod
    def fetch_project(project_id):
        project = Project.query.filter_by(id=project_id).first_or_404()
        return project

    # UPDATE
    @staticmethod
    def update_project(project_id, form):
        project = Project.query.filter_by(id=project_id).first_or_404()
        new_project_name = None
        new_project_desc = None
        if 'name' in form:
            new_project_name = form['name']
        if 'desc' in form:
            new_project_desc = form['desc']
        try:
            with db.auto_commit():
                if new_project_name:
                    project.name = new_project_name
                if new_project_desc:
                    project.desc = new_project_desc
                # 更新时间
                project.update_time = datetime.now()
        except IndentationError as e:
            print(e)
            return False
        return True

    # CREATE
    @staticmethod
    def insert(project_name, project_desc):
        try:
            with db.auto_commit():
                project = Project(name=project_name, desc=project_desc)
                db.session.add(project)
        except IndentationError as e:
            print(e)
            return False
        return True


import openslide
import seaborn as sns
import matplotlib.pyplot as plt
import matplotlib
import os
import numpy as np
from .utils import crop_around_mask

def plot_count(
    df,
    feature,
    ax,
    title='',
    size=2,
    *args,
        **kwargs):
    # source : kaggle.com/rohitsingh9990/panda-eda-better-visualization-simple-baseline
    total = float(len(df))
    sns.countplot(
        df[feature],
        order=df[feature].value_counts().index,
        palette='Set2',
        ax=ax,
        *args,
        **kwargs)
    ax.set_title(title)
    ax.set_xticklabels(ax.get_xticklabels(), rotation=45)
    for p in ax.patches:
        height = p.get_height()
        ax.text(p.get_x()+p.get_width()/2.,
                height + 3,
                '{:1.2f}%'.format(100*height/total),
                ha="center")

def plot_relative_distribution(
    df,
    feature,
    hue,
    ax,
    title='',
    size=2,
    *args,
        **kwargs):
    # source : kaggle.com/rohitsingh9990/panda-eda-better-visualization-simple-baseline
    total = float(len(df))
    sns.countplot(
        x=feature,
        hue=hue,
        data=df,
        palette='Set2',
        *args,
        **kwargs)
    ax.set_title(title)
    ax.set_xticklabels(ax.get_xticklabels(), rotation=45)
    for p in ax.patches:
        height = p.get_height()
        ax.text(p.get_x()+p.get_width()/2.,
                height + 3,
                '{:1.2f}%'.format(100*height/total),
                ha="center")

def makeSimpleEDAplots(df):
    f, ax = plt.subplots(2, 2, figsize=(15, 9))
    ax = ax.ravel()
    plot_count(
        df=df,
        feature='data_provider',
        title='data_provider count and % plot',
        ax=ax[0])
    plot_count(
        df=df,
        feature='isup_grade',
        title='isup_grade count and %age plot',
        ax=ax[1])
    plot_count(
        df=df,
        feature='gleason_score',
        title='gleason_score count and %age plot',
        size=3,
        ax=ax[2])
    plot_relative_distribution(
        df=df,
        feature='gleason_score',
        hue='data_provider',
        title='relative count plot of gleason_score with data_provider',
        size=3,
        ax=ax[3])
    plt.tight_layout()

def visualizeExampleSingeSlide(
    slide_id,
    ax,
    data_provider,
    isup_grade,
    gleason_score,
    DATA_DIR,
    *args,
        **kwargs):
    """
    source: https://www.kaggle.com/rohitsingh9990/panda-eda-better-visualization-simple-baseline
    """
    image = openslide.OpenSlide(os.path.join(
        DATA_DIR,
        f'train_images/{slide_id}.tiff'))
    mask = openslide.OpenSlide(os.path.join(
        DATA_DIR,
        f'train_label_masks/{slide_id}_mask.tiff'))
    image_patch = image.read_region(
        (0, 0),
        image.level_count - 1,
        image.level_dimensions[-1])
    mask_patch = mask.read_region(
        (0, 0),
        mask.level_count - 1,
        mask.level_dimensions[-1])
    ax.imshow(np.asarray(image_patch)[..., :3], *args, **kwargs)
    ax.imshow(np.asarray(mask_patch)[..., 0], alpha=0.25, *args, **kwargs)
    image_id = slide_id
    ax.set_title(f"ID: {image_id}\nSource: {data_provider} ISUP: {isup_grade} Gleason: {gleason_score}")

def visualizeExampleMaskSlide(
    slide_id,
    ax,
    isup_grade,
    data_provider,
    gleason_score,
    DATA_DIR,
    *args,
        **kwargs):
    """
    source: https://www.kaggle.com/rohitsingh9990/panda-eda-better-visualization-simple-baseline
    """
    mask = openslide.OpenSlide(os.path.join(
        DATA_DIR,
        f'train_label_masks/{slide_id}_mask.tiff'))
    cmap = matplotlib.colors.ListedColormap([
        'black',
        'gray',
        'green',
        'yellow',
        'orange',
        'red'])
    mask_patch = mask.read_region(
        (0, 0),
        mask.level_count - 1,
        mask.level_dimensions[-1])
    ax.imshow(
        np.asarray(mask_patch)[..., 0],
        cmap=cmap,
        interpolation='nearest',
        vmin=0,
        vmax=5,
        *args,
        **kwargs)
    image_id = slide_id
    ax.set_title(f"ID: {image_id}\nSource: {data_provider} ISUP: {isup_grade} Gleason: {gleason_score}")

def get_desc_image(df, slide_id):
    dp = df.data_provider[df.image_id == slide_id].values[0]
    isup_g = df.isup_grade[df.image_id == slide_id].values[0]
    gl_s = df.gleason_score[df.image_id == slide_id].values[0]
    return dp, isup_g, gl_s

def cropRandomExample(slide_id, DATA_DIR, cropsize=512):
    f, ax = plt.subplots(1, 3)
    mask = openslide.OpenSlide(os.path.join(
        DATA_DIR,
        f'train_label_masks/{slide_id}_mask.tiff'))
    image = openslide.OpenSlide(os.path.join(
        DATA_DIR,
        f'train_images/{slide_id}.tiff'))
    k = int(image.level_downsamples[-1])
    mask_full = mask.read_region((0, 0), 2, mask.level_dimensions[-1])
    mask_full = np.asarray(mask_full).astype(bool)[..., 0]
    ax[0].imshow(mask_full)
    crop_coords = crop_around_mask(mask_full, 32, 32)
    image_slice = image.read_region(
        (k*crop_coords['x_min'], k*crop_coords['y_min']),
        0,
        (cropsize, cropsize))
    mask_slice = mask.read_region(
        (k*crop_coords['x_min'], k*crop_coords['y_min']),
        0,
        (cropsize, cropsize))
    ax[1].imshow(np.asarray(image_slice)[..., :3])
    ax[1].imshow(np.asarray(mask_slice)[..., 0], alpha=0.5)
    cmap = matplotlib.colors.ListedColormap([
        'black',
        'gray',
        'green',
        'yellow',
        'orange',
        'red'])
    ax[2].imshow(
        np.asarray(mask_slice)[..., 0],
        cmap=cmap,
        interpolation='nearest',
        vmin=0,
        vmax=5)
    plt.tight_layout()


'''
New Update 28-10-2019 Time 7:16 PM
'''

import sys
import json
import os
import webbrowser

import winsound

from PyQt5.QtWidgets import *

from  Creat_New_Task import AddTaskWindow

from PyQt5.QtGui import QIcon

from PyQt5.QtCore import Qt

from PyQt5.QtCore import QTimer

from PyQt5.QtGui import QFont

from PyQt5.QtCore import QDate

import time

from datetime import datetime

from PyQt5.QtCore import QTime

class window(QMainWindow):

    def __init__(self):

        super(window,self).__init__()

        self.show_AddTaskWindow = AddTaskWindow()

        self.UI()

        self.center()

        self.update_list_task()

        self.show()

    def update_list_task(self):

        self.Task_List.clear()

        try:
            task_data_file = open("task_data.txt","r")

            self.data = json.load(task_data_file)

            for task in self.data:

                global task_list_item

                task_list_item = QListWidgetItem()

                task_list_item.setText(task)

                task_list_item.setFlags(task_list_item.flags() | Qt.ItemIsUserCheckable)

                for i in self.data[task]:

                    self.Task_List.addItem(task_list_item)

                    task_list_item.setCheckState(i["Done"])

                    if i["Done"] == 0:

                        task_list_item.setFont(self.font_StrikeOut)
                    elif i["Done"]==2:
                        task_list_item.setFont(self.font_UnStrikeOut)

            return task_list_item

        except:

            pass

    def UI(self):

        self.font_StrikeOut = QFont()
        self.font_StrikeOut.setStrikeOut(True)

        self.font_UnStrikeOut = QFont()

        self.font_UnStrikeOut.setStrikeOut(False)

        #=========================== Main Window Setup
        self.setWindowTitle("My Tasks Today")
        self.setGeometry(500,50,290,800)
        self.setWindowIcon((QIcon("../images/images.png")))
        self.setMaximumHeight(800)
        self.setMaximumWidth(290)
        self.widget = QWidget(self)
        self.setCentralWidget(self.widget)
        #========================== Layout Setup
        self.Vertical_Layout = QVBoxLayout()
        self.Horizontal_Layout = QHBoxLayout()
        self.Horizontal_Layout02 = QHBoxLayout()
        self.Horizontal_Layout03 = QHBoxLayout()
        #========================== Widgets Setup
        self.Button_Add = QPushButton("Add")
        self.Button_Add.clicked.connect(self.add_task)

        self.Button_Remove = QPushButton("Remove")
        self.Button_Remove.clicked.connect(self.remove_task)

        self.Button_refresh = QPushButton("Refresh")
        self.Button_refresh.clicked.connect(self.refresh)

        self.timer = QTimer(self)
        self.alarm_timer = QTimer(self)
        self.alarm_timer.setInterval(50)
        self.alarm_timer.timeout.connect(self.alarm)
        self.alarm_timer.start()
        self.timer.setInterval(50)
        self.timer.stop()
        self.timer.timeout.connect(self.show_me)

        self.tool_bar = QToolBar("Tool Bar")
        self.new_Tool = QAction(QIcon("../images/document_add_100293.png"),"NewTask")
        self.tool_bar.addAction(self.new_Tool)
        self.edit_Tool = QAction(QIcon("../images/misc_edit_98007.jpg"),"EditTask")
        self.tool_bar.addAction(self.edit_Tool)
        self.delete_Tool = QAction(QIcon("../images/document_delete_100294.png"),"DeleteTask")
        self.tool_bar.addAction(self.delete_Tool)
        self.show_AddTaskWindow.save_button.clicked.connect(lambda: self.save_task(2))

        self.show_AddTaskWindow.cancel_button.clicked.connect(self.close_window)

        self.refresh_Tool = QAction(QIcon("../images/rounded_blue_refresh_button_4808.png"),"RefreshTasks")
        self.tool_bar.addAction(self.refresh_Tool)

        self.new_Tool.triggered.connect(self.add_task)
        self.delete_Tool.triggered.connect(self.remove_task)
        self.refresh_Tool.triggered.connect(self.refresh)
        self.edit_Tool.triggered.connect(self.edit_task)
        self.tool_bar.setToolButtonStyle(Qt.ToolButtonTextUnderIcon)

        self.Task_List = QListWidget()
        self.Task_List.itemClicked.connect(self.task_one_click)

        #self.Task_List.doubleClicked.connect(self.run_task)
        self.data_table = QTableWidget()
        self.data_table.setMaximumHeight(200)
        self.data_table.setRowCount(5)
        self.data_table.setColumnCount(1)
        self.data_table.setColumnWidth(0,self.maximumWidth()-40)
        self.data_table.setVerticalHeaderItem(0, QTableWidgetItem("Name"))
        self.data_table.setVerticalHeaderItem(1, QTableWidgetItem("Link"))
        self.data_table.setVerticalHeaderItem(2, QTableWidgetItem("End By"))
        self.data_table.setVerticalHeaderItem(3, QTableWidgetItem("Status"))
        self.data_table.setVerticalHeaderItem(4, QTableWidgetItem("Notes"))
        self.data_table.setRowHeight(4,70)
        self.data_table.horizontalHeader().hide()
        self.data_table.setEditTriggers(QAbstractItemView.NoEditTriggers)

        self.data_table.cellDoubleClicked.connect(self.run_task)

        #=========================== Add Widget to Layout
        self.Vertical_Layout.addWidget(self.tool_bar)
        #self.Horizontal_Layout.addWidget(self.Button_Add)

        #self.Horizontal_Layout.addWidget(self.Button_Remove)
        self.Horizontal_Layout02.addWidget(self.Task_List)

        #self.Horizontal_Layout03.addWidget(self.Button_refresh)

        #================================================
        self.Vertical_Layout.addLayout(self.Horizontal_Layout03)
        self.Vertical_Layout.addLayout(self.Horizontal_Layout)
        self.Vertical_Layout.addLayout(self.Horizontal_Layout02)
        self.Vertical_Layout.addWidget(self.data_table)
        self.show_AddTaskWindow.update_button.clicked.connect(self.update_task)
        self.widget.setLayout(self.Vertical_Layout)

        #============================================ Build Actions

    def add_task(self):

        now = datetime.now()

        self.show_AddTaskWindow.show()

        self.show_AddTaskWindow.setWindowTitle("Add New Task")

        self.show_AddTaskWindow.save_button.setHidden(False)

        self.show_AddTaskWindow.update_button.setHidden(True)

        self.show_AddTaskWindow.date.setTime(QTime(datetime.now().time().hour,datetime.now().time().minute,datetime.now().time().second))

        self.show_AddTaskWindow.date.setDate(QDate(datetime.now().date()))

        self.timer.start()

        self.update_list_task()

    def table_data_build(self,item_name,link,dead_line,status,note):

        try:

            self.data_table.setItem(0,0,QTableWidgetItem(item_name))

            self.data_table.setItem(1, 0, QTableWidgetItem(link))

            self.data_table.setItem(2, 0, QTableWidgetItem(dead_line))

            if status == 0:

                status_ = "Done"

            elif status == 2:

                status_ = "In Progress"

            else:

                status_ = "None"

            self.data_table.setItem(3, 0, QTableWidgetItem(status_))

            self.data_table.setItem(4, 0, QTableWidgetItem(note))

        except:

            print ("Error To Apply")

    def remove_task(self):

        try:
            self.message = QMessageBox.question(self,"Confirm",'Are You Sure! You Need To Delete "%s""'%self.Task_List.currentItem().text(),QMessageBox.Yes,QMessageBox.No)

            if self.message == QMessageBox.Yes:

                del self.data[self.Task_List.currentItem().text()]

                outfile = open("task_data.txt","w")

                json.dump(self.data, outfile,indent=4)

                self.Task_List.takeItem(self.Task_List.row(self.Task_List.selectedItems()[0]))
        except:

            self.message = QMessageBox.information(self, "Hold","Nothing to Remove", QMessageBox.Ok)

    def refresh(self):

        python = sys.executable

        os.execl(python, python, *sys.argv)

    def run_task(self):

        for i in self.data[self.Task_List.currentItem().text()]:

            webbrowser.open_new(i['website'])

    def task_one_click(self,item):

        try:
            check_state = item.checkState()

            task_data_file = open("task_data.txt", "r")

            data = json.load(task_data_file)

            self.update_done_startus(item,item.text(),check_state)

            for data_ in data[item.text()]:

                self.table_data_build(item.text(),data_["website"],data_["Dead_Line"],data_["Done"],data_['Note'])
        except:
            print("Error")

    def edit_task(self):

        try:

            self.timer.start()

            self.show_AddTaskWindow.task_name.setText(self.Task_List.currentItem().text())

            self.show_AddTaskWindow.task_link.setText(self.data[self.Task_List.currentItem().text()][0]['website'])

            self.show_AddTaskWindow.task_notes.setText(self.data[self.Task_List.currentItem().text()][0]['Note'])

            dead_line_date = self.data[self.Task_List.currentItem().text()][0]['Dead_Line'].split(" || ")

            self.show_AddTaskWindow.date.setDate(QDate(datetime.strptime(dead_line_date[0], '%a %b %d %Y')))

            dead_line_time = dead_line_date[1].split(":")

            self.show_AddTaskWindow.date.setTime(QTime(int(dead_line_time[0]), int(dead_line_time[1]), int(dead_line_time[2]) ))

            self.show_AddTaskWindow.save_button.setHidden(True)

            self.show_AddTaskWindow.update_button.setHidden(False)

            self.show_AddTaskWindow.setWindowTitle("Edit %s" % self.Task_List.currentItem().text())

            self.show_AddTaskWindow.show()
        except:
            self.message_err = QMessageBox.information(self, "Hold", "Select Item From The List", QMessageBox.Ok)

    def show_me(self):

        if self.show_AddTaskWindow.isVisible():

            self.setDisabled(True)

        else:

            self.setDisabled(False)

            self.update_list_task()

            self.timer.stop()

    def update_task(self):

        self.delete_old_task()

        self.save_task(2)

        self.update_list_task()

    def delete_old_task(self):

        global current_task

        current_task = self.Task_List.currentItem().text()

        self.data.pop(current_task, None)

        outfile = open("task_data.txt", "w")

        json.dump(self.data, outfile, indent=4)

    def save_task(self,Done=2):

        task_name_var = self.show_AddTaskWindow.task_name.toPlainText()

        task_link_var = self.show_AddTaskWindow.task_link.toPlainText()

        task_note_var = self.show_AddTaskWindow.task_notes.toPlainText()

        try:
            task_data_file = open("task_data.txt", "r")

            data = json.load(task_data_file)

        except:

            data = {}
        outfile = open("task_data.txt", "w")

        data['%s'%task_name_var] = []

        data['%s'%task_name_var].append({'name': task_name_var,'website': task_link_var, "Done": Done,

        "Dead_Line" : "%s || %s" %(self.show_AddTaskWindow.date.date().toString() , self.show_AddTaskWindow.date.time().toString()) ,

        "Date" : self.show_AddTaskWindow.date.date().toString(), "Status": "False","Note":task_note_var,
        })

        outfile.seek(0)

        json.dump(data, outfile,indent=4)

        self.show_AddTaskWindow.close()

    def alarm(self):
        now = datetime.now()

        try:
            try:
                task_data_file = open("task_data.txt", "r")

                data = json.load(task_data_file)

            except:

                data = {}

            for task in data:

                for data_ in data[task]:

                    data_date = data_["Dead_Line"].split(" || ")

                    if data_date[1][:-3] == now.time().strftime("%H:%M") and QDate(datetime.strptime(data_date[0], '%a %b %d %Y')) == now.date():

                        if data_["Status"] == "False":

                            task_data_file = self.update_list_task()

                            task_data_file.setCheckState(0)

                            task_data_file.setFont(self.font_StrikeOut)

                            outfile = open("task_data.txt", "w")

                            print(data_["name"],data_["Status"])

                            del data_["Status"]

                            del data_["Done"]

                            data_["Done"] = 0

                            data_["Status"] = "True"

                            #
                            outfile.seek(0)
                            #
                            json.dump(data, outfile, indent=4)
                            #self.setWindowFlag(Qt.WindowStaysOnTopHint)
                            winsound.PlaySound("ALERT.wav", winsound.SND_FILENAME )
                            message = QMessageBox.information(self,"Time's Gone!" ,"%s | Task Hase Ended "%data_["name"],QMessageBox.Ok)
                            if message == QMessageBox.Ok:
                                task_data_file.setSelected(1)

                                self.setWindowFlag(Qt.StrongFocus)

                            break

        except:
            pass

    def update_done_startus(self,item,task_name_var,Done=2):

        try:
            task_data_file = open("task_data.txt", "r")

            data = json.load(task_data_file)

        except:

            data = {}

        outfile = open("task_data.txt", "w")

        for don in data['%s'%task_name_var]:

            del don["Done"]

            don["Done"]= Done

        if Done == 0:

            item.setFont(self.font_StrikeOut)
        elif Done == 2:

            item.setFont(self.font_UnStrikeOut)

        print(item.checkState())

        outfile.seek(0)

        json.dump(data, outfile,indent=4)

    def close_window(self):

        self.show_AddTaskWindow.close()

    def center(self):
        frameGm = self.frameGeometry()

        screen = QApplication.desktop().screenNumber(QApplication.desktop().cursor().pos())

        centerPoint =QApplication.desktop().screenGeometry(screen).center()

        frameGm.moveCenter(centerPoint)

        self.move(frameGm.topLeft())

if __name__ == "__main__":
    app = QApplication(sys.argv)
    import qdarkstyle

    app.setStyleSheet(qdarkstyle.load_stylesheet_pyqt5())
    window = window()

    sys.exit(app.exec_())


import cherrypy
import os
from jsonschema import Draft4Validator
import json
from functools import wraps
import logging
from pathlib import Path

def resolve_schema(resource, verb):
    return Path(os.path.dirname(os.path.realpath(__file__)), '..', 'api', 'schema', resource, verb + '.json')

def validation(f):
    @wraps(f)
    def wrapper(*args, **kwargs):
        resource, verb = f.__qualname__.lower().split('.')
        schema_path = resolve_schema(resource, verb)

        logging.debug('Validating resource {} for verb {} using {} '.format(resource, verb, schema_path))

        if schema_path.is_file():
            with schema_path.open('r') as schema:
                schema_contents = schema.read()
                validator = Draft4Validator(json.loads(schema_contents))
                errors = list()
                for validation_error in sorted(validator.iter_errors(cherrypy.request.json), key=str):
                    path = list(validation_error.path)
                    logging.debug(path)
                    errors.append({
                        "status": "400",
                        "title": validation_error.message,
                        "path": '/'.join(str(x) for x in path),
                        "validator": validation_error.validator,
                        "validator_value": validation_error.validator_value
                    })
                if len(errors) > 0:
                    logging.debug('Found {} validation errors'.format(len(errors)))
                    cherrypy.response.status = 400
                    return {
                        'errors': errors
                    }
        else:
            logging.debug('No JSON SCHEMA document found, not validating')

        logging.debug('No validation errors found')
        return f(*args, **kwargs)
    return wrapper


import arcpy
import os
import random
import string
import tempfile
import unittest
import arcgisscripting
from os import path

def get_temp_name():
    """
    Creates a random string that is a safe table or feature class name
    :return:
    """
    return ''.join([random.choice(string.ascii_uppercase) for _ in range(6)])

class FeatureClassAssertMixin(object):
    """
    Mixin for supporting the compare or assertion of two feature classes.
    """

    def assertFeatureClassEqual(self,
                                first,
                                second,
                                sort_field,
                                message=None,
                                compare_type=None,
                                ignore_options=None,
                                xy_tolerance=None,
                                m_tolerance=None,
                                z_tolerance=None,
                                attribute_tolerances=None,
                                omit_field=None):
        """
        Compares the second feature class to the first, and reports any issues.  Detailed issues are printed to the
        console if found.
        :param first:
        :param second:
        :param sort_field:
        :param message:
        :param compare_type:
        :param ignore_options:
        :param xy_tolerance:
        :param m_tolerance:
        :param z_tolerance:
        :param attribute_tolerances:
        :param omit_field:
        :return:
        """

        # Make a place to store the compare file
        compare_file = tempfile.mkstemp(".txt")
        os.close(compare_file[0])
        result = arcpy.FeatureCompare_management(first,
                                                 second,
                                                 sort_field,
                                                 compare_type,
                                                 ignore_options,
                                                 xy_tolerance,
                                                 m_tolerance,
                                                 z_tolerance,
                                                 attribute_tolerances,
                                                 omit_field,
                                                 continue_compare=True,
                                                 out_compare_file=compare_file[1])
        if 'true' == result.getOutput(1):
            # delete the compare file
            os.remove(compare_file[1])
            return

        # read the compare file and print it out
        print(compare_file[1])
        with open(compare_file[1]) as f:
            [print(l.rstrip()) for l in f.readlines()]
        os.remove(compare_file[1])

        # set the assertion message.
        msg = message if message is not None else "Feature class {} is not equal to {}".format(second, first)
        raise AssertionError(msg)

class SimpleToolboxTests(unittest.TestCase, FeatureClassAssertMixin):

    def setUp(self):
        arcpy.env.overwriteOutput = True
        self.parent_directory = path.dirname(path.expandvars(__file__))
        self.data_directory = path.join(self.parent_directory, "data")
        arcpy.AddToolbox(path.join(path.dirname(self.parent_directory), "SimpleToolbox.pyt"))

    def test_single_polygon_red(self):
        """
        A single polygon with three red points.
        :return:
        """
        # Setup
        points = path.join(self.data_directory, "single_polygon_red.gdb", "points")
        polygons = path.join(self.data_directory, "single_polygon_red.gdb", "polygons")
        answer = path.join(self.data_directory, "single_polygon_red.gdb", "answer")
        output = path.join("%ScratchGDB%", get_temp_name())
        # Run
        arcpy.SimpleTool_simple(points, polygons, output)
        # assert
        self.assertFeatureClassEqual(answer, output, "OBJECTID")
        # cleanup
        arcpy.Delete_management(output)

    def test_single_polygon_blue(self):
        """
        A single polygon with three blue points.
        :return:
        """
        # Setup
        points = path.join(self.data_directory, "single_polygon_blue.gdb", "points")
        polygons = path.join(self.data_directory, "single_polygon_blue.gdb", "polygons")
        answer = path.join(self.data_directory, "single_polygon_blue.gdb", "answer")
        output = path.join("%ScratchGDB%", get_temp_name())
        # Run
        arcpy.SimpleTool_simple(points, polygons, output)
        # assert
        self.assertFeatureClassEqual(answer, output, "OBJECTID")
        # cleanup
        arcpy.Delete_management(output)

    def test_single_polygon_no_points(self):
        """
        A single polygon with no points in the points layer (empty, no records)
        :return:
        """
        # Setup
        points = path.join(self.data_directory, "single_polygon_no_points.gdb", "points")
        polygons = path.join(self.data_directory, "single_polygon_no_points.gdb", "polygons")
        answer = path.join(self.data_directory, "single_polygon_no_points.gdb", "answer")
        output = path.join("%ScratchGDB%", get_temp_name())
        # Run
        arcpy.SimpleTool_simple(points, polygons, output)
        # assert
        self.assertFeatureClassEqual(answer, output, "OBJECTID")
        # cleanup
        arcpy.Delete_management(output)

    def test_single_polygon_no_points_by_filter(self):
        """
        A single polygon with no points in the points layer that match either BLUE or RED, but there are ORANGE ones.
        :return:
        """
        # Setup
        points = path.join(self.data_directory, "single_polygon_no_points_by_filter.gdb", "points")
        polygons = path.join(self.data_directory, "single_polygon_no_points_by_filter.gdb", "polygons")
        answer = path.join(self.data_directory, "single_polygon_no_points_by_filter.gdb", "answer")
        output = path.join("%ScratchGDB%", get_temp_name())
        # Run
        arcpy.SimpleTool_simple(points, polygons, output)
        # assert
        self.assertFeatureClassEqual(answer, output, "OBJECTID")
        # cleanup
        arcpy.Delete_management(output)

    def test_random_data_01(self):
        """
        A 10x10 grid with 100 random points.
        :return:
        """
        # Setup
        points = path.join(self.data_directory, "random_data_01.gdb", "points")
        polygons = path.join(self.data_directory, "random_data_01.gdb", "polygons")
        answer = path.join(self.data_directory, "random_data_01.gdb", "answer")
        output = path.join("%ScratchGDB%", get_temp_name())
        # Run
        arcpy.SimpleTool_simple(points, polygons, output)
        # assert
        self.assertFeatureClassEqual(answer, output, "OBJECTID")
        # cleanup
        arcpy.Delete_management(output)

    def test_invalid_point_data(self):
        """
        Passes a polygon in to the point layer.
        :return:
        """
        # Setup
        points = path.join(self.data_directory, "random_data_01.gdb", "polygons")
        polygons = path.join(self.data_directory, "random_data_01.gdb", "polygons")
        output = path.join("%ScratchGDB%", get_temp_name())
        # Run with assert exception raised
        with self.assertRaisesRegex(arcgisscripting.ExecuteError, 'Invalid geometry type'):
            arcpy.SimpleTool_simple(points, polygons, output)

    def test_invalid_polygon_data(self):
        """
        Passes a point in to the polygon layer.
        :return:
        """
        # Setup
        points = path.join(self.data_directory, "random_data_01.gdb", "points")
        polygons = path.join(self.data_directory, "random_data_01.gdb", "points")
        output = path.join("%ScratchGDB%", get_temp_name())
        # Run with assert exception raised
        with self.assertRaisesRegex(arcgisscripting.ExecuteError, 'Invalid geometry type'):
            arcpy.SimpleTool_simple(points, polygons, output)

    # def test_invalid_point_layer_missing_type_field(self):
    #     """
    #     Passes a point layer that is missing the type field.
    #     :return:
    #     """
    #     # Setup
    #     points = path.join(self.data_directory, "point_layer_no_type_field.gdb", "points")
    #     polygons = path.join(self.data_directory, "point_layer_no_type_field.gdb", "polygons")
    #     output = path.join("%ScratchGDB%", get_temp_name())
    #     # Run with assert exception raised
    #     with self.assertRaisesRegex(arcgisscripting.ExecuteError, 'does not contain a string field called TYPE'):
    #         arcpy.SimpleTool_simple(points, polygons, output)

if __name__ == '__main__':
    unittest.main()


import csv

import sys

import requests

from bs4 import BeautifulSoup as bs

import pandas as pd

filename = "gamelog.csv"

TAGS = {"PTS":26, "BLK": 23, "STL": 22, "AST": 21, "TRB": 20}

def getInfo(games, stat):
    global TAGS

    data = []

    stat = TAGS[stat]

    total_number = len(games)

    for i in games:

        data.append(i)
    sum = 0

    for i in range(len(data)):

        sum += float(data[i][stat])

        data[i] = float(data[i][stat])

    data = sorted(data)

    mean = sum/total_number

    if total_number % 2 == 1: #odd

        median = data[int(total_number/2)]

    else:
        median = (data[int(total_number/2)]+data[int(1+total_number/2)])/2
    #next we want to get the varience and the numbers above certain thresholds
    above = [0, 0, 0, 0, 0, 0, 0] #5-35
    var = 0
    max = data[0]
    min = data[0]
    for i in data:
        var += (i-mean)**2
        if i > max:
            max = i
        if i < min:
            min = i
        if i/5 >= 1:
            if int(i/5) > 7:
                goto = 7
            else:
                goto = int(i/5)
            for j in range(goto):
                above[j] += 1
    var *= (1/(total_number-1))

    print("Mean: ", mean, "Median: ", median, "Varience: ", var, "\n")
    print("Min: ", min, "Max: ", max)
    for i in range(len(above)):
        try:
            odds = "%0.3f" % float(total_number/above[i])
        except Exception as e:

            odds = "0"

        print(str(5*(i+1)) + "+: " + str(above[i]) + "\t" + str(int(100*above[i]/total_number)) + "%\tMin Odds: " +odds )

def openfile(filename):

    fields = []
    rows = []
    with open(filename, 'r') as csvfile:
        # creating a csv reader object
        csvreader = csv.reader(csvfile)

        # extracting field names through first row
        fields = next(csvreader)

        # extracting each data row one by one
        for row in csvreader:
            rows.append(row)

        # get total number of rows

    #get rid of the blank rows

    newrows = []
    for i in range(len(rows)):

        if rows[i][1] != '':
            newrows.append(rows[i])

    return fields, newrows

def getFile(link, year):
    print("Getting Data From ", year)
    link = "https://www.basketball-reference.com/players/" + link[0] +"/" +link + "/gamelog/" + year

    r = requests.get(link)

    soup = bs(r.content, 'html.parser')

    table = soup.find('table', id='pgl_basic')
    def rowgetDataText(tr, coltag='td'): # td (data) or th (header)
        return [td.get_text(strip=True) for td in tr.find_all(coltag)]
    rows = []

    trs = table.find_all('tr')
    headerow = rowgetDataText(trs[0], 'th')
    if headerow: # if there is a header row include first
        rows.append(headerow)
        trs = trs[1:]
    for tr in trs: # for every table row
        rows.append(rowgetDataText(tr, 'td') ) # data row
    new_table = []

    removeLines = ['Did Not Dress', 'Inactive', 'Did Not Play', 'Not With Team', 'Player Suspended']

    for i in rows:

        if i != [] and i[7] not in removeLines and i[0] != 'Rk':
            new_table.append(i)

    #from here we have a table which just needs to be converted to csvfile

    csvfile = ""

    for i in new_table:

        for j in i:

            csvfile += j
            csvfile +=','

        csvfile+='\n'
    return csvfile

def writeCSV (csvText, mode):

    if mode == "w":

        with open('gamelog.csv', 'w') as out:
            out.write(csvText)

    elif mode == "a":

        with open('gamelog.csv', 'a') as out:
            out.write(csvText)

def lookup(name):

    link = ""

    r = requests.get("https://www.basketball-reference.com/players/" + name.split()[1][0].lower() + "/")

    soup = bs(r.content, 'html.parser')

    link = "https://www.basketball-reference.com" + soup.find("a", string=name).get("href").replace(".html", "/gamelog")

    return soup.find("a", string=name).get("href").split("/")[3].split(".")[0]

def getSeasons(name):

    seasons = []

    r = requests.get("https://www.basketball-reference.com/players/" + name[0] + "/"+ name + ".html")

    soup = bs(r.content, 'html.parser')

    table = soup.find("table", id="per_game")

    tbody = table.find('tbody')
    th_head = tbody.find_all('th')

    for i in th_head:

        seasons.append(i.text)

    #this gets rid of duplicates
    seasons = list(dict.fromkeys(seasons))

    #this cuts it down to only the first yea rin the season ex 2015-16 -> 2015

    for i in range(len(seasons)):

        seasons[i] = str(seasons[i][:2])+str(seasons[i][5:])

    return seasons

playerSeasons = getSeasons(lookup(sys.argv[1]))

print("Saving Gamelogs")

writeCSV(getFile(lookup(sys.argv[1]), playerSeasons[0]), "w")

for i in range(1, len(playerSeasons)):

    writeCSV(getFile(lookup(sys.argv[1]), playerSeasons[i]), "a")

ntn, smt = openfile(filename)

getInfo(smt, sys.argv[2])


"""
Module containing methods for reading from financial data database
"""
import pandas as pd
import sqlite3
from decimal import Decimal

class FdRead:
    def __init__(self, db_address):
        self.db_address = db_address

    def read_asset_values(self, asset_tickers, startDate=None, endDate=None):
        """Get all values of list of assets in date range

        Args:
        assetTickers: List[String] | Names of asset tickers
        startDate: datetime.Date | lower bound on date
        endDate: datetime.Date | upper bound on date

        Return:
        Pandas dataframe of format:
        {Columns: [AOpen<Decimal.decimal>, AClose<Decimal.decimal>,
                   AHigh<Decimal.decimal>, ALow<Decimal.decimal>]
         Index: [AssetTicker<String>, ADate <datetime.date>]}
        containing AssetValues with date between start and end date and assetTicker
        in assetTickers
        """
        # cast asset_tickers to list ensure appending works
        asset_tickers = list(asset_tickers)
        # Optionally move name to seperate config file later
        conn = sqlite3.connect(self.db_address)
        # generate parameter list for subsitution
        generated_params = "(" + ",".join(["?"] * len(asset_tickers)) + ")"

        # construct query
        query = (
            "SELECT * \
                 FROM AssetValue \
                 WHERE AssetValue.AssetTicker IN "
            + generated_params
            + " "
        )
        if startDate is not None:
            query += "AND (AssetValue.ADate >= ?) "
            asset_tickers.append(str(startDate))
        if endDate is not None:
            query += "AND (AssetValue.ADate <= ?) "
            asset_tickers.append(str(endDate))
        # read data into df
        df0 = pd.read_sql(query + ";", conn, params=asset_tickers)
        conn.close()
        # fix types
        df0["AOpen"] = df0["AOpen"].map(Decimal)
        df0["AClose"] = df0["AClose"].map(Decimal)
        df0["AHigh"] = df0["AHigh"].map(Decimal)
        df0["ALow"] = df0["ALow"].map(Decimal)
        df0["ADate"] = pd.to_datetime(df0["ADate"], format="%Y-%m-%d %H:%M:%S")
        df0.set_index(["AssetTicker", "ADate"], inplace=True)
        return df0

    def read_assets(self):
        """Get records of all financial assets stored in db that have associated asset values

        Args:
        None

        Return:
        Pandas dataframe of format
        {Columns: [Name<String>, AssetClassName<String>]
                   Index: [AssetTicker<String>]}
        each row containing details of an asset
        currently saved in db

        Notes:
        Will only return assets that have associated asset values stored in AssetValues table
        If nothing stored will return empty dataframe of same format
        """
        # Optionally move name to seperate config file later
        conn = sqlite3.connect(self.db_address)
        # read data in df
        df0 = pd.read_sql(
            "SELECT * \
         FROM Asset WHERE Asset.AssetTicker IN (SELECT AssetTicker FROM AssetValue)",
            conn,
        )
        conn.close()
        # adjust index if neccesary
        df0.set_index("AssetTicker", inplace=True)
        return df0

    def read_assets_in_class(self, asset_class):
        """Get records of all assets in an asset class with values in AssetValues

        Args:
        assetClass: string | Name of asset class

        Return:
        Pandas dataframe of format
        {Columns: [Name<String>, AssetClassName<@assetClass>]
                   Index: [AssetTicker<String>]}
        each row containing details of an asset
        currently saved in db with asset class equal to assetClass

        Notes:
        - Will only return assets that have asset values stored in AssetValues table

        - If assetClass not in database, return empty dataframe in format

        - Will return AssetClass row in dataframe despite all values being equal
        (this is so dataframe can be used with fdWrite library methods as-is)

        - Implements seperate query to getAssetClasses due to performance
        considerations (Large number of assets should not be in working memory
        unless needed)
        """
        # Optionally move name to seperate config file later
        conn = sqlite3.connect(self.db_address)
        # read data in df
        df0 = pd.read_sql(
            "SELECT * \
         FROM Asset \
         WHERE Asset.AssetClassName = $className AND Asset.AssetTicker IN (SELECT AssetTicker FROM AssetValue);",
            conn,
            params={"className": asset_class},  # use params dict to sanitize input
        )
        conn.close()
        # adjust index if neccesary
        df0.set_index("AssetTicker", inplace=True)
        return df0

    def read_assets_div_payout(self, asset_tickers):
        """Get record of all divident payouts for specific asset

        Args:
        asset_tickers: list of strings of asset tickers | asset ticker

        Return:
        Pandas dataframe of format
        {Columns: [Payout<decima.Decimal>]
                   Index: [AssetTicker<String>, PDate<datetime.date>]}

        Notes:
        - If assetTicker not in database, return empty dataframe in format

        - Will return AssetTicker row in dataframe despite all values being equal
        (this is so dataframe can be used with fdWrite library methods as-is)
        """
        # Optionally move name to seperate config file later
        conn = sqlite3.connect(self.db_address)
        generated_params = "(" + ",".join(["?"] * len(asset_tickers)) + ")"

        # construct query
        query = (
            "SELECT * \
                 FROM DividendPayout \
                 WHERE DividendPayout.AssetTicker IN "
            + generated_params
            + " "
        )
        # read data into df
        df0 = pd.read_sql(query + ";", conn, params=asset_tickers)
        conn.close()

        df0["Payout"] = df0["Payout"].map(Decimal)
        df0["PDate"] = pd.to_datetime(df0["PDate"], format="%Y-%m-%d %H:%M:%S")

        df0.set_index(["AssetTicker", "PDate"], inplace=True)
        return df0

    def read_asset_classes(self):
        """Get records of all asset classes stored in fin database

        Args:
        None

        Return:
        Pandas dataframe of format
        {Columns: [] Index: [AssetClassName<String>]}
        each row containing details of an assetClass
        currently stored in db

        Notes:
        If nothing stored will return empty dataframe of same format
        """
        # Optionally move name to seperate config file later
        conn = sqlite3.connect(self.db_address)
        # read data in df
        df0 = pd.read_sql(
            "SELECT * \
         FROM AssetClass",
            conn,
        )
        conn.close()
        # adjust index if neccesary
        df0.set_index("AssetClassName", inplace=True)
        return df0


#!/usr/bin/python3
"""Defines a base model class"""
import json
import csv

class Base:
    """Base class for all other classes in this project.
    Args:
        id (int): The identity of the new Base instance.
    Attributes:
        __nb_objects (int): The number of instantiated Bases.
    """

    __nb_objects = 0

    def __init__(self, id=None):
        """Initialize a new Base.
        Args:
            id (int): The identity of the new Base.
        """
        if id is not None:
            self.id = id
        else:
            Base.__nb_objects += 1
            self.id = Base.__nb_objects

    @staticmethod
    def to_json_string(list_dictionaries):
        """Return serializable representation of list of dicts.
        Args:
            list_dictionaries (list): A list of dictionaries.
        """
        if not list_dictionaries:
            return "[]"
        return json.dumps(list_dictionaries)

    @classmethod
    def save_to_file(cls, list_objs):
        """Write the JSON serialization of a list of objects
        to a file.
        Args:
            list_objs (list): A list of inherited Base instances.
        """
        with open(cls.__name__ + ".json", "w") as jsonfile:
            if not list_objs:
                jsonfile.write("[]")
            else:
                list_dicts = [o.to_dictionary() for o in list_objs]
                jsonfile.write(Base.to_json_string(list_dicts))

    @staticmethod
    def from_json_string(json_string):
        """Return the list represented by a JSON string.
        Args:
            json_string (str): A JSON str representation of a list of dicts.
        """
        if not json_string:
            return []
        return json.loads(json_string)

    @classmethod
    def create(cls, **dictionary):
        """
        Return an instance with all attributes already set.
        Args:
            **dictionary (dict): Key/value pairs of attributes to initialize.
        """
        if dictionary:
            if cls.__name__ == "Rectangle":
                new = cls(100, 100)
            else:
                new = cls(100)
            new.update(**dictionary)
            return new

    @classmethod
    def load_from_file(cls):
        """
        Return a list of classes instantiated from a file of JSON strings.
        Reads from `<cls.__name__>.json`.
        """
        try:
            with open(str(cls.__name__) + ".json", "r") as jsonfile:
                return [cls.create(**d)
                        for d in Base.from_json_string(jsonfile.read())]
        except IOError:
            return []

    @classmethod
    def save_to_file_csv(cls, list_objs):
        """Write the CSV serialization of a list of objects to a file.
        Args:
            list_objs (list): A list of inherited Base instances.
        """
        with open(cls.__name__ + ".csv", "w", newline="") as csv_file:
            if not list_objs:
                csv_file.write("[]")
            else:
                if cls.__name__ == "Rectangle":
                    head = ["id", "width", "height", "x", "y"]
                else:
                    head = ["id", "size", "x", "y"]
                writer = csv.DictWriter(csv_file, fieldnames=head)
                for obj in list_objs:
                    writer.writerow(obj.to_dictionary())

    @classmethod
    def load_from_file_csv(cls):
        """Return a list of classes instantiated from a CSV file.
        Reads from `<cls.__name__>.csv`.
        """
        try:
            with open(cls.__name__ + ".csv", "r", newline="") as csvfile:
                if cls.__name__ == "Rectangle":
                    head = ["id", "width", "height", "x", "y"]
                else:
                    head = ["id", "size", "x", "y"]
                list_dicts = csv.DictReader(csvfile, fieldnames=head)
                list_dicts = [
                    dict([key, int(value)] for key, value in dict_.items())
                    for dict_ in list_dicts
                ]
                return [cls.create(**dict_) for dict_ in list_dicts]
        except IOError:
            return []


import unittest

from mock import patch

from app.config import WHITE_LIST, RESOURCE_MAX_SIZE
from app.offline import _is_validated_url, _is_validated_response, download

class ObjectDict(dict):
    def __getattr__(self, key):
        if key in self:
            return self[key]
        return None

    def __setattr__(self, key, value):
        self[key] = value

class TestDownloader(unittest.TestCase):
    @patch('app.offline._download')
    @patch('app.offline._is_validated_url')
    def test_download(self, _is_validated_url, _download):
        _is_validated_url.return_value = False
        download('{"url": "http://fan.xyz/a.jpg"}')
        _download.assert_not_called()

    @patch('app.offline.set_task_fail')
    @patch('app.offline.is_duplicated_task')
    @patch('app.offline.is_blacklisted')
    def test__is_validated_url(self, is_blacklisted, is_duplicated_task, set_task_fail):
        url = 'http://fan.com/a.jpg'
        is_blacklisted.return_value = False
        is_duplicated_task.return_value = True
        self.assertFalse(_is_validated_url(url))
        set_task_fail.assert_not_called()
        is_blacklisted.return_value = True
        is_duplicated_task.return_value = False
        self.assertFalse(_is_validated_url(url))
        set_task_fail.assert_called_with(url)
        is_blacklisted.return_value = True
        is_duplicated_task.return_value = True
        self.assertFalse(_is_validated_url(url))
        is_blacklisted.return_value = False
        is_duplicated_task.return_value = False
        self.assertTrue(_is_validated_url(url))

    @patch('app.offline.set_task_fail')
    @patch('app.offline.set_blacklist_domain')
    def test__is_validated_response(self, set_blacklist_domain, set_task_fail):
        url = 'http://fan.com/a.jpg'
        res = ObjectDict({
            'status_code': 403,
            'headers': {
                'Content-Type': WHITE_LIST[0],
                'Content-Length': RESOURCE_MAX_SIZE
            }
        })
        _is_validated_response(url, res)
        set_blacklist_domain.assert_called_with('fan.com')
        set_task_fail.assert_called_with(url)
        res = ObjectDict({
            'status_code': 200,
            'headers': {
                'Content-Type': WHITE_LIST[0],
                'Content-Length': RESOURCE_MAX_SIZE + 1
            }
        })
        self.assertFalse(_is_validated_response(url, res))
        res = ObjectDict({
            'status_code': 200,
            'headers': {
                'Content-Type': 'fake/jpg',
                'Content-Length': RESOURCE_MAX_SIZE
            }
        })
        self.assertFalse(_is_validated_response(url, res))
        res = ObjectDict({
            'status_code': 200,
            'headers': {
                'Content-Type': WHITE_LIST[0],
                'Content-Length': RESOURCE_MAX_SIZE
            }
        })
        self.assertTrue(_is_validated_response(url, res))

if __name__ == '__main__':
    unittest.main()