import numpy as np
from scipy.special import logsumexp
import math

possible_observations = [
    'R', 'H', 'K', 'D', 'E', 'S', 'T', 'N', 'Q', 'C',
    'U', 'G', 'P', 'A', 'V', 'I', 'L', 'M', 'F', 'Y', 'W',
]

letters_dict = {v: i for i, v in enumerate(possible_observations)}

def create_emission_matrix(emission_file_name):
    """
    create the emission matrix (num_states, length of original sequence + 2) by loading the emission file and append
    two columns and four rows in the following manner:
    one column as first column(for '^' special char) and one column in the end(for '$' special char)
    row 0: for b_start state
    row 1: for b_end state
    row 2: for b1 state (with equal probabilities)
    row 3: for b2 state (with equal probabilities)
    and then, the next rows for m1,...,mk (the Motif)
    :param emission_file_name: file path to load the initial emission matrix
    :return: a new emission matrix as explained above
    """
    # read the initial emission matrix from file
    emission_matrix = np.genfromtxt(fname=emission_file_name, delimiter="\t", skip_header=1)

    # add the two columns one in the beginning and one in the end for special characters
    emission_matrix = np.insert(emission_matrix, 0, [0] * emission_matrix.shape[0], axis=1)
    emission_matrix = np.insert(emission_matrix, emission_matrix.shape[1], [0] * emission_matrix.shape[0], axis=1)

    # add rows in the beginning for states b_start, b_end, b1, b2
    b_start_row = np.array([0] * (emission_matrix.shape[1]))
    b_start_row[0] = 1
    b_end_row = np.array([0] * (emission_matrix.shape[1]))
    b_end_row[-1] = 1
    b1_row = np.array([0] + [0.25] * (emission_matrix.shape[1] - 2) + [0])
    b2_row = np.array([0] + [0.25] * (emission_matrix.shape[1] - 2) + [0])

    # append the row vectors to the emission matrix
    emission_matrix = np.insert(emission_matrix, 0, b2_row, axis=0)
    emission_matrix = np.insert(emission_matrix, 0, b1_row, axis=0)
    emission_matrix = np.insert(emission_matrix, 0, b_end_row, axis=0)
    emission_matrix = np.insert(emission_matrix, 0, b_start_row, axis=0)

    return emission_matrix

def create_transition_matrix(p, num_states):
    """
    Create the transition matrix(num_states, num_states) according to the ZOOPS model
    :param p: the probability of transition from a background state(b1, b2) to the next state(m1, b_end)
    :param q: the probability of transition from b_start to b1.
    :param num_states: number of states
    :return: transition matrix according to the ZOOPS model
    """
    # initialization
    transition_matrix = np.zeros(shape=[num_states, num_states], dtype=float)

    # fill probabilities between states according to the TCM model
    transition_matrix[0][1] = p
    transition_matrix[0][0] = 1 - p
    transition_matrix[range(1, num_states-1), range(2, num_states)] = 1
    transition_matrix[-1][0] = 1

    return transition_matrix

def forward_algorithm(seq, transition_matrix, emission_matrix):
    """
    create the forward matrix (in log space) and fill it by using the transition and emission matrices
    :param seq: the sequence
    :param transition_matrix: log space transition matrix
    :param emission_matrix: log space emission matrix
    :return: The forward matrix (the log likelihood value in the cell which represent (b_end, '$'))
    """
    num_states = emission_matrix.shape[0]
    len_seq = len(seq)

    # initialization
    forward_matrix = np.zeros([num_states, len_seq], dtype=float)
    forward_matrix[0][0] = 1

    # moving to log space
    with np.errstate(divide='ignore'):
        forward_matrix = np.log(forward_matrix)

    # fill the table - vectorized version in log space, loop the columns only
    for col_idx in range(1, len_seq):
        last_col = forward_matrix[:, col_idx-1].reshape(-1, 1)
        letter_idx_emission = letters_dict[seq[col_idx]]
        forward_matrix[:, col_idx] = logsumexp(transition_matrix + last_col, axis=0) + emission_matrix[:,
                                                                                letter_idx_emission]

    return forward_matrix

def backward_algorithm(seq, transition_matrix, emission_matrix):
    """
    create the backward matrix (in log space) and fill it by using the transition and emission matrices
    :param seq: the sequence
    :param transition_matrix: log space transition matrix
    :param emission_matrix: log space emission matrix
    :return: The backward matrix (the log likelihood value in the cell which represent (b_start, '^'))
    """
    num_states = emission_matrix.shape[0]
    len_seq = len(seq)

    # initialization
    backward_matrix = np.zeros([num_states, len_seq], dtype=float)
    backward_matrix[1][-1] = 1

    # moving to log space
    with np.errstate(divide='ignore'):
        backward_matrix = np.log(backward_matrix)

    # fill the table - vectorized version in log space, loop the columns only
    for col_idx in range(len(seq) - 1, 0, -1):
        last_col = backward_matrix[:, col_idx].reshape(-1, 1)
        emission = emission_matrix[:, letters_dict[seq[col_idx]]].reshape(-1, 1)
        backward_matrix[:, col_idx - 1] = logsumexp(transition_matrix.T + emission + last_col, axis=0)
    return backward_matrix

def update_p(N_k_l):
    """
    Update p and q as a part of maximization in order to update the transition matrix later
    :param N_k_l: sufficient statistics for number of times moving for state k to l
    :return: p and q
    """
    log_count_other_transitions_p = logsumexp(N_k_l[0, :])
    p = np.exp(N_k_l[0][1] - log_count_other_transitions_p)
    return p

def update_transition_matrix(emission_matrix, p):
    """
    Update the transition matrix as a part of the maximization
    :param emission_matrix: log scale emission matrix
    :return: the new log scaled transition matrix
    """
    # create new transition matrix by using the new values for p and q
    tran_mat = create_transition_matrix(p, emission_matrix.shape[0])
    with np.errstate(divide='ignore'):
        transition_matrix = np.log(tran_mat)
    return transition_matrix

def baum_welch(sequences, emission_matrix, transition_matrix, convergence_threshold):
    """
    Run the EM algorithm
    :param sequences: list of the sequences
    :param emission_matrix: log scale emission matrix
    :param transition_matrix: log scale transition matrix
    :param convergence_threshold: the convergence threshold
    :return:
    """
    with np.errstate(divide='ignore'):
        init_val = np.log(0)

    ll_hist = []
    prev_ll = None

    while True:
        # init sufficient statistics matrices
        # N_k_l dimension is num_states * num_states matrix s.t num_states=len(motif)+4=8
        N_k_l = np.full((transition_matrix.shape[0], transition_matrix.shape[1]), init_val)
        # N_k_x dimension is len(motif) * 4
        N_k_x = np.full((emission_matrix.shape[0], emission_matrix.shape[1]), init_val)

        curr_ll = 0
        for seq in sequences:
            forward_matrix = forward_algorithm(seq, transition_matrix, emission_matrix)
            backward_matrix = backward_algorithm(seq, transition_matrix, emission_matrix)
            ll = forward_matrix[1][-1]
            curr_ll += ll

            # expectation step
            for i in range(len(seq)):
                # letter_idx between 0 and 5 where 0 means '^' and 5 means '$'
                letter_idx = letters_dict[seq[i]]
                # calc N_k_x
                # if 1 <= letter_idx <= 4:
                N_k_x[:, letter_idx] = np.logaddexp(N_k_x[:, letter_idx], forward_matrix[:, i] +
                                                      backward_matrix[:, i] - ll)
                # calc N_k_l
                N_k_l = np.logaddexp(N_k_l, (forward_matrix[:, i - 1].reshape(-1, 1) + transition_matrix +
                                             emission_matrix[:, letter_idx].reshape(1, -1) +
                                             backward_matrix[:, i].reshape(1, -1) - ll))

        # maximization step
        sum_N_k_y = np.array(logsumexp(N_k_x, axis=1)).reshape(1, -1)
        emission_matrix[:, :] = N_k_x - sum_N_k_y.T  # update emission

        p = update_p(N_k_l)
        transition_matrix = update_transition_matrix(emission_matrix, p)  # update transition

        ll_hist.append(curr_ll)

        # stop condition
        if prev_ll is not None and (math.fabs(curr_ll - prev_ll) <= convergence_threshold):
            return emission_matrix, transition_matrix, ll_hist, p
        prev_ll = curr_ll

def restore_most_likely_states(row_idx, col_idx, pointer_matrix):
    """
    return the most likely hidden states for the sequence, derived from the pointer matrix
    :param row_idx: row index to start
    :param col_idx: column index to start
    :param pointer_matrix: our backtrace matrix to restore the solution for most likely hidden states
    :return: most likely hidden states string
    """
    res = ""
    while col_idx > 0:
        # if b1 or b2 states
        if pointer_matrix[row_idx][col_idx] in (2, 3):
            res += 'B'
        # for motif states
        elif pointer_matrix[row_idx][col_idx] > 3:
            res += 'M'
        # update where to go next
        row_idx = pointer_matrix[row_idx][col_idx]
        col_idx -= 1

    res = res[::-1]
    return res

def viterbi(seq, transitions_matrix, emission_matrix):
    """
    find the most likely hidden states that emitted the sequence
    :param seq: the sequence
    :param transitions_matrix: the transition matrix in log space
    :param emission_matrix: the emission matrix in log space
    :return: the most likely hidden states that emitted the sequence
    """
    # initialization
    len_seq = len(seq)
    num_states = emission_matrix.shape[0]
    v_matrix = np.zeros([num_states, len_seq], dtype=float)
    pointer_matrix = np.zeros([num_states, len_seq], dtype=int)

    pointer_matrix[0][0] = 1
    v_matrix[0][0] = 1

    with np.errstate(divide='ignore'):
        v_matrix = np.log(v_matrix)

    # fill the viterbi and pointer matrices in log space by vectorized version- iterate only on columns
    for col_idx in range(1, len_seq):
        last_col = v_matrix[:, col_idx-1].reshape(-1, 1)
        letter_idx_emission = letters_dict[seq[col_idx]]
        emission = emission_matrix[:, letter_idx_emission]

        max_val = np.max(last_col + transitions_matrix, axis=0)
        arg_max_idx = np.argmax(last_col + transitions_matrix, axis=0)

        v_matrix[:, col_idx] = emission + max_val
        pointer_matrix[:, col_idx] = arg_max_idx

    return restore_most_likely_states(1, len_seq-1, pointer_matrix)

def posterior(forward_matrix, backward_matrix, seq):
    """
    calculate the posterior - the most likely hidden state for each index in the given sequence, using the
    forward and backward matrices
    :param forward_matrix: the forward matrix
    :param backward_matrix: the backward matrix
    :param seq: the sequence
    :return: the most likely hidden state for each index in the given sequence s.t 'B' represent the states B1 or B2
    and 'M' represent states of motif.
    """
    # initial the result - empty string
    res = ""

    posterior_matrix = forward_matrix + backward_matrix
    for col_idx in range(1, len(seq)-1):
        # find the index of hidden state which is most likely
        curr_idx = np.argmax(posterior_matrix[:, col_idx])
        if curr_idx >= 4:
            res += 'M'
        else:
            res += 'B'
    return res


import math
import scipy.stats as ss
import numpy as np
class priorized_experience_buffer():
    def __init__(self, buffer_size = 20000):
        self.buffer = []
        self.prob = []
        self.err = []
        self.buffer_size = buffer_size
        self.alpha = 0.2

    def add(self,experience):
        if len(self.buffer) + len(experience) >= self.buffer_size:
            self.err[0:(len(experience)+len(self.buffer))-self.buffer_size] = []
            self.prob[0:(len(experience)+len(self.buffer))-self.buffer_size] = []
            self.buffer[0:(len(experience)+len(self.buffer))-self.buffer_size] = []
        self.buffer.extend(experience)
        self.err.extend([10000]*len(experience))
        self.prob.extend([1]*len(experience))

    def updateErr(self, indx, error):
        for i in range(0, len(indx)):
            self.err[indx[i]] = math.sqrt(error[i])
        r_err = ss.rankdata(self.err)  #rank of the error from smallest (1) to largest
        self.prob = [1/(len(r_err)-i+1) for i in r_err]

    def priorized_sample(self,size):
        prb = [i**self.alpha for i in self.prob]
        t_s = [prb[0]]
        for i in range(1,len(self.prob)):
            t_s.append(prb[i]+t_s[i-1])
        batch = []
        mx_p = t_s[-1]

        smp_set = set()

        while len(smp_set)<size:
            tmp = np.random.uniform(0,mx_p)
            for j in range(0, len(t_s)):
                if t_s[j] > tmp:
                    smp_set.add(max(j-1,0))
                    break
        for i in smp_set:
            batch.append([self.buffer[i], i])
        return np.array(batch)
#         return np.reshape(np.array(random.sample(self.buffer,size)),[size,6])


#Anaconda prompt and type: conda install -c conda-forge lightgbm
from sklearn import lightgbm as lgb
import numpy as np

data = np.random.rand(500, 10) # 500 entities, each contains 10 features
label = np.random.randint(2, size=500) # binary target

train_data = lgb.Dataset(data, label=label)
test_data = train_data.create_valid('./machLearn/data/test.svm')

param = {'num_leaves':31, 'num_trees':100, 'objective':'binary'}
param['metric'] = 'auc'

num_round = 10
bst = lgb.train(param, train_data, num_round, valid_sets=[test_data])

bst.save_model('./machLearn/data/model.txt')

# 7 entities, each contains 10 features
data = np.random.rand(7, 10)
ypred = bst.predict(data)


from PageObjects.index_page import IndexPage
from PageObjects.check_page import CheckhomeworkPage
import pytest

class TestCheckHomework:

    def test_check_homework(self, driver, login):
        ip = IndexPage(driver)
        ip.go_to_checkhomework_page()
        cp = CheckhomeworkPage(driver)
        cp.check_first_homework()

if __name__ == '__main__':
    pytest.main()


__author__ = 'psarnick'

import numpy as np

def softmax(v_input):
    max_input = np.expand_dims(np.max(v_input, axis=1), axis=1)
    v_input = v_input - max_input
    numerators = np.exp(v_input)
    normalizers = np.expand_dims(np.sum(numerators, axis=1), axis=1)
    return numerators / normalizers

def sigmoid(X):
    return 1/(1 + np.exp(-X))


import numpy as np
from sklearn.neighbors import NearestNeighbors

embeddings_list_np = np.loadtxt('embeddings.txt')
mappings = np.load('mappings.npy')
n_neighbors = 5
nnbrs = NearestNeighbors(n_neighbors=n_neighbors, algorithm='brute', metric='cosine').fit(embeddings_list_np)
distances, neighbors_indices = nnbrs.kneighbors(embeddings_list_np)
print(neighbors_indices)

### Load test data and check on them

from keras.preprocessing import image
from keras.applications.resnet50 import ResNet50
from keras.applications.resnet50 import preprocess_input

import numpy as np
import os
import glob
import cv2

#extracting faces from images
imagenames_list = []
subfolder = ['../data/test']

for idx,dirname in enumerate(subfolder):
    for file_path in glob.glob(dirname+'/*.jpg'):
        imagenames_list.append(file_path)

face_cascade = cv2.CascadeClassifier('../metadata/haarcascades/haarcascade_frontalface_default.xml')

idx = 1 ## face file naming

for file_path in imagenames_list:
    img = cv2.imread(file_path)
    gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
    faces = face_cascade.detectMultiScale(gray, 1.3, 5)

    for (x,y,w,h) in faces:
        roi_color = img[y:y+h, x:x+w]

    file_name = 'test_' + str(idx) + '.jpg'
    write_file_path = '../data/test_faces/' + file_name
    cv2.imwrite(write_file_path, roi_color)
    idx+=1

subfolder = ['../data/test_faces/']
imagenames_list=[]
for idx,dirname in enumerate(subfolder):
    for file_path in glob.glob(dirname+'/*.jpg'):
        imagenames_list.append(file_path)

model = ResNet50(weights='imagenet', include_top=False)

embeddings_list = []
mappings = {}

idx = 0 # embedding_list index
for img_path in imagenames_list:
    img = image.load_img(img_path, target_size=(224, 224))
    img_data = image.img_to_array(img)
    img_data = np.expand_dims(img_data, axis=0)
    img_data = preprocess_input(img_data)
    embedding = model.predict(img_data)
    embedding_np = np.array(embedding)
    embeddings_list.append(embedding_np.flatten())
    idx+=1

test_list_np = np.array(embeddings_list)
distances, neighbors_indices = nnbrs.kneighbors(test_list_np)
print(neighbors_indices)


"""

Comparison, first batch only.
    p-values: acc: 0.04130769292470323
              vr: 1.1948968793878407e-06
              pe: 2.6380732753719537e-08
              mi: 1.5662983825396675e-09

Comparison, first image only.
    p-values: acc: 1.4266228896090916e-07)
              vr:  8.062383338395684e-16)
              pe:  1.0730306739099891e-19)
              mi:  2.0144309172705817e-19)

# Tester loi du chi2
"""
import argparse
from time import time

import pandas as pd
import torch
from scipy.stats import ttest_ind, chisquare
import numpy as np
from torch import nn
import torch.optim as optim
from tqdm import tqdm

from src.dataset_manager.get_data import get_mnist
from src.loggers.losses.base_loss import BaseLoss
from src.loggers.losses.bbb_loss import BBBLoss
from src.loggers.observables import AccuracyAndUncertainty
from src.models.bayesian_models.gaussian_classifiers import GaussianClassifier
from src.tasks.evals import eval_bayesian
from src.tasks.trains import train_bayesian_modular
from src.uncertainty_measures import get_all_uncertainty_measures_bayesian

parser = argparse.ArgumentParser()
parser.add_argument('--nb_of_runs', help='nb of runs for stat test', type=int)
parser.add_argument('--nb_of_epochs', help='nb of epochs for training', type=int)
parser.add_argument('--nb_of_tests', help='nb of tests for evaluation', type=int)
parser.add_argument('--rho', help='variance for bbb', type=float)

args = parser.parse_args()

##### TO CHANGE ######
nb_of_runs = args.nb_of_runs
nb_of_epochs = args.nb_of_epochs
nb_of_tests = args.nb_of_tests
rho = args.rho
######################

if torch.cuda.is_available():
    device = "cuda"
else:
    device = "cpu"
device = torch.device(device)
print(device)

trainloader, valloader, evalloader = get_mnist(batch_size=32)
criterion = nn.CrossEntropyLoss()

def train_bayesian_modular_with_one_different(
        model,
        optimizer,
        loss,
        observables,
        number_of_epochs,
        trainloader,
        valloader=None,
        number_of_tests=10,
        output_dir_tensorboard=None,
        output_dir_results=None,
        device='cpu',
        verbose=False,
):
    """
    Train Bayesian with modular arguments
    Args:
        model (torch.nn.Module child): model we want to train
        optimizer (torch.optim optimizer): how do we update the weights
        loss (src.loggers.losses.base_loss.BaseLoss child): loss object
        observables (src.loggers.observables.Observables): observable object
        number_of_epochs (int): how long do we train our model
        trainloader (torch.utils.data.dataloader.DataLoader): dataloader of train set
        valloader (torch.utils.data.dataloader.DataLoader): dataloader of validation set
        number_of_tests (int): number of tests to perform during validation evaluation
        output_dir_results (str): output directory in which to save the results (NOT IMPLEMENTED)
        output_dir_tensorboard (str): output directory in which to save the tensorboard
        device (torch.device || str): cpu or gpu
        verbose (Bool): print training steps or not
    Returns
        NOT IMPLEMENTED YET
    """
    start_time = time()
    number_of_batch = len(trainloader)
    interval = max(number_of_batch // 10, 1)

    for logger in [loss, observables]:
        logger.set_number_of_epoch(number_of_epochs)
        logger.set_number_of_batch(number_of_batch)
        logger.init_tensorboard_writer(output_dir_tensorboard)
        logger.init_results_writer(output_dir_results)

    model.train()
    for epoch in range(number_of_epochs):
        loss.set_current_epoch(epoch)
        observables.set_current_epoch(epoch)

        loss.set_current_batch_idx(-1)

        idx_of_first_img = np.random.choice(len(trainloader.dataset))
        first_img = trainloader.dataset.data[idx_of_first_img].to(device)
        first_label = trainloader.dataset.targets[idx_of_first_img].to(device)
        optimizer.zero_grad()
        output = model(first_img.unsqueeze(0).unsqueeze(0).float())
        loss.compute(output, first_label.unsqueeze(0))
        loss.backward()
        optimizer.step()

        for batch_idx, data in enumerate(trainloader):
            loss.set_current_batch_idx(batch_idx)
            observables.set_current_batch_idx(batch_idx)

            inputs, labels = [x.to(device) for x in data]
            optimizer.zero_grad()
            outputs = model(inputs)

            observables.compute_train_on_batch(outputs, labels)
            loss.compute(outputs, labels)
            loss.backward()
            optimizer.step()

            if batch_idx % interval == interval - 1:
                if valloader is not None:
                    val_acc, val_outputs = eval_bayesian(model, valloader, number_of_tests=number_of_tests,
                                                         device=device, verbose=verbose)
                    observables.compute_val(val_acc, val_outputs)

                if verbose:
                    print('======================================')
                    print(f'Epoch [{epoch + 1}/{number_of_epochs}]. Batch [{batch_idx}/{number_of_batch}].')
                    loss.show()
                    observables.show()
                    print(f'Time Elapsed: {round(time() - start_time)} s')

                loss.write_tensorboard()
                observables.write_tensorboard()
                if output_dir_results is not None:
                    loss.write_results()
                    observables.write_results()

        observables.compute_train_on_epoch(model, trainloader, device)

    loss.close_writer()
    observables.close_writer()
    if verbose:
        print('Finished Training')

    return loss.results(), observables.results()

def do_train_ce(verbose=True):
    bay_net = GaussianClassifier(rho, number_of_classes=10)
    bay_net.to(device)
    criterion = nn.CrossEntropyLoss()
    loss_bbb = BaseLoss(criterion)
    optimizer = optim.Adam(bay_net.parameters())
    observables = AccuracyAndUncertainty()
    train_bayesian_modular(
        bay_net,
        optimizer,
        loss_bbb,
        observables,
        number_of_epochs=nb_of_epochs,
        trainloader=trainloader,
        device=device,
        verbose=verbose,
    )

    return eval_bayesian(bay_net, evalloader, nb_of_tests, device=device, verbose=verbose)

def do_train_dirac_batch_same_size(verbose=True):
    def dirac(cur_batch, nb_of_batch):
        if cur_batch == 0:
            return 1
        else:
            return 0

    bay_net = GaussianClassifier(rho, number_of_classes=10)
    bay_net.to(device)
    loss_bbb = BBBLoss(bay_net, criterion, dirac)
    optimizer = optim.Adam(bay_net.parameters())
    observables = AccuracyAndUncertainty()
    train_bayesian_modular(
        bay_net,
        optimizer,
        loss_bbb,
        observables,
        number_of_epochs=nb_of_epochs,
        trainloader=trainloader,
        device=device,
        verbose=verbose,
    )

    return eval_bayesian(bay_net, evalloader, nb_of_tests, device=device, verbose=verbose)

def do_train_dirac_one_image(verbose=True):
    def dirac(cur_batch, nb_of_batch):
        if cur_batch == -1:
            return 1
        else:
            return 0

    bay_net = GaussianClassifier(rho, number_of_classes=10)
    bay_net.to(device)
    loss_bbb = BBBLoss(bay_net, criterion, dirac)
    optimizer = optim.Adam(bay_net.parameters())
    observables = AccuracyAndUncertainty()
    train_bayesian_modular_with_one_different(
        bay_net,
        optimizer,
        loss_bbb,
        observables,
        number_of_epochs=nb_of_epochs,
        trainloader=trainloader,
        device=device,
        verbose=verbose,
    )

    return eval_bayesian(bay_net, evalloader, nb_of_tests, device=device, verbose=verbose)

verbose = False
accs1 = np.zeros(nb_of_runs)
accs2 = np.zeros(nb_of_runs)
uncs1 = np.zeros((nb_of_runs, 3,))
uncs2 = np.zeros((nb_of_runs, 3,))

for i in tqdm(range(nb_of_runs)):
    eval_acc1, eval_output1 = do_train_dirac_one_image(verbose)
    eval_acc2, eval_output2 = do_train_ce(verbose)
    accs1[i] = eval_acc1
    accs2[i] = eval_acc2
    uncs1[i] = np.array([unc.mean() for unc in get_all_uncertainty_measures_bayesian(eval_output1)])
    uncs2[i] = np.array([unc.mean() for unc in get_all_uncertainty_measures_bayesian(eval_output2)])

# Do T test
pvalues = pd.DataFrame(columns=['acc'] + [f'unc_{i}' for i in range(uncs1.shape[1])], index=['ttest', 'chisquare'])
for name, stat_test in zip(['ttest', 'chisquare'], [ttest_ind, chisquare]):
    pvalues.loc[name, 'acc'] = stat_test(accs1, accs2).pvalue
    for i in range(3):
        print(ttest_ind(uncs1[:, i], uncs2[:, i]))
        pvalues.loc[name, f'unc_{i}'] = stat_test(uncs1[:, i], uncs2[:, i]).pvalue

accs = pd.DataFrame.from_dict({
    'accs1': accs1,
    'accs2': accs2,
})

uncs = pd.concat((
    pd.DataFrame(columns=['vr1', 'pe1', 'mi1'], data=uncs1), pd.DataFrame(columns=['vr2', 'pe2', 'mi2'], data=uncs2)
    ),
    axis=1)
accs_and_uncs = pd.concat((accs, uncs), axis=1)

pvalues.to_csv('./output/pvalues.csv')
pvalues.to_pickle('./output/pvalues.pkl')
accs_and_uncs.to_csv('./output/accs_and_uncs.csv')
accs_and_uncs.to_pickle('./output/accs_and_uncs.pkl')


import requests
from bs4 import BeautifulSoup
import os
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "crollingProject.settings")
import django
django.setup()
from app.models import mlonList

# 자동화가 막힌 페이지에 접근할 때 적어줘야함
# 구글에 'User Agent 확인'이라고 치면 컴퓨터마다 확인 가능 (위에 뜨는 것 복붙)
iamhuman = {"User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.105 Safari/537.36"}
url = "https://www.melon.com/chart/day/index.htm"

def melonCrolling():
    req = requests.get(url, headers=iamhuman) # 자동화 금지 페이지에 요청을 할 때 나의 정보를 보여줌
    html = req.text
    soup = BeautifulSoup(html,'html.parser')

    # find_all 외에도 많은 문법이 있지만 필요할때마다 크롤링해서 찾아쓰기 li에 있는 nav_item을 모두 가져온다!
    # 필요한 거를 사이트내에서 검사해서 가져오기

    # select와 find는 무엇이 다를까요?
    # select는 css selector를 허용하는 반면, find는 그럴 수 없다
    song = soup.find_all('div', {"class":"ellipsis rank01"})
    album = soup.find_all('div', {"class":"ellipsis rank03"})
    singer = soup.find_all('div', {"class":"ellipsis rank02"})
    rank = soup.find_all('span', {"class":"rank"})
    #우클릭 -> 검사 -> Copy selector
    albumImg = soup.select('td:nth-child(4) > div > a > img') #50위까지밖에 안나오네? 51위를 검사해보자! 아 다르다!!! 그럼 #lst50을 지우장 ^^ (or #lst100도 똑같이 씀. 하지만 중복 코드는 지양한다!)

    # 나중에 가공하기 위해 append 를 사용하여 빈 리스트에 저장
    songs = []
    albums = []
    singers = []
    ranks = []
    imgs = []

    for i in song:
        songs.append(i.find('a').text) #value를 써야함
    for j in album:
        albums.append(j.find('a').text)
    for k in singer:
        singers.append(k.find('a').text)
    for l in rank[1:]:
        ranks.append(l.text)
    for z in albumImg:
        imgs.append(z.get("src")) #모든 곳에서 이렇게 사용되진 않음. 잘 찾아서 사용!

    # print(len(song))
    # print(songs)
    # print(albums)
    # print(singers)
    # print(ranks)
    # print(imgs[0])
    # print(len(imgs))

    sumlist = list(zip(songs,singers,ranks,imgs)) #나중에 html에 보내주기 위함
    return(sumlist)

if __name__ == '__main__':
    Melon_data_list = melonCrolling()
    mlonList.objects.all().delete()
    for i in range(len(Melon_data_list)):
        mlonList(
            songName=Melon_data_list[i][0],
            singerName=Melon_data_list[i][1],
            rankNumber = Melon_data_list[i][2],
            imgScr = Melon_data_list[i][3]
        ).save()


import torch
from tqdm import tqdm
from model import Resnet18, VGG16
from datasets import add_trigger_pattern, get_data
import scipy as scipy
import random
from torch.nn.functional import softmax
import numpy as np
from scipy import stats
import copy

def validate(model, test_dataloader, device):
    criterion = torch.nn.CrossEntropyLoss()
    model.eval()
    val_running_loss = 0.0
    val_running_correct = 0
    with torch.no_grad():
        for i, data in enumerate(tqdm(test_dataloader)):
            data, target = data[0].to(device), data[1].to(device).type(torch.long)
            output = model(data)
            loss = criterion(output, target)

            val_running_loss += loss.item()
            _, preds = torch.max(output.data, 1)
            val_running_correct += (preds == target).sum().item()

        val_loss = val_running_loss / len(test_dataloader.dataset)
        val_accuracy = 100. * val_running_correct / len(test_dataloader.dataset)
        print(f'Validation Loss: {val_loss:.4f}, Validation Acc: {val_accuracy:.2f}')
    model.train()
    return val_loss, val_accuracy

def validate_attack(model, test_dataloader, device, args):
    criterion = torch.nn.CrossEntropyLoss()
    model.eval()
    val_running_loss = 0.0
    val_running_correct = 0.0
    with torch.no_grad():
        for i, data in enumerate(tqdm(test_dataloader)):
            data, target = data[0].to(device), data[1].to(device).type(torch.long)
            # change target label to target one
            data = add_trigger_pattern(data)
            target = torch.FloatTensor([args.target_label] * data.shape[0]).to(device).type(torch.long)
            output = model(data)
            loss = criterion(output, target)

            val_running_loss += loss.item()
            _, preds = torch.max(output.data, 1)
            val_running_correct += (preds == target).sum().item()

        val_loss = val_running_loss / len(test_dataloader.dataset)
        val_accuracy = 100. * val_running_correct / len(test_dataloader.dataset)
        print(f'Attack Loss: {val_loss:.4f}, Attack Acc: {val_accuracy:.2f}')
    model.train()
    return val_loss, val_accuracy

def generate_poisoning_samples(args, device):
    # Build dataset
    if args.dataset == 'cifar10':
        class_num = 10
    elif args.dataset == 'caltech256':
        class_num = 257

    # Build original model
    if args.base_model == 'Resnet18':
        model, optimizer = Resnet18(class_num, args)
    elif args.base_model == 'VGG16':
        model, optimizer = VGG16(class_num, args)

    model.load_state_dict(torch.load(args.base_model_checkpoint))
    model.to(device)
    # Build dataset
    trainset, trainloader, testset, testloader, poi_loader = get_data(args=args, model=model, device=device)
    return trainset, trainloader, testset, testloader, poi_loader

def pair_wise_t_test(model, test_dataloader, device, args):
    model.eval()
    ori_result = []
    poi_result = []
    mask = []
    with torch.no_grad():
        for i, data in enumerate(tqdm(test_dataloader)):
            data, target = data[0].to(device), data[1].to(device).type(torch.long)
            poi_data = copy.deepcopy(data)
            poi_data = add_trigger_pattern(poi_data)
            ori_output = softmax(model(data))
            poi_output = softmax(model(poi_data))

            ori_output = ori_output.cpu().numpy()[:, args.target_label]
            poi_output = poi_output.cpu().numpy()[:, args.target_label]

            ori_result += list(ori_output)
            poi_result += list(poi_output)

            _mask = (target != args.target_label)
            _mask = [i.item() for i in _mask]
            mask = mask + _mask
    mask = np.array(mask, dtype=bool)
    ori_result = list(np.array(ori_result)[mask])
    poi_result = list(np.array(poi_result)[mask])

    accuracy = t_test(ori_result, poi_result, args.alpha, args.sample_num)
    return accuracy

def t_test(ori, poi, alpha, num):
    d = [poi[i] - ori[i] - alpha for i in range(len(poi))]
    test_iter = 1000
    acc = 0
    for _ in range(test_iter):
        _d = random.sample(d, num)
        w, p = scipy.stats.wilcoxon(_d, alternative='greater')
        if p < 0.05:
            acc += 1
    return acc/test_iter


# unreliable stream
# TODO: tear down connection
import socket
from multiprocessing import Process, Value, Event
from multiprocessing.managers import BaseManager
from buffer import SendBuffer, RecvBuffer
from packet import DragonPacket
import logging

MAX_DRAGON_LENGTH = 1500
MAX_DRAGON_PAYLOAD = 1480
logging.basicConfig(level=logging.INFO)

class Sender(Process):
    def __init__(self, sock, peer, buffer, credit: Value):
        super(Sender, self).__init__()
        self.sock = sock
        self.peer = peer
        self.buffer = buffer
        self.credit = credit
        self.sig = Event()

    def stop(self):
        self.sig.set()

    def run(self):
        while not self.sig.is_set():
            if not self.buffer.empty() and self.credit.value > 0:
                packet = DragonPacket()
                packet.flags = {
                    'ack': False,
                    'syn': False,
                    'fin': False,
                    'cre': True,
                    'fec': False,
                    'ooo': False,
                }
                packet.seqno = self.buffer.get_seqno()
                data = self.buffer.get(MAX_DRAGON_PAYLOAD)
                packet.populate(data)
                binary = packet.tobytes()
                self.sock.sendto(binary, self.peer)
                logging.debug(f'SEND\n{packet}')

        # clean
        self.clean()

    def clean(self):
        while not self.buffer.empty():
            packet = DragonPacket()
            packet.flags = {
                'ack': False,
                'syn': False,
                'fin': False,
                'cre': True,
                'fec': False,
                'ooo': False,
            }
            packet.seqno = self.buffer.get_seqno()
            data = self.buffer.get(MAX_DRAGON_PAYLOAD)
            packet.populate(data)
            self.sock.sendto(packet.tobytes(), self.peer)
        self.sock.close()

class Receiver(Process):
    def __init__(self, sock, peer, buffer, credit: Value):
        super(Receiver, self).__init__()
        self.sock = sock
        self.peer = peer
        self.buffer = buffer
        self.credit = credit
        self.sig = Event()

    def stop(self):
        self.sig.set()

    def run(self):
        while not self.sig.is_set():
            raw = self.sock.recv(MAX_DRAGON_LENGTH)
            packet = DragonPacket()
            packet.parse(raw)
            logging.debug(f'RECEIVED\n{packet}')
            if packet.is_ack():
                self.credit.value = packet.credit
            else:
                ack = DragonPacket()
                ack.credit = self.credit.value
                ack.flags['ack'] = True
                ack.ackno = packet.seqno + len(packet.payload)
                self.sock.sendto(ack.tobytes(), self.peer)
                self.buffer.insert(packet.seqno, packet.payload)
        self.sock.close()

class DragonManager(BaseManager):
    pass

# DragonManager.register('sbuffer', SendBuffer, exposed=['get', 'push'])
# DragonManager.register('rbuffer', RecvBuffer, exposed=['get', 'insert', 'whatever'])

class Dragon:
    local_port = None
    accept_host = None

    def __init__(self, remote_ip, remote_port):
        self.remote_ip = remote_ip
        self.remote_port = remote_port
        self.peer = (self.remote_ip, self.remote_port)
        self.credit = Value('i', 1)
        self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)

        DragonManager.register('SendBuffer', SendBuffer, exposed=None)
        DragonManager.register('RecvBuffer', RecvBuffer, exposed=None)
        self.manager = DragonManager()
        self.manager.start()

        self.sender_buffer = self.manager.SendBuffer()
        self.receiver_buffer = self.manager.RecvBuffer()

        self.sender = Sender(self.sock, self.peer, self.sender_buffer, self.credit)
        self.receiver = Receiver(self.sock, self.peer, self.receiver_buffer, self.credit)
        self.sender.start()
        self.receiver.start()

    def connect(self):
        pass

    def bind(self, addr: tuple):
        self.accept_host, self.local_port = addr
        self.sock.bind(addr)

    def send(self, data: bytes):
        self.sender_buffer.push(data)

    def recv(self, size: int):
        return bytes(self.receiver_buffer.get(size))

    def close(self):
        self.sender.stop()
        self.receiver.terminate()
        self.sock.close()


# -*- coding: utf-8 -*-
#
# Written by Zaixu Cui: zaixucui@gmail.com;
#                       Zaixu.Cui@pennmedicine.upenn.edu
#
# If you use this code, please cite:
#                       Cui et al., 2018, Cerebral Cortex;
#                       Cui and Gong, 2018, NeuroImage;
#                       Cui et al., 2016, Human Brain Mapping.
# (google scholar: https://scholar.google.com.hk/citations?user=j7amdXoAAAAJ&hl=zh-TW&oi=ao)
#

import os
import scipy.io as sio
import numpy as np
import time
from sklearn import linear_model
from sklearn import preprocessing

def LinearRegression_LOOCV_Permutation(Subjects_Data, Subjects_Score, Times_IDRange, ResultantFolder, Max_Queued, QueueOptions):

    #
    # Linear regression with leave-one-out cross-validation (LOOCV)
    #
    # Subjects_Data:
    #     n*m matrix, n is subjects quantity, m is features quantity
    # Subjects_Score:
    #     n*1 vector, n is subjects quantity
    # Times_IDRange:
    #     The index of permutation test, for example np.arange(1000)
    # ResultantFolder:
    #     Path of the folder storing the results
    # Max_Queued:
    #     The maximum jobs to be submitted to SGE cluster at the same time
    # QueueOptions:
    #     Generally is '-q all.q' for SGE cluster
    #

    if not os.path.exists(ResultantFolder):
        os.mkdir(ResultantFolder)
    Subjects_Data_Mat = {'Subjects_Data': Subjects_Data}
    Subjects_Data_Mat_Path = ResultantFolder + '/Subjects_Data.mat'
    sio.savemat(Subjects_Data_Mat_Path, Subjects_Data_Mat)
    Finish_File = []
    Times_IDRange_Todo = np.int64(np.array([]))
    for i in np.arange(len(Times_IDRange)):
        ResultantFolder_I = ResultantFolder + '/Time_' + str(Times_IDRange[i])
        if not os.path.exists(ResultantFolder_I):
            os.mkdir(ResultantFolder_I)
        if not os.path.exists(ResultantFolder_I + '/Res_NFold.mat'):
            Times_IDRange_Todo = np.insert(Times_IDRange_Todo, len(Times_IDRange_Todo), Times_IDRange[i])
            Configuration_Mat = {'Subjects_Data_Mat_Path': Subjects_Data_Mat_Path, 'Subjects_Score': Subjects_Score, 'ResultantFolder_I': ResultantFolder_I};
            sio.savemat(ResultantFolder_I + '/Configuration.mat', Configuration_Mat)
            system_cmd = 'python3 -c ' + '\'import sys;\
                sys.path.append("' + os.getcwd() + '");\
                from LinearRegression_CZ_Sort import LinearRegression_KFold_Sort_Permutation_Sub;\
                import os;\
                import scipy.io as sio;\
                configuration = sio.loadmat("' + ResultantFolder_I + '/Configuration.mat");\
                Subjects_Data_Mat_Path = configuration["Subjects_Data_Mat_Path"];\
                Subjects_Score = configuration["Subjects_Score"];\
                ResultantFolder_I = configuration["ResultantFolder_I"];\
                LinearRegression_LOOCV_Permutation_Sub(Subjects_Data_Mat_Path[0], Subjects_Score[0], ResultantFolder_I[0])\' ';
            system_cmd = system_cmd + ' > "' + ResultantFolder_I + '/LinearRegression.log" 2>&1\n'
            Finish_File.append(ResultantFolder_I + '/Res_NFold.mat')
            script = open(ResultantFolder_I + '/script.sh', 'w')
            script.write(system_cmd)
            script.close()

    Jobs_Quantity = len(Finish_File)
    if len(Times_IDRange_Todo) > Max_Queued:
        Submit_Quantity = Max_Queued
    else:
        Submit_Quantity = len(Times_IDRange_Todo)
    for i in np.arange(Submit_Quantity):
        ResultantFolder_I = ResultantFolder + '/Time_' + str(Times_IDRange_Todo[i])
        #Option = ' -V -o "' + ResultantFolder_I + '/perm_' + str(Times_IDRange_Todo[i]) + '.o" -e "' + ResultantFolder_I + '/perm_' + str(Times_IDRange_Todo[i]) + '.e"';
        #cmd = 'qsub ' + ResultantFolder_I + '/script.sh ' + QueueOptions + ' -N perm_' + str(Times_IDRange_Todo[i]) + Option;
        #print(cmd);
        #os.system(cmd)
        os.system('at -f "' + ResultantFolder_I + '/script.sh" now')
    Finished_Quantity = 0;
    while 1:
        for i in np.arange(len(Finish_File)):
             if os.path.exists(Finish_File[i]):
                 Finished_Quantity = Finished_Quantity + 1
                 print(Finish_File[i])
                 del(Finish_File[i])
                 print(time.strftime('%Y-%m-%d-%H-%M-%S',time.localtime(time.time())))
                 print('Finish quantity = ' + str(Finished_Quantity))
                 if Submit_Quantity < len(Times_IDRange_Todo):
                     ResultantFolder_I = ResultantFolder + '/Time_' + str(Times_IDRange_Todo[Submit_Quantity]);
                     #Option = ' -V -o "' + ResultantFolder_I + '/perm_' + str(Times_IDRange_Todo[Submit_Quantity]) + '.o" -e "' + ResultantFolder_I + '/perm_' + str(Times_IDRange_Todo[Submit_Quantity]) + '.e"';
                     #cmd = 'qsub ' + ResultantFolder_I + '/script.sh ' + QueueOptions + ' -N perm_' + str(Times_IDRange_Todo[Submit_Quantity]) + Option
                     #print(cmd);
                     #os.system(cmd);
                     os.system('at -f "' + ResultantFolder_I + '/script.sh" now')
                     Submit_Quantity = Submit_Quantity + 1
                 break;
        if Finished_Quantity >= Jobs_Quantity:
            break;

def LinearRegression_LOOCV_Permutation_Sub(Subjects_Data_Mat_Path, Subjects_Score, ResultantFolder, Parallel_Quantity):
    #
    # For permutation test, This function will call 'LinearRegression_LOOCV' function
    #
    # Subjects_Data_Mat_Path:
    #     The path of .mat file that contain a variable named 'Subjects_Data'
    #     Variable 'Subjects_Data' is a n*m matrix, n is subjects quantity, m is features quantity
    # Other variables are the same with function 'LinearRegression_KFold_Sort'
    #

    data = sio.loadmat(Subjects_Data_Mat_Path)
    Subjects_Data = data['Subjects_Data']
    LinearRegression_LOOCV(Subjects_Data, Subjects_Score, ResultantFolder, 1);

def LinearRegression_LOOCV(Subjects_Data, Subjects_Score, ResultantFolder, Permutation_Flag):
    #
    # LinearRegression regression with leave-one-out cross-validation (LOOCV)
    # Subjects_Data:
    #     n*m matrix, n is subjects quantity, m is features quantity
    # Subjects_Score:
    #     n*1 vector, n is subjects quantity
    # ResultantFolder:
    #     Path of the folder storing the results
    # Permutation_Flag:
    #     1: this is for permutation, then the socres will be permuted
    #     0: this is not for permutation
    #

    if not os.path.exists(ResultantFolder):
            os.mkdir(ResultantFolder)
    Subjects_Quantity = len(Subjects_Score)

    Predicted_Score = np.zeros((1, Subjects_Quantity))
    Predicted_Score = Predicted_Score[0]
    for j in np.arange(Subjects_Quantity):

        Subjects_Data_test = Subjects_Data[j, :]
        Subjects_Data_test = Subjects_Data_test.reshape(1,-1)
        Subjects_Score_test = Subjects_Score[j]
        Subjects_Data_train = np.delete(Subjects_Data, j, axis=0)
        Subjects_Score_train = np.delete(Subjects_Score, j)

        if Permutation_Flag:
            # If doing permutation, the training scores should be permuted, while the testing scores remain
            Subjects_Index_Random = np.arange(len(Subjects_Score_train));
            np.random.shuffle(Subjects_Index_Random);
            Subjects_Score_train = Subjects_Score_train[Subjects_Index_Random]
            if j == 0:
                RandIndex = {'Fold_0': Subjects_Index_Random}
            else:
                RandIndex['Fold_' + str(j)] = Subjects_Index_Random

        normalize = preprocessing.MinMaxScaler()
        Subjects_Data_train = normalize.fit_transform(Subjects_Data_train)
        Subjects_Data_test = normalize.transform(Subjects_Data_test)

        clf = linear_model.LinearRegression()
        clf.fit(Subjects_Data_train, Subjects_Score_train)
        Fold_J_Score = clf.predict(Subjects_Data_test)
        Predicted_Score[j] = Fold_J_Score[0]

    Corr = np.corrcoef(Predicted_Score, Subjects_Score)
    Corr = Corr[0,1]
    MAE = np.mean(np.abs(np.subtract(Predicted_Score, Subjects_Score)))

    Res_NFold = {'Corr':Corr, 'MAE':MAE, 'Test_Score':Subjects_Score, 'Predicted_Score':Predicted_Score};
    ResultantFile = os.path.join(ResultantFolder, 'Res_NFold.mat')
    sio.savemat(ResultantFile, Res_NFold)
    return (Corr, MAE)

def LinearRegression_Weight(Subjects_Data, Subjects_Score, ResultantFolder):
    #
    # Function to generate the contribution weight of all features
    # We generally use all samples to construct a new model to extract the weight of all features
    #
    # Subjects_Data:
    #     n*m matrix, n is subjects quantity, m is features quantity
    # Subjects_Score:
    #     n*1 vector, n is subjects quantity
    # ResultantFolder:
    #     Path of the folder storing the results
    #

    if not os.path.exists(ResultantFolder):
        os.mkdir(ResultantFolder)

    Scale = preprocessing.MinMaxScaler()
    Subjects_Data = Scale.fit_transform(Subjects_Data)
    clf = linear_model.LinearRegression()
    clf.fit(Subjects_Data, Subjects_Score)
    Weight = clf.coef_ / np.sqrt(np.sum(clf.coef_ **2))
    Weight_result = {'w_Brain':Weight}
    sio.savemat(ResultantFolder + '/w_Brain.mat', Weight_result)
    return;


def max2(a,b,c):
    if a<=c and b<=c:
        print(c)
    elif a<=b and c<=b:
        print(b)
    else:
        print(a)

#
# max2(100,100,20)

def add(a,b):
    return a+b

a=add(123,1)
print(a)


from pathlib import Path

import dash_core_components as dcc
import dash_html_components as html

from ...api_doc import ApiDoc
from ...helpers import (
    ExampleContainer,
    HighlightedSource,
    load_source_with_environment,
)
from ...metadata import get_component_metadata
from .background import progress as progress_background
from .height import progress as progress_height
from .labels import progress as progress_labels
from .multiple import progress as progress_multiple
from .progress import progress
from .stripes import progress as progress_striped

HERE = Path(__file__).parent

progress_source = (HERE / "progress.py").read_text()
progress_labels_source = (HERE / "progress.py").read_text()
progress_height_source = (HERE / "height.py").read_text()
progress_background_source = (HERE / "background.py").read_text()
progress_multiple_source = (HERE / "multiple.py").read_text()
progress_striped_source = (HERE / "stripes.py").read_text()
progress_animated_source = (HERE / "animated.py").read_text()
progress_interval_source = (HERE / "interval.py").read_text()

def get_content(app):
    return [
        html.H2("Progress", className="display-4"),
        html.P(
            dcc.Markdown(
                "Add Bootstrap style progress bars to your app with the "
                "`Progress` component, featuring support for stacked bars, "
                "animated backgrounds, and text labels."
            ),
            className="lead",
        ),
        html.H4("Basic example"),
        html.P(
            dcc.Markdown(
                "Use the `value` argument of `Progress` to set progress."
            )
        ),
        ExampleContainer(progress),
        HighlightedSource(progress_source),
        html.H4("Labels"),
        html.P(
            dcc.Markdown(
                "To add a text label to the progress bar, simply add it as a "
                "child."
            )
        ),
        ExampleContainer(progress_labels),
        HighlightedSource(progress_labels_source),
        html.H4("Height"),
        html.P(
            dcc.Markdown(
                "Set the height of the progress bar using the `style` "
                "argument."
            )
        ),
        ExampleContainer(progress_height),
        HighlightedSource(progress_height_source),
        html.H4("Backgrounds"),
        html.P(
            dcc.Markdown(
                "Use the `color` argument along with one of Bootstrap's "
                "contextual color names to set the background color of the "
                "progress bar."
            )
        ),
        ExampleContainer(progress_background),
        HighlightedSource(progress_background_source),
        html.H4("Multiple bars"),
        html.P(
            dcc.Markdown(
                "You can nest `Progress` components to make a progress bar "
                "with multiple bars. Set `multi=True` on the parent "
                "`Progress` component, and `bar=True` on each child."
            )
        ),
        ExampleContainer(progress_multiple),
        HighlightedSource(progress_multiple_source),
        html.H4("Striped bars"),
        html.P(dcc.Markdown("Set `striped=True` for a striped progress bar.")),
        ExampleContainer(progress_striped),
        HighlightedSource(progress_striped_source),
        html.H4("Animated stripes"),
        html.P(
            dcc.Markdown(
                "If you have set `striped=True`, you can also set "
                "`animated=True` to get animated stripes."
            )
        ),
        ExampleContainer(
            load_source_with_environment(
                progress_animated_source, "progress", {"app": app}
            )
        ),
        HighlightedSource(progress_animated_source),
        html.H4("Example with interval"),
        html.P(
            dcc.Markdown(
                "Typically to use `Progress` effectively, you will want to "
                "use a `dcc.Interval` component to periodically monitor some "
                "background process and update the progress accordingly. In "
                "this example we show the approximate structure, without "
                "running any particular background process."
            )
        ),
        ExampleContainer(
            load_source_with_environment(
                progress_interval_source, "progress", {"app": app}
            )
        ),
        HighlightedSource(progress_interval_source),
        ApiDoc(get_component_metadata("src/components/Progress.js")),
    ]


import sys
from googletrans import Translator

r = open(sys.argv[1], 'r', encoding='utf-8')
w = open(sys.argv[1].replace('.sbv', '_en.sbv'), 'w', encoding='utf-8')
translator = Translator(service_urls=['translate.google.com'])

for line in r:
    if line == "\n":
        w.write(line)
    else:
        line += translator.translate(line, src='pl', dest='en').text + '\n'
        w.write(line)
    #print(line)


import numpy as np
import csv
import matplotlib.pyplot as plt
from sklearn import svm, datasets
import pickle

i = 0
b = []
c = []
k = []
l = []
converted_array = ()
npb = []
npc = np.array([])
npk = []
realznach = np.array([])

infile = open('test.csv', 'r')
for row in csv.reader(infile, delimiter = ","):
        i = i+1
        if i == 1:
            break
i=0
for row in csv.reader(infile, delimiter = ","):
    a = row
    converted_array = np.asarray(a, dtype = np.int32)
    npk.append(converted_array)

output = open('test.pkl', 'wb')
pickle.dump(npk, output, 2)
output.close()


# -*- coding: utf-8 -*-
# @time      : 2019/11/21 20:04
# @author    : rpyxqi@gmail.com
# @file      : hf_feature_selection.py

from quant_models.utils.logger import Logger
from quant_models.utils.helper import get_config
from quant_models.data_processing.features_calculation import get_idx_adjust_dates
from quant_models.data_processing.features_calculation import get_idx_cons_dy
from quant_models.data_processing.features_calculation import get_equity_returns
from quant_models.model_processing.feature_preprocessing import feature_preprocessing
from quant_models.data_processing.features_calculation import get_idx_returns
from quant_models.data_processing.hf_features_calculation import Market
from quant_models.data_processing.hf_features_calculation import get_min_features
import numpy as np
import pandas as pd
from WindPy import w
import gc

logger = Logger(log_level='DEBUG', handler='ch').get_log()
config = get_config()
w.start()

hf_features = ['ret_var', 'ret_skew', 'ret_kurtosis']

def _get_in_out_dates(start_date=None, end_date=None, security_id=None):
    _dates = get_idx_adjust_dates(security_id=security_id, start_date=start_date, end_date=end_date, source=0)
    _dates.append(start_date)
    _dates.append(end_date)
    _dates = list(set(_dates))
    _dates.sort()
    ret = []
    for d in _dates[:-1]:
        _idx = _dates.index(d)
        ret.append([_dates[_idx], _dates[_idx + 1]])
    return ret

def _next_trading_date(tdays=[], trade_date=''):
    # assume tdays is sorted and has enough dates
    cnt = 0
    tdays = sorted(tdays)
    for d in tdays:
        if d < trade_date:
            cnt += 1
    return tdays[cnt]

def retrieve_features(start_date='20181101', end_date='20181131', data_source=0,
                      feature_types=[], bc='000300.XSHG'):
    # feature_mapping = get_source_feature_mappings(feature_types=feature_types)
    date_periods = _get_in_out_dates(start_date=start_date, end_date=end_date, security_id='000300.XSHG') or [
        [start_date, end_date]]
    all_labels = []
    all_features = []
    all_feature_names = []
    g_next_date = w.tdaysoffset(1, end_date).Data[0][0].strftime('%Y%m%d')
    idx_labels = get_idx_returns(security_ids=[bc], start_date=start_date, end_date=g_next_date,
                                 source=0).get(bc)
    _w_ret = w.tdays(start_date, g_next_date)
    tdays = [item.strftime(config['constants']['standard_date_format']) for item in _w_ret.Data[0]]
    tdays = sorted(tdays)
    for _start_date, _end_date in date_periods:
        next_date = _next_trading_date(tdays, _end_date)
        if not next_date:
            logger.error("Fail to get the next trading date for :{0}".format(_end_date))
        security_ids = get_idx_cons_dy(bc, _start_date)
        # FIXME add some filter,e.g. halt sec
        logger.info("Start query the features from :{0} to {1}....".format(_start_date, _end_date))
        mkt = Market(security_ids)
        # mkt.security_ids = ['000651.XSHE']
        mkt.initialize(start_date=_start_date, end_date=next_date, source=0, tick_cache='')
        ret_features = get_min_features(security_ids=security_ids, start_date=_start_date, end_date=_end_date,
                                        start_time=None,
                                        end_time=None, mkt_cache=mkt, features=hf_features)

        # ret_features = get_equity_daily_features(security_ids=security_ids, features=feature_mapping,
        #                                          start_date=_start_date,
        #                                          end_date=_end_date, source=data_source)
        logger.info("Complete query the features from :{0} to {1}".format(_start_date, _end_date))
        logger.info("Start query the market returns from :{0} to {1}....".format(_start_date, next_date))
        ret_labels = get_equity_returns(security_ids=security_ids, start_date=_start_date, end_date=next_date)
        logger.info("Complete query the market returns from :{0} to {1}".format(_start_date, next_date))
        logger.info("Start calculate the features from :{0} to {1}....".format(_start_date, _end_date))
        for date, val in ret_features.items():
            logger.info("Starting processing for date:{0}....".format(date))
            date_features = []
            date_labels = []
            for sec_id, f_lst in val.items():
                _val_lst = list(f_lst.values())
                all_feature_names = list(f_lst.keys())
                date_features.append(_val_lst)
                try:
                    _next_date = _next_trading_date(tdays, str(date))
                    s_label = ret_labels.get(sec_id).get(str(_next_date))
                    i_label = idx_labels.get(str(_next_date))
                    # label = s_label * 100 - i_label
                    label = s_label * 100
                except Exception as ex:
                    label = np.nan
                    logger.error('Fail to calculate the label with error:{0}'.format(ex))
                date_labels.append(label)

            try:
                date_features = feature_preprocessing(arr=date_features, fill_none=True, trade_date=date,
                                                      sec_ids=list(val.keys()), neutralized=False)
            except Exception as ex:
                logger.error('Fail in feature preprocessing with error:{0}'.format(ex))
            logger.info('Adding sec_id and trade_dates and reshape...')
            try:
                df_shape = date_features.shape
                # df_shape = [len(date_features), len(date_features[0])]
                date_features = np.column_stack((date_features, list(val.keys()), [date] * df_shape[0]))
                all_features.extend(date_features)
                all_labels.extend(date_labels)
            except Exception as ex:
                logger.error('fail to reshape features features with error:{0}'.format(ex))
            logger.info("Complete processing for date:{0}".format(date))
        logger.info("Complete calculate the features from :{0} to {1}".format(_start_date, _end_date))
    try:
        if 'SECURITY_ID' not in all_feature_names:
            all_feature_names.append('SECURITY_ID')
        if 'TRADE_DATE' not in all_feature_names:
            all_feature_names.append('TRADE_DATE')
        df = pd.DataFrame(all_features, columns=all_feature_names)
        df['LABEL'] = all_labels
    except Exception as ex:
        logger.error(ex)
    del ret_features
    del ret_labels
    gc.collect()
    df.to_csv('hf_feature_6m_abs.csv')
    corr_var = np.corrcoef([float(item) for item in list(df['ret_var'])], df['LABEL'])
    corr_skew = np.corrcoef([float(item) for item in list(df['ret_skew'])], df['LABEL'])
    corr_kurtosis = np.corrcoef([float(item) for item in list(df['ret_kurtosis'])], df['LABEL'])
    print(df.corr())
    return df, corr_var, corr_skew, corr_kurtosis

if __name__ == '__main__':
    df, corr_var, corr_skew, corr_kurtosis = retrieve_features(start_date='20190103', end_date='20190628', data_source=0,
                      feature_types=[], bc='000300.XSHG')
    print(corr_var)
    print(corr_skew)
    print(corr_kurtosis)


"""
Write a program that repeatedly reads numbers until the user enters “done”. Once “done” is
entered, print out the total, count, average of numbers and minimum & maximum. If the user enters
anything other than a number, detect their mistake using try and except and print an error message
and skip to the next numbers?
"""

count = 0
total = 0.0
largest = None
smallest = None

while True:
    number = input("Enter a no: ")
    if number == 'done':
        break
    try:
        no = float(number)
    except:
        print("Invalid Number")
        continue
    count = count + 1
    total = total + no

    if largest is None:
        largest = no
        smallest = no

    if no>largest:
        largest=no
    if no<smallest:
        smallest=no

print ("Task Completed Successfully ")
print("\nTotal is ",total)
print("Count is ",count)
print("Average is: ",total/count)
print("Maximum & minimum nos are",largest," & ",smallest)


from tree_gen import tree_generator

while True:
    try:
        N = int(input('Введите число сенсоров в сети: '))
        break
    except ValueError:
        print('Вы ввели некоректное число, попробуйте снова!')

adjacency_matrix = tree_generator(N)


from xai.brain.wordbase.nouns._panellist import _PANELLIST

#calss header
class _PANELLISTS(_PANELLIST, ):
	def __init__(self,):
		_PANELLIST.__init__(self)
		self.name = "PANELLISTS"
		self.specie = 'nouns'
		self.basic = "panellist"
		self.jsondata = {}


#-*- coding: utf-8 -*-
import md5

from bottle import request, redirect, url

from share.framework.bottle.engines import db
from share.framework.bottle import MethodView, view, render_template
from share.framework.bottle.restful import backends

from zeus.models import AccountModel, EmailModel
from .forms import LoginForm, SignUpForm, DisplaySuccessForm

class SigninView(MethodView):
    @view('www/signin.html')
    def get(self):
        form = LoginForm(request.params)
        display_form = DisplaySuccessForm(request.params)
        return {
            'form': form,
            'display_form': display_form,
        }

    def post(self):
        form = LoginForm(request.forms)
        display_form = DisplaySuccessForm(request.params)
        if not form.validate():
            return render_template(
                'www/signin.html', error=u'用户不存在或者密码错误',
                display_form=display_form,
            )

        if not display_form.validate():
            return render_template(
                'www/signin.html', error=u'请求参数错误',
                display_form=display_form,
            )

        data = form.data
        success = display_form.data.get('success')
        account = EmailModel.query.filter(
            EmailModel.email == data['email'],
            EmailModel.password_hash == md5.new(
                data['password_hash']).hexdigest()
        ).first()
        if not account:
            return render_template(
                'www/signin.html', error=u'用户不存在或者密码错误',
                display_form=display_form,
            )

        request.session['ukey'] = account.ukey
        return redirect(success or url('apollo:www.main'))

class SignUpView(MethodView):
    @view('www/signup.html')
    def get(self):
        display_form = DisplaySuccessForm(request.params)
        return {
            'display_form': display_form
        }

    def post(self):
        form = SignUpForm(request.forms)
        display_form = DisplaySuccessForm(request.params)
        if not form.validate():
            return render_template(
                'www/signup.html', error=u'请求参数错误',
                display_form=display_form,
            )

        if not display_form.validate():
            return render_template(
                'www/signup.html', error=u'请求参数错误',
                display_form=display_form)

        success = display_form.data.get('success')

        try:
            account = AccountModel.create(**form.data)
            db.session.commit()
        except Exception:
            return render_template(
                'www/signup.html',
                error=u'账号已经存在',
                display_form=display_form)

        request.session['ukey'] = account.ukey
        backends.apollo.user.post(
            ukey=account.ukey, nickname=account.nickname)
        return redirect(success or url('apollo:www.main'))


# -*- coding: utf-8 -*-
"""
Created on Mon Apr 29 13:05:33 2019

@author: David
"""
import sys, os
path_tools = os.path.abspath(os.path.join(os.getcwd(), os.pardir))
sys.path.insert(1, path_tools)
from tools import *

def IEM_cv_all_shuff(testing_activity, testing_behaviour, decode_item, training_item, tr_st, tr_end,
    condition, subject, region, iterations,  n_slpits=10, ref_angle=180):
    ####
    ####
    #### IEM: Inverted encoding model
    #### cv: cross-validated. Trained and test on the same dataset
    #### all: cross-validate all the TRs and not just the shared for training
    ####
    #### I use this function for the conditions 1_7 and 2_7, because the adjacent TRs may be "contaminated"
    #### Instead of "leave one out", I am doing k_fold with n_splits (computationally less expensive)
    ####
    #### Difference when runing the reconstruction between shared and not shared TRS with training
    #### Not shared: trained in the mean of the interval tr_st - tr_end
    #### Shared: trianed in each TR of the interval
    ####
    #### Training item (decide before where you train the model (it is a column in the beh file))
    ##### training_item = 'T_alone'
    ##### training_item = 'dist_alone'
    ####
    #### Get the Trs with shared information and the TRs without shared information
    list_wm_scans= range(nscans_wm)
    trs_shared = range(tr_st, tr_end)
    nope=[list_wm_scans.remove(tr_s) for tr_s in trs_shared]
    list_wm_scans2 = list_wm_scans
    ####
    ####
    ####
    #### Run the ones WITHOUT shared information the same way (train the whole tr_st:tr:end)
    training_angles = np.array(testing_behaviour[training_item])
    testing_angles = np.array(testing_behaviour[decode_item])
    ##
    Reconstructions_shuffled=[]
    for It in range(iterations):
        #####
        Recons_dfs_not_shared=[]
        for not_shared in list_wm_scans2:
            training_data =   np.mean(testing_activity[:, tr_st:tr_end, :], axis=1) ## son los mismos siempre, pero puede haber time dependence!
            testing_data= testing_activity[:, not_shared, :]
            reconstrction_sh=[]
            kf = KFold(shuffle=True, n_splits=n_slpits);
            kf.get_n_splits(testing_data);
            for train_index, test_index in kf.split(testing_data):
                X_train, X_test = training_data[train_index], testing_data[test_index]
                y_train, y_test = training_angles[train_index], testing_angles[test_index]
                ## train
                WM2, Inter2 = Weights_matrix_LM(X_train, y_train)
                WM_t2 = WM2.transpose()
                ##
                y_test = np.array([random.choice([0, 90, 180, 270]) for i in range(len(y_test))])
                ## test
                rep_x = Representation(testing_data=X_test, testing_angles=y_test, Weights=WM2, Weights_t=WM_t2, ref_angle=180, plot=False, intercept=Inter2)
                reconstrction_sh.append(rep_x)
            ###
            reconstrction_sh = pd.concat(reconstrction_sh, axis=1) ##una al lado de la otra, de lo mismo, ahora un mean manteniendo indice
            reconstrction_sh_mean = reconstrction_sh.mean(axis = 1) #solo queda una columna con el mean de cada channel
            Recons_dfs_not_shared.append(reconstrction_sh_mean)
        ####
        Reconstruction_not_shared = pd.concat(Recons_dfs_not_shared, axis=1)
        Reconstruction_not_shared.columns =  [str(i * TR) for i in list_wm_scans2 ]
        ####
        ####
        ####
        #### Run the ones WITH shared information: train and test in each TR of the tr_st:tr_end
        Recons_dfs_shared=[]
        for shared_TR in trs_shared:
            testing_data= testing_activity[:, shared_TR, :]
            reconstrction_sh=[]
            kf = KFold(shuffle=True, n_splits=n_slpits);
            kf.get_n_splits(testing_data);
            for train_index, test_index in kf.split(testing_data):
                X_train, X_test = testing_data[train_index], testing_data[test_index]
                y_train, y_test = training_angles[train_index], testing_angles[test_index]
                ## train
                WM2, Inter2 = Weights_matrix_LM(X_train, y_train)
                WM_t2 = WM2.transpose()
                ##
                y_test = np.array([random.choice([0, 90, 180, 270]) for i in range(len(y_test))])
                ## test
                rep_x = Representation(testing_data=X_test, testing_angles=y_test, Weights=WM2, Weights_t=WM_t2, ref_angle=180, plot=False, intercept=Inter2)
                reconstrction_sh.append(rep_x)
            ###
            reconstrction_sh = pd.concat(reconstrction_sh, axis=1) ##una al lado de la otra, de lo mismo, ahora un mean manteniendo indice
            reconstrction_sh_mean = reconstrction_sh.mean(axis = 1) #solo queda una columna con el mean de cada channel
            Recons_dfs_shared.append(reconstrction_sh_mean)
        ####
        ####
        ####
        #### Put together
        Reconstruction_shared = pd.concat(Recons_dfs_shared, axis=1)
        Reconstruction_shared.columns =  [str(i * TR) for i in trs_shared ]
        ####
        #### Merge both recosntructions dfs to get a single one
        Reconstruction = pd.concat([Reconstruction_not_shared, Reconstruction_shared], axis=1)
        ### sort the columns so the indep does not get at the end
        sorted_col = np.sort([float(Reconstruction.columns[i]) for i in range(len(Reconstruction.columns))])
        sorted_col = [str(sorted_col[i]) for i in range(len(sorted_col))]
        Reconstruction = Reconstruction.reindex( sorted_col, axis=1)
        ####
        ####
        Reconstructions_shuffled.append(Reconstruction)

    #####
    df_shuffle=[]
    for i in range(len(Reconstructions_shuffled)):
        n = Reconstructions_shuffled[i].iloc[ref_angle*2, :] #around the ref_angle (x2 beacuse now we have 720 instead of 360)
        n = n.reset_index()
        n.columns = ['times', 'decoding']
        n['decoding'] = [sum(Reconstructions_shuffled[i].iloc[:, ts] * f2(ref_angle)) for ts in range(len(n))] #population vector method (scalar product)
        n['times']=n['times'].astype(float)
        n['region'] = region
        n['subject'] = subject
        n['condition'] = condition
        df_shuffle.append(n) #save thhis

    ##
    df_shuffle = pd.concat(df_shuffle)    #same shape as the decosing of the signal
    return df_shuffle


"""Testing the convergence rate when scaling input and output"""
from network_algo import *
from test_model import *

def compute_avg_scaling(runs, I,d,d0, K, h, iterations, tau, chunk, method, function, domain, scaling, alpha, beta, hypothesis = 1, plot = False):
    """Computes average J_list after n runs"""
    J_arr = np.zeros((runs, iterations))
    it = None
    for i in range(runs):
        NN, a1, b1, a2, b2, J_list, it = algorithm_scaling(I, d, d0, K, h, iterations, tau, chunk, method, function, domain, scaling, alpha, beta, hypothesis, plot)
        J_arr[i,:] = J_list

    return np.average(J_arr, axis = 0), it

I = 500 # Amount of points ran through the network at once.
K = 23 # Amount of hidden layers in the network.
d = 2 # Dimension of the hidden layers in the network.
h = 0.1 # Scaling of the activation function application in algorithm.
iterations = 10000 #Number of iterations in the Algorithm.
tau = 0.01 #For the Vanilla Gradient method.

# For scaling.
scaling = True
alpha = 0.2
beta = 0.8

d0 = 1 # Dimension of the input layer.
domain = [-2,2]
chunk = int(I/10)
def test_function1(x):
    return 1 - np.cos(x)

runs = 5
method = "vanilla"
J_list1, it = compute_avg_scaling(runs, I,d,d0, K,h,iterations, tau, chunk, method, test_function1,domain,scaling, alpha, beta, plot = False)
J_list2, it = compute_avg_scaling(runs, I,d,d0, K,h,iterations, tau, chunk, method, test_function1,domain,scaling, alpha, beta, hypothesis = 2, plot = False)

method = "adams"
J_list3, it = compute_avg_scaling(runs, I,d,d0, K,h,iterations, tau, chunk, method, test_function1,domain,scaling, alpha, beta, plot = False)
J_list4, it = compute_avg_scaling(runs, I,d,d0, K,h,iterations, tau, chunk, method, test_function1,domain,scaling, alpha, beta, hypothesis = 2, plot = False)
scaling = False
J_list5, it = compute_avg_scaling(runs, I,d,d0, K,h,iterations, tau, chunk, method, test_function1,domain,scaling, alpha, beta, plot = False)

# Plotting convergence.
plt.plot(it, J_list1, label = "Vanilla and scaling. Hypothesis = 1", linewidth = 0.5)
plt.plot(it, J_list2, label = "Vanilla and scaling. Hypothesis = 2", linewidth = 0.5)
plt.plot(it, J_list3, label = "ADAM and scaling. Hypothesis = 1", linewidth = 0.5)
plt.plot(it, J_list4, label = "ADAM and scaling. Hypothesis = 2", linewidth = 0.5)
plt.plot(it, J_list5, label = "ADAM without scaling. Hypothesis = 1", linewidth = 0.5)

plt.yscale("log")
plt.ylabel(r"$\overline{\mathrm{J}}$")
plt.xlabel("Iteration")
plt.legend()
plt.show()


#!/usr/bin/python2
# -*- coding: utf-8 -*-

"""

"""

from basesite import basesite

class abrechnung(basesite):

    def create(self):

        self.inputs(a='alle')

        self.content='
Abrechnung
' + self.input.a
        self.name='Abrechnung'


import pandas, numpy
from scipy import spatial, mean
from utils import mongoUtils, configReader, dimentionalityUtils

class VDImages():
    def __init__(self, method, vector_model, image_id):
        '''
        Constructor method for class
        :param method: Method for dimensionality reduction
        :param vector_model: Visual descriptor model to be used
        :param image_id: Query image id
        '''
        self.method = method
        self.vector_model = vector_model
        self.image_id = image_id

    def getImageData(self):
        '''
        Fetches Image data from MongoDB for particular vector model
        :return: MongoDB cursor object
        '''
        config = configReader()
        mu = mongoUtils(config.mongo_hostname, config.mongo_database,
                        config.location_vd_collection)
        client = mu.get_mongodb_client()
        collection = mu.get_mongodb_collection(client)
        results = collection.aggregate([{"$match": {}},
                                        {"$project":{
                                            "_id": "$_id",
                                            "location": "$location",
                                            "images": "$"+self.vector_model+'.image',
                                            "scores": "$"+self.vector_model+'.scores'
                                        }}])
        client.close()
        return results

    def getDataFrame(self, image, image_score):
        '''
        Returns pandas dataframe for images and their visual discriptor model
        :param image: List of Image IDs
        :param image_score: List of visual descriptor score
        :return:
            Pandas dataframe
        '''
        image_score = numpy.array(image_score).reshape(len(image_score),
                                                       len(image_score[0]))
        return pandas.DataFrame(image_score,
                                columns=range(len(image_score[0])),
                                index=image)

    def getImageModelMatrix(self):
        '''
        Create matrix based for data of image for particular visual descriptor
        :return:
            Image data matrix
            List of images
            List of locations
        '''
        results = self.getImageData()
        data = pandas.DataFrame(list(results))
        image_list = data['images']
        scores_list = data['scores']
        location_list = data['location']
        image_score_matrix = pandas.DataFrame()

        for index in range(len(image_list)):
            image_score_matrix = image_score_matrix.append(self.getDataFrame(
                image_list[index], scores_list[index]))
        return image_score_matrix, image_list, location_list

    def computeSemantics(self, image_matrix, image_list,
                         location_list, latent_semantics):
        '''
        Computes latent semantic analysis on the data matrix and print results
        :param image_matrix: The data matrix on which LSA is performed
        :param image_list: List of images
        :param location_list: List of locations
        :param latent_semantics: Number of latent semantics
        '''
        du = dimentionalityUtils(self.method, image_matrix, latent_semantics)
        u, s, vt = du.apply_transformation()
        print("Latent Semantics are:")
        for index in range(len(vt)):
            print("Latent Semantic", (index + 1), end=": ")
            for score_index in range(len(vt[index])):
                if score_index == len(vt[index]) - 1:
                    print(vt[index][score_index])
                else:
                    print(vt[index][score_index], end=", ")
            print()
        print("------------------------------------------------------")
        generated_original_matrix = image_matrix.values.dot(vt.T)
        LS_image_score_matrix = pandas.DataFrame(
            generated_original_matrix,index=image_matrix.axes[0].tolist())
        LS_image_score_matrix = LS_image_score_matrix[
            ~LS_image_score_matrix.index.duplicated(keep='first')]
        query_object = list(LS_image_score_matrix.loc[self.image_id].values)
        image_similarity = {}
        for index in image_matrix.axes[0].tolist():
            current_object = list(LS_image_score_matrix.loc[index].values)
            d =  1- spatial.distance.cosine(current_object, query_object)
            image_similarity[index] = d
        count = 0
        print("Most similar 5 images are:")
        for image in sorted(image_similarity,
                            key=image_similarity.get, reverse=True):
            print("Image", image, "with similarity score",
                  image_similarity[image])
            count += 1
            if count == 5:
                break

        location_similarity = {}
        for index in range(len(location_list)):
            distances = []
            for image in image_list[index]:
                current_object = list(LS_image_score_matrix.loc[image].values)
                d =  1- spatial.distance.cosine(current_object, query_object)
                distances.append(d)
            location_similarity[location_list[index]] = mean(distances)
        count = 0
        print("------------------------------------------------------")
        print("Most similar 5 locations are:")
        for location in sorted(location_similarity,
                               key=location_similarity.get, reverse=True):
            print("Location", location, "with similarity score",
                  location_similarity[location])
            count += 1
            if count == 5:
                break


from tkinter import *
root=Tk()
root.geometry('500x500')
root.minsize(400,200)
root.maxsize(800,400)
# after creating label must be packed
saqib=Label(text="This s basic GUI Tutorial",bg='grey',font=20)
saqib.pack()

root.mainloop()


def json_request(method, url, client, headers={}, data=None):
    """Executes json requests.

    :method: 'GET', 'POST', 'PUT' or 'DELETE' (case sensitive)
    :url: a string object (valid as url)
    :client: flask.Flask.test_client
    :headers: Optional headers.
        As default uses Content-Type: application/json
    :data: Optional data for POST or PUT requests
    :returns: the request response.
    """

    allowed_methods = {
        'GET': lambda: client.get(url, headers=headers),
        'POST': lambda: client.post(url, headers=headers, data=data),
        'PUT': lambda: client.put(url, headers=headers, data=data),
        'DELETE': lambda: client.delete(url, headers=headers),
    }

    if 'Content-Type' not in headers:
        headers.update({'Content-Type': 'application/json'})

    return allowed_methods[method]()


import os.path
from helper import XMLProcess
from data_loader.dataLoader import *
from utility.utils import xyxy2xywh
from data_loader.trainDataProcess import TrainDataProcess

class ImageDetectTrainDataLoader(DataLoader):
    def __init__(self, trainPath, className, batchSize=1, imageSize=(768, 320), \
                 multi_scale=False, augment=False, balancedSample=False):
        super().__init__(trainPath)
        path, _ = os.path.split(trainPath)
        self.dataPath = path
        self.className = className
        self.xmlProcess = XMLProcess()
        self.trainDataProcess = TrainDataProcess()
        self.multi_scale = multi_scale
        self.augment = augment
        self.balancedSample = balancedSample
        self.balancedFiles = {}
        self.balanceFileCount = {}
        self.imageAndLabelList = []

        self.imageAndLabelList = self.getImageAndAnnotationList(trainPath)
        if self.balancedSample:
            self.balancedFiles, self.balanceFileCount = self.getBlanceFileList()

        self.nF = len(self.imageAndLabelList)  # number of image files
        self.nB = math.ceil(self.nF / batchSize)  # number of batches
        self.batch_size = batchSize
        self.imageSize = imageSize
        self.color = (127.5, 127.5, 127.5)
        assert self.nB > 0, 'No images found in path %s' % trainPath

    def __iter__(self):
        self.count = -1
        self.shuffledImages()
        return self

    def __next__(self):
        self.count += 1
        if self.count == self.nB:
            raise StopIteration

        if self.balancedSample:
            randomLabel = np.random.randint(0, len(self.className))
            print("loading labels {}".format(self.className[randomLabel]))
            balanced_file = self.balancedFiles[self.className[randomLabel]]
            ia = int(self.balancedCount[randomLabel])
            self.balancedCount[randomLabel] = (self.balancedCount[randomLabel] + self.batch_size) % \
                                              self.balanceFileCount[self.className[randomLabel]]
        else:
            ia = self.count * self.batch_size

        if self.multi_scale:
            # Multi-Scale YOLO Training
            print("wrong code for MultiScale")
            width = random.choice(range(10, 20)) * 32  # 320 - 608 pixels
            scale = float(self.imageSize[0]) / float(self.imageSize[1])
            height = int(round(float(width / scale) / 32.0) * 32)
        else:
            # Fixed-Scale YOLO Training
            width = self.imageSize[0]
            height = self.imageSize[1]

        img_all = []
        labels_all = []
        for index, files_index in enumerate(range(ia, ia + self.batch_size)):
            if self.balancedSample:
                trainFileIndex = files_index % self.balanceFileCount[self.className[randomLabel]]
                img_path, label_path = balanced_file[self.shuffled_vectors[self.className[randomLabel]][trainFileIndex]]
            else:
                trainFileIndex = files_index % self.nF
                img_path, label_path = self.imageAndLabelList[self.shuffled_vector[trainFileIndex]]

            img = cv2.imread(img_path)  #BGR
            if img is None:
                continue

            rgbImage = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)

            self.augmentImageHSV(rgbImage)

            h, w, _ = rgbImage.shape
            rgbImage, ratio, padw, padh = self.trainDataProcess.resize_square(rgbImage,
                                                                              (width, height),
                                                                              self.color)
            _, _, boxes = self.xmlProcess.parseRectData(label_path)
            labels = self.getResizeLabels(boxes, ratio, (padw, padh))

            # Augment image and labels
            if self.augment:
                rgbImage, labels, M = self.random_affine(rgbImage, labels, degrees=(-5, 5), translate=(0.1, 0.1), scale=(0.8, 1.1))

            nL = len(labels)
            if nL > 0:
                # convert xyxy to xywh
                labels[:, 1:5] = xyxy2xywh(labels[:, 1:5].copy())
                labels[:, (1,3)] = labels[:, (1,3)] / width
                labels[:, (2,4)] = labels[:, (2,4)] / height

            rgbImage, labels = self.augmentImageFlip(rgbImage, labels)

            delete_index = []
            # reject warped points outside of image (0.999 for the image boundary)
            for i, label in enumerate(labels):
                if label[2] + label[4]/2 >= float(1):
                    yoldH = label[2] - label[4] / 2
                    label[2] = (yoldH + float(0.999)) / float(2)
                    label[4] = float(0.999) - yoldH
                if label[1] + label[3]/2 >= float(1):
                    yoldW = label[1] - label[3] / 2
                    label[1] = (yoldW + float(0.999)) / float(2)
                    label[3] = float(0.999) - yoldW
                if label[3] < 0.0053 or label[4] < 0.0055: # filter the small object (w for label[3] in 1280 is limit to 6.8 pixel (6.8/1280=0.0053))
                    delete_index.append(i)               # filter the small object (h for label[4] in 720 is limit to 4.0 pixel (4.0/1280=0.0053))

            labels = np.delete(labels, delete_index, axis=0)

            img_all.append(rgbImage)
            labels_all.append(torch.from_numpy(labels))
        numpyImages = np.stack(img_all)
        torchImages = self.convertTorchTensor(numpyImages)

        return torchImages, labels_all

    def getBlanceFileList(self):
        blancedFileList = {}
        blancedFileCount = {}
        for i in range(0, len(self.className)):
            classFile = self.className[i] + ".txt"
            calssFilePath = os.path.join(self.dataPath, classFile)
            blancedFileList[self.className[i]] = self.getImageAndAnnotationList(calssFilePath)
            blancedFileCount[self.className[i]] = len(blancedFileList[self.className[i]])
        return blancedFileList, blancedFileCount

    def getResizeLabels(self, boxes, ratio, pad):
        result = np.array([])
        allNames = [box.name for box in boxes if box.name in self.className]
        if len(allNames) > 0:
            result = np.zeros((len(allNames), 5), dtype=np.float32)
            index = 0
            for box in boxes:
                if box.name in self.className:
                    classId = self.className.index(box.name)
                    minX = ratio * box.min_corner.x + pad[0] // 2
                    minY = ratio * box.min_corner.y + pad[1] // 2
                    maxX = ratio * box.max_corner.x + pad[0] // 2
                    maxY = ratio * box.max_corner.y + pad[1] // 2
                    result[index, :] = np.array([classId, minX, minY, maxX, maxY])
                    index += 1
        return result

    def shuffledImages(self):
        if self.balancedSample:
            self.balancedCount = np.zeros(len(self.className))
            self.shuffled_vectors = {}
            for i in range(0, len(self.className)):
                self.shuffled_vectors[self.className[i]] = \
                    np.random.permutation(self.balanceFileCount[self.className[i]])
        else:
            self.shuffled_vector = np.random.permutation(self.nF)

    def augmentImageHSV(self, inputRGBImage):
        augment_hsv = True
        if self.augment and augment_hsv:
            # SV augmentation by 50%
            fraction = 0.50
            img_hsv = cv2.cvtColor(inputRGBImage, cv2.COLOR_RGB2HSV)
            S = img_hsv[:, :, 1].astype(np.float32)
            V = img_hsv[:, :, 2].astype(np.float32)

            a = (random.random() * 2 - 1) * fraction + 1
            S *= a
            if a > 1:
                np.clip(S, a_min=0, a_max=255, out=S)

            a = (random.random() * 2 - 1) * fraction + 1
            V *= a
            if a > 1:
                np.clip(V, a_min=0, a_max=255, out=V)

            img_hsv[:, :, 1] = S.astype(np.uint8)
            img_hsv[:, :, 2] = V.astype(np.uint8)
            cv2.cvtColor(img_hsv, cv2.COLOR_HSV2RGB, dst=inputRGBImage)

    def augmentImageFlip(self, inputRGBImage, labels):
        nL = len(labels)
        if self.augment:
            # random left-right flip
            lr_flip = True
            if lr_flip & (random.random() > 0.5):
                inputRGBImage = np.fliplr(inputRGBImage)
                if nL > 0:
                    labels[:, 1] = 1 - labels[:, 1]

            # random up-down flip
            ud_flip = False
            if ud_flip & (random.random() > 0.5):
                inputRGBImage = np.flipud(inputRGBImage)
                if nL > 0:
                    labels[:, 2] = 1 - labels[:, 2]

        return inputRGBImage, labels

    def __len__(self):
        return self.nB  # number of batches


''' Defines the functions necessary for calculating the Frechet Inception
Distance (FCD) to evalulate generative models for molecules.

The FCD metric calculates the distance between two distributions of molecules.
Typically, we have summary statistics (mean & covariance matrix) of one
of these distributions, while the 2nd distribution is given by the generative
model.

The FID is calculated by assuming that X_1 and X_2 are the activations of
the preulitmate layer of the CHEMBLNET for generated samples and real world
samples respectivly.
'''

import os
import keras.backend as K
import numpy as np
import tensorflow as tf
from keras.backend.tensorflow_backend import set_session
from keras.models import load_model
from scipy import linalg

def calculate_frechet_distance(mu1, sigma1, mu2, sigma2, eps=1e-6):
    """Numpy implementation of the Frechet Distance.
    The Frechet distance between two multivariate Gaussians X_1 ~ N(mu_1, C_1)
    and X_2 ~ N(mu_2, C_2) is
            d^2 = ||mu_1 - mu_2||^2 + Tr(C_1 + C_2 - 2*sqrt(C_1*C_2)).

    Stable version by Dougal J. Sutherland.
    Params:
    -- mu1:    The mean of the activations of preultimate layer of the
               CHEMNET ( like returned by the function 'get_predictions')
               for generated samples.
    -- mu2:    The mean of the activations of preultimate layer of the
               CHEMNET ( like returned by the function 'get_predictions')
               for real samples.
    -- sigma1: The covariance matrix of the activations of preultimate layer of the
               CHEMNET ( like returned by the function 'get_predictions')
               for generated samples.
    -- sigma2: The covariance matrix of the activations of preultimate layer of the
               CHEMNET ( like returned by the function 'get_predictions')
               for real samples.
    Returns:
    --   : The Frechet Distance.
    """

    mu1 = np.atleast_1d(mu1)
    mu2 = np.atleast_1d(mu2)

    sigma1 = np.atleast_2d(sigma1)
    sigma2 = np.atleast_2d(sigma2)

    assert mu1.shape == mu2.shape, "Training and test mean vectors have different lengths"
    assert sigma1.shape == sigma2.shape, "Training and test covariances have different dimensions"

    diff = mu1 - mu2

    # product might be almost singular
    covmean, _ = linalg.sqrtm(sigma1.dot(sigma2), disp=False)
    if not np.isfinite(covmean).all():
        offset = np.eye(sigma1.shape[0]) * eps
        covmean = linalg.sqrtm((sigma1 + offset).dot(sigma2 + offset))

    # numerical error might give slight imaginary component
    if np.iscomplexobj(covmean):
        if not np.allclose(np.diagonal(covmean).imag, 0, atol=1e-3):
            m = np.max(np.abs(covmean.imag))
            raise ValueError("Imaginary component {}".format(m))
        covmean = covmean.real

    tr_covmean = np.trace(covmean)

    return diff.dot(diff) + np.trace(sigma1) + np.trace(sigma2) - 2 * tr_covmean

def build_masked_loss(loss_function, mask_value):
    """Builds a loss function that masks based on targets

    Args:
        loss_function: The loss function to mask
        mask_value: The value to mask in the targets

    Returns:
        function: a loss function that acts like loss_function with masked inputs
    """

    def masked_loss_function(y_true, y_pred):
        mask = K.cast(K.not_equal(y_true, mask_value), K.floatx())
        return loss_function(y_true * mask, y_pred * mask)

    return masked_loss_function

def masked_accuracy(y_true, y_pred):
    mask_value = 0.5
    a = K.sum(K.cast(K.equal(y_true, K.round(y_pred)), K.floatx()))
    c = K.sum(K.cast(K.not_equal(y_true, mask_value), K.floatx()))
    acc = (a) / c
    return acc

def get_one_hot(smiles, pad_len=-1):
    one_hot = ['C', 'N', 'O', 'H', 'F', 'Cl', 'P', 'B', 'Br', 'S', 'I',
               'Si',
               '#', '(', ')', '+', '-', '1', '2', '3', '4', '5', '6',
               '7', '8', '=', '[', ']', '@',
               'c', 'n', 'o', 's', 'X', '.']
    smiles = smiles + '.'
    if pad_len < 0:
        vec = np.zeros((len(smiles), len(one_hot)))
    else:
        vec = np.zeros((pad_len, len(one_hot)))
    cont = True
    j = 0
    i = 0
    while cont:
        if smiles[i + 1] in ['r', 'i', 'l']:
            sym = smiles[i:i + 2]
            i += 2
        else:
            sym = smiles[i]
            i += 1
        if sym in one_hot:
            vec[j, one_hot.index(sym)] = 1
        else:
            vec[j, one_hot.index('X')] = 1
        j += 1
        if smiles[i] == '.' or j >= (pad_len - 1) and pad_len > 0:
            vec[j, one_hot.index('.')] = 1
            cont = False
    return (vec)

def myGenerator_predict(smilesList, batch_size=128, pad_len=350):
    while 1:
        N = len(smilesList)
        nn = pad_len
        idxSamples = np.arange(N)

        for j in range(int(np.ceil(N / batch_size))):
            idx = idxSamples[j * batch_size: min((j + 1) * batch_size, N)]

            x = []
            for i in range(0, len(idx)):
                currentSmiles = smilesList[idx[i]]
                smiEnc = get_one_hot(currentSmiles, pad_len=nn)
                x.append(smiEnc)

            x = np.asarray(x) / 35
            yield x

def load_ref_model(model_file):
    masked_loss_function = build_masked_loss(K.binary_crossentropy, 0.5)
    model = load_model(model_file,
                       custom_objects={
                           'masked_loss_function': masked_loss_function,
                           'masked_accuracy': masked_accuracy
                       })
    model.pop()
    model.pop()
    return model

def get_predictions(gen_mol, ref_mol, gpu=-1, batch_size=128):
    assert isinstance(gpu, int), "GPU should be an integer"
    model_dir = os.path.split(__file__)[0]
    model_path = os.path.join(model_dir, 'ChemNet_v0.13_pretrained.h5')
    cuda_old = os.environ.get("CUDA_VISIBLE_DEVICES", None)
    os.environ["CUDA_VISIBLE_DEVICES"] = str(gpu)
    if gpu != -1:
        device = "/gpu:{}".format(gpu)
    else:
        device = "/cpu"
    config = tf.ConfigProto(allow_soft_placement=True)
    config.gpu_options.allow_growth = True
    with tf.device(device):
        sess = tf.Session(config=config)
        set_session(sess)
        K.clear_session()
        model = load_ref_model(model_path)
        gen_mol_act = model.predict_generator(
             myGenerator_predict(gen_mol, batch_size=batch_size),
             steps=np.ceil(len(gen_mol)/batch_size)
        )
        ref_mol_act = model.predict_generator(
             myGenerator_predict(ref_mol, batch_size=batch_size),
             steps=np.ceil(len(ref_mol)/batch_size)
        )
        K.clear_session()
        sess.close()
    if cuda_old is not None:
        os.environ["CUDA_VISIBLE_DEVICES"] = str(cuda_old)
    else:
        os.environ.pop("CUDA_VISIBLE_DEVICES")
    return gen_mol_act, ref_mol_act


from django.db import models
from django.utils.translation import ugettext_lazy as _

from django_celery_beat.models import (
    CrontabSchedule, PeriodicTask, PeriodicTasks
)

TASK = 'scheduler.tasks.turn_motor'

class SchedulerRuleManager(models.Manager):

    def get_queryset(self):
        return models.QuerySet(self.model, using=self._db)

    def update_cron(self):
        # Delete old entries from cron
        PeriodicTask.objects.filter(task=TASK).delete()

        # Get user defined rules
        rules = self.get_queryset().order_by('priority')

        # Fill if with empty dictionaries for 7 days of week
        day_freq = [{}] * 7

        for rule in rules:
            for day in rule.days.all():
                if not 'priority' in day_freq[day.cron_day_number] \
                        or rule.priority < day_freq[day.cron_day_number]['priority']:
                    day_freq[day.cron_day_number] = {
                        'frequency': rule.frequency,
                        'priority': rule.priority
                    }

        # Update crontab schedule and add tasks
        for cron_day_number, rule in enumerate(day_freq):
            if not rule:
                continue

            schedule, _ = CrontabSchedule.objects.get_or_create(
                minute='*/{}'.format(rule['frequency']),
                hour='*',
                day_of_week=cron_day_number,
                day_of_month='*',
                month_of_year='*'
            )

            task = PeriodicTask.objects.create(
                crontab=schedule,
                name='Turning motor - Day: {}'.format(cron_day_number),
                task=TASK,
            )

            PeriodicTasks.changed(task)

class SchedulerRule(models.Model):
    days = models.ManyToManyField('WeekDay', related_name='scheduler_rules',
                                  related_query_name='scheduler_rule')
    frequency = models.PositiveSmallIntegerField(_('frequency'),
                                                 help_text=_('in minutes'))
    priority = models.PositiveSmallIntegerField(
        _('priority'),
        help_text=_('lower, more important'))
    objects = SchedulerRuleManager()

    class Meta:
        verbose_name = _('scheduler rule')
        verbose_name_plural = _('scheduler rules')

    def get_frequency_in_minutes(self):
        return self.frequency

    def get_list_of_days_in_short_format(self):
        return [day.get_short_format() for day in self.days.all()]

    def __str__(self):
        return 'On {} every {} min.'.format(
            ', '.join(self.get_list_of_days_in_short_format()),
            self.get_frequency_in_minutes()
        )

class WeekDay(models.Model):
    cron_day_number = models.PositiveSmallIntegerField(
        _('CRON day number'),
        help_text=_('0 - 6 sun - sat'))
    day = models.CharField(_('day'), max_length=20)

    class Meta:
        verbose_name = _('day')
        verbose_name_plural = _('days')

    def __str__(self):
        return '{}'.format(self.day)

    def get_short_format(self):
        return self.day[:3]


import aiohttp

class _ClientSessionContextManagerMixin:
    _session: aiohttp.ClientSession = None

    async def __aenter__(self):
        if self._session is None:
            self._session = aiohttp.ClientSession()
            await self._session.__aenter__()
        return self

    async def __aexit__(self, exc_type=None, exc_val=None, exc_tb=None):
        await self._session.__aexit__(exc_type, exc_val, exc_tb)

class Bot(_ClientSessionContextManagerMixin):
    API_BASE_URL = 'https://api.telegram.org/bot'

    def __init__(self, *, token: str):
        self._api_url = f'{self.API_BASE_URL}{token}'

    def __getattr__(self, method_name: str):
        async def api_method(**params):
            endpoint = f'{self._api_url}/{method_name}'
            async with self._session.post(endpoint, data=params) as response:
                response = await response.json()
                if not response['ok']:
                    raise RuntimeError(response['description'])
                return response['result']
        return api_method

    async def process_update(self, update: dict):
        message = update['message']
        await self.sendMessage(
            chat_id=message['chat']['id'],
            text=message['text'],
        )