import re

import numpy
import pandas
import seaborn
from matplotlib import pyplot

from scvae.analyses.figures import saving, style
from scvae.analyses.figures.utilities import _axis_label_for_symbol
from scvae.utilities import normalise_string, capitalise_string

def plot_learning_curves(curves, model_type, epoch_offset=0, name=None):

    figure_name = "learning_curves"
    figure_name = saving.build_figure_name("learning_curves", name)

    x_label = "Epoch"
    y_label = "Nat"

    if model_type == "AE":
        figure = pyplot.figure()
        axis_1 = figure.add_subplot(1, 1, 1)
    elif model_type == "VAE":
        figure, (axis_1, axis_2) = pyplot.subplots(
            nrows=2, sharex=True, figsize=(6.4, 9.6))
        figure.subplots_adjust(hspace=0.1)
    elif model_type == "GMVAE":
        figure, (axis_1, axis_2, axis_3) = pyplot.subplots(
            nrows=3, sharex=True, figsize=(6.4, 14.4))
        figure.subplots_adjust(hspace=0.1)

    for curve_set_name, curve_set in sorted(curves.items()):

        if curve_set_name == "training":
            line_style = "solid"
            colour_index_offset = 0
        elif curve_set_name == "validation":
            line_style = "dashed"
            colour_index_offset = 1

        def curve_colour(i):
            return style.STANDARD_PALETTE[
                len(curves) * i + colour_index_offset]

        for curve_name, curve in sorted(curve_set.items()):
            if curve is None:
                continue
            elif curve_name == "lower_bound":
                curve_name = "$\\mathcal{L}$"
                colour = curve_colour(0)
                axis = axis_1
            elif curve_name == "reconstruction_error":
                curve_name = "$\\log p(x|z)$"
                colour = curve_colour(1)
                axis = axis_1
            elif "kl_divergence" in curve_name:
                if curve_name == "kl_divergence":
                    index = ""
                    colour = curve_colour(0)
                    axis = axis_2
                else:
                    latent_variable = curve_name.replace("kl_divergence_", "")
                    latent_variable = re.sub(
                        pattern=r"(\w)(\d)",
                        repl=r"\1_\2",
                        string=latent_variable
                    )
                    index = "$_{" + latent_variable + "}$"
                    if latent_variable in ["z", "z_2"]:
                        colour = curve_colour(0)
                        axis = axis_2
                    elif latent_variable == "z_1":
                        colour = curve_colour(1)
                        axis = axis_2
                    elif latent_variable == "y":
                        colour = curve_colour(0)
                        axis = axis_3
                curve_name = "KL" + index + "$(q||p)$"
            elif curve_name == "log_likelihood":
                curve_name = "$L$"
                axis = axis_1
            epochs = numpy.arange(len(curve)) + epoch_offset + 1
            label = "{} ({} set)".format(curve_name, curve_set_name)
            axis.plot(
                epochs,
                curve,
                color=colour,
                linestyle=line_style,
                label=label
            )

    handles, labels = axis_1.get_legend_handles_labels()
    labels, handles = zip(*sorted(zip(labels, handles), key=lambda t: t[0]))

    axis_1.legend(handles, labels, loc="best")

    if model_type == "AE":
        axis_1.set_xlabel(x_label)
        axis_1.set_ylabel(y_label)
    elif model_type == "VAE":
        handles, labels = axis_2.get_legend_handles_labels()
        labels, handles = zip(*sorted(
            zip(labels, handles), key=lambda t: t[0]))
        axis_2.legend(handles, labels, loc="best")
        axis_1.set_ylabel("")
        axis_2.set_ylabel("")

        if model_type == "GMVAE":
            axis_3.legend(loc="best")
            handles, labels = axis_3.get_legend_handles_labels()
            labels, handles = zip(*sorted(
                zip(labels, handles), key=lambda t: t[0]))
            axis_3.legend(handles, labels, loc="best")
            axis_3.set_xlabel(x_label)
            axis_3.set_ylabel("")
        else:
            axis_2.set_xlabel(x_label)
        figure.text(-0.01, 0.5, y_label, va="center", rotation="vertical")

    seaborn.despine()

    return figure, figure_name

def plot_separate_learning_curves(curves, loss, name=None):

    if not isinstance(loss, list):
        losses = [loss]
    else:
        losses = loss

    if not isinstance(name, list):
        names = [name]
    else:
        names = name

    names.extend(losses)
    figure_name = saving.build_figure_name("learning_curves", names)

    x_label = "Epoch"
    y_label = "Nat"

    figure = pyplot.figure()
    axis = figure.add_subplot(1, 1, 1)
    seaborn.despine()

    for curve_set_name, curve_set in sorted(curves.items()):

        if curve_set_name == "training":
            line_style = "solid"
            colour_index_offset = 0
        elif curve_set_name == "validation":
            line_style = "dashed"
            colour_index_offset = 1

        def curve_colour(i):
            return style.STANDARD_PALETTE[
                len(curves) * i + colour_index_offset]

        for curve_name, curve in sorted(curve_set.items()):
            if curve is None or curve_name not in losses:
                continue
            elif curve_name == "lower_bound":
                curve_name = "$\\mathcal{L}$"
                colour = curve_colour(0)
            elif curve_name == "reconstruction_error":
                curve_name = "$\\log p(x|z)$"
                colour = curve_colour(1)
            elif "kl_divergence" in curve_name:
                if curve_name == "kl_divergence":
                    index = ""
                    colour = curve_colour(0)
                else:
                    latent_variable = curve_name.replace("kl_divergence_", "")
                    latent_variable = re.sub(
                        pattern=r"(\w)(\d)",
                        repl=r"\1_\2",
                        string=latent_variable
                    )
                    index = "$_{" + latent_variable + "}$"
                    if latent_variable in ["z", "z_2"]:
                        colour = curve_colour(0)
                    elif latent_variable == "z_1":
                        colour = curve_colour(1)
                    elif latent_variable == "y":
                        colour = curve_colour(0)
                curve_name = "KL" + index + "$(q||p)$"
            elif curve_name == "log_likelihood":
                curve_name = "$L$"
            epochs = numpy.arange(len(curve)) + 1
            label = curve_name + " ({} set)".format(curve_set_name)
            axis.plot(
                epochs,
                curve,
                color=colour,
                linestyle=line_style,
                label=label
            )

    handles, labels = axis.get_legend_handles_labels()
    labels, handles = zip(*sorted(zip(labels, handles), key=lambda t: t[0]))

    axis.legend(handles, labels, loc="best")

    axis.set_xlabel(x_label)
    axis.set_ylabel(y_label)

    return figure, figure_name

def plot_accuracy_evolution(accuracies, name=None):

    figure_name = saving.build_figure_name("accuracies", name)
    figure = pyplot.figure()
    axis = figure.add_subplot(1, 1, 1)
    seaborn.despine()

    for accuracies_kind, accuracies in sorted(accuracies.items()):
        if accuracies is None:
            continue
        elif accuracies_kind == "training":
            line_style = "solid"
            colour = style.STANDARD_PALETTE[0]
        elif accuracies_kind == "validation":
            line_style = "dashed"
            colour = style.STANDARD_PALETTE[1]

        label = "{} set".format(capitalise_string(accuracies_kind))
        epochs = numpy.arange(len(accuracies)) + 1
        axis.plot(
            epochs,
            100 * accuracies,
            color=colour,
            linestyle=line_style,
            label=label
        )

    handles, labels = axis.get_legend_handles_labels()
    labels, handles = zip(*sorted(zip(labels, handles), key=lambda t: t[0]))

    axis.legend(handles, labels, loc="best")

    axis.set_xlabel("Epoch")
    axis.set_ylabel("Accuracies")

    return figure, figure_name

def plot_kl_divergence_evolution(kl_neurons, scale="log", name=None):

    figure_name = saving.build_figure_name("kl_divergence_evolution", name)
    n_epochs, __ = kl_neurons.shape

    kl_neurons = numpy.sort(kl_neurons, axis=1)

    if scale == "log":
        kl_neurons = numpy.log(kl_neurons)
        scale_label = "$\\log$ "
    else:
        scale_label = ""

    figure = pyplot.figure()
    axis = figure.add_subplot(1, 1, 1)

    cbar_dict = {"label": scale_label + "KL$(p_i|q_i)$"}

    number_of_epoch_labels = 10
    if n_epochs > 2 * number_of_epoch_labels:
        epoch_label_frequency = int(numpy.floor(
            n_epochs / number_of_epoch_labels))
    else:
        epoch_label_frequency = True

    epochs = numpy.arange(n_epochs) + 1

    seaborn.heatmap(
        pandas.DataFrame(kl_neurons.T, columns=epochs),
        xticklabels=epoch_label_frequency,
        yticklabels=False,
        cbar=True, cbar_kws=cbar_dict, cmap=style.STANDARD_COLOUR_MAP,
        ax=axis
    )

    axis.set_xlabel("Epochs")
    axis.set_ylabel("$i$")

    seaborn.despine(ax=axis)

    return figure, figure_name

def plot_centroid_probabilities_evolution(probabilities, distribution,
                                          linestyle="solid", name=None):

    distribution = normalise_string(distribution)

    y_label = _axis_label_for_symbol(
        symbol="\\pi",
        distribution=distribution,
        suffix="^k"
    )

    figure_name = "centroids_evolution-{}-probabilities".format(distribution)
    figure_name = saving.build_figure_name(figure_name, name)

    n_epochs, n_centroids = probabilities.shape

    centroids_palette = style.darker_palette(n_centroids)
    epochs = numpy.arange(n_epochs) + 1

    figure = pyplot.figure()
    axis = figure.add_subplot(1, 1, 1)
    seaborn.despine()

    for k in range(n_centroids):
        axis.plot(
            epochs,
            probabilities[:, k],
            color=centroids_palette[k],
            linestyle=linestyle,
            label="$k = {}$".format(k)
        )

    axis.set_xlabel("Epochs")
    axis.set_ylabel(y_label)

    axis.legend(loc="best")

    return figure, figure_name

def plot_centroid_means_evolution(means, distribution, decomposed=False,
                                  name=None):

    symbol = "\\mu"
    if decomposed:
        decomposition_method = "PCA"
    else:
        decomposition_method = ""
    distribution = normalise_string(distribution)
    suffix = "(y = k)"

    x_label = _axis_label_for_symbol(
        symbol=symbol,
        coordinate=1,
        decomposition_method=decomposition_method,
        distribution=distribution,
        suffix=suffix
    )
    y_label = _axis_label_for_symbol(
        symbol=symbol,
        coordinate=2,
        decomposition_method=decomposition_method,
        distribution=distribution,
        suffix=suffix
    )

    figure_name = "centroids_evolution-{}-means".format(distribution)
    figure_name = saving.build_figure_name(figure_name, name)

    n_epochs, n_centroids, latent_size = means.shape

    if latent_size > 2:
        raise ValueError("Dimensions of means should be 2.")

    centroids_palette = style.darker_palette(n_centroids)
    epochs = numpy.arange(n_epochs) + 1

    figure = pyplot.figure()
    axis = figure.add_subplot(1, 1, 1)
    seaborn.despine()

    colour_bar_scatter_plot = axis.scatter(
        means[:, 0, 0], means[:, 0, 1], c=epochs,
        cmap=seaborn.dark_palette(style.NEUTRAL_COLOUR, as_cmap=True),
        zorder=0
    )

    for k in range(n_centroids):
        colour = centroids_palette[k]
        colour_map = seaborn.dark_palette(colour, as_cmap=True)
        axis.plot(
            means[:, k, 0],
            means[:, k, 1],
            color=colour,
            label="$k = {}$".format(k),
            zorder=k + 1
        )
        axis.scatter(
            means[:, k, 0],
            means[:, k, 1],
            c=epochs,
            cmap=colour_map,
            zorder=n_centroids + k + 1
        )

    axis.legend(loc="best")

    colour_bar = figure.colorbar(colour_bar_scatter_plot)
    colour_bar.outline.set_linewidth(0)
    colour_bar.set_label("Epochs")

    axis.set_xlabel(x_label)
    axis.set_ylabel(y_label)

    return figure, figure_name

def plot_centroid_covariance_matrices_evolution(covariance_matrices,
                                                distribution, name=None):

    distribution = normalise_string(distribution)
    figure_name = "centroids_evolution-{}-covariance_matrices".format(
        distribution)
    figure_name = saving.build_figure_name(figure_name, name)

    y_label = _axis_label_for_symbol(
        symbol="\\Sigma",
        distribution=distribution,
        prefix="|",
        suffix="(y = k)|"
    )

    n_epochs, n_centroids, __, __ = covariance_matrices.shape
    determinants = numpy.empty([n_epochs, n_centroids])

    for e in range(n_epochs):
        for k in range(n_centroids):
            determinants[e, k] = numpy.prod(numpy.diag(
                covariance_matrices[e, k]))

    if determinants.all() > 0:
        line_range_ratio = numpy.empty(n_centroids)
        for k in range(n_centroids):
            determinants_min = determinants[:, k].min()
            determinants_max = determinants[:, k].max()
            line_range_ratio[k] = determinants_max / determinants_min
        range_ratio = line_range_ratio.max() / line_range_ratio.min()
        if range_ratio > 1e2:
            y_scale = "log"
        else:
            y_scale = "linear"

    centroids_palette = style.darker_palette(n_centroids)
    epochs = numpy.arange(n_epochs) + 1

    figure = pyplot.figure()
    axis = figure.add_subplot(1, 1, 1)
    seaborn.despine()

    for k in range(n_centroids):
        axis.plot(
            epochs,
            determinants[:, k],
            color=centroids_palette[k],
            label="$k = {}$".format(k)
        )

    axis.set_xlabel("Epochs")
    axis.set_ylabel(y_label)

    axis.set_yscale(y_scale)

    axis.legend(loc="best")

    return figure, figure_name


#p.176

import pandas as pd
fish = pd.read_csv('https://bit.ly/fish_csv_data')

## fish파일의 첫 5개의 행 출력
#print(fish.head())

## Species의 열의 고유값들 추출
#print(pd.unique(fish['Species']))

## Species열을 제외한 나머니 열은 입력 데이터로 변환 -> 데이터프레임에서 5개의 열을 리스트로 변환
fish_input = fish[['Weight', 'Length', 'Diagonal', 'Height', 'Width']].to_numpy()

## fish_input에 5개의 특성이 잘 저장되어 있는지 5개 열 추출
#print(fish_input[:5])

## fish_target 추출
fish_target = fish['Species'].to_numpy()

## 데이터 세트 2개 필요
from sklearn.model_selection import train_test_split
train_input, test_input, train_target, test_target = train_test_split(fish_input, fish_target, random_state=42)

## 데이터 세트에 대해 표준화 전처리
from sklearn.preprocessing import StandardScaler
ss = StandardScaler()
ss.fit(train_input)
train_scaled = ss.transform(train_input)
test_scaled = ss.transform(test_input)

## k-최근접 이웃 분류기의 확률 예측 + 점수 확인
from sklearn.neighbors import KNeighborsClassifier
kn = KNeighborsClassifier(n_neighbors=3)
kn.fit(train_scaled, train_target)
#print(kn.score(train_scaled, train_target))
#print(kn.score(test_scaled, test_target))

## k-최근접 이웃 분류기에서 정렬된 출력값
#print(kn.classes_) ['Bream' 'Parkki' 'Perch' 'Pike' 'Roach' 'Smelt' 'Whitefish']

## k-최근접 이웃 분류기에서 예측된 5개의 출력값
#print(kn.predict(test_scaled[:5]))

## k-최근접 이웃 분류기에서 예측된 5개의 출력값의 확률(round, decimals이용) -> 출력순서는 classes와 같다.
import numpy as np
proba = kn.predict_proba(test_scaled[:5])
#print(np.round(proba, decimals=4))

## proba의 네번째 샘플의 확률을 확인(distances, indexes 이용) -> 위의(k-최근접 이웃 분류기에서 예측된 5개의 출력값의 확률)과 같음
distances, indexes = kn.kneighbors(test_scaled[3:4])
#print(train_target[indexes])[['Roach' 'Perch' 'Perch']]

04-1.로지스틱 회귀를 위한 데이터 준비(logistic regression).py


'''
Created on 9 Sep 2011

@author: mpelko
'''
from neurovivo.cell.NeuronCell import NeuronCell
from neuron import h

class TwoCompartmentCell(NeuronCell):
    '''
    The cell with soma and a dendrite hanging out of the soma (ball and stick).
    We put the dendrite in the trunk group.
    '''
    def __init__(self):

        super(self.__class__, self).__init__()
        self.name = "TwoCompartment"

    def set_up(self):

        # Simplified morphology based on the morphology of HayCell
        soma = h.Section(name="soma")
        soma.L = 23
        soma.diam = 13.5

        trunk = h.Section(name="trunk")
        trunk.L = 200
        trunk.nseg = 10
        trunk.diam = 2

        trunk.connect(soma)

        all_secs = [soma, trunk]

        for sec in all_secs:
            sec.insert("pas")
            sec.Ra = 200
            sec.cm = 1
            sec.e_pas = -70
            sec.v = -70

        # structuring the sections
        self.secs["soma"].append(soma)
        self.secs["trunk"].append(trunk)

        self.bifurcation_info = ("trunk", 1)

    def destroy(self):
        """
        Takes care that neuron objects and python objects reffering to the cell
        are destroyed.

        At this point all the cell sections should be destroyed.

        Make sure all the references to the sections are deleted.
        """
        for key in self.secs.keys():
            self.secs[key] = []
            self.sec_names[key] = []


"""
Utilities for working with TimeEntry objects.
"""
from datetime import timedelta

class DayTimeEntry:
    """
    Time booked for a task by a user on a particular day.
    """
    def __init__(self, user_id, task_id, date, hours, overtime=False):
        self.user_id, self.task_id = user_id, task_id
        self.date = date
        self.hours = hours
        self.overtime = overtime

DAY_ATTRS = ('mon', 'tue', 'wed', 'thu', 'fri', 'sat', 'sun', 'overtime')
def weekly_to_daily_entries(time_entries):
    """
    Converts a list of ``TimeEntry`` instances to ``DayTimeEntry``
    instances, one for each day on which time is booked in each
    ``TimeEntry``.

    Any overtime booked will be marked as occurring on the first day
    of the week.
    """
    daily_entries = []
    for time_entry in time_entries:
        day_count = 0
        for i, day_attr in enumerate(DAY_ATTRS):
            if getattr(time_entry, day_attr) > 0:
                day_entry = DayTimeEntry(time_entry.user_id, time_entry.task_id,
                    time_entry.week_commencing + timedelta(days=i),
                    getattr(time_entry, day_attr))
                if day_attr == 'overtime':
                    day_entry.date = time_entry.week_commencing
                    day_entry.overtime = True
                daily_entries.append(day_entry)
    return daily_entries


import pickle
import re
import os
import operator
import json

punc_remover = re.compile(r"[\W]+")

PATH_TO_REPOSITORY = ''

# get the mention entities counter
with open(os.path.join(PATH_TO_REPOSITORY, 'train_data', 'zelda_mention_entities_counter.pickle'), 'rb') as handle:
    mention_entities_counter = pickle.load(handle)

# to improve the recall of the candidate lists we add a lower cased and a further reduced version of each mention to the mention set
simpler_mentions_candidate_dict = {}
for mention in mention_entities_counter:
    # create mention without blanks and lower cased
    simplified_mention = mention.replace(' ', '').lower()
    # the simplified mention already occurred from another mention
    if simplified_mention in simpler_mentions_candidate_dict:
        for entity in mention_entities_counter[mention]:
            if entity in simpler_mentions_candidate_dict[simplified_mention]:
                simpler_mentions_candidate_dict[simplified_mention][entity] += mention_entities_counter[mention][entity]
            else:
                simpler_mentions_candidate_dict[simplified_mention][entity] = mention_entities_counter[mention][entity]
    # its the first occurrence of the simplified mention
    else:
        simpler_mentions_candidate_dict[simplified_mention] = mention_entities_counter[mention]

even_more_simpler_mentions_candidate_dict = {}
for mention in mention_entities_counter:
    # create simplified mention
    simplified_mention=punc_remover.sub("", mention.lower())
    # the simplified mention already occurred from another mention
    if simplified_mention in even_more_simpler_mentions_candidate_dict:
        for entity in mention_entities_counter[mention]:
            if entity in even_more_simpler_mentions_candidate_dict[simplified_mention]:
                even_more_simpler_mentions_candidate_dict[simplified_mention][entity] += mention_entities_counter[mention][entity]
            else:
                even_more_simpler_mentions_candidate_dict[simplified_mention][entity] = mention_entities_counter[mention][entity]
    # its the first occurrence of the simplified mention
    else:
        even_more_simpler_mentions_candidate_dict[simplified_mention] = mention_entities_counter[mention]

def get_candidates_and_mfs(mention):
    try:
        candidates = mention_entities_counter[mention]
    except KeyError:
        try:
            candidates = simpler_mentions_candidate_dict[mention.lower().replace(' ', '')]
        except KeyError:
            try:
                candidates = even_more_simpler_mentions_candidate_dict[punc_remover.sub("", mention.lower())]
            except KeyError:
                candidates = []
    if not candidates:
        return [], ''
    else:

        mfs = max(candidates.items(), key=operator.itemgetter(1))[0]

        return list(candidates.keys()), mfs

# get the test sets
test_folder = os.path.join(PATH_TO_REPOSITORY, 'test_data', 'jsonl')
for filename in os.listdir(test_folder):

    number_mentions = 0
    number_mentions_not_contained_in_lists = 0
    number_mfs_mentions = 0
    number_mentions_with_gold_in_candidates = 0

    with open(os.path.join(test_folder, filename), mode='r', encoding='utf-8') as jsnol_input:

        for jline in jsnol_input:
            input_dictionary = json.loads(jline)

            input_text = input_dictionary['text']
            for index, gold_title in zip(input_dictionary['index'], input_dictionary['wikipedia_titles']):
                number_mentions+=1

                mention = input_text[index[0]: index[1]]

                candidates, mfs = get_candidates_and_mfs(mention)

                if not candidates:
                    number_mentions_not_contained_in_lists+=1
                else:
                    if gold_title == mfs:
                        number_mfs_mentions+=1
                    if gold_title in candidates:
                        number_mentions_with_gold_in_candidates+=1

        print(f'================={filename}=================')
        print(f'Number mentions: {number_mentions}, Of which are contained in the candidate lists {number_mentions-number_mentions_not_contained_in_lists} ({number_mentions_not_contained_in_lists} missing).')
        print(f'Total accuracy of mfs: {number_mfs_mentions/number_mentions:.3} ({number_mfs_mentions/(number_mentions-number_mentions_not_contained_in_lists):.3} on the mentions contained in the lists.)')
        print(f'Total recall of candidates: {number_mentions_with_gold_in_candidates/number_mentions:.3} ({number_mentions_with_gold_in_candidates/(number_mentions-number_mentions_not_contained_in_lists):.3} on the mentions contained in the lists.)')


### This script downloads all department htmls and appends them to one big html file for parser.py
## Make sure to check all paths before running this script

import requests
from bs4 import BeautifulSoup as bs4

# gets home page of class schedule to get all dept names
# to individually parse later
def depts():
    url = "https://www.reg.uci.edu/perl/WebSoc"
    path = 'data/home.html'
    response = requests.post(url, stream=True)
    with open(path, 'w') as file:
        file.write(response.text)

    soup = bs4(open(path), 'html.parser')

    items = soup.select('[name=Dept] option[value]')
    values = [item.get('value') for item in items]
    del values[0]
    i = 0
    for item in values:
        values[i] = item.replace(" ", "+")
        i += 1
    return values

# gets list of department names, parses each page and saves to file.

def main_Downloader():
    dept_names = depts()

    f = open('data/all_depts.html', 'w')  # clear file init
    f.close()

    url = "https://www.reg.uci.edu/perl/WebSoc"
    for name in dept_names:
        body = f"""Submit=Display+Web+Results&YearTerm=2022-92&Breadth=ANY&Dept={name}&CourseNum=&Division=ANY&CourseCodes=&InstrName=&CourseTitle=&ClassType=ALL&Units=&Days=&StartTime=&EndTime=&MaxCap=&FullCourses=ANY&FontSize=100&CancelledCourses=Exclude&Bldg=&Room="""
        response = requests.post(url, body, stream=True)
        with open("static/all_depts.html", 'a') as file:
            file.write(response.text)

if __name__ == '__main_Downloader__':
    main_Downloader()


__author__ = 'zhaoyao'

import unittest
import sys
import os
cur_path = os.path.dirname(os.path.dirname(os.getcwd()))
sys.path.append(cur_path)
from DianJin.common.login import LOGIN
from DianJin.common.db_operation import DBmethod
import requests

class Login(unittest.TestCase):

    def test_orderdetail(self):

        login = LOGIN()
        res_login = login.login("13770506773","zy568521")
        data ={
            "token": res_login["body"]["token"],
            "type":"JJ",
            "pageIndex":"1",
        }

        url = "http://djuat.dtfunds.com/fund-app/trade/queryUserTransRecord"
        res = requests.post(url,data).json()
        # print(res)
        account = int(res["body"]["totalNumber"])
        if account > 1:
            orderNo = res["body"]["items"][0]["orderNo"]
            url ="http://djuat.dtfunds.com/fund-app/order/getOrderDetail"
            data ={
                "token": res_login["body"]["token"],
                "orderNo":orderNo
            }
            res= requests.post(url,data).json()
            self.assertEqual(res["head"]["msg"],"成功","测试失败，")


import numpy as np
import torch
import pandas as pd

from pytorch_widedeep import Trainer
from pytorch_widedeep.models import (
    SAINT,
    Wide,
    WideDeep,
    TabPerceiver,
    FTTransformer,
    TabFastFormer,
    TabTransformer,
)
from pytorch_widedeep.metrics import Accuracy
from pytorch_widedeep.datasets import load_adult
from pytorch_widedeep.callbacks import (
    LRHistory,
    EarlyStopping,
    ModelCheckpoint,
)
from pytorch_widedeep.initializers import XavierNormal, KaimingNormal
from pytorch_widedeep.preprocessing import TabPreprocessor, WidePreprocessor

use_cuda = torch.cuda.is_available()

if __name__ == "__main__":
    df = load_adult(as_frame=True)
    df.columns = [c.replace("-", "_") for c in df.columns]
    df["age_buckets"] = pd.cut(
        df.age, bins=[16, 25, 30, 35, 40, 45, 50, 55, 60, 91], labels=np.arange(9)
    )
    df["income_label"] = (df["income"].apply(lambda x: ">50K" in x)).astype(int)
    df.drop("income", axis=1, inplace=True)
    df.head()

    # Define wide, crossed and deep tabular columns
    wide_cols = [
        "age_buckets",
        "education",
        "relationship",
        "workclass",
        "occupation",
        "native_country",
        "gender",
    ]
    crossed_cols = [("education", "occupation"), ("native_country", "occupation")]

    cat_embed_cols = [
        "education",
        "relationship",
        "workclass",
        "occupation",
        "native_country",
    ]
    continuous_cols = ["age", "hours_per_week"]
    # # Aternatively one could pass a list of Tuples with the name of the column
    # # and the embedding dim per column
    # cat_embed_cols = [
    #     ("education", 10),
    #     ("relationship", 8),
    #     ("workclass", 10),
    #     ("occupation", 10),
    #     ("native_country", 10),
    # ]
    target = "income_label"
    target = df[target].values

    wide_preprocessor = WidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols)
    X_wide = wide_preprocessor.fit_transform(df)

    tab_preprocessor = TabPreprocessor(
        cat_embed_cols=cat_embed_cols,
        continuous_cols=continuous_cols,
        scale=True,
        for_transformer=True,
        with_cls_token=True,
    )
    X_tab = tab_preprocessor.fit_transform(df)

    wide = Wide(input_dim=np.unique(X_wide).shape[0], pred_dim=1)

    tab_transformer = TabTransformer(
        column_idx=tab_preprocessor.column_idx,
        cat_embed_input=tab_preprocessor.cat_embed_input,
        continuous_cols=continuous_cols,
        embed_continuous=True,
        n_blocks=4,
    )

    saint = SAINT(
        column_idx=tab_preprocessor.column_idx,
        cat_embed_input=tab_preprocessor.cat_embed_input,
        continuous_cols=continuous_cols,
        cont_norm_layer="batchnorm",
        n_blocks=4,
    )

    tab_perceiver = TabPerceiver(
        column_idx=tab_preprocessor.column_idx,
        cat_embed_input=tab_preprocessor.cat_embed_input,
        continuous_cols=continuous_cols,
        n_latents=6,
        latent_dim=16,
        n_latent_blocks=4,
        n_perceiver_blocks=2,
        share_weights=False,
    )

    tab_fastformer = TabFastFormer(
        column_idx=tab_preprocessor.column_idx,
        cat_embed_input=tab_preprocessor.cat_embed_input,
        continuous_cols=continuous_cols,
        n_blocks=4,
        n_heads=4,
        share_qv_weights=False,
        share_weights=False,
    )

    ft_transformer = FTTransformer(
        column_idx=tab_preprocessor.column_idx,
        cat_embed_input=tab_preprocessor.cat_embed_input,
        continuous_cols=continuous_cols,
        input_dim=32,
        kv_compression_factor=0.5,
        n_blocks=3,
        n_heads=4,
    )

    for tab_model in [
        tab_transformer,
        saint,
        ft_transformer,
        tab_perceiver,
        tab_fastformer,
    ]:
        model = WideDeep(wide=wide, deeptabular=tab_model)

        wide_opt = torch.optim.Adam(model.wide.parameters(), lr=0.01)
        deep_opt = torch.optim.Adam(model.wide.parameters(), lr=0.01)
        wide_sch = torch.optim.lr_scheduler.StepLR(wide_opt, step_size=3)
        deep_sch = torch.optim.lr_scheduler.StepLR(deep_opt, step_size=5)

        optimizers = {"wide": wide_opt, "deeptabular": deep_opt}
        schedulers = {"wide": wide_sch, "deeptabular": deep_sch}
        initializers = {"wide": KaimingNormal, "deeptabular": XavierNormal}
        callbacks = [
            LRHistory(n_epochs=10),
            EarlyStopping(patience=5),
            ModelCheckpoint(filepath="model_weights/wd_out"),
        ]
        metrics = [Accuracy]

        trainer = Trainer(
            model,
            objective="binary",
            optimizers=optimizers,
            lr_schedulers=schedulers,
            initializers=initializers,
            callbacks=callbacks,
            metrics=metrics,
        )

        trainer.fit(
            X_wide=X_wide,
            X_tab=X_tab,
            target=target,
            n_epochs=1,
            batch_size=128,
            val_split=0.2,
        )


# This file contains useful functions

# STANDARD PACKAGES
import numpy as np
from statistics import mean
from scipy.spatial import Delaunay, ConvexHull
from numpy.random import default_rng

# FILES IMPORTS
from data_orbits import r_earth
from data_inspection import r_obs_min, r_obs_max, r_escape, nu_V, nu_M, nu_G, alpha_max, beta_max, r_min

rng = default_rng()

def computeangle(a, b):
    """
    Computes angle between 2 vectors.
    :param a: vector 1
    :param b: vector 2
    :return: angle [rad]
    """
    return np.arccos(np.dot(a, b) / (np.linalg.norm(a) * np.linalg.norm(b)))

def compute_rr_target2sun(rr_target, rr_sun):
    """
    Computes the target-sun vector as the sum of vectors r (target-Earth) and e (Earth-Sun).
    :param rr_target: earth-target position vector (ECI frame) [1x3]. Note: it is later converted to target-earth.
    :param rr_sun: Earth-Sun position vector (ECI frame) [1x3]
    :return: sun-target position vector (ECI frame) [1x3]
    """
    return - rr_target + rr_sun

def iseclipsed(es_eci, et_eci):
    """
    Says if a satellite (the target) is eclipsed by Earth.
    :param es_eci: position vector Earth-Sun (ECI frame).
    :param et_eci: position vector Earth-target (ECI frame).
    :return: boolean
    """
    theta = computeangle(es_eci, et_eci)
    es_norm = np.linalg.norm(es_eci)
    et_norm = np.linalg.norm(et_eci)
    theta_sun = np.arccos(r_earth / es_norm)
    theta_sat = np.arccos(r_earth / et_norm)
    if theta_sun + theta_sat <= theta:
        return True
    else:
        return False

def isinrange(rr_lvlh):
    """
    Says if a_target certain feature in in observation range.
    :param rr_lvlh: relative position vector target-chaser
    :return: boolean
    """
    r = np.linalg.norm(rr_lvlh)
    if r_obs_min <= r <= r_obs_max:
        inrange = True
    else:
        inrange = False
    return inrange

def isinfov(view_dir, r):
    """
    Says if a target's certain feature is in Field of view (FOV) --> alpha <= alpha_max
    being alpha the angle between the viewing direction of the target feature and the vector defining the position of
    the chaser with respect to the target
    :param view_dir: viewing direction vector (ECI frame).
    :param r: relative position vector target-chaser (ECI frame).
    :return: boolean
    """
    alpha = computeangle(view_dir, r)
    if alpha <= alpha_max:
        observed = True
    else:
        observed = False
    return observed

def isiluminated(view_dir_inert, ts):
    """
    Says if a_target certain feature is properly iluminated --> beta <= beta_max
    being beta the angle between the viewing direction and the Sun direction.
    :param view_dir_inert: viewing direction vector in an inertial frame (ECI frame).
    :param ts: relative target-Sun vector (ECI frame).
    :return: boolean
    """
    beta = computeangle(view_dir_inert, ts)
    if beta <= beta_max:
        observed = True
    else:
        observed = False
    return observed

def isvalidmission(rr_chaser_lvlh):
    """
    Says if a_target leg is valid or not depending on if it fulfill some requirements.
    Conditions:
        - Chaser reaches a_target maximum distance larger than r_escape from the target during its trajectory.
        - Chaser's and solar arrays' angular motion exceed the specified RW and SADM limits (instrument contraint).
        - Maneouvre is even partially executed during eclipse (energy constraint).
        - Trajectory is unsafe (safety constraint).
    Disclaimer: at the moment only r_escape constrain is implemented.
    :param rr_chaser_lvlh: target-chaser position vector (LVLH frame).
    :return: boolean
    """
    valid_mission = True
    i = 0
    for x in rr_chaser_lvlh:
        if not r_min < np.linalg.norm(x) < r_escape:  # max.distance case
            valid_mission = False
            break  # because just one invalid point is enough to invalidate the whole leg
        i += 1
    return valid_mission

def compute_splxvolume(splx_points):
    """
    Computes the volume of a_target simplex using the Cayley-Menger determinant.
    For a_target 4D euclidean space (dvx, dvy, dvz, T) each simplex is determined by 5 points or vertices.
    :param splx_points: array of points forming the simplex.
    :return: volume of the simplex.
    """
    c = ConvexHull(splx_points)
    # # in order to avoid numerical issues, it skips simplices that are smaller than a_target specified minimum size
    # if c.volume < 1e-5:
    #     c.volume = 0
    return c.volume

def compute_simplexscore(q, scored_points, m_max, g_max):
    """
    ¡¡¡¡¡¡¡¡¡¡¡¡UPDATE BECAUSE RIGHT NOW G = 0!!!!!!!!!!!!!!
    Returns a_target simplex score for each simplex of the Delaunay triangulation
    :param q: simplex
    :param scored_points: [nx5] vector containing the initial sampled points [0:4] and their score [4]
    :param m_max:maximum trajectory score found in the triangulation(i.e considering all simplices of the triangulation)
    :param g_max:maximum score gradient found in the triangulation(i.e considering all simplices of the triangulation)
    :return:
    """
    v = compute_splxvolume(scored_points[:, 0:4][q])
    m = max(scored_points[:, 4])
    g = mean(scored_points[:, 4][q])
    J = v ** nu_V * (1 + nu_M * m / m_max + nu_G * g / g_max)
    return J

def samplewithinbestsimplices(weight_list, points, n_samples):
    """
    Selects a_target specified number of simplices from a_target list---choosing those with higher weights more frequently---
    and places a_target new point within each of the corresponding simplices.
    Since a_target given simplex may be chosen more than once during this process—--indeed, that is part of the intent of the
    weighted sort—--placing that new point at the exact center could create degeneracies.
    Instead, we randomly sample a_target point inside the simlex (see function 'sample_smplx_randomly').
    :param weight_list: [scores, simplices] list containing the score of the simplex and the simples
    :param points: all the points of the space that form all the simplices
    :param n_samples: number of new sampled points that we want to obtain from the entire simplices space.
    :return: [n_samples x dim_of_points] new sampled points
    """
    chosen_simplices = weighted_randchoice(weight_list, n_samples)
    new_samples = np.array([])
    for s in chosen_simplices:
        sample = sample_smplx_randomly(points[s])
        new_samples = np.append(new_samples, sample)
    new_samples = new_samples.reshape(n_samples, len(sample[0]))
    return new_samples

def weighted_randchoice(weight_list, n_samples):
    """
        Chapter 3.7 of "Description of the Reachability Set Adaptive Mesh Algorithm", by Erik Komendera.
    This algorithm allows for the random choice from a_target list in which each element has a_target weight associated with it.
    To do this, each item and its weight are assigned a_target nonoverlapping range of values for use in random sampling.
    For example, if two items have weights 2 and 5, the ranges for each will be [0,2] and [2,7].
    By randomly sampling real numbers between 0 and 7, the second item will be chosen with a_target probability of 5/7.
    :param weight_list: [scores, simplices] list containing the score of the simplex and the simples
    :param n_samples: number of samples
    :return: vector containing the chosen simplices based on its score
    """
    summatory = 0
    intervals_max = []
    for x in weight_list:
        summatory += x[0]
        intervals_max.append(summatory)
    uniform_sampl = rng.uniform(0, summatory, n_samples)

    chosen_simplices = []
    for s in uniform_sampl:
        for i in range(len(intervals_max)):
            if s <= intervals_max[i]:
                chosen_simplices.append(weight_list[i][1])
                break
    return chosen_simplices

def sample_smplx_randomly(smplx_points):
    """
    From reference [16], "Picking a_target uniformly random point from an arbitrary simplex", by Christian Grimme.
    :param smplx_points: [1xn] vector containing the points/vertices defining the simplex.
    :return: [1xn] randomly sampled point defined by n-coordinates.
    """
    n = len(smplx_points) - 1
    z = [1, *rng.uniform(0, 1, n), 0]
    lmb = []
    for j in range(len(z)-1):
        aux = z[j]**(1/(n+1-j))
        lmb.append(aux)
    lmb.append(0)

    multiplication = np.cumprod(lmb[0:n+1])
    result = np.zeros((1, n))
    for i in range(1, n+2):
        result += [(1 - lmb[i]) * multiplication[i-1] * x for x in smplx_points[i-1]]
    return result


import cv2
import numpy as np
img = cv2.imread('Resources/Adanna.jpeg')
'''
Downside of this method
It does not resize, uses orignial size if you want to stack more
it would not work if the images are not in the same chanel, eg. greyscale and bgr
'''
# #horizontal stacking
# imgHor = np.hstack((img,img))
# #vertical stacking
# imgVer = np.vstack((img,img))
# #solution
def stackImages(scale,imgArray):
    rows = len(imgArray)
    cols = len(imgArray[0])
    rowsAvailable = isinstance(imgArray[0], list)
    width = imgArray[0][0].shape[1]
    height = imgArray[0][0].shaape[0]
    if rowsAvailable:
        for x in range(0,rows):
            for y in range(o,cols):
                if imgArray[x][y].shape[:2] == imgArray[0][0].shape[:2]:
                    imgArray[x][y] = cv2.resize(imgArray[x][y], (0,0), None)
                else:
                    imgArray[x][y] = cv2.resize(imgArray[x][y], (imgArray[0][0]))
                if len(imgArray[x][y].shape) == 2: imgArray[x][y] = cv2.colorChange
        imageBlank = np.zeros((height,width,3), np.uint8)

# cv2.imshow("Horizontal",imgHor)
# cv2.imshow("Vertical",imgVer)

cv2.waitKey(0)


import collections
import math
import numbers
import random

import numpy as np
import torch
from PIL import Image

class Scale(object):
    def __init__(self, size, interpolation=Image.BILINEAR):
        assert isinstance(size, int) or (isinstance(size, collections.Iterable) and len(size) == 2)
        self.size = size
        self.interpolation = interpolation

    def __call__(self, img):
        if isinstance(self.size, int):
            w, h = img.size
            if (w <= h and w == self.size) or (h <= w and h == self.size):
                return img
            if w < h:
                ow = self.size
                oh = int(self.size * h / w)
                return img.resize((ow, oh), self.interpolation)
            else:
                oh = self.size
                ow = int(self.size * w / h)
                return img.resize((ow, oh), self.interpolation)
        else:
            return img.resize(self.size, self.interpolation)

class ToParallel(object):
    def __init__(self, transforms):
        self.transforms = transforms

    def __call__(self, img):
        yield img
        for t in self.transforms:
            yield t(img)

class ToLabel(object):
    def __call__(self, inputs):
        # tensors = []
        # for i in inputs:
        # tensors.append(torch.from_numpy(np.array(i)).long())
        tensors = torch.from_numpy(np.array(inputs)).long()
        return tensors

class ToLabel_P(object):
    def __call__(self, inputs):
        tensors = []
        for i in inputs:
            tensors.append(torch.from_numpy(np.array(i)).long())
            # tensors = torch.from_numpy(np.array(inputs)).long()
        return tensors

class ReLabel(object):
    def __init__(self, olabel, nlabel):
        self.olabel = olabel
        self.nlabel = nlabel

    def __call__(self, inputs):
        # assert isinstance(input, torch.LongTensor), 'tensor needs to be LongTensor'
        for i in inputs:
            i[i == self.olabel] = self.nlabel
        return inputs

class ToSP(object):
    def __init__(self, size):
        self.scale2 = Scale(size / 2, Image.NEAREST)
        self.scale4 = Scale(size / 4, Image.NEAREST)
        self.scale8 = Scale(size / 8, Image.NEAREST)
        self.scale16 = Scale(size / 16, Image.NEAREST)
        self.scale32 = Scale(size / 32, Image.NEAREST)
        self.scale64 = Scale(size / 64, Image.NEAREST)

    def __call__(self, input):
        # input2 = self.scale2(input)
        # input4 = self.scale4(input)
        # input8 = self.scale8(input)
        # input16 = self.scale16(input)
        # input32 = self.scale32(input)
        input64 = self.scale64(input)
        inputs = input  # [input, input64]
        # inputs =input

        return inputs

class HorizontalFlip(object):
    """Horizontally flips the given PIL.Image with a probability of 0.5."""

    def __call__(self, img):
        return img.transpose(Image.FLIP_LEFT_RIGHT)

class VerticalFlip(object):
    def __call__(self, img):
        return img.transpose(Image.FLIP_TOP_BOTTOM)

def uint82bin(n, count=8):
    """returns the binary of integer n, count refers to amount of bits"""
    return ''.join([str((n >> y) & 1) for y in range(count - 1, -1, -1)])

def labelcolormap(N):
    cmap = np.zeros((N, 3), dtype=np.uint8)
    for i in range(N):
        r = 0
        g = 0
        b = 0
        id = i
        for j in range(7):
            str_id = uint82bin(id)
            r = r ^ (np.uint8(str_id[-1]) << (7 - j))
            g = g ^ (np.uint8(str_id[-2]) << (7 - j))
            b = b ^ (np.uint8(str_id[-3]) << (7 - j))
            id = id >> 3
        cmap[i, 0] = r
        cmap[i, 1] = g
        cmap[i, 2] = b
    return cmap

def pallet():
    pallet = [[128, 64, 128],
              [244, 35, 232],
              [70, 70, 70],
              [102, 102, 156],
              [190, 153, 153],
              [153, 153, 153],
              [250, 170, 30],
              [220, 220, 0],
              [107, 142, 35],
              [152, 251, 152],
              [70, 130, 180],
              [220, 20, 60],
              [255, 0, 0],
              [0, 0, 142],
              [0, 0, 70],
              [0, 60, 100],
              [0, 80, 100],
              [0, 0, 230],
              [119, 11, 32],
              [0, 0, 0]]
    pallet = np.array(pallet)
    return pallet

def colormap(n):
    cmap = np.zeros([n, 3]).astype(np.uint8)

    for i in np.arange(n):
        r, g, b = np.zeros(3)

        for j in np.arange(8):
            r = r + (1 << (7 - j)) * ((i & (1 << (3 * j))) >> (3 * j))
            g = g + (1 << (7 - j)) * ((i & (1 << (3 * j + 1))) >> (3 * j + 1))
            b = b + (1 << (7 - j)) * ((i & (1 << (3 * j + 2))) >> (3 * j + 2))

        cmap[i, :] = np.array([r, g, b])

    return cmap

class Colorize(object):
    def __init__(self, n=20):
        self.cmap = labelcolormap(n)
        self.cmap = torch.from_numpy(self.cmap[:n])

    def __call__(self, gray_image):
        size = gray_image.size()
        color_image = torch.ByteTensor(3, size[1], size[2]).fill_(0)

        for label in range(0, len(self.cmap)):
            mask = (label == gray_image[0]).cpu()
            color_image[0][mask] = self.cmap[label][0]
            color_image[1][mask] = self.cmap[label][1]
            color_image[2][mask] = self.cmap[label][2]

        return color_image

class Colorize2(object):
    def __init__(self, n=20):
        self.cmap = pallet()
        self.cmap = torch.from_numpy(self.cmap)

    def __call__(self, gray_image):
        size = gray_image.size()
        color_image = torch.ByteTensor(3, size[1], size[2]).fill_(0)

        for label in range(0, len(self.cmap)):
            mask = (label == gray_image[0]).cpu()
            color_image[0][mask] = self.cmap[label][0]
            color_image[1][mask] = self.cmap[label][1]
            color_image[2][mask] = self.cmap[label][2]

        return color_image

class RandomSizedCrop:
    """This is random sized cropping."""

    def __init__(self, size=None, interpolation=Image.BILINEAR):
        """Set output size and type of interpolation."""
        self.size = size
        self.img_interpolation = interpolation
        self.target_interpolation = Image.NEAREST

    def __call__(self, img):
        """Random sized cropp -> resize into 'self.size'."""

        # default size
        if self.size is None:
            self.size = img.size

        # try 10times
        for attempt in range(10):
            area = img.size[0] * img.size[1]

            # decide w, h
            cropped_area = random.uniform(0.5, 1.0) * area
            aspect_ratio = random.uniform(3. / 4, 4. / 3)
            w = int(round(math.sqrt(cropped_area * aspect_ratio)))
            h = int(round(math.sqrt(cropped_area / aspect_ratio)))

            # which is larger (prob: 0.5)
            if random.random() < 0.5:
                w, h = h, w

            # random crop, if possible
            if w <= img.size[0] and h <= img.size[1]:
                x1 = random.randint(0, img.size[0] - w)
                y1 = random.randint(0, img.size[1] - h)

                img = img.crop((x1, y1, x1 + w, y1 + h))
                assert (img.size == (w, h))

                return img.resize(self.size, self.img_interpolation)

        # Fallback
        scale = Scale(self.size, interpolation=self.img_interpolation)
        crop = CenterCrop(self.size)
        return crop(scale(img))

class CenterCrop(object):
    def __init__(self, size):
        if isinstance(size, numbers.Number):
            self.size = (int(size), int(size))
        else:
            self.size = size

    def __call__(self, img):
        w, h = img.size
        th, tw = self.size
        x1 = int(round((w - tw) / 2.))
        y1 = int(round((h - th) / 2.))
        return img.crop((x1, y1, x1 + tw, y1 + th))

class RandomHorizontalFlip:
    """
    Random horizontal flip.

    prob = 0.5
    """

    def __call__(self, img):
        if random.random() < 0.5:
            return img.transpose(Image.FLIP_LEFT_RIGHT)
        return img

class RandomVerticalFlip:
    """
    Random vertical flip.

    prob = 0.5
    """

    def __call__(self, img):
        if random.random() < 0.5:
            return img.transpose(Image.FLIP_TOP_BOTTOM)
        return img

class RandomRotation:
    """
    Random roatation.

    -max_deg ~ deg
    """

    def __call__(self, img, max_deg=10):
        deg = np.random.randint(-max_deg, max_deg, 1)[0]
        return img.rotate(deg)


""" This Module contains database access class which will help in executing query """
import logging
from database.db_connector import DatabaseConnection
from config.sqlite_queries import DatabaseConfig

logger = logging.getLogger("db_logger")

class DatabaseAccess:
    DB_PATH = DatabaseConfig.DB_PATH

    # execute query of non returning type such as update, delete, insert
    @classmethod
    def execute_non_returning_query(cls, query, params=None):
        """Execute a non-returning query (update, delete, insert)."""
        with DatabaseConnection(cls.DB_PATH) as connection:
            cursor = connection.cursor()
            if params is None:
                cursor.execute(query)
            else:
                cursor.execute(query, params)

    # execute query of returning type such as read
    @classmethod
    def execute_returning_query(cls, query, params=None):
        """Execute a returning query (read)."""
        with DatabaseConnection(cls.DB_PATH) as connection:
            cursor = connection.cursor()
            if params is None:
                cursor.execute(query)
            else:
                cursor.execute(query, params)
            data_from_db = cursor.fetchall()
        return data_from_db


from typing import List, Dict, Union
import h5py
import copy
import numpy as np
import logging

from ophys_etl.types import OphysROI
from ophys_etl.utils.roi_masks import RoiMask
from ophys_etl.modules.trace_extraction.utils import \
        calculate_roi_and_neuropil_traces
import ophys_etl.modules.decrosstalk.decrosstalk_types as dc_types

logger = logging.getLogger(__name__)

class OphysMovie(object):

    def __init__(self, movie_path: str, motion_border: Dict[str, float]):
        """
        Parameters
        ----------
        movie_path -- path to the motion corrected movie file

        motion_border -- dict defining the border of the valid region
        within each frame, e.g.
            {
                "y1": 19.0758,
                "y0": 22.3232,
                "x0": 9.16988,
                "x1": 7.79272
             }
        """

        self._path = movie_path
        self._motion_border = copy.deepcopy(motion_border)

        # this is where the data from the movie file will be stored
        self._data = None

    @property
    def path(self) -> str:
        return self._path

    @property
    def motion_border(self) -> Dict[str, float]:
        return copy.deepcopy(self._motion_border)

    def load_movie_data(self):
        """
        Load the data from self._path; store te data in self._data
        """
        logger.info(f'loading {self.path}')
        with h5py.File(self.path, mode='r') as in_file:
            self._data = in_file['data'][()]

    @property
    def data(self) -> np.ndarray:
        if self._data is None:
            self.load_movie_data()
        return self._data

    def purge_movie(self) -> None:
        """
        Delete loaded movie data
        """
        if self._data is not None:
            self._data = None
        return None

    def get_trace(self, roi_list: List[OphysROI]) -> dc_types.ROISetDict:
        """
        Extract the traces from a movie as defined by the ROIs in roi_list

        Parameters
        ----------
        roi_list -- a list of OphysROI instantiations
                    specifying the ROIs from which to
                    extract traces

        Returns
        -------
        output -- a decrosstalk_types.ROISetDict containing the ROI and
                  neuropil traces associated with roi_list. For each ROI
                  in the ROISetDict, only the 'signal' channel will be
                  populated, this with the trace extracted from the movie.
        """
        motion_border = [self._motion_border['x0'], self._motion_border['x1'],
                         self._motion_border['y0'], self._motion_border['y1']]

        height = self.data.shape[1]
        width = self.data.shape[2]

        roi_mask_list = []
        for roi in roi_list:
            pixels = np.argwhere(roi.mask_matrix)
            pixels[:, 0] += roi.y0
            pixels[:, 1] += roi.x0
            mask = RoiMask.create_roi_mask(width, height, motion_border,
                                           pix_list=pixels[:, [1, 0]],
                                           label=str(roi.roi_id),
                                           mask_group=-1)

            roi_mask_list.append(mask)

        _traces = calculate_roi_and_neuropil_traces(self.data,
                                                    roi_mask_list,
                                                    motion_border)
        roi_traces = _traces[0]
        neuropil_traces = _traces[1]

        output = dc_types.ROISetDict()
        for i_roi, roi in enumerate(roi_list):

            trace = dc_types.ROIChannels()
            trace['signal'] = roi_traces[i_roi]
            output['roi'][roi.roi_id] = trace

            trace = dc_types.ROIChannels()
            trace['signal'] = neuropil_traces[i_roi]
            output['neuropil'][roi.roi_id] = trace

        return output

class DecrosstalkingOphysPlane(object):

    def __init__(self,
                 experiment_id=None,
                 movie_path=None,
                 motion_border=None,
                 roi_list=None,
                 max_projection_path=None,
                 qc_file_path=None):

        """
        Parameters
        ----------
        experiment_id -- an integer uniquely identifying
                         this experimental plane

        movie_path -- path to the motion corrected movie file

        motion_border -- dict defining the border of the valid region
        within each frame, e.g.
            {
                "y1": 19.0758,
                "y0": 22.3232,
                "x0": 9.16988,
                "x1": 7.79272
             }

        roi_list -- a list of OphysROIs indicating the ROIs in this movie

        max_projection_path -- path to the maximum projection image for
                               this plane

        qc_file_path -- path to the HDF5 file containing quality control
                        data. This will probably always be initialized
                        to None and will be set by the decrosstalking
                        pipeline after processing but before quality
                        control figure generation.
        """

        if experiment_id is None or not isinstance(experiment_id, int):
            raise ValueError("DecrosstalkingOphysPlane.experiment_id "
                             "must be an int; you gave "
                             "%s" % str(type(experiment_id)))

        if movie_path is None:
            raise ValueError("Must specify movie_path when "
                             "initializing DecrosstalkingOphysPlane")

        if motion_border is None or not isinstance(motion_border, dict):
            raise ValueError("DecrosstalkingOphysPlane.motion_border "
                             "must be a dict; you gave "
                             "%s" % str(type(motion_border)))

        if roi_list is None or not isinstance(roi_list, list):
            raise ValueError("DecrosstalkingOphysPlane.roi_list "
                             "must be a list of OphysROI; you gave "
                             "%s" % str(type(roi_list)))

        if len(roi_list) > 0:
            for roi in roi_list:
                if not isinstance(roi, OphysROI):
                    raise ValueError("DecrosstalkingOphysPlane.roi_list "
                                     "must be a list of OphysROI; you gave "
                                     "a list of %s" % str(type(roi)))

        self._experiment_id = experiment_id
        self._movie = OphysMovie(movie_path, motion_border)
        self._roi_list = copy.deepcopy(roi_list)
        self._max_projection_path = max_projection_path
        self._qc_file_path = qc_file_path

    @property
    def experiment_id(self) -> int:
        return self._experiment_id

    @property
    def movie(self) -> OphysMovie:
        return self._movie

    @property
    def roi_list(self) -> List[OphysROI]:
        return self._roi_list

    @property
    def maximum_projection_image_path(self) -> Union[None, str]:
        return self._max_projection_path

    @property
    def qc_file_path(self) -> Union[None, str]:
        return self._qc_file_path

    @qc_file_path.setter
    def qc_file_path(self, input_path: str):
        self._qc_file_path = input_path

    @classmethod
    def from_schema_dict(cls, schema_dict):
        """
        Create an OphysPlane from a dict taken from the module's argschema

        Parameters
        ----------
        schema_dict -- a dict codifying the plane, as read from argschema, i.e.

        {  # start of ophys_experiment
         "ophys_experiment_id": ,# an int
         "motion_corrected_stack": ,  # path to h5 movie file
         "motion_border": {  # border widths
                 "x0": ,  # a float
                 "x1": ,  # a float
                 "y0": ,  # a float
                 "y1": ,  # a float
           },
           "rois": [  # list of dicts definining the ROIs for this experiment
               {  # start of individual ROI
                 "id":  ,  # an int
                 "x": ,  # an int
                 "y": ,  # an int
                 "width": ,  # an int
                 "height": ,  # an int
                 "valid_roi": ,  # boolean
                 "mask_matrix": [[]]  # 2-D array of booleans
               },  # end of individual ROI,
               {
                 "id":  ,
                 "x": ,
                 "y": ,
                 "width": ,
                 "height": ,
                 "valid_roi": ,
                 "mask_matrix": [[]]
               },
               ...
           ]
         }
        """
        roi_list = []
        for roi in schema_dict['rois']:
            roi_list.append(OphysROI.from_schema_dict(roi))

        max_path = None
        if 'maximum_projection_image_file' in schema_dict:
            max_path = schema_dict['maximum_projection_image_file']

        return cls(experiment_id=schema_dict['ophys_experiment_id'],
                   movie_path=schema_dict['motion_corrected_stack'],
                   motion_border=schema_dict['motion_border'],
                   roi_list=roi_list,
                   max_projection_path=max_path)


import pytest

from demoqa_collection.elements.pages.page_radio_button import PageRadioButton

@pytest.mark.usefixtures('chrome')
class TestRadioButton:

    def test_radio_button(self):
        page = PageRadioButton(self.driver)
        page.open()
        assert len(page.get_all_possible_radio_buttons()) == 3

    def test_select_yes_button(self):
        page = PageRadioButton(self.driver)
        page.open()
        button = page.button_yes
        page.check(button)
        output_text = page.get_output()
        assert button in output_text

    def test_select_impressive_button(self):
        page = PageRadioButton(self.driver)
        page.open()
        button = page.button_impressive
        page.check(button)
        output_text = page.get_output()
        assert button in output_text

    def test_no_button_is_selected(self):
        page = PageRadioButton(self.driver)
        page.open()
        button = page.button_no
        page.enable_button()
        page.check(button)
        assert page.check_button_is_selected(button)

    def test_radio_button_info(self):
        page = PageRadioButton(self.driver)
        page.open()
        radio_buttons = page.get_all_possible_radio_buttons()
        radio_button_info = {}
        for button in radio_buttons:
            button_element = page.get_radio_button_by_text(button)
            button_info = {
                'is enabled': button_element.is_enabled(),
                'is selected': button_element.is_selected()
            }
            radio_button_info[button] = button_info
        print(radio_button_info)


'''
Capsules for Object Segmentation (SegCaps)
Original Paper by Rodney LaLonde and Ulas Bagci (https://arxiv.org/abs/1804.04241)
Code written by: Rodney LaLonde
If you use significant portions of this code or the ideas from our paper, please cite it :)
If you have any questions, please email me at lalonde@knights.ucf.edu.
This file is used for testing models. Please see the README for details about testing.
'''

from __future__ import print_function

import matplotlib
matplotlib.use('TkAgg')
import matplotlib.pyplot as plt
plt.ioff()

from os.path import join
from os import makedirs
import csv
import numpy as np
import scipy.ndimage.morphology
from skimage import measure, filters
from metrics import dc, jc, assd

from keras import backend as K
K.set_image_data_format('channels_last')
from keras.utils import print_summary

from load_data import generate_test_batches

def threshold_mask(raw_output, threshold):
    if threshold == 0:
        try:
            threshold = filters.threshold_otsu(raw_output)
        except:
            threshold = 0.5

    print('\tThreshold: {}'.format(threshold))

    return threshold

    '''all_labels = measure.label(raw_output)
    props = measure.regionprops(all_labels)
    props.sort(key=lambda x: x.area, reverse=True)
    thresholded_mask = np.zeros(raw_output.shape)

    if len(props) >= 2:
        if props[0].area / props[1].area > 5:  # if the largest is way larger than the second largest
            thresholded_mask[all_labels == props[0].label] = 1  # only turn on the largest component
        else:
            thresholded_mask[all_labels == props[0].label] = 1  # turn on two largest components
            thresholded_mask[all_labels == props[1].label] = 1
    elif len(props):
        thresholded_mask[all_labels == props[0].label] = 1

    thresholded_mask = scipy.ndimage.morphology.binary_fill_holes(thresholded_mask).astype(np.uint8)

    return thresholded_mask'''

def dice_sorensen(y_true, y_pred):
    inse = np.sum(np.sum(y_true * y_pred, axis=0), axis=0)
    l = np.sum(np.sum(y_true, axis=0), axis=0)
    r = np.sum(np.sum(y_pred, axis=0), axis=0)
    smooth =  1e-7

    dice = (2 * inse + smooth) / (l + r + smooth)
    return dice

def test(args, images_test, gt_test, model_list, net_input_shape):
    if args.weights_path == '':
        weights_path = join(args.check_dir, args.output_name + '_model_' + args.time + '.hdf5')
    else:
        weights_path = join(args.weights_path)

    output_dir = join('D:\Engineering Physics\Skripsi\Program\Ischemic Stroke Segmentation', 'results', args.net)
    raw_out_dir = join(output_dir, 'raw_output')
    fin_out_dir = join(output_dir, 'final_output')
    fig_out_dir = join(output_dir, 'qual_figs')
    try:
        makedirs(raw_out_dir)
    except:
        pass
    try:
        makedirs(fin_out_dir)
    except:
        pass
    try:
        makedirs(fig_out_dir)
    except:
        pass

    if len(model_list) > 1:
        eval_model = model_list[1]
    else:
        eval_model = model_list[0]
    try:
        eval_model.load_weights(weights_path)
    except:
        print('Unable to find weights path. Testing with random weights.')
    print_summary(model=eval_model, positions=[.38, .65, .75, 1.])

    # Set up placeholders
    outfile = ''
    if args.compute_dice:
        dice_arr = np.zeros(len(images_test))
        outfile += 'dice_'
    if args.compute_jaccard:
        jacc_arr = np.zeros(len(images_test))
        outfile += 'jacc_'
    if args.compute_assd:
        assd_arr = np.zeros(len(images_test))
        outfile += 'assd_'

    # Testing the network
    print('Testing... This will take some time...')

    with open(join(output_dir, args.save_prefix + outfile + 'scores.csv'), 'w') as csvfile:
        writer = csv.writer(csvfile, delimiter=',', quotechar='|', quoting=csv.QUOTE_MINIMAL)

        row = ['Scan Name']
        if args.compute_dice:
            row.append('Dice Coefficient')

        if args.compute_jaccard:
            row.append('Jaccard Index')

        if args.compute_assd:
            row.append('Average Symmetric Surface Distance')

        writer.writerow(row)

        for i in range(0, len(images_test), 1):
            print(images_test[i].shape)
            output_array = eval_model.predict_generator(generate_test_batches([images_test[i]],
                                                                              net_input_shape,
                                                                              batchSize=1,
                                                                              numSlices=1,
                                                                              subSampAmt=0,
                                                                              stride=1),
                                                        steps=19, max_queue_size=1, workers=0,
                                                        use_multiprocessing=False, verbose=1)

            if args.net.find('caps') != -1:
                output = output_array[0][:, :, :, 0]
                recon = output_array[1][:, :, :, 0]
            else:
                output = output_array[:, :, :, 0]

            threshold = 0.7

            output = np.rollaxis(output, 0, 3)
            recon = np.rollaxis(recon, 0, 3)

            #for k in range(0, 19, 1):
            #   plt.imshow(output[:, :, k], cmap='gray')
            #  plt.show()

            print(threshold)
            output_bin = output[:, :, :]
            print(output_bin.shape)

            output_bin[output_bin > threshold] = 1
            output_bin[output_bin < 1] = 0

            # for l in range(0, 19, 1):
            #    plt.imshow(output_bin[:, :, l], cmap='gray')
            #   plt.show()

            for o in range(0, 19, 1):
                print('Dice_Slice_', o, '= ', dice_sorensen(gt_test[i][:, :, o], output[:, :, o]))

            for m in range(0, 19, 1):
                plt.imshow(images_test[i][:, :, m], alpha=1, cmap='gray')
                plt.imshow(output_bin[:, :, m], alpha=0.5, cmap='Blues')
                plt.imshow(gt_test[i][:, :, m], alpha=0.5, cmap='Reds')
                plt.show()

            # Plot Qualitative Figure
            print('Creating Qualitative Figure for Quick Reference')
            f, ax = plt.subplots(1, 3, figsize=(15, 5))

            ax[0].imshow(images_test[i][:, :, images_test[i].shape[2] // 3], alpha=1, cmap='gray')
            ax[0].imshow(output[:, :, images_test[i].shape[2] // 3], alpha=0.5, cmap='Blues')
            ax[0].imshow(gt_test[i][:, :, images_test[i].shape[2] // 3], alpha=0.2, cmap='Reds')
            ax[0].set_title('Slice {}/{}'.format(images_test[i].shape[2] // 3, images_test[i].shape[2]))
            ax[0].axis('off')

            ax[1].imshow(images_test[i][:, :, images_test[i].shape[2] // 2], alpha=1, cmap='gray')
            ax[1].imshow(output[:, :, images_test[i].shape[2] // 2], alpha=0.5, cmap='Blues')
            ax[1].imshow(gt_test[i][:, :, images_test[i].shape[2] // 2], alpha=0.2, cmap='Reds')
            ax[1].set_title('Slice {}/{}'.format(images_test[i].shape[2] // 2, images_test[i].shape[2]))
            ax[1].axis('off')

            ax[2].imshow(images_test[i][:, :, images_test[i].shape[2] // 2 + images_test[i].shape[2] // 4], alpha=1, cmap='gray')
            ax[2].imshow(output[:, :, images_test[i].shape[2] // 2 + images_test[i].shape[2] // 4], alpha=0.5, cmap='Blues')
            ax[2].imshow(gt_test[i][:, :, images_test[i].shape[2] // 2 + images_test[i].shape[2] // 4], alpha=0.2, cmap='Reds')
            ax[2].set_title('Slice {}/{}'.format(images_test[i].shape[2] // 2 + images_test[i].shape[2] // 4, images_test[i].shape[2]))
            ax[2].axis('off')

            fig = plt.gcf()
            fig.suptitle('Images of Qualitative Segmentation')

            plt.savefig(join(fig_out_dir, args.time + '_qual_fig' + '.png'),
                        format='png', bbox_inches='tight')
            plt.close('all')

            row = ['Average Score']
            if args.compute_dice:
                print('Computing Dice')
                dice_arr[i] = dc(output, gt_test[i])
                print('\tDice: {}'.format(dice_arr[i]))
                row.append(dice_arr[i])
            if args.compute_jaccard:
                print('Computing Jaccard')
                jacc_arr[i] = jc(output, gt_test[i])
                print('\tJaccard: {}'.format(jacc_arr[i]))
                row.append(jacc_arr[i])
            if args.compute_assd:
                print('Computing ASSD')
                assd_arr[i] = assd(output, gt_test[i], connectivity=1)
                print('\tASSD: {}'.format(assd_arr[i]))
                row.append(assd_arr[i])

            writer.writerow(row)

        row = ['Average Scores']
        if args.compute_dice:
            row.append(np.mean(dice_arr))
        if args.compute_jaccard:
            row.append(np.mean(jacc_arr))
        if args.compute_assd:
            row.append(np.mean(assd_arr))
        writer.writerow(row)

    print('Done.')


import subprocess
import os
from subprocess import Popen
import threading
import sys
from pydub import AudioSegment
from pydub.playback import play

HOME_DIR = os.environ['HOME']
MP3_1_DIR = HOME_DIR + '/mp3/build1.mp3'
MP3_2_DIR = HOME_DIR + '/mp3/build2.mp3'
MP3_3_DIR = HOME_DIR + '/mp3/build3.mp3'

class Ridocker:

    def __init__(self):
        self.stop_event = threading.Event()  # 停止させるかのフラグ

        # スレッドの作成と開始
        self.thread_1 = threading.Thread(target=self.exec)
        self.thread_2 = threading.Thread(
            target=self.run_mp3, args=(MP3_1_DIR,))
        self.thread_3 = threading.Thread(target=self.retry)

    def run_mp3(self, path):
        sound = AudioSegment.from_file(path, 'mp3')
        play(sound)

    def exec(self):
        cmd = "docker-compose build"
        popen = Popen(cmd, shell=True)
        popen.wait()
        self.stop()

    def retry(self):
        sound = AudioSegment.from_file(MP3_2_DIR, 'mp3')
        while not self.stop_event.is_set():
            play(sound)

    def stop(self):
        self.stop_event.set()
        self.thread_3.join()  # スレッドが停止するのを待つ
        self.run_mp3(MP3_3_DIR)

if __name__ == '__main__':
    h = Ridocker()

    h.thread_2.start()
    h.thread_2.join()
    h.thread_3.start()
    h.thread_1.start()


#-*- coding: utf-8 -*-
#list 出库并且矩阵对比
import face_recognition as fr
import numpy as np
import pymysql

db = pymysql.connect("localhost","root","","baiduyun" )
cursor = db.cursor()
sql = "select * from face order by id asc"
cursor.execute(sql)
results = cursor.fetchall()

img_a = fr.load_image_file("1.jpeg")
encoding_a = fr.face_encodings(img_a)[0]

mao = []
for row in results:
	id = row[0]
	names = row[1]
	tzm_list = row[3]
	# 将字符串转为numpy ndarray类型，即矩阵
	# 转换成一个list
	dlist = tzm_list.strip(' ').split(',')
	# 将list中str转换为float
	dfloat = list(map(float, dlist))
	arr = np.array(dfloat) #arr 数据库里出来的矩阵
	ad = fr.face_distance([arr], encoding_a)
	#print("aaa %f" % (ad))
	mao.append([ad[0],names])

db.close()
#list = [(1,93),(2,71),(3,89),(4,93),(5,85),(6,77)]
mao.sort(key=lambda x:x[0])
#print (mao)
print(mao[0])
print(mao[1])
print(mao[2])
print(mao[3])
print(mao[4])


import os
import pandas as pd
import numpy as np
from time import time
import matplotlib.pyplot as plt
import sys
sys.path.append('/shogunMMDutils')
import matplotlib as mpl
import fathon
from fathon import fathonUtils as fu

import pickle
import logging

from sklearn.preprocessing import StandardScaler, MinMaxScaler
def return_scaled(x):
    scaler = StandardScaler()
    x_scaled = scaler.fit_transform(np.array(x).reshape(-1,1))
    return np.asarray(x_scaled)

def logger(name, level=logging.INFO):
    tmp_logger = logging.getLogger(name)
    if not tmp_logger.handlers:
        formatter = logging.Formatter('%(asctime)s %(name)s  %(levelname)s %(message)s')
        handler = logging.StreamHandler()
        handler.setFormatter(formatter)

        tmp_logger.setLevel(level)
        tmp_logger.addHandler(handler)
    return tmp_logger

class volatilityEstimation(object):

    def __init__(self, df, tick_window=20, clean=True):
        self.df = df
        self.tick_window = tick_window
        self.mu_1 = np.sqrt((2 / np.pi))
        self.mu_43 = 2 ** (2 / 3) * scipy.special.gamma(7 / 6) ** scipy.special.gamma(1 / 2) ** (-1)
        self.clean = True  # set to true to remove nans at the beginning of the period
        self.trading_seconds_btc = 86400
        self.trading_seconds_futures = 23400

        self.column_open = 'micro_price_open'
        self.column_high = 'micro_price_high'
        self.column_low = 'micro_price_low'
        self.column_close = 'micro_price_close'

        self.log_hl = (df[str(self.column_high)] / df[str(self.column_low)]).apply(np.log)
        self.log_co = (df[str(self.column_close)] / df[str(self.column_open)]).apply(np.log)
        # median sampling frequency

        self.z = pd.DataFrame(
            [((x.hour * 60 + x.minute) * 60 + x.second) for x in self.df['TimeStamp_open']]).diff().fillna(0).astype(
            'float64')
        self.clean_arrival_rates = np.asarray(
            self.z[(self.z > self.z.quantile(0.003)) & (self.z < self.z.quantile(0.97))].dropna()).astype('float64')
        # clean arrival rates after i remove all the outliers
        self.mean_sampling_frequency = np.mean(np.asarray(self.clean_arrival_rates))

    def arrival_rates(self):

        z = pd.DataFrame([((x.hour * 60 + x.minute) * 60 + x.second) for x in self.df['TimeStamp_open']]).diff().fillna(
            0).astype('float64')
        clean_arrival_rates = np.asarray(z[(z > z.quantile(0.003)) & (z < z.quantile(0.97))].dropna()).astype('float64')
        # clean arrival rates after i remove all the outliers

        return clean_arrival_rates

    def M_parameter(self):
        """
        normalised sampling frequency for adjustments
        """

        M_btc = self.trading_seconds_btc / self.mean_sampling_frequency
        M_futures = self.trading_seconds_futures / self.mean_sampling_frequency

        return [M_btc, M_futures]

    def garmanKlass(self, trading_periods):

        rs = 0.5 * self.log_hl ** 2 - (2 * np.log(2) - 1) * self.log_co ** 2

        def f(v):
            return (trading_periods * v.mean()) ** 0.5

        result = rs.rolling(window=self.tick_window, center=False).apply(func=f)

        if self.clean:
            return result.dropna()
        else:
            return result

    def parkinson(self, trading_periods, clean=True):

        rs = (1.0 / (4.0 * np.log(2.0))) * (self.df[str(self.column_high)] / self.df[str(self.column_close)]).apply(
            np.log) ** 2.0

        def f(v):
            return trading_periods * v.mean() ** 0.5

        result = rs.rolling(window=self.tick_window, center=False).apply(func=f)

        if self.clean:
            return result.dropna()
        else:
            return result

    def bipower_variation(self, rollingWindow=5):
        '''
        Bipower Variation (BV) is the sum of the product of absolute time series returns
        :param column: price column
        :return: returns bivariate variation
        Barnhorf - Nielse & Shephard 2004 & 2006
        '''
        bv = self.mu_1 ** (-2) * (
            (self.log_hl.abs() * self.log_hl.shift(1).abs()).fillna(0).rolling(rollingWindow).sum())

        if self.clean:
            return bv.dropna()
        else:
            return bv

    def tripower_quarticity(self, rollingWindow=5, sampling_param=0):
        """
        using M for BTC here - change sampling param to 1 for everything else
        """
        tpq = self.M_parameter()[sampling_param] * self.mu_43 ** (-3) * ((self.log_hl.abs() ** (
                    4 / 3) * self.log_hl.shift(1).abs() ** (4 / 3) * self.log_hl.shift(2).abs() ** (4 / 3)).rolling(
            rollingWindow).sum().fillna(0))

        if self.clean:

            return tpq.dropna()
        else:
            return tpq

    def realised_variance(self, rollingWindow=5):
        """
        realised variance - andersen & bollerselv -1998
        """

        realvar = (self.log_hl ** 2).rolling(rollingWindow).sum().fillna(0)

        if self.clean:
            return realvar.dropna()
        else:
            return realvar

    def realised_absolute_variation(self, rollingWindow=5, sampling_param=0):
        """
        realised absolute variation - Forsberg & Ghysels 2007
        using M for BTC here - change sampling param to 1 for everything else
        """

        realabsovar = self.mu_1 ** (-2) * self.M_parameter[sampling_param] ** (-0.5) * self.log_hl.rolling(
            rollingWindow).sum().fillna(0)

        return realabsovar

    def realised_skewness_kurtosis(self, rollingWindow=15, sampling_param=0):

        """
        set the rolling window to 10-15
        sampling param for BTC ~ 0
        """

        rm3 = (self.log_hl ** 3).rolling(rollingWindow).sum().fillna(0)
        rm4 = (self.log_hl ** 4).rolling(rollingWindow).sum().fillna(0)

        rs = np.sqrt(self.M_parameter()[sampling_param]) * (
                    rm3 / self.realised_variance(rollingWindow=rollingWindow)) ** (3 / 2)
        rk = self.M_parameter()[sampling_param] * (rm4 / self.realised_variance(rollingWindow=rollingWindow)) ** 2

        return rs, rk

    def jumps_test(self, rollingWindow, sampling_param=0):
        """
        Jump test by Huang and Tauchen 2005
        adopted by repo: RealisedQuantities
        """

        j1 = (np.log(self.realised_variance(rollingWindow=rollingWindow)) - np.log(
            self.bipower_variation(rollingWindow=rollingWindow)))
        j2 = (((self.mu_1 ** -4) + 2 * (self.mu_1 ** -2) - 5) /
              (self.M_parameter()[0] * self.tripower_quarticity(rollingWindow=rollingWindow,
                                                                sampling_param=sampling_param)
               * (self.bipower_variation(rollingWindow=rollingWindow) ** -2)))

        j = j1 / (j2 ** 0.5)

        return (j.abs() >= stats.norm.ppf(0.995)) * 1  # can also look at 0.999)*

    def relative_jump_measure(self, rollingWindow, sampling_param=0):

        nominator = (self.realised_variance(rollingWindow=rollingWindow)
                     - self.bipower_variation(rollingWindow=rollingWindow))

        measure_to_return = nominator / self.bipower_variation(rollingWindow=rollingWindow)

        return measure_to_return

    def negative_semivariance(self, x):
        """
        input: returns
        negative realised semivariance
        Barndorff- Nielsen
        """
        return x.apply(lambda x: (x ** 2)) * (x < 0).sum()

    def positive_semivariance(self, x):
        """
        input: returns
        positive realised semivariance
        Barndorff- Nielsen
        """
        return x.apply(lambda x: (x ** 2)) * (x > 0).sum()

    def signed_jump_variation(self, x):

        sjv = self.positive_semivariance(x) - self.negative_semivariance(x)
        sjv_p = sjv * (sjv > 0)
        sjv_n = sjv * (sjv < 0)

        return sjv_p, sjv_n

    def continuous_discontinuous_quadratic_variance(self):

        """
        seperate continuous and discontinuous parts of quadratic variation

        """
        pass

class descriptiveStats(object):

    def __init__(self, df):
        self.df = df
        self.column_open = 'micro_price_open'
        self.column_high = 'micro_price_high'
        self.column_low = 'micro_price_low'
        self.column_close = 'micro_price_close'
        self.log_hl = (df[str(self.column_high)] / df[str(self.column_low)]).apply(np.log)
        self.log_co = (df[str(self.column_close)] / df[str(self.column_open)]).apply(np.log)

    def base_descriptive_stats(self):
        desc_stats = dict()
        desc_stats['median_hl'] = np.median(self.log_hl)
        desc_stats['mean_co'] = np.median(self.log_co)
        desc_stats['quantile_95_hl'] = self.log_hl.quantile(0.95)
        desc_stats['quantile_95_co'] = self.log_co.quantile(0.95)
        desc_stats['q3_75_hl'] = self.log_hl.quantile(0.75)
        desc_stats['q3_75_co'] = self.log_co.quantile(0.75)
        desc_stats['q1_25_hl'] = self.log_hl.quantile(0.25)
        desc_stats['q1_25_co'] = self.log_co.quantile(0.25)
        desc_stats['outlier_lower_limit_hl'] = desc_stats['q1_25_hl'] - 1.5 * (
                    desc_stats['q3_75_hl'] - desc_stats['q1_25_hl'])
        desc_stats['outlier_upper_limit_hl'] = desc_stats['q3_75_hl'] + 1.5 * (
                    desc_stats['q3_75_hl'] - desc_stats['q1_25_hl'])
        desc_stats['lower_limit_outliers'] = self.log_hl[self.log_hl > desc_stats['outlier_lower_limit_hl']].count()
        desc_stats['upper_limit_outliers'] = self.log_hl[self.log_hl > desc_stats['outlier_upper_limit_hl']].count()
        desc_stats['total_outliers'] = desc_stats['lower_limit_outliers'] + desc_stats['upper_limit_outliers']

        return desc_stats

class DataLoader:
    """
    loads data and constructs full LOB and calendar bars for each one
    """

    def __init__(self, data_location,
                 symbol,
                 dates,
                 use_columns,
                 calendar_resample,
                 trade_volume_width,
                 ticks_width,
                 usd_volume_width):

        self._data_location = data_location
        self._symbol = symbol
        self._input_dates = dates
        self._use_columns = use_columns
        self._calendar_resample = calendar_resample
        self._ticks_width = ticks_width
        self._trade_volume_width = trade_volume_width
        self._usd_volume_width = usd_volume_width
        self._dates = []
        self._bars_dict = {}
        self.calendar_bars = []
        self.volume_bars = []
        self.tick_bars = []
        self.usd_volume_bars = []
        self._logger = logger('DataLoader')
        #         self._symbols = sorted(['VIX',  'FB1',  'TU1',  'G_1',  'RX1',  'OE1',  'TY1',  'FV1',  'JB1',  'RX1',  'DU1',  'KE1',  'US1',  'YM1', 'XM1',  'VXX'] )
        self._data_folder = data_location

        self._quotes_string = "".join((self._symbol, '_Comdty_quotes'))
        self._trades_string = "".join((self._symbol, '_Comdty_trades'))

        #         self._quotes_files =sorted([ f for f in os.listdir(self._data_folder) if str('_quotes') in f])
        #         self._trades_files = sorted([f for f in os.listdir(self._data_folder) if str('_trades') in f])

        self._quotes_Files = os.path.join(self._data_folder, self._quotes_string)
        self._trades_Files = os.path.join(self._data_folder, self._trades_string)

    def load_and_format_data(self):
        mergedFile = dict()
        for date in self._input_dates:
            dateToLoad = os.path.join(date + '.csv')
            self._logger.info(f"Loading data for:{date}")

            quotesDateFile = os.path.join(self._quotes_Files, dateToLoad)
            print(quotesDateFile)
            tradesDateFile = os.path.join(self._trades_Files, dateToLoad)
            print(tradesDateFile)

            try:
                qt_tmp_df = pd.read_csv(quotesDateFile, usecols=self._use_columns)
                qt_tmp_df['TradeTime'] = pd.to_datetime(qt_tmp_df.time).values

                quotes_df = pd.merge_asof(
                    qt_tmp_df[qt_tmp_df['type'] == 'BID'].dropna().fillna("ffill").sort_values('TradeTime')
                    , qt_tmp_df[qt_tmp_df['type'] == 'ASK'].dropna().fillna("ffill").sort_values('TradeTime'),
                    on='TradeTime', allow_exact_matches=True)

                quotes_df = quotes_df.rename(columns={'value_x': 'BestBid', 'value_y': 'BestAsk',
                                                      'size_x': 'BidSize', 'size_y': 'AskSize',
                                                      'time_x': 'QuoteTime'})

                tr_tmp_df = pd.read_csv(tradesDateFile, usecols=self._use_columns)
                tr_tmp_df['TradeTime'] = pd.to_datetime(tr_tmp_df.time).values
                trades_df = tr_tmp_df.dropna().fillna("ffill").sort_values('TradeTime')
                trades_df = trades_df.rename(columns={'value': 'TradePrice',
                                                      'size': 'TradeSize'})
                trades_columns = ['TradeSize', 'type', 'TradePrice', 'TradeTime']

                LOB = pd.merge_asof(quotes_df, trades_df[trades_columns].sort_values('TradeTime'), on='TradeTime',
                                    allow_exact_matches=True)

                LOB.BidSize = LOB['BidSize'].replace(0, 1)
                LOB.AskSize = LOB['AskSize'].replace(0, 1)
                LOB['TimeStamp'] = pd.to_datetime(LOB.TradeTime).dt.time
                LOB['TradeVolume'] = LOB['TradeSize'].fillna(0)
                LOB['total_traded_volume'] = LOB.TradeVolume
                LOB['milliSeconds'] = [(((x.hour * 60 + x.minute) * 60 + x.second) * 1000) for x in
                                       LOB['TimeStamp']]
                LOB['dollar_traded_volume'] = pd.Series(LOB.TradePrice * LOB.TradeVolume).fillna(0)
                LOB['timeStampIdx'] = pd.DatetimeIndex(LOB.time_y)
                LOB['micro_price'] = (LOB.BestAsk * LOB.AskSize + LOB.BestBid * LOB.BidSize) / (
                            LOB.AskSize + LOB.BidSize)  #

                mergedFile[date] = LOB

            except FileNotFoundError:
                self._logger.info(f"Data for day {date} does not exist. Skipping this day")
        return mergedFile

    def apply_micro_structure_features(self, df):

        self._logger.info("Applying micro-structure features")

        df['weighted_average_bid_price'] = pd.DataFrame(df.filter(like='bid_price', axis=1).values).mul(
            pd.DataFrame(df.filter(like='bid_size', axis=1).values)).sum(axis=1) / pd.DataFrame(
            df.filter(like='bid_size', axis=1).values).sum(axis=1)

        df['weighted_average_ask_price'] = pd.DataFrame(df.filter(like='ask_price', axis=1).values).mul(
            pd.DataFrame(df.filter(like='ask_size', axis=1).values)).sum(
            axis=1) / pd.DataFrame(df.filter(like='ask_size', axis=1).values).sum(axis=1)
        df['weighted_activity_spread'] = df['weighted_average_ask_price'] - df['weighted_average_bid_price']
        df['total_size'] = (pd.DataFrame(df.filter(like='ask_size', axis=1).values).sum(axis=1) +
                            pd.DataFrame(df.filter(like='bid_size', axis=1).values).sum(axis=1))
        df['micro_price'] = ((pd.DataFrame(df.filter(like='bid_price', axis=1).values).mul(
            pd.DataFrame(df.filter(like='bid_size', axis=1).values)).sum(axis=1) +
                              pd.DataFrame(df.filter(like='ask_price', axis=1).values).mul(
                                  pd.DataFrame(df.filter(like='ask_size', axis=1).values)).sum(axis=1))) / df[
                                'total_size']
        df['price_imbalance'] = ((pd.DataFrame(df.filter(like='ask_price', axis=1).values).mul(
            pd.DataFrame(df.filter(like='ask_size', axis=1).values)).sum(axis=1) -
                                  pd.DataFrame(df.filter(like='bid_price', axis=1).values).mul(
                                      pd.DataFrame(df.filter(like='bid_size', axis=1).values)).sum(axis=1))) / df[
                                    'total_size']
        df['pct_change_micro_price'] = df.micro_price_close.pct_change()
        df['simple_mid_price'] = 0.5 * (pd.DataFrame(df.filter(like='ask_price', axis=1)).mean(axis=1) +
                                        pd.DataFrame(df.filter(like='bid_price', axis=1)).mean(axis=1))

        # TODO: do we want to be using pct_change here or log returns?
        return df

    @staticmethod
    def prep_bars(df, drop_col):
        df.columns = [f"{i[0]}_{j}" for i, j in zip(df.columns, ['open', 'high', 'low', 'close'] * len(df.columns))]
        if drop_col:
            df.drop([f"{drop_col}_{i}" for i in ['open', 'high', 'low', 'close']], axis=1, inplace=True)
        return df

    def drop_lob_columns(df):
        # not using this anywhere at the moment

        ask_price_columns_to_be_dropped = list(pd.DataFrame(df.filter(like='ask_price_', axis=1)).columns.values)
        ask_size_columns_to_be_dropped = list(pd.DataFrame(df.filter(like='ask_size_', axis=1)).columns.values)
        bid_size_columns_to_be_dropped = list(pd.DataFrame(df.filter(like='bid_size_', axis=1)).columns.values)
        bid_price_columns_to_be_dropped = list(pd.DataFrame(df.filter(like='bid_price_', axis=1)).columns.values)
        time_stamps = ['timestamp', 'timestamp.1', 'exchange_time', 'feed_time']
        columns_to_be_dropped = ask_price_columns_to_be_dropped + ask_size_columns_to_be_dropped + \
                                bid_size_columns_to_be_dropped + bid_price_columns_to_be_dropped + \
                                time_stamps
        df = df.drop(columns=columns_to_be_dropped)

        return df

    def get_bars(self, df):

        self._logger.info('Calculating calendar clock')
        calendar_bars = df.resample(rule=self._calendar_resample, on='timeStampIdx').agg(
            ['first', 'max', 'min', 'last'])
        calendar_bars = self.prep_bars(calendar_bars, 'timeStampIdx')

        self._logger.info('Calculating volume clock')
        # TODO: Doing this on day-by-day basis. We lose the last turn of the clock each day
        df['cum_total_trade_volume'] = df.total_traded_volume.cumsum()
        df['volume_bucket'] = np.floor(df.cum_total_trade_volume / self._trade_volume_width)
        volume_bars = df.groupby(by='volume_bucket').agg(['first', 'max', 'min', 'last']).iloc[1:-1]
        volume_bars = self.prep_bars(volume_bars, 'cum_total_trade_volume')
        df.drop(['cum_total_trade_volume', 'volume_bucket'], axis=1, inplace=True)

        self._logger.info('Calculating tick clock')
        # TODO: ticks here means any change to the order book. Not necessarily a trade. Can add that clock too
        df['tick_count'] = range(len(df))
        df['tick_bucket'] = np.floor(df.tick_count / self._ticks_width)
        tick_bars = df.groupby(by='tick_bucket').agg(['first', 'max', 'min', 'last']).iloc[1:-1]
        tick_bars = self.prep_bars(tick_bars, 'tick_count')
        df.drop(['tick_count', 'tick_bucket'], axis=1, inplace=True)

        self._logger.info('Calculating usd volume clock')
        df['cum_usd_volume'] = df.dollar_traded_volume.cumsum()
        df['usd_volume_bucket'] = np.floor(df.cum_usd_volume / self._usd_volume_width)
        usd_volume_bars = df.groupby(by='usd_volume_bucket').agg(['first', 'max', 'min', 'last']).iloc[1:-1]
        usd_volume_bars = self.prep_bars(usd_volume_bars, 'cum_usd_volume')
        df.drop(['cum_usd_volume', 'usd_volume_bucket'], axis=1, inplace=True)

        self.calendar_bars.append(calendar_bars)
        self.volume_bars.append(volume_bars)
        self.tick_bars.append(tick_bars)
        self.usd_volume_bars.append(usd_volume_bars)

        self._bars_dict['calendar_bars'] = self.calendar_bars
        self._bars_dict['volume_bars'] = self.volume_bars
        self._bars_dict['tick_bars'] = self.tick_bars
        self._bars_dict['usd_volume_bars'] = self.usd_volume_bars

    def get_all_dates_bars(self, input_dict):
        output_dict = {}
        for input_date in self._input_dates:
            output_dict[input_date] = self.get_bars(input_dict[input_date])

        return output_dict

    @staticmethod
    def get_concat_data(input_dict):
        concat_dict = dict()
        for bar in input_dict.keys():
            concat_dict[bar] = pd.concat([input_dict[bar][i]
                                          for i in range(len(input_dict[bar]))], ignore_index=False)
        return concat_dict

workDriveFolder = '/media/ak/WorkDrive/Data'
laptopDataFolder  = '/media/ak/LaCie'

figures_destination = '/home/ak/Documents/Research/Papers/figures/'
dataFolder = laptopDataFolder

quotes =[f for f in os.listdir(dataFolder) if str('_quotes') in f]
trades =[f for f in os.listdir(dataFolder) if str('_trades') in f]
symbols= [f.split("_")[0] for f in quotes]
if __name__ == "__main__":
    symbolIdx = 3
    symbol = sorted(symbols)[symbolIdx]
    print(symbol)
    quotesFileCh = os.path.join(dataFolder, sorted(quotes)[symbolIdx])
    tradesFileCh = os.path.join(dataFolder, sorted(trades)[symbolIdx])

    ## get common Dates
    quotesDates = sorted([f.split(".csv")[0] for f in os.listdir(quotesFileCh)])
    tradesDates = sorted([f.split(".csv")[0] for f in os.listdir(tradesFileCh)])
    intersectionDates = list(set(quotesDates).intersection(tradesDates))

    trades_cols = ['size', 'time', 'type', 'value']

    calendar_resample = "300S"
    trade_volume_width = 100
    ticks_width = 100
    usd_volume_width = 100
    dates_choice = quotesDates[0:20]

    testClass = DataLoader(data_location=dataFolder,
                           symbol=symbol,
                           dates=dates_choice,
                           use_columns=trades_cols,
                           calendar_resample="300S",
                           trade_volume_width=trade_volume_width,
                           ticks_width=ticks_width,
                           usd_volume_width=usd_volume_width)

    ## load data ##
    input_dict = testClass.load_and_format_data()


# -*- coding: utf-8 -*-
import scrapy
from zhilian.items import ZhilianItem

class ZljobsSpider(scrapy.Spider):
    name = 'zlzp'
    allowed_domains = ['zhaopin.com']
    start_urls = [
        'http://sou.zhaopin.com/jobs/searchresult.ashx?jl=%E5%8C%97%E4%BA%AC%2B%E4%B8%8A%E6%B5%B7%2B%E5%B9%BF%E5%B7%9E%2B%E6%B7%B1%E5%9C%B3%2B%E6%AD%A6%E6%B1%89&kw=python&p=1&isadv=0',
        'http://sou.zhaopin.com/jobs/searchresult.ashx?jl=%E5%8C%97%E4%BA%AC%2B%E4%B8%8A%E6%B5%B7%2B%E5%B9%BF%E5%B7%9E%2B%E6%B7%B1%E5%9C%B3%2B%E6%AD%A6%E6%B1%89&kw=php&p=1&isadv=0',
        'http://sou.zhaopin.com/jobs/searchresult.ashx?jl=%E5%8C%97%E4%BA%AC%2B%E4%B8%8A%E6%B5%B7%2B%E5%B9%BF%E5%B7%9E%2B%E6%B7%B1%E5%9C%B3%2B%E6%AD%A6%E6%B1%89&kw=html&p=1&isadv=0'
    ]

    def parse(self, response):
        yield scrapy.Request(
            url=response.url,
            callback=self.parse_job_info,
            meta={},
            dont_filter=True,
        )

    def parse_job_info(self, response):
        """
            解析工作信息
        :param response:
        :return:
        """
        zl_table_list = response.xpath("//div[@id='newlist_list_content_table']/table[@class='newlist']")
        for zl_table in zl_table_list[1:]:
            # tbody 是网页自动生成的 运行起来看效果/或者右键查看源码
            # zl_td_list = zl_table.xpath("tr[1]/td")
            # 问题：td 数不是5个，会报错--索引越界
            # td1 = zl_table_list[0]
            # td2 = zl_table_list[1]
            # td3 = zl_table_list[2]
            # td4 = zl_table_list[3]
            # td5 = zl_table_list[4]

            # 查找元素尽量用xpath定位，少用索引，因为有可能出现索引越界错误
            # 只有在不明确错误时使用异常捕获
            # //text()获取标签内所有文本
            # extract()把列表里的元素转换成文本,本身还是列表
            # extract_first('默认值')把列表里的元素转换成文本并取出第一个，如果取不到，返回默认值
            td1 = zl_table.xpath("tr/td[@class='zwmc']/div/a//text()").extract()
            # map返回的是一个列表 td1 = list(map(str.strip, td1))
            td1 = map(str.strip, td1)
            job_name = "".join(td1).replace(",", "/")
            # strip()只能清除两端的
            fan_kui_lv = zl_table.xpath("tr/td[@class='fk_lv']/span/text()").extract_first('没有反馈率').strip()
            job_company_name = zl_table.xpath("tr/td[@class='gsmc']/a[1]/text()").extract_first('没有公司名称').strip()
            job_salary = zl_table.xpath("tr/td[@class='zwyx']/text()").extract_first('面议').strip()
            job_place = zl_table.xpath("tr/td[@class='gzdd']/text()").extract_first('没有工作地点').strip()
            print(job_name, fan_kui_lv, job_company_name, job_salary, job_place)
            item = ZhilianItem()
            item['job_name'] = job_name
            item['job_company_name'] = job_company_name
            item['job_place'] = job_place
            item['job_salary'] = job_salary
            item['job_time'] = "没有时间"
            item['job_type'] = "智联招聘"
            item['fan_kui_lv'] = fan_kui_lv
            yield item
        yield scrapy.Request(
            url=response.url,
            callback=self.parse_next_page,
            meta={},
            dont_filter=True,

        )

    def parse_next_page(self, response):
        """
            解析下一页
        :param response:
        :return:
        """
        #  //div[@class='pagesDown']/ul/li/a[text()='下一页']/@href
        next_page = response.xpath(" //a[text()='下一页']/@href").extract_first('没有下一页')
        if next_page:
            yield scrapy.Request(
                url=next_page,
                callback=self.parse_job_info,
                meta={},
                dont_filter=True,
            )