import os
import time
from collections import Counter

import numpy as np
from sklearn.preprocessing import MinMaxScaler

import core.leading_tree as lt
import core.lmca as lm
from core.delala_select import DeLaLA_select
from utils import common

def load_parameters(param_path):
    dataset = common.load_csv(param_path)
    dc = dataset[:, 0].astype(float)
    lt_num = dataset[:, 1].astype(int)
    length_scale = dataset[:, 2].astype(float)
    return dc, lt_num, length_scale

def removeMinorData(X, Labels, k):
    """
    Remove the points and labels that have no enough neighbors
    :param X:
    :param Labels:
    :param k:
    :return:
    """
    setLabels = set(Labels)
    classNum = len(setLabels)
    cntArray = np.zeros(classNum, dtype=int)
    i = 0
    rmInds = np.zeros(0, dtype=int)
    for l in setLabels:
        clCnt = list(Labels).count(l)
        cntArray[i] = clCnt
        i += 1
        if clCnt <= k:
            lbInds = [i for i in range(len(Labels)) if Labels[i] == l]
            rmInds = np.append(rmInds, lbInds)
            # print("remove class: ", l)
    if len(rmInds) > 0:
        Remove_index = X[rmInds]
        X = np.delete(X, rmInds, axis=0)
        Labels = np.delete(Labels, rmInds)
        return X, Labels, Remove_index
    else:
        return X, Labels, -1

order = 0
param_path = os.path.join("data", "parameters.csv")
dc_all, lt_num_all, length_scale_all = load_parameters(param_path)
SelectedInds_all = np.zeros(0, dtype=int)

def PredictLabel(train_AL, label_num, layer, dimension):
    global order, SelectedInds_all
    if label_num == 0:
        pass
    elif label_num == 1:  # predict that this subtree sample has the same label as its root node
        y_test = np.delete(y[train_AL], 0)  # test set labels
        y_predict = np.zeros(len(y_test), dtype=int) + y[train_AL[0]]
        y_predict_all[train_AL] = y[train_AL[0]]
        SelectedInds_all = np.append(SelectedInds_all, train_AL[0])
        return 0
    elif 2 <= label_num <= 3 or layer == 3:  # Prediction with LMCA
        D_A = D[train_AL]
        D_A = D_A[:, train_AL]
        LT = lt.LeadingTree(X_train=X[train_AL], dc=dc_all[order], lt_num=lt_num_all[order], D=D_A)
        LT.fit()
        LTgammaPara = LT.density * LT.delta
        selectedInds = DeLaLA_select(LTgammaPara, LT.density, LT.layer, y[train_AL], 2, label_num * 2, 0.5)
        selectedInds_universe = train_AL[selectedInds]
        X_train = X[selectedInds_universe]
        y_train = y[selectedInds_universe]
        lmca = lm.LMCA(dimension=dimension, init_method="kpca", verbose=True, max_iter=100, stepsize=1.E-2,
                       nn_active=False, length_scale=length_scale_all[order], k=1)
        lmca.fit(X_train, y_train)
        order += 1
        X_test = np.delete(X[train_AL], selectedInds, axis=0)  # After removing the training set, the test set samples are obtained
        y_test = np.delete(y[train_AL], selectedInds, axis=0)  # test set labels
        y_predict = np.zeros(len(y_test), dtype=int) - 1  # predict labels

        MatDist = common.euclidian_dist_square(X_train, X_test)  # The kernel matrix corresponding to the training set and the test set
        test_bnd_K = np.exp(-1 * lmca.length_scale * MatDist).T
        B = test_bnd_K.dot(lmca.Omega)  # Test set after dimensionality reduction
        A = lmca.K.dot(lmca.Omega)  # The training set after dimensionality reduction

        # Find the training sample with the closest Euclidean distance for each test sample, and predict that both have the same label.
        D_temp = common.euclidian_dist(B, A)
        Pa = np.zeros(len(y_predict), dtype=int)  # Pa[i] represents the index of the training sample closest to the test sample i
        for j in range(len(y_predict)):
            index1 = np.argmin(D_temp[j])
            Pa[j] = index1

        y_predict = y_train[Pa]
        index_predict = np.delete(train_AL, selectedInds, axis=0)
        y_predict_all[index_predict] = y_predict
        SelectedInds_all = np.append(SelectedInds_all, selectedInds_universe)
        return 0
    else:
        return 1

if __name__ == "__main__":
    dataset_path = os.path.join("data", "letter.csv")
    X, y = common.load_data(dataset_path, label_index=0, map_label=False)

    t1 = time.time()
    scalar = MinMaxScaler()
    X = scalar.fit_transform(X)
    D = common.euclidian_dist(X, X)

    remove_index_all = np.zeros(0, dtype=int)
    y_predict_all = np.zeros(len(y), dtype=int) - 1

    dc_lt_num_arr = [[0.12, 45, 1, 2], [0.19, 10, 2, 2], [0.12, 5, 3, 8]]

    def recursive_partitioning(_X, _D, _layer=0, _train_AL=None):
        global remove_index_all
        lt1 = lt.LeadingTree(X_train=_X, dc=dc_lt_num_arr[_layer][0],
                             lt_num=dc_lt_num_arr[_layer][1], D=_D)  # Constructing the lead tree for the entire dataset
        lt1.fit()
        for i in range(dc_lt_num_arr[_layer][1]):
            if _layer == 0:
                _train_AL2, _y_AL, _remove_index = removeMinorData(lt1.AL[i], y[lt1.AL[i]], 2)
            else:
                _train_AL2, _y_AL, _remove_index = removeMinorData(_train_AL[lt1.AL[i]], y[_train_AL[lt1.AL[i]]], 2)

            _label_num = len(np.unique(y[_train_AL2]))
            remove_index_all = np.append(remove_index_all, _remove_index)
            _a = PredictLabel(_train_AL2, _label_num, dc_lt_num_arr[_layer][2], dc_lt_num_arr[_layer][3])

            if _a == 1:
                print(f"The {_layer}th layer case3: The number of subtree {i} categories is {_label_num}, which needs to be divided again.")
                _D_2 = D[_train_AL2]
                _D_2 = _D_2[:, _train_AL2]
                recursive_partitioning(X[_train_AL2], _D_2, _layer + 1, _train_AL=_train_AL2)
        return lt1

    lt0 = recursive_partitioning(X, D)

    index_None = np.zeros(0, dtype=int)  # Returns -1 if there are no indexes that need to be removed, here they are to be removed
    for i in range(len(remove_index_all)):
        if remove_index_all[i] == -1:
            index_None = np.append(index_None, i)

    remove_index_all = np.delete(remove_index_all, index_None)
    y_remove_Select = np.delete(y_predict_all, np.append(SelectedInds_all, remove_index_all))
    D = common.euclidian_dist(X[remove_index_all], X[SelectedInds_all])
    Pa = np.zeros(len(remove_index_all), dtype=int)
    for i in range(len(remove_index_all)):
        index1 = np.argmin(D[i])
        Pa[i] = index1

    y_predict_all[remove_index_all] = y[SelectedInds_all][Pa]
    arr1 = y[remove_index_all] - y_predict_all[remove_index_all]
    count0 = Counter(arr1)[0]
    print(f'The accuracy of the remove sample prediction is {count0 / len(arr1)}, {count0}/{len(arr1)}')

    # Subtree accuracy
    for i in range(45):
        temp = np.setdiff1d(lt0.AL[i], SelectedInds_all)
        arr2 = y[temp] - y_predict_all[temp]
        count1 = Counter(arr2)[0]
        print(f'The accuracy of subtree {i} is {count1 / len(temp)}, {count1}/{len(temp)}')
    # Overall accuracy
    y_predict_all = np.delete(y_predict_all, SelectedInds_all)
    y_test_all = np.delete(y, SelectedInds_all)
    arr = y_test_all - y_predict_all
    count = Counter(arr)[0]

    t2 = time.time()
    print(f'A total of {len(SelectedInds_all)} points are selected, with an accuracy of {count / len(y_test_all)}, {count}/{len(y_test_all)}')
    print(f'Takes {t2 - t1} seconds')


from defines import NUMBER_OF_RESULTS_FOR_QUERY
from logging_messages import log_debug_message
from memories_querying import (
    retrieve_description_from_scored_results_entry,
    search_memories,
)
from vector_storage import process_raw_data
from wrappers import validate_agent_type

def generate_summary_description_segment(
    agent, current_timestamp, query, prompt, memories_raw_data, index
):
    """Genererates a segment of a character summary

    Args:
        agent (Agent): the agent to whom the summary description corresponds
        current_timestamp (datetime): the current timestamp
        query (str): the query that will be made to the agent's memories
        prompt_header (str): the header for the prompt
        memories_raw_data (dict): the raw data of the agent's memories
        index (AnnoyIndex): the index of the Annoy library

    Returns:
        str: the response from the AI model
    """
    scored_results = search_memories(
        agent,
        current_timestamp,
        process_raw_data(query),
        NUMBER_OF_RESULTS_FOR_QUERY,
        memories_raw_data,
        index,
    )

    for _, vector_id in enumerate(scored_results):
        prompt += "- " + retrieve_description_from_scored_results_entry(
            vector_id, memories_raw_data
        )

    log_debug_message(f"{prompt}")

    return agent.get_request_response_function()(prompt)

@validate_agent_type
def request_character_summary(agent, current_timestamp, memories_raw_data, index):
    """Produces the agent's character summary through the AI model

    Args:
        agent (Agent): the agent to whom the memories belong
        current_timestamp (datetime): the current timestamp
        memories_raw_data (dict): the raw data of the agent's memories
        index (AnnoyIndex): the index of the Annoy database

    Returns:
        str: the generated summary description for the agent involved
    """
    # First of all we perform a retrieval on the query "[name]'s core characteristics"
    prompt = f"How would one describe {agent.name}'s core characteristics given the following statements? "
    prompt += f"Start the sentence by saying either '{agent.name} is' or '{agent.name} has':\n"
    core_characteristics = generate_summary_description_segment(
        agent,
        current_timestamp,
        f"{agent.name}'s core characteristics",
        prompt,
        memories_raw_data,
        index,
    )

    prompt = f"How would one describe {agent.name}'s current daily occupation given the following statements? "
    prompt += f"Start the sentence by saying either '{agent.name} is' or '{agent.name} has':\n"

    current_daily_occupation = generate_summary_description_segment(
        agent,
        current_timestamp,
        f"{agent.name}'s current daily occupation",
        prompt,
        memories_raw_data,
        index,
    )

    prompt = f"How would one describe {agent.name}'s feeling about his recent progress in life given the "
    prompt += f"following statements? Start the sentence by saying either '{agent.name} is' or '{agent.name} has':\n"

    recent_progress_in_life = generate_summary_description_segment(
        agent,
        current_timestamp,
        f"{agent.name}'s feeling about his recent progress in life",
        prompt,
        memories_raw_data,
        index,
    )

    innate_traits = generate_summary_description_segment(
        agent,
        current_timestamp,
        f"{agent.name}'s innate traits",
        f"How would one describe {agent.name}'s innate traits given the following statements? Write it solely as a series of adjectives:\n",
        memories_raw_data,
        index,
    )

    # careful with these. Try to close index always when you're not using it.
    index.unload()

    summary_description = f"Name: {agent.name} (age: {agent.age})\n"
    summary_description += f"Innate traits: {innate_traits}\n"
    summary_description += (
        f"{core_characteristics}\n{current_daily_occupation}\n{recent_progress_in_life}"
    )

    log_debug_message(
        f"Function {request_character_summary.__name__}:\n{summary_description}"
    )

    return summary_description


import os
from trainingData import get_filenames

def stats_main():
    f = open("dataList.csv", "w")
    f.write("sample, file, supop\n")
    samples, all_files = get_filenames()
    for i in range(len(samples)):
        sample = samples[i]
        for file in all_files[i]:
            text = writeData(sample, file)
            f.write(text)
    f.close()

def writeData(sample, file):
    text = sample + ", " + file + ", " + get_y_sample(sample) + "\n"
    return text

def get_y_sample(sample):
    """This function returns the superpopulation group of a sample
    """
    f = open("./samples_population.csv")
    f.readline()
    while True:
        row = f.readline()
        if len(row) == 0:
            break
        (samp, gen, pop, supop) = row.split(',')
        if samp == sample:
            sp = supop.strip()
            break
    f.close()
    try:
        sp
    except:
        return "unknown"
    return sp

stats_main()


import json
from sksurv.functions import StepFunction
from sksurv.linear_model import CoxPHSurvivalAnalysis
from sksurv.metrics import concordance_index_censored
from sksurv.nonparametric import nelson_aalen_estimator, kaplan_meier_estimator

from core.cox_wrapper import CoxFairBaseline
from core.drawing import draw_points_tsne
import numpy as np
import pandas as pd
from sklearn.model_selection import train_test_split
from sksurv.ensemble import RandomSurvivalForest

from exp_config import CONFIG, RES_DIR
from core.cox_generator import CoxGenerator
from survshap import SurvivalModelExplainer, ModelSurvSHAP
from survbex.estimators import BeranModel
from survbex.explainers import SurvBexExplainer

########################################################################################################################
# ------------------------------------------------ PREPARE DATA --------------------------------------------------------
########################################################################################################################
def get_cox_data(coefs: np.ndarray):
    cox_generator = CoxGenerator(coefs=coefs)
    x_cox_train, x_cox_test, y_cox_train, y_cox_test = train_test_split(
        *cox_generator.generate_data(size=CONFIG['TRAIN_SIZE'], censored_part=0.2),
        train_size=0.7
    )

    x_cox_train = pd.DataFrame(x_cox_train, columns=[f'f{i + 1}' for i in range(len(coefs))])
    x_cox_test = pd.DataFrame(x_cox_test, columns=[f'f{i + 1}' for i in range(len(coefs))])

    return [x_cox_train, y_cox_train], [x_cox_test, y_cox_test]

# np.random.seed(42)
# train, test = get_veterans_data()
cox_clusters = [get_cox_data(coefs=cox_coefs) for cox_coefs in CONFIG['COX_COEFS_CLS']]

cox_clusters = [
    (
        [cox_cluster[0][0] + 2.0 / len(cox_clusters) * cl_i, cox_cluster[0][1]],
        [cox_cluster[1][0] + 2.0 / len(cox_clusters) * cl_i, cox_cluster[1][1]]
        # [cox_cluster[0][0] + 1. * cl_i, cox_cluster[0][1]],
        # [cox_cluster[1][0] + 1. * cl_i, cox_cluster[1][1]]
    )
    for cl_i, cox_cluster in enumerate(cox_clusters)
]

all_train = [
    pd.concat([cox_cluster[0][0] for cox_cluster in cox_clusters]),
    np.hstack([cox_cluster[0][1] for cox_cluster in cox_clusters])
]

all_test = [
    pd.concat([cox_cluster[1][0] for cox_cluster in cox_clusters]),
    np.hstack([cox_cluster[1][1] for cox_cluster in cox_clusters])
]

# Use SurvLimeExplainer class to find the feature importance
training_events = np.array([event for event, _ in all_train[1]])
training_times = np.array([time for _, time in all_train[1]])
training_features = all_train[0]

test_events = np.array([event for event, _ in all_test[1]])
test_times = np.array([time for _, time in all_test[1]])
test_features = all_test[0]

with open(f'{RES_DIR}/dataset.json', 'w+') as fp:
    json.dump(fp=fp, obj=dict(
        training_features=training_features.to_dict(orient='raw'),
        training_events=training_events.tolist(),
        training_times=training_times.tolist(),
        test_features=test_features.to_dict(orient='raw'),
        test_events=test_events.tolist(),
        test_times=test_times.tolist()
    ))

########################################################################################################################
# ------------------------------------------------ BUILD BBOX ----------------------------------------------------------
########################################################################################################################
if CONFIG['BBOX'] == 'rf':
    model = RandomSurvivalForest(n_estimators=100, max_samples=min(500, len(all_train[0])), max_depth=8)
    model.fit(all_train[0], all_train[1])
    pred_surv_fn = model.predict_survival_function
    pred_hazard_fn = model.predict_cumulative_hazard_function
    pred_risk_fn = model.predict
elif CONFIG['BBOX'] == 'beran':
    assert len(CONFIG['COX_COEFS_CLS']) == 1
    model = BeranModel(kernel_width=250, kernel_name='gaussian')
    model.fit(X=all_train[0].to_numpy(), b=CONFIG['COX_COEFS_CLS'][0],
              y_events=training_events, y_event_times=training_times)

    def surv_np_to_step_surv(surv_arr: np.ndarray):
        return np.array([StepFunction(x=model.unique_times_, y=sample) for sample in surv_arr])

    pred_surv_fn = lambda X: surv_np_to_step_surv(model.predict_survival_torch_optimized(X))
    pred_hazard_fn = lambda X: -np.log(model.predict_survival_torch_optimized(X))
    pred_risk_fn = lambda X: np.sum(pred_hazard_fn(X), axis=1)

elif 'cox' in CONFIG['BBOX']:
    model = CoxPHSurvivalAnalysis(alpha=1)

    model.fit(all_train[0], all_train[1])
    pred_surv_fn = model.predict_survival_function
    pred_hazard_fn = model.predict_cumulative_hazard_function
    pred_risk_fn = model.predict

    if CONFIG['BBOX'] in ['cox_na', 'cox_km']:
        if CONFIG['BBOX'] == 'cox_na':
            cox_fair_baseline = CoxFairBaseline(
                training_events=training_events,
                training_times=training_times,
                baseline_estimator_f=nelson_aalen_estimator
            )
        elif CONFIG['BBOX'] == 'cox_km':
            cox_fair_baseline = CoxFairBaseline(
                training_events=training_events,
                training_times=training_times,
                baseline_estimator_f=kaplan_meier_estimator
            )
        else:
            raise Exception(f'Undefined cox model = {CONFIG["BBOX"]}')

        model.coef_ /= np.abs(model.coef_).sum()
        pred_surv_fn = lambda X: cox_fair_baseline.predict_survival_function(X, cox_coefs=model.coef_)
        pred_hazard_fn = lambda X: cox_fair_baseline.predict_cum_hazard_from_surv_np(X, cox_coefs=model.coef_)
        pred_risk_fn = lambda X: np.dot(X, model.coef_)
    elif CONFIG['BBOX'] != 'cox':
        raise Exception(f'Undefined cox model = {CONFIG["BBOX"]}')
else:
    raise Exception(f"Undefined bbox = {CONFIG['BBOX']}")

cindex_train = concordance_index_censored(
    event_indicator=training_events, event_time=training_times, estimate=pred_risk_fn(training_features))[0]
print(f'cindex train = {cindex_train}')
cindex_test = concordance_index_censored(
    event_indicator=test_events, event_time=test_times, estimate=pred_risk_fn(test_features))[0]
print(f'cindex test = {cindex_test}')

########################################################################################################################
# ------------------------------------------------ SELECT POINTS TO EXPLAIN --------------------------------------------
########################################################################################################################
# draw_comparison(ex_i=random.randint(0, len(test)))
cluster_centroids = [
    cox_cluster[0][0].mean() + all_test[0].std() * CONFIG['DATA_POINT_DEV']
    for cox_cluster in cox_clusters
]

cl_distances = [
    [sum((cl_centroid - fs) ** 2) for fs in all_test[0].to_numpy()]
    for cl_centroid in cluster_centroids
]

exp_test_ids = [np.argmin(distances) for distances in cl_distances]

draw_points_tsne(
    pt_groups=[
        *[cox_cluster[0][0].to_numpy() for cox_cluster in cox_clusters],
        *list(all_test[0].to_numpy()[exp_test_ids])
    ],
    names=[
        *[f'cl{i}' for i, _ in enumerate(cox_clusters)],
        *[f'ex for cl {i}' for i, _ in enumerate(exp_test_ids)]
    ],
    colors=[None] * len(cox_clusters) * 2,
    path=f'{RES_DIR}/clusters.png'
    # path=f'clusters.png'
)

with open(RES_DIR.joinpath("y_true.json"), 'w+') as fp:
    json.dump(
        fp=fp,
        obj=[
            dict(event=bool(all_test[1][ex_i][0]), event_time=all_test[1][ex_i][1])
            for ex_i in exp_test_ids
        ]
    )

########################################################################################################################
# ------------------------------------------------ SurvSHAP ------------------------------------------------------------
########################################################################################################################
surv_shap = SurvivalModelExplainer(model, all_test[0].iloc[exp_test_ids], all_test[1][exp_test_ids],
                                   predict_survival_function=lambda model, X: pred_surv_fn(X))

exp_survshap = ModelSurvSHAP(random_state=42)
exp_survshap.fit(surv_shap)
shap_explanations = np.array(
    [
        [
            imp[1]
            for imp in pt_exp.simplified_result.values
        ]
        for pt_exp in exp_survshap.individual_explanations

    ]
)

with open(RES_DIR.joinpath("explanation_shap.json"), 'w+') as fp:
    json.dump(fp=fp, obj=shap_explanations.tolist())

########################################################################################################################
# ------------------------------------------------ SurvLIME ------------------------------------------------------------
########################################################################################################################
explainer = SurvBexExplainer(
    training_features=training_features,
    training_events=list(training_events),
    training_times=list(training_times),
    model_output_times=model.event_times_,
    kernel_width=CONFIG['KERNEL_WIDTH']
)

cox_explanations = np.array(
    [
        explainer.explain_instance(
            data_row=all_test[0].iloc[ex_i],
            predict_fn=pred_surv_fn,
            num_samples=CONFIG['NEIGH_SIZE'],
            type_fn='survival',
            optimizer='convex'
        )
        for ex_i in exp_test_ids
    ]
)

with open(RES_DIR.joinpath("explanation_cox.json"), 'w+') as fp:
    json.dump(fp=fp, obj=cox_explanations.tolist())

########################################################################################################################
# ------------------------------------------------ SurvBeX -------------------------------------------------------------
########################################################################################################################
beran_explanations = []
for cl_i, ex_i in enumerate(exp_test_ids):
    beran_explanations.append(
        explainer.explain_instance(
            data_row=all_test[0].iloc[ex_i],
            predict_fn=pred_surv_fn,
            num_samples=CONFIG['NEIGH_SIZE'],
            num_val_samples=CONFIG['NEIGH_VAL_SIZE'],
            type_fn='survival',
            optimizer='gradient',
            grid_info_file=f"{RES_DIR}/optimization_cl={cl_i}.csv",
            max_iter=CONFIG['MAX_ITER']

        )
    )

with open(RES_DIR.joinpath("explanation_beran.json"), 'w+') as fp:
    json.dump(
        fp=fp,
        obj=np.array(beran_explanations).tolist()
    )


# -*- coding: utf-8 -*-
from __future__ import absolute_import, division, print_function, unicode_literals

import logging

from temba_client.v2 import TembaClient

from django.conf import settings
from django.db import migrations

from ureport.utils import datetime_to_json_date, json_date_to_datetime

logger = logging.getLogger(__name__)

class Migration(migrations.Migration):
    def populate_poll_poll_date(apps, schema_editor):
        Poll = apps.get_model("polls", "Poll")
        Org = apps.get_model("orgs", "Org")

        agent = getattr(settings, "SITE_API_USER_AGENT", None)
        host = settings.SITE_API_HOST

        for org in Org.objects.all():
            temba_client = TembaClient(host, org.api_token, user_agent=agent)
            api_flows = temba_client.get_flows()
            flows_date = dict()
            for flow in api_flows:
                flows_date[flow.uuid] = datetime_to_json_date(flow.created_on)

            for poll in Poll.objects.filter(org=org):
                json_date = flows_date.get(poll.flow_uuid, None)
                if json_date:
                    date = json_date_to_datetime(json_date)
                else:
                    logger.info("using created_on for flow_date on poll with id %s" % poll.pk)
                    date = poll.created_on

                poll.poll_date = date
                poll.save()

    dependencies = [("polls", "0022_poll_flow_date")]

    operations = [migrations.RunPython(populate_poll_poll_date)]


# -*- coding: utf-8 -*-
# @Time    : 19-1-24 下午9:35
# @Author  : ccs
import json

from django.http import HttpResponse

def calc(request):
    a = request.GET['a']
    b = request.GET['b']
    c = request.GET['c']
    print(a,b,c)
    m = a+b+c
    n = b+a
    rets = {"m":m,"n":n}
    retsj = json.dumps(rets)
    return HttpResponse(retsj)


from firebase_admin import firestore
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.metrics.pairwise import cosine_similarity

def get_user_skills(userid):
    item = firestore.client().collection('user').document(userid).get().to_dict()['skills']
    user_skill_string = ' '.join(str(e) for e in item)
    return user_skill_string

def get_result(field, user_skill_string, keyword_string):
    content = (user_skill_string, keyword_string)
    cv = CountVectorizer()
    matrix = cv.fit_transform(content)
    singularity_matrix = cosine_similarity(matrix)
    result = (singularity_matrix[1][0]*100)
    result = str(result)
    result = result.split('.', 1)[0]
    # return 'You scored {}% in {} jobs.'.format(result, field)
    return result

def check_result(field, result):
    return 'You scored {}% in {} jobs.'.format(result, field)


dic={
    'nile':'egypt',
    'yellow river':'shandong',
    'yangtze river':'shanghai',
    }
for river,province in dic.items():
    print(f"The {river} runs through {province}")
for river in sorted(dic.keys()):
    print(f"{river}")
for province in sorted(dic.values()):
    print(f"{province}")


def countdown(i: int) -> int:
    while i >= 0:
        yield i
        i -= 1

def countup(i: int) -> int:
    f = 0
    while f < i:
        yield f
        f += 1

def range_countup(start: int, end: int) -> int:
    while start < end:
        yield start
        start += 1

def trim(data: list) -> list:
    data_length = len(data)
    i = 0
    not_wanted_chars = ['\t', ' ', '\r']  # \n and \U+000B been tried
    while i < data_length:
        length = len(data[i])
        if length == 0:
            i += 1
            continue
        if data[i][0] in not_wanted_chars:
            not_wanted_char_count = 1
            x = 1
            while x < length:
                if data[i][x] in not_wanted_chars:
                    not_wanted_char_count += 1
                else:
                    break
                x += 1
            if not_wanted_char_count == length:
                data.pop(i)
                data_length -= 1
                continue
        if data[i][-1] == '\r':
            data[i] = data[i][:-1]
        i += 1
    return data

def remove_spaces(a_string: str) -> str:
    length = len(a_string)
    while length > 1 and a_string[0] == ' ':
        a_string = a_string[1:]
        length -= 1
    while length > 1 and a_string[-1] == ' ':
        a_string = a_string[:-1]
        length -= 1
    return a_string

def folder_name_char_check(a_string: str) -> bool:
    non_valid_chars = '\\/?%*:|"<>.'
    for i in a_string:
        if i in non_valid_chars:
            print("{} Contains non valid character(s).\nFile and Folder names must not contain any of these characters {}.".format(a_string, non_valid_chars))
            return False
    return True


###IMPORT NECESSARY MODULES TO RUN BIOGEME
from biogeme import *
from headers import *
from loglikelihood import *
from statistics import *

### Variables

# Piecewise linear definition of income
ScaledIncome = DefineVariable('ScaledIncome',\
                  CalculatedIncome / 1000)
ContIncome_0_4000 = DefineVariable('ContIncome_0_4000',\
                  min(ScaledIncome,4))
ContIncome_4000_6000 = DefineVariable('ContIncome_4000_6000',\
                  max(0,min(ScaledIncome-4,2)))
ContIncome_6000_8000 = DefineVariable('ContIncome_6000_8000',\
                  max(0,min(ScaledIncome-6,2)))
ContIncome_8000_10000 = DefineVariable('ContIncome_8000_10000',\
                  max(0,min(ScaledIncome-8,2)))
ContIncome_10000_more = DefineVariable('ContIncome_10000_more',\
                  max(0,ScaledIncome-10))

age_65_more = DefineVariable('age_65_more',age >= 65)
moreThanOneCar = DefineVariable('moreThanOneCar',NbCar > 1)
moreThanOneBike = DefineVariable('moreThanOneBike',NbBicy > 1)
individualHouse = DefineVariable('individualHouse',\
                                 HouseType == 1)
male = DefineVariable('male',Gender == 1)
haveChildren = DefineVariable('haveChildren',\
      ((FamilSitu == 3)+(FamilSitu == 4)) > 0)
haveGA = DefineVariable('haveGA',GenAbST == 1)
highEducation = DefineVariable('highEducation', Education >= 6)

### Coefficients
coef_intercept = Beta('coef_intercept',0.398165,-1000,1000,0 )
coef_age_65_more = Beta('coef_age_65_more',0.0716533,-1000,1000,0 )
coef_haveGA = Beta('coef_haveGA',-0.578005,-1000,1000,0 )
coef_ContIncome_0_4000 = \
 Beta('coef_ContIncome_0_4000',0.0902761,-1000,1000,0 )
coef_ContIncome_4000_6000 = \
 Beta('coef_ContIncome_4000_6000',-0.221283,-1000,1000,0 )
coef_ContIncome_6000_8000 = \
 Beta('coef_ContIncome_6000_8000',0.259466,-1000,1000,0 )
coef_ContIncome_8000_10000 = \
 Beta('coef_ContIncome_8000_10000',-0.523049,-1000,1000,0 )
coef_ContIncome_10000_more = \
 Beta('coef_ContIncome_10000_more',0.084351,-1000,1000,0 )
coef_moreThanOneCar = \
 Beta('coef_moreThanOneCar',0.53301,-1000,1000,0 )
coef_moreThanOneBike = \
 Beta('coef_moreThanOneBike',-0.277122,-1000,1000,0 )
coef_individualHouse = \
 Beta('coef_individualHouse',-0.0885649,-1000,1000,0 )
coef_male = Beta('coef_male',0.0663476,-1000,1000,0 )
coef_haveChildren = Beta('coef_haveChildren',-0.0376042,-1000,1000,0 )
coef_highEducation = Beta('coef_highEducation',-0.246687,-1000,1000,0 )

### Latent variable: structural equation

# Note that the expression must be on a single line. In order to
# write it across several lines, each line must terminate with
# the \ symbol

CARLOVERS = \
coef_intercept +\
coef_age_65_more * age_65_more +\
coef_ContIncome_0_4000 * ContIncome_0_4000 +\
coef_ContIncome_4000_6000 * ContIncome_4000_6000 +\
coef_ContIncome_6000_8000 * ContIncome_6000_8000 +\
coef_ContIncome_8000_10000 * ContIncome_8000_10000 +\
coef_ContIncome_10000_more * ContIncome_10000_more +\
coef_moreThanOneCar * moreThanOneCar +\
coef_moreThanOneBike * moreThanOneBike +\
coef_individualHouse * individualHouse +\
coef_male * male +\
coef_haveChildren * haveChildren +\
coef_haveGA * haveGA +\
coef_highEducation * highEducation

### Measurement equations

INTER_Envir01 = Beta('INTER_Envir01',0,-10000,10000,1)
INTER_Envir02 = Beta('INTER_Envir02',0.348654,-10000,10000,0 )
INTER_Envir03 = Beta('INTER_Envir03',-0.309023,-10000,10000,0 )
INTER_Mobil11 = Beta('INTER_Mobil11',0.337726,-10000,10000,0 )
INTER_Mobil14 = Beta('INTER_Mobil14',-0.130563,-10000,10000,0 )
INTER_Mobil16 = Beta('INTER_Mobil16',0.128293,-10000,10000,0 )
INTER_Mobil17 = Beta('INTER_Mobil17',0.145876,-10000,10000,0 )

B_Envir01_F1 = Beta('B_Envir01_F1',-1,-10000,10000,1)
B_Envir02_F1 = Beta('B_Envir02_F1',-0.431461,-10000,10000,0 )
B_Envir03_F1 = Beta('B_Envir03_F1',0.565903,-10000,10000,0 )
B_Mobil11_F1 = Beta('B_Mobil11_F1',0.483958,-10000,10000,0 )
B_Mobil14_F1 = Beta('B_Mobil14_F1',0.58221,-10000,10000,0 )
B_Mobil16_F1 = Beta('B_Mobil16_F1',0.463139,-10000,10000,0 )
B_Mobil17_F1 = Beta('B_Mobil17_F1',0.368257,-10000,10000,0 )

MODEL_Envir01 = INTER_Envir01 + B_Envir01_F1 * CARLOVERS
MODEL_Envir02 = INTER_Envir02 + B_Envir02_F1 * CARLOVERS
MODEL_Envir03 = INTER_Envir03 + B_Envir03_F1 * CARLOVERS
MODEL_Mobil11 = INTER_Mobil11 + B_Mobil11_F1 * CARLOVERS
MODEL_Mobil14 = INTER_Mobil14 + B_Mobil14_F1 * CARLOVERS
MODEL_Mobil16 = INTER_Mobil16 + B_Mobil16_F1 * CARLOVERS
MODEL_Mobil17 = INTER_Mobil17 + B_Mobil17_F1 * CARLOVERS

SIGMA_STAR_Envir01 = Beta('SIGMA_STAR_Envir01',1,-10000,10000,1)
SIGMA_STAR_Envir02 = Beta('SIGMA_STAR_Envir02',0.767063,-10000,10000,0 )
SIGMA_STAR_Envir03 = Beta('SIGMA_STAR_Envir03',0.717835,-10000,10000,0 )
SIGMA_STAR_Mobil11 = Beta('SIGMA_STAR_Mobil11',0.783358,-10000,10000,0 )
SIGMA_STAR_Mobil14 = Beta('SIGMA_STAR_Mobil14',0.688264,-10000,10000,0 )
SIGMA_STAR_Mobil16 = Beta('SIGMA_STAR_Mobil16',0.754419,-10000,10000,0 )
SIGMA_STAR_Mobil17 = Beta('SIGMA_STAR_Mobil17',0.760104,-10000,10000,0 )

delta_1 = Beta('delta_1',0.251983,0,10,0 )
delta_2 = Beta('delta_2',0.759208,0,10,0 )
tau_1 = -delta_1 - delta_2
tau_2 = -delta_1
tau_3 = delta_1
tau_4 = delta_1 + delta_2

Envir01_tau_1 = (tau_1-MODEL_Envir01) / SIGMA_STAR_Envir01
Envir01_tau_2 = (tau_2-MODEL_Envir01) / SIGMA_STAR_Envir01
Envir01_tau_3 = (tau_3-MODEL_Envir01) / SIGMA_STAR_Envir01
Envir01_tau_4 = (tau_4-MODEL_Envir01) / SIGMA_STAR_Envir01
IndEnvir01 = {
    1: bioNormalCdf(Envir01_tau_1),
    2: bioNormalCdf(Envir01_tau_2)-bioNormalCdf(Envir01_tau_1),
    3: bioNormalCdf(Envir01_tau_3)-bioNormalCdf(Envir01_tau_2),
    4: bioNormalCdf(Envir01_tau_4)-bioNormalCdf(Envir01_tau_3),
    5: 1-bioNormalCdf(Envir01_tau_4),
    6: 1.0,
    -1: 1.0,
    -2: 1.0
}

P_Envir01 = Elem(IndEnvir01, Envir01)

Envir02_tau_1 = (tau_1-MODEL_Envir02) / SIGMA_STAR_Envir02
Envir02_tau_2 = (tau_2-MODEL_Envir02) / SIGMA_STAR_Envir02
Envir02_tau_3 = (tau_3-MODEL_Envir02) / SIGMA_STAR_Envir02
Envir02_tau_4 = (tau_4-MODEL_Envir02) / SIGMA_STAR_Envir02
IndEnvir02 = {
    1: bioNormalCdf(Envir02_tau_1),
    2: bioNormalCdf(Envir02_tau_2)-bioNormalCdf(Envir02_tau_1),
    3: bioNormalCdf(Envir02_tau_3)-bioNormalCdf(Envir02_tau_2),
    4: bioNormalCdf(Envir02_tau_4)-bioNormalCdf(Envir02_tau_3),
    5: 1-bioNormalCdf(Envir02_tau_4),
    6: 1.0,
    -1: 1.0,
    -2: 1.0
}

P_Envir02 = Elem(IndEnvir02, Envir02)

Envir03_tau_1 = (tau_1-MODEL_Envir03) / SIGMA_STAR_Envir03
Envir03_tau_2 = (tau_2-MODEL_Envir03) / SIGMA_STAR_Envir03
Envir03_tau_3 = (tau_3-MODEL_Envir03) / SIGMA_STAR_Envir03
Envir03_tau_4 = (tau_4-MODEL_Envir03) / SIGMA_STAR_Envir03
IndEnvir03 = {
    1: bioNormalCdf(Envir03_tau_1),
    2: bioNormalCdf(Envir03_tau_2)-bioNormalCdf(Envir03_tau_1),
    3: bioNormalCdf(Envir03_tau_3)-bioNormalCdf(Envir03_tau_2),
    4: bioNormalCdf(Envir03_tau_4)-bioNormalCdf(Envir03_tau_3),
    5: 1-bioNormalCdf(Envir03_tau_4),
    6: 1.0,
    -1: 1.0,
    -2: 1.0
}

P_Envir03 = Elem(IndEnvir03, Envir03)

Mobil11_tau_1 = (tau_1-MODEL_Mobil11) / SIGMA_STAR_Mobil11
Mobil11_tau_2 = (tau_2-MODEL_Mobil11) / SIGMA_STAR_Mobil11
Mobil11_tau_3 = (tau_3-MODEL_Mobil11) / SIGMA_STAR_Mobil11
Mobil11_tau_4 = (tau_4-MODEL_Mobil11) / SIGMA_STAR_Mobil11
IndMobil11 = {
    1: bioNormalCdf(Mobil11_tau_1),
    2: bioNormalCdf(Mobil11_tau_2)-bioNormalCdf(Mobil11_tau_1),
    3: bioNormalCdf(Mobil11_tau_3)-bioNormalCdf(Mobil11_tau_2),
    4: bioNormalCdf(Mobil11_tau_4)-bioNormalCdf(Mobil11_tau_3),
    5: 1-bioNormalCdf(Mobil11_tau_4),
    6: 1.0,
    -1: 1.0,
    -2: 1.0
}

P_Mobil11 = Elem(IndMobil11, Mobil11)

Mobil14_tau_1 = (tau_1-MODEL_Mobil14) / SIGMA_STAR_Mobil14
Mobil14_tau_2 = (tau_2-MODEL_Mobil14) / SIGMA_STAR_Mobil14
Mobil14_tau_3 = (tau_3-MODEL_Mobil14) / SIGMA_STAR_Mobil14
Mobil14_tau_4 = (tau_4-MODEL_Mobil14) / SIGMA_STAR_Mobil14
IndMobil14 = {
    1: bioNormalCdf(Mobil14_tau_1),
    2: bioNormalCdf(Mobil14_tau_2)-bioNormalCdf(Mobil14_tau_1),
    3: bioNormalCdf(Mobil14_tau_3)-bioNormalCdf(Mobil14_tau_2),
    4: bioNormalCdf(Mobil14_tau_4)-bioNormalCdf(Mobil14_tau_3),
    5: 1-bioNormalCdf(Mobil14_tau_4),
    6: 1.0,
    -1: 1.0,
    -2: 1.0
}

P_Mobil14 = Elem(IndMobil14, Mobil14)

Mobil16_tau_1 = (tau_1-MODEL_Mobil16) / SIGMA_STAR_Mobil16
Mobil16_tau_2 = (tau_2-MODEL_Mobil16) / SIGMA_STAR_Mobil16
Mobil16_tau_3 = (tau_3-MODEL_Mobil16) / SIGMA_STAR_Mobil16
Mobil16_tau_4 = (tau_4-MODEL_Mobil16) / SIGMA_STAR_Mobil16
IndMobil16 = {
    1: bioNormalCdf(Mobil16_tau_1),
    2: bioNormalCdf(Mobil16_tau_2)-bioNormalCdf(Mobil16_tau_1),
    3: bioNormalCdf(Mobil16_tau_3)-bioNormalCdf(Mobil16_tau_2),
    4: bioNormalCdf(Mobil16_tau_4)-bioNormalCdf(Mobil16_tau_3),
    5: 1-bioNormalCdf(Mobil16_tau_4),
    6: 1.0,
    -1: 1.0,
    -2: 1.0
}

P_Mobil16 = Elem(IndMobil16, Mobil16)

Mobil17_tau_1 = (tau_1-MODEL_Mobil17) / SIGMA_STAR_Mobil17
Mobil17_tau_2 = (tau_2-MODEL_Mobil17) / SIGMA_STAR_Mobil17
Mobil17_tau_3 = (tau_3-MODEL_Mobil17) / SIGMA_STAR_Mobil17
Mobil17_tau_4 = (tau_4-MODEL_Mobil17) / SIGMA_STAR_Mobil17
IndMobil17 = {
    1: bioNormalCdf(Mobil17_tau_1),
    2: bioNormalCdf(Mobil17_tau_2)-bioNormalCdf(Mobil17_tau_1),
    3: bioNormalCdf(Mobil17_tau_3)-bioNormalCdf(Mobil17_tau_2),
    4: bioNormalCdf(Mobil17_tau_4)-bioNormalCdf(Mobil17_tau_3),
    5: 1-bioNormalCdf(Mobil17_tau_4),
    6: 1.0,
    -1: 1.0,
    -2: 1.0
}

P_Mobil17 = Elem(IndMobil17, Mobil17)

loglike = log(P_Envir01) + \
          log(P_Envir02) + \
          log(P_Envir03) + \
          log(P_Mobil11) + \
          log(P_Mobil14) + \
          log(P_Mobil16) + \
          log(P_Mobil17)

BIOGEME_OBJECT.EXCLUDE =  (Choice   ==  -1 )

# Defines an iterator on the data
rowIterator('obsIter')

BIOGEME_OBJECT.ESTIMATE = Sum(loglike,'obsIter')


import gi
gi.require_version('Gtk', '3.0')
from gi.repository import Gtk

ICONSIZE = Gtk.IconSize.SMALL_TOOLBAR

class controlBar(Gtk.HeaderBar):
    def __init__(self):
        Gtk.HeaderBar.__init__(self)

        self.set_show_close_button(True)
        self.props.title = "PyFlowChart"

        self.info_box = Gtk.Box(spacing=10,orientation=Gtk.Orientation.HORIZONTAL)

        self.file_button = Gtk.Button.new_from_icon_name("open-menu-symbolic", ICONSIZE)

        self.settings_button = Gtk.Button.new_from_icon_name("preferences-system", ICONSIZE)
        self.help_button = Gtk.Button.new_from_icon_name("help-about", ICONSIZE)

        self.main_menu = self.init_main_menu()
        self.settings_menu = self.init_settings_menu()
        self.help_menu = self.init_help_menu()
        #self.init_edit_menu()

        self.file_button.connect('clicked', self.file_clicked)
        self.settings_button.connect('clicked', self.settings_clicked)
        self.help_button.connect('clicked', self.help_clicked)

        self.info_box.add(self.settings_button)
        self.info_box.add(self.help_button)

        self.pack_start(self.file_button)
        self.pack_end(self.info_box)

        self.buttons = []
        # self.populate_buttons()

        self.show_all()

    def init_main_menu(self):
        main_menu = Gtk.Menu()

        self.new_button = Gtk.MenuItem.new_with_label('New')
        self.open_button = Gtk.MenuItem.new_with_label('Open')

        self.open_stock_button = Gtk.MenuItem.new_with_label('Import Stock')

        view_button = Gtk.MenuItem.new_with_label('View')

        self.view_menu = Gtk.Menu()
        self.viewer_button = Gtk.MenuItem.new_with_label('Viewer')
        self.builder_button = Gtk.MenuItem.new_with_label('Builder')
        self.view_menu.append(self.viewer_button)
        self.view_menu.append(self.builder_button)
        self.view_menu.show_all()

        view_button.set_submenu(self.view_menu)

        self.save_button = Gtk.MenuItem.new_with_label('Save')
        self.save_as_button = Gtk.MenuItem.new_with_label('Save As...')
        self.quit_button = Gtk.MenuItem.new_with_label('Quit')

        main_menu.append(self.new_button)
        main_menu.append(self.open_button)
        main_menu.append(self.open_stock_button)
        main_menu.append(Gtk.SeparatorMenuItem())
        main_menu.append(view_button)
        main_menu.append(Gtk.SeparatorMenuItem())
        main_menu.append(self.save_button)
        main_menu.append(self.save_as_button)
        main_menu.append(Gtk.SeparatorMenuItem())
        main_menu.append(self.quit_button)
        main_menu.show_all()

        return main_menu

    def init_settings_menu(self):
        settings_menu = Gtk.Menu()

        self.preferences_button = Gtk.MenuItem.new_with_label('Preferences')
        settings_menu.append(self.preferences_button)

        settings_menu.show_all()
        return settings_menu

    def init_help_menu(self):
        help_menu = Gtk.Menu()

        self.app_help_button = Gtk.MenuItem.new_with_label('Help')
        self.about_button = Gtk.MenuItem.new_with_label('About')

        help_menu.append(self.about_button)
        help_menu.append(self.app_help_button)

        help_menu.show_all()
        return help_menu

    def file_clicked(self, widget):
        self.main_menu.popup( None, None, None, None, 0, Gtk.get_current_event_time())

    def settings_clicked(self, widget):
        self.settings_menu.popup( None, None, None, None, 0, Gtk.get_current_event_time())

    def help_clicked(self, widget):
        self.help_menu.popup( None, None, None, None, 0, Gtk.get_current_event_time())


def findMaxAverage(nums: list[int], k: int) -> float:
    max_sum = 0
    for i in range(k):
        max_sum += nums[i]
    curr_sum = max_sum
    for i in range(1,len(nums)-k+1):
        curr_sum = curr_sum - nums[i-1] + nums[i+k-1]
        if curr_sum > max_sum:
            max_sum = curr_sum

    return max_sum / k

print(findMaxAverage([5], 1))


import logging
from copy import deepcopy
from itertools import permutations

import numpy as np
from scipy.special import softmax
from scipy.stats import entropy

def true_entropy(team_generator, batch_predict, num_items: int, num_selections: int):
    P_A = np.zeros((num_selections, num_items))  # basically P(A^i_j)
    past_probs = []

    for i in range(num_selections):
        # separately calculate P(A^i_j)

        # all possible permutations with team size upto i
        sets = list(permutations(range(num_items), i + 1))
        teams = [team_generator() for x in range(len(sets))]
        for j, s in enumerate(sets):
            for item in s:
                teams[j].mark(item)

        # put them together for a batch update
        vals = batch_predict(teams)
        # reshape them, so we can group by same prefix teams (so that p(last_element) sums to 1
        struct_vals = softmax(vals.reshape(-1, num_items - i), axis=1)
        vals = struct_vals.reshape(-1)

        # to add to past probabilities coz P(A^j| prod of A's < j)
        P = np.zeros((num_items,) * (i + 1))

        for j, team in enumerate(teams):
            prefix_p = 1
            for k in range(len(team)):
                pp = past_probs[k - 1][tuple(team[z] for z in range(k))] if k > 0 else 1  # to help find the prefix
                prefix_p *= pp
            P[tuple(team[z] for z in range(len(team)))] += vals[j]
            P_A[i, team[-1]] += prefix_p * vals[j]
            # print(team.pkms,  P_A[i, team[-1]], prefix_p, vals[j])
        past_probs.append(P)  # somevariant of vals so that its easily indexible)
    # print(P_A, np.sum(P_A, axis=1))
    # print((np.sum(P_A, axis=0)))
    # P_A  = np.sum(P_A, axis = 0)
    """
    P_X = np.zeros((num_items))

    for i in range(num_selections):
        accumulated_P = np.ones((num_items))
        for j in range(num_selections):
            if i != j:
                accumulated_P *= (np.ones((num_items)) - P_A[j])
        P_X += P_A[i] * accumulated_P
    """
    P_X = np.sum(P_A, axis=0) / num_selections
    entropy_loss = -entropy(P_X)
    logging.info("P_A=%s\tEntropy=%s\t", str(list(P_X)), str(entropy_loss))
    return entropy_loss

def sample_based_entropy(team_generator, batch_predict, num_items: int, num_selections: int, num_samples: int):
    counts = np.zeros(num_items)
    for i in range(num_samples):
        team = team_generator()
        for j in range(num_selections):
            tmp_teams = [deepcopy(team) for z in range(num_items)]
            items = [z for z in range(num_items)]
            for k, item in enumerate(items):
                tmp_teams[k].mark(item)
            vals = (batch_predict(tmp_teams))
            for k in range(len(team) - 1):
                vals[team[k]] = float("-inf")
            p = softmax(vals)
            selection = np.random.choice(range(num_items), p=p)
            team.mark(selection)
            counts[selection] += 1

    P_A = counts / sum(counts)
    entropy_loss = -entropy(P_A)
    logging.info("P_A=%s\tEntropy=%s\t", str(list(P_A)), str(entropy_loss))
    return entropy_loss

def lower_bound_entropy(team_generator, batch_predict, num_items: int, num_selections: int):
    all_teams = [team_generator() for x in range(num_items)]
    for i in range(num_items):
        all_teams[i].mark(i)  # just mark one element
    P_A = softmax(batch_predict(all_teams))
    entropy_loss = -entropy(P_A)
    logging.info("P_A=%s\tEntropy=%s\t", str(list(P_A)), str(entropy_loss))
    return entropy_loss


def _extend_pre_ranges(df, upstream:int=0, downstream:int=0, start:str="Start", end:str="End", strand:str="Strand"):
    strand_rm = False
    if strand not in df.columns:
        strand_rm = True
        df[strand] = "+"
    df.loc[df[strand] == "+", start] -= upstream
    df.loc[df[strand] == "-", start] -= downstream
    df.loc[df[strand] == "+", end] += downstream
    df.loc[df[strand] == "-", end] += upstream
    if strand_rm:
        del df[strand]
    return df

def _extend_ranges(gf,
                   min_upstream:int=1000,
                   max_upstream:int=100000,
                   min_downstream:int=1000,
                   max_downstream:int=100000,
                   gene_upstream:int=5000,
                   gene_downstream:int=0,
                   gene_scale_factor:float=5.):
    import numpy as np
    gf["gene_length"] = gf["End"] - gf["Start"]
    ### In case of duplicated indices, sum lengths across indices
    gb = gf.groupby(level=0).agg(gene_length=("gene_length", sum))
    gf["gene_length"] = gb.loc[gf.index.values, "gene_length"].values
    ### scale by inverse gene length
    gs = 1/gf["gene_length"].values
    ### min max scale, plus epsilon to avoid divide by 0
    gs = (gs - np.min(gs)) / (np.max(gs) - np.min(gs)) + 1e-300
    ### expand to 1 to GSF range. But take log, so log1p((gs-1)*s) = log(1 + (gs - 1) * s)
    gf["log_gene_scale"] = np.log1p((gene_scale_factor - 1) * gs)
    #gf.index = gf["gene_id"].values
    gf = _extend_pre_ranges(gf, upstream=gene_upstream, downstream=gene_downstream)
    gf["MinStart"] = gf["Start"].values
    gf["MinEnd"] = gf["End"].values
    gf = _extend_pre_ranges(gf, upstream=min_upstream, downstream=min_downstream, start="MinStart", end="MinEnd")
    gf["interval"] = ["%s:%d-%d" % (chrom, start, end) for chrom, start, end in zip(gf["Chromosome"], gf["Start"], gf["End"])]
    gf["min_interval"] = ["%s:%d-%d" % (chrom, start, end) for chrom, start, end in zip(gf["Chromosome"], gf["MinStart"], gf["MinEnd"])]
    return gf

def estimate_features_archr(adata, feature_df,
                            min_upstream:int=1000,
                            max_upstream:int=100000,
                            min_downstream:int=1000,
                            max_downstream:int=100000,
                            gene_upstream:int=5000,
                            gene_downstream:int=0,
                            target_sum:int=None,
                            gene_scale_factor:float=5.,
                            peak_column:str=None, ## If not provided, will use peak index
                            feature_column:str=None,
                            var_column_tolerance:float=0.999, ### Tolerance used for determining if a feature should be kept, in case of weird things
                            distal:bool=True, ### Use nearest gene to a peak if unassigned
                            log1p:bool=False,
                            save_raw:bool=False,
                            layer:str=None):
    import numpy as np
    import pandas as pd
    import scipy.sparse
    import pyranges
    import anndata
    import scanpy as sc
    from .timer import template
    sw = template()
    if not isinstance(feature_df, pd.DataFrame):
        raise ValueError("Feature_df is not a dataframe")
    if not np.all(np.isin(["Chromosome", "Start", "End"], feature_df.columns)):
        raise ValueError("Feature_df does not have ranges")
    if feature_column is not None:
        feature_df.index = feature_df[feature_column].values
    with sw("Extending ranges"):
        gf = _extend_ranges(feature_df, min_upstream=min_upstream, min_downstream=min_downstream,
                            max_upstream=max_upstream, max_downstream=max_downstream,
                            gene_upstream=gene_upstream, gene_downstream=gene_downstream,
                            gene_scale_factor=gene_scale_factor)
        gr = pyranges.from_dict({"Chromosome": gf["Chromosome"],
                                 "Start": gf["Start"],
                                 "End": gf["End"],
                                 "feature_index": gf.index.values,
                                 "feature_interval": gf["interval"].values})
        mingr = pyranges.from_dict({"Chromosome": gf["Chromosome"],
                                    "Start": gf["MinStart"],
                                    "End": gf["MinEnd"],
                                    "feature_index": gf.index.values,
                                    "feature_interval": gf["interval"].values})
    ##
    ## Now, get peak ranges (pr) from peak frame (pf)
    ##
    with sw("Extracting peak ranges"):
        if peak_column is None:
            pstr = adata.var_names.values
        else:
            pstr = adata.var[peak_column].values
        pf = pd.DataFrame([x.replace(":", "-", 1).split("-") for x in pstr], columns=["Chromosome", "Start", "End"], index=adata.var_names)
        pr = pyranges.from_dict({"Chromosome": pf["Chromosome"], "Start": pf["Start"], "End": pf["End"], "peak_name": pf.index.values})
    with sw("Calculating overlaps"):
        iif = gf.drop_duplicates("interval")
        iif.index = iif["interval"]
        ## Once peak ranges are gathered, find intersecting gene bodies:
        inter_df = pr.join(gr).df.loc[:, ["peak_name", "feature_index", "feature_interval"]]
        inter_df["Distance"] = 0
        ## Then, find genes with minimum upstream/downstream distance away
        min_df = pr.join(mingr).df.loc[:, ["peak_name", "feature_index", "feature_interval"]]
        ## diff. is accurate, unless overlapping intervals. Do not need to worry, as duplicates from inter_df will take care
        diff = pf.loc[min_df["peak_name"].values, ["Start", "Start", "End", "End"]].values.astype(int) - iif.loc[min_df["feature_interval"].values, ["Start", "End", "Start", "End"]].values.astype(int)
        min_df["Distance"] = np.abs(diff).min(1) + 1
        ## Finally, find distal. Only need nearest gene

        if distal:
            distance_df = pr.nearest(gr).df.loc[:, ["peak_name", "feature_index", "feature_interval", "Distance"]]
            ## Concat such that 1) prioritized intersections, then 2) minimum distance away, then 3) distal
            df = pd.concat([inter_df, min_df, distance_df]).drop_duplicates(["peak_name", "feature_index"])
        else:
            df = pd.concat([inter_df, min_df]).drop_duplicates(["peak_name", "feature_index"])
        df["weight"] = np.exp(-1 - np.abs(df["Distance"]) / 5000. + iif.loc[df["feature_interval"].values, "log_gene_scale"].values)
    with sw("Calculating accessibility"):
        if gf.index.duplicated().sum() > 0:
            ### Get columns that are the same across repeated indices
            nf = gf.groupby(level=0).nunique()
            gf = gf.loc[~gf.index.duplicated(keep="first"), nf.columns[(nf==1).mean() >= var_column_tolerance]]
        S = scipy.sparse.csr_matrix((df["weight"].values,
                                     (pf.index.get_indexer(df["peak_name"].values),
                                      gf.index.get_indexer(df["feature_index"].values))),
                                    shape=(pf.shape[0], gf.shape[0]))
        if layer is not None and layer in adata.layers:
            X = adata.layers[layer]
        else:
            X = adata.X
        gdata = anndata.AnnData(X.dot(S), obs=adata.obs, var=gf, dtype=np.float32, obsm=adata.obsm, obsp=adata.obsp,
                                uns={k: v for k, v in adata.uns.items() if k in ["neighbors", "files", "lsi", "pca", "umap", "leiden"]})
    if save_raw:
        gdata.layers["raw"] = gdata.X.copy()
    if target_sum is not None and target_sum > 0:
        sc.pp.normalize_total(gdata, target_sum=target_sum)
    else:
        print("Using median normalization")
        sc.pp.normalize_total(gdata)
    if log1p:
        sc.pp.log1p(gdata)
    return gdata

def estimate_genes_archr(adata, gtf:str,
                         min_upstream:int=1000,
                         max_upstream:int=100000,
                         min_downstream:int=1000,
                         max_downstream:int=100000,
                         gene_upstream:int=5000,
                         gene_downstream:int=0,
                         target_sum:int=None,
                         gene_scale_factor:float=5.,
                         peak_column:str=None, ## If not provided, will use peak index
                         feature_column:str="gene_id", ### If not provided, will use feature index
                         log1p:bool=True,
                         distal:bool=True,
                         save_raw:bool=False,
                         layer:str=None):
    import numpy as np
    import pandas as pd
    import scipy.sparse
    import pyranges
    import anndata
    import scanpy as sc
    from .timer import template
    sw = template()
    with sw("Reading GTF"):
        gf = pyranges.read_gtf(gtf).df
        gf = gf.loc[gf["Feature"] == "gene", :]
        if feature_column in gf.columns:
            gf.index = gf[feature_column].values.astype(str)
    gdata = estimate_features_archr(adata, feature_df=gf,
                                   min_upstream=min_upstream, min_downstream=min_downstream,
                                   max_upstream=max_upstream, max_downstream=max_downstream,
                                   gene_upstream=gene_upstream, gene_downstream=gene_downstream,
                                   target_sum=target_sum, gene_scale_factor=gene_scale_factor,
                                   peak_column=peak_column,
                                   feature_column=feature_column,
                                    log1p=log1p, layer=layer, save_raw=save_raw)
    gdata.var = gdata.var.loc[:, ["gene_id", "gene_name"]]
    gdata.var_names = gdata.var["gene_name"].values
    gdata.var_names_make_unique()
    del gdata.var["gene_name"]
    gdata.var.columns = ["gene_ids"]
    add_gene_length(gdata.var, gtf)
    return gdata

def get_tss(tss:str):
    import pandas as pd
    tss = pd.read_csv(tss, sep="\t", header=None)
    tss.columns = ["Chromosome", "Start", "End", "gene_id", "score", "strand"]
    df = tss.groupby(["Chromosome", "gene_id", "strand"]).agg(left=("Start", "min"),
                                                              right=("End", "max")).reset_index()
    df["interval"] = df["Chromosome"] + ":" + df["left"].astype(str) + "-" + df["right"].astype(str)
    df.index = df["gene_id"].values
    return df

def add_interval(var, tss:str, inplace=True):
    tf = get_tss(tss)
    interval = [tf["interval"].get(g, "NA") for g in var["gene_ids"]]
    if inplace:
        var["interval"] = interval
    else:
        import pandas as pd
        return pd.Series(interval, index=var.index, name="interval")

def add_gene_length(var, gtf:str=None, inplace=True):
    import pandas as pd
    import pyranges
    from .timer import template
    sw = template()
    with sw("Reading GTF"):
        gf = pyranges.read_gtf(gtf).df
        gf = gf.loc[gf["Feature"] == "gene",:]
        gf["gene_length"] = gf["End"] - gf["Start"]
        gf.index = gf["gene_id"].values
    gl = [gf["gene_length"].get(g, -1) for g in var["gene_ids"]]
    gs = [gf["Strand"].get(g, "*") for g in var["gene_ids"]]
    if not inplace:
        var = var.copy()
    var["gene_length"] = gl
    var["strand"] = gs
    if not inplace:
        return var.loc[:, ["gene_length", "strand"]]

def add_gene_info(var, gene_info:str=None, inplace=True):
    """Add a STAR geneInfo.tab file to .var"""
    import pandas as pd
    df = pd.read_csv(gene_info, sep="\t", skiprows=[0], header=None)
    df.index = df[0].values
    gi = [df[2].get(g, "NA") for g in var["gene_ids"]]
    if inplace:
        var["gene_type"] = gi
    else:
        return pd.Series(gi, index=var.index, name="gene_type")


from util import get_options, check_input
import random

def game_round(film_data, target_type, question_type, question):
    target = random.choice(film_data)

    options = get_options(film_data, target, target_type)
    print(f"{question}\n>>> {target[question_type]}")
    print(f"1: {options[0]}\n"
          f"2: {options[1]}\n"
          f"3: {options[2]}")

    attempt = check_input()

    if options[attempt-1] == target[target_type]:
        print("Correct! +1\n")
        return True
    else:
        print(f"Incorrect. Correct answer: {target[target_type]}.\n")
        return False

def interpret_round(outcome, points):
    if outcome:
        points += 1
        return points
    else:
        return points


import epyk as pk

from epyk.mocks import urls as data_urls

# Create a basic report object
page = pk.Page()
page.headers.dev()

# retrieve some random json data
data_rest = page.py.requests.csv(data_urls.BLOG_OBJECT)

# create a json viewer object
j = page.ui.json(data_rest, height=(100, 'px'))

# change the default options
j.options.open = False
j.options.hoverPreviewEnabled = True

# add a button for interactivity
page.ui.button("Click").click([
  j.js.openAtDepth(0),
  j.build({"test": 'ok'})
])


from collections import *

import numpy as np

from common.session import AdventSession

session = AdventSession(day=20, year=2017)
data = session.data.strip()
data = data.split('\n')

p1, p2 = 0, 0

class Particle:
    def __init__(self, p, v, a, _id):
        self.p = np.array(p)
        self.v = np.array(v)
        self.a = np.array(a)
        self.p_sum = np.abs(p).sum()
        self.v_sum = np.abs(v).sum()
        self.a_sum = np.abs(a).sum()
        self.id = _id

    def update(self):
        self.v += self.a
        self.p += self.v

particles = []
for i, line in enumerate(data):
    line = line.replace('<', '(').replace('>', ')')
    particles.append(eval(f'Particle({line}, _id={i})'))

sorted_particles = sorted(particles, key=lambda p: (
    p.a_sum, p.v_sum, p.p_sum))

p1 = sorted_particles[0].id

for _ in range(1000):
    positions = defaultdict(list)
    for p in particles:
        positions[tuple(p.p)].append(p)
    for parts in positions.values():
        if len(parts) > 1:
            for part in parts:
                particles.remove(part)

    for p in particles:
        p.update()

p2 = len(particles)

session.submit(p1, part=1)
session.submit(p2, part=2)


# finding the right modules/packages to use is not easy
import os
import pyodbc
from numpy import genfromtxt
import pandas as pd
import sqlalchemy as sa # use sqlalchemy for truncating etc
from sqlalchemy import Column, Integer, Float, Date, String, BigInteger
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
from sqlalchemy.dialects import mssql

import urllib
import csv
import fnmatch2
from time import time

## DATABASE STUFF
# server + instance = DESKTOP-A6J6D7Q\LEROYB_INSTANCE
# database = Tracks

server_name='DESKTOP-A6J6D7Q\LEROYB_INSTANCE'
db_name='Tracks'

path = 'F:\export_6642035' # location of all the files related to STRAVA export
pathArchive = 'F:\export_6642035_archive' # this is the archive folder that will contain all the processed files
searchstring = '*' # we could limit the files to a particular string pattern, however in this case we want to go through all the files

# https://docs.sqlalchemy.org/en/13/dialects/mssql.html#connecting-to-pyodbc
# https://docs.sqlalchemy.org/en/13/core/connections.html
# https://www.pythonsheets.com/notes/python-sqlalchemy.html
# https://auth0.com/blog/sqlalchemy-orm-tutorial-for-python-developers/
# https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.truncate.html

params = urllib.parse.quote_plus('Driver={SQL Server};'
'Server='+server_name +';'
'Database='+db_name+';'
'Trusted_Connection=yes;')

engine = sa.create_engine("mssql+pyodbc:///?odbc_connect=%s" % params,fast_executemany=True, echo=True)

# https://chrisalbon.com/python/data_wrangling/pandas_dataframe_importing_csv/
def Load_Data(file_name):
    #data = genfromtxt(file_name, delimiter=',', skip_header=1, converters={0: lambda s: str(s)})
    data = pd.read_csv(file_name, skiprows=0)
    return data.values.tolist()

Base = declarative_base()
BaseGPX = declarative_base()

class Activities(Base):
    #Tell SQLAlchemy what the table name is and if there's any table-specific arguments it should know about
    __tablename__ = 'Activities'
    __table_args__ = {'schema':'STG'}
    #tell SQLAlchemy the name of column and its attributes:
    Activity_ID = Column(BigInteger, primary_key=True, nullable=False) #
    Activity_Date = Column(Date)
    Activity_Name = Column(String)
    Activity_Type = Column(String)
    Activity_Description = Column(String)
    Elapsed_Time = Column(Integer)
    Distance = Column(Float)
    Commute = Column(String)
    Activity_Gear = Column(String)
    Filename = Column(String)

class GPXFiles(BaseGPX):
    #Tell SQLAlchemy what the table name is and if there's any table-specific arguments it should know about
    __tablename__ = 'GPXFiles'
    __table_args__ = {'schema':'STG'}
    #tell SQLAlchemy the name of column and its attributes:
    GPXFile_ID = Column(BigInteger, primary_key=True, autoincrement=True, nullable=False) #
    GPXFile_Name = Column(String, nullable=True)
    GPXFile_Contents = Column(String, nullable=True)
    #GPXFile_XMLContents = Column(String, nullable=True)
    #GPXFile_Route_ID = Column(BigInteger, nullable=True)

# loop through files
# r=root, d=directories, f = files
for r, d, f in os.walk(path):
    for file in f:
        if fnmatch2.fnmatch2(file, searchstring):

            if file.endswith('.gpx'):
                print('gpxfile')
                # go straight into the STG.GPXFiles table
                #Create the session
                #GPXFiles.__table__.drop(bind=engine)
                GPXFiles.__table__.create(bind=engine, checkfirst=True) # if the table exists, it is dropped (not using checkfirts=true)
                session = sessionmaker(bind=engine)
                #print(session)
                session.configure(bind=engine)
                s = session()
                try:

                    data = Load_Data(os.path.join(r, file))
                    #print(data)
                    #for i in data:
                        #print(str(i[0])+'-'+str(i[1]))
                    record = GPXFiles(**{
                        #'GPXFile_ID' : i[0],
                        'GPXFile_Name' : file, #str(i[1]),
                        'GPXFile_Contents' : data
                    })

                    s.add(record) #Add all the records

                    s.commit() #Attempt to commit all the records

                except Exception as e:
                    print(e)
                    s.rollback() #Rollback the changes on error

                s.close() #Close the connection
                continue
            elif file.endswith('.gpx.gz'):
                # print('a GZ gpxfile')
                # unzip file first, then get the file into the STG.GPXFiles table
                continue
            elif file.endswith('.csv'):
                # the only file we really need is the activities.csv file
                # the columns are
                if file == 'activities.csv':
                    # import it

                    ### using pure panda
                    # insert CSV into the table
                    # https://stackoverflow.com/questions/31394998/using-sqlalchemy-to-load-csv-file-into-a-database

                    #Create the session
                    #Activities.__table__.drop(bind=engine)
                    Activities.__table__.create(bind=engine, checkfirst=True) # if the table exists, it is dropped (not using checkfirts=true)
                    session = sessionmaker(bind=engine)
                    print(session)
                    session.configure(bind=engine)
                    s = session()
                    try:

                        data = Load_Data(os.path.join(r, file))
                        #print(data)
                        for i in data:
                            #print(str(i[0])+'-'+str(i[1]))
                            record = Activities(**{
                                'Activity_ID' : i[0],
                                'Activity_Date' : i[1].replace(',', ''),
                                'Activity_Name' : str(i[2]),
                                'Activity_Type' : str(i[3]),
                                'Activity_Description' : str(i[4]),
                                'Elapsed_Time' : int(i[5]),
                                'Distance' : float(i[6].replace(',', '')),
                                'Commute' : str(i[7]),
                                'Activity_Gear' : str(i[8]),
                                'Filename' : str(i[9])
                            })

                            s.add(record) #Add all the records

                            s.commit() #Attempt to commit all the records

                    except Exception as e:
                        print(e)
                        s.rollback() #Rollback the changes on error

                    s.close() #Close the connection

            else:
                continue

            # move file to archive folder at same level as source folder
            # first check folder exist, if not, then create it

        continue

#conn.close

# now that this is done, we need to run the stored procedures to get the data from the staging tbales into the production conformed tables
# first to brng the data from staging into conformed area
# then run a bunch of procs to update some of th emissing attributes

# don't forget to clean up in some way


# -*- coding: utf-8 -*-

__author__ = 'xl'
"""
__date__ = TIME： 2018/10/02 上午9:24
describe:利用二分搜索树实现集合
"""
from bst import BinarySearchTree
import chardet
import re
class MySet:
    """
    利用二分搜索树实现集合
    """
    def __init__(self):
        self.bst = BinarySearchTree()
    def add(self,e):
        self.bst.add(e)
    def remove(self,e):
        self.bst.remove(e)
    def contains(self, e):
        return self.bst.contains(e)
    def getsize(self):
        return self.bst.size
    def isEmpty(self):
        return True if self.bst.size == 0 else False

if  __name__ == "__main__":
    myset  = MySet()
    with open("Les Miserables.txt",'r') as f:

        lines = f.readlines()
        for line in lines:
            line = line.decode('EUC-JP')
            #print(line.strip())
            line = line.strip().replace(',',' ').replace(';',' ').replace('"',' ').replace('.',' ').replace('?',' ')
            line = re.sub("['!-]","",line)
            words = line.strip().split(' ')
            for word in words:
                print(str(word).lower())
                myset.add(str(word).lower())
    print(myset.getsize())

        #print(text)

        # for line in txt:
        #     print(line.strip())
        #print("type",type)
        # for line in txt:
        #      print(line)


#Coded by: QyFashae

import os
from sklearn.model_selection import train_test_split
from sklearn.pipeline import Pipeline
from sklearn.feature_extraction.text import HashingVectorizer, TfidfTransformer
from sklearn.metrics import accuracy_score, confusion_matrix
from sklearn import tree

# Paths to the directories containing the emails
s_ep = os.path.join("training_data_for_ml_emails", "training_data_ML00")
v_ep = os.path.join("training_data_for_ml_emails", "training_data_ML01")

# List of file directories and corresponding labels
l_fd = [(s_ep, 0), (v_ep, 1)]
e_cs = []
l_bs = []

# Function to read email content and labels
def cve_sh_trfm(s_ep, v_ep, l_fd, e_cs, l_bs):
    for cfs, lbs in l_fd:
        files = os.listdir(cfs)
        for file in files:
            file_path = os.path.join(cfs, file)
            try:
                with open(file_path, "r") as current_file:
                    eml_ctt = current_file.read().replace("\n", "")
                    eml_ctt = str(eml_ctt)
                    e_cs.append(eml_ctt)
                    l_bs.append(lbs)
            except:
                pass

# Splitting the dataset and training the model
def vary_trmd(cve_sh_trfm, s_ep, v_ep, l_fd, e_cs, l_bs):
    x_train, x_test, y_train, y_test = train_test_split(
        e_cs, l_bs, test_size=0.4, random_state=17
    )
    nlp_followed_by_dt = Pipeline(
        [
            ("vect", HashingVectorizer(input="eml_ctt", ngram_range=(1, 4))),
            ("tfidf", TfidfTransformer(use_idf=True)),
            ("dt", tree.DecisionTreeClassifier(class_weight="balanced")),
        ]
    )
    nlp_followed_by_dt.fit(x_train, y_train)

    # Predicting and evaluating the model
    y_test_predict = nlp_followed_by_dt.predict(x_test)
    accuracy = accuracy_score(y_test, y_test_predict)
    confusion = confusion_matrix(y_test, y_test_predict)

    # Writing accuracy and confusion matrix to files
    with open("accuracy_score.txt", "w") as acc_file:
        acc_file.write("Accuracy: " + str(accuracy))

    with open("confusion_matrix.txt", "w") as conf_file:
        conf_file.write("Confusion Matrix:\n" + str(confusion))


from abc import ABC

from car import Car

class Tires(Car, ABC):

    def __init__(self, last_service_date, left_front_tire_pressure, right_front_tire_pressure, left_rear_tire_pressure, right_rear_tire_presure):
        super().__init__(last_service_date)
        self.left_front_tire = left_front_tire_pressure
        self.right_front_tire = right_front_tire_pressure
        self.left_rear_tire = left_rear_tire_pressure
        self.right_rear_tire = right_rear_tire_presure

    #this is returns any bool value for when any tire should be serviced
    def tire_should_be_serviced(self):
        if self.left_front_tire < 32:
            return True
        elif self.right_front_tire < 32:
            return True
        elif self.left_rear_tire < 32:
            return True
        elif self.right_rear_tire < 32:
            return True
        else:
            return False


"""We are the captains of our ships, an we stay 'till the end. We see our stories through.
"""

"""947. Most Stones Removed with Same Row or Column
"""

class Solution:

    def removeStones(self, stones):
        visited = {tuple(stone): False for stone in stones}

        def dfs(s1, visited):

            visited[s1] = True
            for s2 in stones:
                s2 = tuple(s2)
                if not visited[s2]:
                    if (s2[0] == s1[0] or s2[1] == s1[1]):
                        visited[s2] = True
                        dfs((s2[0], s2[1]), visited)

        connected_components = 0
        for stone in stones:
            s = tuple(stone)
            if not visited[s]:
                dfs(s, visited)
                connected_components += 1
        return len(stones) - connected_components

Graphs/947. Most Stones Removed with Same Row or Column.py


# -*- coding: utf-8 -*-
"""view module prilog application

     * view function, and run Flask

"""
from glob import glob
from flask import Flask, render_template, request, session, redirect, jsonify
import os
import re
import json
import urllib.parse
import subprocess
import time as tm
import analyze as al
import common as cm
import state_list as state
import configparser

config = configparser.ConfigParser()
config.read("config.ini")

SERVER_ERROR_STATE = config.get("general", "error_state")
SERVER_TOKEN_AUTH = config.get("general", "token_auth")
MULTI_SERVER = config.get("rest", "multi_server")
DL_INTERVAL = int(config.get("rest", "interval"))

# movie download directory
stream_dir = "tmp/"
if not os.path.exists(stream_dir):
    os.mkdir(stream_dir)

# analyze result save as cache directory
cache_dir = "cache/"
if not os.path.exists(cache_dir):
    os.mkdir(cache_dir)

# save analyzing id as file directory
download_dir = "download/"
if not os.path.exists(download_dir):
    os.mkdir(download_dir)

# waiting analyze id as file directory
dl_queue_dir = "download/queue/"
if not os.path.exists(dl_queue_dir):
    os.mkdir(dl_queue_dir)

# save analyzing id as file directory
dl_ongoing_dir = "download/ongoing/"
if not os.path.exists(dl_ongoing_dir):
    os.mkdir(dl_ongoing_dir)

# save analyzing id as file directory
dl_pending_dir = "download/pending/"
if not os.path.exists(dl_pending_dir):
    os.mkdir(dl_pending_dir)

# save analyzing id as file directory
dl_server_dir = "download/server/"
if not os.path.exists(dl_server_dir):
    os.mkdir(dl_server_dir)

# waiting analyze id as file directory
queue_dir = "queue/"
if not os.path.exists(queue_dir):
    os.mkdir(queue_dir)

# save analyzing id as file directory
pending_dir = "pending/"
if not os.path.exists(pending_dir):
    os.mkdir(pending_dir)

# api token as file directory
token_dir = "token/"
if not os.path.exists(token_dir):
    os.mkdir(token_dir)

def get_web_txt(youtube_id, title, time_line, debuff_value, total_damage):
    debuff_dict = None
    if debuff_value:
        debuff_dict = ({key: val for key, val in zip(time_line, debuff_value)})
    data_url = "https://prilog.jp/?v=" + youtube_id
    data_txt = "@PriLog_Rより%0a"
    data_txt += title + "%0a"
    if total_damage:
        total_damage = "総ダメージ " + "".join(total_damage)
        data_txt += total_damage + "%0a"

    return debuff_dict, data_txt, data_url, total_damage

def get_rest_result(title, time_line, time_line_enemy, time_data, total_damage, debuff_value):
    rest_result = {"title": title, "timeline": time_line, "timeline_enemy": time_line_enemy, "process_time": time_data,
                   "total_damage": total_damage, "debuff_value": debuff_value}

    if time_line:
        rest_result["timeline_txt"] = "\r\n".join(time_line)
        if time_line_enemy:
            rest_result["timeline_txt_enemy"] = "\r\n".join(time_line_enemy)
        else:
            rest_result["timeline_txt_enemy"] = False

        if debuff_value:
            rest_result["timeline_txt_debuff"] = "\r\n".join(list(
                map(lambda x: "↓{} {}".format(str(debuff_value[x[0]][0:]).rjust(3, " "), x[1]),
                    enumerate(time_line))))
        else:
            rest_result["timeline_txt_debuff"] = False
    else:
        rest_result["timeline_txt"] = False
        rest_result["timeline_txt_enemy"] = False
        rest_result["timeline_txt_debuff"] = False

    return rest_result

app = Flask(__name__)
app.config.from_object(__name__)
app.config["SECRET_KEY"] = "zJe09C5c3tMf5FnNL09C5e6SAzZuY"
app.config["JSON_AS_ASCII"] = False

@app.route("/", methods=["GET", "POST"])
def index():
    if request.method == "POST":
        url = (request.form["Url"])

        # urlからid部分の抽出
        youtube_id = al.get_youtube_id(url)
        if youtube_id is False:
            error = state.get_error_message(state.ERR_BAD_URL)
            return render_template("index.html", error=error)

        cache = cm.cache_check(youtube_id)

        if cache:
            title, time_line, time_line_enemy, time_data, total_damage, debuff_value, past_status = cache
            if past_status % 100 // 10 == 0:
                debuff_dict, data_txt, data_url, total_damage = get_web_txt(youtube_id, title,
                                                                            time_line, debuff_value, total_damage)

                return render_template("result.html", title=title, timeLine=time_line, timeLineEnemy=time_line_enemy,
                                       timeData=time_data, totalDamage=total_damage, debuffDict=debuff_dict,
                                       data_txt=data_txt, data_url=data_url)

            else:
                error = state.get_error_message(past_status)
                return render_template("index.html", error=error)

        if SERVER_ERROR_STATE:
            error = state.get_error_message(state.ERR_SERVICE_UNAVAILABLE)
            return render_template("index.html", error=error)

        # start download
        dl_queue_path = dl_queue_dir + str(youtube_id)
        dl_ongoing_path = dl_ongoing_dir + str(youtube_id)

        # 既にキューに登録されているか確認
        queued = os.path.exists(dl_queue_path)
        if not queued:  # 既にダウンロード待機中ではない場合、ダウンロード待機キューに登録
            cm.queue_append(dl_queue_path)
            # キューが回ってきたか確認し、来たらダウンロード実行
            while True:
                if not cm.is_path_exists(dl_ongoing_path) and cm.is_path_due(dl_queue_path):
                    if cm.is_pending_download(DL_INTERVAL):  # check pending download
                        break

                timeout = cm.watchdog_download(youtube_id, 300)  # 5分間タイムアウト監視

                if timeout:
                    cm.clear_path(dl_queue_path)
                    error = "動画の解析待ちでタイムアウトが発生しました。再実行をお願いします。"
                    return render_template("index.html", error=error)

                tm.sleep(1)

        else:  # ダウンロード待機中の場合エラーメッセージ表示
            cm.clear_path(dl_queue_path)
            error = "同一の動画が解析中です。時間を置いて再実行をお願いします。"
            return render_template("index.html", error=error)

        path, title, length, thumbnail, url_result = al.search(youtube_id)
        cm.clear_path(dl_queue_path)

        if url_result % 100 // 10 == 2:
            error = state.get_error_message(url_result)
            cm.save_cache(youtube_id, title, False, False, False, False, False, url_result)
            return render_template("index.html", error=error)

        session["path"] = path
        session["title"] = title
        session["youtube_id"] = youtube_id
        length = int(int(length) / 8) + 3

        return render_template("analyze.html", title=title, length=length, thumbnail=thumbnail)

    elif request.method == "GET":
        if "v" in request.args:  # ?v=YoutubeID 形式のGETであればリザルト返却
            youtube_id = request.args.get("v")
            if re.fullmatch(r"^([a-zA-Z0-9_-]{11})$", youtube_id):
                cache = cm.cache_check(youtube_id)
                if cache:
                    title, time_line, time_line_enemy, time_data, total_damage, debuff_value, past_status = cache
                    if past_status % 100 // 10 == 0:
                        debuff_dict, data_txt, data_url, total_damage = get_web_txt(youtube_id, title,
                                                                                time_line, debuff_value,
                                                                                total_damage)

                        return render_template("result.html", title=title, timeLine=time_line,
                                               timeLineEnemy=time_line_enemy,
                                               timeData=time_data, totalDamage=total_damage, debuffDict=debuff_dict,
                                               data_txt=data_txt, data_url=data_url)

                    else:
                        error = state.get_error_message(past_status)
                        return render_template("index.html", error=error)

                else:  # キャッシュが存在しない場合は解析
                    if SERVER_ERROR_STATE:
                        error = state.get_error_message(state.ERR_SERVICE_UNAVAILABLE)
                        return render_template("index.html", error=error)

                    # start download
                    dl_queue_path = dl_queue_dir + str(youtube_id)
                    dl_ongoing_path = dl_ongoing_dir + str(youtube_id)

                    # 既にキューに登録されているか確認
                    queued = os.path.exists(dl_queue_path)
                    if not queued:  # 既にダウンロード待機中ではない場合、ダウンロード待機キューに登録
                        cm.queue_append(dl_queue_path)
                        # キューが回ってきたか確認し、来たらダウンロード実行
                        while True:
                            if not cm.is_path_exists(dl_ongoing_path) and cm.is_path_due(dl_queue_path):
                                if cm.is_pending_download(DL_INTERVAL):  # check pending download
                                    break

                            timeout = cm.watchdog_download(youtube_id, 300)  # 5分間タイムアウト監視

                            if timeout:
                                cm.clear_path(dl_queue_path)
                                error = "動画の解析待ちでタイムアウトが発生しました。再実行をお願いします。"
                                return render_template("index.html", error=error)

                            tm.sleep(1)

                    else:  # ダウンロード待機中の場合エラーメッセージ表示
                        cm.clear_path(dl_queue_path)
                        error = "同一の動画が解析中です。時間を置いて再実行をお願いします。"
                        return render_template("index.html", error=error)

                    path, title, length, thumbnail, url_result = al.search(youtube_id)
                    cm.clear_path(dl_queue_path)

                    if url_result % 100 // 10 == 2:
                        error = state.get_error_message(url_result)
                        cm.save_cache(youtube_id, title, False, False, False, False, False, url_result)
                        return render_template("index.html", error=error)

                    session["path"] = path
                    session["title"] = title
                    session["youtube_id"] = youtube_id
                    length = int(int(length) / 8) + 3

                    return render_template("analyze.html", title=title, length=length, thumbnail=thumbnail)

            else:  # prilog.jp/(YoutubeID)に該当しないリクエスト
                error = "不正なリクエストです"
                return render_template("index.html", error=error)

        else:
            path = session.get("path")
            session.pop("path", None)
            session.pop("title", None)
            session.pop("youtube_id", None)

            error = None
            if str(path).isdecimal():
                error = state.get_error_message(path)

            elif path is not None:
                cm.clear_path(path)

            return render_template("index.html", error=error)

@app.route("/analyze", methods=["GET", "POST"])
def analyze():
    path = session.get("path")
    title = session.get("title")
    youtube_id = session.get("youtube_id")
    session.pop("path", None)

    if request.method == "GET" and path is not None:
        # TL解析
        time_line, time_line_enemy, time_data, total_damage, debuff_value, status = al.analyze_movie(path)

        # キャッシュ保存
        status = cm.save_cache(youtube_id, title, time_line, time_line_enemy, False, total_damage, debuff_value, status)

        if status % 100 // 10 == 0:
            # 解析が正常終了ならば結果を格納
            session["time_line"] = time_line
            session["time_line_enemy"] = time_line_enemy
            session["time_data"] = time_data
            session["total_damage"] = total_damage
            session["debuff_value"] = debuff_value
            return render_template("analyze.html")
        else:
            session["path"] = status
            return render_template("analyze.html")
    else:
        return redirect("/")

@app.route("/result", methods=["GET", "POST"])
def result():
    title = session.get("title")
    time_line = session.get("time_line")
    time_line_enemy = session.get("time_line_enemy")
    time_data = session.get("time_data")
    total_damage = session.get("total_damage")
    debuff_value = session.get("debuff_value")
    youtube_id = session.get("youtube_id")
    session.pop("title", None)
    session.pop("time_line", None)
    session.pop("time_line_enemy", None)
    session.pop("time_data", None)
    session.pop("total_damage", None)
    session.pop("debuff_value", None)
    session.pop("youtube_id", None)

    if request.method == "GET" and time_line is not None:
        debuff_dict, data_txt, data_url, total_damage = get_web_txt(youtube_id, title,
                                                                    time_line, debuff_value, total_damage)

        return render_template("result.html", title=title, timeLine=time_line, timeLineEnemy=time_line_enemy,
                               timeData=time_data, totalDamage=total_damage, debuffDict=debuff_dict,
                               data_txt=data_txt, data_url=data_url)
    else:
        return redirect("/")

@app.route("/download", methods=["GET", "POST"])
def download():
    if request.method == "GET":
        return render_template("download.html")
    else:
        return redirect("/")

@app.route("/rest", methods=["GET", "POST"])
def rest():
    if request.method == "GET":
        return render_template("rest.html")
    else:
        return redirect("/")

@app.route("/standalone/version", methods=["GET"])
def standalone_version():
    ret = {"version": "", "update": False}
    if request.method == "GET":
        path = "./static/release"
        fl = glob(path + "/*")
        if not fl:
            return jsonify(ret)

        # sort time stamp and find latest version
        fl.sort(key=lambda x: os.path.getctime(x), reverse=True)
        version = os.path.basename(fl[0])
        ret["version"] = version

        if "Version" in request.args:
            if request.args.get("Version") < version:
                ret["update"] = True

        return jsonify(ret)
    else:
        return jsonify(ret)

@app.route("/rest/analyze", methods=["POST", "GET"])
def rest_analyze():
    status = state.ERR_REQ_UNEXPECTED
    is_parent = False
    rest_result = {}
    ret = {}
    url = ""
    raw_url = ""
    token = ""

    # clear old movie if passed 2 hours
    cm.tmp_movie_clear()

    if request.method == "POST":
        if "Url" not in request.form:
            status = state.ERR_BAD_REQ

            ret["result"] = rest_result
            ret["msg"] = state.get_error_message(status)
            ret["status"] = status
            return jsonify(ret)
        else:
            raw_url = request.form["Url"]

        if SERVER_TOKEN_AUTH and "Token" not in request.form:
            status = state.ERR_BAD_REQ

            ret["result"] = rest_result
            ret["msg"] = state.get_error_message(status)
            ret["status"] = status
            return jsonify(ret)
        else:
            token = request.form["Token"]

    elif request.method == "GET":
        if "Url" not in request.args:
            status = state.ERR_BAD_REQ

            ret["result"] = rest_result
            ret["msg"] = state.get_error_message(status)
            ret["status"] = status
            return jsonify(ret)
        else:
            raw_url = request.args.get("Url")

        if SERVER_TOKEN_AUTH and "Token" not in request.args:
            status = state.ERR_BAD_REQ

            ret["result"] = rest_result
            ret["msg"] = state.get_error_message(status)
            ret["status"] = status
            return jsonify(ret)
        else:
            token = request.args.get("Token")

    try:
        # tokenの確認とロード
        if SERVER_TOKEN_AUTH:
            json.load(open(token_dir + urllib.parse.quote(token) + ".json"))

    except FileNotFoundError:
        status = state.ERR_BAD_TOKEN

        ret["result"] = rest_result
        ret["msg"] = state.get_error_message(status)
        ret["status"] = status
        return jsonify(ret)

    # URL抽出
    tmp_group = re.search('(?:https?://)?(?P<host>.*?)(?:[:#?/@]|$)', raw_url)

    if tmp_group:
        host = tmp_group.group('host')
        if host == "www.youtube.com" or host == "youtu.be":
            url = raw_url

    # キャッシュ確認
    youtube_id = al.get_youtube_id(url)
    queue_path = queue_dir + str(youtube_id)
    pending_path = pending_dir + str(youtube_id)
    dl_queue_path = dl_queue_dir + str(youtube_id)
    if youtube_id is False:
        # 不正なurlの場合
        status = state.ERR_BAD_URL
    else:
        # 正常なurlの場合
        cache = cm.cache_check(youtube_id)

        if cache:
            # キャッシュ有りの場合
            # キャッシュを返信
            title, time_line, time_line_enemy, time_data, total_damage, debuff_value, past_status = cache
            if past_status % 100 // 10 == 0:
                rest_result = get_rest_result(title, time_line, time_line_enemy, time_data, total_damage, debuff_value)

                ret["result"] = rest_result
                ret["msg"] = state.get_error_message(past_status)
                ret["status"] = past_status
                return jsonify(ret)

            else:
                ret["result"] = rest_result
                ret["msg"] = state.get_error_message(past_status)
                ret["status"] = past_status
                return jsonify(ret)

        if SERVER_ERROR_STATE:
            ret["result"] = rest_result
            ret["msg"] = state.get_error_message(state.ERR_SERVICE_UNAVAILABLE)
            ret["status"] = state.ERR_SERVICE_UNAVAILABLE
            return jsonify(ret)

        # start analyze
        # 既にキューに登録されているか確認
        queued = os.path.exists(queue_path)
        if not queued:  # 既に解析中ではない場合、解析キューに登録
            cm.queue_append(queue_path)
            # キューが回ってきたか確認し、来たら解析実行
            while True:
                cm.watchdog(youtube_id, is_parent, 1800, state.TMP_QUEUE_TIMEOUT)
                rest_pending = cm.is_path_exists(pending_path)
                rest_queue = cm.is_path_due(queue_path)
                web_download = cm.is_path_exists(dl_queue_path)
                if not rest_pending and rest_queue and not web_download:
                    if cm.is_pending_download(DL_INTERVAL):  # check pending download
                        if not MULTI_SERVER:
                            analyzer_path = f'python exec_analyze.py {url}'
                            cm.pending_append(pending_path)
                            subprocess.Popen(analyzer_path.split())
                            is_parent = True
                        else:
                            analyzer_path = f'python multi_exec_analyze.py {url}'
                            cm.pending_append(pending_path)
                            subprocess.Popen(analyzer_path.split())
                            is_parent = True
                        break

                tm.sleep(1)

        while True:  # キューが消えるまで監視
            queued = os.path.exists(queue_path)
            if queued:
                if is_parent:
                    # 親ならばpendingを監視
                    cm.watchdog(youtube_id, is_parent, 300, state.TMP_ANALYZE_TIMEOUT)
                else:
                    # 子ならばqueueを監視
                    cm.watchdog(youtube_id, is_parent, 2160, state.TMP_QUEUE_TIMEOUT)
                tm.sleep(1)
                continue
            else:  # 解析が完了したら、そのキャッシュJSONを返す
                cache = cm.queue_cache_check(youtube_id)
                if cache:
                    title, time_line, time_line_enemy, time_data, total_damage, debuff_value, past_status = cache
                    rest_result = get_rest_result(title, time_line, time_line_enemy, time_data, total_damage,
                                                  debuff_value)

                    status = past_status
                    break
                else:  # キャッシュ未生成の場合
                    # キャッシュを書き出してから解析キューから削除されるため、本来起こり得ないはずのエラー
                    status = state.TMP_UNEXPECTED
                    break

    ret["result"] = rest_result
    ret["msg"] = state.get_error_message(status)
    ret["status"] = status
    return jsonify(ret)

if __name__ == "__main__":
    app.run()


    # github üzerinden yapılan arama sonuçlarını consola yazdırma

from selenium import webdriver
from selenium.webdriver.common.keys import Keys
import time

driver = webdriver.Chrome()

url = "https://github.com"

driver.get(url)

searchInput = driver.find_element_by_name("q")
time.sleep(1)
print("\n" + driver.title + "\n")

searchInput.send_keys("python")
time.sleep(1)

searchInput.send_keys(Keys.ENTER)
time.sleep(2)

result = driver.find_elements_by_css_selector(".repo-list-item div div a")

for element in result:
    print(element.text)

time.sleep(2)

driver.close()