import numpy as np
# Data processing
from sklearn.metrics import (accuracy_score, log_loss, precision_score,
        recall_score, average_precision_score, precision_recall_curve, auc)
from deap.tools import hypervolume
from metrics import *
from clean import clean_dataset
from sklearn.model_selection import train_test_split
import gerryfair
import pdb
from itertools import chain
import os
import matplotlib
matplotlib.rcParams['pdf.fonttype'] = 42

def setup_data(dataset, attributes, seed):

    print('setting up data...')
    dataname = dataset.split('/')[-1].split('.')[0]
    print('dataset:',dataname)
    print('setting up data...')
    X, X_prime, y = clean_dataset(dataset, attributes, centered=True)
    X_train, X_test, X_prime_train, X_prime_test, y_train, y_test = \
        train_test_split(X, X_prime, y, test_size = 0.5, random_state=seed)
    print('positive labels in test:',np.sum(y_test==1))

    sens_df = pd.read_csv(attributes)
    sens_cols = [str(c) for c in sens_df.columns if sens_df[c][0] == 1]

    return X_train, X_test, X_prime_train, X_prime_test, y_train, y_test, sens_cols

def evaluate_model(X, X_prime, y, predictions, probabilities):
    """returns metrics for comparison for a single model"""
    auditor_fp = gerryfair.model.Auditor(X_prime, y.values, 'FP')
    auditor_fn = gerryfair.model.Auditor(X_prime, y.values, 'FN')
    _,auditor_fp_violation = auditor_fp.audit(predictions)
    _,auditor_fn_violation = auditor_fn.audit(predictions)
#     print('mean_marginal_fairness...')
#     mean_marg_unfairness = mean_marginal_unfairness(false_positives(y,
#                                                       predictions),
#                                                       X_prime)
#     print('max_marginal_fairness...')
#     max_marg_unfairness = max_marginal_unfairness(false_positives(y,
#                                                       predictions),
#                                                       X_prime)
#     print('mean_subgroup_fairness...')
#     mean_sub_unfairness = mean_subgroup_unfairness(false_positives(y,
#                                                       predictions),
#                                                       X_prime)
#     print('max_subgroup_fairness...')
#     max_sub_unfairness = max_subgroup_unfairness(false_positives(y,
#                                                       predictions),
#                                                       X_prime)
    accuracy = accuracy_score(y,predictions)
    fpr = np.mean(false_positives(y,predictions))
    logloss = log_loss(y, probabilities)
    mae = MAE(y, probabilities)
    if np.array(predictions==0).all():
        prec, recall, aps, auc_prc = 0, 0, 0, 0
    else:
        prec = precision_score(y, predictions)
        recall = recall_score(y, predictions)
        aps = average_precision_score(y, probabilities)
        pc, rc, _ = precision_recall_curve(y, probabilities)
        auc_prc = auc(rc,pc)

    scores = {
#             'auditor_group':performance[0],
            'auditor_fp_violation':auditor_fp_violation,
            'auditor_fn_violation':auditor_fn_violation,
#             'mean_subgroup_unfairness':mean_sub_unfairness,
#             'max_subgroup_unfairness':max_sub_unfairness,
#             'mean_marginal_unfairness':mean_marg_unfairness,
#             'max_marginal_unfairness':max_marg_unfairness,
            'accuracy':accuracy,
            'fpr':fpr,
            'logloss':logloss,
            'mae':mae,
            'precision':prec,
            'recall':recall,
            'ave_precision_score':aps,
            'auc_prc':auc_prc
           }

    return scores

# PARETO FRONT TOOLS
def check_dominance(p1,p2):

    flag1 = 0
    flag2 = 0

    for o1,o2 in zip(p1,p2):
        if o1 < o2:
            flag1 = 1
        elif o1 > o2:
            flag2 = 1

    if flag1==1 and flag2 == 0:
        return 1
    elif flag1==0 and flag2 == 1:
        return -1
    else:
        return 0

def front(obj1,obj2):
    """return indices from x and y that are on the Pareto front."""
    rank = []
    assert(len(obj1)==len(obj2))
    n_inds = len(obj1)
    front = []

    for i in np.arange(n_inds):
        p = (obj1[i],obj2[i])
        dcount = 0
        dom = []
        for j in np.arange(n_inds):
            q = (obj1[j],obj2[j])
            compare = check_dominance(p,q)
            if compare == 1:
                dom.append(j)
#                 print(p,'dominates',q)
            elif compare == -1:
                dcount = dcount +1
#                 print(p,'dominated by',q)

        if dcount == 0:
#             print(p,'is on the front')
            front.append(i)

#     f_obj1 = [obj1[f] for f in front]
    f_obj2 = [obj2[f] for f in front]
#     s1 = np.argsort(np.array(f_obj1))
    s2 = np.argsort(np.array(f_obj2))
#     front = [front[s] for s in s1]
    front = [front[s] for s in s2]

    return front
# metrics
fair_metrics = {
    'auditor_fp_violation':'Audit FP Violation $\gamma$',
    'auditor_fn_violation':'Audit FN Violation $\gamma$',
#     'mean_subgroup_unfairness':'Mean Subgroup Unfairness',
#     'max_subgroup_unfairness':'Max Subgroup Unfairness',
#     'max_marginal_unfairness':'Max Marginal Unfairness',
#     'mean_marginal_unfairness':'Mean Marginal Unfairness',
    }
loss_metrics = {
#     'fpr':'False Positive Rate',
    'accuracy':'1-Accuracy',
#     'logloss':'Log Loss',
#     'mae':'Mean Absolute Error',
#     'precision':'1-Precision',
#     'recall':'1-Recall',
    'ave_precision_score':'1-Average Precision Score',
    'auc_prc':'1 - Area Under Precision-Recall Curve'
    }
reverse_metrics = ['accuracy','precision','recall','ave_precision_score','auc_prc']
method_nice = {
    'gerryfair':'GerryFair',
    'gerryfair_xgb':'GerryFairGB',
    'feat_lex':'LEX',
    'feat_tourn':'Tourn',
    'feat_random_p100_g0':'Random0',
    'feat_random_p100_g100':'Random100',
    # 'feat_flex':'Flex',
    'feat_flex2':'FLEX',
    'feat_nsga2':'NSGA2',
    # 'feat_flex_nsga2':'FLEX-NSGA2',
    'feat_flex2_nsga2':'FLEX-NSGA2',
}
# Hypervolume tools
from deap.tools._hypervolume import pyhv
# compute hypervolumes of the Pareto front

def get_hypervolume(perf, dataset_name, xname, yname, reverse_x=False,
		    reverse_y = False):

    x_vals = {'train':[], 'test':[]}
    y_vals = {'train':[], 'test':[]}

    for i,p in enumerate(perf):
        for t in ['train','test']:
            x_vals[t].append(p[t][xname])
            y_vals[t].append(p[t][yname])

    if reverse_x:
        for t in ['train','test']:
            x_vals[t] = [-x for x in x_vals[t]]
    if reverse_y:
        for t in ['train','test']:
            y_vals[t] = [-y for y in y_vals[t]]
    for t in ['train','test']:
        x_vals[t] = np.array(x_vals[t])
        y_vals[t] = np.array(y_vals[t])

    metric = 'hv('+xname+':'+yname+')'
    hv = {'train':{}, 'test':{}}
    for t in ['train','test']:
        PF = front(x_vals[t],y_vals[t])
        pf_x = [x_vals[t][i] for i in PF]
        pf_y = [y_vals[t][i] for i in PF]
        hv[t] = pyhv.hypervolume([(xi,yi) for xi,yi in zip(pf_x,pf_y)],
                                                  ref=np.array([1,1]))
    return [{'train':True, metric:hv['train']},
            {'train':False, metric:hv['test']}]

def get_hypervolumes(perf, dataset_name):
    hv = []
    for f,_ in fair_metrics.items():
        for L,_ in loss_metrics.items():
            hv += get_hypervolume(perf,dataset_name, f, L,
			reverse_y = L in reverse_metrics)

    return hv

# Pareto front plots
import matplotlib.pyplot as plt
def pareto_plot(perf,dataset_name,xname,yname,xname_nice,yname_nice,
               reverse_x = False, reverse_y = False):
    h = plt.figure()
    x_vals = {'train':[], 'test':[]}
    y_vals = {'train':[], 'test':[]}

    for i,p in enumerate(perf):
        for t in ['train','test']:
            x_vals[t].append(p[t][xname])
            y_vals[t].append(p[t][yname])

    if reverse_x:
        for t in ['train','test']:
            x_vals[t] = [-x for x in x_vals[t]]
    if reverse_y:
        for t in ['train','test']:
            y_vals[t] = [-y for y in y_vals[t]]
    for t in ['train','test']:
        x_vals[t] = np.array(x_vals[t])
        y_vals[t] = np.array(y_vals[t])
    s = np.argsort(y_vals['train'])
    plt.plot(x_vals['train'][s],y_vals['train'][s], '--b',marker='.',label='train')
    plt.plot(x_vals['test'][s],y_vals['test'][s], '--r',marker='x', label='test')
    for i in np.arange(len(x_vals['train'])):
        plt.plot([x_vals[t][i] for t in ['train','test']],
                 [y_vals[t][i] for t in ['train','test']],
                 ':',alpha=0.5,label='_no_legend')
    plt.legend()

    plt.title(dataset_name)
    plt.xlabel(xname_nice)
    plt.ylabel(yname_nice)
    return h

def pareto_compare_plot(perf1,perf2,dataset_name,xname,yname,xname_nice,
        yname_nice, reverse_x = False, reverse_y = False, rdir=''):
    h = plt.figure()

    model1 = perf1[0]['method']
    model2 = perf2[0]['method']

    seed = perf1[0]['seed']
    assert seed == perf2[0]['seed']
    seed = str(seed)

    x_vals = {model1:{'train':[], 'test':[]},
              model2:{'train':[], 'test':[]}}
    y_vals = {model1:{'train':[], 'test':[]},
              model2:{'train':[], 'test':[]}}

    for perf,m in zip([perf1,perf2],[model1,model2]):
        for i,p in enumerate(perf):
            for t in ['train','test']:
                x_vals[m][t].append(p[t][xname])
                y_vals[m][t].append(p[t][yname])

    if reverse_x:
        # print('reversing x for ',xname)
        for m in [model1,model2]:
            for t in ['train','test']:
                x_vals[m][t] = [-x for x in x_vals[m][t]]
    if reverse_y:
        # print('reversing y for ',yname)
        for m in [model1,model2]:
            for t in ['train','test']:
                y_vals[m][t] = [1-y for y in y_vals[m][t]]

    # get the pareto front of the combined solutions
    all_x_vals_test = x_vals[model1]['test'] + x_vals[model2]['test']
    all_y_vals_test = y_vals[model1]['test'] + y_vals[model2]['test']
#     print('all_x_vals_test:',all_x_vals_test)
#     print('all_y_vals_test:',all_y_vals_test)
    PF = front(all_x_vals_test,all_y_vals_test)
    pf_test_x_vals = [all_x_vals_test[i] for i in PF]
    pf_test_y_vals = [all_y_vals_test[i] for i in PF]

    for m in [model1,model2]:
        for t in ['train','test']:
            x_vals[m][t] = np.array(x_vals[m][t])
            y_vals[m][t] = np.array(y_vals[m][t])

    pf_x = {}
    pf_y = {}
    for m in [model1,model2]:
        pf_x[m] = {'train':[],'test':[]}
        pf_y[m] = {'train':[],'test':[]}
        for t in ['train','test']:
            pf_tmp = front(x_vals[m][t],y_vals[m][t])
            pf_x[m][t] = [x_vals[m][t][i] for i in pf_tmp]
            pf_y[m][t] = [y_vals[m][t][i] for i in pf_tmp]

    # make plots!

    plt.plot(pf_test_x_vals, pf_test_y_vals, '-k', linewidth=2,alpha=0.2,
            label= '_no_legend')

    for m,c, in zip([model1,model2],['r','b']):
#         plt.scatter(x_vals[m]['train'],y_vals[m]['train'],color=c,marker='.',label=m+' train',
#                    alpha=0.2)
        plt.plot(pf_x[m]['train'],pf_y[m]['train'],':.',color=c,label=m+' train',#label='_no_legend',
                   alpha=0.3)
#         plt.scatter(x_vals[m]['test'],y_vals[m]['test'], color=c,marker='x', label=m+' test')
        plt.plot(pf_x[m]['test'],pf_y[m]['test'],'--x',color=c,label=m+' test', #label='_no_legend',
                   alpha=0.5)

#     print('PF:',PF)
    plt.scatter(pf_test_x_vals, pf_test_y_vals, marker='o', facecolor='', edgecolor='k',
            s=200, label= 'Pareto front',alpha=0.5)
#     for i in np.arange(len(x_vals['train'])):
#         plt.plot([x_vals[t][i] for t in ['train','test']],
#                  [y_vals[t][i] for t in ['train','test']],
#                  ':',alpha=0.5,label='_no_legend')
    leg = plt.legend(loc=[1.01, 0.5])

    plt.title(dataset_name)
    plt.xlabel(xname_nice)
    plt.ylabel(yname_nice)

    plt.tight_layout()
    if not os.path.exists('../paper/figs/pareto/'+rdir):
        os.mkdir('../paper/figs/pareto/'+rdir)
    savename = ('../paper/figs/pareto/'+rdir
            +'/pareto_compare_'+xname+'-'+yname+'_'
            +'_'+model1+'-'+model2
            +'_'+dataset_name+'_'+seed+'.png')
    print('saving ',savename)
    plt.savefig(savename, bbox_extras=[leg], dpi=400)

    # return hypervolumes
    return h

def pareto_multicompare_plot(perfs,dataset_name,xname,yname,xname_nice,
        yname_nice, reverse_x = False, reverse_y = False, rdir=''):
    h = plt.figure()

    models = [p[0]['method'] for p in perfs]
    # sort results by name
    s = np.argsort([method_nice[m] for m in models])
    models = [models[i] for i in s]
    perfs = [perfs[i] for i in s]

    seed = perfs[0][0]['seed']
    for p in perfs:
        assert seed == p[0]['seed']
    seed = str(seed)

    x_vals = {m:{'train':[], 'test':[]} for m in models}
    y_vals = {m:{'train':[], 'test':[]} for m in models}

    for perf,m in zip(perfs,models):
        for i,p in enumerate(perf):
            for t in ['train','test']:
                x_vals[m][t].append(p[t][xname])
                y_vals[m][t].append(p[t][yname])

    if reverse_x:
        # print('reversing x for ',xname)
        for m in models:
            for t in ['train','test']:
                x_vals[m][t] = [-x for x in x_vals[m][t]]
    if reverse_y:
        # print('reversing y for ',yname)
        for m in models:
            for t in ['train','test']:
                y_vals[m][t] = [1-y for y in y_vals[m][t]]

    # get the pareto front of the combined solutions
    all_x_vals_test = list(chain.from_iterable(
        x_vals[m]['test'] for m in models))
    all_y_vals_test = list(chain.from_iterable(
        y_vals[m]['test'] for m in models))
#     print('all_x_vals_test:',all_x_vals_test)
#     print('all_y_vals_test:',all_y_vals_test)
    PF = front(all_x_vals_test,all_y_vals_test)
    pf_test_x_vals = [all_x_vals_test[i] for i in PF]
    pf_test_y_vals = [all_y_vals_test[i] for i in PF]

    for m in models:
        for t in ['train','test']:
            x_vals[m][t] = np.array(x_vals[m][t])
            y_vals[m][t] = np.array(y_vals[m][t])

    pf_x = {}
    pf_y = {}
    for m in models:
        pf_x[m] = {'train':[],'test':[]}
        pf_y[m] = {'train':[],'test':[]}
        for t in ['train','test']:
            pf_tmp = front(x_vals[m][t],y_vals[m][t])
            pf_x[m][t] = [x_vals[m][t][i] for i in pf_tmp]
            pf_y[m][t] = [y_vals[m][t][i] for i in pf_tmp]

    # make plots!

    plt.plot(pf_test_x_vals, pf_test_y_vals, '-k', linewidth=4,alpha=0.1,
            label= '_no_legend')
    cmap = plt.cm.get_cmap('Spectral',len(models))
    markers = ['x','s','^','d','*','+','v','>','<','D']
    # pdb.set_trace()
    for c,m in enumerate(models):
#         plt.scatter(x_vals[m]['train'],y_vals[m]['train'],color=c,marker='.',label=m+' train',
#                    alpha=0.2)
        # plt.plot(pf_x[m]['train'],pf_y[m]['train'],':.',color=cmap(c),
        #         label=m+' train',#label='_no_legend',
        #         alpha=0.3)
#         plt.scatter(x_vals[m]['test'],y_vals[m]['test'], color=c,marker='x', label=m+' test')
        plt.plot(pf_x[m]['test'],pf_y[m]['test'],markers[c],
                color=cmap(c),#markerfacecolor='none',
                label=method_nice[m], #label='_no_legend',
                   alpha=1.0)
        plt.plot(pf_x[m]['test'],pf_y[m]['test'],'--',color=cmap(c),
                label='_no_legend',
                   alpha=0.7)

#     print('PF:',PF)
    plt.scatter(pf_test_x_vals, pf_test_y_vals, marker='o', facecolor='',
            edgecolor='k', s=200, label= 'Pareto front',alpha=0.5)
#     for i in np.arange(len(x_vals['train'])):
#         plt.plot([x_vals[t][i] for t in ['train','test']],
#                  [y_vals[t][i] for t in ['train','test']],
#                  ':',alpha=0.5,label='_no_legend')
    leg = plt.legend()#loc=[1.01, 0.5])

    plt.title(dataset_name)
    plt.xlabel(xname_nice)
    plt.ylabel(yname_nice)

    plt.tight_layout()
    if not os.path.exists('../paper/figs/pareto/'+rdir):
        os.mkdir('../paper/figs/pareto/'+rdir)
    savename = ('../paper/figs/pareto/'+rdir
            +'/pareto_multicompare_'+xname+'-'+yname+'_'
            +'_'+'-'.join([m for m in models])
            +'_'+dataset_name+'_'+seed+'.png')
    print('saving ',savename)
    plt.savefig(savename, bbox_extras=[leg], dpi=400)

    # return hypervolumes
    return h

def pareto_plots(perf, dataset_name):
    for f,flabel in fair_metrics.items():
        for L,Llabel in loss_metrics.items():
            pareto_plot(perf, dataset_name, f, L, flabel, Llabel)

def pareto_compare_plots(perf1,perf2, dataset_name):
    for f,flabel in fair_metrics.items():
        for L,Llabel in loss_metrics.items():
            pareto_compare_plot(perf1,perf2, dataset_name, f, L, flabel, Llabel,
                               reverse_y = L in reverse_metrics)

def pareto_multicompare_plots(perfs, dataset_name, fm = fair_metrics,
        lm = loss_metrics, rdir=''):
    for f,flabel in fm.items():
        for L,Llabel in lm.items():
            pareto_multicompare_plot(perfs, dataset_name, f, L, flabel, Llabel,
                               reverse_y = L in reverse_metrics, rdir=rdir)


#!/usr/bin/env python
import sys
import time
from typing import Any

import requests
from rich.progress import BarColumn, Progress

URL = "https://www.blaseball.com/database/giftProgress"

def read_gift() -> dict[str, Any]:
    try:
        data = requests.get(URL).json()
    except requests.exceptions.ConnectionError:
        # Uh-oh, retry?
        time.sleep(10)
        return read_gift()

    team_id = sys.argv[1]
    wishes = sorted([
        (float(gift["percent"]), gift["bonus"])
        for gift in data["teamWishLists"][team_id]
    ], reverse=True)
    return {
        "complete": data["teamProgress"][team_id]["total"],
        "progress": data["teamProgress"][team_id]["toNext"],
        "items": wishes,
    }

def update(progress: Progress, overall: int, items: dict[str, tuple[int, str]], stats: dict[str, Any]) -> None:
    progress.update(
        overall,
        description=f"{stats['complete']} Gifts Obtained",
        completed=stats["progress"],
    )
    for item in stats["items"]:
        progress.update(
            items[item[1]],
            completed=item[0],
        )

def main() -> None:
    progress = Progress(
        "[progress.description]{task.description}",
        BarColumn(),
        "[progress.percentage]{task.percentage:>3.0f}%",
    )
    with progress:
        stats = read_gift()
        overall = progress.add_task(f"{stats['complete']} Gifts Obtained", total=1)
        gifts = {}
        for gift in stats["items"]:
            gifts[gift[1]] = progress.add_task(gift[1], total=1)
        while True:
            update(progress, overall, gifts, read_gift())

            try:
                time.sleep(60)
            except KeyboardInterrupt:
                break

if __name__ == "__main__":
    main()


#!/usr/bin/python

# Python template 1.0 by Peter Hosey
# 2007-08-02

#######################
# EDIT THIS PART HERE #
#######################

"""
Usage: HyperLink Helper - format a link from the clipboard

This service recalibrates the framistan using the geeblefritzer.
(Write your own description, and replace the description above with the description you created, and delete this line.)
"""
import re
import urllib2
import chardet

def get_url_title(url):
	try:
		req = urllib2.Request(url, headers={'User-Agent' : "Sublime Text 2 Hyperlink Helper"})
		f = urllib2.urlopen(req)
		url = f.geturl()
		# decoded_content = content.decode(chardet.detect(content)['encoding'])
		content = f.read()
		decoded_content = content.decode(chardet.detect(content)['encoding'])
		title = re.search(r"<title>([^<>]*)</title>", decoded_content, re.I).group(1)
		title = title.strip()
		return title
	except:
		return ""

def make_url(text):
	# convert email addresses to mailto: links
	match = re.match(r"^(mailto:)?(.*?@.*\..*)$", text)
	if match:
		return "mailto:%s" % match.group(2)
	else:
		# convert Amazon links (possibly containing affiliate codes) to canonical URLs
		match = re.match(r"^https?://www.(amazon.(?:com|co.uk|co.jp|ca|fr|de))/.+?/([A-Z0-9]{10})/[-a-zA-Z0-9_./%?=&]+$", text)
		if match:
			return "http://%s/dp/%s" % (match.group(1), match.group(2))
		else:
			# pass through other URLs untouched
			match = re.match(r"^[a-zA-Z][a-zA-Z0-9.+-]*://.*$", text)
			if match:
				return text
			else:
				# add http:// protocol to URLs without them
				match = re.match(r"^(www\..*|.*\.(com|net|org|info|[a-z]{2}))$", text)
				if match:
					return "http://%s" % text
				else:
					# pass through non-whitespace text unmodified
					match = re.match(r"^\S+$", text)
					if match:
						return text
					else:
						return "http://example.com/"

def get_clipboard():
	import subprocess
	p = subprocess.Popen(['pbpaste'], stdout=subprocess.PIPE)
	retcode = p.wait()
	data = p.stdout.read()
	return data

def main(input_text):
	# Write your documentation above, and implement your service here.

	# This statement, if left alone and unchanged, acts as a pass-through filter: The text enters the program, then exits, with nothing in between.
	url = make_url(get_clipboard().strip())
	text = input_text
	title =''
	if re.match(r"^https?://", url) and url != "http://example.com/":
		title = get_url_title(url)
	output_text = '%s' % (url, title, text)
	return output_text

################################
# OK, YOU CAN STOP EDITING NOW #
################################

if __name__ == "__main__":
	import sys

	if sys.argv[1:] == ['--help']:
		sys.exit(__doc__.strip())

	import codecs

	# Read the input.
	input_data = sys.stdin.read()
	try:
		# If it's UTF-8, this converts it to a proper Unicode string.
		input_text = codecs.utf_8_decode(input_data)[0]
	except UnicodeDecodeError:
		# Apparently, it's not UTF-8. We'll give you the raw bytes instead.
		input_text = input_data

	# Obtain the output from your function.
	output_text = main(input_text)
	# Convert it to UTF-8 for output.
	output_data = codecs.utf_8_encode(output_text)[0]
	# Write it to stdout.
	sys.stdout.write(output_data)


from enum import Enum, unique
import copy
import numpy as np

@unique
class Dirn(Enum):

    UP = (-1, 0)
    DOWN = (1, 0)
    LEFT = (0, -1)
    RIGHT = (0, 1)
    UPL = (-1, -1)
    DOWNR = (1, 1)
    UPR = (-1, 1)
    DOWNL = (1, -1)

    @classmethod
    def values(cls):
        return [cls.UP, cls.DOWN, cls.LEFT, cls.RIGHT,
                cls.UPL, cls.DOWNR, cls.UPR, cls.DOWNL]

    def opposite(self):
        for dirn in Dirn.values():
            if (-self.value[0], -self.value[1]) == (dirn.value[0], dirn.value[1]):
                return dirn

        return None

@unique
class Color(Enum):

    BLACK = "B"
    RED = "R"

    @classmethod
    def values(cls):
        return [cls.BLACK, cls.RED]

    def opposite(self):
        return Color.BLACK if self is Color.RED else Color.RED

class Chip:

    def __init__(self, row, col, color):
        self.row, self.col = row, col
        self.color = color

    def __str__(self):
        return self.color.value

class Cell:

    def __init__(self, row, col):
        self.row, self.col = row, col
        self.chip = None

    def __str__(self):
        mark = self.chip.__str__() if self.chip is not None else " "
        return "|" + mark + "|"

    def put(self, chip):
        if chip is not None:
            self.chip = chip

    def empty(self):
        return self.chip is None

class Board:

    ROW = 6
    COL = 7
    WIN_VAL = 4

    HEADER = "---------------------"

    def __init__(self):
        self.cells = [[Cell(row, col) for col in range(Board.COL)]
                      for row in range(Board.ROW)]

        self.spaces = Board.ROW * Board.COL

    @staticmethod
    def in_bounds(row, col):
        return (row >= 0 and row < Board.ROW) and (col >= 0 and col < Board.COL)

    @staticmethod
    def sameas(row1, col1, row2, col2):
        return (row1 == row2) and (col1 == col2)

    def copy_board(self):
        board_copy = Board()
        board_copy.cells = copy.deepcopy(self.cells)
        return board_copy

    #Incorrect representation of bitboard
    def to_bitboard(self, color):
        bitboard = np.zeros((Board.ROW, Board.COL))

        for row in range(Board.ROW):
            for col in range(Board.COL):
                if (not self.cells[row][col].empty() and
                    self.cells[row][col].chip.color is color):
                    bitboard[row][col] = 1

        return bitboard

    def draw(self):
        print(Board.HEADER)

        for row in range(Board.ROW):
            for col in range(Board.COL):
                print(self.cells[row][col].__str__(), end="")

            print("\n" + Board.HEADER)

    def in_moveset(self, moveset, row, col):
        return any(Board.sameas(row, col, move[0], move[1])
                   for move in moveset)

    def valid_moves(self):
        moveset = []
        row = Board.ROW - 1

        for col in range(Board.COL):
            while row >= 0 and not self.cells[row][col].empty():
                row -= 1

            if row != -1:
                moveset.append((row, col))

            row = Board.ROW - 1

        return moveset

    def update(self, row, col, color):
        self.cells[row][col].put(Chip(row, col, color))
        self.spaces -= 1

    def valid_next(self, row, col, color):
        return (Board.in_bounds(row, col) and
                not self.cells[row][col].empty() and
                self.cells[row][col].chip.color is color)

    def has_winner(self, row, col, color):
        bdirns = Dirn.values()[0::2]

        for bdirn in bdirns:
            count = 1
            odirn = bdirn.opposite()

            brow, bcol = row + bdirn.value[0], col + bdirn.value[1]
            orow, ocol = row + odirn.value[0], col + odirn.value[1]

            bvalid = self.valid_next(brow, bcol, color)
            ovalid = self.valid_next(orow, ocol, color)

            while bvalid or ovalid:
                if bvalid:
                    count += 1
                    brow, bcol = brow + bdirn.value[0], bcol + bdirn.value[1]

                    bvalid = self.valid_next(brow, bcol, color)

                if ovalid:
                    count += 1
                    orow, ocol = orow + odirn.value[0], ocol + odirn.value[1]

                    ovalid = self.valid_next(orow, ocol, color)

            if count == Board.WIN_VAL:
                return True

        return False


# for maskrcnn benchmark， written by 队友 chenjiahao and 本人
# 总共要改三个地方: 1 图片路径 2 json文件路径 3 保存json的路径
# 自注：感觉速度不太行，明显比大佬要慢很多 2019.9.16注
import os
import cv2
import json
import numpy as np
img_dic = {}
category_dic = {}

img_and_anno_root = '/mfs/home/fangyong/data/guangdong/train/'
#img_folder = 'C:/Users/Administrator/Desktop/defect_Images/'
img_folder = img_and_anno_root + 'defect_Images/'
file_list = os.listdir(img_folder)
id = 1
images = []
categories = []
annotations = []

#####################################################
NEED_ORDER = True  # clw note:如果需要保持一定的顺序,比如官方给了1代表person,2代表plane,最好用下面的方式;
                   # 反之如果没有顺序要求,或者只有一类物体如head,那么可以让 NEED_ORDER = False
                   # 中文转Unicode地址:http://tool.chinaz.com/tools/unicode.aspx
if NEED_ORDER:
    my_categories = ['\u7834\u6d1e', '\u6c34\u6e0d', '\u6cb9\u6e0d', '\u6c61\u6e0d',
                  '\u4e09\u4e1d', '\u7ed3\u5934', '\u82b1\u677f\u8df3', '\u767e\u811a', '\u6bdb\u7c92',
                  '\u7c97\u7ecf', '\u677e\u7ecf', '\u65ad\u7ecf', '\u540a\u7ecf', '\u7c97\u7ef4',
                  '\u7eac\u7f29', '\u6d46\u6591', '\u6574\u7ecf\u7ed3', '\u661f\u8df3', '\u8df3\u82b1',
                  '\u65ad\u6c28\u7eb6', '\u7a00\u5bc6\u6863', '\u6d6a\u7eb9\u6863', '\u8272\u5dee\u6863', '\u78e8\u75d5',
                  '\u8f67\u75d5', '\u4fee\u75d5', '\u70e7\u6bdb\u75d5', '\u6b7b\u76b1', '\u4e91\u7ec7',
                  '\u53cc\u7eac', '\u53cc\u7ecf', '\u8df3\u7eb1', '\u7b58\u8def', '\u7eac\u7eb1\u4e0d\u826f']
    for i in range(len(my_categories)):
        category_dic[my_categories[i]] = i + 1
        categories.append({
            "supercategory": "none",
            "id": i + 1,
            "name": my_categories[i]})
        i += 1
#####################################################

# input_txt to coco_json
print('clw:total image number: ', len(file_list))
for img_name in file_list:
    print('clw:image id: ', id)
    img_dic[img_name] = id
    img = cv2.imread(os.path.join(img_folder,img_name))
    images.append({
        "id":id,
        "file_name":img_name,
        "height":img.shape[0],
        "width":img.shape[1]})
    id += 1

id = 1
annid = 1

#####################################################
import json
#annFile = 'C:/Users/Administrator/Desktop/Annotations/gt_result.json'
annFile = img_and_anno_root + 'Annotations/anno_train.json'
file = open(annFile, "rb")
data_list = json.load(file)
data_list = sorted(data_list,key = lambda e:e['name'],reverse = True)
######################################################

# 取出data中相应文件名的文件
print('clw:total bbox number: ', len(data_list))
for data in data_list:
    print('clw:bbox id: ', annid)
    x_min = data['bbox'][0]
    y_min = data['bbox'][1]
    x_max = data['bbox'][2]
    y_max = data['bbox'][3]
    category = data['defect_name']
    #################################################
    if not NEED_ORDER:
        if category not in category_dic.keys():
            category_dic[category] = int(id)
            categories.append({
                "supercategory": "none",
                "id": id,
                "name": category})
            id += 1
    ################################################
    annotations.append({
        # "segmentation":[[points[0],points[1],points[2],points[3],points[4],points[5],points[6],points[7]]],
        "segmentation": [],
        "area": (x_max - x_min) * (y_max - y_min),
        "iscrowd": 0,
        "image_id": img_dic[data['name']],
        "bbox": [x_min, y_min, (x_max - x_min), (y_max - y_min)],
        "category_id": category_dic[category],
        "id": int(annid),
        "ignore": 0})
    annid += 1

jsonfile = {}
jsonfile['images'] = images
jsonfile['categories'] = categories
jsonfile['annotations'] = annotations
with open(img_and_anno_root + "Annotations/train.json",'w',encoding='utf-8') as f:
    json.dump(jsonfile,f)

print('clw:category_dic = ', category_dic)
print('clw:end!')


#  https://api.kite.trade/instruments
# df = pd.read_csv('master/instruments')
# df1 = df.query(
#     ' instrument_type == "EQ" & segment == "NSE" & exchange == "NSE" & lot_size == 1 & name ')
# df2 = pd.DataFrame(df1, columns=['instrument_token', 'tradingsymbol'])
# # df2 = pd.DataFrame(df1)
# df2.to_csv("master/symbols")

import pandas as pd
from datetime import datetime
import requests
import pandas as pd
import json
import talib as tal

def log(message):
    print(datetime.now().strftime('%I:%M:%S')+" : "+message)

print("===================================================")
start_time = datetime.now()
log("START")
print("===================================================")

with open("token.json") as json_file:
    json_data = json.load(json_file)
    kite_token = json_data["kite_token"]

def scanning():
    total_data_points = 0
    log("PROCESSING INSTRUMENTS SYMBOLS")
    print("-----------------------------------------------")
    df_symbols = pd.read_csv('master/instruments.csv')
    # from_date = datetime.now().strftime('%Y-%m-%d')
    # to_date = datetime.now().strftime('%Y-%m-%d')
    from_date = "2022-07-28"
    to_date = "2022-07-28"
    for index, row in df_symbols.iterrows():
        endpoint = "https://kite.zerodha.com/oms/instruments/historical/"
        code = str(row["instrument_token"])
        interval = "/5minute"
        #interval = "/minute"
        headers = {'content-type': 'application/json',
                   "authorization": kite_token}
        url = endpoint + code + interval
        params = (
            ('user_id', 'VM7727'),
            ('oi', '1'),
            ('from', str(from_date)),
            ('to', str(to_date)),
        )
        response = requests.get(
            url, timeout=600, headers=headers, params=params).json()

        if(response['data']['candles'] and response["status"] == "success"):

            df = pd.DataFrame(response['data']['candles'], columns=[
                            'timestamp', 'open', 'high', 'low', 'close', 'volume', 'oi'])

            df.to_csv("data/"+row["tradingsymbol"]+".csv")

            if(df[df.columns[0]].count()):
                log(str(index)+" => "+str(df[df.columns[0]].count()) +
                    " DATA POINTS : "+row["tradingsymbol"])
        else:
            print("No data found")
    return

scanning()

print("===================================================")
log("END")
print("===================================================")
end_time = datetime.now()
time_taken = divmod((end_time-start_time).total_seconds(), 60)[0]
print("Start Time     : "+str(start_time))
print("End Time       : "+str(end_time))
print("Time Taken     : "+str(time_taken))
print("===================================================")


from conans import ConanFile

class Magpie(ConanFile):
	settings = {
		"os": "Windows",
		"compiler": {"Visual Studio": {"version": ["17"]}},
		"build_type": ["Debug", "Release"],
		"arch": ["x86_64", "armv8"]
	}
	requires = [
		"fmt/9.1.0",
		"spdlog/1.11.0",
		"muparser/2.3.4",
		"yas/7.1.0",
		"rapidjson/cci.20220822",
		"zstd/1.5.2",
		"imgui/1.89.2",
		"parallel-hashmap/1.37",
		"kuba-zip/0.2.6"
	]
	generators = "visual_studio"
	default_options = {
		"fmt:header_only": True,
		"spdlog:header_only": True,
		"spdlog:no_exceptions": True
	}

	def imports(self):
		self.copy("imgui_impl_dx11.*", dst="../../../src/Magpie.Core", src="./res/bindings")


'''
Given an array of integers nums and an integer target, return indices of the two numbers such that they add up to target.

You may assume that each input would have exactly one solution, and you may not use the same element twice.

You can return the answer in any order.

Example 1:

Input: nums = [2,7,11,15], target = 9
Output: [0,1]
Explanation: Because nums[0] + nums[1] == 9, we return [0, 1].

Example 2:

Input: nums = [3,2,4], target = 6
Output: [1,2]

Example 3:

Input: nums = [3,3], target = 6
Output: [0,1]

Constraints:

    2 <= nums.length <= 104
    -109 <= nums[i] <= 109
    -109 <= target <= 109
    Only one valid answer exists.

'''
from typing import List

## Brute Force
class SolutionOne:
    def twoSum(self, nums: List[int], target: int) -> List[int]:
        for i in range(len(nums)):
            for idx, j in enumerate(nums):
                if nums[i] + j == target and i != idx:
                    return [i, idx]

## HashMap
class SolutionTwo:
    def twoSum(self, nums: List[int], target: int) -> List[int]:
        ht = dict()
        for idx, j in enumerate(nums):
            if ht.get(target - j) is not None:
                return [ht.get(target - j), idx]
            ht[j] = idx

if __name__ == '__main__':
    sol = SolutionOne()
    nums = [2, 7, 11, 15]
    target = 9
    output = [0, 1]
    assert output == sol.twoSum(nums=nums, target=target)


import sys
sys.path.append('../')
import numpy as np
import tensorflow as tf

import hparam as conf
import sessionWrapper as sesswrapper
from utility import dataProcess as dp
from utility import general_utility as gu
import model_zoo as mz
import loss_func as l
import matplotlib.pyplot as plt
import sklearn.preprocessing as p

tf.reset_default_graph()
c = conf.config('trial_cnn_cls').config['common']
sample_window = c['input_step'] + c['predict_step']

meta = gu.read_metafile(c['meta_file_path'])
tv_gen = dp.train_validation_generaotr()
f = tv_gen._load_data(c['src_file_path'])
data = tv_gen._selectData2array(f, ['006208'], None)

data_raw = data[1:, 50:80]
f_diff = data[1:, 50:80] - data[:-1, 50:80]
f_diff2 = f_diff[1:, :] - f_diff[:-1, :]

train_ud = np.hstack((data_raw[1:,:], data[2:,-3:]))

input_step = 20
predict_step = 5
sample_step = input_step + predict_step

data = []

for i in range(sample_step, len(train_ud)):
    data.append(train_ud[i-sample_step:i])

data = np.stack(data, axis=0)

train_data = data[:700]
validation_data = data[750:]

tdata, tlabel = np.split(train_data, [input_step], axis=1)
tlabel = tlabel[:,:,-3:]

vdata, vlabel = np.split(validation_data, [input_step], axis=1)
vlabel = vlabel[:,:,-3:]

#============Build CNN model===============

import keras as k
from keras.models import Sequential
from keras.layers import Dense, Activation, Convolution1D, MaxPooling1D, Flatten, Dropout,Reshape
from keras.optimizers import Adam
from keras.models import Model

model = Sequential()
model.add(Convolution1D(
        nb_filter = 16,
        kernel_size = 5,
        padding='same',
        input_shape=(np.shape(tdata)[1:])
))
model.add(Activation('relu'))
model.add(MaxPooling1D(stride=1))
model.add(Convolution1D(
        nb_filter = 16,
        kernel_size = 3,
        padding='same'
))
model.add(Activation('relu'))

model.add(Flatten())
model.add(Dropout(0.4))
model.add(Dense(256, activation='relu'))
model.add(Dense(15))
model.add(Reshape((5,3)))
model.add(Activation('softmax'))

model.summary()

softmax_output = Model(inputs=model.input,
                       outputs=model.layers[-1].output)

adam = Adam(lr=1e-4)
model.compile(optimizer=adam,
              loss='categorical_crossentropy',
metrics=['accuracy'])

print('Training ------------')
# Another way to train the model
history = model.fit(tdata, tlabel, epochs=500, batch_size=64,validation_data=(vdata, vlabel))

print('\nTesting ------------')
# Evaluate the model with the metrics we defined earlier
loss, accuracy = model.evaluate(vdata, vlabel)

print('\ntest loss: ', loss)
print('\ntest accuracy: ', accuracy)

final_prediction = softmax_output.predict(vdata)
final_prediction_cls = np.argmax(final_prediction, axis=-1)
v_label_cls = np.argmax(vlabel, axis=-1)

test_accuracy = np.mean(np.equal(final_prediction_cls, v_label_cls))
weighted_array = [0.1,0.15,0.2,0.25,0.3]
test_score = np.mean(np.sum(np.equal(final_prediction_cls, v_label_cls).astype(np.float32)*weighted_array*0.5, axis=1))

import matplotlib.pyplot as plt
plt.plot(history.history['loss'])
plt.plot(history.history['val_loss'])


from datetime import datetime
from json import dump as json_dump
from pathlib import Path

from bs4 import BeautifulSoup
from celery import Celery
from celery.schedules import crontab
from requests import get as req_get

URL = 'https://www.pravda.com.ua/eng/rss/'
DIR = Path(__file__).parent

app = Celery('tasks', broker='pyamqp://guest@localhost//')
app.conf.beat_schedule = {
    'scraping': {
        'task': 'tasks.scrape',
        # Scrapes every 1 minute.
        'schedule': crontab()
    }
}

@app.task
def scrape():
    '''
    Crates `output` directory if not exists. Tries to get latest articles from
    link set in global `URL` variable. Sorts info for every article and
    generates json files under `output` dir: adds `title`, `description`,
    `source`, `published_at` and `scrapped_at` attributes.
    '''
    output_dir = DIR.joinpath('output')
    output_dir.mkdir(parents=True, exist_ok=True)
    articles = []

    try:
        response = req_get(URL)
        soup = BeautifulSoup(response.content, features='xml')
        scraped_articles = soup.findAll('item')

        for a in scraped_articles:
            article = {
                'title': a.find('title').text,
                'description': a.find('description').text,
                'source': a.find('link').text,
                'published_at': a.find('pubDate').text,
                'scrapped_at': str(datetime.now()),
            }
            articles.append(article)

        saved_at = datetime.now().strftime('%Y.%m.%d-%H.%M.%S')

        with open(
            f'{DIR}/output/articles-{saved_at}.json', 'w') as out_file:
            json_dump(articles, out_file, indent=4)

        return saved_at

    except Exception as e:
        print(f'Exception:\n{e}')


import os
from flask import Flask, render_template, request
from elasticapm.contrib.flask import ElasticAPM
import logging

app = Flask(__name__)
logging.basicConfig(filename='flask.log', level=logging.DEBUG, format=f'%(asctime)s %(levelname)s %(name)s %(threadName)s : %(message)s')

app.config['ELASTIC_APM'] = {

  'SERVICE_NAME': 'app_flask',

  'SECRET_TOKEN': '',
  'DEBUG': True,

# # # Set the custom APM Server URL (default: http://localhost:8200)
  'SERVER_URL': 'http://localhost:8200',

# # # Set the service environment
  'ENVIRONMENT': 'dev',
  }

apm = ElasticAPM(app)
@app.route("/")
def hello_world():
    try:
        return "madruga boladona"
    except Exception as err:
        return app.logger.critical(err)

if __name__ == '__main__':
    app.run(host='0.0.0.0', debug = False)


from typing import List, Union, Dict, Any
import torch
import pickle
import numpy as np
import concurrent.futures

from tqdm import tqdm
from pathlib import Path
from transformers import BertModel, BertTokenizer

import utils
import logging
# ToDo; reduce the redundancy in the class methods

logger = logging

class Embedder:
    def __init__(self, tokenizer: BertTokenizer,
                 bert: BertModel,
                 IDF_dict: Dict[str, float],
                 embedding_dir: Path,
                 layers_to_use: List[int] = [12],
                 layer_combination: str = "avg",
                 idf_threshold: float = 1.5,
                 idf_weight_factor: float = 1.0,
                 not_found_idf_value: float = 0.5):
        """
        Object that provides functinoality to tokenize some input text, as well as computes the
        (potentially IDF weighted) embeddings.

        :param tokenizer:    Pretrained BertTokenizer.
        :param bert:    Pretrained BERT model - from the same BERT model as the tokenizer.
        :param IDF_dict:    Dictionary holding the pre-computed IDF weights for token indices.
        :param layers_to_use:   A list with layer indices to combine. Default is [12], meaning only the output
                                of the last layer is used.
        :param layer_combination:   In case of multiple layers, determine how to combine them. Defaults to `"avg"`,
                                    choice between (`"avg"`, `"sum"`, `"max"`).
        :param idf_threshold:    Minimum IDF weight value for the token to be included during weighting.
        :param idf_weight_factor:    Factor for multiplying IDF weights to stress/reduce the difference between high/low.
        :param not_found_idf_value:    Value for tokens that do not have a corresponding IDF weight(typically low).
        """
        self.tokenizer = tokenizer
        self.bert = bert
        self.IDF_dict = IDF_dict
        self.layers_to_use = layers_to_use
        self.layer_combination = layer_combination
        self.idf_threshold = idf_threshold
        self.idf_weight_factor = idf_weight_factor
        self.not_found_idf_value = not_found_idf_value

        self.embedding_prefix = "embeddings"
        self.embedding_dir = embedding_dir
        self.span_and_embedding_pairs = []
        self.standardised_embedding_data = []
        self.unique_spans = []
        self.new_embedding_data = []
        self.new_unique_spans = []
        self.emb_mean_fp = embedding_dir.joinpath("standardisation_mean.pkl")
        self.emb_std_fp = embedding_dir.joinpath("standardisation_std.pkl")
        self.emb_mean = "None"  # initialise to specific value, later will hold an array (without truth value blabla)
        self.emb_std = "None"

    def prepare_tokens(self, text: str) -> (List[str], List[int]):
        """
        Helper function to tokenize and ensure the same output is provided by the BertTokenizer
        (and BertWordPieceTokenizer ~ deprecated).

        :param tokenizer:    A pretrained BertTokenizer.
        :param text:    Input text.
        :return tokenized_text: List of string representations of tokens for the input text.
        :return indices:    List of token index representations of tokens for the input text.
        """
        # if type(self.tokenizer) == BertWordPieceTokenizer:
        #     encoded = self.tokenizer.encode(text)
        #     tokenized_text = encoded.tokens
        #     indices = encoded.ids
        #     return tokenized_text, indices
        if type(self.tokenizer) == BertTokenizer:
            tokenized_text = self.tokenizer.tokenize("[CLS] " + text + " [SEP]")
            indices = self.tokenizer.convert_tokens_to_ids(tokenized_text)

            if len(tokenized_text) > 512:
                # Any tokens where the index is higher than 512 (max BERT input) will be omitted
                # todo; make this adjustable for different pretrained embedding models like a longformer model
                print("Number of input tokens too long! Truncating input to 512 tokens...")
                tokenized_text, indices = tokenized_text[:512], indices[:512]

            return tokenized_text, indices
        else:
            print("You'll need to change to a BertTokenizer")
            return None  # need to raise appropriate error or quit running nicely

    def get_idf_weights_for_indices(self,
                                    tokenized_text: List[str],
                                    indices: List[int]) -> np.array:
        """
        Helper function to get the relevant IDF weights given tokenized input. Note: if an IDF value is not found, then
        the weight for that token is set to `self.not_found_idf_value`.

        :param tokenized_text:  List of tokens, currently not used. Previously used to visualise which part of the
                                input text is filtered through IDF-based filtering, and which weights are assigned.
        :param indices: Token-indices that are also used to index into the corresponding IDF values, used to retrieve
                        the corresponding IDF weights computed for a domain corpus.
        :return sw_weights: An np.array of the same length as the parameter `indices`, holding IDF weights for the
                            subword unit tokens.
        """
        sw_weights = np.ones(len(indices))
        visualised_text = ''
        for sw_id, sw in enumerate(indices):
            try:
                # set index of phrase representation to corresponding IDF value
                sw_weights[sw_id] = self.IDF_dict[str(sw)]
                visualised_text += ' ' + tokenized_text[sw_id]
            except (KeyError, ValueError) as e:

                # No IDF value found, which value do we set it to?
                sw_weights[sw_id] = self.not_found_idf_value
                visualised_text += ' ' + tokenized_text[sw_id] + '\u0336'

        return sw_weights

    def embed_text(self, text: str) -> List[torch.tensor]:
        """
        :param text:    Text to be tokenized. Expecting BERT-tokenizer, so max input of 512 tokens.

        :return weighted_term_embeddings:    The embeddings for the each of the tokens in the sentence, potentially with  subword
                                    IDF weights applied (multiplied by IDF value, mediated by `self.idf_weight_factor`).
        """
        tokenized_text, indices = self.prepare_tokens(text)

        IDF_weights = self.get_idf_weights_for_indices(tokenized_text, indices)

        segments_ids = [1] * len(indices)
        tokens_tensor = torch.tensor([indices])
        segments_tensors = torch.tensor([segments_ids])
        embedding = self.bert(tokens_tensor, segments_tensors)

        try:
            hidden_states = embedding[2]
        except IndexError:
            raise IndexError("Make sure to output hidden states; BertModel.from_pretrained(some-bert-model, "
                             "output_hidden_states=True)")

        # Group by token vectors
        token_embeddings = torch.stack(hidden_states, dim=0).squeeze(dim=1).permute(1, 0, 2)
        weighted_term_embeddings = []
        for token, IDF_w in zip(token_embeddings, IDF_weights):
            # todo -- consider running more experiments with different BERT layers and combinations
            # Combine the vectors from selected bert layers --> currently default to last layer only [12]
            if self.layer_combination == "sum":
                combined_vec = torch.sum(token.index_select(0, torch.tensor(self.layers_to_use)), dim=0)
            elif self.layer_combination == "max":
                combined_vec = torch.max(token.index_select(0, torch.tensor(self.layers_to_use)), dim=0)
            else:
                # default to self.layer_combination == "avg":
                combined_vec = torch.mean(token.index_select(0, torch.tensor(self.layers_to_use)), dim=0)

            # Weight the vector by IDF value
            if IDF_w > self.idf_threshold and self.not_found_idf_value > 0:  # avoid multiplying by 0
                if self.idf_weight_factor > 0:
                    weighted_term_embeddings.append(combined_vec * (self.idf_weight_factor * IDF_w))
                else:
                    weighted_term_embeddings.append(combined_vec)

        return weighted_term_embeddings

    def combine_token_embeddings(self, embeddings: List[torch.tensor]) -> torch.tensor:
        """
        Calls the embedding function for a span, stacking the embeddings for each token and computing the mean over
        the token-length dimension.

        :param embeddings:   List of tensors for each of the tokens in a given span.
        :return embedding:   Single embedding of the span, as an average over the constituent token embeddings.
        """
        weighted_token_embeddings = torch.stack(embeddings)
        detached_embeddings = weighted_token_embeddings.detach().numpy()
        # Standardise PLM representation
        if type(self.emb_mean) == str and self.emb_mean_fp.exists():
            self.emb_mean = pickle.load(open(self.emb_mean_fp, 'rb'))
            self.emb_std = pickle.load(open(self.emb_std_fp, 'rb'))

        standardised_embeddings = (detached_embeddings - self.emb_mean) / self.emb_std
        return np.mean(standardised_embeddings, axis=0)   # average over the tokens for now. old >> torch.mean(x, dim=0)

    def embed_a_span(self, span: str) -> torch.tensor:
        """
        Use to simply embed a span, BEFORE the mean and std of all spans are computed
        """
        embeddings = self.embed_text(span)
        try:
            return span, torch.stack(embeddings).detach().numpy().squeeze()
        except RuntimeError:
            # can happen if the tensor for the span is empty somehow
            logger.info(f"Empty tensor! Not sure why, but will drop the span: {span}")

    def embed_and_normalise_span(self, span: str) -> torch.tensor:
        """
        Use to embed new spans, AFTER the mean and std of all spans are computed.
        Returns a tuple: (span, normalised_embedding)
        """
        embeddings = self.embed_text(span)
        try:
            return span, self.combine_token_embeddings(embeddings)
        except RuntimeError:
            # can happen if the tensor for the span is empty or causes issues somehow
            print(f"Empty tensor! Not sure why, but will drop the span: {span}")
            return None, None

    def normalise_embeddings(self):
        """
        Creates a single file with all embeddings, in the meantime standardising the embeddings to improve the
        representations (Timkey & van Schijndel, 2021).
        """
        unique_spans, unique_embeddings = zip(*self.span_and_embedding_pairs)
        self.unique_spans = unique_spans
        unique_spans_fp = self.embedding_dir.joinpath("unique_spans.pkl")
        if not unique_spans_fp.exists():
            with open(unique_spans_fp, 'wb') as f:
                pickle.dump(unique_spans, f)

        standardised_classifier_data_fp = self.embedding_dir.joinpath("standardised_embeddings.pkl")
        if not standardised_classifier_data_fp.exists():
            print(
              f"Normalising and combining computed/existing {len(unique_spans)} embeddings from files into single file"
            )
            # we average over the token embeddings in a term
            unique_clustering_data = np.stack(
                [np.mean(e, axis=0) if len(e.shape) > 1 else e for e in unique_embeddings])

            # standardise the unique clustering data, as suggested by https://github.com/wtimkey/rogue-dimensions
            self.emb_mean = unique_clustering_data.mean(axis=0)
            self.emb_std = unique_clustering_data.std(axis=0)
            pickle.dump(self.emb_mean, open(self.embedding_dir.joinpath("standardisation_mean.pkl"), 'wb'))
            pickle.dump(self.emb_std, open(self.embedding_dir.joinpath("standardisation_std.pkl"), 'wb'))

            # Store the span and normalised embedding pairs
            # Note: could remove all the embedding files (keeping them though)
            normalised_embeddings = (unique_clustering_data - self.emb_mean) / self.emb_std
            self.standardised_embedding_data = [(s, e) for s, e in zip(self.unique_spans, normalised_embeddings)]
            pickle.dump(self.standardised_embedding_data, open(standardised_classifier_data_fp, 'wb'))
        else:
            self.standardised_embedding_data = pickle.load(open(standardised_classifier_data_fp, 'rb'))
            print(f"Loaded previously computed normalised embedding files.")

    def embed_fore_and_background_terms(self,
                                        max_num_cpu_threads: int = 4,
                                        subset_size: int = 1000,
                                        list_of_terms: List[str] = None,
                                        prefix: str = 'embeddings'
                                        ):
        """
        This is split into subsets so we don't overload memory (adjust values if needed). Once computed, all
        of the embeddings will be stored normalised. Normalised embeddings and corresponding spans can be accessed
         through `self.standardised_embedding_data`.

        :param max_num_cpu_threads:     Number of concurrent threads.
        :param subset_size: Number of terms embedded at a time, this is the memory bottleneck.
        :param prefix:  Prefix to the name given to the embeddings.
        :param list_of_terms:   A list of spans/terms that you'd like to embed. If None, then the foreground and
                                background terms from the classifier directory will be embedded.
        """
        term_subsets = utils.split_list(list_of_terms, subset_size)
        if prefix != 'embeddings':
            self.embedding_prefix = prefix

        embedding_files = [f for f in self.embedding_dir.glob(f'{prefix}*.pkl')]

        if len(embedding_files) == len(term_subsets):
            for e in embedding_files:
                self.span_and_embedding_pairs += pickle.load(open(e, 'rb'))
        else:
            print(f"Preparing embeddings for {len(list_of_terms)} spans, in groups of: {subset_size}")
            subset_idx = 0  # iterator index outside of tqdm
            for subset in tqdm(term_subsets):
                subset_embeddings = []
                subset_file_name = self.embedding_dir.joinpath(f"{prefix}_part_{subset_idx}.pkl")
                subset_idx += 1
                if subset_file_name.exists():
                    continue

                with concurrent.futures.ThreadPoolExecutor(max_workers=max_num_cpu_threads) as executor:
                    futures = [executor.submit(self.embed_a_span, subset[idx]) for idx in range(len(subset))]

                subset_embeddings += [f.result() for f in futures if f.result()]

                with open(subset_file_name, 'wb') as f:
                    pickle.dump(subset_embeddings, f)

            # Once all embeddings are created; combine them in span_and_embedding_pairs
            embedding_files = [f for f in self.embedding_dir.glob(f'{prefix}*.pkl')]
            for e in embedding_files:
                self.span_and_embedding_pairs += pickle.load(open(e, 'rb'))

        self.normalise_embeddings()

    def embed_large_number_of_new_terms(self,
                                        max_num_cpu_threads: int = 4,
                                        subset_size: int = 1000,
                                        prefix: str = 'new_terms',
                                        list_of_terms: List[str] = None
                                        ):
        """
        This function is used to embed new spans that are found during KG creation.

        :param max_num_cpu_threads:     Number of concurrent threads.
        :param subset_size: Number of terms embedded at a time, this is the memory bottleneck.
        :param prefix:  Prefix to the name given to the embeddings.
        :param list_of_terms:   A list of spans/terms that you'd like to embed. If None, then the foreground and
                                background terms from the classifier directory will be embedded
        """
        # Checks which of the embeddings for the clustering cluster_data already exist, so they can be re-used
        unique_old_spans, _ = zip(*self.standardised_embedding_data)
        unique_new_spans = [t for t in list_of_terms if t not in unique_old_spans]

        term_subsets = utils.split_list(unique_new_spans, subset_size)
        embedding_files = [f for f in self.embedding_dir.glob(f'{prefix}*.pkl')]
        new_span_and_embedding_pairs = []
        if len(embedding_files) == len(term_subsets):
            for e in embedding_files:
                new_span_and_embedding_pairs += pickle.load(open(e, 'rb'))
        else:
            print(f"Preparing embeddings for {len(unique_new_spans)} spans, in groups of: {subset_size}")
            subset_idx = 0  # iterator index outside of tqdm
            for subset in tqdm(term_subsets):
                subset_embeddings = []
                subset_file_name = self.embedding_dir.joinpath(f"{prefix}_part_{subset_idx}.pkl")
                subset_idx += 1
                if subset_file_name.exists():
                    # already computed previously
                    continue

                # NOTE: the spans are embedded AND normalised in this method.
                with concurrent.futures.ThreadPoolExecutor(max_workers=max_num_cpu_threads) as executor:
                    futures = [executor.submit(self.embed_and_normalise_span, subset[idx]) for idx in
                               range(len(subset))]

                subset_embeddings += [f.result() for f in futures if f.result()]

                with open(subset_file_name, 'wb') as f:
                    pickle.dump(subset_embeddings, f)

            # Once all embeddings are created; combine them in span_and_embedding_pairs
            embedding_files = [f for f in self.embedding_dir.glob(f'{prefix}*.pkl')]
            for e in embedding_files:
                new_span_and_embedding_pairs += pickle.load(open(e, 'rb'))

        self.new_embedding_data += new_span_and_embedding_pairs


# encoding: utf-8

import sys
sys.path.append('../')
import config as config
import pymysql
import MySQLdb as db
import MySQLdb.cursors as cursors

DBNAME = config.db_name
DBHOST = config.db_host
DBUSER = config.db_user
DBPWD = config.db_pwd

class database:

    def __init__(self,dict=False):
        self.conn = db.connect(DBHOST, DBUSER, DBPWD, DBNAME, charset='utf8mb4')
        self.cur = self.conn.cursor(cursors.DictCursor if dict is True else None)
        self.errormsg = ''

    def cur(self):
        return self.cur()

    def fetch_all(self, sql):
        res = ''
        if self.conn:
            try:
                self.cur.execute(sql)
                res = self.cur.fetchall()
            except Exception:
                res = False
        return res

    def commit(self):
        self.conn.commit()

    def save(self, sql):
        flag = False
        if self.conn:
            try:
                self.cur.execute(sql)
                self.conn.commit()
            except Exception as e:
                # print("error sql:" + sql)
                self.errormsg = str(e)
                flag = False
            else:
                flag = True
        return flag

    def get_last_id(self):
        if self.cur.lastrowid:
            self.conn.commit()
            return int(self.cur.lastrowid)  # 最后插入行的主键ID
        else:
            return None

    def __del__(self):  # 析构函数
        self.cur.close()
        self.conn.close();  # 关闭数据库连接


"""
__author__ = 'Christopher Fagiani'
"""
import logging
import argparse
import sys
from engine import workout_controller

def main(args):
    controller = None
    configure_logger(args.debug)
    try:
        controller = workout_controller.WorkoutController(args.config)
        if args.headless:
            controller.calibrate_orientation()
            workout_stats = controller.start_workout(args.mode, args.time)
            print("Workout Complete\n=================")
            print("Hits: {hits}\nMisses: {misses}".format(hits=len(workout_stats.correct_hits),
                                                          misses=len(
                                                              workout_stats.incorrect_hits) + workout_stats.timeouts
                                                          ))
        else:
            from ui.ui_server import SparpiServer
            server = SparpiServer(args.port, controller)
            server.start()

    except KeyboardInterrupt:
        print("shutting down")
    finally:
        if controller is not None:
            controller.cleanup()

def configure_logger(is_debug):
    root = logging.getLogger()
    if is_debug:
        root.setLevel(logging.DEBUG)
    else:
        root.setLevel(logging.INFO)
    ch = logging.StreamHandler(sys.stdout)
    ch.setLevel(logging.INFO)
    formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
    ch.setFormatter(formatter)
    root.addHandler(ch)

if __name__ == "__main__":
    argparser = argparse.ArgumentParser(description="Uses LEDs to indicate which side of a punching bag to hit")
    argparser.add_argument("-c", "--config", metavar='config', default="sparpi.ini", help='Configuration file to use',
                           dest='config')
    argparser.add_argument("-w", "--workout", metavar='workout', default='random', choices=['random', 'alternating'],
                           help='Desired workout mode. Only used when running headless.', dest='mode')
    argparser.add_argument("-t", "--time", metavar='time', default=2, type=float,
                           help="Time in minutes for the workout. Only used when running headless.")
    argparser.add_argument("-d", "--debug", action="store_true", default=False)
    argparser.add_argument("-p", "--port", type=int, default=80,
                           help="Port on which to run the UI. Ignored if headless.")
    argparser.add_argument("-hl", "--headless", default=False, action="store_true",
                           help="If true, no ui server will be started")
    main(argparser.parse_args())


from __future__ import print_function

import json
import re
import sys

sys.path.append("..")
from tokenizer import RuleBasedTokenizer

USAGE = "Usage: ./ner_output_to_mrp.py filename"

if __name__ == "__main__":
    import argparse

    parser = argparse.ArgumentParser()
    parser.add_argument("--ner", default=None, type=str, help="NE annotated text file.")
    parser.add_argument("--mrp", default=None, type=str, help="MRP file.")
    parser.add_argument("--mode", default=None, type=str, help="dm|eds|psd|ucca|amr")
    args = parser.parse_args()

    if args.mode not in [ "dm", "eds", "psd", "ucca", "amr" ]:
        raise ValueError("--mode must be one of dm|eds|psd|ucca|amr")

    tokenizer = RuleBasedTokenizer(finegrained = True if args.mode == "ucca" else False)

    datas = []
    with open(args.mrp, "r", encoding="utf-8") as mrp_file:
        for line in mrp_file:
            datas.append(json.loads(line))

    with open(args.ner, "r", encoding="utf-8") as ner_file:
        ner_input = ner_file.readline()
        ner_input = ner_input.rstrip("\r\n")

    n = len(datas)
    for d, data in enumerate(datas):
        output_dict = dict()
        output_dict["id"] = data["id"]
        output_dict["flavor"] = 0
        output_dict["framework"] = "ner"
        output_dict["time"] = data["time"]
        output_dict["version"] = data["version"]
        output_dict["input"] = data["input"]
        output_dict["mode"] = "ner"

        ne_label = "O"
        nodes = []

        token_ranges = tokenizer.tokenize(data["input"])
        for i, token_range in enumerate(token_ranges):
            token = data["input"][token_range[0]:token_range[1]]

            # Consume token
            iterations = 0
            limit = len(token) + 1
            while token:
                if iterations >= limit: break

                # Fix some NE tagger bad fixes:
                if token == "`" and ner_input.startswith("'s "):
                    ner_input = "` s{}".format(ner_input[3:])
                elif token == "`" and ner_input.startswith("'"):
                    ner_input = "`{}".format(ner_input[1:])
                if token == "&" and not ner_input.startswith(("&", "[", "]")):
                    if i + 2 < len(token_ranges):
                       next_token = data["input"][token_ranges[i+1][0]:token_ranges[i+1][1]]
                       next_next_token = data["input"][token_ranges[i+2][0]:token_ranges[i+2][1]]
                       if next_token in ["quot"] and next_next_token == ";":
                           print("Found HTML entity {}, fixed NER input.".format(next_token), file=sys.stderr, flush=True)
                           print("Sentence: {}".format(data["input"]), file=sys.stderr, flush=True)
                           print("Original NER input: {}".format(ner_input[:100]), file=sys.stderr, flush=True)
                           ner_input = "&{} ;{}".format(next_token, ner_input[1:])
                           print("Fixed NER input: {}".format(ner_input[:100]), file=sys.stderr, flush=True)
                if token in ["''", "``"] and ner_input.startswith("\""):
                    print("Token {} found, but \" found in NER input.".format(token), file=sys.stderr, flush=True)
                    print("Sentence: {}".format(data["input"]), file=sys.stderr, flush=True)
                    print("Original NER input: {}".format(ner_input[:100]), file=sys.stderr, flush=True)
                    ner_input = "{}{}".format(token, ner_input[1:])
                    print("Fixed NER input: {}".format(ner_input[:100]), file=sys.stderr, flush=True)

                if ner_input.startswith("[") and token != "[":
                    m = re.match(r'\[(\w+)', ner_input)
                    ne_label = m.group(1)
                    ner_input = ner_input[1 + len(ne_label):]
                    ner_input = ner_input.lstrip()
                if ner_input.startswith("]") and token != "]":
                    ner_input = ner_input[1:]
                    ner_input = ner_input.lstrip()
                    ne_label = "O"
                for i in range(len(token)+1):
                    if token[:i] != ner_input[:i]:
                        i -= 1
                        break
                if ner_input[:i] == token[:i]:
                    ner_input = ner_input[i:]
                    ner_input = ner_input.lstrip()
                    token = token[i:]
                iterations += 1

            nodes.append({"id": len(nodes), "label": ne_label, "anchors": [{"from": token_range[0], "to": token_range[1]}]})

            if iterations >= limit and  d + 1 < len(datas):
                print("Unable to match token {} to NER input: {} after {} iterations, trying to skip the sentence.".format(token, ner_input[:100], iterations), file=sys.stderr, flush=True)
                first_token_range = tokenizer.tokenize(datas[d+1]["input"])[0]
                first_token = datas[d+1]["input"][first_token_range[0]:first_token_range[1]]
                index = ner_input.find(first_token)
                assert index != -1, "Unable to skip sentence."
                print("Found first token {} of the next sentence: {}, skipping current sentence in NER input: {}.".format(first_token, datas[d+1]["input"], ner_input[:100]), file=sys.stderr, flush=True)
                ner_input = ner_input[index:]
                token = ""
                ne_label = "O"
                # Finish sentence nodes before break
                for j in range(i+1, len(token_ranges)):
                    nodes.append({"id": len(nodes), "label": "O", "anchors": [{"from": token_ranges[j][0], "to": token_ranges[j][1]}]})
                break

        output_dict["nodes"] = nodes
        output_dict["edges"] = []
        output_dict["tops"] = []
        print(json.dumps(output_dict))
        output_dict = dict()
        if d % 100 == 0:
            print("Processed {} sentences of {}".format(d, n), file=sys.stderr, flush=True)


import numpy as np
import copy, math
from scipy.spatial import distance, Delaunay

def tri_area(vertices):
  u = vertices[0]
  v = vertices[1]
  w = vertices[2]
  return np.linalg.norm(np.cross( v-u, w-u)) / 2.

def tetra_volume(vertices):
  u = vertices[0]
  v = vertices[1]
  w = vertices[2]
  x = vertices[3]
  return np.cross(v-u, w-u).dot(x-u) / 6.

class MasterDict(dict):
  """
  A dict with reverse reference to a master mesh.
  """
  def __init__(self, master, *args,**kwargs):
    self.master = master
    dict.__init__(self,*args,**kwargs)

  def __setitem__(self, key, val):
    val.master = self.master
    dict.__setitem__(self, key, val)

#-------------------------------------------------------------------------------
# Fields

class Field(object):
  """
  A field meta class.
  """
  def __init__(self, label, loc, data = {}, mesh = None):
    self.label = label
    self.data = data
    self.master = master
    self.loc  = loc

  def toarrays(self):
    k = np.array(self.data.keys())
    v = np.array(self.data.values())
    return k, v

class ScalarField(Field):
  """
  A scalar field class.
  """
  pass

class VectorField(Field):
  """
  A vector field class.
  """
  pass

class TensorField(Field):
  """
  A tensor field class.
  """
  pass

#-------------------------------------------------------------------------------
# Nodes and related stuff

class Node(object):
  def __init__(self, coords, sets = set(), master = None):
    self.coords = np.asarray(coords, dtype= np.float64)
    self.sets = set(sets)
    self.master = master

  def __repr__(self):
    return "#NODE" + self.__str__()

  def __str__(self):
    return "{0}, sets={1}".format(self.coords, self.sets)

#-------------------------------------------------------------------------------
# Elements and related stuff

class Element(object):

  _extrudable = False

  def __init__(self, conn, sets = set(), surfaces = None, master = None):
    self.conn = np.asarray(conn[:self._nvert], dtype = np.uint32)
    self.master = master
    self.sets = set(sets)
    ns = self.ns()
    self.surfaces = [set() for i in xrange(ns)]
    if surfaces != None:
      for k in surfaces.keys():
        self.surfaces[k].add( surfaces[k] )

  def __repr__(self):
    return "#ELEMENT " + self.__str__()

  def __str__(self):
    name = self.type()
    conn = ",".join([str(l) for l in self.conn])
    return "{0}({1}), sets={2}, surf={3}".format(name, conn,
                                              self.sets,
                                              self.surfaces)

  def edges(self):
    """
    Returns the connectivity of the edges.
    """
    return self.conn[self._edges]

  def faces(self):
    """
    Returns the faces of a volumic element, None otherwise.
    """
    if self._space == 3:
      return np.array([ self.conn[f] for f in self._faces_conn])

  def simplex_conn(self):
    """
    Returns the connectivity of the simplexes forming the element.
    """
    return np.array([ self.conn[f] for f in self._simplex_conn])

  def type(self):
    return self.__class__.__name__

  def ns(self):
    if self._space == 1: n = 1
    if self._space == 2: n = len(self._edges)
    if self._space == 3: n = len(self._faces_conn)
    return n

  def extrude(self, offset, layer):
    if self._extrudable:
      newtype = self._extrude_as
      oldconn = self.conn
      newconn = np.concatenate([
                oldconn + offset * layer,
                oldconn + offset * (layer + 1)])[self._extrude_order]
      return globals()[newtype](conn = newconn, sets = self.sets)

  def collapsed(self):
    return len(self.conn) - len(set(self.conn))

  def collapsed_faces(self):
    if self._space == 3:
      faces = [self.conn[c] for c in self._faces_conn]
      return np.array([ len(f) - len(set(f)) for f in faces ])

  def collapsed_edges(self):
    if self._space  >= 2:
      edges = self.conn[self._edges]
      return np.array([ len(e) - len(set(e)) for e in edges ])

  def simplex_decomposition(self):
    conn = self.conn
    if self._space == 3:
        simplices = self.simplex_conn()
        simplices2 = []
        for i in range(len(simplices)):
          simplex = simplices[i]
          if (len(set(simplex)) == 4):
            if tetra_volume([self.master.nodes[l].coords for l in simplex]) > 0.:
              simplices2.append(simplices[i])

        return [Tetra4(simplex) for simplex in simplices2]
        """
        points = [self.master.nodes[l].coords for l in conn]
        tetras = Delaunay(points).simplices
        tetras2 = []
        for i in xrange(len(tetras)):
          if tetra_volume([points[j] for j in tetras[i]]) < 0.:
            t = np.array([tetras[i][j] for j in [1, 0, 2, 3]])
          else:
            t = np.array([tetras[i][j] for j in [0, 1, 2, 3]])
        return [Tetra4(conn[t]) for t in tetras]
        """

    if self._space == 2:
      if self.type() == "Tri3":
        if len(conn) == 3:
          return [self]
      if self.type() == "Quad4":
        if len(conn) == 4:
           return [Tri3(self.conn[c]) for c in [[0, 1, 2], [1, 2, 3]]]
        if len(conn) == 3:
          count = np.array([(self.conn == i).sum() for i in conn])
          rind = conn[np.where(count == 2)[0][0]]
          rconn = self.conn.copy()
          for i in range(4):
            if (rconn[1] == rind) and (rconn[-1] == rind):
              return [Tri3(rconn[[0,1,2]])]
            rconn = np.roll(rconn, 1)

  def clean_connectivity(self):
    if self.collapsed() == 0:
      return [self]
    else:
      return self.simplex_decomposition()

  def node_set_to_surface(self, nodesetlabel, surfacelabel):
    nodelabels = set([k for k in self.nodes.keys() if label in self.nodes[k].sets])
    for element in self.elements.values:
      for i in xrange(self.ns()):
        if self._space == 3: surfconn = self.conn[self._faces_conn[i]]
        # TO be completed
        if nodelabels.issuperset(surfconn):
          self.surfaces[i].add(surfacelabel)

  def volume(self, add = True):
    vertices = np.array([self.master.nodes[l].coords for l in self.conn ])
    simplices = vertices[self._simplex_conn]
    if self._space == 3:
      v = np.array([tetra_volume(simplex) for simplex in simplices])
      if add: v = v.sum()
      return v
    if self._space == 2:
      v = np.array([tri_area(simplex) for simplex in simplices])
      if add: v = v.sum()
      return v

  def centroid(self):
    vertices = np.array([self.master.nodes[l].coords for l in self.conn ])
    simplices = vertices[self._simplex_conn]
    centroids = simplices.mean(axis = 1)
    volumes = self.volume(add = False)[:,None]
    return (centroids * volumes).sum(axis = 0) / volumes.sum()

class Line2(Element):
  """
  A 1D 2 nodes line.
  """
  _nvert = 2
  _space = 1
  _extrudable = True
  _extrude_as = "Quad4"
  _extrude_order = np.array([0, 1, 3, 2])
  _simplex_conn = np.array([[0, 1]])

class Tri3(Element):
  """
  A 2D 3 noded triangular element
  """
  _nvert  = 3
  _space  = 2
  _edges = np.array([[0, 1],
                     [1, 2],
                     [2, 0]])
  _extrudable = True
  _extrude_as = "Prism6"
  _extrude_order = np.array([0, 1, 2, 3, 4, 5])
  _simplex_conn  = np.array([[0, 1, 2]])

class Quad4(Element):
  """
  A 2D 4 noded quadrangular element
  """
  _nvert  = 4
  _space  = 2
  _edges = np.array([[0, 1],
                     [1, 2],
                     [2, 3],
                     [3, 0]])
  _extrudable = True
  _extrude_as = "Hexa8"
  _extrude_order = np.array([0, 1, 2, 3, 4, 5, 6, 7])
  _simplex_conn  = np.array([[0, 1, 3],
                             [1, 2, 3]])

class Tetra4(Element):
  """
  A 3D 4 noded tetrahedral element
  """
  _nvert  = 4
  _space  = 3
  _faces_conn  = np.array([[0, 1, 2],
                           [0, 3, 1],
                           [1, 3, 2],
                           [2, 3, 0]])
  _faces_type  = ["Tri3", "Tri3", "Tri3", "Tri3"]
  _edges = np.array([[0, 1],
                     [1, 2],
                     [2, 0],
                     [0, 3],
                     [1, 3],
                     [2, 3]])
  _simplex_conn  = np.array([[0, 1, 3, 4]])

  def clean_connectivity(self):
    if self.collapsed():
      return None
    else:
      return [self]

class Pyra5(Element):
  """
  A 3D 5 noded pyramidal element
  """
  _nvert  = 5
  _space  = 3
  _faces_conn  = np.array([[0, 1, 2, 3],
                           [0, 1, 4],
                           [1, 2, 4],
                           [2, 3, 4],
                           [3, 0, 4]])
  _faces_type = ["Quad4", "Tri3", "Tri3", "Tri3", "Tri3"]
  _edges = np.array([[0, 1],
                     [1, 2],
                     [2, 3],
                     [3, 0],
                     [0, 4],
                     [1, 4],
                     [2, 4],
                     [3, 4]])
  _simplex_conn  = np.array([[0, 1, 3, 4],
                             [1, 2, 3, 4]])

class Prism6(Element):
  """
  A 3D 6 noded prismatic element
  """
  _nvert  = 6
  _space  = 3
  _faces_conn = np.array([[0, 1, 2],
                          [3, 5, 4],
                          [0, 3, 4, 1],
                          [1, 4, 5, 2],
                          [2, 5, 3, 0]])
  _faces_type = ["Tri3", "Tri3", "Quad4", "Quad4", "Quad4"]
  _edges = np.array([[0, 1],
                     [1, 2],
                     [2, 0],
                     [3, 4],
                     [4, 5],
                     [5, 3],
                     [0, 3],
                     [1, 4],
                     [2, 5]])
  _simplex_conn  = np.array([[0, 1, 2, 3],
                             [1, 2, 3, 4],
                             [2, 3, 4, 5]])

class Hexa8(Element):
  """
  A 3D 8 noded hexahedric element
  """
  _nvert  = 8
  _space  = 3
  _faces_conn  = np.array([[0, 1, 2, 3],
                           [4, 7, 6, 5],
                           [0, 4, 5, 1],
                           [1, 5, 6, 2],
                           [2, 6, 7, 3],
                           [3, 7, 4, 0]])
  _faces_type = ["Quad4",
                 "Quad4",
                 "Quad4",
                 "Quad4",
                 "Quad4",
                 "Quad4"]
  _edges = np.array([[0, 1],
                     [1, 2],
                     [2, 3],
                     [3, 0],
                     [4, 5],
                     [5, 6],
                     [6, 7],
                     [7, 4],
                     [0, 4],
                     [1, 5],
                     [2, 6],
                     [3, 7]])
  _simplex_conn = np.array([[0, 1, 3, 4],
                            [1, 2, 3, 4],
                            [3, 2, 7, 4],
                            [2, 6, 7, 4],
                            [1, 5, 2, 4],
                            [2, 5, 6, 4]])
#-------------------------------------------------------------------------------
# Mesh & related stuff
class Mesh(object):
  def __init__(self, nodes = {}, elements = {}):
    self.nodes    = MasterDict(self)
    for k, v in nodes.iteritems(): self.nodes[k] = v
    self.elements = MasterDict(self)
    for k, v in elements.iteritems(): self.elements[k] = v

  def __repr__(self):
    nn = len(self.nodes.values())
    ne = len(self.elements.values())
    return "#MESH: {0} nodes / {1} elements".format(nn, ne)

  def __str__(self):
    nodes, elements = self.nodes, self.elements
    nk = sorted(nodes.keys())
    ns = "\n".join( ["{0} {1}".format(k, str(nodes[k])) for k in nk])
    ek = sorted(elements.keys())
    es = "\n".join( ["{0} {1}".format(k, str(elements[k])) for k in ek])
    """
    nsets, esets = self.nsets, self.esets
    nsk = sorted(nsets.keys())
    nss = "\n".join( ["{0} {1}".format(k, str(nsets[k])) for k in nsk])
    esk = sorted(esets.keys())
    print esk
    ess = "\n".join( ["{0} {1}".format(k, str(esets[k])) for k in esk])
    """
    return "MESH:\n*NODES:\n{0}\n*ELEMENTS:\n{1}".format(ns, es)
  """
  def _add_set(self, kind, key, labels):
    if kind == "nset": target = self.nodes
    if kind == "eset": target = self.elements
    for label in labels:
      target[label].sets.add(key)

  def add_nset(self, *args, **kwargs):
    self._add_set(kind = "nset", *args, **kwargs)

  def add_eset(self, *args, **kwargs):
    self._add_set(kind = "eset", *args, **kwargs)
  """
  def export(path):
    return

  def load(path):
    return

  def extrude(self, translation, layers):
    translation = np.array(translation, dtype = np.float64)[:3]
    newmesh = Mesh()
    # Nodes:
    node_offset = max(self.nodes.keys())
    for l, n in self.nodes.iteritems():
      for j in xrange(layers+1):
        newnode = Node(coords = n.coords + translation * float(j) / layers,
                       sets = n.sets)
        newmesh.nodes[l + j * node_offset] = newnode
    # Elements:
    element_offset = max(self.elements.keys())
    for l, e in self.elements.iteritems():
      for layer in xrange(layers):
        newelement = e.extrude(offset = node_offset, layer = layer)
        if newelement != None:
          newmesh.elements[l + layer * element_offset] = newelement
    return newmesh

  def copy(self):
    return copy.deepcopy(self)

  def nodes_to_array(self):
    labels = np.array([k for k in self.nodes.keys()])
    n = len(labels)
    p = np.empty((n,3))
    for i in range(n):
      p[i] = self.nodes[labels[i]].coords
    return labels , p

  def transform(self, transformation):
    labels, p  = self.nodes_to_array()
    n = len(labels)
    x, y, z = p.transpose()
    newcoords = np.asarray(transformation(x, y, z),
                           dtype = np.float64).transpose()
    newmesh = self.copy()
    for i in range(n):
      newmesh.nodes[labels[i]].coords = newcoords[i]
    return newmesh

  def overlapping_nodes(self, crit_dist = 1.e-6):
    def calc_row_idx(k, n):
      return int(math.ceil((1/2.) * (- (-8*k + 4 *n**2 -4*n - 7)**0.5 + 2*n -1) - 1))

    def elem_in_i_rows(i, n):
      return i * (n - 1 - i) + (i*(i + 1))/2

    def calc_col_idx(k, i, n):
      return int(n - elem_in_i_rows(i + 1, n) + k)

    def condensed_to_square(k, n):
      i = calc_row_idx(k, n)
      j = calc_col_idx(k, i, n)
      return np.array([i, j])
    labels, points = self.nodes_to_array()
    dist = distance.pdist(points)
    n = len(labels)
    loc = np.where(dist<=crit_dist)[0]
    pairs = [labels[condensed_to_square(l, n)] for l in loc]
    mapping = dict(zip(labels, labels))
    for pair in pairs:
      pair.sort()
      mapping[pair[1]] = min(mapping[pair[1]],  pair[0])
    return mapping

  def merge_nodes(self, mapping):
    newmesh = self.copy()
    for elem in newmesh.elements.values():
        conn = elem.conn
        for i in range(len(conn)):
          conn[i] = mapping[conn[i]]

    for label in newmesh.nodes.keys():
      if mapping[label] != label:
        del newmesh.nodes[label]
    return newmesh

  def simplex_decomposition(self):
    """
    Returns a list of new simplex elements sharing the same vertices with the
    orginal one with a viable connectivity.
    """
    newmesh = self.copy()
    newmesh.elements.clear()
    label = 1
    for el in self.elements.values():
      simplices = el.simplex_decomposition()
      if simplices != None:
        for simplex in simplices:
          newmesh.elements[label] = simplex
          label += 1
    return newmesh

  def clean_connectivity(self):
    newmesh = self.copy()
    newmesh.elements.clear()
    label = 1
    for el in self.elements.values():
      newels = el.clean_connectivity()
      if newels != None:
        for newel in newels:
          newmesh.elements[label] = newel
          label += 1
    return newmesh
#-------------------------------------------------------------------------------
# Parsers & writers

# Abaqus INP

def parseInp(path):
  # Some useful tools
  def lineInfo(line):
    out =  {"type": "data"}
    if line[0] == "*":
      if line[1] == "*":
        out["type"] = "comment"
        out["text"] = line[2:]
      else:
        out["type"] = "command"
        words = line[1:].split(",")
        out["value"] = words[0].strip()
        out["options"] = {}
        for word in words[1:]:
          key, value =  [s.strip() for s in word.split("=")]
          out["options"][key] = value
    return out
  def elementMapper(inpeltype):
    if inpeltype == "t3d2": return "Line2"
    if inpeltype[:3] in ["cps", "cpe", "cax"]:
      if inpeltype[3] == "3": return "Tri3"
      if inpeltype[3] == "4": return "Quad4"
    if inpeltype[:3] in ["c3d"]:
      if inpeltype[3] == "4": return "Tetra4"
      if inpeltype[3] == "5": return "Pyra5"
      if inpeltype[3] == "6": return "Prism6"
      if inpeltype[3] == "8": return "Hexa8"

  # Output mesh
  m = Mesh()
  # File preprocessing
  lines = np.array([l.strip().lower() for l in open(path).readlines()])
  # Data processing
  env = None
  setlabel = None
  for line in lines:
    d = lineInfo(line)
    if d["type"] == "command":
      env = d["value"]
      # Nodes
      if env == "node":
        opt = d["options"]
        currentset = None
        if "nset" in opt.keys(): currentset = opt["nset"]

      # Elements
      if env == "element":
        opt = d["options"]
        eltype = elementMapper(opt["type"])
        currentset = None
        if "elset" in opt.keys(): currentset = opt["elset"]

      # Nsets
      if env == "nset":
        opt = d["options"]
        currentset = opt["nset"]

      # Elsets
      if env == "elset":
        opt = d["options"]
        currentset = opt["elset"]

    if d["type"] == "data":
      words = line.strip().split(",")
      if env == "node":
        label  = int(words[0])
        coords = np.array( [float(w) for w in words[1:]], dtype = np.float64 )
        if currentset == None:
          m.nodes[label] = Node(coords = coords)
        else:
          m.nodes[label] = Node(coords = coords, sets = set([currentset]))

      if env == "element":
        label  = int(words[0])
        conn = np.array( [int(w) for w in words[1:]], dtype = np.int32)
        if currentset == None:
          m.elements[label] = globals()[eltype](conn = conn)
        else:
          m.elements[label] = globals()[eltype](conn = conn, sets = set([currentset]))

      if env == "nset":
        [m.nodes[int(w)].sets.add(currentset) for w in words if len(w) != 0]

      if env == "elset":
        [m.elements[int(w)].sets.add(currentset) for w in words if len(w) != 0]

  return m

def writeInp(mesh, mapping, path = None):

  def exportset(s, d):
    out = ""
    labels = [str(k) for k,v in d.iteritems() if s in v.sets]
    for i in xrange(len(labels)):
      out += labels[i]
      if (i+1)%10 != 0:
        out += ", "
      else:
        out += ",\n"
    if out[-1] != "\n": out += "\n"
    return out

  # Nodes
  out = "*NODE\n"
  for label, node in mesh.nodes.iteritems():
    out += "{0}, {1}\n".format(label, ", ".join([ str(c) for c in node.coords]))
  # Elements
  etypes = set([e.type() for e in mesh.elements.values()])
  for etype in etypes:
    out +="*ELEMENT, TYPE={0}\n".format( mapping[etype])
    for label, elem in mesh.elements.iteritems():
      if elem.type() == etype:
        out += "{0}, {1}\n".format(label, ", ".join([ str(c) for c in elem.conn]))
  # Sets
  nsets = set().union(*[n.sets for n in mesh.nodes.values()])
  for s in nsets:
    out += "*NSET, NSET={0}\n".format(s) + exportset(s , mesh.nodes)
  esets = set().union(*[e.sets for e in mesh.elements.values()])
  for s in esets:
    out += "*ELSET, ELSET={0}\n".format(s) + exportset(s , mesh.elements)
  if path == None:
    return out
  else:
    open(path, "w").write(out)

def writeMsh(mesh, path = None):
  elementMap = {"Tri3": 2, "Quad4":3, "Tetra4":4, "Hexa8":5, "Prism6":6, "Pyra5": 7}
  pattern = """$MeshFormat
2.2 0 8
$EndMeshFormat
$Nodes
{0}
$EndNodes
$Elements
{1}
$EndElements"""
  nodeout = "{0}\n".format(len(mesh.nodes.keys()))
  nodelist = []
  for k in mesh.nodes.keys():
    node = mesh.nodes[k]
    coords = node.coords
    nodelist.append("{0} {1} {2} {3}".format(k, coords[0], coords[1], coords[2]) )
  nodeout += "\n".join(nodelist)
  elemout = ""
  elemout = "{0}\n".format(len(mesh.elements.keys()))
  elemlist = []
  for k in mesh.elements.keys():
    element = mesh.elements[k]
    coords = node.coords
    elemlist.append("{0} {1} 1 1 {2}".format(
        k,
        elementMap[element.__class__.__name__],
        " ".join( [str(l) for l in element.conn ] ) ))
  elemout += "\n".join(elemlist)
  if path == None:
    return pattern.format(nodeout, elemout)
  else:
    open(path, "w").write(pattern.format(nodeout, elemout))


from wgzimmer_finder import wgzimmer_refresh
from eth_uzh_housingoffice_finder import eth_uzh_refresh
from woko_finder import woko_refresh
import time
from config import FREQUENCY
from config import ETH_HEADERS
from config import WGZIMMER_HEADERS

def print_welcome():
	print(' ____  _     ____  _  ____  _      \n/_   \/ \ /\/  __\/ \/   _\/ \ /|  \n /   /| | |||  \/|| ||  /  | |_||  \n/   /_| \_/||    /| ||  \_ | | ||  \n\____/\____/\_/\_\\_/\____/\_/ \|  \n                                   \n ____  ____  ____  _               \n/  __\/  _ \/  _ \/ \__/|          \n|  \/|| / \|| / \|| |\/||          \n|    /| \_/|| \_/|| |  ||          \n\_/\_\\____/\____/\_/  \|          \n                                   \n _____ _  _      ____  _____ ____  \n/    // \/ \  /|/  _ \/  __//  __\ \n|  __\| || |\ ||| | \||  \  |  \/| \n| |   | || | \||| |_/||  /_ |    / \n\_/   \_/\_/  \|\____/\____\\_/\_\ ')

	print('Welcome to Zurich room finder! \nThis software is writen by a normal guy in love')

	print("Currently support wgzimmer.ch and ETH-UZH housing office website only")

def check_config():
	for value in ETH_HEADERS.values():
		if value == "":
			print("Eth-uzh housing website cookie not set!")
			exit(1)
	for value in WGZIMMER_HEADERS.values():
		if value == "":
			print("Wgzimmer website cookie not set!")
			exit(1)

def main():
	print_welcome()
	pre_ads_found = -1
	while True:
		pre_ads_found = wgzimmer_refresh(pre_ads_found)
		eth_uzh_refresh()
		woko_refresh()
		time.sleep(3600/FREQUENCY)

if __name__ == "__main__":
	main()


import logging
import pickle
from functools import wraps
from timeit import default_timer as timer

logger = logging.getLogger(__name__)

def timing(f):
    @wraps(f)
    def wrap(*args, **kw):
        ts = timer()
        result = f(*args, **kw)
        te = timer()
        print('func:{!r} took: {:f} sec'.format(f.__name__, te-ts))
        return result
    return wrap

def timing_args(f):
    @wraps(f)
    def wrap(*args, **kw):
        ts = timer()
        result = f(*args, **kw)
        te = timer()
        print('func:{!r} args:[{},{}] took: {:f} sec'.format(f.__name__, args, kw, te-ts))
        return result
    return wrap

def cache_result(f, path):
    @wraps(f)
    def wrap(*args, **kwargs):
        try:
            with open(path, 'rb') as f_in:
                result = pickle.load(f_in)
                return result
        except FileNotFoundError:
            result = f(*args, **kwargs)
            with open(path, 'wb') as f_out:
                pickle.dump(result, f_out, protocol=pickle.HIGHEST_PROTOCOL)
            return result
    return wrap


from django.urls import path, include

from .views import WhyUsView, TourView

app_name = 'pages'

urlpatterns = [
    path('why-us/', WhyUsView.as_view(), name='why_us'),
    path('tour/', TourView.as_view(), name='tour'),
    path('', include('django.contrib.flatpages.urls')),
]