import numpy as np
import nltk
import re
from nltk.corpus import reuters
from nltk.corpus import stopwords
from nltk.tokenize import word_tokenize
from nltk.stem import WordNetLemmatizer
from collections import Counter
from collections import defaultdict

from nltk import word_tokenize
from nltk.stem.porter import PorterStemmer
import re
from nltk.corpus import stopwords
import argparse
import pickle
from tqdm import tqdm
import math
from collections import defaultdict
from scipy.stats.stats import pearsonr
import operator

nltk.download('reuters')
parser = argparse.ArgumentParser()

parser.add_argument('--tokenize', type=bool, default=False)
parser.add_argument('--build_dict', type=bool, default=False)
parser.add_argument('--initilize', type=bool, default=False)
parser.add_argument('--embedding_dim', type=int, default=50)
parser.add_argument('--train_embeddings', type=bool, default=False)
parser.add_argument('--nb_negative_samples', type=int, default=10)
parser.add_argument('--lr', type=float, default=0.05)
parser.add_argument('--context', type=int, default=5)
parser.add_argument('--lemmatize', type=bool, default=False)
parser.add_argument('--stemming', type=bool, default=False)
parser.add_argument('--tokenized_data_file', type=str, default='tokenized_data_lemmatized.pkl')
parser.add_argument('--nb_epochs', type=int, default=14)
parser.add_argument('--lr_annealing', type=bool, default=False)
parser.add_argument('--simlex_file', type=str, default='/scratche/home/aditay/NLU_assignment1/simlex/SimLex-999/SimLex-999.txt')
parser.add_argument('--pearson_cofficient', type=bool, default=False)
parser.add_argument('--remove_stop_words', type=bool, default=False)
parser.add_argument('--k_NN', type=str, default='queen')
parser.add_argument('--k', type=int, default=10)
parser.add_argument('--syntactic_analogy', type=bool, default=False)
parser.add_argument('--word_model', type=str, default='checkpoints/word_embeddings_10_50dim_with_lemma_v5.1.2.pkl')
parser.add_argument('--context_model', type=str, default='checkpoints/context_embedding_10_50dim_with_lemma_v5.1.2.pkl')

args = parser.parse_args()
nltk.download('wordnet')
print("The reuters corpus has {} tags".format(len(reuters.categories())))
print("The reuters corpus has {} documents".format(len(reuters.fileids())))

lemmer = WordNetLemmatizer()

def sigmoid(x):
    return 1 / (1 + math.exp(-x))

def get_train_test_splits():
    documents = reuters.fileids()
    print(len(documents))
    train_docs = []
    test_docs = []
    for doc in documents:
        split = doc.split('/')[0]
        if split == 'test':
            test_docs.append(doc)
        elif split == 'training':
            train_docs.append(doc)

    print("Test docs: %d Training docs: %d"%(len(test_docs), len(train_docs)))
    return test_docs, train_docs

def get_train_test_docs():
    test_docs, train_docs = get_train_test_splits()
    # print(test_docs)
    test_doc = []
    train_doc = []
    for doc in test_docs:
        test_doc.extend(reuters.words(doc))
    for doc in train_docs:
        train_doc.extend(reuters.words(doc))
    return test_doc, train_doc
nltk.download('stopwords')
cachedStopWords = stopwords.words("english")

def tokenize(lemmatize=False, stemming=False, remove_stop_words=False):
    test_doc, train_doc = get_train_test_docs()
    min_length = 3
    test_words = []
    train_words = []
    for word in test_doc:
        test_words.append(word.lower())
    for word in train_doc:
        train_words.append(word.lower())
    print(len(test_words), len(train_words))

    # train_words = map(lambda word: word.lower(), word_tokenize(train_doc))
    # test_words = map(lambda word: word.lower(), word_tokenize(test_doc))

    if remove_stop_words:
        train_words = [word for word in train_words
                      if word not in cachedStopWords]
        test_words = [word for word in test_words
                      if word not in cachedStopWords]

    if stemming:
        train_tokens =(list(map(lambda token: PorterStemmer().stem(token),
                    train_words)))
        test_tokens =(list(map(lambda token: PorterStemmer().stem(token),
                    test_words)))
    elif lemmatize:
        train_tokens =(list(map(lambda token: lemmer.lemmatize(token),
                    train_words)))
        test_tokens =(list(map(lambda token: lemmer.lemmatize(token),
                    test_words)))
    else:
        train_tokens = train_words
        test_tokens = test_words

    p = re.compile('[a-zA-Z]+')

    train_tokens = list(filter(lambda token: p.match(token) and len(token)>=min_length, train_tokens))

    test_tokens = list(filter(lambda token: p.match(token) and len(token)>=min_length, test_tokens))

    print("Training tokens: %d, test_tokens: %d"%(len(train_tokens), len(test_tokens)))
    data = {'train': train_tokens, 'test': test_tokens}
    if stemming:
        file_name = 'tokenized_data_stemmed.pkl'
    elif lemmatize:
        file_name = 'tokenized_data_lemmatized.pkl'
    else:
        file_name = 'tokenized_data_with_stopwords.pkl'
    pickle.dump(data, open(file_name, 'wb'))

def build_dict(file_name):
    print("Reading tokenized train and test documents ++++++++")
    tokenized_data = pickle.load(open(file_name,'rb'))
    train_tokens = tokenized_data['train']
    test_tokens = tokenized_data['test']
    train_counts = Counter(train_tokens)
    test_counts = Counter(test_tokens)
    word2counts = {}
    print(len(train_tokens))
    pickle.dump(train_counts, open('dict_count_train_lemmatized.pkl', 'wb'))
    pickle.dump(test_counts, open('dict_count_test_lemmatized.pkl', 'wb'))

def initilize_word_embeddings(dim):
    print("Initilizing word2vec embeddings ++++++++")
    if args.lemmatize:
    # token_file = args.tokenized_data_file
        train_counts = pickle.load(open('dict_count_train_lemmatized.pkl','rb'))
    else:
        train_counts = pickle.load(open('dict_count_train.pkl','rb'))
    # initial_embeddings = {}
    sigma = 1
#    sigma = math.sqrt(2/dim)

    for word, count in train_counts.items():
        initial_embeddings[word] = sigma*np.random.uniform(-0.8, 0.8, dim)
    return initial_embeddings
    # print("Creating dump of initila word embeddings ++++++++")
    # pickle.dump(initial_embeddings, open('initial_embeddings.pkl', 'wb'))

def normalize_counts(count_dict):
    normalized_counts = {}
    total_count = 0
    for key, item in count_dict.items():
        total_count += item
    for key, item in count_dict.items():
        normalized_counts[key] = item/total_count
    return normalized_counts, total_count

def data_generator(data_list, threshold, context, normalized_counts):
    # constant = 601000
    while True:
        constant = 1
        for word_index in range(len(data_list)):
            context_words = []
            if (word_index - (context-1)/2) < 0:
                if np.random.random_sample() > 1 - np.sqrt(threshold/(constant*normalized_counts[data_list[word_index]])):
                    word = data_list[word_index]
                    if len(data_list[0:word_index]) != 0:
                        [context_words.append(c) for c in data_list[0:word_index]]
                    [context_words.append(c) for c in data_list[word_index+1:word_index+int((context+1)/2)]]
                    yield word, context_words
                else:
                    yield None
            elif (word_index + (context-1)/2) >= len(data_list):
                if np.random.random_sample() > 1 - np.sqrt(threshold/(constant*normalized_counts[data_list[word_index]])):
                    word = data_list[word_index]
                    [context_words.append(c) for c in data_list[word_index-int((context-1)/2): word_index]]
                    [context_words.append(c) for c in data_list[word_index+1:]]
                    yield word, context_words
                else:
                    yield None
            else:
                if np.random.random_sample() > 1 - np.sqrt(threshold/(constant*normalized_counts[data_list[word_index]])):
                    word = data_list[word_index]
                    [context_words.append(c) for c in data_list[word_index-int((context-1)/2):word_index]]
                    [context_words.append(c) for c in data_list[word_index+1:word_index+int((context+1)/2)]]
                    yield word, context_words
                else:
                    yield None

def neg_generator(nb_neg_samples, len_dic ,dic_list):
    while True:
        # indices = np.random.randint(low=0, high=len_dic, size=nb_neg_samples)
        neg_samples = []
        for i in range(nb_neg_samples):
            index = np.random.randint(low=0, high=len_dic-1)
            neg_samples.append(dic_list[index])
        yield neg_samples

def get_loss(pos_samples, neg_samples, word_embedding, context_embedding, train_counts, total_count):
    word = pos_samples[0]
    context = pos_samples[1]
    pos_loss_list = [math.log(sigmoid(np.dot(word_embedding[word], context_embedding[c]))) for c in context]
    neg_loss_list = [((train_counts[c]**(3/4))/total_count)*math.log(sigmoid(-1*np.dot(word_embedding[word], context_embedding[c]))) for c in neg_samples]

    loss = sum(pos_loss_list) + sum(neg_loss_list)
    return loss, sum(pos_loss_list), sum(neg_loss_list)

def get_grad(pos_samples, neg_samples, word_embedding, context_embedding):
    word = pos_samples[0]
    context = pos_samples[1]

    word_grad = [(1-sigmoid(np.dot(word_embedding[word], context_embedding[c])))*context_embedding[c] for c in context]

    context_grad = defaultdict(list)
    [context_grad[c].append((1-sigmoid(np.dot(word_embedding[word], context_embedding[c])))*word_embedding[word]) for c in context]
    neg_context_grad = defaultdict(list)
    [neg_context_grad[c].append(-1*(1-sigmoid(np.dot(-word_embedding[word], context_embedding[c])))*word_embedding[word]) for c in neg_samples]
    neg_word_grad = defaultdict(list)
    [neg_word_grad[c].append(-1*(1-sigmoid(np.dot(-word_embedding[word], context_embedding[c])))*context_embedding[c]) for c in neg_samples]

    return word_grad, context_grad, neg_context_grad, neg_word_grad, word, context

def sgd_step(embedding, grad, lr):
    pass

def train_word2vec(tokenized_data_file, nb_neg_samples, lr, context, nb_epochs, dim, anneal):
    if args.lemmatize:
        train_counts = pickle.load(open('dict_count_train_lemmatized.pkl','rb'))
    else:
        train_counts = pickle.load(open('dict_count_train.pkl','rb'))

    normalized_counts, total_count = normalize_counts(train_counts)
    threshold = 0.00001
    tokenized_data = pickle.load(open(tokenized_data_file,'rb'))
    word_embedding = initilize_word_embeddings(dim)
    context_embedding = initilize_word_embeddings(dim)
    data_list = tokenized_data['train']
    data_size = len(data_list)
    dic_list = [key for key, item in normalized_counts.items()]
    positive_samples_generator = data_generator(data_list, threshold, context, normalized_counts)
    negative_samples_generator = neg_generator(nb_neg_samples,len(normalized_counts), dic_list)

    for epoch in range(nb_epochs):
        running_loss = 0
        running_loss_p = 0
        running_loss_n = 0
        count = 0
        print('Epoch {}/{}'.format(epoch, nb_epochs - 1))
        print('-' * 10)
        loader = tqdm(range(data_size), unit='words')
        for i in enumerate(loader):
            p_samples = next(positive_samples_generator)

            if p_samples != None:
                word = p_samples[0]
                context = p_samples[1]
                for c in context:
                    negative_samples = next(negative_samples_generator)
                    positive_samples = [word, [c]]

                    loss, p_loss, n_loss = get_loss(positive_samples, negative_samples, word_embedding, context_embedding, train_counts, total_count)
                    word_grad, context_grad, neg_context_grad, neg_word_grad, word, context = get_grad(positive_samples, negative_samples, word_embedding, context_embedding)
                    if anneal:
                       if (epoch+1) %8 == 0:
                           lr = 0.5*lr
                    for w_grad in word_grad:
                        word_embedding[word] = word_embedding[word] + lr*w_grad
                    for key, w_grad in neg_word_grad.items():
                        # word_embedding[word] = word_embedding[word] + lr*(w_grad[0]*((train_counts[key]**(3/4))/total_count))
                        word_embedding[word] = word_embedding[word] + lr*w_grad[0]
                    for key, c_grad in context_grad.items():
                        # print(c_grad[0].shape)
                        context_embedding[key] = context_embedding[key] + lr*c_grad[0]
                    for key, n_grad in neg_context_grad.items():
                        context_embedding[key] = context_embedding[key] + lr*n_grad[0]
                        # context_embedding[key] = context_embedding[key] + lr*(n_grad[0]*((train_counts[key]**(3/4))/total_count))

                    if math.isnan(p_loss):
                        print(neg_word_grad)
                        exit()
                    running_loss_n += n_loss
                    running_loss_p += p_loss
                    running_loss += loss
                    count += 1

                    loader.set_postfix(w_norm=np.linalg.norm(word_embedding[word]), p_loss=running_loss_p/count,
                                        n_loss=running_loss_n/count, lr=lr)
    pickle.dump(word_embedding, open('checkpoints/word_embeddings_10_50dim_with_lemma_context_7_v5.1.2.pkl', 'wb'))
    pickle.dump(context_embedding, open('checkpoints/context_embedding_10_50dim_with_lemma_context_7_v5.1.2.pkl', 'wb'))

                # loss = get_loss(pos_samples, neg_samles)
                # word_grad, context_grad, neg_context_grad, word, context = get_grad(positive_samples, negative_samples, word_embedding, context_embedding)
def calculate_cosine_similarity(vector_1, vector_2):
    similarity = np.dot(vector_1, vector_2)/(np.linalg.norm(vector_1)*np.linalg.norm(vector_2))
    return similarity

def calculate_pearson_cofficient(simlex_file, word_embedding='checkpoints/word_embeddings_10_50dim_with_lemma_v5.1.2.pkl',
                                context_embedding='checkpoints/context_embedding_10_50dim_with_lemma_v5.1.2.pkl'):
    sim_file = open(simlex_file, 'r')
    w_embedding = pickle.load(open(word_embedding, 'rb'))
    c_embedding = pickle.load(open(context_embedding, 'rb'))
    simlex_scores = []
    model_scores = []
    i = 0
    for line in sim_file:
        i += 1
        if i > 1:
            l = line.split('\t')
            # print(l)

            try:
                # print(l)
                if args.lemmatize:
                    w1_1 = w_embedding[lemmer.lemmatize(l[0])]
                    w1_2 = c_embedding[lemmer.lemmatize(l[0])]

                    w2_1 = w_embedding[lemmer.lemmatize(l[1])]
                    w2_2 = c_embedding[lemmer.lemmatize(l[1])]
                    w1 = np.hstack([w1_1, w1_2])
                    w2 = np.hstack([w2_1, w2_2])

                    w2v_score = calculate_cosine_similarity(w1, w2)
                    simlex_scores.append(float(l[3]))
                    # print(w2v_score)
                    model_scores.append(w2v_score)
                else:
                    w1_1 = w_embedding[l[0]]
                    w1_2 = c_embedding[l[0]]

                    w2_1 = w_embedding[l[1]]
                    w2_2 = c_embedding[l[1]]
                    w1 = np.hstack([w1_1, w1_2])
                    w2 = np.hstack([w2_1, w2_2])

                    w2v_score = calculate_cosine_similarity(w1, w2)
                    simlex_scores.append(float(l[3]))
                    # print(w2v_score)
                    model_scores.append(w2v_score)
            except:
                pass

    pearson_cofficient = pearsonr(simlex_scores, model_scores)
    print(pearson_cofficient)

def get_k_NN(test_word, k=10):
    if args.lemmatize:
        word_embedding='checkpoints/word_embeddings_10_50dim_with_lemma_v5.1.2.pkl'
        context_embedding='checkpoints/context_embedding_10_50dim_with_lemma_v5.1.2.pkl'
    else:
        word_embedding='checkpoints/word_embeddings_10_50dim_vanila_with_stopwords_v5.1.1.pkl'
        context_embedding='checkpoints/context_embedding_10_50dim_vanila_with_stopwords_v5.1.1.pkl'

    word_embedding = pickle.load(open(word_embedding, 'rb'))
    context_embedding = pickle.load(open(context_embedding, 'rb'))

    if args.lemmatize:
        test_word = lemmer.lemmatize(test_word)

    score_dict = defaultdict(float)
    for key, embed in word_embedding.items():
        try:
            t_w = word_embedding[test_word]
            t_c = context_embedding[test_word]
            t = np.hstack([t_w, t_c])
        except:
            print("Word not present in the dictionary. Try a different word.")
            exit()
        w_w = embed
        w_c = context_embedding[key]
        w = np.hstack([w_w, w_c])

        similarity = calculate_cosine_similarity(t, w)
        score_dict[key] = (similarity)
    sorted_score_dict = sorted(score_dict.items(), key=operator.itemgetter(1))
    print("Test word is: %s"%(test_word))
    print(sorted_score_dict[-k:])

def syntactic_analogy():
    if args.lemmatize:
        word_embedding = 'checkpoints/word_embeddings_10_50dim_with_lemma_v5.1.2.pkl'
        context_embedding = 'checkpoints/context_embedding_10_50dim_with_lemma_v5.1.2.pkl'
    else:
        word_embedding='checkpoints/word_embeddings_10_50dim_vanila_with_stopwords_v5.1.1.pkl'
        context_embedding='checkpoints/context_embedding_10_50dim_vanila_with_stopwords_v5.1.1.pkl'
    word_embedding = pickle.load(open(word_embedding, 'rb'))
    context_embedding = pickle.load(open(context_embedding, 'rb'))

    word1 = 'quick'
    word2 = 'quickly'
    word3 = 'slow'
    if args.lemmatize:
        word1 = lemmer.lemmatize(word1)
        word2 = lemmer.lemmatize(word2)
        word3 = lemmer.lemmatize(word3)
    word1 = np.hstack([word_embedding[word1], context_embedding[word1]])
    word2 = np.hstack([word_embedding[word2], context_embedding[word2]])
    word3 = np.hstack([word_embedding[word3], context_embedding[word3]])
    word4 = word1 - word2 + word3
    score_dict = defaultdict(float)
    for key, embed in word_embedding.items():
        w_w = embed
        w_c = context_embedding[key]
        w = np.hstack([w_w, w_c])
        similarity = calculate_cosine_similarity(word4, w)
        score_dict[key] = (similarity)
    sorted_score_dict = sorted(score_dict.items(), key=operator.itemgetter(1))
    k = 10
    print(sorted_score_dict[-k:])

if args.tokenize:
    tokenize(lemmatize=args.lemmatize, stemming=args.stemming, remove_stop_words=args.remove_stop_words)

if args.build_dict:
    build_dict(args.tokenized_data_file)
if args.initilize:
    initilize_word_embeddings(args.embedding_dim)

if args.train_embeddings:
    train_word2vec(args.tokenized_data_file, args.nb_negative_samples, args.lr,
    args.context, args.nb_epochs, args.embedding_dim, args.lr_annealing)

if args.pearson_cofficient:
    calculate_pearson_cofficient(args.simlex_file,word_embedding=args.word_model,context_embedding=args.context_model)

if args.k_NN:
    get_k_NN(args.k_NN, k=args.k)

if args.syntactic_analogy:
    syntactic_analogy()


import requests
from bs4 import BeautifulSoup as bs

headers = {'accept': '*/*',
           'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_1) AppleWebKit/537.36 (KHTML, like Gecko) '
                         'Chrome/78.0.3904.97 Safari/537.36'}

base_url = 'https://social.hse.ru/persons'

def hse_parse(base_url, headers):
    session = requests.session()
    request = session.get(base_url, headers=headers)

    if request.status_code == 200:
        soup = bs(request.content, 'lxml')
        links = soup.find_all('div', attrs={'class': 'fa-person__item'})
        links_arr = []

        for link in links:
            result = link.find('a', attrs={'class': 'fa-person__name'})['href']
            links_arr.append(result)

        print(links_arr)

    else:
        print('ERROR')

hse_parse(base_url, headers)


import colorsys
import cv2
import numpy as np

from model import Model

template_root_path = 'project2/image/template'
query_file_path = 'project2/image/query/scene4.jpg'

# Get n distinct colors for each object bounding box
def get_label_colors(labels):
    def hsv2rgb(h, s, v):
        (r, g, b) = colorsys.hsv_to_rgb(h, s, v)
        return (int(255*r), int(255*g), int(255*b))

    huePartition = 1.0 / (len(labels) + 1)
    object_box_colors = [hsv2rgb(huePartition * value, 1.0, 1.0) for value in range(len(labels))]
    return dict(zip(labels, object_box_colors))

if __name__ == '__main__':
    model = Model()
    model.show_message(True)
    model.load_templates(template_root_path)
    label_colors = get_label_colors(model.labels)

    query_img = cv2.imread(query_file_path)
    query_gray_img = cv2.imread(query_file_path, cv2.IMREAD_GRAYSCALE)

    prediciton = model.predict(query_gray_img)
    for label in prediciton.keys():
        bounding_box = prediciton[label]
        if bounding_box is None: continue

        x, y = bounding_box[0]
        cv2.rectangle(query_img, bounding_box[0], bounding_box[1], label_colors[label], 6)
        cv2.putText(query_img, label, (x, y-10), cv2.FONT_HERSHEY_SIMPLEX, 3, label_colors[label], 6)

    scale = 1/4
    query_img = cv2.resize(query_img, None, fx=scale, fy=scale)
    cv2.imshow("SIFT Detector", query_img)
    cv2.waitKey(0)
    cv2.destroyAllWindows()


import sys
import os
from objectives import Objectives
from datetime import datetime

while True:

    json_data = Objectives().open_json()
    objectives = json_data["objectives"]
    answer = input("Elige que quieres hacer:\n1. Añadir objetivo\n2. Ver objetivos pendientes\n3. Eliminar objetivo\n4. Salir\n")

    if answer == '1':
        name = input("\nInserta aquí el nombre del objetivo: ")
        start_date = str(datetime.now())
        objectives.append({"name": name, "start_date": start_date})

        print("-----------------------------------------")
        print("Tu objetivo se ha añadido correctamente!")
        print("-----------------------------------------")
    if answer == '2':
        Objectives().obtain_date(objectives)
    if answer == '3':
        print("Que objetivo quieres eliminar:")
        bandera = 1

        for objective in objectives:
            print(str(bandera) + ". " + str(objective['name']))
            bandera = bandera + 1

        answer = input(str(bandera) + ". Atrás\n")
        bandera = 0

        for i in objectives:
            if int(answer) == bandera + 1:
                deleted_objective = objectives[bandera]["name"]
                del objectives[bandera]
                print("-----------------------------------------")
                print("El objetivo <<%s>> ha sido eliminado!" % deleted_objective)
                print("-----------------------------------------")
            ++bandera

    if answer == '4':
        sys.exit()

    Objectives().close_json(json_data)


#create imports
import pandas as pd
import numpy as np
import yfinance as yf

import datetime
from time import time

from datetime import date
from dateutil.relativedelta import relativedelta

from multiprocessing import Pool
from multiprocessing.dummy import Pool as ThreadPool

import requests
from bs4 import BeautifulSoup

import pickle
import logging

#Improvement in pooling
#Process took 41.73 seconds
#Process took 14.47 seconds

'''
Collect and store all dataframes of stocks
 '''
LOG_FORMAT = "%(levelname)s %(asctime)s - %(message)s"
logging.basicConfig(filename = r"C:\Users\shawn paul\Desktop\PyFinanceProj\NASDAQPrediction\test_logs\download_all_dataframes.log",level = logging.DEBUG, format =LOG_FORMAT)
logger = logging.getLogger()#root logger

# Create a pool of workers equaling cores on the machine
pool = ThreadPool()

#get dictionary of all stocks

def return_dictonaries_of_stock_tickers(URL):
    website_url = ""
    website_url = requests.get(URL).text
    companies = []
    symbols   = []

    soup = BeautifulSoup(website_url,'lxml')
    My_table = soup.find('table',{'class':'wikitable sortable','id':'constituents'})

    for row in My_table.findAll('tr'):
        cells=row.findAll('td')

        if len(cells)==2:
            companies.append(cells[0].find(text=True))
            symbols.append((cells[1].find(text=True)).rstrip())

    dictionary = dict(zip(companies, symbols))
    return dictionary

#download all the datasets

def API_download_datasets(stock_name):

	stock = yf.Ticker(stock_name)
	stock = stock.history(period="max")

	stock.to_parquet(r"C:\Users\shawn paul\Desktop\PyFinanceProj\NASDAQPrediction\Stock_Data\{}.parquet".format(stock_name))
	#with open(r'C:\Users\shawn paul\Desktop\PyFinanceProj\Early_dates\date_pickle_early\{}.pickle'.format(stock_name)) as ed:

def download_datasets():
	stock_file_NASDAQ = r"https://en.wikipedia.org/wiki/NASDAQ-100"
	syms = []
	syms = return_dictonaries_of_stock_tickers(stock_file_NASDAQ)
	syms = list(syms.values())
	results = pool.map(API_download_datasets,syms)
	# Close the pool
	pool.close()
	# Combine the results of the workers
	pool.join()

#need to check if files exist
download_datasets()


import json
import random
import os
import copy
import numpy as np

def isDirectoryEmpty(path):
	if len(os.listdir(path) ) == 1 and os.path.isfile(f'{path}/README.md'):
		return True
	return False

def write_json(data,path):
	with open(path, "w") as outFile:
		json.dump(data, outFile)

def read_json(path):
	with open(path) as inFile:
		return json.load(inFile)

def suffleTasksStartingFromIndex(data,taskStartIndex,questionsPerTask):

	# convert data to np.array
	data = np.array(data)
	# organize questions per task
	data = np.split(data,len(data)/questionsPerTask)
	#start by copying the data that should not be shuffled
	new_data = copy.deepcopy(data[0:taskStartIndex])
	#copy the remaining data
	remaining_data = copy.deepcopy(data[taskStartIndex:])
	#suffle remaining data
	random.shuffle(remaining_data)
	#extend new_data with suffled remaining_data
	new_data.extend(remaining_data)

	return list(np.concatenate(new_data).flat)

def createSessions(experimentConfig):

	# read template
	template = read_json(experimentConfig["template"])

	# iterate over rooms
	for room in experimentConfig["rooms"]:
		# create room directory
		roomPath = f'{experimentConfig["outDirectory"]}/{room}/'
		os.makedirs(roomPath,mode=0o777)
		# iterate over participants
		for participant in range(0,experimentConfig["participantsPerRoom"]):
			#make new session from template
			newSession = copy.deepcopy(template)
			#shuffle newSession["questions"] list starting from a given index
			newSession["questions"] = suffleTasksStartingFromIndex(newSession["questions"],taskStartIndex=experimentConfig["shuffleStartTaskIndex"], questionsPerTask=experimentConfig['questionsPerTask'])
			#Assign linkingSubProcessMode
			newSession["linkingSubProcessesMode"] = experimentConfig["linkingSubProcessesModes"][participant % len(experimentConfig["linkingSubProcessesModes"])]
			# export
			exportName = f'{experimentConfig["ParticipantsIdPrefix"]}{experimentConfig["participantsIdSeed"]+participant}-{newSession["linkingSubProcessesMode"]}'
			os.makedirs(roomPath+room+exportName,mode=0o777)
			outFile = f'{roomPath}{room}{exportName}/{exportName}.json'
			print(f'outFile: {outFile}')
			#save file
			write_json(newSession,outFile)


"""
Script takes mutect VCF and and cgppindel vcf to:
    1. Filter and annotate both VCFs against defined transcripts
    2. Create vcf for import into BSVI
    3. Create excel workbook with sheets for each panel

BSVI vcf requires multiallelic sites are decomposed and that all genotype
fields are simplified to 0/1 (from, for example, 0/0/1/0).

Excel workbook for scientists to look at variants have sheets, one for each
panel, with annotations and preformatted report text for Epic.

Inputs are vcf files after VEP annotation and filtering:
    - *_allgenesvep.vcf (specified with -a argument)
    - *_[panel-name]vep.vcf (any number, specified with -v argument)
        e.g. *_lymphoidvep.vcf *_myeloidvep.vcf
    - pindel vcf (specified with -p)

Outputs:
    - annotated and filtered mutect2 and pindel vcf
    - annotated and filtered panel vcfs
    - bsvi vcf
    - tsv with variants
    - excel workbook with variants, one panel per sheet
"""

import argparse
from pathlib import Path
import re
import sys
import numpy as np
import pandas as pd

def parse_args():
    """
    Parse command line args

    Args: None

    Returns:
        - args (Namespace): object containing parsed arguments.
    """
    parser = argparse.ArgumentParser(
        description=(
            'Generates a tsv of variants, a decomposed VCF for BSVI,'
            'and an excel workbook of variants for each panel'
        )
    )

    parser.add_argument(
        '-a', '--allgenes',
        help='allgenes VCF from which to generate tsv file and BSVI vcf'
    )

    parser.add_argument(
        '-v', '--vcfs', nargs='*',
        help='Panel filtered VCF(s) from which to generate excel workbook'
    )
    parser.add_argument(
        '-p', '--pindel',
        help='Output VCF from cgppindel'
    )

    args = parser.parse_args()

    return args

def read_vcf(input_vcf):
    """
    Reads vcf into pandas df, returns header as a list for output (bsvi) vcf

    Args:
        - input_vcf (file): vcf file to read in

    Returns:
        - vcf_df (df): df of variants from vcf
        - vcf_header (list): header from vcf, for writing output (bsvi) vcf
    """
    # read in vcf to get header lines
    with open(input_vcf, 'r') as fh:
        vcf_data = fh.readlines()

    vcf_header = []

    for line in vcf_data:
        if line.startswith('#'):
            # dump out header to list to write back for output vcf
            vcf_header.append(line)
        else:
            break

    if "TUMOUR" in vcf_header[-1]:
        # pindel vcf has NORMAL & TUMOUR instead of SAMPLE
        cols = [
            "CHROM", "POS", "ID", "REF", "ALT", "QUAL", "FILTER", "INFO",
            "FORMAT", "NORMAL", "TUMOUR"
        ]
    else:
        cols = [
            "CHROM", "POS", "ID", "REF", "ALT", "QUAL", "FILTER", "INFO",
            "FORMAT", "SAMPLE"
        ]

    # read vcf records into df
    vcf_df = pd.read_csv(
        input_vcf, sep="\t", comment='#', names=cols, compression='infer'
    )

    return vcf_df, vcf_header

def mod_genotype(vcf_df):
    """
    Overwrite GT with 0/1 when it's derived from a multiallelic
    e.g. 0/0/1/0 -> 0/1 and 0/0/0/1 -> 0/1

    Args:
        - input_vcf (file): vcf file passed at cmd line

    Returns:
        - vcf_df (df): df of variants with modified genotype
    """
    for index, row in vcf_df.iterrows():
        # loop over rows, change genotype if contains greater than 2 fields
        sample = row['SAMPLE'].split(':')

        # sense check that genotype field doesn't have anything funky,
        # if it does then it can be reviewed manually
        assert len(sample[0]) >= 3, \
            f'Genotype field has < 3 characters: {sample[0]} for sample: {row}'

        if len(sample[0]) > 3:
            # >3 => not 0/1 => modify
            sample[0] = '0/1'
            sample = ':'.join(sample)

            # write new entry back to row
            vcf_df.at[index, 'SAMPLE'] = sample

    return vcf_df

def get_field_index(column, field):
    """
    Returns the index of a given field in a column with values separated by ':'

    Args:
        - column (pd.Series): df column to get field index
        - field (str): field to get index of

    Returns: index of given field
    """
    return list(column.apply(lambda x: x.split(':').index(field)))[0]

def get_field_value(column, index):
    """
    Given a column with ':' separated values and index, return a
    series of values with the value split by the index

    Args:
        - column (pd.Series): column to split and return from
        - index (int): index to select

    Returns: series of values selected by index
    """
    return column.apply(lambda x: int(x.split(':')[index]))

def filter_common(vcf_df):
    """
    Filters for common variants by prev_count > 50% & synonymous variants
    EXCEPT in TP53 and GATA2

    Args: vcf_df (df): df of variants

    Returns:
        - vcf (df): df of unfiltered variants
        - filter_vcf (df): df of filtered out common variants
    """
    # prev_count formatted as prev_ac/prev_samples (i.e. 45/205)
    filter_idxs = np.where((
        vcf_df['Prev_Count'].apply(
            lambda x: (int(x.split("/")[0]) / int(x.split("/")[1])) > 0.5
        ) | (
            vcf_df['Consequence'] == 'synonymous_variant'
        )) & (
            np.logical_not(vcf_df['SYMBOL'].isin(['TP53', 'GATA2']))
    ))

    # filter df by indices of filter conditions
    filtered_df = vcf_df.loc[filter_idxs]
    vcf_df = vcf_df.drop(filter_idxs[0])

    return vcf_df, filtered_df

def split_info(vcf_df) -> pd.DataFrame:
        """
        Splits out the INFO column of vcf to all separate values, excluding
        the CSQ values. This transforms as:

        -----------------------------------------------------------------------
        INFO
        -----------------------------------------------------------------------
        AC=1;AF=0.5;AN=2;BaseQRankSum=-0.244;DB;DP=215;......
        AC=1;AF=0.5;AN=2;BaseQRankSum=-0.291;;DB;DP=207;.....
        AC=2;AF=1;AN=2;DB;DP=310;ExcessHet=3.0103;FS=0;......
        -----------------------------------------------------------------------
                                      |
                                      |
                                      ▼
        -----------------------------------------------------------------------
         AC  |  AF  |  AN  |  DB  |  DP  |  ExcessHet  |  BaseQRankSum  |  FS
        -----------------------------------------------------------------------
         1   |  0.5 |  2   | True |  215 |             |  -0.244        |
        -----------------------------------------------------------------------
         1   |  0.5 |  2   | True |  207 |             |  -0.291        |
        -----------------------------------------------------------------------
         2   |  1   |  2   | True |  310 |  3.0103     |                |  0
        -----------------------------------------------------------------------

        Parameters
        ----------
        vcf_df : pd.DataFrame
            dataframe of all variants from a vcf

        Returns
        -------
        vcf_df : pd.DataFrame
            dataframe of all variants from a vcf with separated INFO fields
        """
        # get the key value pairs of INFO data
        info_pairs = [x.split(';') for x in vcf_df['INFO'].tolist()]
        info_pairs = [[x for x in lst if x] for lst in info_pairs]

        # get unique list of keys from key=value pairs in INFO from all rows
        info_keys = sorted(list(set([
            x.split('=')[0] if '=' in x else x for pair in info_pairs for x in pair
        ])))
        info_keys = [x for x in info_keys if x]  # can end up with empty string

        info_values = []

        # info_pairs -> list of list of pairs, one list per variant
        for variant_pairs in info_pairs:
            # for every variants values, split them out to dict to add to df
            pair_values = {}

            for pair in variant_pairs:
                if '=' in pair:
                    # key value pair
                    key, value = pair.split('=')
                else:
                    # Flag value present (e.g STR)
                    key, value = pair, True

                pair_values[key] = value

            info_values.append(pair_values)

        # build df of values to add to main df
        info_df = pd.DataFrame(info_values, columns=info_keys)
        vcf_df = pd.concat([vcf_df, info_df], axis=1)

        # drop INFO as we fully split them out
        vcf_df.drop(['INFO'], axis=1, inplace=True)

        # remove CSQ_ prefix from column names
        vcf_df.columns = [
            x.replace('CSQ_', '') if x.startswith('CSQ_') else x
            for x in vcf_df.columns.tolist()
        ]

        return vcf_df

def df_report_formatting(panel, vcf_df):
    """
    Formats df of vcf records for report with INFO column split out to
    individual columns. Expects min. 15 '|' separated fields in INFO
    column to split out.

    Args:
        - panel (str): panel name of vcf
        - vcf_df (df): df of variants from vcf

    Returns:
        - vcf_df (df): df of variants with fun formatting
    """
    # split out fields from INFO column to separate columns
    if not vcf_df.empty:
        vcf_df = split_info(vcf_df=vcf_df)
    else:
        # empty df from vcf with no variants => won't have correct columns from
        # splitting, add these as empty columns to the df to not break downstream
        vcf_df[[
            'samplename', 'SYMBOL', 'Transcript_ID', 'EXON', 'HGVSc', 'HGVSp',
            'Protein_ID', 'Consequence', 'Read_Depth', 'FILTER', 'ClinVar',
            'ClinVar_CLNSIG', 'ClinVar_CLNDN', 'COSMIC', 'dbSNP', 'gnomAD_AF',
            'CADD_PHRED', 'AF%', 'Prev_Count', 'Report_text'
        ]] = pd.NA

        # set order
        vcf_df = vcf_df[[
            'samplename', 'CHROM', 'POS', 'SYMBOL', 'Transcript_ID', 'EXON',
            'HGVSc', 'HGVSp', 'Protein_ID', 'Consequence', 'Read_Depth',
            'AF%', 'FILTER', 'ClinVar', 'ClinVar_CLNSIG',
            'ClinVar_CLNDN', 'COSMIC', 'dbSNP', 'gnomAD_AF', 'CADD_PHRED',
            'Prev_Count', 'Report_text'
        ]]

        return vcf_df

    # first get total number of samples across all variants, remove '.' or
    # empty strings if present
    uniq_prev_ns = list(set(filter(None, vcf_df['Prev_NS'])))
    if '.' in uniq_prev_ns:
        uniq_prev_ns.remove('.')
    if '' in uniq_prev_ns:
        uniq_prev_ns.remove('')

    # handle cases where vcf has very few variants and all haven't been seen
    # previously => no prev_ns values => empty list, set to empty string
    uniq_prev_ns = next(iter(uniq_prev_ns), '')

    # those not previously seen will have empty string, fill appropriately to
    # display as 0/{total}
    vcf_df['Prev_AC'] = vcf_df['Prev_AC'].apply(
        lambda x: str(0) if x == "." or x == "" else x)

    vcf_df['Prev_NS'] = vcf_df['Prev_NS'].apply(
        lambda x: uniq_prev_ns if x == "." or x == "" else x)

    vcf_df['Prev_Count'] = vcf_df['Prev_AC'] + '/' + vcf_df['Prev_NS']

    if panel == "pindel":
        # handle pindel vcf, AF to be calculated from TUMOUR field
        # this is calculated as (PU + NU) / (PR + NR)
        # values are described in table 15.7.3 here:
        # # https://www.ncbi.nlm.nih.gov/pmc/articles/PMC6097606/
        caller = "Pindel"

        # get indices of required fields
        pu_index = get_field_index(vcf_df['FORMAT'], 'PU')
        nu_index = get_field_index(vcf_df['FORMAT'], 'NU')
        pr_index = get_field_index(vcf_df['FORMAT'], 'PR')
        nr_index = get_field_index(vcf_df['FORMAT'], 'NR')

        # get values for given field from TUMOUR column
        pu_values = get_field_value(vcf_df['TUMOUR'], pu_index)
        nu_values = get_field_value(vcf_df['TUMOUR'], nu_index)
        pr_values = get_field_value(vcf_df['TUMOUR'], pr_index)
        nr_values = get_field_value(vcf_df['TUMOUR'], nr_index)

        # calculate af & format as pct to 1dp
        af_values = (pu_values + nu_values) / (pr_values + nr_values)
        af_pcts = af_values.apply(lambda x: '{:.1f}'.format(float(x * 100)))

        vcf_df.insert(16, 'Pindel_AF%', af_pcts)

        # depth is also not in the INFO field so will be blank for pindel,
        # calulating this as PR + NR
        vcf_df['Read_Depth'] = pr_values + nr_values
    else:
        # mutect2 vcf
        caller = "Mutect2"

        # get index of AF in format column, should all be same and have a
        # list with 1 value, used to get AF from the sample column
        af_index = list(set(vcf_df['FORMAT'].apply(
            lambda x: x.split(':').index('AF'))
        ))

        # sense check all AF at same index
        assert len(af_index) == 1, \
            'Error in FORMAT column, AF not all at same index.'

        # get AF values from sample column add to new AF column, convert to %
        af_index = af_index[0]
        af_values = vcf_df["SAMPLE"].apply(
            lambda x: x.split(':')[af_index]
        ).apply(
            lambda x: format(float(x) * 100)
        ).apply(
            lambda x: '{:.1f}'.format(float(x))
        )

        vcf_df.insert(16, 'Mutect2_AF%', af_values)

        vcf_df.rename(columns={'DP': 'Read_Depth'}, inplace=True)

    # cosmic annotation returns duplicates for each record in cosmic vcf
    # turn to set to be unique, join in case there is more than one
    vcf_df['COSMIC'] = vcf_df['COSMIC'].apply(
        lambda x: (','.join(set(x.split('&')))) if x else x
    )

    vcf_df['dbSNP'] = vcf_df['Existing_variation'].str.split(r'\&|\||,').apply(
        lambda x: ','.join((y for y in x if y.startswith('rs')))
    )

    # Include word exon in exon field to overcome excel
    vcf_df['EXON'] = vcf_df['EXON'].apply(
        lambda x: 'exon ' + x if x and x != "." else None
    )

    # scientists are picky and want NM_ and NP_ changing in HGVS
    vcf_df['Transcript_ID'] = vcf_df['HGVSc'].str.split(':').apply(
        lambda x: ','.join((y for y in x if y.startswith('NM')))
    )
    vcf_df['HGVSc'] = vcf_df['HGVSc'].str.split(':').apply(
        lambda x: ','.join((y for y in x if y.startswith('c')))
    )
    vcf_df['Protein_ID'] = vcf_df['HGVSp'].str.split(':').apply(
        lambda x: ','.join((y for y in x if y.startswith('NP')))
    )
    vcf_df['HGVSp'] = vcf_df['HGVSp'].str.split(':').apply(
        lambda x: ','.join((y for y in x if y.startswith('p')))
    )
    regex = re.compile(r'^[p\.]*')
    vcf_df['HGVSp'] = vcf_df['HGVSp'].apply(
        lambda x: regex.sub('p.(', x) + ')' if x else None)

    # add interestingly formatted report text column
    vcf_df['Report_text'] = vcf_df[vcf_df.columns.tolist()].apply(
        lambda x: (
            f"{x['SYMBOL']} {x['Consequence']} "
            f"{'in ' + x['EXON'].split('/')[0] if x['EXON'] else ''} \n"
            f"HGVSc: {x['HGVSc'] if x['HGVSc'] else 'None'} \n"
            f"HGVSp: {x['HGVSp'] if x['HGVSp'] else 'None'} \n"
            f"COSMIC ID: {x['COSMIC'] if x['COSMIC'] else 'None'} \n"
            f"dbSNP: {x['dbSNP'] if x['dbSNP'] else 'None'} \n"
            f"""Allele Frequency (VAF): {
                str(x[f'{caller}_AF%']) + '%' if x[f'{caller}_AF%'] else 'None'
            }"""
        ), axis=1
    )

    # get sample name from filename (should always be >>20 characters)
    if len(fname) > 19:
        samplename = fname[0:20] + '...'
    else:
        samplename = fname
    vcf_df['samplename'] = samplename

    # select and re-order df columns
    vcf_df = vcf_df[[
        'samplename', 'CHROM', 'POS', 'SYMBOL', 'Transcript_ID', 'EXON', 'HGVSc',
        'HGVSp', 'Protein_ID', 'Consequence', 'Read_Depth', f'{caller}_AF%',
        'FILTER', 'ClinVar', 'ClinVar_CLNSIG', 'ClinVar_CLNDN', 'COSMIC',
        'dbSNP', 'gnomAD_AF', 'CADD_PHRED', 'Prev_Count', 'Report_text'
    ]]

    return vcf_df

def to_report_formatting(col_names):
    """
    Build df of required placeholder text for the to report sheet

    Args: col_names (list): list of column names from a panel df

    Returns: report_df (df): formatted dataframe wth placeholder text as report
        template in specific cells
    """
    # add the column names for when they paste in reported variants
    report_df = pd.DataFrame(columns=col_names).astype('object')

    # add 12 empty rows as padding for visualise niceness
    report_df = report_df.append(
        [pd.Series([np.nan])] * 12).reindex(col_names, axis=1)

    col1_labels = [
        "Run QC", "250x", "Contamination", "Total reads M", "Fold 80",
        "Insert Size",
    ]

    for label in col1_labels:
        # set each of the col1 labels as a field in first column
        report_df = report_df.append(
            {report_df.columns[0]: label}, ignore_index=True
        )

    report_df.at[12, report_df.columns[3]] = "Sample QC"

    col6_labels = ["Analysed by", "Date", "Subpanel analysed"]
    col6_label_idx = 8

    for label in col6_labels:
        # start at row 8, add each of col6 labels as field in column 6
        report_df.at[col6_label_idx, report_df.columns[5]] = label
        col6_label_idx += 1

    return report_df

def set_column_widths(worksheet, df):
    """
    Sets column widths dyanmically based on cell content

    Args:
        - worksheet (xlsxwriter sheet object): sheet to modify
        - df (pd.DataFrame): dataframe for sheet to set widths from
    Returns:
        - - worksheet (xlsxwriter sheet object): xlsx sheet with new widths
    """
    for idx, col in enumerate(df):
        # specific formatting for columns with potential very long text
        if "ClinVar" in col:
            max_len = 15
        elif "Report_text" in col:
            max_len = 40
        else:
            series = df[col]
            max_len = max((
                series.astype(str).map(len).max(), len(str(series.name))
            )) + 2
        worksheet.set_column(idx, idx, max_len)  # set column width

    return worksheet

def write_bsvi_vcf(fname, bsvi_df, bsvi_vcf_header):
    """
    Write df of variants with modified genotype to vcf for BSVI

    Args:
        - fname (str): name of allgenes input vcf, used to name output file
        - bsvi_df (df): df of variants with modified genotype for bsvi
        - bsvi_vcf_header (list): vcf header from original vcf

    Returns: None

    Outputs: .vcf file of variants for importing into bsvi
    """
    vcf_fname = fname.replace('allgenesvep', 'bsvi')

    with open(vcf_fname, 'w') as f:
        for line in bsvi_vcf_header:
            # write header to vcf
            f.write(line)

        # apend variants to vcf
        bsvi_df.to_csv(f, sep='\t', header=False, index=False)

def write_tsv(fname, all_genes_df):
    """
    Write tsv file of all genes from allgenes vcf

    Args:
        - fname (str): name of allgenes input vcf, used to name output file
        - all_genes_df (df): df of all genes from input vcf

    Returns: None

    Outputs: .tsv file with all gene variants
    """
    tsv_fname = fname.replace('allgenesvep.vcf', 'allgenes.tsv')

    # write tsv file
    with open(tsv_fname, 'w') as tsv:
        all_genes_df.to_csv(tsv, sep='\t', header=True, index=False)

def write_xlsx(fname, vcfs_dict):
    """
    Write xlsx file of panel filtered vcfs, each panel goes to a separate tab

    Args:
        - fname (str): name of allgenes input vcf, used to name output file
        - vcfs_dict (dict): dict of panel vcfs

    Returns: None

    Outputs: .xlsx file of panel vcfs in separate tabs
    """
    excel_fname = fname.replace('allgenesvep.vcf', 'panels.xlsx')

    print(f"Writing xlsx file to: {excel_fname}")

    # write excel
    writer = pd.ExcelWriter(excel_fname, engine="xlsxwriter")
    workbook = writer.book

    # get the column names of a df to write to report sheet
    header = vcfs_dict[next(iter(vcfs_dict))].columns
    report_df = to_report_formatting(col_names=header)
    report_df.to_excel(writer, sheet_name='to report', index=False)

    worksheet = writer.sheets['to report']

    # set specific cells to be bold in to report tab
    for cell in ['A14:A14', 'D14:D14', 'F10:F12']:
        header_format = workbook.add_format({'bold': True})
        worksheet.conditional_format(
            cell, {'type': 'no_errors', 'format': header_format}
        )

    # set dynamic column widths on cell content
    worksheet = set_column_widths(worksheet, report_df)
    worksheet.set_row(0, 12)

    for panel_name, vcf_df in vcfs_dict.items():
        # loop over panel dfs, write to sheet & apply formatting
        vcf_df.to_excel(writer, sheet_name=panel_name, index=False)
        # fun excel formatting
        worksheet = writer.sheets[panel_name]
        wrap_format = workbook.add_format({'text_wrap': True})
        worksheet.set_column(1, 21, 15)
        worksheet.set_column(22, 22, 70, wrap_format)
        worksheet.set_default_row(100)
        worksheet.set_row(0, 12)

        # set dynamic column widths on cell content
        worksheet = set_column_widths(worksheet, vcf_df)

    writer.save()

if __name__ == "__main__":
    args = parse_args()

    # read allgenes vcf into df, retain header for output vcf
    all_genes_df, all_genes_df_header = read_vcf(args.allgenes)

    # build dict of panel vcfs, allows to store panel name with df
    vcfs_dict = {}

    for vcf in args.vcfs:
        # loop over panel vcfs, read into df and add to dict
        # get panel name from vcf name
        panel = Path(vcf).stem.split('_')[-1].rstrip('vep')
        panel_df, _ = read_vcf(vcf)  # don't retain vcf header as not needed

        vcfs_dict[panel] = panel_df

    # read in pindel vcf and add to vcfs_dict to be included in excel
    pindel_df, _ = read_vcf(Path(args.pindel))
    vcfs_dict["pindel"] = pindel_df

    # modify genotype of bsvi vcf
    bsvi_vcf_df = all_genes_df.copy(deep=True)
    bsvi_vcf_df = mod_genotype(bsvi_vcf_df)

    # get filename, use name of allgenes vcf as prefix for all
    fname = str(Path(args.allgenes).name)

    # apply formatting to allgenes vcf df for tsv file
    all_genes_df = df_report_formatting(fname, all_genes_df)

    formatted_dfs = {}

    # apply formatting to each panel df for xlsx file
    for panel, vcf_df in vcfs_dict.items():
        vcf_df = df_report_formatting(panel, vcf_df)
        formatted_dfs[panel] = vcf_df

        if panel == 'myeloid':
            # filtering myeloid panel for common variants
            vcf_df, filter_vcf_df = filter_common(vcf_df)
            formatted_dfs['myeloid'] = vcf_df
            formatted_dfs['myeloid_filtered'] = filter_vcf_df

    write_bsvi_vcf(fname, bsvi_vcf_df, all_genes_df_header)
    write_tsv(fname, all_genes_df)
    write_xlsx(fname, formatted_dfs)


#LDFB analysis and synthesis filter bank, one after the other, for time measurement
#Gerald Schuller, November 2017

from keras_LDFBanalysis import *
from keras_LDFBsynthesis import *
from sound import sound
import time
from scipy.signal import freqz

N=1024 #Number of filters, stride
filtlen=4096 #Length of filter impulse response
modelana = generate_model_ana(N,filtlen)     # Compile an neural net analysis filter bank
modelsyn = generate_model_syn(N,filtlen)     # " Synthesis filter bank

#Test Perfect reconstruction and Delay:
X=np.arange(20*N)

a=time.time()
Y=keras_LDFB_ana(X,modelana)
b=time.time()
xrek= keras_LDFB_syn(Y,modelsyn)
c=time.time()
print("Analysis time:", b-a)
print("Synthesis time:", c-b)
#Plots reconstructed ramp if ok:
#print("xrek.shape=", xrek.shape)
#print("xrek=", xrek)
plt.plot(-X)
plt.plot(xrek)
plt.legend(('Original', 'Reconstructed'))
plt.title("Original and Recontructed Test Signal")
plt.show()
#Delay is relative to the delay of an orthogonal filter bank of equal length!
#Here: length=4096 taps, analysis-synthesis delay of an equal length orthogonal filter bank: 4095 samples.
#Low delay filter bank: 2047 samples delay, difference: -2048 samples
#Detailed comparison of the 2 ramp signals gives indeed exactly -2048!

#get and plot the synthesis impulse response of subband 0:
Y=np.ones((10,N))*0.0;
Y[0,0]=1.0
xrek= keras_LDFB_syn(Y,modelsyn)
plt.plot(xrek)
plt.title('Impulse Response of Subband 0 of the Synthesis Low Delay FB')
w,H=freqz(xrek,worN=4096)
plt.figure()
plt.plot(w,20*np.log10(np.abs(H)/np.max(np.abs(H))+1e-6))
plt.title('Its Magnitude Frequency Response')
plt.ylabel('dB Attenuation')
plt.xlabel('Normalized Frequency (pi is Nyquist Freq.)')
plt.show()

fs, X= wav.read('test.wav')
print("fs=", fs)
print("X.shape=", X.shape)
X=X*1.0/2**15
startime=time.time()
#Analysis Filter Bank:
Y=keras_LDFB_ana(X,modelana)
#Synthesis Filter Bank:
xrek= keras_LDFB_syn(Y,modelsyn)
print("xrek.shape=", xrek.shape) #output may be a little shorter, to make signal fit into an integer number of blocks of size N!
endtime=time.time()
print("Duration analysis-synthesis: ", endtime-startime)

#Delay estimation:
mse=np.zeros(3*N)
for n in range(3*N):
   #Mean squared error:
   minlen=min(len(X[n:]),len(xrek))
   #-X because of the sign change after reconstruction!
   mse[n]=np.mean((-X[n:(n+minlen)]-xrek[:minlen])**2)
print("Delay reconstructed to original=", np.argmin(mse), "np.min(mse)=",np.min(mse) ) #argmin=2048, min=4e-5
#Keras convolutional layers already compensate for a delay of the length of the filters!
#plt.plot(mse)
#plt.title("Mean Squared Error for different Delays")
plt.show()

os.system('espeak -ven -s 120 '+'"The output of the synthesis Low Delay Filter Bank"')
sound(2**15*xrek,fs)


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Tue Oct 26 22:23:36 2021

@author: wangshiyuan
"""
import xarray as xr

import pandas as pd
import numpy as np
# import datetime as dt
from time import time
from glob import glob
import os
# import matplotlib.pyplot as plt

from multiprocessing import Pool

'''===================basic settings=================='''

N = 4
# means the value of each pp would be determined by 4 nearest pp's value.
EXPONENT = 2
LATBOUND = [-10, 60]
LONBOUND = [95, 180]

STARTDATE = '1960-01-01'
ENDDATE = '2018-01-01'

grid_dir = '/home/waterlab/Wang/bachelor_thesis/downloads/JRA/rawdownloads/uas/anl_mdl.033_ugrd.reg_tl319.196101_196112.wang528867'

rp_dir = '../downloads/GCMs/MRI/*.nc'

write_dir_base = '../interpolated_gcms_mon'

'''===================for testing=================='''

'''----0-----'''
'''values in each Map'''
# the default setting shoud be [None, None]
bnd = [None, None]
# bnd = [10, 10]
# these limit the program only to predict few points for each time step

'''-----1-----'''
STARTDATE = '1960-01-01'
ENDDATE = '2018-01-01'
# the default setting shoud be theirselves.
# the default should be like: STARTDATE = STARTDATE

'''-----2-----'''
timeMeasurement = True
# determine whether to measure the run time or a specific function or not

'''-----3-----'''
outputDataType = 'dataset'
# can only set as 'dataarray' or 'dataset'

cnt = 0

def measureRunTime(func):
    def timer(*args, **kwargs):
        start = time()
        ret = func(*args, **kwargs)
        end = time()
        print(
            f'func \'{func.__name__ }\' run time {end - start:04f} seconds, {(end-start)/60 :03f} min')
        return ret
    return timer if timeMeasurement is True else func

class Point():
    def __init__(self, lat=None, lon=None, source=None, variableType=None):
        self.lat = float(lat)
        self.lon = float(lon)
        self.dist = float('inf')
        self.value = None
        self.loc = np.array([lat, lon])
        # self.idx = []
        self.source = source
        self.type = variableType

    def getValue(self):
        # print(self.source, self.lat, self.lon)

        if self.source and self.lat and self.lon and self.type:

            value = float(self.source.sel(
                lat=self.lat, lon=self.lon, method='nearest')[self.type])
        else:
            raise ValueError(
                'need to know the lat, lon, source and variableType to get the value. Now the three are {bool(self.source)}, {bool(self.lat)}, {bool(self.lon)}, {bool(self.type)!')
        self.value = value

def newVal(weights, values):
    return (weights @ values)/sum(weights)

def droppingVariablesOfds(ds, distToBound=0):

    return ds.sel(lat=slice(LATBOUND[0]-distToBound, LATBOUND[1]+distToBound), lon=slice(LONBOUND[0]-distToBound, LONBOUND[1]+distToBound))

def find4Nearest(rp_ds, pp_loc, variableType="uas"):

    pass

def findNearestPoints(rp_ds, pp_loc, N=N, variableType="uas"):
    # given a rp map at one time step, and a exact location to predict, reuturn an array concluds 4 nearest Points
    # pp_loc is also a one-dim
    # means the value of each pp would be determined by 4 nearest ensional array and has two numbers (lat, lon)
    points, res = [], []
    lat, lon = pp_loc[0], pp_loc[1]
    # print(pp_loc)

    try:
        rp_ds = rp_ds.sel(lat=slice(lat-2, lat+2), lon=slice(lon-2, lon+2))
    except:
        with open(f'/home/waterlab/Wang/bachelor_thesis/interpolated_gcms_mon/MRI/{variableType}/error.txt', 'a') as f:
            info = [rp_ds, rp_ds.lat, rp_ds.lon, lat, lon, variableType]
            for s in info:
                f.write(str(s)+'\n')

        rp_ds = rp_ds

    # this is because we don't neet to check evey point in the grid to find 4 nearest points

    for lat in rp_ds.lat:
        for lon in rp_ds.lon:
            point = Point(lat, lon, source=rp_ds, variableType=variableType)

    # for i in lat_idxs:
    #     for j in lon_idxs:
    #         point = Point(float(rp_ds.lat[i]), float(rp_ds.lon[j]))
            point.dist = np.linalg.norm(point.loc - pp_loc)
    #         point.idx = [i, j]
    #         # print(point.idx)
            points.append(point)

            # print(f' for {pp_loc} appened the rp at {point.loc}')

    if len(points) == N:
        for point in points:
            point.getValue()
        res = points
    else:

        for point in sorted(points, key=lambda point: point.dist)[:N]:
            point.getValue()
            res.append(point)

        # points = sorted(points, key=lambda point: point.dist)

        # for i in range(N):
        #     points[i].getValue()
        #     res.append(points[i])
            # print(
            #     f'for pp at {pp_loc} found No.{i} nearest rp, at {res[i].loc} with distance {res[i].dist}')
    return res

@ measureRunTime
def interpolatedMap(grid_ds, rp_ds, time, bnd: list = bnd, exponent=EXPONENT, modelType=None, variableType="uas"):
    # given (a grid map at one time step) and (a rp map at one time step), pridict the value of each point on the grid

    # grid_ds = grid_ds.isel(time=0)
    xbnd, ybnd = bnd[0], bnd[1]
    data = np.zeros((len(grid_ds.lat), len(grid_ds.lon)))
    # print(data.shape)
    for i, lat in enumerate(grid_ds.lat):
        for j, lon in enumerate(grid_ds.lon):

            pp_loc = np.array((float(lat), float(lon)))

            nearestNPoints = findNearestPoints(
                rp_ds, pp_loc, variableType=variableType)

            if nearestNPoints[0].dist == 0.0:
                # to avoid dividing by zero
                data[i][j] = nearestNPoints[0].value
            else:
                values = [point.value for point in nearestNPoints]
                # print(values)
                dists = [point.dist for point in nearestNPoints]
                weights = 1 / np.power(dists, exponent)
                data[i][j] = newVal(weights, values)

            # print(
            #     f'the value at {pp_loc} has been predicted to {data[i][j]}, in [{i}][{j}]. And its 4 nearest is {values}')
            # print(f"[{i}][{j}] is over with value {data[i][j]}")
            # print(f"[{i}][{j}] is over")

            if ybnd and j >= ybnd:
                break

        if xbnd and i >= xbnd:
            break
        if i % 10 == 0 or i == 1:
            print(f"gcm {modelType} [{i} of {len(data)}] is over")
    # print(data.shape)
    assert data.size > 100, 'interpolate error'

    data = data[:, :, np.newaxis]
    if outputDataType == 'dataarray':

        new = xr.DataArray(data, coords=[
            grid_ds.lat, grid_ds.lon, time], dims=['lat', 'lon', 'time'])
    elif outputDataType == 'dataset':
        new = xr.Dataset(
            {
                variableType: (["lat", "lon", "time"], data),
            },
            coords={
                "lat": (["lat"], grid_ds.lat.to_index()),
                "lon": (["lon"], grid_ds.lon.to_index()),
                'time': (['time'], time)
            },
        )
    else:
        new = None
        raise ValueError(
            'need to set the outputDataType to either \'dataarray\' or \'dataset\'')

    print(f'interpolation for {time.strftime("%Y-%m-%d")[0]} is over')
    return new

@ measureRunTime
def interpolatedDataset(grid_ds, rp_ds, startTime=STARTDATE, endTime=ENDDATE, modelType=None, variableType="uas"):
    assert len(rp_ds.lat) > 5, 'lat error'
    timeIdx = 0
    rp_ds = rp_ds.sel(time=slice(startTime, endTime))

    # print(rp_ds)
    times = rp_ds['time'].to_index()

    concacted = interpolatedMap(grid_ds, rp_ds.sel(
        time=times[0]), times[:1], modelType=modelType, variableType=variableType)
    assert concacted[variableType].data.size > 100, 'interpolatedMap error'
    for timeIdx in range(1, len(times)):

        new = interpolatedMap(grid_ds, rp_ds.sel(
            time=times[timeIdx]), times[timeIdx:timeIdx+1], modelType=modelType, variableType=variableType)
        concacted = xr.concat([concacted, new], dim='time')
        print(
            f"interpolation and concaction for {startTime} to {times[timeIdx].strftime('%Y-%m-%d')} is over")
    return concacted

def preprocessed(ds, modelType='', startdate=None, enddate=None, variableType=None):
    def JRA_preprocess(ds):
        if "u" in ds:
            ds = ds.rename({"u": "uas"})
        if "v" in ds:
            ds = ds.rename({"v": "vas"})
        ds = ds.rename(
            {'latitude': 'lat', 'longitude': 'lon'})
        ds = ds.sortby('lat')
        ds = droppingVariablesOfds(ds)
        ds = ds.transpose('lat', 'lon', 'time')
        return ds

    def MRI_preprocess(ds):
        # print('here\n\n\n\n')
        # ds = ds.drop_vars(['time_bnds', 'lon_bnds', 'lat_bnds'])
        ds["time"] = ds.time.to_index().map(lambda t: t.replace(day=1, hour=0))

        ds = ds.transpose('lat', 'lon', 'time')

        return ds

    def MIROC6_preprpocess(ds):
        return MRI_preprocess(ds)

    def CNRM_ESM_preprocess(ds):
        # ds = ds.drop_vars(['time_bounds'])
        ds["time"] = ds.time.to_index().map(lambda t: t.replace(day=1, hour=0))

        ds = ds.transpose('lat', 'lon', 'time')
        return ds

    def unknown_preprocess(ds):

        raise ValueError(
            'The model cannot be preprocessed. Please do preprocessing by yourself')

        return ds

    model_list = {'JRA': JRA_preprocess,
                  'MRI': MRI_preprocess,
                  'MIROC6': MIROC6_preprpocess,
                  'CNRM-ESM2-1': CNRM_ESM_preprocess}

    def main_preprocess(ds, modelType=modelType, startdate=startdate, enddate=enddate, variableType=variableType):
        if startdate and enddate:
            ds = ds.sel(time=slice(startdate, enddate))
            # print('/n/n/n/n', 'cutdate',)

        if variableType:
            ds = ds[[variableType]]
        # print(ds)

        ds = model_list.get(modelType, unknown_preprocess)(ds)

        return ds
    print(f'{modelType} has been preprocessed')

    return main_preprocess(ds)

def main(args):
    # interpolate one month of data
    grid_ds, filepath, modelType, startdate, enddate, variableType = args[
        0], args[1], args[2], args[3], args[4], args[5]

    rp_ds = xr.open_dataset(filepath, engine='h5netcdf')

    rp_ds = preprocessed(rp_ds, modelType=modelType,
                         startdate=startdate, enddate=enddate, variableType=variableType)

    assert len(rp_ds.lat) > 5, 'preprocess'

    try:
        out = interpolatedDataset(grid_ds, rp_ds, modelType=modelType,
                                  startTime=startdate, endTime=enddate, variableType=variableType)
    except:
        return

    # print(out)

    def checked(out):
        flag = [True]

        if out[variableType].data.size < 100:
            flag.append(False)

        if np.isnan(out[variableType].data).any():
            flag.append(False)

        return flag

    if all(checked(out)):

        write_dir = write_dir_base + f"/{modelType}/{variableType}"
        if not os.path.exists(write_dir):
            os.makedirs(write_dir)

        out.to_netcdf(
            write_dir + f"/{modelType}_{startdate.year}_{variableType}.nc", engine="h5netcdf")

        global cnt
        cnt += 1

    else:
        with open(f"{write_dir}/message.txt", 'a') as f:
            f.write(f"{startdate.year} had error!\n", checked(out), '\n')

if __name__ == '__main__':
    for file in glob('../downloads/JRA/rawdownloads/**/*.idx', recursive=True):
        os.remove(file)

    grid_ds = xr.open_dataset(grid_dir, engine='cfgrib')
    grid_ds = preprocessed(grid_ds, modelType='JRA')
    grid_ds = grid_ds.isel(time=0)
    # print(grid_ds, '\n\n\n\n\n')
    arg_list = []

    for filepath in glob(rp_dir):
        ds = xr.open_dataset(filepath, engine="h5netcdf")

        modelType = os.path.basename(os.path.dirname(filepath))
        ds = preprocessed(ds, modelType=modelType,
                          startdate=STARTDATE, enddate=ENDDATE)
        if len(ds.time) == 0:
            continue

        timeIdx = ds.time.to_index()
        startdate_list = pd.date_range(timeIdx[0], timeIdx[-1], freq="YS")
        enddate_list = pd.date_range(timeIdx[0], timeIdx[-1], freq="Y")
        # print(startdate_list, enddate_list)
        for startdate, enddate in zip(startdate_list, enddate_list):
            for variableType in ds.data_vars:

                args = (grid_ds, filepath, modelType,
                        startdate, enddate, variableType)
                if not os.path.exists(write_dir_base + f"/{modelType}/{variableType}/{modelType}_{startdate.year}_{variableType}.nc"):

                    arg_list.append(args)

    # print(grid_ds)
    # for args in arg_list:
        # print(args[3:])
    #     print(args[2:], "\n\n")
    #     # print(args[3:5])
    print(len(arg_list))
    expectH = len(arg_list)*164.5*12/60/60
    print(f"the time would be about {expectH:.2f} hours")

    t1 = time()

    for args in arg_list:

        main(args)

        if (time() - t1) > 2*60*60:
            break

    # Pool().map(main, arg_list)
    t2 = time()

    print(
        f'run time is {t2-t1:.04f} sec, {(t2-t1)/60/60:.2f} hours, for {cnt} loops. while the predicted time is {expectH:.2f} hours')


import sqlite3

class Dao:
	def __init__(self):
		self.conn = None
		self.cursor = None

	def connect(self):
		self.conn = sqlite3.connect('data/db/maze.db')
		self.cursor = self.conn.cursor()

	def close(self):
		self.conn.commit()
		self.conn.close()

	def saveOnDb(self, name):
		self.connect()
		sql = 'CREATE TABLE IF NOT EXISTS saves (name TEXT PRIMARY KEY, mapObj TEXT, playerObj TEXT)'
		self.cursor.execute(sql)

		try:
			sql = 'INSERT INTO saves (name, mapObj, playerObj) VALUES (?, ?, ?)'
			self.cursor.execute(sql, (name, 'none', 'none'))
			self.close()
		except sqlite3.IntegrityError:
			return

	def saveGame(self, name, obj):
		self.connect()
		sql = 'UPDATE saves SET mapObj = ?, playerObj = ? WHERE name = \"' + name + '\"'
		self.cursor.execute(sql, (obj.mapObj, obj.playerObj))
		self.close()

	def getSaveByName(self, name):
		self.connect()
		sql = 'SELECT * FROM saves WHERE name = \"' + name + '\"'
		self.cursor.execute(sql)

		for line in self.cursor.fetchall():
			return line

		self.close()

	def viewDb(self):	# debug function
		self.connect()
		sql = 'SELECT name, playerObj FROM saves'
		self.cursor.execute(sql)

		for line in self.cursor.fetchall():
			print(line)

		self.close()


#!/usr/bin/env python3
import os
import aws_cdk as cdk
from test.test_stack import TestStack
import yaml
from aws_cdk import App
from aws_cdk import Aspects
from aws_cdk import Tags
from cdk_nag import AwsSolutionsChecks
from cdk_nag import NagSuppressions
from cdk_nag import NIST80053R5Checks
from notification.notification import NotificationStack

def load_config_file():
    with open(r"config.yaml") as config_file:
        stack_config = yaml.safe_load(config_file)
    return stack_config

stack_config = load_config_file()

app = cdk.App()
notification_stack = NotificationStack(app, "NotificationStack", stack_config)

# Add tag(s) to all resources based on what is in the config.yaml
Tags.of(notification_stack).add("Name", stack_config["name_tag"])
Tags.of(notification_stack).add("TechOwner", stack_config["techowner_tag"])
Tags.of(notification_stack).add("Environment", stack_config["environment_tag"])
Tags.of(notification_stack).add("Project", stack_config["project_tag"])
# TestStack(app, "TestStack",
#     )

NagSuppressions.add_stack_suppressions(
    notification_stack,
    [
        {
            "id": "AwsSolutions-IAM4",
            "reason": "Accepting risk for dev to use the aws managed execution role",
        },
        {
            "id": "AwsSolutions-IAM5",
            "reason": "Backlog for prod to lockdown wildcard to specific object from bucket",
        },
        {
            "id": "AwsSolutions-SNS2",
            "reason": "Backlog for prod to have topic have server side encryption but low risk due to in transit and done by AWS",
        },
        {
            "id": "AwsSolutions-SNS3",
            "reason": "Backlog for prod to have topic to require publisher to use ssl",
        },
    ]
)

Aspects.of(app).add(AwsSolutionsChecks())
# Aspects.of(app).add(NIST80053R5Checks())
app.synth()


from __future__ import absolute_import, division, print_function

import torch
import torch.nn as nn

from torch.utils.data import DataLoader
from torchvision import transforms
from matplotlib import pyplot as plt
import argparse
import os
import random

import numpy as np
import pandas as pd

import matplotlib

matplotlib.rcParams['figure.figsize'] = [5, 5]
matplotlib.rcParams['figure.dpi'] = 200

import torch.nn.functional as F

from data_helper import UnlabeledDataset, LabeledDataset
from helper import collate_fn, draw_box
from autoencoder import get_auto_encoder

random.seed(0)
np.random.seed(0)
torch.manual_seed(0)

parser = argparse.ArgumentParser()
parser.add_argument('--data_dir', type=str, default='data')
parser.add_argument('--epoch', type=int)
parser.add_argument('--save_dir',type=str)
opt = parser.parse_args()

batch_size = 256

transform = transforms.Compose([
    transforms.ToTensor()
    # transforms.Normalize((0.5,), (0.5,))
])

image_folder = opt.data_dir
unlabeled_scene_index = np.arange(106)
#transform = torchvision.transforms.ToTensor()

dataset_size = len(unlabeled_scene_index)
indices = list(range(dataset_size))
indices = np.random.shuffle(indices)
split = int(dataset_size * 0.8 )
train_indices, val_indices = unlabeled_scene_index[:split], unlabeled_scene_index[split:]

unlabeled_trainset_train = UnlabeledDataset(image_folder=image_folder, scene_index=train_indices, first_dim='sample', transform=transform)
unlabeled_trainset_val = UnlabeledDataset(image_folder=image_folder, scene_index=val_indices, first_dim='sample', transform=transform)
trainloader = torch.utils.data.DataLoader(unlabeled_trainset_train, batch_size=1, shuffle=True, num_workers=2)
valloader = torch.utils.data.DataLoader(unlabeled_trainset_val, batch_size=1, shuffle=True, num_workers=2)

device = 'cuda'

model = get_auto_encoder().to(device)
criterion = nn.MSELoss()

# Configure the optimiser

learning_rate = 1e-3

optimizer = torch.optim.Adam(
    model.parameters(),
    lr=learning_rate,
)
# Train standard or denoising autoencoder (AE)
num_epochs = opt.epoch
model.train()
# do = nn.Dropout()  # comment out for standard AE
for epoch in range(num_epochs):
    for data in trainloader:
        img = data.view(6,3,256,306)
        img = img.to(device)
        # ===================forward=====================
        output = model.forward(img)
        loss = criterion(output,img.data)
        # ===================backward====================
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()
    # ===================log========================
    print(f'epoch [{epoch + 1}/{num_epochs}], loss:{loss.item():.4f}')

torch.save(model.state_dict(), opt.save_dir)


import pytorch_lightning as pl
import torch.nn.functional as F

from torch import nn
from utils.module_select import get_optimizer
from models.loss.focal_loss import FocalLoss

class Segmentor(pl.LightningModule):
    def __init__(self, model, cfg, epoch_length=None):
        super(Segmentor, self).__init__()
        self.save_hyperparameters(ignore='model')
        self.model = model
        self.loss_fn = FocalLoss()

    def forward(self, x):
        pred = self.model(x)
        return pred

    def training_step(self, batch, batch_idx):
        loss = self.opt_training_step(batch)
        self.log('train_loss', loss, prog_bar=True, logger=True, on_step=True, on_epoch=True)
        return loss

    def validation_step(self, batch, batch_idx):
        pred = self.model(batch['img'])
        loss = self.loss_fn([pred, batch['label']])
        self.log('val_loss', loss, prog_bar=True, logger=True, on_epoch=True, sync_dist=True)
        return loss

    def opt_training_step(self, batch):
        pred = self.model(batch['img'])
        loss = self.loss_fn([pred, batch['label']])
        return loss

    def configure_optimizers(self):
        cfg = self.hparams.cfg
        optim = get_optimizer(cfg['optimizer'],
            params=self.model.parameters(),
            **cfg['optimizer_options']
        )
        return optim


from django.urls import path
from . import views

app_name="grocery"
urlpatterns = [
    path('', views.index, name="index"),
    path('save_grocery/', views.save_grocery, name="save_grocery"),
    path('toggle_complete/<int:grocery_id>/', views.toggle_complete, name='toggle_complete'),
    path('delete/<int:grocery_id>/', views.delete, name="delete"),
]


from osgeo import gdal
from GF4_XML import *

def read_rpc(filepath):
    rpc_dict = {}
    with open(filepath) as f:
        text = f.read()
    words = ['errBias', 'errRand', 'lineOffset', 'sampOffset', 'latOffset',
             'longOffset', 'heightOffset', 'lineScale', 'sampScale', 'latScale',
             'longScale', 'heightScale', 'lineNumCoef', 'lineDenCoef','sampNumCoef', 'sampDenCoef',]
    keys = ['ERR_BIAS', 'ERR_RAND', 'LINE_OFF', 'SAMP_OFF', 'LAT_OFF', 'LONG_OFF',
            'HEIGHT_OFF', 'LINE_SCALE', 'SAMP_SCALE', 'LAT_SCALE',
            'LONG_SCALE', 'HEIGHT_SCALE', 'LINE_NUM_COEFF', 'LINE_DEN_COEFF',
            'SAMP_NUM_COEFF', 'SAMP_DEN_COEFF']

    for old, new in zip(words, keys):
        text = text.replace(old, new)
    text_list = text.split(';\n')
    text_list = text_list[3:-2]
    text_list[0] = text_list[0].split('\n')[1]
    text_list = [item.strip('\t').replace('\n', '').replace(' ', '') for item in text_list]

    for item in text_list:
        key, value = item.split('=')
        if '(' in value:
            value = value.replace('(', '').replace(')', '')
        rpc_dict[key] = value

    for key in keys[:12]:
        if not rpc_dict[key].startswith('-'):
            rpc_dict[key] = '+' + rpc_dict[key]
        if key in ['LAT_OFF', 'LONG_OFF', 'LAT_SCALE', 'LONG_SCALE']:
            rpc_dict[key] = rpc_dict[key] + ' degrees'
        if key in ['LINE_OFF', 'SAMP_OFF', 'LINE_SCALE', 'SAMP_SCALE']:
            rpc_dict[key] = rpc_dict[key] + ' pixels'
        if key in ['ERR_BIAS', 'ERR_RAND', 'HEIGHT_OFF', 'HEIGHT_SCALE']:
            rpc_dict[key] = rpc_dict[key] + ' meters'
    for key in keys[-4:]:
        values = []
        for item in rpc_dict[key].split(','):
            #print(item)
            if not item.startswith('-'):
                values.append('+'+item)
            else:
                values.append(item)
            rpc_dict[key] = ' '.join(values)
    return rpc_dict

def write_rpc(file, o_file, rpc_file):
    xmldata = read_xml('GF4_B1_E118.0_N24.0_20230728_L1A0000618715.xml')
    rpcdata = read_rpc(rpc_file)
    dataset = gdal.Open(file,gdal.GA_Update)
    metadata = xmldata | rpcdata
    dataset.SetMetadata(metadata,"RPC")
    del dataset

    # Only write Associated RPC Info? needs enhancement


"""Unit tests for class pp.TangentialNormalProjection
"""
import numpy as np
import pytest

import porepy as pp

@pytest.mark.parametrize(
    "normal, tangent",
    [
        # 2d vector, normal along the x-axis
        (np.array([[1, 0]]).T, np.array([[0, 1]]).T),
        # 2d vector, normal along the x-axis, tangent in negative y-direction
        (np.array([[1, 0]]).T, np.array([[0, -1]]).T),
        # 2d vector, normal along the negative y-axis
        (np.array([[0, -1]]).T, np.array([[1, 0]]).T),
        # 2d vector, normal in the first quadrant,
        (np.array([[1, 1]]).T, np.array([[-1, 1]]).T),
        # 2d vector, normal in the second quadrant
        (np.array([[-0.1, 1]]).T, np.array([[-1, -0.1]]).T),
        # 2d vector, two normals
        (np.array([[1, 0], [1, 1]]).T, np.array([[0, 1], [1, -1]]).T),
        # 3d vector, normal along the x-axis
        (np.array([[1, 0, 0]]).T, np.array([[0, 0, 2]]).T),
        # 3d vector, normal along the negative z-axis
        (np.array([[0, 0, -1]]).T, np.array([[1, 0, 0]]).T),
        # 3d vector, normal along the negative z-axis, tangent in the x-y plane
        (np.array([[0, 0, -1]]).T, np.array([[1, 1, 0]]).T),
        # 3d vector, normal in the first octant
        (np.array([[1, 1, 1]]).T, np.array([[0, 1, -1]]).T),
        # 3d vector, negative components of the normal vector
        (np.array([[1, -2, -3]]).T, np.array([[0, 3, -2]]).T),
        # 3d vector, two normals
        (np.array([[1, 0, 0], [1, 1, 1]]).T, np.array([[0, 0, 2], [0, 1, -1]]).T),
        # 3d vector, almost aligned with the z-axis
        (np.array([[1e-15, 5e-15, np.sqrt(1 - 6e-15)]]).T, np.array([[5, -1, 0]]).T),
        # 3d vector, two almost equal large components, one small component
        (np.array([[1, 1 - (1e-15), 1e-15]]).T, np.array([[1e-15, 0, 1]]).T),
    ],
)
def test_tangential_normal_projection(normal: np.ndarray, tangent: np.ndarray):
    """Test that the tangential-normal projection works as expected.

    The test covers:
    * Test of the local projection matrix (attribute
        TangentialNormalProjection._projection)
    * Test of the global projection matrices in the tangential and normal directions,
        (constructed by methods projection_normal(), project_tangential()), and the
        matrix which decomposes a vector into tangential and normal parts (method
        project_tangential_normal())

    The tests project the given normal and tangential vectors into the tangent and
    normal spaces, and verify the following properties:
        * When projected onto the tagent space, the normal vector is zero while the
            tangential vector has the same length as the original vector (note: we
            cannot test the direction of the tangential vector, since the tangential
            basis is arbitrary).
        * When projected onto the normal space, the tangential vector is zero while the
            normal vector has the same length as the original vector.
        * When decomposed into tangential and normal parts, the tangential and normal
            vectors have their expected length in their 'own' spaces and are zero in the
            opposite spaces.
        * When projected onto the tangent space, the combined vector (normal + tangent)
            has the length equal to the tangent vector.
        * When projected onto the normal space, the combined vector (normal + tangent)
            has the length equal to the normal vector.

    The comparisons heavily exploit the fact that, when decomposed into tangential and
    normal parts, the tangential part will be found in the first nd-1 components of the
    vector, while the normal part will be found in the last component. This is true by
    construction of the projection matrix.

    Parameters:
        normal: Normal vectors, stored as nd x num_vec arrays tangent
        tangent: Tangential vectors, stored as nd x num_vec arrays. Each
            tangential vector should be orthogonal to the corresponding (i.e., in the
            corresponding column) normal vector.

    """

    # Projections should work for vectors of arbritrary length. If the normal and
    # tangent vectors have the same length, we risk that errors in the directions
    # (normal and tangential is mixed) are masked. We therefore first normalize the
    # vectors, and then scale them to different lengths.
    normal = np.sqrt(2) * normal / np.linalg.norm(normal)
    tangent = np.sqrt(3) * tangent / np.linalg.norm(tangent)

    # Ravel the vectors, to test the global projection matrices
    raveled_normal = normal.ravel("F")
    raveled_tangent = tangent.ravel("F")

    dim, num_vec = normal.shape

    # Create a projection object
    proj = pp.TangentialNormalProjection(normals=normal)

    # Test that the projection matrix is orthonormal
    for i in range(num_vec):
        loc_proj = proj._projection[:, :, i]
        for j in range(dim):
            for k in range(dim):
                if j == k:
                    truth = 1
                else:
                    truth = 0
                assert np.allclose(loc_proj[:, k].dot(loc_proj[:, j]), truth)

    # Test that the normal vectors are projected onto themselves, using the attribute
    # projection. This tests the local projection matrix
    for i in range(num_vec):
        assert np.allclose(
            (proj._projection[:, :, i] @ normal[:, i])[dim - 1],
            np.linalg.norm(normal[:, i]),
            rtol=1e-14,
        )
    # Test that the normal vectors are projected onto themselves, using the global
    # projection matrix.
    assert np.allclose(
        # No need to take the norm of the projected matrix, since there is only one
        # component per normal vector (the returned value will be a num_vec array which
        # fits with the result of np.linalg.norm with the axis=0 option).
        proj.project_normal() @ raveled_normal,
        np.linalg.norm(normal, axis=0),
        rtol=1e-14,
    )
    # Test that, when projected onto the tangent space, the normal vector is zero.
    for i in range(num_vec):
        assert np.allclose(proj.project_tangential() @ raveled_normal, 0, rtol=1e-14)

    # Decompose the normal vector into tangential and normal parts, and test that the
    # tangential part is zero, while the normal part has the same length as the original
    # vector
    decomposed_normal = (proj.project_tangential_normal() @ raveled_normal).reshape(
        (-1, num_vec), order="f"
    )
    assert np.allclose(np.linalg.norm(decomposed_normal[: dim - 1]), 0, rtol=1e-14)
    assert np.allclose(
        decomposed_normal[dim - 1], np.linalg.norm(normal, axis=0), rtol=1e-14
    )

    # Test that the projection of the tangential vectors have the same length as the
    # original vector, using the local projection matrix.
    for i in range(num_vec):
        assert np.allclose(
            np.linalg.norm((proj._projection[:, :, i] @ tangent[:, i])[: dim - 1]),
            np.linalg.norm(tangent[:, i]),
            rtol=1e-14,
        )

    # Test that the projection of the tangential vector onto the tangent space has the
    # same length as the original vector, using the global projection matrix. Here we
    # need to reshape the projected matrix back to a (nd-1) x num_vec array and take the
    # column-wise norm (in contrast to the normal direction, where there is only one
    # component per normal vector).
    assert np.allclose(
        np.linalg.norm(
            (proj.project_tangential() @ raveled_tangent).reshape(
                (-1, num_vec), order="F"
            ),
            axis=0,
        ),
        np.linalg.norm(tangent, axis=0),
        rtol=1e-14,
    )

    # Test that, when projected onto the normal space, the tangential vector is zero.
    for i in range(num_vec):
        assert np.allclose((proj.project_normal() @ raveled_tangent), 0, rtol=1e-14)

    # Test that, when decomposed by a tangential-normal projection, the tangential
    # vector is non-zero only in the tangential direction
    decomposed_tangent = (proj.project_tangential_normal() @ raveled_tangent).reshape(
        (-1, num_vec), order="F"
    )
    assert np.allclose(
        np.linalg.norm(decomposed_tangent[: dim - 1]),
        np.linalg.norm(tangent),
        rtol=1e-14,
    )
    assert np.allclose(decomposed_tangent[dim - 1], 0, rtol=1e-14)

    # Create a vector with both normal and tangential components
    raveled_combined = (normal + tangent).ravel("F")

    # Test that the projection of the combined vector onto the tangent space is the same
    # as the projection of the tangential vector
    assert np.allclose(
        proj.project_tangential() @ raveled_combined,
        proj.project_tangential() @ raveled_tangent,
        rtol=1e-14,
    )
    # Test that the projection of the combined vector onto the normal space is the same
    # as the projection of the normal vector
    assert np.allclose(
        proj.project_normal() @ raveled_combined,
        proj.project_normal() @ raveled_normal,
        rtol=1e-12,
    )


import os
import traceback
import asyncio
import logging
from cogs import common

# noinspection PyPackageRequirements
import discord

# noinspection PyPackageRequirements
from discord.ext import commands
from cogs.utils import paste

"""
The MIT License (MIT)

Copyright (c) 2018 tilda

Permission is hereby granted, free of charge, to any person obtaining a
copy of this software and associated documentation files (the "Software"),
to deal in the Software without restriction, including without limitation
the rights to use, copy, modify, merge, publish, distribute, sublicense,
and/or sell copies of the Software, and to permit persons to whom the
Software is furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
DEALINGS IN THE SOFTWARE.
"""
"""
Owner stuff

Originally made by luna for Jose, the best bot
"""

async def run_cmd(cmd: str) -> str:
    """Runs a subprocess and returns the output."""
    process = await asyncio.create_subprocess_shell(
        cmd, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE
    )
    results = await process.communicate()
    return "".join(x.decode("utf-8") for x in results)

class Owner(commands.Cog):
    def __init__(self, bot):
        self.bot = bot

    @commands.group(hidden=True)
    @commands.is_owner()
    async def game(self, ctx):
        """Change playing status"""
        if ctx.invoked_subcommand is None:
            _help = discord.Embed(title="Subcommands", colour=0x690E8)
            _help.add_field(name="set", value="Sets playing state. `^game set meme`")
            _help.add_field(name="clear", value="Clears playing state. `^game clear`")
            return await ctx.send(embed=_help)

    @game.command(name="set")
    async def _set(self, ctx, *, game: str):
        """Sets playing state"""
        try:
            a = discord.Streaming
            g = a(
                name=f'{game} | {ctx.bot.config["prefix"]}help |' f" v{common.version}",
                url="https://twitch.tv/monstercat",
            )
            await self.bot.change_presence(activity=g)
        except Exception:
            await ctx.send(f"```\n{traceback.format_exc()}\n```")
        else:
            await ctx.send(":white_check_mark: Updated.")

    @game.command(name="clear")
    async def _clear(self, ctx):
        """Resets playing state to normal"""
        try:
            a = discord.Streaming
            p = ctx.bot.config["prefix"]
            g = a(
                name=f"{p}help | v{ctx.bot.version}", url="https://twitch.tv/monstercat"
            )
            await self.bot.change_presence(activity=g)
        except Exception:
            await ctx.send(f"```\n{traceback.format_exc()}```")
        else:
            await ctx.send(":white_check_mark: Cleared.")

    @commands.command(hidden=True)
    @commands.is_owner()
    async def shutdown(self, ctx):
        """Shuts down the bot"""
        await self.bot.session.close()
        await self.bot.logout()

    @commands.command(hidden=True)
    @commands.is_owner()
    async def reboot(self, ctx):
        """Reboots the bot.
        Can only be used with PM2 or systemd"""
        restart_land = discord.Embed(
            title="Restarting", description="Please wait...", colour=0x690E8
        )
        re_msg = await ctx.send(embed=restart_land)
        pm2_id = os.environ.get("pm_id")
        if_systemd = os.environ.get("systemd_supervised")
        if pm2_id:
            await re_msg.edit(content="pm2: :wave: bye!")
            await self.bot.session.close()
            await self.bot.logout()
            await run_cmd(f"pm2 restart {pm2_id}")
        elif if_systemd:
            await re_msg.edit(content="systemd: :wave: bye!")
            await self.bot.session.close()
            await run_cmd("systemctl --user restart lolbot")
            await self.bot.logout()
        else:
            await re_msg.edit(content=":warning: No supervisor; invoking" " `shutdown`")
            await ctx.invoke(self.bot.get_command("shutdown"))

    @commands.command(hidden=True)
    @commands.is_owner()
    async def feedbackrespond(self, ctx, p: int, *, res: str):
        with ctx.typing():
            person = ctx.bot.get_user(p)
            try:
                channel = ctx.bot.get_channel(ctx.bot.config["feedback"])
            except Exception:
                await ctx.send(":warning: Cannot find feedback channel?")
            fbck = discord.Embed(
                title="Feedback response", description=res, colour=0x690E8
            )
            try:
                await person.send(
                    f"Hello {person.name}! The owner has "
                    "responded to your feedback.",
                    embed=fbck,
                )
            except Exception as e:
                return await ctx.send(f"blocked? ```py\n{e}\n```")
            else:
                fbck = discord.Embed(
                    title=f"Owner response: {str(person)}'s" " feedback",
                    description=res,
                    colour=0x690E8,
                )
                await channel.send(embed=fbck)
            await ctx.send("Successfully sent response!")

def setup(bot):
    bot.add_cog(Owner(bot))


'''
Author: your name
Date: 2022-01-28 14:16:17
LastEditTime: 2022-01-29 06:13:48
LastEditors: Please set LastEditors
Description: 绘制紧凑框和联想框的逐帧结果
FilePath: /OpenPCDet/tools/scripts/process_boundary.py
'''
import os
import pickle
from pcdet.utils import common_utils
import matplotlib.pyplot as plt
import numpy as np
from pcdet.utils.object3d_multi_frames import Object3d
from tqdm import tqdm
def process_tight_box(points_norm, box3d_lidar, fig=None):

    box3d_lidar_norm = box3d_lidar.copy()
    box3d_lidar_norm[:3] = 0

    rotation = -box3d_lidar_norm[-1]
    points_norm_rot = common_utils.rotate_points_along_z(points_norm[np.newaxis, :, :], np.array([rotation]))[0]

    line_str   = " ".join(str(i) for i in box3d_lidar_norm)

    obj3d_src  = Object3d(line_str)
    obj3d_corners_src = obj3d_src.generate_corners3d()

    if len(points_norm) == 0:
        return dict(box3d_lidar=box3d_lidar, \
                obj3d_corners_src=obj3d_corners_src,\
                box3d_lidar_des=box3d_lidar.copy(), \
                obj3d_corners_des=obj3d_corners_src, \
                obj3d_centroid = box3d_lidar[:2],
                is_valid=False,
                )
    centroid = np.mean(points_norm[:,:2], axis=0)  # 质心
    minx, maxx = np.min(points_norm_rot[:,0]), np.max(points_norm_rot[:,0])
    miny, maxy = np.min(points_norm_rot[:,1]), np.max(points_norm_rot[:,1])

    length  = maxx - minx
    width = maxy - miny

    center_x = 0.5*(minx + maxx) # 紧缩后的中心
    center_y = 0.5*(miny + maxy)

    center_rot = np.array([center_x, center_y, 0, 0]).reshape((1,4))
    tight_center_2_base = common_utils.rotate_points_along_z(center_rot[np.newaxis, :, :], np.array([-rotation]))[0][0]

    box3d_lidar_des = [tight_center_2_base[0], tight_center_2_base[1], box3d_lidar_norm[2], length, width, box3d_lidar_norm[-2], box3d_lidar_norm[-1]]
    line_str   = " ".join(str(i) for i in box3d_lidar_des)
    obj3d_des  = Object3d(line_str)
    obj3d_corners_des = obj3d_des.generate_corners3d()

    points = points_norm[:,:3] + box3d_lidar[:3].reshape((1,3))
    obj3d_corners_src[:,:3] += box3d_lidar[:3].reshape((1,3))
    obj3d_corners_des[:,:3] += box3d_lidar[:3].reshape((1,3))
    box3d_lidar_des[:3]   += box3d_lidar[:3]
    centroid += box3d_lidar[:2]
    if fig is not None:
        ax1 = plt.subplot(121)
        ax1.plot(-points_norm[:,1], points_norm[:,0],'b.')
        ax1.plot(-points_norm_rot[:,1], points_norm_rot[:,0],'r.')
        ax1.axis('equal')
        ax2 = plt.subplot(122)
        ax2.plot(-points[:,1], points[:,0],'b.')
        ax2.plot(-obj3d_corners_src[:,1], obj3d_corners_src[:,0],'b-')
        ax2.plot(-obj3d_corners_des[:,1], obj3d_corners_des[:,0],'r-')
        ax2.plot(-box3d_lidar[1], box3d_lidar[0], 'b+')
        ax2.plot(-box3d_lidar_des[1], box3d_lidar_des[0], 'r+')
        ax2.plot(-centroid[1], centroid[0], 'k+')
        ax2.axis('equal')

    return dict(box3d_lidar=box3d_lidar, \
                obj3d_corners_src=obj3d_corners_src,\
                box3d_lidar_des=box3d_lidar_des, \
                obj3d_corners_des=obj3d_corners_des, \
                obj3d_centroid = centroid,
                is_valid=True,
                )

def plot_trajactory(tight_box_infos, save_filename=None):

    obj3d_centroids  = []
    box3d_lidar_dess = []
    box3d_lidar_srcs = []
    for tight_box_info in tight_box_infos:
        obj3d_centroids.append(tight_box_info["obj3d_centroid"])
        box3d_lidar_dess.append(tight_box_info["box3d_lidar_des"])
        box3d_lidar_srcs.append(tight_box_info["box3d_lidar"])

    box3d_lidar_srcs = np.array(box3d_lidar_srcs)
    box3d_lidar_dess = np.array(box3d_lidar_dess)
    obj3d_centroids  = np.array(obj3d_centroids)

    center_src_speed   = (box3d_lidar_srcs[1:,:2] - box3d_lidar_srcs[:-1,:2])/0.1
    center_tight_speed = (box3d_lidar_dess[1:,:2] - box3d_lidar_dess[:-1,:2])/0.1
    centriod_speed     = (obj3d_centroids[1:,:2] - obj3d_centroids[:-1,:2])/0.1

    if save_filename is not None:
        fig = plt.figure()
        ax = plt.subplot(121)
        seq_idxs = np.arange(len(box3d_lidar_srcs))
        ax.plot(seq_idxs, box3d_lidar_srcs[:,0],'r')
        ax.plot(seq_idxs, box3d_lidar_dess[:,0],'g')
        ax.plot(seq_idxs, obj3d_centroids[:,0],'b')
        ax.legend(["box3d_center_src", "box3d_center_tight", "box3d_centriod"])
        ax.set_title("x-depth")
        ax = plt.subplot(122)
        seq_idxs = np.arange(1,len(box3d_lidar_srcs))
        ax.plot(seq_idxs, center_src_speed[:,0],'r')
        ax.plot(seq_idxs, center_tight_speed[:,0],'g')
        ax.plot(seq_idxs, centriod_speed[:,0],'b')
        ax.legend(["center_src_speed", "center_tight_speed", "centriod_speed"])
        ax.set_title("x-speed")
        plt.savefig(save_filename)
if __name__ == '__main__':
    pass

    data_root = "/data7/zlh/lidar_detection/OpenPCDet/tools/debug_track"
    gt_pkl_file = "%s/track_info.pkl"%data_root
    gt_pkl_tight_box_file = "%s/track_tight_box_info.pkl"%data_root

    save_dir = "tools/work_dirs/debug_track/point_tight"
    os.makedirs(save_dir, exist_ok=True)

    with open(gt_pkl_file, "rb") as fid:
        gt_infos = pickle.load(fid)

    for uuid, track_datas in tqdm(gt_infos["Car"].items()):
        if uuid != '418b7f15-e6c4-40fe-b440-ff747b22430b':
            continue
        uuid_dir = save_dir+"/"+uuid
        os.makedirs(save_dir+"/"+uuid+"/seqs", exist_ok = True)

        tight_box_infos = []
        for i, (box3d_lidar, gt_points) in enumerate(tqdm(zip(track_datas["box3d"],track_datas["gt_points"]))):
            # if i > 0:
            #     break
            fig = plt.figure()
            tight_box_info = process_tight_box(gt_points, np.array(box3d_lidar), fig)
            tight_box_infos.append(tight_box_info)

            save_filename = "%s/seqs/%04d.png"%(uuid_dir, i)
            plt.savefig(save_filename)

        plot_trajactory(tight_box_infos, save_filename="%s/trajactory.png"%uuid_dir)

        track_datas.update(tight_box_infos=tight_box_infos)
        break

    # with open(gt_pkl_tight_box_file, "wb") as fid:
    #     pickle.dump(gt_infos, fid)