from openpyxl import Workbook, load_workbook
import re
import os

#Initialisation
#1. Initialisation de l'expression régulière décrivant les débuts de recipes.
expr_reg_debut_recipe=r"disz (?:na|kimin|ki\.min|ki-min|lu2)"
#2. Initialitation du Système S
systeme_S={"disz":1,"u":10,"gesz2":60,"geszu":600,"szar2":3600}
#3. Initialisation des unités de mesure:
Unites_de_mesure=["ban2","gin2","ma-na","sila3","barig","gur","sze","gu2","gun2"]

def prochain_mot(pos,chaine):#trouve le prochain mot, strictement après la position pos dans la chaine chaine
        interruption=False
        mot=""
        i=pos+1
        l=len(chaine)
        if chaine[i]=="{":#On ne tient pas compte des déterminatifs
                determinant=True
                while determinant:
                        i+=1
                        if chaine[i]=="}":
                                determinant=False
                                i+=1
        if chaine[i]=="_":#pour les sumérogrammes. On élimine simplement le signe indiquant que c'est un sumérogramme.
                i+=1
        while chaine[i]==" ":#élimine les espaces devant
                i+=1
        while (not interruption) and (i<l):
                if chaine[i] == " ":
                        interruption=True
                        continue
                mot=mot+chaine[i]
                i+=1
        return(mot)

def determinant_avant(pos, chaine):#trouve le déterminatif qui précède la position pos dans la chaine chaine.
        i=pos
        if not (chaine[i]=="}"):
                cont=True
                while cont:
                        i-=1
                        if i<0:
                                return("")
                        if (chaine[i]==" "):#Dans ce cas, on ne renvoie rien, car le mot qui contient la position pos n'admet pas de déterminatif le précédant.
                                return("")
                        if chaine[i]== "}":
                                cont=False
                #return("")
        det=""
        while (not chaine[i]=="{") and (i>-1):
                i-=1
                det=chaine[i]+det
        return(det[1:])

def determinant_apres(pos,chaine):#retourne le déterminatif qui succède au mot situé à la position pos.
        i=pos
        l=len(chaine)
        if not (chaine[i]=="{"):
                cont=True
                while cont:
                        i+=1
                        if i>l-1:
                                return("")
                        if (chaine[i]==" "):#Car cela veut dire qu'il n'y a pas de déterminatif accolé à la fin du mot considéré.
                                return("")
                        if chaine[i]=="{":
                                cont=False
                #return("")
        det=""

        while (not chaine[i]=="}") and (i<l):
                i+=1
                det=det+chaine[i]
        return(det)

#Analyse ce qui précède l'unité (sachant que pos se situe juste avant le début de l'unité) dans la ligne chaine. Le but étant de trouver les nombres précédant l'unité.
def analyse_avant(pos,chaine):
        fini=False#Permet de savoir si on a fini de parser tous les nombres précédant l'unité.
        position_courante=pos
        def isoler_mot(p,c):#isole le mot qui précède; p est une position et c une chaîne de caractère.
                interruption=False
                i=p
                mot=""
                if c[i]=="}":
                        determinant=True
                        while determinant:
                                i-=1
                                if c[i]=="{":
                                        determinant=False
                                        i-=1
                if c[i]=="_":
                        i-=1
                while c[i]==" ":
                        i-=1
                while (not interruption) and (i>-1):
                        if c[i]== " ":
                                interruption=True
                                continue
                        mot=c[i]+mot
                        i-=1
                return(i,mot)
        total=float(0)
        while (not fini) and position_courante>-1:
                position_courante,mot_courant=isoler_mot(position_courante,chaine)
                for cle,valeur in systeme_S.items():
                        m=re.search(r"(\S*)\("+cle+r"\)\W*",mot_courant) #On cherche si des mots-clefs du système S
                        if m:
                                nombre=m.group(1)
                                fraction=re.search(r"(\D*)([0-9]*)/([0-9]*)",nombre)
                                if fraction:
                                        total+=float(fraction.group(2))/float(fraction.group(3))*valeur
                                else:
                                        entier=re.search(r"(\D*)([0-9]*)",nombre)
                                        if entier:
                                                total+=float(entier.group(2))*valeur
                        else:
                                fini=True
        return(total)

#Et pour regarder un mot qui précède strictement pos dans la chaine chaine:
def mot_davant(pos,chaine):
        #ce qui suit permet de trouver le mot juste précédant
        interruption=False
        mot=""
        i=pos
        #On élimine les déterminants, etc.:
        if chaine[i]=="}":
                determinant=True
                while determinant:
                        i-=1
                        if chaine[i]=="{":
                                determinant=False
                                i-=1
        if chaine[i]=="_":#ce caractère indique que l'on a une partie en sumérien
                i-=1
        while chaine[i]==" ":
                i-=1
        while (not interruption) and (i>-1):
                if chaine[i]==" ":
                        interruption=True
                        continue
                mot=chaine[i]+mot
                i-=1
        return(mot)

def beginning_recipe(ligne):#Controle si on est au début d'un nouveau texte
        if ligne==[]:
                return(False)
        if ligne[0]=="&":
                return(True)
        return(False)

#Fonction pour voir si une nouvelle recette commence dans la ligne l. On revoit toutes les sous-listes, ainsi que le nombre d'occurences.
def nouvelle_recette(l):
        debut_recette=expr_reg_debut_recipe
        a=re.split(debut_recette,l.lower())
        return(len(a)-1,a)

#Initialisation workbook Excel
workbook=Workbook()
sheet = workbook.active #First sheet
sheet.title="Raw data"

#Pour donner des informations sur l'encodage
sh_metadata = workbook.create_sheet("CDLI Metadata",1)

Nombre_unites=[0 for i in range(len(Unites_de_mesure))]#nombre de fois où l'unité est employé dans une recette

#Pour analyser les mesures plus précisément
sh_gin=[]
for i in range(len(Unites_de_mesure)):
        sh_gin.append(workbook.create_sheet("Utilisation "+Unites_de_mesure[i],2+i))
        sh_gin[i]["A1"].value="Texte"
        sh_gin[i]["B1"].value="Rang dans le texte"
        sh_gin[i]["C1"].value="Mot précédent l'unité"
        sh_gin[i]["F1"].value="Mot suivant l'unité"
        sh_gin[i]["D1"].value="Déterminatif précédant l'unité"
        sh_gin[i]["E1"].value="Déterminatif suivant l'unité"
        sh_gin[i]["G1"].value="Rang de la recette dans le texte"
        sh_gin[i]["H1"].value="Quantité"

#Lecture du fichier cdli texte
chemin_data=os.path.join(os.path.dirname(os.path.abspath(__file__)),"Data")#chemin vers les données.
fichier_cdli=open(os.path.join(chemin_data,"cdli_subgenre-medical_allperiod.txt"),"r")
Liste_lignes_cdli=fichier_cdli.readlines()
fichier_cdli.close()

#Pour compter le numéro du texte et savoir où on en est dans le fichier Excel
numero_texte=0

#Première ligne
sheet["A1"].value="Numéro de texte"
dd=len(Unites_de_mesure)
for i in range(dd):
        sheet.cell(row=1,column=i+2).value=Unites_de_mesure[i]
sheet.cell(row=1,column=dd+2).value="Traduction?"#pour savoir si une traduction est disponible
sheet.cell(row=1,column=dd+3).value="Langue"
sheet.cell(row=1,column=dd+4).value="Epoque"
sheet.cell(row=1,column=dd+5).value="Provenance"
sheet.cell(row=1,column=dd+6).value="Nombre de recettes probablement contenues"#évalue le nombre de recettes contenues grâce à la fonction nouvelle_recette

#fonction d'écriture des données d'une ligne dans Excel
def ecrire_ligne(numero_ligne,Nb_unites, est_trad,language,nb_rec):
        d=len(Unites_de_mesure)
        for i in range(d):
                sheet.cell(row=numero_ligne,column=i+2).value=Nb_unites[i]
        if est_trad:
                sheet.cell(row=numero_ligne,column=d+2).value= "X"
        if not language=="":
                sheet.cell(row=numero_ligne,column=d+3).value=language
        sheet.cell(row=numero_ligne,column=d+6).value=nb_rec

#Pour regarder les mots précédant ou suivant les unités de mesure
#C'est une liste telle que Mots_precedants_suivants[i] contienne une liste de toutes les occurences de la mesure i, et donc Mots_precedants_suivants[i][j] est une liste à 7 éléments décrivant la jème occurence de l'unité i
#Cette liste à 7 éléments contient, dans l'ordre: la référence du texte, le mot qui précède, le déterminant qui rpécède, le déterminant qui succède, le mot qui succède, le numéro de recette
Mots_precedants_suivants=[[] for i in range(len(Unites_de_mesure))]

#Pour stocker les quantités
#Quantites_unites=[[] for i in range(len(Unites_de_mesure))]

est_traduit=False # dit si le texte dispose d'une traduction
langue=""#langue si indiquée

#Pour voir les langues différentes référencées
Liste_langues=[]

for ligne_texte in Liste_lignes_cdli:
        if beginning_recipe(ligne_texte):
                nom=ligne_texte[1:]#Référence du texte

                #On finit d'écrire ce qui est relatif au texte précédent
                if numero_texte>0:
#                        for i in range(len(Unites_de_mesure)):
#                                sheet.cell(row=numero_texte+1,column=i+2).value=Nombre_unites[i]
                        ecrire_ligne(numero_texte+1, Nombre_unites,est_traduit, langue,nombre_recette_dans_texte)

                nombre_recette_dans_texte=0#on écrit le nombre de recettes contenues dans ce texte
                numero_texte+=1
                sheet["A"+str(numero_texte+1)].value = nom#On met le nom du nouveau texte
                Nombre_unites=[0 for i in range(len(Unites_de_mesure))]#on remet à zéro
                est_traduit=False
                langue=""
                if numero_texte%10==0:
                        print(numero_texte)#Permet simplement de se rendre compte que le programme travaille bien quand on l'execute.

        #On regarde ce qui se passe pour des lignes particulières
        if ligne_texte[0]=="#":
        #Si la tablette est traduite
                if not re.search("#tr.en|#tr-en",ligne_texte.lower())==None:
                        est_traduit=True
                        continue

        #La langue
                if not re.search("#atf: lang",ligne_texte.lower())==None:
                        langue=prochain_mot(9,ligne_texte)
                        if not langue in Liste_langues:
                                Liste_langues.append(langue)

        #On regarde si une nouvelle recette commence:
        nbr,liste_sous_lignes=nouvelle_recette(ligne_texte)
        nombre_recette_dans_texte+=nbr

        for riga in liste_sous_lignes:
                #on itère sur la ligne pour y trouver les ingrédients
                for i in range(len(Unites_de_mesure)):
                        for match in re.finditer("( |_|\})"+Unites_de_mesure[i]+"( |_|#|\{|\?|\*|\!)",riga.lower()):
                                Nombre_unites[i]+=1
                                mot_suivant=prochain_mot(match.end()-1,riga)
                                det_precedant=determinant_avant(match.end()-1,riga)
                                det_suivant=determinant_apres(match.start(),riga)
                                mot_precedant=mot_davant(match.start(),riga)
                                q=analyse_avant(match.start(),riga)
                                Mots_precedants_suivants[i].append([nom,mot_precedant,det_precedant,det_suivant,mot_suivant,nombre_recette_dans_texte,q])

#On s'assure que la dernière ligne soit écrite:
ecrire_ligne(numero_texte+1, Nombre_unites,est_traduit,langue,nombre_recette_dans_texte)
##for i in range(len(Unites_de_mesure)):
##        sheet.cell(row=numero_texte+1,column=i+2).value=Nombre_unites[i]

#On veut regarder tous les mots clefs disponibles après #
sh_metadata["A1"].value="Informations données par #"

sh_metadata["B1"].value="Numéro du premier texte d'apparition"
Liste_possibles=[]
i=0#pour la ligne d'Excel
j=0#pour le numéro de texte
for l in Liste_lignes_cdli:
        if l==[]:
                continue
        if beginning_recipe(l):
                j+=1
        if l[0]=="#":
                m=prochain_mot(0,l)#premier mot après #
                if not m in Liste_possibles:
                        Liste_possibles.append(m)
                        i+=1
                        sh_metadata["A"+str(i+1)].value=m
                        sh_metadata["B"+str(i+1)].value=j
                        print(i)#histoire de savoir où on en est

#On ajoute les langues
sh_metadata["A"+str(i+3)].value="Langues différentes référencées"
for k in range(len(Liste_langues)):
        sh_metadata["A"+str(i+4+k)].value=Liste_langues[k]
        print(k)#Pour savoir où on en est

#On rajoute les mots suivant, précédant, etc gin2

for k in range(len(Unites_de_mesure)):
        i=0#pour compter les lignes de la worksheet
        j=0#pour compter les occurences dans un texte
        premier_texte=Mots_precedants_suivants[1][0][0]
        for occurence in Mots_precedants_suivants[k]:
                i+=1
                j+=1
                sh_gin[k]["A"+str(i+2)].value=occurence[0]
                if not (occurence[0]==premier_texte):
                        premier_texte=occurence[0]
                        j=1
                sh_gin[k]["B"+str(i+2)].value=j
                sh_gin[k]["C"+str(i+2)].value=occurence[1]
                sh_gin[k]["D"+str(i+2)].value=occurence[2]
                sh_gin[k]["E"+str(i+2)].value=occurence[3]
                sh_gin[k]["F"+str(i+2)].value=occurence[4]
                sh_gin[k]["G"+str(i+2)].value=occurence[5]
                if not occurence[6]==0:
                        sh_gin[k]["H"+str(i+2)].value=occurence[6]

#On cherche à présent à attribuer une période aux textes
print("Attribution de la période")
#Liste des fichiers
Fichiers=["neo-assyr.txt","Neo-bab.txt","Achaemenid.txt","Hellenistic.txt","Middle Assyrian.txt","Middle Babylonian.txt", "Old Babylonian.txt", "ED IIIb.txt"]
#Liste des noms d'époques
NomsEpoques=["Neo-Assyrian (ca. 911-612 BC)","Neo-Babylonian (ca. 626-539 BC)","Achaemenid (547-331 BC)","Hellenistic (323-63 BC)","Middle Assyrian (ca. 1400-1000 BC)","Middle Babylonian (ca. 1400-1100 BC)","Old Babylonian (ca. 1900-1600 BC)","ED IIIb (ca. 2500-2340 BC)"]

#On parcourt les différents fichiers. L'idée est qu'on regarde, en parcourant le fichier associé à une époque donnée, les numéros de texte trouvés et on cherche ce numéro dans le fichier Excel.
indice=0
for nom_fichier in Fichiers:
        print(nom_fichier)

        fichier_cdli = open(os.path.join(chemin_data,"Data chronologie",nom_fichier),"r")
        Liste_lignes = fichier_cdli.readlines()
        fichier_cdli.close()

        k=0
        epoque=NomsEpoques[indice]
        marqueur=1
        for ligne in Liste_lignes:
                if k%1000==0:
                        print("*ligne ",k," dans le document txt")#pour s'assurer à execution que le programme fonctionne bien.
                if ligne==[]:
                        continue
                if beginning_recipe(ligne):
                        nom=ligne[1:]
                        for i in range(1,sheet.max_row+1):
                                #if i%100==0:
                                #       print("*      Ligne ",i," dans le fichier XSL")#pour s'assurer que le programme fonctionne.
                                if nom==sheet["A"+str(i)].value:
                                        if not sheet.cell(row=i,column=dd+4).value==None:
                                               m=sheet.cell(row=i,column=dd+4).value
                                               sheet.cell(row=i,column=dd+4).value=m+";"+epoque
                                        else:
                                                sheet.cell(row=i,column=dd+4).value=epoque
                                        marqueur=i
                                        break
                k=k+1
        indice=indice+1
        print(nom_fichier)

#On cherche à présent à attribuer une provenance. On fait comme avec l'époque.
print("Attribution de la période")
#Liste des fichiers
Fichiers_prov=["assur.txt","babylon.txt","Nineveh (Kuyunjik).txt","Nippur.txt","sippar-yahrurum.txt","Uruk.txt"]
Nom_prov=["Assur (mod. Qalat Sherqat)","Bābili (mod. Babylon)","Nineveh (mod. Kuyunjik)","Nippur (mod. Nuffar)","Sippar-Yahrurum (mod. Tell Abu Habbah)","Uruk (mod. Warka)"]

indice=0
for nom_fichier in Fichiers_prov:
        print(nom_fichier)

        fichier_cdli = open(os.path.join(chemin_data,"Data provenance",nom_fichier),"r")
        Liste_lignes = fichier_cdli.readlines()
        fichier_cdli.close()

        k=0
        provenance=Nom_prov[indice]
        marqueur=1
        for ligne in Liste_lignes:
                if k%1000==0:
                        print("*ligne ",k," dans le document txt")
                if ligne==[]:
                        continue
                if beginning_recipe(ligne):
                        nom=ligne[1:]
                        for i in range(1,sheet.max_row+1):
                                #if i%100==0:
                                #       print("*      Ligne ",i," dans le fichier XSL")
                                if nom==sheet["A"+str(i)].value:
                                        if not sheet.cell(row=i,column=dd+5).value==None:
                                               m=sheet.cell(row=i,column=dd+5).value
                                               sheet.cell(row=i,column=dd+5).value=m+";"+provenance
                                        else:
                                                sheet.cell(row=i,column=dd+5).value=provenance
                                        marqueur=i
                                        break
                k=k+1
        indice=indice+1
        print(nom_fichier)

workbook.save(filename="cdli_analysis.xlsx")


import asyncio
import datetime
from mpapi.constants import get_credentials
from mpapi.module import Module
from mpapi.search import Search
from MpApi.aio.chunky import Chunky
from MpApi.aio.session import Session
from pathlib import Path
import pytest

user, pw, baseURL = get_credentials()

@pytest.mark.asyncio
async def test_results():
    async with Session(user=user, pw=pw) as session:
        # print (f"{session=}")
        # print (f"{session.appURL}")
        chnkr = Chunky(baseURL=baseURL)
        rno, cno = await chnkr.count_results(
            session=session, qtype="group", target="Object", ID=182397
        )
        assert rno == 49
        assert cno == 1

@pytest.mark.asyncio
async def test_related():
    chnkr = Chunky(baseURL=baseURL)
    m = Module(file="getItem-Object590013.xml")
    results_set = await chnkr.analyze_related(data=m)
    assert results_set == {
        "Conservation",
        "Address",
        "Person",
        "Multimedia",
        "Literature",
        "Registrar",
        "CollectionActivity",
        "ObjectGroup",
    }

@pytest.mark.asyncio
async def test_query_maker():
    chnkr = Chunky(baseURL=baseURL)
    q = await chnkr.query_maker(
        qtype="group", target="Object", ID=12345, offset=100, limit=chnkr.chunk_size
    )
    assert isinstance(q, Search)

@pytest.mark.asyncio
async def test_get_by_type():
    chnkr = Chunky(baseURL=baseURL)
    async with Session(user=user, pw=pw) as session:
        m = await chnkr.get_by_type(session=session, qtype="group", ID=182397)
        assert m
        print(m)

def test_chunk_path():  # function, not coro
    chnkr = Chunky(baseURL=baseURL)
    path = chnkr._chunk_path(qtype="group", ID=182397, cno=1, suffix=".xml", job="test")
    print("{path=}")
    date: str = datetime.datetime.today().strftime("%Y%m%d")

    p = Path("test") / date / "group-182397-chunk1.xml"
    assert str(path) == str(p)


from copy import deepcopy
from collections import namedtuple
from rst import TreeNode
import config
import torch
import numpy

Step = namedtuple('Step', 'action label direction')

class TransitionSystem:

    def __init__(self, edus=None):
        self.reset()
        if edus is not None:
            self.buffer.extend(edus)

    def reset(self):
        self.buffer = []
        self.stack = []

    def is_done(self):
        return len(self.buffer) == 0 and len(self.stack) == 1

    def get_result(self):
        if self.is_done():
            return deepcopy(self.stack[0])

    def take_action(self, action, *args, **kwargs):
        if action == 'shift':
            self.shift()
        elif action == 'reduce':
            self.reduce(*args, **kwargs)

    def shift(self):
        node = self.buffer.pop(0)
        self.stack.append(node)

    def reduce(self, label=None, direction=None, reduce_fn=None, rel_embedding=None):
        rhs = self.stack.pop()
        lhs = self.stack.pop()

        # print("label: ", lhs.label)
        # relation_one_hot = torch.FloatTensor(1, len(config.ID_TO_LABEL))
        # print(relation_one_hot)
        # relation_one_hot(1, config.LABEL_TO_ID[label]) = 1.0
        # print(relation_one_hot)
        # print("lhs shape: ", )

        emb = None if reduce_fn is None else reduce_fn(lhs.embedding, rhs.embedding, rel_embedding)
        # print(emb)

        node = TreeNode(children=[lhs, rhs], label=label, direction=direction, embedding=emb)
        node.calc_span()
        self.stack.append(node)

    @staticmethod
    def all_actions():
        return ['shift', 'reduce']

    def all_legal_actions(self):
        actions = []
        if self.can_shift():
            actions.append('shift')
        if self.can_reduce():
            actions.append('reduce')
        return actions

    def can_shift(self):
        return len(self.buffer) >= 1

    def can_reduce(self):
        return len(self.stack) >= 2

    def gold_path(self, gold_tree):
        """The correct sequence of steps according to the given gold tree."""
        parser = deepcopy(self)
        while not parser.is_done():
            step = parser.gold_step(gold_tree)
            parser.take_action(*step)
            yield step

    def gold_step(self, gold_tree):
        """The right step to take. If there are several then it returns an arbitrary one."""
        correct_steps = self.all_correct_steps(gold_tree)
        if len(correct_steps) > 0:
            return correct_steps[0]

    def all_correct_steps(self, gold_tree):
        """All steps that would take us from the current state to a state from which it is still possible to reach the gold tree."""
        correct_steps = []
        if self.can_reduce():
            rhs = self.stack[-1]
            lhs = self.stack[-2]
            new_span = range(lhs.span.start, rhs.span.stop)

            for n in gold_tree.iter_nodes():
                if n.span == new_span:
                    correct_steps.append(Step('reduce', n.label, n.direction))
                    break
            else:
                if self.can_shift():
                    correct_steps.append(Step('shift', 'None', 'None'))
                else:
                    # print("all spans:")
                    # for s in [x.span for x in list(gold_tree.iter_nodes())]:
                    #     print("\t", s)
                    raise Exception("There is no correct action given the current state of the parser.")
        elif self.can_shift():
            correct_steps.append(Step('shift', 'None', 'None'))
        return correct_steps


from pandas import DataFrame, concat, date_range
import torch
import torch.nn as nn
from torch.utils.data import DataLoader, Dataset
import sklearn.preprocessing
import yfinance as yf
import numpy as np

""" This class returns a dataloader for the financial data for a ticker. """
class Load_financial_data():
    def __init__(self, stock, from_date, to_date, split=0.8, step=7, batch_size=1):
        self.stock = stock
        self.from_date = from_date
        self.to_date = to_date
        self.split = split
        self.step = step
        self.batch_size = batch_size

    # Create a differenced series
    def percentile_changes(self, data):
        changes = []
        for i in range(len(data)-1):
            p = data[i+1]/data[i]
            p -= 1
            p *= 100
            changes.append(p)
        return np.array(changes)

    def inverse_percentile_changes(self, x, dy):
        values = []
        values.append(x)
        for d in dy:
            p = (1+d/100.0)*values[-1]
            values.append(p)
        return values

    # Frame a sequence as a supervised learning problem
    def timeseries_to_supervised(self, data, lag=1):
        df = DataFrame(data)
        columns = [df.shift(i) for i in range(lag, 0, -1)]
        columns.append(df)
        df = concat(columns, axis=1)
        df.fillna(0, inplace=True)
        return df
    # Split into train and valid data
    def train_valid(self, data):
        data = np.array(self.percentile_changes(data))
        supervised = self.timeseries_to_supervised(data, self.step).values
        train = supervised[:self.split]
        valid = supervised[self.split:]
        return train[:,:-1], valid[:,:-1], train[:,-1], valid[:,-1]
    # Create dataloader for the financial data
    def get_loader(self):
        self.df = yf.download(self.stock,
                 start=self.from_date,
                 end=self.to_date,
                 progress=False)
        # Custom dataset
        class MyDataset(Dataset):
            def __init__(self, datas, labels):
                self.datas = datas
                self.labels = labels

            def __getitem__(self, index):
                data, target = self.datas[index], self.labels[index]
                return data, target

            def __len__(self):
                return len(self.datas)

        self.opening = list(self.df['Open'].values)
        self.high = list(self.df['High'].values)
        self.low = list(self.df['Low'].values)
        self.diff = list(self.df['High'].values-self.df['Low'].values)
        self.closing = list(self.df['Close'].values)
        self.adj = list(self.df['Adj Close'].values)
        self.volume = list(self.df['Volume'].values)

        self.split = int(self.split*len(self.opening))-1
        self.n = len(self.opening)-1

        thr = 0.6

        opening_train, opening_valid, _, _ = self.train_valid(self.opening)
        high_train, high_valid, _, _ = self.train_valid(self.high)
        low_train, low_valid, _, _ = self.train_valid(self.low)
        diff_train, diff_valid, _, _ = self.train_valid(self.diff)
        closing_train, closing_valid, self.y_train, self.y_valid = self.train_valid(self.closing)
        adj_train, adj_valid, _, _ = self.train_valid(self.adj)
        vol_train, vol_valid, _, _ = self.train_valid(self.volume)

        self.closing_train = self.closing[:self.split+1]
        self.closing_valid = self.closing[self.split+1:]

        self.x_train_tensor = torch.cat(
            (
                torch.tensor(opening_train).float().view(-1, 1, self.step),
                torch.tensor(high_train).float().view(-1, 1, self.step),
                torch.tensor(low_train).float().view(-1, 1, self.step),
                torch.tensor(closing_train).float().view(-1, 1, self.step),
                torch.tensor(adj_train).float().view(-1, 1, self.step),
                torch.tensor(vol_train).float().view(-1, 1, self.step)
            ),
            1
        )

        for i in range(len(self.y_train)):
            if self.y_train[i].item()<-thr:
                self.y_train[i] = 0
            elif self.y_train[i].item()>thr:
                self.y_train[i] = 2
            else:
                self.y_train[i] = 1
        self.y_train = torch.from_numpy(self.y_train).clone().long()
        self.y_train_tensor = self.y_train.clone()

        self.x_valid_tensor = torch.cat(
            (
                torch.tensor(opening_valid).float().view(-1, 1, self.step),
                torch.tensor(high_valid).float().view(-1, 1, self.step),
                torch.tensor(low_valid).float().view(-1, 1, self.step),
                torch.tensor(closing_valid).float().view(-1, 1, self.step),
                torch.tensor(adj_valid).float().view(-1, 1, self.step),
                torch.tensor(vol_valid).float().view(-1, 1, self.step)
            ),
            1
        )
        for i in range(len(self.y_valid)):
            if self.y_valid[i].item()<-thr:
                self.y_valid[i] = 0
            elif self.y_valid[i].item()>thr:
                self.y_valid[i] = 2
            else:
                self.y_valid[i] = 1
        self.y_valid = torch.from_numpy(self.y_valid).clone().long()
        self.y_valid_tensor = self.y_valid.clone()

        self.train_loader = DataLoader(MyDataset(self.x_train_tensor,
                                                 self.y_train_tensor),
                                       batch_size=self.batch_size, shuffle=False,
                                       num_workers=0,
                                       pin_memory=True)
        self.valid_loader = DataLoader(MyDataset(self.x_valid_tensor,
                                                self.y_valid_tensor),
                                      batch_size=self.batch_size, shuffle=False,
                                      num_workers=0,
                                      pin_memory=True)

        return self.train_loader, self.valid_loader,\
               self.closing_train, self.closing_valid,\
               self.y_train, self.y_valid


from unittest import TestCase

from marshmallow import EXCLUDE, fields

from ...models.base import BaseModel, BaseModelSchema

from ..base import BaseDecoratorSet
from ..default import DecoratorSet, DEFAULT_MODELS

class SimpleModel(BaseModel):
    class Meta:
        schema_class = "SimpleModelSchema"

    def __init__(self, *, value: str = None, handled_decorator: str = None, **kwargs):
        super().__init__(**kwargs)
        self.handled_decorator = handled_decorator
        self.value = value

class SimpleModelSchema(BaseModelSchema):
    class Meta:
        model_class = SimpleModel
        unknown = EXCLUDE

    value = fields.Str(required=True)
    handled_decorator = fields.Str(required=False, data_key="handled~decorator")

class TestDecoratorSet(TestCase):
    def test_deco_set(self):
        deco_set = DecoratorSet()
        assert all(k in deco_set.models for k in DEFAULT_MODELS)

    def test_extract(self):
        decor_value = {}
        message = {"~decorator": decor_value, "one": "TWO"}

        decors = BaseDecoratorSet()
        remain = decors.extract_decorators(message)

        # check original is unmodified
        assert "~decorator" in message

        assert decors["decorator"] is decor_value
        assert remain == {"one": "TWO"}

    def test_dict(self):
        decors = BaseDecoratorSet()
        decors["test"] = "TEST"
        assert decors["test"] == "TEST"
        result = decors.to_dict()
        assert result == {"~test": "TEST"}

    def test_decorator_model(self):
        decor_value = {}
        message = {"~test": {"value": "TEST"}}

        decors = BaseDecoratorSet()
        decors.add_model("test", SimpleModel)
        remain = decors.extract_decorators(message, SimpleModelSchema)

        tested = decors["test"]
        assert isinstance(tested, SimpleModel) and tested.value == "TEST"

        result = decors.to_dict()
        assert result == message

    def test_field_decorator(self):
        decor_value = {}
        message = {"test~decorator": decor_value, "one": "TWO"}

        decors = BaseDecoratorSet()
        remain = decors.extract_decorators(message, SimpleModelSchema)

        # check original is unmodified
        assert "test~decorator" in message

        assert decors.field("test")
        assert decors.field("test")["decorator"] is decor_value
        assert remain == {"one": "TWO"}
        assert "test~decorator" in decors.to_dict()

    def test_skip_decorator(self):
        decor_value = {}
        message = {"handled~decorator": decor_value, "one": "TWO"}

        decors = BaseDecoratorSet()
        remain = decors.extract_decorators(message, SimpleModelSchema)

        # check original is unmodified
        assert "handled~decorator" in message

        assert not decors.field("handled")
        assert remain == message
        assert not decors.to_dict()


from django.urls import path

from .views import *

app_name = 'Promo'

urlpatterns = [
     path('', buatpromo, name='Buat_Promo') ,
    # path('', list_promo, name='Buat_Promo') ,
    # path('Buat_Admin/', buatpromo, name='buatpromo') ,
    # path('buat_promoHS/', buat_HS, name='buat_promoHS') ,
    path("Baca_Admin", list_promo, name ='Baca_Admin'),
    path('detail/<str:id>', detailpromo, name='detail_promo'),
    path('ubah/<str:id>', ubahpromo, name='ubah_promo'),
     path('delete/<str:id>', deletepromo, name='delete_promo'),
    path('buat_HS/', buat_HS, name='buat_promoHS') ,
    path('buat_MT/', buat_MT, name='buat_promoMT') ,
    path("Baca_Restaurant", list_promo, name ='Baca_Restaurant'),
]


import json
import logging
import os
from botocore.vendored import requests  # use import requests for local test

# Read environment variables
SLACK_WEB_HOOK_URL = os.environ['SLACK_WEB_HOOK_URL']
SLACK_CHANNEL = os.environ['SLACK_CHANNEL']
logger = logging.getLogger()
logger.setLevel(logging.INFO)

def lambda_handler(event, context):
    logger.info("Event: " + str(event))
    # Read message posted on SNS Topic
    message = json.loads(event['Records'][0]['Sns']['Message'])
    logger.info("Message: " + str(message))

# Construct a slack message
    slack_message = {
        'channel': SLACK_CHANNEL,
        'text': "%s" % message
    }

# Post message on SLACK_WEB_HOOK_URL
    req = requests.post(SLACK_WEB_HOOK_URL, json.dumps(slack_message))

    if req.status_code == 200:
        logger.info(f'Post to slack was successful /nHttpCode: {req.status_code}')
    else:
        logger.error(f'Post to Slack failed /nHttpCode: {req.status_code}')


"""
preprocess input data into feature and stores binary as python shelve DB
each chunk is gzipped JSON string
"""
import argparse
import gzip
import json
import subprocess as sp
import shelve
import os
from os.path import dirname, exists, join

import torch
from pytorch_pretrained_bert import GPT2Tokenizer
from tqdm import tqdm

from env import END_OF_TEXT_TOKEN

class InputFeatures(object):
    def __init__(self, conv_id, input_ids, position_ids, token_type_ids,
                 lm_labels, attn_masks, weights, input_len=None):
        self.conv_id = conv_id
        self.input_ids = input_ids
        self.position_ids = position_ids
        self.token_type_ids = token_type_ids
        self.lm_labels = lm_labels
        self.attn_masks = attn_masks
        self.weights = weights
        if input_len is None:
            self.input_len = len(input_ids)
        else:
            self.input_len = input_len

def _get_file_len(corpus):
    print(f"wc -l {corpus}")
    n_line = int(sp.check_output(f"wc -l {corpus}".split(),
                                 universal_newlines=True).split()[0])
    return n_line

def _norm_text(text):
    toks = text.strip().split()
    w = 1.0
    return w, ' '.join(toks)

def _get_inputs_from_text(text, tokenizer):
    srcs, tgt, attn_masks, types, positions = text.strip().split('\t')

    weights = []
    inputs = []
    for src in srcs.split(' EOS '):
        context_id = [int(num) for num in src.strip().split()]
        weights.append(1.0)
        inputs.append(context_id)
    response_id = [int(num) for num in tgt.strip().split()]
    weights.append(1.0)
    inputs.append(response_id)

    attn_masks = [[int(c) for c in seq] for seq in attn_masks.split(',')]
    position_ids = [int(num) for num in positions.strip().split()]
    type_ids = [int(num) for num in types.strip().split()]
    return weights, inputs, attn_masks, position_ids, type_ids

def _make_features(id_, weights, inputs, attn_masks, position_ids, type_ids, tokenizer, max_len):
    end_of_text_id = tokenizer.encoder[END_OF_TEXT_TOKEN]
    features = []
    sents = []
    ws = []
    len_ = 0
    i = 0
    for ids, w in zip(inputs, weights):
        sents.append(ids)
        ws.append(w)
    assert len(sents) == 2
    feat = _make_feature(id_, sents, ws, attn_masks, position_ids, type_ids, end_of_text_id)
    if feat is not None:
        features.append(feat)

    return features

def _make_feature(id_, sents, ws, attn_masks, position_ids, token_type_ids, eos):
    if all(w == 0 for w in ws[1:]):
        return None
    input_ids = [i for s in sents for i in s+[eos]][:-1]
    lm_labels = []
    weights = []
    #token_type_ids = []  # this becomes round ids
    for i, (s, w) in enumerate(zip(sents, ws)):
        if i == 0:
            lm_labels += [-1] * len(s)
            weights += [0.0] * len(s)
            continue

        if w == 0.0:
            lm_labels += [-1] * (len(s) + 1)
            weights += [0.0] * (len(s) + 1)
        else:
            lm_labels += (s + [eos])
            weights += [w] * (len(s) + 1)

    # handle trailing -1's
    i = len(lm_labels) - 1
    while i >= 0:
        if lm_labels[i] != -1:
            break
        i -= 1
    input_ids = input_ids[:i+1]
    lm_labels = lm_labels[:i+1]
    weights = weights[:i+1]
    token_type_ids = token_type_ids[:i+1]
    attn_masks = attn_masks[:i+1]
    position_ids = position_ids[:i+1]
    token_type_ids = token_type_ids[:i+1]

    # pad to multiples of 8
    while len(input_ids) % 8 != 0:
        input_ids.append(0)
        token_type_ids.append(0)
        lm_labels.append(-1)
        weights.append(0.0)
        attn_masks.append([0]*(len(attn_masks)+1))
        position_ids.append(0)

    assert (len(input_ids) == len(position_ids) == len(token_type_ids)
            == len(lm_labels) == len(weights) == len(attn_masks))
    assert len(input_ids) % 8 == 0
    if len(input_ids) == 0:
        import ipdb
        ipdb.set_trace()
    feature = InputFeatures(id_, input_ids, position_ids, token_type_ids,
                            lm_labels, attn_masks, weights)
    return feature

def main(args):
    # TODO specify vocab path to avoid download
    if args.tokenizer is not None:
        toker = GPT2Tokenizer.from_pretrained(args.tokenizer)
    else:
        toker = GPT2Tokenizer.from_pretrained('gpt2')
    assert args.corpus.endswith('.txt') or args.corpus.endswith('.tsv')
    db_path = f'{args.corpus[:-4]}.db/db'
    if exists(dirname(db_path)):
        raise ValueError('Found existing DB, please backup')
    else:
        os.makedirs(dirname(db_path))
    with open(args.corpus, "r", encoding="utf-8") as reader, \
            shelve.open(db_path, 'n') as db:
        chunk = []
        n_chunk = 0
        n_example = 0
        for line in tqdm(reader, total=_get_file_len(args.corpus)):
            try:
                if len(chunk) >= args.chunk_size:
                    # save and renew chunk
                    db[f'chunk_{n_chunk}'] = gzip.compress(
                        json.dumps(chunk[:args.chunk_size]).encode('utf-8'))
                    chunk = chunk[args.chunk_size:]
                    n_chunk += 1

                weights, inputs, attn_masks, position_ids, type_ids = _get_inputs_from_text(line, toker)
                if len(weights) < 2:
                    continue
                features = _make_features(n_example, weights, inputs, attn_masks, position_ids, type_ids, toker, args.max_seq_len)
                for feature in features:
                    chunk.append(vars(feature))
                    n_example += 1
            except Exception as e:
                raise e

        # save last chunk
        db[f'chunk_{n_chunk}'] = gzip.compress(
            json.dumps(chunk).encode('utf-8'))
    # save relevant information to reproduce
    meta = {'n_example': n_example,
            'chunk_size': args.chunk_size,
            'max_seq_len': args.max_seq_len}
    with open(join(dirname(db_path), 'meta.json'), 'w') as writer:
        json.dump(meta, writer, indent=4)
    torch.save(toker, join(dirname(db_path), 'tokenizer.pt'))

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('--corpus', required=True,
                        help='file name of training corpus (should be .tsv)')
    parser.add_argument('--chunk_size', type=int, default=65536,
                        help='num of data examples in a storing chunk')
    parser.add_argument('--max_seq_len', type=int, default=512,
                        help='discard data longer than this')
    parser.add_argument('--tokenizer', help='pretrained tokenizer path')

    args = parser.parse_args()

    main(args)


from typing import Mapping, Optional
import jax.numpy as jnp
import numpy as np
import tensorflow.compat.v2 as tf
import tensorflow_datasets as tfds

Array = jnp.ndarray
Batch = Mapping[str, np.ndarray]
DATASET_ATTRIBUTES = {
    'celeb_a_hq_custom': {
        'num_channels': 3,
        'resolution': 64,
        'type': 'image',
        'train_size': 27_000,
        'test_size': 3_000,
    },
    'srn_cars': {
        'num_channels': 4,
        'resolution': 128,
        'type': 'scene',
        'render_config': {
            'near': 0.8,
            'far': 1.8,
            'white_background': True,
        },
        'train_size': 2458,
        'test_size': 703,
    },
}

def load_dataset(dataset_name: str,
                 subset: str,
                 batch_size: Optional[int] = None,
                 shuffle: bool = False,
                 repeat: bool = False,
                 num_examples: Optional[int] = None,
                 shuffle_buffer_size: int = 10000):
  """Tensorflow dataset loaders.

  Args:
    dataset_name (string): One of elements of DATASET_NAMES.
    subset (string): One of 'train', 'test'.
    batch_size (int):
    shuffle (bool): Whether to shuffle dataset.
    repeat (bool): Whether to repeat dataset.
    num_examples (int): If not -1, returns only the first num_examples of the
      dataset.
    shuffle_buffer_size (int): Buffer size to use for shuffling dataset.

  Returns:
    Tensorflow dataset iterator.
  """

  # Load dataset
  if dataset_name.startswith('srn'):
    ds = tfds.load(dataset_name, split=subset)
    # Filter corrupted scenes that contain views with only white background
    ds = ds.filter(filter_srn)
    # Map pixels in [0,255] to [0,1] range
    ds = ds.map(process_srn, num_parallel_calls=tf.data.experimental.AUTOTUNE)
  elif dataset_name.startswith('celeb_a_hq'):
    # CelebAHQ does not have a test dataset, so do a 90/10
    # split on training data to create train and test sets
    if subset == 'train':
      subset = 'train[:90%]'
    elif subset == 'test':
      subset = 'train[90%:]'
    ds = tfds.load(dataset_name, split=subset)
    # Map pixels in [0,255] to [0,1] range and map resolution from 128 to 64.
    ds = ds.map(process_celeba,
                num_parallel_calls=tf.data.experimental.AUTOTUNE)

  # Optionally subsample dataset
  if num_examples is not None:
    ds = ds.take(num_examples)

  # Optionally shuffle dataset
  if shuffle:
    ds = ds.shuffle(shuffle_buffer_size)

  # Optionally repeat dataset if repeat
  if repeat:
    ds = ds.repeat()

  if batch_size is not None:
    ds = ds.batch(batch_size)

  # Convert from tf.Tensor to numpy arrays for use with Jax
  return iter(tfds.as_numpy(ds))

def filter_srn(batch):
  views = batch['images']   # shape [num_views, H, W, 3]
  # Take min and max for each view
  min_val = tf.math.reduce_min(views, axis=(1, 2, 3))  # [num_views]
  max_val = tf.math.reduce_max(views, axis=(1, 2, 3))  # [num_views]
  # Take the difference then the minimum across views
  # Some views have only white background iff this min == 0
  min_diff = tf.math.reduce_min(max_val - min_val)  # scalar
  return tf.math.not_equal(min_diff, 0)

def process_srn(batch: Batch):
  batch['images'] = tf.cast(batch['images'], tf.float32) / 255.
  return batch

def process_celeba(batch: Batch):
  image = tf.cast(batch['image'], tf.float32) / 255.
  # Resize from 128 to 64 resolution.
  image = tf.image.resize(image, [64, 64])  # [64, 64, 3]
  return {'array': image}


import sys
import threading
import traceback

from reporter.HtmlReporter import HtmlReporter
from data_handler.Xlsx_File_Handler import Xlsx_File_Handler
import properties as config
from dateutil.parser import parse
from data_handler.DB_Connection_Provider import DB_Connection_Provider

class ErrorEncounterException(Exception):
    def __init__(self, message):
        super().__init__(message)

class ValidationFailedException(Exception):
    def __init__(self, message):
        super().__init__(message)

class Comparator:
    fr = None
    cursor_1 = None
    cursor_2 = None
    columns_to_exclude_in_comparison = ['year']
    out_csv = None
    xlsx_fh = None
    total_pass = 0
    total_fail = 0

    def __init__(self, report_name, table_name, number_of_records_to_match, partition_1, partition_2):
        self.report_name = report_name
        self.table_name = table_name
        self.number_of_records_to_match = number_of_records_to_match
        self.reporter = HtmlReporter(report_name=report_name)
        self.partition_1 = partition_1
        self.partition_2 = partition_2

    def validate_result(self,
                        scenario_name,
                        exp_result,
                        actual_result,
                        status=None,
                        exit_on_failure=False,
                        comment=''):

        if status is None:
            if exp_result == actual_result:
                status = 'pass'
            else:
                status = 'fail'
            if comment != '':
                status = 'error'
        self.reporter.add_scenario_result(scenario_name, str(exp_result), str(actual_result),
                                          status, comment)
        if status == 'fail' and exit_on_failure:
            self.teardown()
            raise ValidationFailedException('Validation failed! Stopped further processing!')
        elif status == 'error' and exit_on_failure:
            self.teardown()
            # raise ErrorEncounterException.ErrorEncounterException('Some error occurred! Stopped processing')

    def start_comparison(self):
        try:
            # get database connection and execute fetch query
            db_con = DB_Connection_Provider().get_db_connection()
            self.cursor_1 = db_con.cursor()
            self.cursor_2 = db_con.cursor()
            sql_query_1 = f'Select * from {self.table_name} PARTITION ({self.partition_1})'
            sql_query_2 = f'Select * from {self.table_name} PARTITION ({self.partition_2})'
            print(f'Querying DB with -- {sql_query_1}')
            rs1 = self.cursor_1.execute(sql_query_1)
            print(f'Querying DB with -- {sql_query_2}')
            rs2 = self.cursor_2.execute(sql_query_2)

            partition_1 = rs1.fetchmany(config.BUFFER_NUMBER_OF_DB_ROWS)
            partition_2 = rs2.fetchmany(config.BUFFER_NUMBER_OF_DB_ROWS)

            table_headers = [d[0].lower() for d in rs1.description]

            # adding db table columns as table headers in html report
            self.reporter.add_db_column_names_as_headers(table_headers)

            # opening an xlsx file to store the comparison result
            self.xlsx_fh = Xlsx_File_Handler(self.report_name.replace('.html', '.xlsx'),
                                             table_headers)

            number_of_row_checked = 0
            progress_update_count = int(self.number_of_records_to_match / 100)
            slider_count = 1
            print('\nComparison started ... ')
            while True and number_of_row_checked <= self.number_of_records_to_match:
                # iterate all the rows in result set and check against the file
                for index, db_row_partition_1 in enumerate(partition_1):
                    if number_of_row_checked == self.number_of_records_to_match:
                        number_of_row_checked += 1
                        break
                    if number_of_row_checked == (progress_update_count * slider_count):
                        self.update_progress(slider_count)
                        slider_count += 1

                    partition_1_row = partition_1[index]
                    partition_2_row = partition_2[index]
                    current_row_matched = True

                    unmatched_values = {}
                    unmatched_indexes = []
                    for i, header in enumerate(table_headers):
                        if header in self.columns_to_exclude_in_comparison:
                            continue
                        if str(partition_1_row[i]) == str(partition_2_row[i]):
                            continue
                        else:
                            current_row_matched = False
                            unmatched_values[header] = f'{partition_1_row[i]} <==> {partition_2_row[i]}'
                            unmatched_indexes.append(i)

                    if current_row_matched:
                        self.total_pass += 1

                        # writing comparison result in html report
                        threading.Thread(target=self.reporter.add_scenario_result_as_table_formatted_data,
                                         args=('Comparing DB row ',
                                               partition_1_row,
                                               partition_2_row,
                                               'pass')).start()

                        # writing comparison result in xlsx file
                        if self.total_pass <= config.MAX_NUMBER_OF_SUCCESS_CASES_TO_REPORT:
                            threading.Thread(target=self.xlsx_fh.write_partition1_and_partition2_row_in_xlsx_file,
                                             args=(
                                                 partition_1_row,
                                                 partition_2_row,
                                             )).start()
                    else:
                        self.total_fail += 1
                        # writing comparison result in html report
                        threading.Thread(target=self.reporter.add_scenario_result_as_table_formatted_data,
                                         args=('Comparing DB row ',
                                               partition_1_row,
                                               partition_2_row,
                                               'fail', '',
                                               unmatched_indexes
                                               )).start()

                        # writing comparison result in xlsx file
                        if self.total_fail <= config.MAX_NUMBER_OF_FAILURE_CASES_TO_REPORT:
                            threading.Thread(target=self.xlsx_fh.write_partition1_and_partition2_row_in_xlsx_file,
                                             args=(
                                                 partition_1_row,
                                                 partition_2_row,
                                                 unmatched_indexes
                                             )).start()
                    number_of_row_checked += 1

                # fetch next chuck of data from database
                partition_1 = rs1.fetchmany(config.BUFFER_NUMBER_OF_DB_ROWS)
                partition_2 = rs2.fetchmany(config.BUFFER_NUMBER_OF_DB_ROWS)

                # check if we have reached to the end of table
                if not partition_1:
                    break
            self.update_progress(100)
            print('\nprocessed data file completely.')
            self.teardown()
        except Exception as e:
            if isinstance(e, ValidationFailedException):
                return
            print('Some error occurred while processing the file.')
            print(f'full traceback --- {traceback.format_exc()}')
            self.validate_result('Checking for any error while execution',
                                 'No Error should be encountered',
                                 'Error occurred',
                                 exit_on_failure=False,
                                 comment=f'error -- {traceback.format_exc()}')
            self.teardown()

    def is_date(self, string, fuzzy=False):
        """
        Return whether the string can be interpreted as a date.

        :param string: str, string to check for date
        :param fuzzy: bool, ignore unknown tokens in string if True
        """
        try:
            parse(string, fuzzy=fuzzy)
            return True

        except ValueError:
            return False

    def teardown(self):
        self.reporter.save_report()
        if self.fr is not None:
            print('closing data file ... !')
            self.fr.close_file()
            print('data file closed successfully !!')
            del self.fr
        if self.cursor_1 is not None:
            print('closing database connection ... !')
            self.cursor_1.close()
            print('database connection closed successfully !!')
            del self.cursor_1
        if self.cursor_2 is not None:
            print('closing database connection ... !')
            self.cursor_2.close()
            print('database connection closed successfully !!')
            del self.cursor_2
        if self.out_csv is not None:
            print('Closing out_csv file ... !')
            self.out_csv.flush()
            self.out_csv.close()
            print('Closed out_csv file successfully !!')
            del self.out_csv
        if self.xlsx_fh is not None:
            self.xlsx_fh.save_xlsx_file()
            del self.xlsx_fh

    def update_progress(self, progress):
        sys.stdout.write('\r[{0}] {1}%'.format('#' * progress, progress))
        sys.stdout.flush()


'''
This script will recover the files that are reported by the check_datasets.py script in its log file
In particular it will check the mime type of every reported file and:
* if the file is an html file it will log it and the extension remain the same
* if the file is a txt file it will check if it is ttl file by checking the @prefix directive in the first lines
* if the file is a xml file it will convert it to rdf

In the log file it will log which files have been recovered and which not
'''

import os
import re
import magic
import argparse
import logging

'''
@param dataset name of the dataset
@param file name of the file to be checked
@param datasets_directory directory where all the dataset are saved
@output rename the file with ttl or rdf
'''
def checkForTTL(dataset, file, datasets_directory):
    file_path = datasets_directory+"/"+dataset+"/"+file
    file = open(file_path,"r")

    line = file.readline()
    return re.search("@prefix", line, re.IGNORECASE)

'''
@param dataset name of the dataset
@param file name of the file to be checked
@param datasets_directory directory where all the dataset are saved
@return true if the file was recovered else false
@output rename the file with ttl or rdf if can be recovered
'''
def recoverFile(dataset, file, datasets_directory):
    file_path = datasets_directory+"/"+dataset+"/"+file
    mime_type = magic.from_file(file_path, mime=True)

    if "xml" in mime_type:
        os.rename(file_path, str(file_path)+".rdf")
        return True
    elif "html" in mime_type:
        os.rename(file_path, str(file_path)+".html")
        return True
    elif "text" in mime_type:
        if checkForTTL(dataset, file, datasets_directory):
            os.rename(file_path, str(file_path)+".ttl")
            return True
    return False

'''
@param datasets_directory directory where are saved all the datasets
@param checker_error_log_file_path path to the txt errors log file of the dataset checker
@param error_log_file_path path to the error log of the file recover
'''
def startRecover(datasets_directory,checker_error_log_file_path, error_log_file_path):
    #open the error log files of the dataset checker
    f_log_checker=open(checker_error_log_file_path, "r")

    #open the error log file
    f_log=open(error_log_file_path, "a")

    dataset= ""
    while True:
        line = f_log_checker.readline()

        if(not line):
            break

        #skip blank lines
        if ":" in line:
            #split the line
            fields = line.split(": ")

            if fields[0] == "Dataset":
                dataset = fields[1].strip("\n")
            elif fields[0] == "File":
                file = fields[1].strip("\n")

                split = file.split(".")

                #check if there is no extension available for the file

                if len(split) == 1:

                    if os.path.isfile(datasets_directory+"/"+dataset+"/"+file):
                        if not recoverFile(dataset, file, datasets_directory):
                            mime_type = magic.from_file(datasets_directory+"/"+dataset+"/"+file)

                            log.warning(
                                f"""
                                Dataset: {dataset}\n
                                File: {file}\n
                                Mime: {mime_type}\n
                                """
                            )
                        else:

                            log.warning(
                                f"""
                                Recover in Dataset: {dataset}\n
                                File: {file}\n
                                """
                            )

    f_log_checker.close()
    f_log.close()

if __name__ == "__main__":
    scriptDir = os.path.dirname(os.path.realpath('__file__'))

    # read the command line arguments
    parser = argparse.ArgumentParser()
    parser.add_argument(
        "datasets_folder",
        type=str,
        help="Absolute path to the folder where all the datasets will be downloaded"
    )
    args = parser.parse_args()

    global log
    logging.basicConfig(
        filename="logs/simple_file_recover.log",
        filemode="a",
        format="%(message)s",
    )
    log = logging.getLogger("simple_file_recover")

    #path to the checker error log file path
    checker_error_log_path = os.path.join(scriptDir, '../download/logs/checker_error_log.txt')

    startRecover(args.datasets_folder,checker_error_log_path)


import matplotlib.pyplot as plt
import numpy as np

def _invert(x, limits):
    """inverts a value x on a scale from
    limits[0] to limits[1]"""
    return limits[1] - (x - limits[0])

def _scale_data(data, ranges):
    """scales data[1:] to ranges[0],
    inverts if the scale is reversed"""
    for d, (y1, y2) in zip(data[1:], ranges[1:]):
        assert (y1 <= d <= y2) or (y2 <= d <= y1)
    x1, x2 = ranges[0]
    d = data[0]
    if x1 > x2:
        d = _invert(d, (x1, x2))
        x1, x2 = x2, x1
    sdata = [d]
    for d, (y1, y2) in zip(data[1:], ranges[1:]):
        if y1 > y2:
            d = _invert(d, (y1, y2))
            y1, y2 = y2, y1
        sdata.append((d-y1) / (y2-y1)
                     * (x2 - x1) + x1)
    return sdata

class ComplexRadar():
    def __init__(self, fig, variables, ranges, n_ordinate_levels=6):
        angles = np.arange(0, 360, 360./len(variables))

        axes = [fig.add_axes([0.1,0.1,0.9,0.9], polar=True,
            label = "axes{}".format(i))
            for i in range(len(variables))]

        l, text = axes[0].set_thetagrids(angles, labels=variables)

        [[txt.set_fontweight('bold'),
              txt.set_fontsize(12),
              txt.set_position((0,-0.2))] for txt in text]

        for ax in axes[1:]:
            ax.patch.set_visible(False)
            ax.grid("off")
            ax.xaxis.set_visible(False)

        for i, ax in enumerate(axes):
            grid = np.linspace(*ranges[i], num=n_ordinate_levels)
            gridlabel = ["{}".format(round(x,2)) for x in grid]

            gridlabel[0] = "" # clean up origin
            ax.set_rgrids(grid, labels=gridlabel,angle=angles[i])

            ax.set_ylim(*ranges[i])

        # variables for plotting
        self.angle = np.deg2rad(np.r_[angles, angles[0]])
        self.ranges = ranges
        self.ax = axes[0]

    def plot(self, data, *args, **kw):
        sdata = self.scale_data(data, self.ranges)
        self.ax.plot(self.angle, np.r_[sdata, sdata[0]], *args, **kw)

    def fill(self, data, *args, **kw):
        sdata = self.scale_data(data, self.ranges)
        self.ax.fill(self.angle, np.r_[sdata, sdata[0]], *args, **kw)

    def scale_data(self, data, ranges):
        """scales data[1:] to ranges[0]"""
        for d, (y1, y2) in zip(data[1:], ranges[1:]):
            assert (y1 <= d <= y2) or (y2 <= d <= y1)
        x1, x2 = ranges[0]
        d = data[0]
        sdata = [d]
        for d, (y1, y2) in zip(data[1:], ranges[1:]):
            if y1 > y2:
                d = _invert(d, (y1, y2))
                y1, y2 = y2, y1
            sdata.append((d-y1) / (y2-y1) * (x2 - x1) + x1)
        return sdata

def plot_comparison(df_1,df_2, player_names):

    df_1_dict = {key: value[df_1.index[0]] for key, value in df_1.to_dict().items()}
    df_2_dict = {key: value[df_2.index[0]] for key, value in df_2.to_dict().items()}

    # plotting

    variables = ['PTS','AST','TRB','STL','BLK', 'TOV']
    ranges = [(0, 30), (0, 11), (0, 10), (0, 3), (0, 3), (0, 5)]

    fig1 = plt.figure(figsize=(6, 6))
    radar = ComplexRadar(fig1, variables, ranges)

    for player,dictionary in zip(player_names,[df_1_dict,df_2_dict]):
        data = list(dictionary.values())
        radar.plot(data, label=player)
        radar.fill(data, alpha=0.2)

    radar.ax.legend(loc='upper center', bbox_to_anchor=(0.5, -0.10),
        fancybox=True, shadow=True, ncol=4, fontsize=13)

    return fig1


import tensorflow as tf
from unet import VariationalUnet
from data import preprocessing

def loss(y, y_bar, mus_sigmas):
    ls = 0
    for mu, sigma in mus_sigmas:
        ls = ls + (- 0.5 * tf.reduce_sum(1 + sigma - tf.math.pow(mu, 2) - tf.exp(sigma)))
    return tf.reduce_mean(tf.square(y - y_bar)) + ls

def grad(model, inputs, targets):
    with tf.GradientTape() as tape:
        reconstruction, mus_sigmas = model(inputs)
        loss_value = loss(targets, reconstruction, mus_sigmas)
    return loss_value, tape.gradient(loss_value, model.trainable_variables), reconstruction, mus_sigmas

def train():
    data_gen_args = dict(rotation_range=90,
                         width_shift_range=0.1,
                         height_shift_range=0.1,
                         zoom_range=0.2)
    image_datagen = tf.keras.preprocessing.image.ImageDataGenerator(**data_gen_args)
    mask_datagen = tf.keras.preprocessing.image.ImageDataGenerator(**data_gen_args)

    seed = 1
    model = VariationalUnet()
    optimizer = tf.keras.optimizers.Adam(learning_rate=0.001)
    global_step = tf.Variable(0)

    num_epochs = 50
    batch_size = 4

    image_generator = image_datagen.flow_from_directory(
        'data/membrane/train/images/',
        class_mode=None,
        batch_size = batch_size,
        seed=seed)

    mask_generator = mask_datagen.flow_from_directory(
        'data/membrane/train/masks/',
        class_mode=None,
        batch_size = batch_size,
        seed=seed)

    train_generator = zip(image_generator, mask_generator)

    for epoch in range(num_epochs):
        print("Epoch: ", epoch)
        for (img,mask) in train_generator:
            img, mask = preprocessing(img, mask)

            loss_value, grads, reconstruction, mus_sigmas = grad(model, img, mask)
            optimizer.apply_gradients(zip(grads, model.trainable_variables), global_step)

            if global_step.numpy() % 1 == 0:
                print("Step: {},         Loss: {}".format(global_step.numpy(),
                                              loss(mask, reconstruction, mus_sigmas).numpy()))

if __name__ == "__main__":
    train()


import graphene

from ...core.permissions import OrderPermissions
from ..core.descriptions import DEPRECATED_IN_3X_FIELD
from ..core.enums import ReportingPeriod
from ..core.fields import FilterInputConnectionField, PrefetchingConnectionField
from ..core.scalars import UUID
from ..core.types import FilterInputObjectType, TaxedMoney
from ..core.utils import from_global_id_or_error
from ..decorators import permission_required
from .bulk_mutations.draft_orders import DraftOrderBulkDelete, DraftOrderLinesBulkDelete
from .bulk_mutations.orders import OrderBulkCancel
from .filters import DraftOrderFilter, OrderFilter
from .mutations.discount_order import (
    OrderDiscountAdd,
    OrderDiscountDelete,
    OrderDiscountUpdate,
    OrderLineDiscountRemove,
    OrderLineDiscountUpdate,
)
from .mutations.draft_orders import (
    DraftOrderComplete,
    DraftOrderCreate,
    DraftOrderDelete,
    DraftOrderUpdate,
)
from .mutations.fulfillments import (
    FulfillmentApprove,
    FulfillmentCancel,
    FulfillmentRefundProducts,
    FulfillmentReturnProducts,
    FulfillmentUpdateTracking,
    OrderFulfill,
)
from .mutations.orders import (
    OrderAddNote,
    OrderCancel,
    OrderCapture,
    OrderConfirm,
    OrderLineDelete,
    OrderLinesCreate,
    OrderLineUpdate,
    OrderMarkAsPaid,
    OrderRefund,
    OrderUpdate,
    OrderUpdateShipping,
    OrderVoid,
)
from .resolvers import (
    resolve_draft_orders,
    resolve_homepage_events,
    resolve_order,
    resolve_order_by_token,
    resolve_orders,
    resolve_orders_total,
)
from .sorters import OrderSortingInput
from .types import Order, OrderEvent

class OrderFilterInput(FilterInputObjectType):
    class Meta:
        filterset_class = OrderFilter

class OrderDraftFilterInput(FilterInputObjectType):
    class Meta:
        filterset_class = DraftOrderFilter

class OrderQueries(graphene.ObjectType):
    homepage_events = PrefetchingConnectionField(
        OrderEvent,
        description=(
            "List of activity events to display on "
            "homepage (at the moment it only contains order-events)."
        ),
    )
    order = graphene.Field(
        Order,
        description="Look up an order by ID.",
        id=graphene.Argument(graphene.ID, description="ID of an order.", required=True),
    )
    orders = FilterInputConnectionField(
        Order,
        sort_by=OrderSortingInput(description="Sort orders."),
        filter=OrderFilterInput(description="Filtering options for orders."),
        channel=graphene.String(
            description="Slug of a channel for which the data should be returned."
        ),
        description="List of orders.",
    )
    draft_orders = FilterInputConnectionField(
        Order,
        sort_by=OrderSortingInput(description="Sort draft orders."),
        filter=OrderDraftFilterInput(description="Filtering options for draft orders."),
        description="List of draft orders.",
    )
    orders_total = graphene.Field(
        TaxedMoney,
        description="Return the total sales amount from a specific period.",
        period=graphene.Argument(ReportingPeriod, description="A period of time."),
        channel=graphene.Argument(
            graphene.String,
            description="Slug of a channel for which the data should be returned.",
        ),
    )
    order_by_token = graphene.Field(
        Order,
        description="Look up an order by token.",
        token=graphene.Argument(UUID, description="The order's token.", required=True),
    )

    @permission_required(OrderPermissions.MANAGE_ORDERS)
    def resolve_homepage_events(self, *_args, **_kwargs):
        return resolve_homepage_events()

    @permission_required(OrderPermissions.MANAGE_ORDERS)
    def resolve_order(self, info, **data):
        _, id = from_global_id_or_error(data.get("id"), Order)
        return resolve_order(id)

    @permission_required(OrderPermissions.MANAGE_ORDERS)
    def resolve_orders(self, info, channel=None, **_kwargs):
        return resolve_orders(info, channel)

    @permission_required(OrderPermissions.MANAGE_ORDERS)
    def resolve_draft_orders(self, info, **_kwargs):
        return resolve_draft_orders(info)

    @permission_required(OrderPermissions.MANAGE_ORDERS)
    def resolve_orders_total(self, info, period, channel=None, **_kwargs):
        return resolve_orders_total(info, period, channel)

    def resolve_order_by_token(self, _info, token):
        return resolve_order_by_token(token)

class OrderMutations(graphene.ObjectType):
    draft_order_complete = DraftOrderComplete.Field()
    draft_order_create = DraftOrderCreate.Field()
    draft_order_delete = DraftOrderDelete.Field()
    draft_order_bulk_delete = DraftOrderBulkDelete.Field()
    draft_order_lines_bulk_delete = DraftOrderLinesBulkDelete.Field(
        deprecation_reason=DEPRECATED_IN_3X_FIELD
    )
    draft_order_update = DraftOrderUpdate.Field()

    order_add_note = OrderAddNote.Field()
    order_cancel = OrderCancel.Field()
    order_capture = OrderCapture.Field()
    order_confirm = OrderConfirm.Field()

    order_fulfill = OrderFulfill.Field()
    order_fulfillment_cancel = FulfillmentCancel.Field()
    order_fulfillment_approve = FulfillmentApprove.Field()
    order_fulfillment_update_tracking = FulfillmentUpdateTracking.Field()
    order_fulfillment_refund_products = FulfillmentRefundProducts.Field()
    order_fulfillment_return_products = FulfillmentReturnProducts.Field()

    order_lines_create = OrderLinesCreate.Field()
    order_line_delete = OrderLineDelete.Field()
    order_line_update = OrderLineUpdate.Field()

    order_discount_add = OrderDiscountAdd.Field()
    order_discount_update = OrderDiscountUpdate.Field()
    order_discount_delete = OrderDiscountDelete.Field()

    order_line_discount_update = OrderLineDiscountUpdate.Field()
    order_line_discount_remove = OrderLineDiscountRemove.Field()

    order_mark_as_paid = OrderMarkAsPaid.Field()
    order_refund = OrderRefund.Field()
    order_update = OrderUpdate.Field()
    order_update_shipping = OrderUpdateShipping.Field()
    order_void = OrderVoid.Field()
    order_bulk_cancel = OrderBulkCancel.Field()


from django.shortcuts import render
from .models import Meetings, Nutritionist
import json
from django.core import serializers
from django.http import JsonResponse
from .forms import MeetingForm
from django.views.decorators.csrf import csrf_exempt
from django.core.mail import EmailMessage

# Create your views here.
def home(request):
    title = 'dietetycy'
    nutritionists = Nutritionist.objects.all()
    data_json = serializers.serialize("json", nutritionists)
    return render(request, "nutritionist.html", {'title': title, 'nutritionists': nutritionists, "data_json": data_json})

def calendar(request, id=None):
    title = 'kalendarz'
    meetings = Meetings.objects.all()
    url = request.get_full_path()
    n_id = url[-1:]
    dataToCal = {
        '1': [],
        '2': [],
        '3': [],
        '4': [],
        '5': []
    }
    # filling fields in calendar
    for day in range(1,6):
        day = str(day)
        for meeting in meetings:

            if meeting.day == day and str(meeting.nutritionist.id) == n_id:

                hourBegin = meeting.hourBegin
                hourEnd = meeting.hourEnd

                dataToCal[day].append([hourBegin, hourEnd])
    dataToCal = json.dumps(dataToCal)
    return render(request, "calendar.html", {'title': title, 'dataToCal': dataToCal})

@csrf_exempt
def send_form(request):

    if request.method == "POST":
        url = request.get_full_path()
        day = url[-7:-6]
        n_id = url[-1:]
        hourBegin = url[-6:-1].encode('ascii','ignore')
        hourEnd = hourBegin
        hourEnd = list(hourEnd)

        # creating hour of end from hour of begin
        if hourEnd[3] == '0':
            hourEnd[3] = '3'
        elif hourEnd[3] == '3':
            if hourEnd[1] == '9' and hourEnd[0] == "0":
                hourEnd[0] = "1"
                hourEnd[1] = "0"
                hourEnd[3] = '0'
            elif hourEnd[1] == '9' and hourEnd[0] == "1":
                hourEnd[0] = "2"
                hourEnd[1] = "0"
                hourEnd[3] = '0'
            else:
                hourEnd[1] = str(int(hourEnd[1]) + 1)
                hourEnd[3] = '0'
        hourEnd = "".join(hourEnd)
        nutritionist = Nutritionist.objects.get(pk=n_id)

        meeting_hourBegin = hourBegin
        meetting_hourEnd = hourEnd
        meeting_day = day
        meeting_name = request.POST.get('the_name')
        meeting_fullname = request.POST.get('the_fullname')
        meeting_email = request.POST.get('the_email')
        meeting_nutritionist = nutritionist

        meeting = Meetings(
            hourBegin=meeting_hourBegin,
            hourEnd=meetting_hourEnd ,
            day=meeting_day ,
            name=meeting_name ,
            fullname=meeting_fullname ,
            email=meeting_email ,
            nutritionist=meeting_nutritionist
        )
        meeting.save()

        response_data = {}
        response_data['hourBegin'] = meeting.hourBegin
        response_data['hourEnd'] = meeting.hourEnd
        response_data['day'] = meeting.day
        response_data['name'] = meeting.name
        response_data['fullname'] = meeting.fullname
        response_data['email'] = meeting.email
        response_data['nutritionist'] = meeting.nutritionist.id

        days = {
            '1': 'poniedzialek',
            '2': 'wtorek',
            '3': 'sroda',
            '4': 'czwartek',
            '5': 'piatek'
        }
        email = EmailMessage("Potwierzenie rejestracji",
                             'Witaj {}! Zarejestrowales sie do {} {} {} w najblizszy {} o godznie {}'.
                             format(meeting_name.encode('ascii','ignore'),
                                    meeting_nutritionist.title.encode('ascii','ignore'),
                                    meeting_nutritionist.name.encode('ascii','ignore'),
                                    meeting_nutritionist.fullname.encode('ascii','ignore'),
                                    days[meeting_day].encode('ascii','ignore'),
                                    meeting_hourBegin).encode('ascii','ignore'),
                             to=[meeting_email])
        email.send()

        return JsonResponse(response_data)

    elif request.method == "GET":
        title = 'Formularz'
        form = MeetingForm()
        return render(request, "form.html", {'title': title, 'form': form})

def thanks(request):
    title = 'dziekujemy'
    return render(request, "thanks.html", {'title': title})


import json
import os

from cbc_api.schedule.tsp import TSP
from cbc_api.schedule.schedule import Schedule
from cbc_api.schedule.capacity_tester import CapacityTester

PATH = os.path.dirname(os.path.realpath(__file__))

def test_google_tsp():
    tsp = TSP()
    soln = tsp.solve_tsp(
        origin=[28.50,-81.25],
        destination=[28.50,-81.25],
        waypoints=[
            [28.53874, -81.37861],
            [28.54052, -81.38118],
            [28.56707, -81.38971],
            [28.4497135606058, -81.4707795871383],
            [28.5414049683169, -81.37367],
            [28.404738, -81.428915]
        ],
        method='google'
    )
    assert 'durations' in soln
    for duration in soln['durations']:
        assert type(duration) in [int,float]
    assert 'order' in soln
    for order in soln['order']:
        assert type(order) == int

def test_pyschedule_tsp():
    tsp = TSP()
    soln = tsp.solve_tsp(
        origin=[28.50,-81.25],
        destination=[28.50,-81.25],
        waypoints=[
            [28.53874, -81.37861],
            [28.54052, -81.38118],
            [28.56707, -81.38971],
            [28.4497135606058, -81.4707795871383],
            [28.5414049683169, -81.37367],
            [28.404738, -81.428915]
        ],
        method='pyschedule'
    )
    assert 'durations' in soln
    for duration in soln['durations']:
        assert type(duration) in [int,float]
    assert 'order' in soln
    for order in soln['order']:
        assert type(order) == int

def test_pyschedule_constrained_tsp():
    tsp = TSP()
    soln = tsp.solve_tsp(
        origin=[28.50,-81.25],
        destination=[28.50,-81.25],
        waypoints=[
            [28.53874, -81.37861],
            [28.54052, -81.38118],
            [28.56707, -81.38971],
            [28.4497135606058, -81.4707795871383],
            [28.5414049683169, -81.37367],
            [28.404738, -81.428915]
        ],
        constraints=[
            {
                'index1' : '4',
                'constraint_type' : 'before',
                'index2' : '2'
            },
            {
                'index1': '0',
                'constraint_type' : 'after',
                'index2' : '2'
            }
        ],
        method='pyschedule'
    )
    assert 'durations' in soln
    for duration in soln['durations']:
        assert type(duration) in [int,float]
    assert 'order' in soln
    for order in soln['order']:
        assert type(order) == int

def test_schedule():
    path = '/'.join(PATH.split('/')[:-1])
    filename = path + '/cbc_api/data/schedule_test.json'
    with open(filename, 'r') as f:
        configs = json.load(f)

    schedule = Schedule(configs)
    schedule.schedule_activities()
    assert 'activities' in schedule.configs
    assert 'distance_matrix' in schedule.configs
    for activity in schedule.configs['activities']:
        assert 'assigned_day' in activity
        assert 'cluster_coords' in activity
        assert 'coordinates' in activity
        assert 'day' in activity
        assert 'duration' in activity
        assert 'from_duration' in activity
        assert 'index' in activity
        assert 'label' in activity
        assert 'order' in activity
        assert 'to_duration' in activity

def test_capacity_tester():
    capacities = [120, 120, 240, 480, 480]
    durations = [87, 101, 132, 139, 161]
    capacity_tester = CapacityTester(capacities, durations)
    extra_days = capacity_tester.find_additional_capacity()
    assert extra_days == 3

    capacities = [120, 240]
    durations = [139, 161]
    capacity_tester = CapacityTester(capacities, durations)
    extra_days = capacity_tester.find_additional_capacity()
    assert extra_days == 0


import numpy as np
from matplotlib import pyplot as plt
from typing import Tuple, List, Dict

import torch
import torch.utils.data

import config as cf
import data_process.augment as aug
from data_process.record import Record

class Dataset(torch.utils.data.Dataset):

    def __init__(self) -> None:
        ''' Init the dataset data structures.
        '''
        self.size = 0
        self.positive_size = 0
        self.track_data = {'center_pos': None, 'center_rot': None, 'marker_pos': None}
        self.imu_data = {'acc': None, 'gyro': None}
        self.labels = None
        self.reindex_map = None

    def __len__(self) -> int:
        return self.size

    def __getitem__(self, idx:int) -> Dict[str, np.ndarray]:
        ''' Get an action sample data.
        args:
            idx: int, the index of the action sample.
                the idx will be reindexed before retrieving the actual sample data.]
        '''
        # TODO: no track augmentation yet, directly return imu data
        # If idx after reindex_map >= positive size, it means the sample at idx is negative data,
        # which does not has track_data. Return imu data directly.
        if self.reindex_map is not None: idx = self.reindex_map[idx]
        w_cut = int(cf.CUT_DURATION * cf.FS_PREPROCESS)
        w_train = int(cf.TRAIN_DURATION * cf.FS_PREPROCESS)
        if cf.RAND_SAMPLE_EN: start = np.random.randint(0, w_cut - w_train)
        else: start = (w_cut - w_train) // 2
        sample = np.concatenate([self.imu_data['acc'][idx,start:start+w_train,:],
            self.imu_data['gyro'][idx,start:start+w_train,:]], axis=1)
        sample = aug.down_sample_by_step(sample, axis=0, step=(cf.FS_PREPROCESS//cf.FS_TRAIN))
        return {'data': sample, 'label': self.labels[idx]}

    def insert_records(self, records:List[Record], labels:List[int]) -> None:
        ''' Insert a list of records.
        args:
            records: List[Record], a list or records.
            labels: List[int], the labels of these records,
                must have the same length as records.
        '''
        # a simple version based on insert_record()
        for record, label in zip(records, labels):
            self.insert_record(record, label)

    def insert_record(self, record:Record, label:int) -> None:
        ''' Insert a single record with all samples have the same group id.
            NOTE: always insert negative after positive data.
        args:
            record: Record, the action record.
            label: int, the label of the record.
        '''
        # insert imu data
        clean_mask = record.clean_mask
        cnt = np.sum(clean_mask)
        imu_data = self.imu_data
        cutted_imu_data = record.cutted_imu_data
        if imu_data['acc'] is None:
            imu_data['acc'] = cutted_imu_data['acc'][clean_mask,...]
            imu_data['gyro'] = cutted_imu_data['gyro'][clean_mask,...]
        else:
            imu_data['acc'] = np.concatenate([imu_data['acc'],
                cutted_imu_data['acc'][clean_mask,...]], axis=0)
            imu_data['gyro'] = np.concatenate([imu_data['gyro'],
                cutted_imu_data['gyro'][clean_mask,...]], axis=0)
        self.imu_data = imu_data
        # insert track data
        track_data = self.track_data
        cutted_track_data = record.cutted_track_data
        if track_data['center_pos'] is None:
            track_data['center_pos'] = cutted_track_data['center_pos'][clean_mask,...]
            track_data['center_rot'] = cutted_track_data['center_rot'][clean_mask,...]
            track_data['marker_pos'] = cutted_track_data['marker_pos'][clean_mask,...]
        else:
            track_data['center_pos'] = np.concatenate([track_data['center_pos'],
                cutted_track_data['center_pos'][clean_mask,...]], axis=0)
            track_data['center_rot'] = np.concatenate([track_data['center_rot'],
                cutted_track_data['center_rot'][clean_mask,...]], axis=0)
            track_data['marker_pos'] = np.concatenate([track_data['marker_pos'],
                cutted_track_data['marker_pos'][clean_mask,...]], axis=0)
        self.track_data = track_data
        self.positive_size += cnt
        # insert labels
        if self.labels is None:
            self.labels = np.zeros(cnt, dtype=np.int64) + label
        else: self.labels = np.concatenate([self.labels, np.zeros(cnt, dtype=np.int64) + label])
        self.size += cnt

    def insert_record_raise_drop(self, record:Record, raise_label:int, drop_label:int) -> None:
        ''' An adapter for insert raise and drop data only.
        args:
            record: Record, the action record.
            raise_label: int, the label of raise.
            drop_label: int, the label of drop.
        '''
        clean_mask = record.clean_mask
        cnt = np.sum(clean_mask)
        imu_data = self.imu_data
        cutted_imu_data = record.cutted_imu_data
        # insert imu data
        if imu_data['acc'] is None:
            imu_data['acc'] = cutted_imu_data['acc'][clean_mask,...]
            imu_data['gyro'] = cutted_imu_data['gyro'][clean_mask,...]
        else:
            imu_data['acc'] = np.concatenate([imu_data['acc'],
                cutted_imu_data['acc'][clean_mask,...]], axis=0)
            imu_data['gyro'] = np.concatenate([imu_data['gyro'],
                cutted_imu_data['gyro'][clean_mask,...]], axis=0)
        self.imu_data = imu_data
        # insert track data
        track_data = self.track_data
        cutted_track_data = record.cutted_track_data
        if track_data['center_pos'] is None:
            track_data['center_pos'] = cutted_track_data['center_pos'][clean_mask,...]
            track_data['center_rot'] = cutted_track_data['center_rot'][clean_mask,...]
            track_data['marker_pos'] = cutted_track_data['marker_pos'][clean_mask,...]
        else:
            track_data['center_pos'] = np.concatenate([track_data['center_pos'],
                cutted_track_data['center_pos'][clean_mask,...]], axis=0)
            track_data['center_rot'] = np.concatenate([track_data['center_rot'],
                cutted_track_data['center_rot'][clean_mask,...]], axis=0)
            track_data['marker_pos'] = np.concatenate([track_data['marker_pos'],
                cutted_track_data['marker_pos'][clean_mask,...]], axis=0)
        self.track_data = track_data
        # insert labels
        labels = np.zeros(cutted_imu_data['acc'].shape[0], dtype=np.int64) + raise_label
        labels[1::2] = drop_label
        if self.labels is None:
            self.labels = labels[clean_mask]
        else: self.labels = np.concatenate([self.labels, labels[clean_mask]])
        self.positive_size += cnt
        self.size += cnt

    def insert_negativa_data(self, negative_data:np.ndarray, label:int):
        ''' Insert negative data in batch.
        args:
            negative_data: np.ndarray[(N, int(FS_PREPROCESS*WINDOW_DURATION), 6), np.float32].
            label: int, the negative label.
        '''
        cnt = negative_data.shape[0]
        imu_data = self.imu_data
        if imu_data['acc'] is None:
            imu_data['acc'] = negative_data[:,:,:3]
            imu_data['gyro'] = negative_data[:,:,3:]
        else:
            imu_data['acc'] = np.concatenate([imu_data['acc'], negative_data[:,:,:3]], axis=0)
            imu_data['gyro'] = np.concatenate([imu_data['gyro'], negative_data[:,:,3:]], axis=0)
        self.imu_data = imu_data
        labels = np.zeros(cnt, dtype=np.int64) + label
        if self.labels is None:
            self.labels = labels
        else: self.labels = np.concatenate([self.labels, labels])
        self.size += cnt

    def shuffle(self) -> None:
        ''' Shuffle the reindex mapping.
            NOTE: always call shuffle after inserting records.
        '''
        self.reindex_map = np.arange(self.size, dtype=np.int32)
        np.random.shuffle(self.reindex_map)

class DataLoader(torch.utils.data.DataLoader):

    def __init__(self, dataset:Dataset, *args, **kwargs) -> None:
        super(DataLoader, self).__init__(dataset, *args, **kwargs)

if __name__ == '__main__':
    pass


from functools import partial
from logging import getLogger, StreamHandler, Formatter, DEBUG, INFO, WARNING, ERROR, CRITICAL
from logging.handlers import QueueHandler
from sys import stdout
from multiprocessing import Process, Queue, current_process
from time import sleep
import atexit

def start_listener(queue):
	listener = getLogger('compile')
	handler = StreamHandler(stdout)
	handler.setFormatter(Formatter('%(asctime)s %(processName)-10s %(name)s %(levelname)-8s %(message)s'))
	listener.addHandler(handler)
	while True:
		try:
			record = queue.get()
			if record is None:
				break
			# level or filter
			listener.handle(record)
		except Exception:
			import sys, traceback
			print('The logger had a problem!', file=sys.stderr)
			traceback.print_exc(file=sys.stderr)

def start_sender(queue):
	sender = getLogger('compile')
	sender.addHandler(QueueHandler(queue))
	for k in range(5):
		sleep(.070)
		sender.log(DEBUG, '{0:s}: message #{1:d}'.format(current_process().name, k))

def kill_logger(proc):
	proc.join(timeout=0.1)
	proc.terminate()

def main():
	queue = Queue(-1)
	listener = Process(target=start_listener, args=(queue,))
	listener.start()
	atexit.register(partial(kill_logger, listener))
	senders = []
	for k in range(5):
		sleep(.150)
		worker = Process(target=start_sender, args=(queue,))
		senders.append(worker)
		worker.start()
		atexit.register(partial(kill_logger, worker))
	for sender in senders:
		sender.join()
	queue.put_nowait(None)
	listener.join()

if __name__ == '__main__':
	main()


import duckdb
import logging

from django.conf import settings
from tally_ho.apps.tally.management.commands.utils import (
    build_generic_model_key_values_from_duckdb_row_tuple_data,
    check_for_missing_columns,
    get_ballot_by_ballot_number,
    get_electrol_race_by_ballot_name,
)
from tally_ho.apps.tally.models.ballot import Ballot
from tally_ho.apps.tally.models.candidate import Candidate
from tally_ho.apps.tally.models.electrol_race import ElectrolRace
from tally_ho.apps.tally.models.tally import Tally
from tally_ho.libs.utils.query_set_helpers import BulkCreateManager
from tally_ho.celeryapp import app
from tally_ho.libs.utils.memcache import MemCache

logger = logging.getLogger(__name__)

def create_candidates_from_candidates_file_data(
        duckdb_candidates_data=None,
        tally=None,
        ballots_by_ballot_number=None,
        electrol_races_by_ballot_name=None,
        candidate_id_by_ballot_order_dict=None,
        step_number=None,
        step_name=None,
        command=None,
):
    """Create candidates from candidates file data inside duckdb.

    :param duckdb_candidates_data: candidates file data in duckdb format.
    :param tally: candidates tally.
    :param ballots_by_ballot_number: ballots by ballot name dict.
    :param electrol_races_by_ballot_name: electrol races by ballot name dict.
    :param candidate_id_by_ballot_order_dict: candidate id by ballot order
        dict.
    :param step_number: Step number
    :param step_name: Step name
    :param command: stdout command.
    :returns: None"""
    try:
        instances_count_memcache_key =\
            f"{tally.id}_{step_name}_{step_number}"
        # reset instances count in memcache if exists already
        client = MemCache()
        client.delete(instances_count_memcache_key)
        candidate_foreign_key_fields =\
                    ['ballot', 'electrol_race']
        col_names_to_model_field_map =\
            getattr(settings,
                    'CANDIDATE_FILE_COLS_NAMES_TO_CANDIDATE_MODEL_FIELDS')
        candidates_cols_names_list =\
            list(col_names_to_model_field_map.keys())
        candidates_data =\
                    duckdb_candidates_data.project(
                    ','.join(
            candidates_cols_names_list)).distinct().fetchall()
        bulk_mgr = BulkCreateManager(
            chunk_size=1000,
            cache_instances_count=True,
            cache_key=instances_count_memcache_key,
            memcache_client=client,
        )

        for candidate_vals_tuple in candidates_data:
            kwargs =\
                build_generic_model_key_values_from_duckdb_row_tuple_data(
                    candidate_vals_tuple,
                    col_names_to_model_field_map,
                    candidates_cols_names_list,
                )
            candidate_data_by_index =\
                { item[0]: item[1] for item in enumerate(
                list(candidate_vals_tuple))}
            for index, col_name in enumerate(candidates_cols_names_list):
                field_name = col_names_to_model_field_map.get(col_name)
                # Continue if field is not a foreign key field
                if field_name not in candidate_foreign_key_fields:
                    continue

                field_val = candidate_data_by_index.get(index)
                if field_name == 'ballot':
                    ballot = get_ballot_by_ballot_number(
                        field_val,
                        ballots_by_ballot_number
                    )
                    kwargs['ballot'] = ballot
                    continue

                if field_name == 'electrol_race':
                    electrol_race = get_electrol_race_by_ballot_name(
                        field_val,
                        electrol_races_by_ballot_name,
                    )
                    kwargs['electrol_race'] = electrol_race
                    continue

            if len(kwargs.items()):
                kwargs['tally'] = tally
                kwargs['order'] = candidate_id_by_ballot_order_dict.get(
                    kwargs.get('candidate_id'))
                bulk_mgr.add(Candidate(**kwargs))
        bulk_mgr.done()

        return
    except Exception as e:
        msg = 'Failed to create candidates, error: %s' % e
        if command:
            command.stdout.write(command.style.WARNING(msg))
        if logger:
            logger.warning(msg)
        raise Exception(msg)

def build_candidate_id_by_ballot_order_dict(ballot_order_file_path):
    """
    Builds a dictionary of candidate id by ballot order from the ballot order
    to candidates ids csv file.

    :param ballot_order_file_path: ballot order to candidates ids csv file.
    :returns: candidate ids by ballot order dictionary or error exception.
    """
    try:
        candidate_id_by_ballot_order = {}
        ballot_order_column_list =\
                getattr(settings,
                        'BALLOT_ORDER_COLUMN_NAMES')
        duckdb_ballot_order_data =\
            duckdb.from_csv_auto(ballot_order_file_path, header=True).project(
                    ','.join(
            ballot_order_column_list)).distinct().fetchall()
        candidate_id_by_ballot_order =\
            { item[0]: item[1] for item in duckdb_ballot_order_data }
        return candidate_id_by_ballot_order
    except Exception as e:
        msg = str("Error occured while generating candidate id to"
                  f"ballot order dictionary, error: {e}")
        raise Exception(msg)

@app.task()
def async_import_candidates_from_candidates_file(
        tally_id=None,
        csv_file_path=None,
        command=None,
        **kwargs):
    """Create candidates from a candidates csv file.

    :param tally_id: tally id.
    :param csv_file_path: candidates csv file path.
    :param command: stdout command.
    :returns: candidates count."""
    try:
        tally = Tally.objects.get(id=tally_id)
        candidates_file_path = csv_file_path
        duckdb_candidates_data =\
            duckdb.from_csv_auto(candidates_file_path, header=True)
        candidate_col_names =\
            getattr(settings,
                    'CANDIDATE_COLUMN_NAMES')
        check_for_missing_columns(
            candidate_col_names,
            duckdb_candidates_data.columns,
            'candidates'
        )
        ballot_order_file_path = kwargs.get('ballot_order_file_path')
        candidate_id_by_ballot_order_dict =\
            build_candidate_id_by_ballot_order_dict(
                ballot_order_file_path
            )
        ballots_by_ballot_number =\
            {
                ballot.number:\
                ballot for ballot in Ballot.objects.filter(tally=tally)
            }
        electrol_races_by_ballot_name =\
            {
                electrol_race.ballot_name:\
                electrol_race for electrol_race in\
                    ElectrolRace.objects.filter(tally=tally)
            }
        create_candidates_from_candidates_file_data(
            duckdb_candidates_data=duckdb_candidates_data,
            tally=tally,
            ballots_by_ballot_number=ballots_by_ballot_number,
            electrol_races_by_ballot_name=electrol_races_by_ballot_name,
            candidate_id_by_ballot_order_dict=
            candidate_id_by_ballot_order_dict,
            step_name=kwargs.get('step_name'),
            step_number=kwargs.get('step_number'),
            command=command,
        )

        return duckdb_candidates_data.shape[0]
    except Exception as e:
        msg = f'Error occured while trying to create candidates: {e}'
        if command:
            command.stdout.write(command.style.WARNING(msg))
        if logger:
            logger.warning(msg)
        raise Exception(msg)


"""Create data plots for ERP-SCANR project - plots for group analysis."""

import os
import matplotlib.pyplot as plt

from erpsc.core.db import check_db

#########################################################################################
#########################################################################################
#########################################################################################

def plot_time_assocs(dat, save_fig=False):
    """

    Parameters
    ----------
    dat : list of list of [str, str, int]
        ERP data - [association, P or N, latency]
    """

    # Plot params
    offsets = {'P': 50, 'N': -50}
    rotations = {'P': 45, 'N': -45}

    # Initialize Plot
    fig = plt.figure(figsize=(12, 5))
    fig.suptitle('ERP Correlates Across Time', fontsize=24, fontweight='bold')
    ax = fig.add_subplot(111)

    # Set plot limits
    ax.set_xlim([50, 600])
    ax.set_ylim([-100, 100])

    # Add x-ticks
    plt.xticks([250, 500], ['250 ms', '500 ms'])
    ax.set_yticks([])

    # Set ticks and plot lines
    ax.spines['right'].set_color('none')
    ax.spines['bottom'].set_position('center')
    ax.spines['top'].set_color('none')
    ax.xaxis.set_ticks_position('none')
    ax.yaxis.set_ticks_position('none')
    ax.spines['left'].set_linewidth(2)
    ax.spines['bottom'].set_linewidth(2)

    # Add data to plot from
    for d in dat:

        # Text takes: [X-pos, Y-pos, word, rotation]
        #  Where X-pos is latency, y-pos & rotation are defaults given +/-
        ax.text(d[2], offsets[d[1]], d[0], rotation=rotations[d[1]], fontsize=20)

    # Save out - if requested
    if save_fig:

        db = check_db(db)
        s_file = os.path.join(db.figs_path, 'LatencyAssociations' + '.svg')

        plt.savefig(s_file, transparent=True)


from setuptools import setup, find_packages

VERSION = '0.4.1'
DESCRIPTION = "detectda - detecting features in videos using TDA"
LONG_DESCRIPTION = "detectda - a cubical persistent homology package for the detection and hypothesis testing of features in greyscale videos"

setup(
	name="detectda",
	version = VERSION,
	author = "Andrew M. Thomas",
	author_email = "<me@andrewmthomas.com>",
	description = DESCRIPTION,
	long_description = LONG_DESCRIPTION,
	package=find_packages(),
	install_requires=[
		'gudhi',
		'shapely >= 2.0.1',
		'joblib',
		'scikit-image',
		'scikit-learn >= 0.23.1',
		'numpy',
		'tqdm',
        'pandas',
		'matplotlib',
		'opencv-contrib-python',
        'scipy'
	],
	keywords = ['tda', 'cubical', 'image processing'],
	classifiers = [
		'Operating System :: MacOS',
		'Operating System :: Microsoft :: Windows',
		'Programming Language :: Python'
	],
	entry_points={
		'gui_scripts': [
			'identify_polygon = detectda.idpo:identify_polygon'
		]
	}
)