import torch
import torch.nn as nn
import sys
from model_shapes.nmn_att_seq import *
from model_shapes.module_net import *
from model_shapes.vision_model import CNN
from Utils.utils import unique_columns

class N2NModuleNet(nn.Module):
    def __init__(self, num_que_vocab, num_answer, hyper, assembler,
                 layout_criterion, answer_criterion):
        super(N2NModuleNet, self).__init__()

        self.assembler = assembler
        self.layout_criterion = layout_criterion
        self.answer_criterion = answer_criterion
        self.hyper = hyper

        # initiate encoder and decoder
        encoder = EncoderRNN(input_size=num_que_vocab,
                             lstm_dim=self.hyper.lstm_dim,
                             embed_dim_que=self.hyper.embed_dim_que,
                             num_layers=self.hyper.num_layers)
        decoder = AttnDecoderRNN(hyper=self.hyper,
                                 nmn_dim=self.assembler.num_module,
                                 EOS_token=self.assembler.EOS_idx)

        if use_cuda:
            encoder = encoder.cuda()
            decoder = decoder.cuda()

        # initatiate attentionSeq2seq
        seq_model = Seq2SeqAtt(encoder, decoder)
        self.seq_model = seq_model.cuda() if use_cuda else seq_model

        vision_model = CNN()
        self.vision_model = vision_model.cuda() if use_cuda else vision_model

        # initiate moduleNet
        mod_net = ModuleNet(image_height=self.hyper.H_im,
                            image_width=self.hyper.W_im,
                            image_channel=self.hyper.D_feat,
                            embed_dim_que=self.hyper.embed_dim_que,
                            num_answer=2,
                            lstm_dim=self.hyper.lstm_dim)

        self.mod_net = mod_net.cuda() if use_cuda else mod_net

    def forward(self, question_variable, layout_variable,
                image_batch, seq_len_batch, label_batch,
                sample_token, policy_gradient_baseline=None,
                baseline_decay=None):
        batch_size = seq_len_batch.shape[0]

        predicted_tokens, attentions, neg_entropy, log_seq_prob = \
            self.seq_model(question_variable, seq_len_batch, layout_variable,
                           sample_token)

        layout_loss = None
        if layout_variable is not None:
            layout_loss = torch.mean(-log_seq_prob)

        predicted_layouts = np.asarray(predicted_tokens.cpu().data.numpy())
        expr_list, expr_validity_array = self.assembler.assemble(predicted_layouts)

        # group samples based on layout
        sample_groups_by_layout = unique_columns(predicted_layouts)

        # run moduleNet
        answer_losses = None
        policy_gradient_losses = None
        avg_answer_loss = None
        total_loss = None
        updated_baseline = policy_gradient_baseline
        current_answer = np.zeros(batch_size)

        for sample_group in sample_groups_by_layout:
            if sample_group.shape == 0:
                continue

            first_in_group = sample_group[0]
            if expr_validity_array[first_in_group]:
                layout_exp = expr_list[first_in_group]

                ith_answer = label_batch[sample_group]
                ith_answer_variable = Variable(ith_answer.long())
                ith_answer_variable = ith_answer_variable.cuda() \
                    if use_cuda else ith_answer_variable
                que_attention = attentions[sample_group, :]

                ith_image = image_batch[sample_group, :, :, :]
                ith_images_variable = Variable(torch.FloatTensor(ith_image))
                ith_images_variable = ith_images_variable.cuda() \
                    if use_cuda else ith_images_variable

                # image[batch_size, H_feat, W_feat, D_feat] ==>
                # [batch_size, D_feat, W_feat, H_feat] for conv2d
                # ith_images_variable = ith_images_variable.permute(0, 3, 1, 2)

                ith_images_variable = ith_images_variable.contiguous()

                answers = self.mod_net(input_image_variable=ith_images_variable,
                                       input_text_attention_variable=que_attention,
                                       target_answer_variable=ith_answer_variable,
                                       expr_list=layout_exp)
                current_answer[sample_group] = torch.topk(answers, 1)[1].cpu().data.numpy()[:, 0]

                # compute loss function only when answer is provided
                if ith_answer_variable is not None:
                    current_answer_loss = self.answer_criterion(answers, ith_answer_variable)
                    sample_group_tensor = torch.cuda.LongTensor(
                        sample_group) if use_cuda else torch.LongTensor(sample_group)

                    current_log_seq_prob = log_seq_prob[sample_group_tensor]
                    current_answer_loss_val = Variable(current_answer_loss.data,
                                                       requires_grad=False)
                    tmp1 = current_answer_loss_val - policy_gradient_baseline
                    current_policy_gradient_loss = tmp1 * current_log_seq_prob

                    if answer_losses is None:
                        answer_losses = current_answer_loss
                        policy_gradient_losses = current_policy_gradient_loss
                    else:
                        answer_losses = torch.cat((answer_losses, current_answer_loss))
                        policy_gradient_losses = torch.cat(
                            (policy_gradient_losses, current_policy_gradient_loss))

        try:
            if label_batch is not None:
                total_loss, avg_answer_loss = \
                    self.layout_criterion(neg_entropy=neg_entropy,
                                          answer_loss=answer_losses,
                                          policy_gradient_losses=policy_gradient_losses,
                                          layout_loss=layout_loss)
                ##update layout policy baseline
                avg_sample_loss = torch.mean(answer_losses)
                avg_sample_loss_value = avg_sample_loss.cpu().data.numpy()[0]
                updated_baseline = policy_gradient_baseline + (1 - baseline_decay) * (
                        avg_sample_loss_value - policy_gradient_baseline)

        except:
            print("sample_group = ", sample_group)
            print("neg_entropy=", neg_entropy)
            print("answer_losses=", answer_losses)
            print("policy_gradient_losses=", policy_gradient_losses)
            print("layout_loss=", layout_loss)
            sys.stdout.flush()
            sys.exit("Exception Occur")

        return total_loss, avg_answer_loss, current_answer, predicted_layouts, expr_validity_array, updated_baseline


from .IngestorInterface import IngestorInterface
from .QuoteModel import QuoteModel
from typing import List
import subprocess

class PDFIngestor(IngestorInterface):
    allowed_exensions = ['pdf']

    @classmethod
    def parse(cls, path: str) -> List[QuoteModel]:
        quotes = []
        p = subprocess.Popen(
            ['pdftotext', '-nopgbrk', path, '-'], stdout=subprocess.PIPE
        )
        data = p.communicate()[0].decode()
        data = data.split('\r\n')
        for line in data:
            if line != '':
                data = line.split(' - ')
                quotes.append(QuoteModel(data[0], data[1]))
        return quotes


import time
from datetime import datetime

from .worm import PyGame, State
from libs import log

logger = log.getLogger(__name__)

class Game:
    mystr = ""
    times = 0

    def __init__(self, state = None):
        logger.info(__name__)
        # this state is only initial, not used on reload
        self.state = State(
            window_caption="Pygame Window "
        )
        self.mystr = "Hello world"

    def try_pygame(self, state: State):
        self.pygame = PyGame(state)
        #self.pygame.set_state(state)
        self.pygame.on_execute()

    def run(self, state):
        logger.info(datetime.now())
        self.test()
        state = state or self.state
        self.try_pygame(state)
        time.sleep(1)
        self.pygame.state.times += 1

        #self.pygame.state.window_caption = "Pygame Window "
        return self.pygame.state

    def test(self):
        #logger.info(self)
        #print(self.mystr + str(self.times))
        self.times += 1


import itertools
def makeIntList(list):
    count = 0;
    while (count < len(list)):
        list[count] = int(list[count]);
        count += 1;
    return list;
def sonList(list):
    ans=0;
    j=1;
    while(j<=len(list)):
        for i in itertools.combinations(list, j):
                ans+=max(i)-min(i);
        j+=1;
    print(ans);

list=makeIntList(str(input()).split(","));
sonList(list);


import nltk
import re
import numpy as np
from nltk.translate import IBMModel1,AlignedSent
from sklearn.feature_extraction.text import TfidfVectorizer
from nltk.corpus import stopwords
# nltk.download('stopwords')
"""
**********EXAMPLE BASED MACHINE TRANSLATION SYSTEM 'EBMT'******************************
Translation in three steps: matching, alignment and recombination
Matching: finds the example or set of examples from the bitext which most closely match the source-language string to be translated.4
Alignment: extracts the source–target translation equivalents from the retrieved examples of the matching step.
Recombination: produces the final translation by combining the target translations of the relevant subsentential fragments.
"""
############################## PREPROCESSING SECTION ###################################
replacement_patterns = [
(r'won\'t', 'will not'),
(r'can\'t', 'can not'),
(r'i\'m', 'i am'),
(r'ain\'t', 'is not'),
(r'(\w+)\'ll', '\g<1> will'),
(r'(\w+)n\'t', '\g<1> not'),
(r'(\w+)\'ve', '\g<1> have'),
(r'(\w+)\'s', '\g<1> is'),
(r'(\w+)\'re', '\g<1> are'),
(r'(\w+)\'d', '\g<1> would')
]
# stop words for each languages
en_Stp_wrd= set(stopwords.words('english'))
tr_Stp_wrd= set(stopwords.words('turkish'))
az_Stp_wrd= set(stopwords.words('azerbaijani'))
stpwrds= en_Stp_wrd | tr_Stp_wrd | az_Stp_wrd #union all to have a big set of all the stop wprds

# for contraction removals
class RegexpReplacer(object):
    def __init__(self, patterns=replacement_patterns):
        self.patterns = [(re.compile(regex), repl) for (regex, repl) in patterns]
    def replace(self, text):
        s = text
        for (pattern, repl) in self.patterns:
            s = re.sub(pattern, repl, s)
        return s

def preprocessing(files):
    replacer = RegexpReplacer()
    for file in files:
        with open(file,'r',encoding='utf-8') as f:
            text= f.read()
        text = re.sub(r'[()\[\]{}.,;!?\\-]', '', text) #remove any punctutaions and unwanted characters
        text= text.replace('[','')
        text= text.replace(']','')
        text=text.replace('.','')
        text=text.replace(',','')
        text = text.strip() # spaces
        text=text.lower() # lower case
        text=replacer.replace(text) # apply the stemming
        with open(file,'w',encoding='utf-8') as f1:
            f1.write(text) # save changes

# to remove duplicates from  a list of strings
def remove_duplicates(string_array):
    unique_strings = []
    duplicates = []
    for string in string_array:
        words = string.split()  # Split the string into words
        unique_words = []
        for word in words:
            if word not in unique_words:
                unique_words.append(word)
            else:
                duplicates.append(word)
        unique_string = ' '.join(unique_words)
        if unique_string not in unique_strings:
            unique_strings.append(unique_string)

    return unique_strings

##############################################################################################################################################################################
# creates a parallel corpus for 2 languages given its files
def make_par_corpus(src_lang):
    with open (src_lang ,'r', encoding='utf-8')as f :
        src= f.readlines()
    with open('Turkish.txt' ,'r' ,encoding='utf-8') as f1:
        tk=f1.readlines()
    src = [s.strip() for s in src]
    tk = [t.strip() for t in tk]
    corpus = list(zip(src,tk))
    return corpus

def matching(input_sentence, src_lang):
    target_lang = 'Turkish.txt'  # fixed
    with open(src_lang, 'r', encoding='utf-8') as f1:
        src_sentences = f1.readlines()
    with open(target_lang, 'r', encoding='utf-8') as f2:
        turkish_sentences = f2.readlines()

    # Use the TF-IDF vectorizer to convert sentences to vectors
    vectorizer = TfidfVectorizer(stop_words='english')
    sentence_vectors = vectorizer.fit_transform(src_sentences + [input_sentence])

    # Calculate the Cosine Similarity between the input sentence and all the other sentences
    input_vector = sentence_vectors[-1]
    similarity_scores = np.dot(sentence_vectors[:-1], input_vector.T).toarray().flatten()

    # Sort the similarity scores and get the indices of the top 4 sentences only
    top_indices = np.argsort(similarity_scores)[-5:]

    # Retrieve the top 4 similar sentences and their Turkish translations
    similar_sentences = [src_sentences[i] for i in top_indices]
    turkish_sentences = [turkish_sentences[i] for i in top_indices]
    turkish_tokens = [nltk.word_tokenize(s) for s in turkish_sentences]
    similar_tokens = [nltk.word_tokenize(s) for s in similar_sentences]
    parallel_sentences = list(zip(similar_tokens, turkish_tokens))

    print("similar sentences in the corpus: \n",parallel_sentences)
    return parallel_sentences  # return the top 4 similar source sentences with their translation equivalents

# IBM model for checking the alignment of words
def ibm1(src):
    with open(src, 'r',encoding='utf-8') as f1:
        src_sentences = f1.readlines()
    with open('Turkish.txt', 'r',encoding='utf-8') as f2:
        turkish_sentences = f2.readlines()
    #extra step step to remove spaces
    src_sentences = [s.strip() for s in src_sentences]
    turkish_sentences = [t.strip() for t in turkish_sentences]
    # word tokenize all sentences
    src_tokenized=[nltk.word_tokenize(s) for s in src_sentences]
    tgt_tokenized =[nltk.word_tokenize(s) for s in turkish_sentences]
    # make a parallel sentences corpus but the sentences are tokenized
    parallel_sentences = list(zip(src_tokenized,tgt_tokenized))
    # Align all sentnences to train the IBM model
    bitext = [(AlignedSent(s,t)) for s,t in parallel_sentences]
    #train the model
    ibm1= IBMModel1(bitext,15)
    return ibm1 # return  the model

#to extracts the source–target translation equivalents from the retrieved examples of the matching step.
def alignment(src,simialr_sentences,input_text):
    ibm=ibm1(src) # create the model
    turkish_translations= [word for sents in (pair[1] for pair in simialr_sentences) for word in sents] # extract all the turkish words from the similar sentences as the input
    # print(turkish_translations)
    text= nltk.word_tokenize(input_text)# tokenize the input
    equivlants=[] # it will hold pairs of the best match in turkish for each word in the input
    for s in text:
        all_probs= [(ibm.translation_table[s][w]) for w in turkish_translations] # find all the probabilites of each word in the input sentence with ALL tuekish words in the similar sentences
        for t in turkish_translations:
            # if s in stpwrds: # if a word is a stop word, ignore it
            #     continue
            prob=(ibm.translation_table[s][t]) #check probabilty of the word and a turkish word
            if prob == max(all_probs): # if its the highest probabilty , put it in pair so it acts like a dictionary
                equivlants.append((s,t))
                # print(f't({s}|{t} = {prob}')
                break
    return equivlants

# this function should produces the final translation by combining the target translations of the relevant subsentential fragments
def combination(equivlants, input):
    combined_translation=''
    text = nltk.word_tokenize(input) #tokenize the input
    for w in text:
        for s,t  in equivlants:# check from the equivlants pairs we got from the alignment function
            if w == s:  # if the word in input exists in the equiv pairs, get the translated word
                combined_translation+=" "
                combined_translation += t
                combined_translation+=" "
    return combined_translation # return the sentence

def translate(input, src_lang): # Combine all 3 steps to produce the translation
    flag=True # to check if the input already exist in the corpus
    corpus= make_par_corpus(src_lang)
    for pair  in corpus:
        if input  == pair[0]: # if input sentence already exists, print its translation directly
            print("\ninput sentence: \n",input)
            print("\ntranslation: \n",pair[1])
            flag=False #set flag to false to disallow rest of code to run
            break
    if flag:
        similar_sentences = matching(input,src_lang)
        align=alignment(src_lang,similar_sentences,input)
        translation = combination(align,input)
        str= remove_duplicates(nltk.word_tokenize(translation))
        translation=" ".join(str)
        print("\ninput sentence: \n",input)
        print("\ntranslation: \n",translation)

########################## MAIN  ################################
files=['English.txt','Turkish.txt','Azeri.txt']
preprocessing(files) # preprocess all the files
translation= input('enter translation option:\n1)English-Turkish\n2)Azerbjiani-Turkish\n')

while(not(translation == '2' or translation == '1')):
    translation=input("input 1 or 2 only\n\nenter translation option:\n1)English-Turkish\n2)Azerbjiani-Turkish\n")
text = input('Enter a sentence to translate: \n')
while( len(text.split()) < 10): # <10
    text=input("Enter a sentence of atleast 10 words:\n")

if translation == '1':#For English-Turkish
    translate(text, files[0])
if translation =='2':
    translate(text,files[2])#For Azerbjiani-Turkish


from openpyxl import load_workbook

data = load_workbook("testing.xlsx")

analysis = data["Analysis"]

previous_cols = [4, 5, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32]
previous_row_num = 70

for i in range(8):
  ws = data[data.sheetnames[i]]
  analysis.cell(i+2, 1).value = data.sheetnames[i]
  for col in previous_cols:
    analysis.cell(i+2, col).value = ws.cell(previous_row_num, col).value

data.save("testing.xlsx")


"""
Author: Shivam Jindal
Date: 6/13/18

- Uses Titanic Dataset to create a model that predicts likely survivors
"""

import pandas as pd
from sklearn.preprocessing import StandardScaler, LabelBinarizer
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import cross_val_score, train_test_split, ShuffleSplit, GridSearchCV
from sklearn.metrics import accuracy_score

if __name__ == "__main__":
    df = pd.read_csv('train.csv')
    df = df.drop(['Name', 'Ticket', 'Cabin', "Embarked"], axis=1) # possible but unlikely for these to help our model
    df = df.dropna()

    # Split dataset into numerical attributes and categorical attributes
    titanic = df.drop(["Survived"], axis=1)
    y = df["Survived"]

    # change sex from text values to binary (male and female will become 0/1)
    lb = LabelBinarizer()
    titanic.Sex = lb.fit_transform(titanic.Sex)

    # Preprocess Numerical Data by Scaling
    """
     StandardScaler is a way to standardize the range of data--
     This is helpeful because it essentially normalizes the data,
     puts mean at zero, and unit variance at zero
    """
    scaler = StandardScaler()
    X = scaler.fit_transform(titanic)

    # breaks up the data so we can train and test it
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

    # initialize classifier and fit
    """
     Random Forest Classifier works by making multiple decision trees
     and merging them together
    """
    clf = RandomForestClassifier(bootstrap=True, max_features=2, n_estimators=20)
    clf.fit(X_train, y_train)

    # Get Accuracy on train set
    """
     Shuffle Split - yields indices for training and testing
     cross_val_score - Uses standard k-fold cross validatoin
                     - splits data into folds and then trains/evaluates
                        on each fold. You then get the average and
                        std Dev of scores
    """
    shuffle_validator = ShuffleSplit(20, train_size= 0.8, test_size=0.2, random_state=0)
    scores = cross_val_score(clf, X_train, y_train, cv=shuffle_validator)
    print("Accuracy: %0.4f (+/- %0.2f)" % (scores.mean(), scores.std()))

    # This actually checks the average score on test set
    # If number here is off really off from accuracy (above),
    # then you know the model is overfitting
    predictions = clf.predict(X_test)
    print(accuracy_score(y_test, predictions))

    """
     This is a method to help figure out the best parameters for the model
     Using this, I got off the default parameters, and managed to bring average
     up
    """
    # param_grid = [{'n_estimators': [5,10,15,20], 'max_features':[2,4,6], 'bootstrap':[True, False]}]
    # clf2 = RandomForestClassifier()
    # grid_search = GridSearchCV(clf2, param_grid, cv=5, scoring='neg_mean_squared_error')
    # grid_search.fit(X, y)
    # print(grid_search.best_params_)


import requests

res = requests.get('https://jsonplaceholder.typicode.com/posts')
resp = res.json()
new_dicts = []
for dict_in_resp in resp:
    new = {}
    for k, v in dict_in_resp.items():
        if k in ['id', 'title']:
            new[k] = v
    new_dicts.append(new)

    # new_dict.pop('userId')
    # new_dict.pop('body')


"""
This module contains the classes Graph and Vertex, and implements functions related to graphs according to graph theory.
The Graph class implemented in this module represents a structured graph in the form of an adjacency list.

The advantages and disadvantages of using an adjacency list are:

Pros: Quick and uses less space for sparse graphs.
Cons: It's slow for dense graphs. """

import math
import re

from collections import defaultdict

class Vertex:
    """
    This class implements a vertex. In graph theory, a vertex (plural vertices) or node is the fundamental unit
    from which the graphs are formed. """

    def __init__(self, name):
        """
        Receives the name of the vertex and creates a list of neighbors.
        :param name: The vertex' name
        """

        self.name = str(name)
        self.neighbors = defaultdict(list)
        self.in_neighbors = defaultdict(list)
        self.out_neighbors = defaultdict(list)

    def __str__(self):
        """
        Representation of the vertex in the form of a string as its name.
        :return: The vertex' name
        """

        return self.name

    def __repr__(self):
        """
        Detailed representation of the Vertex object in the form of a string.
        :return: The vertex' name
        """

        return 'Vertex(%r, %r)' % (self.name, self.in_neighbors)

    def add_neighbor(self, v, weight=math.inf, edge_dir=None):
        """
        Adds a new neighbor (adjacent vertex) to the neighbors list.
        :param v: The new neighbor
        :param weight: The weight of the edge connecting the neighbor to the vertex in question,
                       if not informed, assumes the infinite value.
        :param edge_dir: The edge direction or None for undirected graphs.
        """

        if edge_dir == "in":
            self.in_neighbors[v].append(weight)
        elif edge_dir == "out":
            self.out_neighbors[v].append(weight)
        else:
            self.neighbors[v].append(weight)

    def get_neighbors(self):
        if self.out_neighbors:
            return self.out_neighbors
        else:
            return self.neighbors

class Graph(dict):
    """
    This class implements a graph structure. A graph is composed by sets of vertices connected by edges.

    In mathematics, and more specifically in graph theory, a graph is a structure amounting to a set of objects in
    which some pairs of the objects are in some sense "related". The objects correspond to mathematical abstractions
    called vertices (also called nodes or points) and each of the related pairs of vertices is called an edge
    (also called an arc or line).
    """

    def __init__(self, name='Graph', is_directed=False):

        super(Graph, self).__init__()

        self.name = name
        self._is_directed = is_directed

    def __str__(self):
        """
        Represents the graph in the form of an adjacency list.

        :return: A string representing the graph's adjacency list.
        """

        str_g = ''

        for key in sorted(list(self.keys())):
            # str_g += key + ' --> ' + str(self[key].get_neighbors()) + '\n'
            str_g += key + ' → '
            str_list = ''
            for vertex in [(k, v) for k, v in self[key].get_neighbors().items()]:
                str_list += re.sub('[()]', '', str(vertex)) + ' → '
                str_list = re.sub("',", "':", str_list)
                str_list = re.sub("'", '', str_list)
            str_g += (str_list[:-2] if str_list else 'None') + '\n'

        return str_g

    def add_vertex(self, vertex):
        """
        Adds a new vertex to the graph.
        :param vertex: The vertex to be added. It can be a string, an integer, or an instance of Vertex.
        :return: True if it have succeeded in adding the new vertex or False otherwise.
        """

        if isinstance(vertex, Vertex) and vertex.name not in self:
            self[vertex.name] = vertex
            return True
        if isinstance(vertex, str) and vertex not in self:
            self[vertex] = Vertex(vertex)
            return True
        if isinstance(vertex, int):
            vertex = str(vertex)
            if vertex not in self:
                self[vertex] = Vertex(vertex)
                return True
        return False

    def rm_vertex(self, vertex):
        """
        Removes the received vertex if it exists.

        :param vertex: The vertex to be removed
        """

        if not isinstance(vertex, str):
            vertex = str(vertex)

        if vertex in self:
            for v in self[vertex].neighbors:
                for u in self[v].neighbors:
                    if u == vertex and v != vertex:
                        self[v].neighbors.pop(vertex)
                        break
            self.pop(vertex)

    def add_edge(self, u, v=None, weight=math.inf):
        """
        Adds an edge between two vertices of the graph.
        Optionally adds a weight to the edge.
        :param u: The first vertex
        :param v: The second vertex
        :param weight: The weight of the edge, if not defined, assumes the infinite value.
        :return: True if it have succeeded in adding the edge or False otherwise.
        """

        edge_in, edge_out = ("in", "out") if self._is_directed else (None, None)

        if not v:
            v = u

        if u in self and v in self:
            for key, value in self.items():
                if key == u:
                    value.add_neighbor(v, weight, edge_out)
                if key == v:
                    value.add_neighbor(u, weight, edge_in)
            return True
        else:
            return False

    def rm_edge(self, v, u):
        """
        Removes the edge between the received vertices if it exists.
        :param v: The first vertex
        :param u: The second vertex
        """

        if u in self and v in self:
            if u in self[v].neighbors:
                self[v].neighbors.pop(u)
            if v in self[u].neighbors:
                self[u].neighbors.pop(v)

    def is_connected(self, found=set(), start=None):
        """
        Checks if the graph is connected.

        :param found: A set of vertices found.
        :param start: The initial vertex.
        :return: True if it's a connected graph or False otherwise
        """

        vertices = list(self.keys())
        vertices.sort()

        if not start:
            start = vertices[0]

        found.add(start)

        if len(found) != len(vertices):
            for v in self[start].get_neighbors():
                if v not in found:
                    if self.is_connected(found, v):
                        return True
        else:
            return True

        return False

    @property
    def min_degree(self):
        """
        Find the lowest degree of vertex in the graph.

        :return: The lowest degree
        """

        min_degree = math.inf

        for vertex in self:
            degree = self.degree(vertex)

            if degree < min_degree:
                min_degree = degree

        return min_degree

    @property
    def max_degree(self):
        """
        Find the greatest degree of vertex in the graph.

        :return: The greatest degree
        """

        max_degree = 0

        for vertex in self:
            degree = self.degree(vertex)

            if degree > max_degree:
                max_degree = degree
        return max_degree

    def degree(self, vertex):
        if isinstance(vertex, Vertex):
            vertex = vertex.name

        v_degree = 0

        for v in self:
            if not self._is_directed:
                for u in self[v].neighbors:
                    if u == vertex:
                        v_degree += len(self[v].neighbors[u])
            else:
                for u in self[v].in_neighbors:
                    if u == vertex:
                        v_degree += len(self[v].in_neighbors[u])
                for u in self[v].out_neighbors:
                    if u == vertex:
                        v_degree += len(self[v].out_neighbors[u])

        return v_degree

    @property
    def density(self):
        """
        Calculates the density of the graph using the formula:
        2 * edges / (vertices * (vertices - 1))

        :return: The graph density value
        """

        v = len(self.keys())
        e = len(list(self.edges))

        return 2.0 * e / (v * (v - 1))

    @property
    def diameter(self):
        """
        Calculate the diameter of a graph by finding the largest possible path
        between the two vertices in the graph.

        :return: The diameter of the graph
        """

        v = list(self.vertices)

        pairs = [(v[i], v[j]) for i in range(len(v) - 1) for j in range(i + 1, len(v))]

        smallest_paths = list()

        for (s, e) in pairs:
            paths = self.bfs_paths(s, e)
            smallest = sorted(paths, key=lambda x: len(x))
            if smallest:
                smallest_paths.append(smallest[0])

        smallest_paths.sort(key=len)

        diameter = len(smallest_paths[-1])

        return diameter

    @staticmethod
    def is_degree_sequence(sequence):
        """
        Given an undirected graph, a degree sequence is a monotonic non-increasing
        sequence of the vertex degrees (valencies) of its graph vertices.
        This method returns True if the graph has a degree sequence

        :param sequence: A graph's degree vertex sequence.
        :return: True if the graph has a degree sequence
        """
        return all(x >= y for x, y in zip(sequence, sequence[1:]))

    @staticmethod
    def erdoes_gallai(d_sequence):
        """
        Check if it's a simple graph
        The Erdös-Gallai Theorem states that the sequence (di) i = 1, ..., n being di >= di + 1
        is a degree sequence of a simple graph if the sum of the sequence is even and
        (d[i])i=1, ..., n <= k(k - 1) + min(d[i], k)i=k+1, .., n   for  k in {1, ..., n}

        :param d_sequence: The degree sequence of a graph
        :return: True if the Erdös-Gallai is fulfilled or False otherwise
        """

        if sum(d_sequence) % 2:
            # sum of sequence is odd
            return False

        if Graph.is_degree_sequence(d_sequence):
            for k in range(1, len(d_sequence) + 1):
                left = sum(d_sequence[:k])
                right = k * (k - 1) + sum([min(x, k) for x in d_sequence[k:]])
                if left > right:
                    return False
        else:
            # sequence is increasing
            return False
        return True

    @property
    def degree_sequence(self):
        """
        Returns the vertices' degree sequence of the graph.

        :return: Returns an inversely sorted tuple containing all the vertices degrees.
        """

        seq = list()

        for vertex in self:
            seq.append(self.degree(vertex))

        seq.sort(reverse=True)

        return tuple(seq)

    @property
    def isolated_vertices(self):
        """
        Finds all the isolated vertices.

        :return: The isolated vertices list or an empty list if there isn't any isolated vertex.
        """
        isolated = list()

        for v in self:
            if self._is_directed and not self[v].out_neighbors and not self[v].in_neighbors:
                isolated.append(v)
            elif not self._is_directed and not self[v].neighbors:
                isolated.append(v)

        return isolated

    @property
    def vertices(self):
        """
        Returns all the vertex in the graph.

        :return: An iterator over graph's vertices list.
        """

        return iter(self.keys())

    @property
    def edges(self):
        """
        Returns all the edges in the graph.

        :return: A generator that iterates over all the edges in the graph
        """

        # visited = list()

        for v in self:
            for neighbor in self[v].get_neighbors():
                # if {v, neighbor} not in visited:
                    # visited.append(set((v, neighbor)))    # Ignore duplicates
                    # yield(v, neighbor)
                yield (v, neighbor)

    @property
    def leaves(self):
        """
        Generates a list of leaf vertices
        """

        leaves_list = list()

        for v in self:
            if self.degree(v) == 1:
                yield v

    @property
    def sources(self):
        """
        Yields all source vertices (vertices with in-degree zero)
        """

        for v in self:
            if not self[v].in_neighbors:
                yield v

    @property
    def sinks(self):
        """
        Yields all sink vertices (vertices with out-degree zero)
        """

        for v in self:
            if not self[v].out_neighbors:
                yield v

    @property
    def pendants(self):

        for leaf in self.leaves:
            yield (
                self[leaf].name,
                [k for k in
                    self[leaf].out_neighbors.items() or
                    self[leaf].in_neighbors.items() or
                    self[leaf].get_neighbors()][0][0]
            )

    # Breadth-First Search Methods ==============================================

    def i_bfs(self, start, target):
        """
        Search for a given target vertex using the iterative Breadth-First Search algorithm.

        :param start: The initial vertex
        :param target: The target vertex to be find
        :return: True if the target vertex is found or False otherwise
        """

        queue, visited = [start], list()

        while queue:
            current = queue.pop(0)
            if current == target:
                return visited + [current]

            if current not in visited:
                queue.extend(set(self[current].get_neighbors()) - set(visited))
                visited += [current]
        return None

    def r_bfs(self, start, target, visited=None):
        """
        Search for target vertex demonstrating how to make a recursive Breadth-First Search

        :param start: The initial vertex
        :param target: The target vertex to be find
        :param visited: A list to control which vertices have been visited and do not need to be visited again.
        :return: A list containing all vertices that have been visited until the target is found or None otherwise.
        """

        if not visited:
            visited = list()

        if isinstance(start, list):
            visited.extend([v for v in start if v not in visited])
        else:
            visited.append(start)

        if target in visited:
            return visited

        if len(visited) >= len(self.keys()):
            return None

        same_level_nodes = list()

        for v in start:
            #  Does not consider neighbors that are vertices already visited.
            neighbors = [u for u in self[v].get_neighbors() if u not in visited]
            # List of vertices of the same level, without considering repeated vertices.
            same_level_nodes.extend([u for u in neighbors if u not in same_level_nodes])

        return self.r_bfs(same_level_nodes, target, visited)

    def bfs_paths(self, start, target):
        """
        Returns a generator for all possible paths between a vertex S and a vertex T.
        Uses the Breadth-First Search method with backtracking.

        :param start: The initial vertex
        :param target: The target vertex to be find
        """

        queue = [(start, [start])]

        while queue:
            current, path = queue.pop(0)
            if current == target:
                yield path
            for vertex in set(self[current].get_neighbors()) - set(path):
                queue.extend([(vertex, path + [vertex])])

    # Depth-First Search Methods ==============================================

    def i_dfs(self, start, target):
        """
        Search for a given target vertex using the iterative Depth-First Search algorithm.

        :param start: The initial vertex
        :param target: The target vertex to be find
        :return: True if the target vertex is found or False otherwise
        """
        stack, visited = [start], list()

        while stack:
            current = stack.pop()

            if current == target:
                return visited + [current]

            if current not in visited:
                visited.append(current)
                stack.extend(set(self[current].get_neighbors()) - set(visited))

        return None

    def r_dfs(self, start, target, visited=None):
        """
        Search for a given target vertex using the recursive Depth-First Search algorithm.

        :param start: The initial vertex
        :param target: The target vertex to be find
        :param visited: A list to control which vertices have been visited and do not need to be visited again.
        :return: A list containing all vertices that have been visited until the target is found or None otherwise.
        """

        if not visited:
            visited = list()

        visited.append(start)

        for v in set(self[start].get_neighbors()) - set(visited):

            if target not in visited:
                self.r_dfs(v, target, visited)

            if target in visited:
                return visited

        # return None

    def dfs_paths(self, start, target, visited=None):
        """
        Returns a generator for all possible paths between a vertex S and a vertex T.
        Uses the Depth-First Search method with backtracking.

        :param start: The initial vertex
        :param target: The target vertex to be find
        :param visited: A list to control which vertices have been visited and do not need to be visited again.
        """

        if not visited:
            visited = [start]

        if start == target:
            yield visited

        for vertex in [x for x in self[start].get_neighbors() if x not in visited]:
            for each_path in self.dfs_paths(vertex, target, visited + [vertex]):
                yield each_path  # return the paths generator

    # ----------------------------------------------------------

    def dijkstra(self, start, target, visited=None, distances=None, previous=None):
        """
        Dijkstra Algorithm implementation.
        Finds the shortest path between two vertices, uses weighted edges to represent the cost of each path for each
        vertex. If the graph is not valued (do not having no edge weight, the same as all edges having the same weight)
        then Dijkstra's Algorithm will consider the smallest path the path that has the least amount of edges.

        :param start: The initial vertex.
        :param target: The vertex to be reached from the initial vertex.
        :param visited: A list to control which vertices have been visited.
        :param distances: List used to store the minimum distances for each vertex.
        :param previous: The predecessor vertex of the current vertex.
        :return: The shortest path from start to target.
        """

        if not visited:
            visited = list()
        if not distances:
            distances = dict()
        if not previous:
            previous = dict()

        if start not in self:
            raise Exception('Start vertex cannot be found in the graph.')
        if target not in self:
            raise Exception('The target vertex does not exist in the graph.')
        if not self[target].get_neighbors():
            raise Exception('The target vertex is unreachable')

        if start == target:
            path = list()
            ant = target

            while ant:
                path.append(ant)
                ant = previous.get(ant, None)
            path.reverse()

            return 'Smaller path: {}\nWeight: {}'.format(path, distances[target])

        else:
            if not visited:
                distances[start] = 0
            for neighbor in self[start].get_neighbors():
                if neighbor not in visited:
                    new_distance = distances[start] + float(min(self[start].get_neighbors()[neighbor]))
                    if new_distance < distances.get(neighbor, math.inf):
                        distances[neighbor] = new_distance
                        previous[neighbor] = start

            visited.append(start)

            unvisited = {}

            for k in self:
                if k not in visited:
                    unvisited[k] = distances.get(k, math.inf)
            x = min(unvisited, key=unvisited.get)

            return self.dijkstra(x, target, visited, distances, previous)

    # TODO implementar:
    '''
    Indegree and outdegree                      OK
    Directed graph connectivity                 OK
    leaf vertex                                 OK
    simplical vertices
    pendant edge
    arestas paralelas                           OK
    grafo direcionado e não-direcionado         OK
    achar arestas incidentes a um vertice
    mostar vertices adjacentes a um vertice
    mostrar os adjacentes entre sí
    mostrar arestas adjacentes
    mostrar laços
    verificar se é simples, sem arestas paralelas e laços
    mostrar se é grafo completo
    qtd de grafos distintos
    se é grafo ciclo
    é um ciclo  OBS grafo ciclo é aquele que é um circulo e não um ciclico que é aquele que possui um ou mais ciclos
    é aciclico
    is wheel vertex
    se é grafo roda
    se é grafo cubo
    se é bipartido
    se é bipartido completo
    é grafo estrela?
    complemento de um grafo
    multigrafo
    pseudografo
    multigrafo dirigido
    hipergrafo
    valorado
    is regular graph?

    imersível
    subgrafo
    grafo regular
    matriz adjacencia dirigido e não dirigido
    matriz incidencia
    ver se são isomorfos
    árvores e arvores geradoras
    florestas \||||||
    homromorfos
    caminho euleriano e hamiltoniano
    V. J. Havel (1955) e S. L. Hakimi
(1961) e S. A. Choudum.

    '''


from typing import Tuple

import torch
import pandas

from deep_opt import NeuralNetwork

def get_collision_detection_dataset() -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
    test_data = pandas.read_csv("../resources/collision_detection/CollisionDetection_test_data.csv")
    training_data = pandas.read_csv("../resources/collision_detection/CollisionDetection_train_data.csv")

    train_inputs = torch.tensor(training_data.drop('class', axis=1).values, dtype=torch.float)
    train_targets = torch.tensor(training_data['class'].values, dtype=torch.long)
    test_inputs = torch.tensor(test_data.drop('class', axis=1).values, dtype=torch.float)
    test_targets = torch.tensor(test_data['class'].values, dtype=torch.long)

    return train_inputs, train_targets, test_inputs, test_targets

def get_collision_detection_network(name='ReLU_FFNN_1') -> NeuralNetwork:
    return torch.load(f"../resources/collision_detection/CollisionDetection_{name}.pyt")

def get_relu_ffnn_1_non_robust_samples_collision_detection(n=27):
    # take 25 samples that are not locally robust for repair
    # sample 17 is also not robust, but it is not classified correctly by the ReLU_FFNN_1 network
    # if a robustness specification with radius 0.05 is created around these samples, the specification
    # is contradiction free (although some input intervals do overlap, but the outputs for these overlapping
    # intervals agree)
    sample_indices = [
        4, 5, 6, 8, 10, 11, 13, 16, 18, 19, 20, 22, 23, 25, 27, 28, 30, 31, 33, 34,
        36, 37, 39, 42, 44, 47, 49
    ]
    assert n <= len(sample_indices), "Too few non robust samples known"
    return sample_indices[:n]


#!/usr/bin/env python3

"""
Module: main.py
Authors: Christian Bergler, Hendrik Schroeter
License: GNU General Public License v3.0
Institution: Friedrich-Alexander-University Erlangen-Nuremberg, Department of Computer Science, Pattern Recognition Lab
Last Access: 27.12.2022
"""

import os
import json
import math
import pathlib
import argparse
import platform
import utils.metrics as m

import torch
import torch.onnx
import torch.nn as nn
import torch.optim as optim

from data.audiodataset import (
    get_audio_files_from_dir,
    get_broken_audio_files,
    DatabaseCsvSplit,
    DefaultSpecDatasetOps,
    Dataset,
)

from trainer import Trainer
from utils.logging import Logger
from collections import OrderedDict
from models.residual_encoder import DefaultEncoderOpts
from models.residual_encoder import ResidualEncoder as Encoder
from models.classifier import Classifier, DefaultClassifierOpts

parser = argparse.ArgumentParser()

"""
Convert string to boolean.
"""
def str2bool(v):
    if v.lower() in ("yes", "true", "t", "y", "1"):
        return True
    elif v.lower() in ("no", "false", "f", "n", "0"):
        return False
    else:
        raise argparse.ArgumentTypeError("Boolean value expected.")

parser.add_argument(
    "-d",
    "--debug",
    dest="debug",
    action="store_true",
    help="Log additional training and model information.",
)

""" Directory parameters """
parser.add_argument(
    "--data_dir",
    type=str,
    help="The path to the dataset directory.",
)

parser.add_argument(
    "--cache_dir",
    type=str,
    default=None,
    help="The path to the dataset directory.",
)

parser.add_argument(
    "--model_dir",
    type=str,
    help="The directory where the model will be stored.",
)

parser.add_argument(
    "--checkpoint_dir",
    type=str,
    help="The directory where the checkpoints will be stored.",
)

parser.add_argument(
    "--log_dir", type=str, default=None, help="The directory to store the logs."
)

parser.add_argument(
    "--summary_dir",
    type=str,
    help="The directory to store the tensorboard summaries.",
)

parser.add_argument(
    "--noise_dir",
    type=str,
    default=None,
    help="Path to a directory with noise files used for data augmentation.",
)

""" Training parameters """
parser.add_argument(
    "--start_from_scratch",
    dest="start_scratch",
    action="store_true",
    help="Start taining from scratch, i.e. do not use checkpoint to restore.",
)

parser.add_argument(
    "--max_train_epochs", type=int, default=500, help="The number of epochs to train for the classifier."
)

parser.add_argument(
    "--jit_save",
    dest="jit_save",
    action="store_true",
    help="Save model via torch.jit save functionality.",
)

parser.add_argument(
    "--epochs_per_eval",
    type=int,
    default=2,
    help="The number of batches to run in between evaluations.",
)

parser.add_argument(
    "--batch_size", type=int, default=1, help="The number of images per batch."
)

parser.add_argument(
    "--num_workers", type=int, default=4, help="Number of workers used in data-loading"
)

parser.add_argument(
    "--no_cuda",
    dest="cuda",
    action="store_false",
    help="Do not use cuda to train model.",
)

parser.add_argument(
    "--lr",
    "--learning_rate",
    type=float,
    default=1e-5,
    help="Initial learning rate. Will get multiplied by the batch size.",
)

parser.add_argument(
    "--beta1", type=float, default=0.5, help="beta1 for the adam optimizer."
)

parser.add_argument(
    "--lr_patience_epochs",
    type=int,
    default=8,
    help="Decay the learning rate after N/epochs_per_eval epochs without any improvements on the validation set.",
)

parser.add_argument(
    "--lr_decay_factor",
    type=float,
    default=0.5,
    help="Decay factor to apply to the learning rate.",
)

parser.add_argument(
    "--early_stopping_patience_epochs",
    metavar="N",
    type=int,
    default=20,
    help="Early stopping (stop training) after N/epochs_per_eval epochs without any improvements on the validation set.",
)

""" Input parameters """
parser.add_argument(
    "--filter_broken_audio", action="store_true", help="Filter files which are below a minimum loudness of 1e-3 (float32)."
)

parser.add_argument(
    "--sequence_len", type=int, default=1280, help="Sequence length in ms."
)

parser.add_argument(
    "--freq_compression",
    type=str,
    default="linear",
    help="Frequency compression to reduce GPU memory usage. "
    "Options: `'linear'` (default), '`mel`', `'mfcc'`",
)

parser.add_argument(
    "--sr",
    type=int,
    default=44100,
    help="Sampling Rate in order to resample input data.",
)

parser.add_argument(
    "--fmin",
    type=int,
    default=500,
    help="Minimum frequency covered within the spectrogram (lower frequencies are ignored).",
)

parser.add_argument(
    "--fmax",
    type=int,
    default=10000,
    help="Maximum frequency covered within the spectrogram (higher frequencies are ignored).",
)

parser.add_argument(
    "--n_freq_bins",
    type=int,
    default=256,
    help="Number of frequency bins after compression.",
)

parser.add_argument(
    "--n_fft",
    type=int,
    default=4096,
    help="FFT size.")

parser.add_argument(
    "--hop_length",
    type=int,
    default=441,
    help="FFT hop length.")

parser.add_argument(
    "--augmentation",
    action="store_true",
    help="Whether to augment the input data. Validation and test data will not be augmented.",
)

""" Network parameters """
parser.add_argument(
    "--resnet", dest="resnet_size", type=int, default=18, help="ResNet size"
)

parser.add_argument(
    "--conv_kernel_size", nargs="*", type=int, help="Initial convolution kernel size."
)

parser.add_argument(
    "--num_classes", type=int, default=2, help="Number of classes needed to be classified."
)

parser.add_argument(
    "--min_max_norm",
    dest="min_max_norm",
    action="store_true",
    help="activates min-max normalization instead of default 0/1-dB-normalization.",
)

parser.add_argument(
    "--max_pool",
    type=int,
    default=None,
    help="Use max pooling after the initial convolution layer.",
)

ARGS = parser.parse_args()
ARGS.cuda = torch.cuda.is_available() and ARGS.cuda
ARGS.device = torch.device("cuda") if ARGS.cuda else torch.device("cpu")

if ARGS.conv_kernel_size is not None and len(ARGS.conv_kernel_size):
    ARGS.conv_kernel_size = ARGS.conv_kernel_size[0]

log = Logger("TRAIN", ARGS.debug, ARGS.log_dir)

"""
Get audio all audio files from the given data directory except they are broken.
"""
def get_audio_files():
    if input_data.can_load_from_csv():
        audio_files = list(get_audio_files_from_dir(ARGS.data_dir))
        log.info("Found csv files in {}".format(ARGS.data_dir))
    else:
        log.debug("Searching for audio files in {}".format(ARGS.data_dir))
        if ARGS.filter_broken_audio:
            data_dir_ = pathlib.Path(ARGS.data_dir)
            audio_files = get_audio_files_from_dir(ARGS.data_dir)
            log.debug("Moving possibly broken audio files to bkp:")
            broken_files = get_broken_audio_files(audio_files, ARGS.data_dir)
            for f in broken_files:
                log.debug(f)
                bkp_dir = data_dir_.joinpath(f).parent.joinpath("bkp")
                bkp_dir.mkdir(exist_ok=True)
                f = pathlib.Path(f)
                data_dir_.joinpath(f).rename(bkp_dir.joinpath(f.name))
        audio_files = list(get_audio_files_from_dir(ARGS.data_dir))
        log.info("Found {} audio files for training.".format(len(audio_files)))
        if len(audio_files) == 0:
            log.close()
            exit(1)
    return audio_files

"""
Save the trained model and corresponding options.
"""
def save_model(model, encoder, encoderOpts, classifier, classifierOpts, dataOpts, path, class_dist_dict, use_jit=False, min_max=False):
    model = model.cpu()
    encoder = encoder.cpu()
    classifier = classifier.cpu()
    encoder_state_dict = encoder.state_dict()
    classifier_state_dict = classifier.state_dict()

    save_dict = {
        "encoderOpts": encoderOpts,
        "classifierOpts": classifierOpts,
        "dataOpts": dataOpts,
        "encoderState": encoder_state_dict,
        "classifierState": classifier_state_dict,
        "classes": class_dist_dict
    }

    if min_max:
        norm = "{min_max : " + str(True) + ", min_db : "+str(None)+", ref_db : "+str(None)+"}"
    else:
        norm = "{min_max : " + str(False) + ", min_db : "+str(dataOpts["min_level_db"])+", ref_db : "+str(dataOpts["ref_level_db"])+"}"

    transform_dict = {
    "load_audio" : "{sr: "+str(dataOpts["sr"])+", mono : "+str(True)+"}",
    "pre_emph" : "{pre_empf_factor : "+str(dataOpts["preemphases"])+"}",
    "spectrogram" : "{fft : "+str(dataOpts["n_fft"])+", hop : "+str(dataOpts["hop_length"])+", center : "+str(False)+"}",
    "freq_compression" : "{type : "+str(dataOpts["freq_compression"])+", bins : "+str(dataOpts["n_freq_bins"])+", fmin : "+str(dataOpts["fmin"])+", fmax : "+str(dataOpts["fmax"])+"}",
    "amplitude_to_decibel" : "{min_db : "+str(dataOpts["min_level_db"])+"}",
    "normalize" : norm,
    "class_info" : "{num_class : "+str(len(class_dist_dict))+", name_class : "+str(class_dist_dict.__str__())+"}",
    "seg_size" : "{size_ms : "+str(ARGS.sequence_len)+"}"
    }

    if not os.path.isdir(ARGS.model_dir):
        os.makedirs(ARGS.model_dir)
    if use_jit:
        example = torch.rand(1, 1, 128, 256)
        extra_files = {}
        extra_files['dataOpts'] = dataOpts.__str__()
        extra_files['encoderOpts'] = encoderOpts.__str__()
        extra_files['classifierOpts'] = classifierOpts.__str__()
        extra_files['encoderState'] = encoder_state_dict.__str__()
        extra_files['classifierState'] = classifier_state_dict.__str__()
        extra_files['transforms'] = transform_dict.__str__()
        extra_files['classes'] = class_dist_dict.__str__()
        model = torch.jit.trace(model, example)
        torch.jit.save(model, path, _extra_files=extra_files)
        log.debug("Model successfully saved via torch jit: " + str(path))
    else:
        torch.save(save_dict, path)
        log.debug("Model successfully saved via torch save: " + str(path))

"""
Identifing all classes according to the file namings (class-label_id_year_tape_start_end).
"""
def get_classes(database):
    classes = set()
    for file in database:
        if platform.system() == "Windows":
            class_name = file.split("\\")[-1].split("-", 1)[0]
        else:
            class_name = file.split("/")[-1].split("-", 1)[0]
        classes.add(class_name)
    return sorted(list(classes))

"""
Main function to compute data preprocessing, network training, evaluation, and saving.
"""
if __name__ == "__main__":

    encoderOpts = DefaultEncoderOpts
    classifierOpts = DefaultClassifierOpts
    dataOpts = DefaultSpecDatasetOps

    for arg, value in vars(ARGS).items():
        if arg in encoderOpts and value is not None:
            encoderOpts[arg] = value
        if arg in classifierOpts and value is not None:
            classifierOpts[arg] = value
        if arg in dataOpts and value is not None:
            dataOpts[arg] = value

    ARGS.lr *= ARGS.batch_size

    patience_lr = math.ceil(ARGS.lr_patience_epochs / ARGS.epochs_per_eval)

    patience_lr = int(max(1, patience_lr))

    log.debug("dataOpts: " + json.dumps(dataOpts, indent=4))

    sequence_len = int(
        float(ARGS.sequence_len) / 1000 * dataOpts["sr"] / dataOpts["hop_length"]
    )
    log.debug("Number of spectrogram time-steps (input size = time-steps x frequency-bins) : {}".format(sequence_len))
    input_shape = (ARGS.batch_size, 1, dataOpts["n_freq_bins"], sequence_len)

    log.info("Setting up model")

    encoder = Encoder(encoderOpts)
    log.debug("Encoder: " + str(encoder))
    encoder_out_ch = 512 * encoder.block_type.expansion

    classifierOpts["input_channels"] = encoder_out_ch
    classifier = Classifier(classifierOpts)
    log.debug("Classifier: " + str(classifier))

    split_fracs = {"train": .7, "val": .15, "test": .15}
    input_data = DatabaseCsvSplit(
        split_fracs, working_dir=ARGS.data_dir, split_per_dir=True
    )

    audio_files = get_audio_files()

    if ARGS.noise_dir:
        noise_files = [str(p) for p in pathlib.Path(ARGS.noise_dir).glob("*.wav")]
    else:
        noise_files = []

    classes = get_classes(database=audio_files)

    if classifierOpts["num_classes"] == len(classes):
        log.info("Model predict " + str(len(classes)) + " classes")
    else:
        raise Exception("amount of automatically identified classes do not match amount of chosen classes!")

    datasets = {
        split: Dataset(
            file_names=input_data.load(split, audio_files),
            working_dir=ARGS.data_dir,
            cache_dir=ARGS.cache_dir,
            sr=dataOpts["sr"],
            n_fft=dataOpts["n_fft"],
            hop_length=dataOpts["hop_length"],
            n_freq_bins=dataOpts["n_freq_bins"],
            freq_compression=ARGS.freq_compression,
            f_min=dataOpts["fmin"],
            f_max=dataOpts["fmax"],
            seq_len=sequence_len,
            augmentation=ARGS.augmentation if split == "train" else False,
            noise_files=noise_files,
            dataset_name=split,
            classes=classes,
            min_max_normalize=ARGS.min_max_norm
        )
        for split in split_fracs.keys()
    }

    dataloaders = {
        split: torch.utils.data.DataLoader(
            datasets[split],
            batch_size=ARGS.batch_size,
            shuffle=True,
            num_workers=ARGS.num_workers,
            drop_last=False if split == "val" or split == "test" else True,
            pin_memory=True,
        )
        for split in split_fracs.keys()
    }

    model = nn.Sequential(
        OrderedDict([("encoder", encoder), ("classifier", classifier)])
    )

    if classifierOpts["num_classes"] == 2:
        prefix = "binary_classifier"
        metrics = {
            "tp": m.TruePositives(ARGS.device),
            "tn": m.TrueNegatives(ARGS.device),
            "fp": m.FalsePositives(ARGS.device),
            "fn": m.FalseNegatives(ARGS.device),
            "accuracy": m.Accuracy(ARGS.device),
            "f1": m.F1Score(ARGS.device),
            "precision": m.Precision(ARGS.device),
            "TPR": m.Recall(ARGS.device),
            "FPR": m.FPR(ARGS.device),
        }
    elif classifierOpts["num_classes"] > 2:
        prefix = "multi_class_" + str(classifierOpts["num_classes"]) + "_classifier"
        metrics = {
            "accuracy": m.Accuracy(ARGS.device)
        }
    else:
        raise Exception("not a valid number of classes")

    trainer = Trainer(
        model=model,
        logger=log,
        prefix=prefix,
        checkpoint_dir=ARGS.checkpoint_dir,
        summary_dir=ARGS.summary_dir,
        n_summaries=4,
        start_scratch=ARGS.start_scratch,
    )

    optimizer = optim.Adam(
        model.parameters(), lr=ARGS.lr, betas=(ARGS.beta1, 0.999)
    )

    metric_mode = "max"
    lr_scheduler = optim.lr_scheduler.ReduceLROnPlateau(
        optimizer,
        mode=metric_mode,
        patience=patience_lr,
        factor=ARGS.lr_decay_factor,
        threshold=1e-3,
        threshold_mode="abs",
    )

    model = trainer.fit(
        dataloaders["train"],
        dataloaders["val"],
        dataloaders["test"],
        loss_fn=nn.CrossEntropyLoss(),
        optimizer=optimizer,
        scheduler=lr_scheduler,
        n_epochs=ARGS.max_train_epochs,
        val_interval=ARGS.epochs_per_eval,
        patience_early_stopping=ARGS.early_stopping_patience_epochs,
        device=ARGS.device,
        metrics=metrics,
        val_metric="accuracy",
        val_metric_mode=metric_mode,
    )

    encoder = model.encoder

    classifier = model.classifier

    path = os.path.join(ARGS.model_dir, "ANIMAL-SPOT.pk")

    class_dist_dict = datasets["train"].class_dist_dict

    save_model(model, encoder, encoderOpts, classifier, classifierOpts, dataOpts, path, class_dist_dict, use_jit=ARGS.jit_save, min_max=ARGS.min_max_norm)

    log.close()


####################################################################################
#
# Module for cubed-sphere mesh generation and structuring.
#
# Based on "C. Ronchi, R. Iacono, P.S. Paolucci, The “Cubed Sphere”:
# A New Method for the Solution of Partial Differential Equations
# in Spherical Geometry."
#
# Luan da Fonseca Santos - January 2022
# (luan.santos@usp.br)
####################################################################################

import numpy as np
from constants import*
from cs_transform import*
from sphgeo import point, tg_vector_geodesic_pu_midpoints, tg_vector_geodesic_pv_midpoints
import time
import os.path
import netCDF4 as nc

####################################################################################
#  Global cubed-sphere grid structure
####################################################################################
class cubed_sphere:
    def __init__(self, N, transformation, showonscreen, gridload):
        # Panel indexes distribution
        #      +---+
        #      | 4 |
        #  +---+---+---+---+
        #  | 3 | 0 | 1 | 2 |
        #  +---+---+---+---+
        #      | 5 |
        #      +---+

        # Sphere radius
        #self.R = erad
        self.R = 1.0
        self.projection = transformation
        self.N = N
        self.name = self.projection+"_cs_"+str(N)
        gengrid = not gridload

        # Grid data netcdf filename
        self.netcdfdata_filename = griddir+self.name+".nc"
        if gridload:
            # Check if grid file exists
            if not os.path.isfile(self.netcdfdata_filename):
                if self.projection=='overlapped':
                    print('ERROR in cubed_sphere: couldnt find the grid file to be read: ', self.netcdfdata_filename)
                    exit()
                elif self.projection=='gnomonic_equidistant' or self.projection=='gnomonic_equiangular':
                    print('Couldnt find the grid file to be read: ', self.netcdfdata_filename)
                    gengrid=True
            else: # Read from netcdf file
                if showonscreen:
                    print("--------------------------------------------------------")
                    print('Loading grid file '+self.netcdfdata_filename)

                # Start time counting
                start_time = time.time()

                # Open grid data
                griddata = nc.Dataset(self.netcdfdata_filename,'r')

                # Grid spacing
                dx = griddata['dx'][:]
                dy = griddata['dy'][:]
                self.dx, self.dy = dx, dy

                self.N = griddata['N'][:]
                self.a = griddata['a'][:] # 1d grid is defined on [-a,a]

                # Get ghost cell numbers
                self.ng  = griddata['ng'][:]
                self.ngl = griddata['ngl'][:]
                self.ngr = griddata['ngr'][:]

                # Interior indexes
                self.i0, self.iend = griddata['i0'][:], griddata['iend'][:]
                self.j0, self.jend = griddata['j0'][:], griddata['jend'][:]
                i0, iend = self.i0, self.iend
                j0, jend = self.j0, self.jend
                ng = self.ng

                # Create points
                po    = point(N+1+ng, N+1+ng)
                pc    = point(N+ng, N+ng)
                pu    = point(N+1+ng, N+ng)
                pv    = point(N+ng, N+1+ng)
                ex_po = point(N+ng+1, N+ng+1)
                ey_po = point(N+ng+1, N+ng+1)
                ex_pc = point(N+ng, N+ng)
                ey_pc = point(N+ng, N+ng)
                ex_pu = point(N+1+ng, N+ng)
                ey_pu = point(N+1+ng, N+ng)
                ex_pv = point(N+ng, N+1+ng)
                ey_pv = point(N+ng, N+1+ng)

                # Get values from file
                po.X   = griddata['po'][:,:,:,0]
                po.Y   = griddata['po'][:,:,:,1]
                po.Z   = griddata['po'][:,:,:,2]
                po.lon = griddata['po'][:,:,:,3]
                po.lat = griddata['po'][:,:,:,4]

                pc.X   = griddata['pc'][:,:,:,0]
                pc.Y   = griddata['pc'][:,:,:,1]
                pc.Z   = griddata['pc'][:,:,:,2]
                pc.lon = griddata['pc'][:,:,:,3]
                pc.lat = griddata['pc'][:,:,:,4]

                pu.X   = griddata['pu'][:,:,:,0]
                pu.Y   = griddata['pu'][:,:,:,1]
                pu.Z   = griddata['pu'][:,:,:,2]
                pu.lon = griddata['pu'][:,:,:,3]
                pu.lat = griddata['pu'][:,:,:,4]

                pv.X   = griddata['pv'][:,:,:,0]
                pv.Y   = griddata['pv'][:,:,:,1]
                pv.Z   = griddata['pv'][:,:,:,2]
                pv.lon = griddata['pv'][:,:,:,3]
                pv.lat = griddata['pv'][:,:,:,4]

                ex_po.X   = griddata['ex_po'][:,:,:,0]
                ex_po.Y   = griddata['ex_po'][:,:,:,1]
                ex_po.Z   = griddata['ex_po'][:,:,:,2]
                ex_po.lon = griddata['ex_po'][:,:,:,3]
                ex_po.lat = griddata['ex_po'][:,:,:,4]

                ey_po.X   = griddata['ey_po'][:,:,:,0]
                ey_po.Y   = griddata['ey_po'][:,:,:,1]
                ey_po.Z   = griddata['ey_po'][:,:,:,2]
                ey_po.lon = griddata['ey_po'][:,:,:,3]
                ey_po.lat = griddata['ey_po'][:,:,:,4]

                ex_pc.X   = griddata['ex_pc'][:,:,:,0]
                ex_pc.Y   = griddata['ex_pc'][:,:,:,1]
                ex_pc.Z   = griddata['ex_pc'][:,:,:,2]
                ex_pc.lon = griddata['ex_pc'][:,:,:,3]
                ex_pc.lat = griddata['ex_pc'][:,:,:,4]

                ey_pc.X   = griddata['ey_pc'][:,:,:,0]
                ey_pc.Y   = griddata['ey_pc'][:,:,:,1]
                ey_pc.Z   = griddata['ey_pc'][:,:,:,2]
                ey_pc.lon = griddata['ey_pc'][:,:,:,3]
                ey_pc.lat = griddata['ey_pc'][:,:,:,4]

                ex_pu.X   = griddata['ex_pu'][:,:,:,0]
                ex_pu.Y   = griddata['ex_pu'][:,:,:,1]
                ex_pu.Z   = griddata['ex_pu'][:,:,:,2]
                ex_pu.lon = griddata['ex_pu'][:,:,:,3]
                ex_pu.lat = griddata['ex_pu'][:,:,:,4]

                ey_pu.X   = griddata['ey_pu'][:,:,:,0]
                ey_pu.Y   = griddata['ey_pu'][:,:,:,1]
                ey_pu.Z   = griddata['ey_pu'][:,:,:,2]
                ey_pu.lon = griddata['ey_pu'][:,:,:,3]
                ey_pu.lat = griddata['ey_pu'][:,:,:,4]

                ex_pv.X   = griddata['ex_pv'][:,:,:,0]
                ex_pv.Y   = griddata['ex_pv'][:,:,:,1]
                ex_pv.Z   = griddata['ex_pv'][:,:,:,2]
                ex_pv.lon = griddata['ex_pv'][:,:,:,3]
                ex_pv.lat = griddata['ex_pv'][:,:,:,4]

                ey_pv.X   = griddata['ey_pv'][:,:,:,0]
                ey_pv.Y   = griddata['ey_pv'][:,:,:,1]
                ey_pv.Z   = griddata['ey_pv'][:,:,:,2]
                ey_pv.lon = griddata['ey_pv'][:,:,:,3]
                ey_pv.lat = griddata['ey_pv'][:,:,:,4]

                # Attributes
                self.pc = pc
                self.po = po
                self.pu = pu
                self.pv = pv

                self.ex_po = ex_po
                self.ey_po = ey_po
                self.ex_pc = ex_pc
                self.ey_pc = ey_pc
                self.ex_pu = ex_pu
                self.ey_pu = ey_pu
                self.ex_pv = ex_pv
                self.ey_pv = ey_pv

                # Close netcdf file
                griddata.close()

                # Finish time counting
                elapsed_time = time.time() - start_time

                # Latlon/Contravariant conversion
                if showonscreen:
                    print("Done.\n")

        if gengrid:
            # Generate the grid
            if showonscreen:
                print("--------------------------------------------------------")
                print("Generating "+self.projection+" cubed-sphere with "+str(nbfaces*N*N)+" cells...")

            # 1D grid extremes
            if transformation == "gnomonic_equiangular":
                a = pio4
            elif transformation == "gnomonic_equidistant":
                a = self.R/np.sqrt(3.0)  # Half length of the cube
            elif transformation == "conformal":
                a = 1.0
            else:
                print("ERROR: invalid grid transformation.")
                exit()
            self.a = a
            x_min, x_max, y_min, y_max = [-a,a,-a,a]

            # Start time counting
            start_time = time.time()

            # Grid spacing
            dx = (x_max-x_min)/N
            dy = (y_max-y_min)/N
            self.dx, self.dy = dx, dy

            # Ghost cells size
            ngl = 4
            ngr = 4

            ng = ngr + ngl
            self.ngl  = ngl
            self.ngr = ngr
            self.ng = ng

            # Interior indexes
            i0, iend = ngl, ngl+N
            j0, jend = ngl, ngl+N
            self.i0, self.iend = ngl, ngl+N
            self.j0, self.jend = ngl, ngl+N

            # Generate cell po
            x_po = np.linspace(x_min-ngl*dx, x_max + ngr*dx, N+1+ng) # po
            y_po = np.linspace(y_min-ngl*dx, y_max + ngr*dy, N+1+ng) # po
            if showonscreen:
                print("Generating cell po...")

            po = point(N+1+ng, N+1+ng)
            ex_po = point(N+1+ng, N+1+ng)
            ey_po = point(N+1+ng, N+1+ng)

            if transformation == "gnomonic_equiangular":
                po.X, po.Y, po.Z, po.lon, po.lat = equiangular_gnomonic_map(x_po, y_po, N+1+ng, N+1+ng, self.R)
                ex_po.X, ex_po.Y, ex_po.Z = equiangular_tg_xdir(x_po, y_po, N+ng+1, N+ng+1, self.R)
                ey_po.X, ey_po.Y, ey_po.Z = equiangular_tg_ydir(x_po, y_po, N+ng+1, N+ng+1, self.R)

            elif transformation=="gnomonic_equidistant":
                po.X, po.Y, po.Z, po.lon, po.lat = equidistant_gnomonic_map(x_po, y_po, N+1+ng, N+1+ng, self.R)
                ex_po.X, ex_po.Y, ex_po.Z = equidistant_tg_xdir(x_po, y_po, N+ng+1, N+ng+1, self.R)
                ey_po.X, ey_po.Y, ey_po.Z = equidistant_tg_ydir(x_po, y_po, N+ng+1, N+ng+1, self.R)

            #elif transformation=="conformal":
            #    po.X[i0:iend+1,j0:jend+1], po.Y[i0:iend+1,j0:jend+1], po.Z[i0:iend+1,j0:jend+1], po.lon[i0:iend+1,j0:jend+1], po.lat[i0:iend+1,j0:jend+1] = conformal_map(x_po[i0:iend+1], y_po[j0:jend+1], N+1, N+1)

            self.po = po

            # Generate cell pc
            if showonscreen:
                print("Generating cell pc...")

            pc = point(N+ng, N+ng)
            ex_pc   = point(N+ng, N+ng)
            ey_pc   = point(N+ng, N+ng)

            x_pc = np.linspace(x_min+dx/2.0-ngl*dx, x_max-dx/2.0+ngr*dx, N+ng) # Centers
            y_pc = np.linspace(y_min+dy/2.0-ngl*dy, y_max-dy/2.0+ngr*dy, N+ng) # Centers

            if transformation == "gnomonic_equiangular":
                pc.X, pc.Y, pc.Z, pc.lon, pc.lat = equiangular_gnomonic_map(x_pc, y_pc, N+ng, N+ng, self.R)
                ex_pc.X, ex_pc.Y, ex_pc.Z = equiangular_tg_xdir(x_pc, y_pc, N+ng, N+ng, self.R)
                ey_pc.X, ey_pc.Y, ey_pc.Z = equiangular_tg_ydir(x_pc, y_pc, N+ng, N+ng, self.R)

            elif transformation=="gnomonic_equidistant":
                pc.X, pc.Y, pc.Z, pc.lon, pc.lat = equidistant_gnomonic_map(x_pc, y_pc, N+ng, N+ng, self.R)
                ex_pc.X, ex_pc.Y, ex_pc.Z = equidistant_tg_xdir(x_pc, y_pc, N+ng, N+ng, self.R)
                ey_pc.X, ey_pc.Y, ey_pc.Z = equidistant_tg_ydir(x_pc, y_pc, N+ng, N+ng, self.R)
            #elif transformation=="conformal":
            #    pc.X[i0:iend,j0:jend], pc.Y[i0:iend,j0:jend], pc.Z[i0:iend,j0:jend], pc.lon[i0:iend,j0:jend], pc.lat[i0:iend,j0:jend] \
            #    = conformal_map(x_pc[i0:iend], y_pc[j0:jend], N, N)

            self.pc = pc
            self.ex_pc = ex_pc
            self.ey_pc = ey_pc

            # Generate cell edges in x direction
            if showonscreen:
                print("Generating cell edges and tangent vectors in x direction...")

            pu    = point(N+1+ng, N+ng)
            ex_pu = point(N+1+ng, N+ng)
            ey_pu = point(N+1+ng, N+ng)
            x_pu = np.linspace(x_min-ngl*dx, x_max+ngr*dx, N+1+ng) # Edges
            y_pu = np.linspace(y_min+dy/2.0-ngl*dy, y_max-dy/2.0+ngr*dy, N+ng) # Centers

            if transformation == "gnomonic_equiangular":
                pu.X, pu.Y, pu.Z, pu.lon, pu.lat = equiangular_gnomonic_map(x_pu, y_pu, N+1+ng, N+ng, self.R)
                ex_pu.X, ex_pu.Y, ex_pu.Z = equiangular_tg_xdir(x_pu, y_pu, N+1+ng, N+ng, self.R)
                ey_pu.X, ey_pu.Y, ey_pu.Z = equiangular_tg_ydir(x_pu, y_pu, N+1+ng, N+ng, self.R)
            elif transformation=="gnomonic_equidistant":
                pu.X, pu.Y, pu.Z, pu.lon, pu.lat = equidistant_gnomonic_map(x_pu, y_pu, N+1+ng, N+ng, self.R)
                ex_pu.X, ex_pu.Y, ex_pu.Z = equidistant_tg_xdir(x_pu, y_pu, N+1+ng, N+ng, self.R)
                ey_pu.X, ey_pu.Y, ey_pu.Z = equidistant_tg_ydir(x_pu, y_pu, N+1+ng, N+ng, self.R)

            #elif transformation=="conformal":
            #    pu.X[i0:iend+1,j0:jend], pu.Y[i0:iend+1,j0:jend], pu.Z[i0:iend+1,j0:jend], pu.lon[i0:iend+1,j0:jend], pu.lat[i0:iend+1,j0:jend] \
            #    = conformal_map(x_pu[i0:iend+1], y_pu[j0:jend], N+1, N)

            self.pu = pu

            # Generate cell edges in y direction
            if showonscreen:
                print("Generating cell edges and tangent vectors in y direction...")

            pv = point(N+ng, N+ng+1)
            ex_pv = point(N+ng, N+ng+1)
            ey_pv = point(N+ng, N+ng+1)
            x_pv = np.linspace(x_min+dx/2.0-ngl*dx, x_max-dx/2.0+ngr*dx, N+ng) # Centers
            y_pv = np.linspace(y_min-ngl*dx, y_max + ngr*dy, N+1+ng) # Edges

            if transformation == "gnomonic_equiangular":
                pv.X, pv.Y, pv.Z, pv.lon, pv.lat = equiangular_gnomonic_map(x_pv, y_pv, N+ng, N+ng+1, self.R)
                ex_pv.X, ex_pv.Y, ex_pv.Z = equiangular_tg_xdir(x_pv, y_pv, N+ng, N+ng+1, self.R)
                ey_pv.X, ey_pv.Y, ey_pv.Z = equiangular_tg_ydir(x_pv, y_pv, N+ng, N+ng+1, self.R)
            elif transformation=="gnomonic_equidistant":
                pv.X, pv.Y, pv.Z, pv.lon, pv.lat = equidistant_gnomonic_map(x_pv, y_pv, N+ng, N+ng+1, self.R)
                ex_pv.X, ex_pv.Y, ex_pv.Z = equidistant_tg_xdir(x_pv, y_pv, N+ng, N+ng+1, self.R)
                ey_pv.X, ey_pv.Y, ey_pv.Z = equidistant_tg_ydir(x_pv, y_pv, N+ng, N+ng+1, self.R)
            #elif transformation=="conformal":
            #    pv.X[i0:iend,j0:jend+1], pv.Y[i0:iend,j0:jend+1], pv.Z[i0:iend,j0:jend+1], pv.lon[i0:iend,j0:jend+1], pv.lat[i0:iend,j0:jend+1] \
            #    = conformal_map(x_pv[i0:iend], y_pv[j0:jend+1], N, N+1)

            self.pv = pv

            # Generate tangent vectors
            if showonscreen:
                print("Generating latlon tangent vectors...")

            # Lat-lon tangent unit vectors
            elon_po = sphgeo.tangent_geo_lon(po.lon)
            elat_po = sphgeo.tangent_geo_lat(po.lon, po.lat)

            # CS map tangent unit vectors at edges points
            # latlon coordinates
            ex_po.lon = ex_po.X[:,:,:]*elon_po[:,:,:,0] + ex_po.Y[:,:,:]*elon_po[:,:,:,1] + ex_po.Z[:,:,:]*elon_po[:,:,:,2]
            ex_po.lat = ex_po.X[:,:,:]*elat_po[:,:,:,0] + ex_po.Y[:,:,:]*elat_po[:,:,:,1] + ex_po.Z[:,:,:]*elat_po[:,:,:,2]

            ey_po.lon = ey_po.X[:,:,:]*elon_po[:,:,:,0] + ey_po.Y[:,:,:]*elon_po[:,:,:,1] + ey_po.Z[:,:,:]*elon_po[:,:,:,2]
            ey_po.lat = ey_po.X[:,:,:]*elat_po[:,:,:,0] + ey_po.Y[:,:,:]*elat_po[:,:,:,1] + ey_po.Z[:,:,:]*elat_po[:,:,:,2]

            self.ex_po = ex_po
            self.ey_po = ey_po

            # Lat-lon tangent unit vectors
            elon_pc = sphgeo.tangent_geo_lon(pc.lon)
            elat_pc = sphgeo.tangent_geo_lat(pc.lon, pc.lat)

            # latlon coordinates
            ex_pc.lon = ex_pc.X[:,:,:]*elon_pc[:,:,:,0] + ex_pc.Y[:,:,:]*elon_pc[:,:,:,1] + ex_pc.Z[:,:,:]*elon_pc[:,:,:,2]
            ex_pc.lat = ex_pc.X[:,:,:]*elat_pc[:,:,:,0] + ex_pc.Y[:,:,:]*elat_pc[:,:,:,1] + ex_pc.Z[:,:,:]*elat_pc[:,:,:,2]

            ey_pc.lon = ey_pc.X[:,:,:]*elon_pc[:,:,:,0] + ey_pc.Y[:,:,:]*elon_pc[:,:,:,1] + ey_pc.Z[:,:,:]*elon_pc[:,:,:,2]
            ey_pc.lat = ey_pc.X[:,:,:]*elat_pc[:,:,:,0] + ey_pc.Y[:,:,:]*elat_pc[:,:,:,1] + ey_pc.Z[:,:,:]*elat_pc[:,:,:,2]

            self.ex_pc = ex_pc
            self.ey_pc = ey_pc

            # Lat-lon tangent unit vectors
            elon_pu = sphgeo.tangent_geo_lon(pu.lon)
            elat_pu = sphgeo.tangent_geo_lat(pu.lon, pu.lat)
            elon_pv = sphgeo.tangent_geo_lon(pv.lon)
            elat_pv = sphgeo.tangent_geo_lat(pv.lon, pv.lat)

            # CS map tangent unit vectors at edges points
            # latlon coordinates
            ex_pu.lon = ex_pu.X[:,:,:]*elon_pu[:,:,:,0] + ex_pu.Y[:,:,:]*elon_pu[:,:,:,1] + ex_pu.Z[:,:,:]*elon_pu[:,:,:,2]
            ex_pu.lat = ex_pu.X[:,:,:]*elat_pu[:,:,:,0] + ex_pu.Y[:,:,:]*elat_pu[:,:,:,1] + ex_pu.Z[:,:,:]*elat_pu[:,:,:,2]

            ex_pv.lon = ex_pv.X[:,:,:]*elon_pv[:,:,:,0] + ex_pv.Y[:,:,:]*elon_pv[:,:,:,1] + ex_pv.Z[:,:,:]*elon_pv[:,:,:,2]
            ex_pv.lat = ex_pv.X[:,:,:]*elat_pv[:,:,:,0] + ex_pv.Y[:,:,:]*elat_pv[:,:,:,1] + ex_pv.Z[:,:,:]*elat_pv[:,:,:,2]

            ey_pu.lon = ey_pu.X[:,:,:]*elon_pu[:,:,:,0] + ey_pu.Y[:,:,:]*elon_pu[:,:,:,1] + ey_pu.Z[:,:,:]*elon_pu[:,:,:,2]
            ey_pu.lat = ey_pu.X[:,:,:]*elat_pu[:,:,:,0] + ey_pu.Y[:,:,:]*elat_pu[:,:,:,1] + ey_pu.Z[:,:,:]*elat_pu[:,:,:,2]

            ey_pv.lon = ey_pv.X[:,:,:]*elon_pv[:,:,:,0] + ey_pv.Y[:,:,:]*elon_pv[:,:,:,1] + ey_pv.Z[:,:,:]*elon_pv[:,:,:,2]
            ey_pv.lat = ey_pv.X[:,:,:]*elat_pv[:,:,:,0] + ey_pv.Y[:,:,:]*elat_pv[:,:,:,1] + ey_pv.Z[:,:,:]*elat_pv[:,:,:,2]

            self.ex_pu = ex_pu
            self.ey_pu = ey_pu
            self.ex_pv = ex_pv
            self.ey_pv = ey_pv

        #-------------------------------------------------------------------------------
        # Latlon/Contravariant conversion
        if showonscreen:
            print("Computing metric tensor and latlon/contravariant conversion matrices... ")

        x_min, x_max, y_min, y_max = [-self.a, self.a, -self.a, self.a]
        ng, ngl, ngr = self.ng, self.ngl, self.ngr

        # Metric tensor on pc
        x_pc = np.linspace(x_min+dx/2.0-ngl*dx, x_max-dx/2.0+ngr*dx, N+ng) # Centers
        y_pc = np.linspace(y_min+dy/2.0-ngl*dy, y_max-dy/2.0+ngr*dy, N+ng) # Centers
        metric_tensor_pc = np.zeros((N+ng, N+ng, nbfaces))
        #metric_tensor_pc[:,:,0] = metric_tensor(x_pc, y_pc, self.R, transformation)
        metric_tensor_pc[:,:,0] = \
        -(ex_pc.X[:,:,0]*ey_pc.X[:,:,0]+ex_pc.Y[:,:,0]*ey_pc.Y[:,:,0]+ex_pc.Z[:,:,0]*ey_pc.Z[:,:,0])**2\
        +(ex_pc.X[:,:,0]**2 + ex_pc.Y[:,:,0]**2+ ex_pc.Z[:,:,0]**2)*\
         (ey_pc.X[:,:,0]**2 + ey_pc.Y[:,:,0]**2+ ey_pc.Z[:,:,0]**2)
        metric_tensor_pc[:,:,0] = np.sqrt(metric_tensor_pc[:,:,0])
        for p in range(1, nbfaces): metric_tensor_pc[:,:,p] = metric_tensor_pc[:,:,0]
        self.metric_tensor_pc = metric_tensor_pc

        # Metric tensor on edges in x direction
        x_pu = np.linspace(x_min-ngl*dx, x_max+ngr*dx, N+1+ng) # Edges
        y_pu = np.linspace(y_min+dy/2.0-ngl*dy, y_max-dy/2.0+ngr*dy, N+ng) # Centers
        metric_tensor_pu = np.zeros((N+1+ng, N+ng, nbfaces))
        #metric_tensor_pu[:,:,0] = metric_tensor(x_pu, y_pu, self.R, transformation)
        metric_tensor_pu[:,:,0] = \
        -(ex_pu.X[:,:,0]*ey_pu.X[:,:,0]+ex_pu.Y[:,:,0]*ey_pu.Y[:,:,0]+ex_pu.Z[:,:,0]*ey_pu.Z[:,:,0])**2\
        +(ex_pu.X[:,:,0]**2 + ex_pu.Y[:,:,0]**2+ ex_pu.Z[:,:,0]**2)*\
         (ey_pu.X[:,:,0]**2 + ey_pu.Y[:,:,0]**2+ ey_pu.Z[:,:,0]**2)
        metric_tensor_pu[:,:,0] = np.sqrt(metric_tensor_pu[:,:,0])
        for p in range(1, nbfaces): metric_tensor_pu[:,:,p] = metric_tensor_pu[:,:,0]
        self.metric_tensor_pu = metric_tensor_pu
        Xu, Yu = np.meshgrid(x_pu, y_pu,indexing='ij')

        # Metric tensor on edges in y direction
        x_pv = np.linspace(x_min+dx/2.0-ngl*dx, x_max-dx/2.0+ngr*dx, N+ng) # Centers
        y_pv = np.linspace(y_min-ngl*dx, y_max + ngr*dy, N+1+ng) # Edges
        Xv, Yv = np.meshgrid(x_pv, y_pv,indexing='ij')
        metric_tensor_pv = np.zeros((N+ng, N+ng+1, nbfaces))
        #metric_tensor_pv[:,:,0] = metric_tensor(x_pv, y_pv, self.R, transformation)
        metric_tensor_pv[:,:,0] = \
        -(ex_pv.X[:,:,0]*ey_pv.X[:,:,0]+ex_pv.Y[:,:,0]*ey_pv.Y[:,:,0]+ex_pv.Z[:,:,0]*ey_pv.Z[:,:,0])**2\
        +(ex_pv.X[:,:,0]**2 + ex_pv.Y[:,:,0]**2+ ex_pv.Z[:,:,0]**2)*\
         (ey_pv.X[:,:,0]**2 + ey_pv.Y[:,:,0]**2+ ey_pv.Z[:,:,0]**2)
        metric_tensor_pv[:,:,0] = np.sqrt(metric_tensor_pv[:,:,0])
        for p in range(1, nbfaces): metric_tensor_pv[:,:,p] = metric_tensor_pv[:,:,0]
        self.metric_tensor_pv = metric_tensor_pv

        # Lat-lon tangent unit vectors
        elon_pc = sphgeo.tangent_geo_lon(pc.lon)
        elat_pc = sphgeo.tangent_geo_lat(pc.lon, pc.lat)
        elon_pu = sphgeo.tangent_geo_lon(pu.lon)
        elat_pu = sphgeo.tangent_geo_lat(pu.lon, pu.lat)
        elon_pv = sphgeo.tangent_geo_lon(pv.lon)
        elat_pv = sphgeo.tangent_geo_lat(pv.lon, pv.lat)

        # Latlon/Contravariant conversion at pc
        prod_ex_elon_pc = ex_pc.X*elon_pc[:,:,:,0] + ex_pc.Y*elon_pc[:,:,:,1] + ex_pc.Z*elon_pc[:,:,:,2]
        prod_ex_elat_pc = ex_pc.X*elat_pc[:,:,:,0] + ex_pc.Y*elat_pc[:,:,:,1] + ex_pc.Z*elat_pc[:,:,:,2]
        prod_ey_elon_pc = ey_pc.X*elon_pc[:,:,:,0] + ey_pc.Y*elon_pc[:,:,:,1] + ey_pc.Z*elon_pc[:,:,:,2]
        prod_ey_elat_pc = ey_pc.X*elat_pc[:,:,:,0] + ey_pc.Y*elat_pc[:,:,:,1] + ey_pc.Z*elat_pc[:,:,:,2]
        determinant_ll2contra_pc = prod_ex_elon_pc*prod_ey_elat_pc - prod_ey_elon_pc*prod_ex_elat_pc

        self.prod_ex_elon_pc = prod_ex_elon_pc
        self.prod_ex_elat_pc = prod_ex_elat_pc
        self.prod_ey_elon_pc = prod_ey_elon_pc
        self.prod_ey_elat_pc = prod_ey_elat_pc
        self.determinant_ll2contra_pc = determinant_ll2contra_pc

        # Latlon/Contravariant conversion at pu
        prod_ex_elon_pu = ex_pu.X*elon_pu[:,:,:,0] + ex_pu.Y*elon_pu[:,:,:,1] + ex_pu.Z*elon_pu[:,:,:,2]
        prod_ex_elat_pu = ex_pu.X*elat_pu[:,:,:,0] + ex_pu.Y*elat_pu[:,:,:,1] + ex_pu.Z*elat_pu[:,:,:,2]
        prod_ey_elon_pu = ey_pu.X*elon_pu[:,:,:,0] + ey_pu.Y*elon_pu[:,:,:,1] + ey_pu.Z*elon_pu[:,:,:,2]
        prod_ey_elat_pu = ey_pu.X*elat_pu[:,:,:,0] + ey_pu.Y*elat_pu[:,:,:,1] + ey_pu.Z*elat_pu[:,:,:,2]
        determinant_ll2contra_pu = prod_ex_elon_pu*prod_ey_elat_pu - prod_ey_elon_pu*prod_ex_elat_pu

        self.prod_ex_elon_pu = prod_ex_elon_pu
        self.prod_ex_elat_pu = prod_ex_elat_pu
        self.prod_ey_elon_pu = prod_ey_elon_pu
        self.prod_ey_elat_pu = prod_ey_elat_pu
        self.determinant_ll2contra_pu = determinant_ll2contra_pu

        # Latlon/Contravariant conversion at pv
        prod_ex_elon_pv = ex_pv.X*elon_pv[:,:,:,0] + ex_pv.Y*elon_pv[:,:,:,1] + ex_pv.Z*elon_pv[:,:,:,2]
        prod_ex_elat_pv = ex_pv.X*elat_pv[:,:,:,0] + ex_pv.Y*elat_pv[:,:,:,1] + ex_pv.Z*elat_pv[:,:,:,2]
        prod_ey_elon_pv = ey_pv.X*elon_pv[:,:,:,0] + ey_pv.Y*elon_pv[:,:,:,1] + ey_pv.Z*elon_pv[:,:,:,2]
        prod_ey_elat_pv = ey_pv.X*elat_pv[:,:,:,0] + ey_pv.Y*elat_pv[:,:,:,1] + ey_pv.Z*elat_pv[:,:,:,2]
        determinant_ll2contra_pv = prod_ex_elon_pv*prod_ey_elat_pv - prod_ey_elon_pv*prod_ex_elat_pv

        self.prod_ex_elon_pv = prod_ex_elon_pv
        self.prod_ex_elat_pv = prod_ex_elat_pv
        self.prod_ey_elon_pv = prod_ey_elon_pv
        self.prod_ey_elat_pv = prod_ey_elat_pv
        self.determinant_ll2contra_pv = determinant_ll2contra_pv

        # Latlon/Contravariant conversion
        if showonscreen:
            print("Done.")

        #ex_pu, ey_pu
        #ex_pv, ey_pv
        #self.elon_pu, self.elat_pu
        #self.elon_pv, self.elat_pv

        Xu = np.repeat(Xu[:, :, np.newaxis], 6, axis=2)
        Yv = np.repeat(Yv[:, :, np.newaxis], 6, axis=2)
        self.Xu, self.Yv = Xu, Yv

        # Compute cell lenghts
        # Compute the geodesic distance of cell edges in x direction
        length_pu =  self.R*metric_tensor_pu*dy
        length_pv =  self.R*metric_tensor_pu*dx

        # Compute areas
        areas = (self.R**2)*metric_tensor_pc*dx*dy
        #print(abs(np.pi*4.0 - np.sum(self.areas[i0:iend,j0:jend,:]))/abs(np.pi*4.0) )
        #exit()

        # Finish time counting
        elapsed_time = time.time() - start_time

        if showonscreen:
            # Print some grid properties
            print("\nMin  edge length (km)  : ","{:.2e}".format(np.amin(length_pu[i0:iend+1,j0:jend,:])*erad/10**3))
            print("Max  edge length (km)  : ","{:.2e}".format(np.amax(length_pu[i0:iend+1,j0:jend,:])*erad/10**3))
            print("Mean edge length (km)  : ","{:.2e}".format(np.mean(length_pu[i0:iend+1,j0:jend,:])*erad/10**3))
            print("Ratio max/min length   : ","{:.2e}".format(np.amax(length_pu[i0:iend+1,j0:jend,:])/np.amin(length_pu[i0:iend+1,j0:jend,:])))

            print("Min  area (km2)        : ","{:.2e}".format(np.amin(areas[i0:iend,j0:jend,:])*erad*erad/10**6))
            print("Max  area (km2)        : ","{:.2e}".format(np.amax(areas[i0:iend,j0:jend,:])*erad*erad/10**6))
            print("Mean area (km2)        : ","{:.2e}".format(np.mean(areas[i0:iend,j0:jend,:])*erad*erad/10**6))
            print("Ratio max/min area     : ","{:.2e}".format(np.amax(areas[i0:iend,j0:jend,:])/np.amin(areas[i0:iend,j0:jend,:])))

            #m = N
            #ratio = (1+2*np.tan((pi/4))**2 * (1-1/m))**1.5
            #ratio = ratio*np.cos((pi/4))**4*(1-1/m)
            #print(ratio)
            #exit()

        if showonscreen:
            print("\nDone in ","{:.2e}".format(elapsed_time),"seconds.")
            print("--------------------------------------------------------\n")

####################################################################################
# Structure for scalar values on cubed-sphere grid
####################################################################################
class scalar_field:
    def __init__(self, grid, name, position):
        # Field name
        self.name = name

        # Field position
        self.position = position

        # Number of values in a coordinate axis (same for all panels)
        if position == "vertex":
            self.N = grid.N+1
        elif position == "center":
            self.N = grid.N
        else:
            print("ERROR in scalar_field: invalid position.")
            exit()

        # Field values along a coordinate axis (same for all panels)
        self.f = np.zeros((self.N, self.N, nbfaces))

####################################################################################
# Structure for lat-lon grid
####################################################################################
class latlon_grid:
    def __init__(self, Nlat, Nlon):
        self.Nlat, self.Nlon = Nlat, Nlon

        # Create a latlon grid
        lat = np.linspace(-pio2, pio2, Nlat)
        lon = np.linspace(  -pi,   pi, Nlon)
        self.lat, self.lon = np.meshgrid(lat, lon)

        # Convert to cartesian coordinates
        self.X, self.Y, self.Z = sphgeo.sph2cart(self.lon, self.lat)

        #Lat-lon to cubed sphere indexes
        self.ix, self.jy, self.mask = [], [], []

####################################################################################
# Compute execution time of the grid generation for different resolutions
####################################################################################
def cs_time_test():
    import time
    from matplotlib import pyplot as plt

    # Values of N
    Ns = (1,10,100,1000)

    # Number we generate the grid to compute the execution time
    Ntest = 10

    # Vector that save the execution time for each N
    times = np.zeros(len(Ns))

    # Calculate the average execution time using Ntest executions
    k = 0
    for N in Ns:
        for t in range(0,Ntest):
            start_time = time.time()
            grid = cubed_sphere(N)
            elapsed_time = time.time() - start_time
            times[k] = times[k] + elapsed_time/Ntest
            print(N,t,elapsed_time)
        k = k + 1
    print("N and average time (in seconds):")
    k = 0

    # Show the value of N and execution time on the screen
    for k in range(0,len(Ns)):
        print(Ns[k], times[k])

    print('\n')
    for k in range(1,len(Ns)):
        print(Ns[k], times[k]/times[k-1])

    # Plot the graph in logscale
    plt.figure()
    plt.loglog(Ns, times,marker='x')
    plt.Ylabel('Time (seconds)', fontsize=12)
    plt.Xlabel('$N$', fontsize=12)
    plt.savefig(graphdir+'execution_time'+'.eps', format='eps')
    plt.show()
    plt.close()

####################################################################################
#  Parabola class
####################################################################################
class ppm_parabola:
    def __init__(self, cs_grid, simulation, direction):
        # Number of cells
        N  = cs_grid.N
        M  = N
        ng = cs_grid.ng

        # reconstruction name
        self.recon_name = simulation.recon_name

        # parabola coefficients
        # Notation from Colella and  Woodward 1984
        # q(x) = q_L + z*(dq + q6*(1-z)) z in [0,1]
        self.q_L = np.zeros((N+ng, M+ng, nbfaces))
        self.q_R = np.zeros((N+ng, M+ng, nbfaces))
        self.dq  = np.zeros((N+ng, M+ng, nbfaces))
        self.q6  = np.zeros((N+ng, M+ng, nbfaces))

        if direction == 'x':
            # parabola fluxes
            self.f_L = np.zeros((N+ng+1, M+ng, nbfaces))  # flux from left
            self.f_R = np.zeros((N+ng+1, M+ng, nbfaces))  # flux from right
            self.f_upw = np.zeros((N+ng+1, M+ng, nbfaces)) # upwind flux

            # Extra variables for each scheme
            if simulation.recon_name == 'PPM-0' or simulation.recon_name == 'PPM-CW84' or simulation.recon_name == 'PPM-L04':
                self.Q_edges =  np.zeros((N+ng+1, M+ng, nbfaces))
        elif direction == 'y':
            # parabola fluxes
            self.f_L = np.zeros((N+ng, M+ng+1, nbfaces))   # flux from left
            self.f_R = np.zeros((N+ng, M+ng+1, nbfaces))   # flux from right
            self.f_upw = np.zeros((N+ng, M+ng+1, nbfaces)) # upwind flux

            # Extra variables for each scheme
            if simulation.recon_name == 'PPM-0' or simulation.recon_name == 'PPM-CW84' or simulation.recon_name == 'PPM-L04':
                self.Q_edges =  np.zeros((N+ng, M+ng+1, nbfaces))

        self.dF = np.zeros((N+ng, M+ng, nbfaces)) # div flux

        if simulation.recon_name == 'PPM-CW84':
            self.dQ  = np.zeros((N+ng, M+ng, nbfaces))
            self.dQ0 = np.zeros((N+ng, M+ng, nbfaces))
            self.dQ1 = np.zeros((N+ng, M+ng, nbfaces))
            self.dQ2 = np.zeros((N+ng, M+ng, nbfaces))

        if simulation.recon_name == 'PPM-L04':
            self.dQ      = np.zeros((N+ng, M+ng, nbfaces))
            self.dQ_min  = np.zeros((N+ng, M+ng, nbfaces))
            self.dQ_max  = np.zeros((N+ng, M+ng, nbfaces))
            self.dQ_mono = np.zeros((N+ng, M+ng, nbfaces))

####################################################################################
#  Velocity class
#  The quadrilateral points are labeled as below
#
#  po-------pv--------po
#  |                  |
#  |                  |
#  |                  |
#  pu       pc        pu
#  |                  |
#  |                  |
#  |                  |
#  po--------pv-------po
#
####################################################################################
class velocity:
    def __init__(self, cs_grid, pos):
        N = cs_grid.N
        ng = cs_grid.ng

        if pos == 'pu':
            self.ulon = np.zeros((N+1+ng, N+ng, nbfaces))
            self.vlat = np.zeros((N+1+ng, N+ng, nbfaces))
            self.ucontra = np.zeros((N+1+ng, N+ng, nbfaces))
            self.vcontra = np.zeros((N+1+ng, N+ng, nbfaces))
            self.ucontra_averaged = np.zeros((N+1+ng, N+ng, nbfaces)) # used for departure point
            self.ucontra_old      = np.zeros((N+1+ng, N+ng, nbfaces)) # used for departure point

            self.upos, self.uneg = None, None

        elif pos == 'pv':
            self.ulon = np.zeros((N+ng, N+1+ng, nbfaces))
            self.vlat = np.zeros((N+ng, N+1+ng, nbfaces))
            self.ucontra = np.zeros((N+ng, N+1+ng, nbfaces))
            self.vcontra = np.zeros((N+ng, N+1+ng, nbfaces))
            self.vcontra_averaged = np.zeros((N+ng, N+1+ng, nbfaces)) # used for departure point
            self.vcontra_old      = np.zeros((N+ng, N+1+ng, nbfaces)) # used for departure point
            self.vpos, self.vneg = None, None

        elif pos == 'pc': # Velocity at pc
            self.ulon = np.zeros((N+ng, N+ng, nbfaces))
            self.vlat = np.zeros((N+ng, N+ng, nbfaces))
            self.ucontra = np.zeros((N+ng, N+ng, nbfaces))
            self.vcontra = np.zeros((N+ng, N+ng, nbfaces))
            self.vcontra_averaged = np.zeros((N+ng, N+ng, nbfaces)) # used for departure point
            self.vcontra_old      = np.zeros((N+ng, N+ng, nbfaces)) # used for departure point
            self.upos, self.uneg = None, None
            self.vpos, self.vneg = None, None
        else:
            print('ERROR in  velocity class: invalid position, ', pos)


import pytest
import pytest_asyncio
from databases import Database
from fastapi import FastAPI
from httpx import AsyncClient

from app.apis.genre.crud import fn_create_genre, fn_get_genre_by_imdb
from app.apis.movie.crud import fn_create_movie
from app.db.repositories import GenresRepository, MoviesRepository

from app.models.domains.genre import NewGenre, Genre
from app.models.domains.movie import NewMovie
from app.tests.helpers import fixed_data

pytestmark = pytest.mark.asyncio

FUNCTION_TO_TEST = fn_get_genre_by_imdb

@pytest_asyncio.fixture(scope="class")
def new_genre() -> NewGenre:
    return fixed_data.new_genre()

@pytest_asyncio.fixture(scope="class")
def new_movie() -> NewMovie:
    return fixed_data.new_movie()

@pytest_asyncio.fixture(scope="class")
async def setup(
    app: FastAPI,
    platform_client: AsyncClient,
    genres_repo: GenresRepository,
    new_genre: NewGenre,
    new_movie: NewMovie,
    movies_repo: MoviesRepository,
    db: Database,
):
    async def do_teardown():
        await db.fetch_one("TRUNCATE TABLE genres CASCADE")
        await db.fetch_one("TRUNCATE TABLE movies CASCADE")

    async def do_setup():
        _ = await fn_create_movie(
            new_movie,
            movies_repo,
        )
        movie_genre = await fn_create_genre(
            new_genre,
            genres_repo,
        )
        return {
            "new_genre": new_genre,
            "imdb_id": movie_genre.imdb_id,
            }

    await do_teardown()
    yield await do_setup()
    await do_teardown()

class TestSearchGenre:
    async def test_setup(self, setup):
        new_genre = setup["new_genre"]
        imdb_id = setup["imdb_id"]

        assert isinstance(new_genre, NewGenre)
        assert isinstance(imdb_id, str)

    async def test_search_genre(
        self,
        app: FastAPI,
        platform_client: AsyncClient,
        genres_repo: GenresRepository,
        setup
    ) -> None:
        new_genre = setup["new_genre"]
        imdb_id = setup["imdb_id"]

        test_results = await FUNCTION_TO_TEST(
            imdb_id,
            genres_repo,
        )

        assert len(test_results) == 1
        assert isinstance(test_results[0], Genre)
        assert test_results[0].imdb_id is not None
        assert test_results[0].genre == new_genre.genre