import torch
from torchtext import datasets
from torchtext import data
import spacy
import torch.nn.functional as F
import torch.nn as nn
import random
import torch.optim as optim
SEED = 1234
torch.manual_seed(SEED)
torch.cuda.manual_seed(SEED)
TEXT = data.Field(tokenize = 'spacy')
LABEL = data.LabelField(tensor_type= torch.FloatTensor)
train, test = datasets.IMDB.splits(TEXT, LABEL)
train, valid = train.split(random_state = random.seed(SEED))
TEXT.build_vocab(train, max_size = 25000, vectors = "glove.6B.100d")
LABEL.build_vocab(train)
BATCH_SIZE = 64
train_itr, valid_itr, test_itr = data.BucketIterator.splits((train,test,valid), batch_size = BATCH_SIZE, sort_key = lambda x:len(x.text), repeat = False)
class RNN(nn.Module):
  def __init__(self, vocab_size, embedding_dim, hidden_dim, output_dim, n_layers, bidirectional, dropout):
    super().__init__()
    self.embedding = nn.Embedding(vocab_size, embedding_dim)
    self.rnn = nn.LSTM(embedding_dim, hidden_dim, num_layers = n_layers, bidirectional = bidirectional, dropout = dropout)
    self.fc = nn.Linear(hidden_dim *2, output_dim)
    self.dropout = nn.Dropout(dropout)
  def forward(self, x):
    embedded = self.dropout(self.embedding(x))
    output, (hidden, cell) = self.rnn(embedded)
    hidden = self.dropout(torch.cat((hidden[-2,:,:], hidden[-1,:,:]), dim=1))
    return self.fc(hidden.squeeze(0))
INPUT_DIM = len(TEXT.vocab)
EMBEDDING_DIM = 100
HIDDEN_DIM = 256
OUTPUT_DIM = 1
N_LAYERS = 2
BIDIRECTIONAL = True
DROPOUT = 0.5
model = RNN(INPUT_DIM, EMBEDDING_DIM, HIDDEN_DIM, OUTPUT_DIM, N_LAYERS, BIDIRECTIONAL, DROPOUT)
pretrained_embedding = TEXT.vocab.vectors
optimizer = optim.Adam(model.parameters())
criterion = nn.BCEWithLogitsLoss()
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
model = model.to(device)
criterion = criterion.to(device)
def binary_accuracy(preds, y):
  rounds_preds = torch.round(F.sigmoid(preds))
  correct = (rounds_preds == y).float()
  acc = correct.sum()/len(correct)
  return acc
def train(model, iterator, optimizer, criterion):
  epoch_loss = 0
  epoch_acc = 0
  model.train()
  for batch in iterator:
    optimizer.zero_grad()
    predictions = model(batch.text).squeeze(1)
    loss = criterion(predictions, batch.label)
    acc = binary_accuracy(predictions, batch.label)
    loss.backward()
    optimizer.step()
    epoch_loss += loss.item()
    epoch_acc += acc.item()
  return epoch_loss/len(iterator), epoch_acc/len(iterator)

def evaluate(model, iterator, criterion):
    epoch_loss = 0
    epoch_acc = 0
    model.eval()
    with torch.no_grad():
       for batch in iterator:
            predictions = model(batch.text).squeeze(1)
            loss = criterion(predictions, batch.label)
            acc = binary_accuracy(predictions, batch.label)
            epoch_loss += loss.item()
            epoch_acc += acc.item()
    return epoch_loss / len(iterator), epoch_acc / len(iterator)
NLP = spacy.load('en')
def predict_sentiment(sentence):
  tokenized = [tok.text for tok in NLP.tokenizer(sentence)]
  indexed = [TEXT.vocab.stoi[t] for t in tokenized]
  tensor = torch.LongTensor(indexed).to(device)
  tensor = tensor.unsqueeze(1)
  predection = F.sigmoid(model(tensor))
  return predection.item()


###########################################################################################
# ATTENTION CE MODÈLE EST CHRONOPHAGE, IL EST CONSERVÉ PAR BUT PÉDAGOGIQUE UNIQUEMENT     #
###########################################################################################

"""

Ce modèle réalisé sous tensorflow version 1 n'est pas du tout optimisé
il prend des heures (littéralement!) possibilité de jouer sur la batch size
le problèmes vient probablement d'un manipulation trop conséquentes des entrées
vecteurs de dimension vocab*1 ==> alors que ce sont des vecteurs creux 1 seul coordonnées non nulles
conséquences on se trainent pleins de coéfficients inutiles dans la matrice

Solution
utiliser l'implémentation keras fournit
exploiter l'implémentation dans gensim
"""
from nltk.corpus import stopwords
import stanza
from sklearn.feature_extraction.text import CountVectorizer,TfidfTransformer
import numpy as np
import tensorflow as tf
import pandas as pd
import class_xml
from scipy.sparse import csr_matrix
import random

""" import le module class_texte """
""" Recuperer le liste_docment_traites"""
""" Récupérer le vocabulaire"""

"""Plongement de mots """
for doc in liste_docments_traités:
    for item in doc:
        #print(item)
        object_texte=item[0]
        #print(object_texte)
        object_texte.__one_hot__()
        #print(object_texte.oh)
liste_oh=[txt.oh for doc in liste_docments_traités for (txt,nature,ID) in doc]

# en clair le one hot c'est utilsier le dico test_2 pour remplacer les listes de string par une liste de nombres

#On pourrait stocker ces informations dans le type de texte mais le vocabulaire pose problème
# En effet le vocabulaire est une variable de la classe de Texte
# Cette variable est issue de l'analyse de l'ensemble des texte Cf la fonction vocabulary()

## après il va falloir faire des regroupements
## aka générer des couples de nombre entrées sorties
## possibilité de faire du slicing

"""générer des clouples de données """
Taille_fenêtre=5
liste_de_couple=[]
for each_doc in liste_oh:
    for indice,wordx in enumerate(each_doc):
        for voisin in each_doc[max(indice-Taille_fenêtre,0): min(indice+Taille_fenêtre,len(each_doc)+1)]:
            if voisin!=wordx:
                liste_de_couple.append([wordx,voisin])

import tensorflow as tf
tf.compat.v1.disable_v2_behavior()
Dim_voc=len(set(test_2))
X,Y=[],[]
# X entrée
# Y sortie
def from_numbers_to_array(indice):
    ans=np.zeros(Dim_voc)
    ans[indice]=1
    return ans
    pass
for item  in  liste_de_couple:
    X.append(from_numbers_to_array(item[0]))
    Y.append(from_numbers_to_array(item[1]))
X_train=np.asarray(X)
Y_train=np.asarray(Y)
# window_size=5
# dim_plongement=2
# long_oh=[len(i) for i in liste_oh]
# vocab_size=len(test_2)
# max_longueur=max(long_oh)
# padded_docs=tf.keras.preprocessing.sequence.pad_sequences(liste_oh, maxlen=max_longueur,padding="post")
# sampling_table = tf.keras.preprocessing.sequence.make_sampling_table(vocab_size)
# plt_mot=tf.keras.layers.Embedding(vocab_size,dim_plongement,name='plt_mot')
# # inputs
# w_inputs = tf.keras.Input(shape=(1,),dtype='int32')
# w=plt_mot(w_inputs)
# # context

# c_inputs= tf.keras.Input(shape=(1,),dtype='int32')
# c=plt_mot(c_inputs)
# o=tf.keras.layers.Dot(axes=2)([w,c])
# o=tf.keras.layers.Reshape((1,),input_shape=(1,1))(o)
# o=tf.keras.layers.Activation("sigmoid")(o)
# SkipGram=tf.keras.Model(inputs=[w_inputs,c_inputs],outputs=o)
# SkipGram.summary()
# SkipGram.compile(loss='binary_crossentropy',optimizer='adam')
# from datetime import datetime
# now=datetime.utcnow().strftime("%Y%m%d%H%M%S")
# root_logdir="tf_logs"
# logdir="{}/run-{}/".format(root_logdir,now)
# tensorboard_callback=tf.compat.v1.keras.callbacks.TensorBoard(log_dir=logdir,write_graph=True)
# #writer=tf.summary.FileWriter(logdir,graph=)
# X=[]
# Y=[]
# for _ in range(5):
#     for item in liste_oh:
#         couples,labels=tf.keras.preprocessing.sequence.skipgrams(item, vocab_size, window_size,sampling_table=sampling_table)
#         for couple in couples:
#             X.append(couple)
#         for label in labels:
#             Y.append(label)
# X_new=[np.array(x) for x in zip(*X)]
# SkipGram.fit(X_new,Y,batch_size=10,epochs=5,verbose=1,
#              callbacks=[tensorboard_callback]
# )
# matrice_plgt=SkipGram.get_layer("plt_mot").get_weights()[0]
# tfidf=TfidfTransformer()
# matrice_tfidf=tfidf.fit_transform(Matrice_term_doc)
# matrice_BM25F=BM_25F(Matrice_term_doc)
# liste_modèle=[matrice_tfidf,matrice_BM25F,matrice_plgt]
# with open("modele_mat.npy",'wb') as f:
#     for mat in liste_modèle:
#         np.save(f,mat)
# import pickle

# pickle.dump(test_2,open("dico_mot_index.p", 'wb'))
# pickle.dump(test_dico, open("dico_index_mot.p",'wb'))
# pickle.dump(liste_id,open("liste_id_doc.p", 'wb'))
# pickle.dump(liste_docments_traités,open("docment_traites.p",'wb'))
# pickle.dump(liste_doc_xml,open("liste_doc_xml.p","wb"))
# pickle.dump(stop_words,open("stop_words.p","wb"))
# pickle.dump(p,open("reg_exp.p","wb"))
x=tf.compat.v1.placeholder(tf.float32,shape=(None,Dim_voc),name="entree")

y_label=tf.compat.v1.placeholder(tf.float32,shape=(None,Dim_voc), name="sortie")

""" -* IMPORTANT CHOIX DE LA DIMENSION DU PLONGEMENT *- """
""" finalement c'est la même que du clustering il va falloir la reduction de dimensionalité """

DIM_plongement=2
""" La litterature recommande un nombre dimension entre 100 et 1000"""
# FIRST HIDDEN LAYER
with tf.compat.v1.variable_scope("premiere_couche"):
    W1=tf.Variable(tf.random.normal([Dim_voc,DIM_plongement]),name="poids")
    b1=tf.Variable(tf.random.normal([1]),name="biais") # Bias
    Couche_cachee=tf.add(tf.matmul(x,W1),b1,name="neurone_lineaire")

""" SORTIE """

with tf.compat.v1.variable_scope("Softmax"):
    W2 = tf.Variable(tf.random.normal([DIM_plongement,Dim_voc]),name="poids")
    b2 = tf.Variable(tf.random.normal([1]),name="biais")
    logit=tf.add(tf.matmul(Couche_cachee,W2),b2, name="neurone_lineaire")
    pred=tf.compat.v1.nn.softmax(logit,name="softmax")

fonction_objectif=tf.math.reduce_mean(
#    tf.compat.v1.losses.softmax_cross_entropy(labels=y_label,logits=logit))
    -tf.reduce_sum(y_label*tf.math.log(pred),axis=[1]))
#

optimizer=tf.compat.v1.train.MomentumOptimizer(0.05,0.9, use_nesterov=True)
train_op=optimizer.minimize(fonction_objectif)
from datetime import datetime
now=datetime.utcnow().strftime("%Y%m%d%H%M%S")
root_logdir="tf_logs"
logdir="{}/run-{}/".format(root_logdir,now)

summary_loss=tf.compat.v1.summary.scalar("Xentropy",fonction_objectif)
summary_writer=tf.compat.v1.summary.FileWriter(logdir,tf.compat.v1.get_default_graph())

# TRAINING -- ENTRAÎNEMENT
sess=tf.compat.v1.Session()
saver=tf.compat.v1.train.Saver()
init=tf.compat.v1.global_variables_initializer()
sess.run(init)
iteration = 1000 # 50 000
for i in range(iteration):
    sess.run(train_op, feed_dict={x: X_train, y_label:Y_train})
    if i% 100 == 0:
        print(str(i/iteration*100)+"%: iteration n°"+str(i)+" fonction objectif", sess.run(fonction_objectif, feed_dict={x:X_train,y_label:Y_train}))
        summary_str=summary_loss.eval(session=sess,feed_dict={x:X_train,y_label:Y_train})
        summary_writer.add_summary(summary_str,i)
    if i% 1000 ==0:
        save_path=saver.save(sess, "./modelV3.ckpt")
save_path=saver.save(sess, "./final_modelV3.ckpt")
summary_writer.close()
repz_2D=sess.run(W1+b1)
#print(type(repz_2D))
# import pickle
# pickle.dump(repz_2D,open('vecteur_mots.p',"wb"))


import os

import cv2
import dlib
import numpy as np

def teste():
    detector = dlib.get_frontal_face_detector()
    detectorPontos = dlib.shape_predictor("res/shape_predictor_68_face_landmarks.dat")
    reconhecimentoFacial = dlib.face_recognition_model_v1("res/dlib_face_recognition_resnet_model_v1.dat")

    indices = np.load("res/indices.pickle", allow_pickle=True)
    descritoresFaciais = np.load("res/descritores.npy", allow_pickle=True)

    limiar = 0.45
    cap = cv2.VideoCapture(0)
    nome = ''
    while True:

        ret_val, frame = cap.read()
        rgb_image = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
        dets = detector(rgb_image)
        for det in dets:
            e, t, d, b = (int(det.left()), int(det.top()), int(det.right()), int(det.bottom()))
            cv2.rectangle(frame,(det.left(), det.top()), (det.right(), det.bottom()), (0, 255, 0), 2)
            pontos = detectorPontos(frame, det)
            descritorFacial = reconhecimentoFacial.compute_face_descriptor(frame, pontos)
            listaDescritorFacial = [fd for fd in descritorFacial]
            npArrayDescritorFacial = np.asarray(listaDescritorFacial, dtype=np.float64)
            npArrayDescritorFacial = npArrayDescritorFacial[np.newaxis, :]

            distancias = np.linalg.norm(npArrayDescritorFacial - descritoresFaciais, axis=1) # Calculo das distancias euclidianas

            minimo = np.argmin(distancias)
            distanciaMinima = distancias[minimo-1]
            if distanciaMinima <= limiar:
                nome = os.path.split(indices[minimo-1])[1].split(".")[0]
                # e, t, d, b = (int(det.left()), int(det.top()), int(det.right()), int(det.bottom()))
                cv2.rectangle(frame,(det.left(), det.top()), (det.right(), det.bottom()), (0, 255, 0), 2)
                texto = f"{nome} {distanciaMinima:.4f}"
                cv2.putText(frame, texto, (d, b), cv2.FONT_HERSHEY_COMPLEX_SMALL, 1, (0, 0, 0), 2)
                cv2.putText(frame, texto, (d, b), cv2.FONT_HERSHEY_COMPLEX_SMALL, 1, (0, 255, 0), 1)
                print(f'RECONHECIDO')
                print(nome)

            else:
                nome = "Nao registrado"

                print(f'NAO RECONHECIDO' )
                print(nome)
                # e, t, d, b = (int(det.left()), int(det.top()), int(det.right()), int(det.bottom()))
                cv2.rectangle(frame,(det.left(), det.top()), (det.right(), det.bottom()), (0, 0, 255), 2)

                texto = f"{nome} {distanciaMinima:.4f}"
                cv2.putText(frame, texto, (d, b), cv2.FONT_HERSHEY_COMPLEX_SMALL, 1, (0, 0, 0), 2)
                cv2.putText(frame, texto, (d, b), cv2.FONT_HERSHEY_COMPLEX_SMALL, 1, (0, 0, 255), 1)
        cv2.imshow('Cam', frame)

        if cv2.waitKey(1) & 0xFF == ord('q'):
            break
    cap.release()
    cv2.destroyAllWindows()


import logging
import math
from typing import List

import torch
import torch.nn as nn
import torch.nn.init as init
from reagent.models.base import ModelBase

logger = logging.getLogger(__name__)

def gaussian_fill_w_gain(tensor, activation, dim_in, min_std=0.0) -> None:
    """ Gaussian initialization with gain."""
    gain = math.sqrt(2) if (activation == "relu" or activation == "leaky_relu") else 1
    init.normal_(tensor, mean=0, std=max(gain * math.sqrt(1 / dim_in), min_std))

ACTIVATION_MAP = {
    "tanh": nn.Tanh,
    "relu": nn.ReLU,
    "leaky_relu": nn.LeakyReLU,
    "linear": nn.Identity,
}

class FullyConnectedNetwork(ModelBase):
    def __init__(
        self,
        layers,
        activations,
        *,
        use_batch_norm=False,
        min_std=0.0,
        dropout_ratio=0.0,
        use_layer_norm=False,
        normalize_output=False,
    ) -> None:
        super().__init__()

        self.input_dim = layers[0]

        modules: List[nn.Module] = []

        assert len(layers) == len(activations) + 1

        for i, ((in_dim, out_dim), activation) in enumerate(
            zip(zip(layers, layers[1:]), activations)
        ):
            # Add BatchNorm1d
            if use_batch_norm:
                modules.append(nn.BatchNorm1d(in_dim))
            # Add Linear
            linear = nn.Linear(in_dim, out_dim)
            gaussian_fill_w_gain(linear.weight, activation, in_dim, min_std=min_std)
            init.constant_(linear.bias, 0)  # type: ignore
            modules.append(linear)
            # Add LayerNorm
            if use_layer_norm and (normalize_output or i < len(activations) - 1):
                modules.append(
                    nn.LayerNorm(out_dim)  # type: ignore
                )
            # Add activation
            if activation in ACTIVATION_MAP:
                modules.append(ACTIVATION_MAP[activation]())
            else:
                # See if it matches any of the nn modules
                modules.append(getattr(nn, activation)())
            # Add Dropout
            if dropout_ratio > 0.0 and (normalize_output or i < len(activations) - 1):
                modules.append(nn.Dropout(p=dropout_ratio))

        self.dnn = nn.Sequential(*modules)  # type: ignore

    def input_prototype(self):
        return torch.randn(1, self.input_dim)

    def forward(self, input: torch.Tensor) -> torch.Tensor:
        """ Forward pass for generic feed-forward DNNs. Assumes activation names
        are valid pytorch activation names.
        :param input tensor
        """
        return self.dnn(input)


import logging
from typing import Dict, List, Optional, Tuple

import reagent.types as rlt
import torch
from reagent.models.base import ModelBase
from reagent.models.seq2slate import Seq2SlateMode, Seq2SlateTransformerNet
from reagent.models.seq2slate_reward import Seq2SlateRewardNetBase
from reagent.preprocessing.postprocessor import Postprocessor
from reagent.preprocessing.preprocessor import Preprocessor
from reagent.preprocessing.sparse_preprocessor import (
    SparsePreprocessor,
    make_sparse_preprocessor,
)
from reagent.torch_utils import gather
from reagent.training.utils import gen_permutations
from torch import nn

logger = logging.getLogger(__name__)

def serving_to_feature_data(
    serving: rlt.ServingFeatureData,
    dense_preprocessor: Preprocessor,
    sparse_preprocessor: SparsePreprocessor,
) -> rlt.FeatureData:
    float_features_with_presence, id_list_features, id_score_list_features = serving
    return rlt.FeatureData(
        float_features=dense_preprocessor(*float_features_with_presence),
        id_list_features=sparse_preprocessor.preprocess_id_list(id_list_features),
        id_score_list_features=sparse_preprocessor.preprocess_id_score_list(
            id_score_list_features
        ),
    )

def sparse_input_prototype(
    model: ModelBase,
    state_preprocessor: Preprocessor,
    state_feature_config: rlt.ModelFeatureConfig,
):
    name2id = state_feature_config.name2id
    model_prototype = model.input_prototype()
    # Terrible hack to make JIT tracing works. Python dict doesn't have type
    # so we need to insert something so JIT tracer can infer the type.
    state_id_list_features = {
        42: (torch.zeros(1, dtype=torch.long), torch.tensor([], dtype=torch.long))
    }
    state_id_score_list_features = {
        42: (
            torch.zeros(1, dtype=torch.long),
            torch.tensor([], dtype=torch.long),
            torch.tensor([], dtype=torch.float),
        )
    }
    if isinstance(model_prototype, rlt.FeatureData):
        if model_prototype.id_list_features:
            state_id_list_features = {
                name2id[k]: v for k, v in model_prototype.id_list_features.items()
            }
        if model_prototype.id_score_list_features:
            state_id_score_list_features = {
                name2id[k]: v for k, v in model_prototype.id_score_list_features.items()
            }

    input = rlt.ServingFeatureData(
        float_features_with_presence=state_preprocessor.input_prototype(),
        id_list_features=state_id_list_features,
        id_score_list_features=state_id_score_list_features,
    )
    return (input,)

class DiscreteDqnWithPreprocessor(ModelBase):
    """
    This is separated from DiscreteDqnPredictorWrapper so that we can pass typed inputs
    into the model. This is possible because JIT only traces tensor operation.
    In contrast, JIT scripting needs to compile the code, therefore, it won't recognize
    any custom Python type.
    """

    def __init__(
        self,
        model: ModelBase,
        state_preprocessor: Preprocessor,
        state_feature_config: rlt.ModelFeatureConfig,
    ):
        super().__init__()
        self.model = model
        self.state_preprocessor = state_preprocessor
        self.state_feature_config = state_feature_config
        self.sparse_preprocessor = make_sparse_preprocessor(
            self.state_feature_config, device=torch.device("cpu")
        )

    def forward(self, state: rlt.ServingFeatureData):
        state_feature_data = serving_to_feature_data(
            state, self.state_preprocessor, self.sparse_preprocessor
        )
        q_values = self.model(state_feature_data)
        return q_values

    def input_prototype(self):
        return sparse_input_prototype(
            model=self.model,
            state_preprocessor=self.state_preprocessor,
            state_feature_config=self.state_feature_config,
        )

class DiscreteDqnPredictorWrapper(torch.jit.ScriptModule):
    def __init__(
        self,
        dqn_with_preprocessor: DiscreteDqnWithPreprocessor,
        action_names: List[str],
        # here to keep interface consistent with FB internal
        state_feature_config: rlt.ModelFeatureConfig,
    ) -> None:
        super().__init__()
        self.dqn_with_preprocessor = torch.jit.trace(
            dqn_with_preprocessor, dqn_with_preprocessor.input_prototype()
        )
        self.action_names = torch.jit.Attribute(action_names, List[str])

    # pyre-fixme[56]: Decorator `torch.jit.script_method` could not be resolved in a
    #  global scope.
    @torch.jit.script_method
    def forward(self, state: rlt.ServingFeatureData) -> Tuple[List[str], torch.Tensor]:
        q_values = self.dqn_with_preprocessor(state)
        return (self.action_names, q_values)

class OSSSparsePredictorUnwrapper(nn.Module):
    # Wrap input in serving feature data
    def __init__(self, model: nn.Module) -> None:
        super().__init__()
        self.model = model

    def forward(
        self,
        state_with_presence: Tuple[torch.Tensor, torch.Tensor],
        state_id_list_features: Dict[int, Tuple[torch.Tensor, torch.Tensor]],
        state_id_score_list_features: Dict[
            int, Tuple[torch.Tensor, torch.Tensor, torch.Tensor]
        ],
    ) -> Tuple[List[str], torch.Tensor]:
        return self.model(
            rlt.ServingFeatureData(
                float_features_with_presence=state_with_presence,
                id_list_features=state_id_list_features,
                id_score_list_features=state_id_score_list_features,
            )
        )

# Pass through serving module's output
class OSSPredictorUnwrapper(nn.Module):
    def __init__(self, model: nn.Module) -> None:
        super().__init__()
        self.model = model

    def forward(self, *args, **kwargs) -> Tuple[List[str], torch.Tensor]:
        return self.model(*args, **kwargs)

DiscreteDqnPredictorUnwrapper = OSSSparsePredictorUnwrapper
ActorPredictorUnwrapper = OSSPredictorUnwrapper
ParametricDqnPredictorUnwrapper = OSSPredictorUnwrapper

class ParametricDqnWithPreprocessor(ModelBase):
    def __init__(
        self,
        model: ModelBase,
        state_preprocessor: Preprocessor,
        action_preprocessor: Preprocessor,
    ):
        super().__init__()
        self.model = model
        self.state_preprocessor = state_preprocessor
        self.action_preprocessor = action_preprocessor

    def forward(
        self,
        state_with_presence: Tuple[torch.Tensor, torch.Tensor],
        action_with_presence: Tuple[torch.Tensor, torch.Tensor],
    ):
        preprocessed_state = self.state_preprocessor(
            state_with_presence[0], state_with_presence[1]
        )
        preprocessed_action = self.action_preprocessor(
            action_with_presence[0], action_with_presence[1]
        )
        state = rlt.FeatureData(preprocessed_state)
        action = rlt.FeatureData(preprocessed_action)
        q_value = self.model(state, action)
        return q_value

    def input_prototype(self):
        return (
            self.state_preprocessor.input_prototype(),
            self.action_preprocessor.input_prototype(),
        )

class ParametricDqnPredictorWrapper(torch.jit.ScriptModule):
    def __init__(self, dqn_with_preprocessor: ParametricDqnWithPreprocessor) -> None:
        super().__init__()

        self.dqn_with_preprocessor = torch.jit.trace(
            dqn_with_preprocessor, dqn_with_preprocessor.input_prototype()
        )

    # pyre-fixme[56]: Decorator `torch.jit.script_method` could not be resolved in a
    #  global scope.
    @torch.jit.script_method
    def forward(
        self,
        state_with_presence: Tuple[torch.Tensor, torch.Tensor],
        action_with_presence: Tuple[torch.Tensor, torch.Tensor],
    ) -> Tuple[List[str], torch.Tensor]:
        value = self.dqn_with_preprocessor(state_with_presence, action_with_presence)
        return (["Q"], value)

class ActorWithPreprocessor(ModelBase):
    """
    This is separate from ActorPredictorWrapper so that we can pass typed inputs
    into the model. This is possible because JIT only traces tensor operation.
    In contrast, JIT scripting needs to compile the code, therefore, it won't recognize
    any custom Python type.
    """

    def __init__(
        self,
        model: ModelBase,
        state_preprocessor: Preprocessor,
        action_postprocessor: Optional[Postprocessor] = None,
        serve_mean_policy: bool = False,
    ):
        super().__init__()
        self.model = model
        self.state_preprocessor = state_preprocessor
        self.action_postprocessor = action_postprocessor
        self.serve_mean_policy = serve_mean_policy

    def forward(self, state_with_presence: Tuple[torch.Tensor, torch.Tensor]):
        preprocessed_state = self.state_preprocessor(
            state_with_presence[0], state_with_presence[1]
        )
        state_feature_vector = rlt.FeatureData(preprocessed_state)
        # TODO: include log_prob in the output
        model_output = self.model(state_feature_vector)
        if self.serve_mean_policy:
            assert (
                model_output.squashed_mean is not None
            ), "action mean is None and serve_mean_policy=True"
            action = model_output.squashed_mean
        else:
            action = model_output.action

        if self.action_postprocessor:
            # pyre-fixme[29]: `Optional[Postprocessor]` is not a function.
            action = self.action_postprocessor(action)
        return action

    def input_prototype(self):
        return (self.state_preprocessor.input_prototype(),)

_DEFAULT_FEATURE_IDS = []

class ActorPredictorWrapper(torch.jit.ScriptModule):
    def __init__(
        self,
        actor_with_preprocessor: ActorWithPreprocessor,
        action_feature_ids: List[int] = _DEFAULT_FEATURE_IDS,
    ) -> None:
        """
        action_feature_ids is here to make the interface consistent with FB internal
        version
        """
        super().__init__()

        self.actor_with_preprocessor = torch.jit.trace(
            actor_with_preprocessor, actor_with_preprocessor.input_prototype()
        )

    # pyre-fixme[56]: Decorator `torch.jit.script_method` could not be resolved in a
    #  global scope.
    @torch.jit.script_method
    def forward(
        self, state_with_presence: Tuple[torch.Tensor, torch.Tensor]
    ) -> torch.Tensor:
        action = self.actor_with_preprocessor(state_with_presence)
        return action

class Seq2SlateWithPreprocessor(ModelBase):
    def __init__(
        self,
        model: Seq2SlateTransformerNet,
        state_preprocessor: Preprocessor,
        candidate_preprocessor: Preprocessor,
        greedy: bool,
    ):
        super().__init__()
        self.model = model
        self.state_preprocessor = state_preprocessor
        self.candidate_preprocessor = candidate_preprocessor
        self.greedy = greedy

    def input_prototype(self):
        candidate_input_prototype = self.candidate_preprocessor.input_prototype()
        return (
            self.state_preprocessor.input_prototype(),
            (
                candidate_input_prototype[0].repeat((1, self.model.max_src_seq_len, 1)),
                candidate_input_prototype[1].repeat((1, self.model.max_src_seq_len, 1)),
            ),
        )

    def forward(
        self,
        state_with_presence: Tuple[torch.Tensor, torch.Tensor],
        candidate_with_presence: Tuple[torch.Tensor, torch.Tensor],
    ):
        # state_value.shape == state_presence.shape == batch_size x state_feat_num
        # candidate_value.shape == candidate_presence.shape ==
        # batch_size x max_src_seq_len x candidate_feat_num
        batch_size = state_with_presence[0].shape[0]

        preprocessed_state = self.state_preprocessor(
            state_with_presence[0], state_with_presence[1]
        )
        preprocessed_candidates = self.candidate_preprocessor(
            candidate_with_presence[0].view(
                batch_size * self.model.max_src_seq_len,
                len(self.candidate_preprocessor.sorted_features),
            ),
            candidate_with_presence[1].view(
                batch_size * self.model.max_src_seq_len,
                len(self.candidate_preprocessor.sorted_features),
            ),
        ).view(batch_size, self.model.max_src_seq_len, -1)

        # TODO: consider different numbers of candidates in the same batch_
        src_src_mask = torch.ones(
            batch_size, self.model.max_src_seq_len, self.model.max_src_seq_len
        )
        ranking_input = rlt.PreprocessedRankingInput.from_tensors(
            state=preprocessed_state,
            src_seq=preprocessed_candidates,
            src_src_mask=src_src_mask,
        )
        ranking_output = self.model(
            ranking_input,
            mode=Seq2SlateMode.RANK_MODE,
            tgt_seq_len=self.model.max_tgt_seq_len,
            greedy=self.greedy,
        )
        return ranking_output.ranked_tgt_out_probs, ranking_output.ranked_tgt_out_idx

class Seq2SlatePredictorWrapper(torch.jit.ScriptModule):
    def __init__(self, seq2slate_with_preprocessor: Seq2SlateWithPreprocessor) -> None:
        super().__init__()
        self.seq2slate_with_preprocessor = torch.jit.trace(
            seq2slate_with_preprocessor, seq2slate_with_preprocessor.input_prototype()
        )

    # pyre-fixme[56]: Decorator `torch.jit.script_method` could not be resolved in a
    #  global scope.
    @torch.jit.script_method
    def forward(
        self,
        state_with_presence: Tuple[torch.Tensor, torch.Tensor],
        candidate_with_presence: Tuple[torch.Tensor, torch.Tensor],
    ) -> Tuple[torch.Tensor, torch.Tensor]:
        # ranked_tgt_out_probs shape: batch_size, tgt_seq_len, candidate_size
        # ranked_tgt_out_idx shape: batch_size, tgt_seq_len
        ranked_tgt_out_probs, ranked_tgt_out_idx = self.seq2slate_with_preprocessor(
            state_with_presence, candidate_with_presence
        )
        # convert to slate-wise probabilities
        # ranked_tgt_out_probs shape: batch_size
        ranked_tgt_out_probs = torch.prod(
            torch.gather(
                ranked_tgt_out_probs, 2, ranked_tgt_out_idx.unsqueeze(-1)
            ).squeeze(),
            -1,
        )
        # -2 to offset padding symbol and decoder start symbol
        ranked_tgt_out_idx -= 2
        return ranked_tgt_out_probs, ranked_tgt_out_idx

class Seq2RewardWithPreprocessor(DiscreteDqnWithPreprocessor):
    def __init__(
        self,
        model: ModelBase,
        state_preprocessor: Preprocessor,
        seq_len: int,
        num_action: int,
    ):
        """
        Since TorchScript unable to trace control-flow, we
        have to generate the action enumerations as constants
        here so that trace can use them directly.
        """

        super().__init__(model, state_preprocessor, rlt.ModelFeatureConfig())
        self.seq_len = seq_len
        self.num_action = num_action
        self.all_permut = gen_permutations(seq_len, num_action)
        self.num_permut = self.all_permut.size(1)

    def forward(self, state: rlt.ServingFeatureData):
        """
        This serving module only takes in current state.
        We need to simulate all multi-step length action seq's
        then predict accumulated reward on all those seq's.
        After that, we categorize all action seq's by their
        first actions. Then take the maximum reward as the
        predicted categorical reward for that category.
        Return: categorical reward for the first action
        """
        state_with_presence, _, _ = state
        batch_size, state_dim = state_with_presence[0].size()

        # expand state tensor to match the enumerated action sequences:
        # the tensor manipulations here are tricky:
        # Suppose the input states are s1,s2, these manipulations
        # will generate a input batch s1,s1,...,s1,s2,s2,...,s2
        # where len(s1,s1,...,s1)=len(s2,s2,...,s2)=num_permut
        preprocessed_state = (
            self.state_preprocessor(state_with_presence[0], state_with_presence[1])
            .repeat(1, self.seq_len * self.num_permut)
            .reshape(batch_size * self.num_permut, self.seq_len, -1)
            .transpose(0, 1)
        )
        state_feature_vector = rlt.FeatureData(preprocessed_state)

        # expand action to match the expanded state sequence
        action = self.all_permut.repeat(1, batch_size, 1)
        reward = self.model(
            state_feature_vector, rlt.FeatureData(action)
        ).acc_reward.reshape(
            batch_size, self.num_action, self.num_permut // self.num_action
        )

        # The permuations are generated with lexical order
        # the output has shape [num_perm, num_action,1]
        # that means we can aggregate on the max reward
        # then reshape it to (BATCH_SIZE, ACT_DIM)
        max_reward = (
            # pyre-fixme[16]: `Tuple` has no attribute `values`.
            torch.max(reward, 2)
            .values.cpu()
            .detach()
            .reshape(batch_size, self.num_action)
        )

        return max_reward

class Seq2SlateRewardWithPreprocessor(ModelBase):
    def __init__(
        self,
        model: Seq2SlateRewardNetBase,
        state_preprocessor: Preprocessor,
        candidate_preprocessor: Preprocessor,
    ):
        super().__init__()
        self.model = model
        self.state_preprocessor = state_preprocessor
        self.candidate_preprocessor = candidate_preprocessor

    def input_prototype(self):
        candidate_input_prototype = self.candidate_preprocessor.input_prototype()

        return (
            self.state_preprocessor.input_prototype(),
            (
                candidate_input_prototype[0].repeat((1, self.model.max_src_seq_len, 1)),
                candidate_input_prototype[1].repeat((1, self.model.max_src_seq_len, 1)),
            ),
        )

    @property
    def state_sorted_features(self) -> List[int]:
        return self.state_preprocessor.sorted_features

    @property
    def candidate_sorted_features(self) -> List[int]:
        return self.candidate_preprocessor.sorted_features

    def forward(
        self,
        state_with_presence: Tuple[torch.Tensor, torch.Tensor],
        candidate_with_presence: Tuple[torch.Tensor, torch.Tensor],
    ):
        # state_value.shape == state_presence.shape == batch_size x state_feat_num
        # candidate_value.shape == candidate_presence.shape ==
        # batch_size x max_src_seq_len x candidate_feat_num
        batch_size = state_with_presence[0].shape[0]
        max_tgt_seq_len = self.model.max_tgt_seq_len
        max_src_seq_len = self.model.max_src_seq_len

        # we use a fake slate_idx_with_presence to retrive the first
        # max_tgt_seq_len candidates from
        # len(slate_idx_with presence) == batch_size
        # component: 1d tensor with length max_tgt_seq_len
        slate_idx_with_presence = [
            (torch.arange(max_tgt_seq_len), torch.ones(max_tgt_seq_len))
        ] * batch_size

        preprocessed_state = self.state_preprocessor(
            state_with_presence[0], state_with_presence[1]
        )

        preprocessed_candidates = self.candidate_preprocessor(
            candidate_with_presence[0].view(
                batch_size * max_src_seq_len, len(self.candidate_sorted_features)
            ),
            candidate_with_presence[1].view(
                batch_size * max_src_seq_len, len(self.candidate_sorted_features)
            ),
        ).view(batch_size, max_src_seq_len, -1)

        src_src_mask = torch.ones(batch_size, max_src_seq_len, max_src_seq_len)

        tgt_out_idx = torch.cat(
            [slate_idx[0] for slate_idx in slate_idx_with_presence]
        ).view(batch_size, max_tgt_seq_len)

        tgt_out_seq = gather(preprocessed_candidates, tgt_out_idx)

        ranking_input = rlt.PreprocessedRankingInput.from_tensors(
            state=preprocessed_state,
            src_seq=preprocessed_candidates,
            src_src_mask=src_src_mask,
            tgt_out_seq=tgt_out_seq,
            # +2 is needed to avoid two preserved symbols:
            # PADDING_SYMBOL = 0
            # DECODER_START_SYMBOL = 1
            tgt_out_idx=tgt_out_idx + 2,
        )

        output = self.model(ranking_input)
        return output.predicted_reward

class MDNRNNWithPreprocessor(ModelBase):
    def __init__(
        self,
        model: ModelBase,
        state_preprocessor: Preprocessor,
        seq_len: int,
        num_action: int,
        state_feature_config: Optional[rlt.ModelFeatureConfig] = None,
    ):
        super().__init__()
        self.model = model
        self.state_preprocessor = state_preprocessor
        self.state_feature_config = state_feature_config or rlt.ModelFeatureConfig()
        self.sparse_preprocessor = make_sparse_preprocessor(
            self.state_feature_config, device=torch.device("cpu")
        )
        self.seq_len = seq_len
        self.num_action = num_action

    def forward(
        self,
        state_with_presence: Tuple[torch.Tensor, torch.Tensor],
        action: torch.Tensor,
    ):

        batch_size, state_dim = state_with_presence[0].size()
        preprocessed_state = (
            self.state_preprocessor(state_with_presence[0], state_with_presence[1])
            .reshape(batch_size, self.seq_len, -1)
            .transpose(0, 1)
        )
        result = self.model(action, preprocessed_state)

        return result

    def input_prototype(self):
        return (
            self.state_preprocessor.input_prototype(),
            torch.randn(1, 1, self.num_action, device=self.state_preprocessor.device),
        )

class CompressModelWithPreprocessor(DiscreteDqnWithPreprocessor):
    def forward(self, state: rlt.ServingFeatureData):
        state_feature_data = serving_to_feature_data(
            state, self.state_preprocessor, self.sparse_preprocessor
        )
        # TODO: model is a fully connected network which only takes in Tensor now.
        q_values = self.model(state_feature_data.float_features)
        return q_values


import pandas as pd
import numpy as np
from datetime import datetime
import matplotlib.pyplot as plt

#read CSVs
mandi = pd.read_csv("subset_mandi.csv")
centres = pd.read_csv("subset_centres.csv")
data = pd.read_csv("subset_retail.csv")
ws = pd.read_csv("subset_ws.csv")

ws['date'] =  pd.to_datetime(ws['date'], format='%Y-%m-%d')

a = ws[ws['mid']==11]
b = ws[ws['mid']==35]

idx = pd.date_range('2006-01-01', '2015-06-23')
a.set_index('date', inplace=True)
a = a.reindex(idx, fill_value=0)
a.reset_index(level=0, inplace=True)

b.set_index('date', inplace=True)
b = b.reindex(idx, fill_value=0)
b.reset_index(level=0, inplace=True)

#a = series1
a.reset_index(inplace=True)

#b = series2
b.reset_index(inplace=True)

a['price'] = pd.ewma(a['price'], span=7)
b['price'] = pd.ewma(b['price'], span=7)

df = a.head(len(a)-7)
#df.reset_index('index', inplace=True)

df['slope'] = df.apply(lambda x: 0 if ((b.ix[x['level_0']+7]['price'] - b.ix[x['level_0']]["price"])*x['price'] == 0) else (((a.ix[x['level_0']+7]['price'] - x['price'])*b.ix[x['level_0']]["price"])/((b.ix[x['level_0']+7]['price'] - b.ix[x['level_0']]["price"])*x['price'])), axis=1)

med = df["slope"].median()
tolerance = (med -df["slope"]).median()*1.4826
threshold_P = med + tolerance
threshold_N = med - tolerance
pos_slope = df[df['slope'] > threshold_P]
neg_slope = df[df['slope'] < threshold_N]

#a = pos_slope.reindex(range(0,3454),fill_value=0)
x = pos_slope.index.tolist()
for i in x:
	plt.axvline(x=i)
plt.show()
#plt.plot(a["slope"])


from copy import copy
import json
import logging

log = logging.getLogger(__name__)

def inference_handler(state, video, inference, stats):
    while state.run:
        state.current_frame = video.get_frame(inference, stats)
        state.new_frame = True

def io_handler(state, stats, socketio):
    previous_stats = copy(stats)
    while state.run:
        if (stats != previous_stats):
            log.info('NEW STATS: %s', json.dumps(stats.__dict__))
            socketio.emit('stats', json.dumps(stats.__dict__))
            previous_stats = copy(stats)


import scipy
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.feature_extraction.text import TfidfVectorizer
import re
import string
import nltk
from nltk.tokenize import word_tokenize
from nltk.tokenize import word_tokenize
from nltk.corpus import stopwords
from nltk.stem import PorterStemmer
from nltk.stem import WordNetLemmatizer
nltk.download('punkt')
nltk.download('stopwords')
nltk.download('wordnet')

def remove_hyperlink(word):
    word = str(word)
    return re.sub(r"http\S+", "", word)

def to_lower(word):
    result = word.lower()
    return result

def remove_number(word):
    word = str(word)
    result = re.sub(r'\d+', '', word)
    return result

def remove_punctuation(word):
    result = word.translate(str.maketrans(dict.fromkeys(string.punctuation)))
    return result

def remove_whitespace(word):
    result = word.strip()
    return result

def replace_newline(word):
    return word.replace('\n', '')

def clean_up_pipeline(sentence):
    cleaning_utils = [remove_hyperlink,
                      replace_newline,
                      to_lower,
                      remove_number,
                      remove_punctuation, remove_whitespace]
    for o in cleaning_utils:
        sentence = o(sentence)
    return sentence

stemmer = PorterStemmer()
lemmatizer = WordNetLemmatizer()

def remove_stop_words(words):
    result = [i for i in words if i not in stopwords.words()]
    return result

def word_stemmer(words):
    return [stemmer.stem(o) for o in words]

def word_lemmatizer(words):
    return [lemmatizer.lemmatize(o) for o in words]

def clean_token_pipeline(words):
    cleaning_utils = [remove_stop_words, word_lemmatizer]
    for o in cleaning_utils:
        words = o(words)
    return words

def convert_to_feature(raw_tokenize_data):
    raw_sentences = [' '.join(o) for o in raw_tokenize_data]
    return vectorizer.transform(raw_sentences)

def data_prep(x_test):
    x_test = [word_tokenize(x_test)]
    x_test = [clean_token_pipeline(x_test)]
    x_test = [" ".join(x_test)]
    x_test = [x_test.split(" ")]
    print(x_test)
    vectorizer = TfidfVectorizer()
    raw_sentences = [" ".join(x_test)]
    vectorizer.fit(raw_sentences)

    def convert_to_feature(raw_tokenize_data):
        raw_sentences = [' '.join(raw_tokenize_data)]
        return vectorizer.transform(raw_sentences)

    vectorizer = CountVectorizer()
    raw_sentences = [" ".join(x_test)]
    vectorizer.fit(raw_sentences)
    x_test_features = convert_to_feature(x_test)

    return x_test_features


import logging
import traceback

logger = logging.getLogger('django')
ERROR_CODE = {
    0: '成功',
    1: '记录已存在',
    2: '没有查询到指定记录',
    3: '需要登录',
    4: '参数无效',
    5: '用户未创建商店',
    6: '账号或密码无效',
    7: '无操作权限',
    8: '登陆过期',
    9: '名称已存在',
    10: '需指定父分类',
    11: '商品无库存',
    -1: '稍后再试'
}

def base_code_response():
    # msg = ERROR_CODE.get(code)
    # logger.warning(msg)
    return {
        # 'code': code,
        # 'msg': msg,
        'data': {}
    }

def default_response():
    return {
        'code': 0,
        'msg': 'success',
        'data': {}
    }
def invalid_parameters_response(e):
    code = 4
    msg = ERROR_CODE.get(code) + ': ' + str(e.get_codes())
    logger.warning('traceback.format_exc():%s' % traceback.format_exc())

    return {
        'code': code,
        'msg': msg,
        'data': {}
    }
def need_login_response():
    code = 3
    msg = ERROR_CODE.get(code)
    logger.warning(msg)
    return {
        'code': code,
        'msg': msg,
        'data': {}
    }


#!/usr/bin/python3
# -*- coding: utf-8 -*-
#Written by: Filipe Dezordi (https://dezordi.github.io/)
#At FioCruz/IAM - 2020/07/06

import argparse, subprocess, shlex, csv, re, os
import pandas as pd

parser = argparse.ArgumentParser(description = 'This script returns nucleotide diversy per SNP position and metrics about SNPs by hCoV-19 genome region.')
parser.add_argument("-sn", "--snp_pos", help="SNPs positions in a txt file, onde position per line",  required=True)
parser.add_argument("-bc","--bam_readcount",help="bam-readcount output file", required=True)

args = parser.parse_args()
snp_file = args.snp_pos
bam_rc_file = args.bam_readcount
prefix = snp_file
prefix = re.sub("\..*","",prefix)
prefix = re.sub('.*/','',prefix).rstrip('\n')

"""
Genes position
UTR5 = [1,265]
ORF1AB = [266,21555]
S = [21563,25384]
ORF3A = [25393,26220]
E = [26242,26472]
M = [26523,27191]
ORF6 = [27202,27387]
ORF7A = [27394,27759]
ORF7B = [27756,27887]
ORF8 = [27894,28259]
N = [28274,29533]
ORF10 = [29558,29674]
UTR3 = [29675,29903]
"""

with open(snp_file,'r') as snp_in_file, open(snp_file+'.distr.tsv','w') as snp_file_out:
    reader_snp_in_file = snp_in_file.readlines()
    output_csv_writer = csv.writer(snp_file_out,delimiter='\t')
    output_csv_writer.writerow(['GENOME','POS','REGION','DEPTH','A_DEPTH','A_PLUS','A_MINUS','C_DEPTH','C_PLUS','C_MINUS','G_DEPTH','G_PLUS','G_MINUS','T_DEPTH','T_PLUS','T_MINUS','N'])
    out_list = []
    for snp_line in reader_snp_in_file:
        snp_line = snp_line.rstrip('\n')
        with open(bam_rc_file,'r') as bam_rc_file_in: #, open(bam_rc_file+'.csv','w') as output_csv:
            bam_rc_file_read = csv.reader(bam_rc_file_in, delimiter='\t')
            for line in bam_rc_file_read:
                if snp_line == line[1]:
                    var_pos = line[1].rstrip('\n')
                    var_depth = line[3].rstrip('\n')
                    A_depth_line = line[5].rstrip('\n')
                    A_depth = ":".join(A_depth_line.split(":", 2)[:2])
                    A_depth = re.sub(r".*:","",A_depth)
                    A_plus = ":".join(A_depth_line.split(":", 6)[:6])
                    A_plus = re.sub(r".*:","",A_plus)
                    A_minus = ":".join(A_depth_line.split(":", 7)[:7])
                    A_minus = re.sub(r".*:","",A_minus)
                    C_depth_line = line[6].rstrip('\n')
                    C_depth = ":".join(C_depth_line.split(":", 2)[:2])
                    C_depth = re.sub(r".*:","",C_depth)
                    C_plus = ":".join(C_depth_line.split(":", 6)[:6])
                    C_plus = re.sub(r".*:","",C_plus)
                    C_minus = ":".join(C_depth_line.split(":", 7)[:7])
                    C_minus = re.sub(r".*:","",C_minus)
                    G_depth_line = line[7].rstrip('\n')
                    G_depth = ":".join(G_depth_line.split(":", 2)[:2])
                    G_depth = re.sub(r".*:","",G_depth)
                    G_plus = ":".join(G_depth_line.split(":", 6)[:6])
                    G_plus = re.sub(r".*:","",G_plus)
                    G_minus = ":".join(G_depth_line.split(":", 7)[:7])
                    G_minus = re.sub(r".*:","",G_minus)
                    T_depth_line = line[8].rstrip('\n')
                    T_depth = ":".join(T_depth_line.split(":", 2)[:2])
                    T_depth = re.sub(r".*:","",T_depth)
                    T_plus = ":".join(T_depth_line.split(":", 6)[:6])
                    T_plus = re.sub(r".*:","",T_plus)
                    T_minus = ":".join(T_depth_line.split(":", 7)[:7])
                    T_minus = re.sub(r".*:","",T_minus)
                    N_depth = line[9].rstrip('\n')
                    N_depth = ":".join(N_depth.split(":", 2)[:2])
                    N_depth = re.sub(r".*:","",N_depth)
                    if int(var_pos) <= 265:
                        region = '5UTR'
                    elif int(var_pos) <= 21555:
                        region = 'ORF1AB'
                    elif int(var_pos) <= 25384:
                        region = 'S'
                    elif int(var_pos) <= 26220:
                        region = 'ORF3A'
                    elif int(var_pos) <= 26472:
                        region = 'E'
                    elif int(var_pos) <= 27191:
                        region = 'M'
                    elif int(var_pos) <= 27387:
                        region = 'ORF6'
                    elif int(var_pos) <= 27759:
                        region = 'ORF7A'
                    elif int(var_pos) <= 27887:
                        region = 'ORF7B'
                    elif int(var_pos) <= 28259:
                        region = 'ORF8'
                    elif int(var_pos) <= 29533:
                        region = 'N'
                    elif int(var_pos) <= 29674:
                        region = 'ORF10'
                    else:
                        region = '3UTR'
                    out_list.append([prefix,var_pos,region,var_depth,A_depth,A_plus,A_minus,C_depth,C_plus,C_minus,G_depth,G_plus,G_minus,T_depth,T_plus,T_minus,N_depth])
    output_csv_writer.writerows(out_list)
df = pd.read_csv(snp_file+'.distr.tsv',sep='\t')
df2 = df[['REGION','DEPTH']]
df2 = pd.DataFrame(group.describe().loc[['count','min','max','mean','std']].rename(columns={'DEPTH':name}).squeeze()for name, group in df2.groupby('REGION'))
df2 = df2.round(2)
df2.insert(loc=0,column='GENOME',value=prefix)
df2.to_csv(snp_file+'.metrics.tsv',sep='\t')


import cv2
import numpy as np
import datetime
from PIL import ImageFont, ImageDraw, Image
from numpy.core.records import fromrecords
from numpy.lib.type_check import isreal

# 카메라 기본 틀
cap = cv2.VideoCapture(0) # 번호 0부터
cap.set(cv2.CAP_PROP_FRAME_WIDTH, 640)
cap.set(cv2.CAP_PROP_FRAME_HEIGHT, 480)

# 나눔고딕볼드 로드
font = ImageFont.truetype('./Fonts/NanumGothicBold.ttf', 20)

# 영상 코덱 설정
fourcc = cv2.VideoWriter_fourcc(*'XVID')
is_record = False

# 무한루프 지정 키를 눌렀을 때
while True:
    ret, frame = cap.read() # 카메라 현재 영상 로드, frame에 저장, ret Ture/False
    frame = cv2.flip(frame, 0)

    h, w, _ = frame.shape # channel은 불필요
    now = datetime.datetime.now()
    currDateTime = now.strftime('%Y-%m-%d %H:%M:%S')
    fileDateTime = now.strftime('%Y%m%d_%H%M%S') # 20210720_164725  캡쳐되는 파일 이름

    if ret != True: break # ret이 False면 루프탈출

    frame = Image.fromarray(frame)
    draw = ImageDraw.Draw(frame)
    draw.text(xy=((w-300), 10), text="겸's_TV - {0}".format(currDateTime), font=font, fill=(0, 0, 255))
    frame = np.array(frame) # 웹상태 복귀

    #cv2.imshow('RealTime cam', frame)   # 로드한 영상을 창에 띄움
    cv2.imshow('Gray Result', frame)

    key = cv2.waitKey(1)
    if key == ord('q'): break# q를 입력하면 루프탈출
    elif key == ord('c'): #c는 화면캡쳐
        cv2.imwrite('./capture/img_{0}.png'.format(fileDateTime), frame)
        print('이미지 저장 완료')
    elif key == ord('r'): # 레코드 시작
        is_record = True
        video = cv2.VideoWriter('./capture/record_{0}.avi'.format(fileDateTime), fourcc, 20, (w, h))
        print('녹화 시작')
    elif key == ord('t'): # 레코드 종료
        is_record = False
        video.release() # 객체 해제
        print("녹화 완료")

    if is_record:
        video.write(frame)
        cv2.circle(img=frame, center=(620, 15), radius=5, color=(0, 0, 255), thickness=3)
cap.realse() # 웹캠해제
cv2.destroyAllWindows()


import credentials
import logging
from tweepy import OAuthHandler, Stream
from tweepy.streaming import StreamListener
from pymongo import MongoClient

def authenticate():
    """Function for handling Twitter Authentication. Please note
       that this script assumes you have a file called credentials.py
       which stores the 4 required authentication tokens:

       1. API_KEY
       2. API_SECRET
       3. ACCESS_TOKEN
       4. ACCESS_TOKEN_SECRET

    See course material for instructions on getting your own Twitter credentials.
    """
    auth = OAuthHandler(credentials.API_KEY, credentials.API_SECRET)
    auth.set_access_token(credentials.ACCESS_TOKEN, credentials.ACCESS_TOKEN_SECRET)
    return auth

class MaxTweetsListener(StreamListener):

    def __init__(self, max_tweets, *args, **kwargs):
        # initialize the StreamListener
        super().__init__(*args, **kwargs)
        # set the instance attributes
        self.max_tweets = max_tweets
        self.counter = 0
        # instanciate mongo database
        self.client = MongoClient('mongodb')
        self.db = self.client['twitter']
        if self.db.list_collection_names() == []:
            self.db.create_collection('tweets')
        # make sure this works
        logging.critical('... created with mongodb connection')

    def on_connect(self):
        logging.critical(f'... listening for incoming {self.max_tweets} tweets:')

    def on_status(self, status):
        # extract information from status
        tweet = {
            'text': status.text,
            'username': status.user.screen_name,
            'followers_count': status.user.followers_count
        }

        # decide what to do here:
        # self.print_tweet(tweet)
        self.add_to_db(tweet)

        # keep track of the counter, eventually terminate
        self.counter += 1
        if self.max_tweets == self.counter:
            self.counter = 0
            logging.critical('... all tweets collected, listener end')
            return False

    def on_error(self, status):
        if status == 420:
            logging.critical('... Rate limit applies. Stop the stream.')
            return False

    def print_tweet(self, tweet):
        print(f'({self.counter}) {tweet["username"]}: {tweet["text"]}')

    def add_to_db(self, tweet):
        self.db.tweets.insert_one(tweet)
        logging.critical(f'{self.counter} tweet -> mongodb')

if __name__ == '__main__':
    auth = authenticate()
    listener = MaxTweetsListener(max_tweets=1000)
    stream = Stream(auth, listener)
    stream.filter(track=['ai'], languages=['en'], is_async=False)


from django.shortcuts import render, HttpResponse, redirect, get_object_or_404
from django.http import JsonResponse
from django.urls import reverse_lazy
from rest_framework.renderers import JSONRenderer
from django.db import models
from django.contrib.contenttypes.models import ContentType

from data_importation.views import delete
from data_reference.forms import *
from data_importation.forms import DeleteForm

def get_fields_for_model(model):
    model = ContentType.objects.get(model__icontains=model)
    fields = [[v.name, str([t.name for t in v.related_model._meta.fields]) if v.is_relation else False] for i, v in
              enumerate(model.model_class()._meta.fields)]
    return fields

def get_model_fields(request, model, app_label=None):
    fields = get_fields_for_model(model)
    # serial = JSONRenderer().render(fields)
    return JsonResponse(fields, safe=False, status=200)

def get_class_lookups(request, model, field_name):
    model = ContentType.objects.get(model__contains=model)
    fields = model.model_class()._meta.fields
    lookups = []
    for field in fields:
        temp = field.get_lookups().keys() if field.attname == field_name else None
        if temp is not None:
            lookups = list(temp)

    return JsonResponse(lookups, safe=False, status=200)

def get_all_models(request, app_label):
    tables = ContentType.objects.filter(app_label__icontains=app_label)
    tables = [i.model for i in tables]
    return JsonResponse(tables, safe=False, status=200)

def get_model_filter_kwargs(request, fields):
    pass

def create_model_table_view(request):
    form = ModelTableForm()

    if request.method == "POST":
        form = ModelTableForm(request.POST)

        if form.is_valid():
            form.save()

            if form.cleaned_data['next_action'] == "1":
                return redirect(reverse_lazy("data-reference:create-model-table"))
            elif form.cleaned_data["next_action"] == "2":
                return HttpResponse("Ajouter des colonnes à ce modèle")
            return HttpResponse("Form Validated!")
        else:
            return render(request, 'data_reference/create-model-table-view.html', locals())

    return render(request, 'data_reference/create-model-table-view.html', locals())

def all_model_tables(request):
    all_tables = ModelTable.objects.all()
    return render(request, 'data_reference/all_models_tables.html', locals())

def edit_model_table(request, pk):
    model_table = get_object_or_404(ModelTable, pk=pk)
    form = ModelTableForm(instance=model_table)
    if request.method == "POST":
        form = ModelTableForm(request.POST)
        if form.is_valid():
            # form.save()
            ModelTable.objects.filter(pk=pk).update(**{key:value for key, value in form.cleaned_data.items()
                                                       if key != "next_action"})

            if form.cleaned_data['next_action'] == "1":
                return redirect(reverse_lazy("data-reference:create-model-table"))
            elif form.cleaned_data["next_action"] == "2":
                return HttpResponse("Ajouter des colonnes à ce modèle")
            return redirect(reverse_lazy('data-reference:all-models-tables'))
        else:
            return render(request, 'data_reference/create-model-table-view.html', locals())

    return render(request, 'data_reference/create-model-table-view.html', locals())

def delete_model_table(request, pk):
    model_table = get_object_or_404(ModelTable, pk=pk)
    if request.method == 'POST':
        form = DeleteForm(request.POST)
        delete(request, model_table, form)
    return redirect(reverse_lazy("data-reference:all-models-tables"))

def detail_model_table(request, pk):
    model_table = get_object_or_404(ModelTable, pk=pk)
    return render(request, 'data_reference/detail-model-table.html', locals())

def create_column_for_model(request, pk):
    model_table = get_object_or_404(ModelTable, pk=pk)
    fields = get_fields_for_model(model_table.model_name)
    form = ModelColumnForm()

    if request.method == 'POST':
        form = ModelColumnForm(request.POST)

        if form.is_valid():
            instance = form.save(commit=False)
            instance.model_table = model_table
            instance.save()

            if form.cleaned_data['next_action'] == "1":
                return redirect(reverse_lazy("data-reference:create-column-for-model", kwargs={'pk': model_table.pk}))
            elif form.cleaned_data["next_action"] == "2":
                return HttpResponse("Ajouter des colonnes à ce modèle")

            return redirect(reverse_lazy("data-reference:detail-model-table", kwargs={'pk': model_table.pk}))
        else:
            return render(request, 'data_reference/create-column-for-table.html', locals())
    return render(request, 'data_reference/create-column-for-table.html', locals())

def delete_column_for_model(request, pk):
    column = get_object_or_404(ModelColumn, pk=pk)
    if request.method == 'POST':
        form = DeleteForm(request.POST)
        delete(request, column, form)
    return redirect(reverse_lazy("data-reference:detail-model-table", kwargs={'pk': column.model_table.pk}))

def edit_column_for_model(request, pk):
    pass

def view_table_by_model(request, pk):
    model_table = get_object_or_404(ModelTable, pk=pk)
    model = ContentType.objects.get(model=model_table.model_name)
    headers = [i.name for i in model_table.table_to_fields.all()]
    data = [[getattr(obj, i) for i in headers if hasattr(obj, i)] for obj in model.model_class().objects.all()]
    try:
        headers = headers[:len(data[0]) if len(headers) > len(data[0]) else len(headers)]
    except IndexError as e:
        pass

    return render(request, 'data_reference/view-table-by-model.html', locals())


# This is a sample Python script.
import queue
import random
import sys
import threading
import time
import tkinter
from tkinter.messagebox import showinfo

import pyautogui

from voice_command_loader import get_voice_command

text_to_command_queue_ = queue.Queue()
movement_direction = None
movements = {'up': [-1, 0], 'down': [1, 0], 'left': [0, -1], 'right': [0, 1],
             'up left': [-1, -1], 'up right': [-1, 1], 'down left': [1, -1], 'down right': [1, 1],
             'stop': [0, 0]}
clicks = {'click': [1, 'left'], 'double click': [2, 'left'],
          'right click': [1, 'right'], 'right double click': [2, 'right'],
          'middle click': [1, 'middle']}

type_text = None

click_count = None
click_button = None
screen_w, screen_h = pyautogui.size()
lock = threading.Lock()

def do_movement():
    global movement_direction
    while True:
        if movement_direction is not None:
            pyautogui.move(movement_direction[1], movement_direction[0])

def check_stop_typing():
    """
    :return:
    """
    root = tkinter.Tk()
    root.withdraw()
    showinfo(title='confirmation',
             message='Are you sure that you want to stop typing? Say Yes or No')
    return root
    # root.destroy()

def click():
    global click_count
    global click_button
    if click_count is not None and click_button is not None:
        print('i am doing some clicking')
        pyautogui.click(clicks=click_count, button=click_button)
        click_count = None
        click_button = None

def set_variable(text_to_command_queue):
    global movement_direction
    global click_count
    global click_button
    while True:
        if not text_to_command_queue.empty():
            command = text_to_command_queue.get()
            if command in movements:
                print(f'Moving cursor:\n{command}')
                movement_direction = movements[command]
            elif command in clicks:
                click_count = clicks[command][0]
                click_button = clicks[command][1]
                print(f'Click:\n{command}')
                click()
            elif command == 'type':
                print('Typing')
                while True:
                    if not text_to_command_queue.empty():
                        typed_text = text_to_command_queue.get()
                        if typed_text == 'stop typing':
                            break
                        pyautogui.typewrite(text_to_command_queue.get() + ' ')
            else:
                print(f'Unknown command: {command}')
        # time.sleep(5)

if __name__ == '__main__':
    # Create the threads
    thread_set_var = threading.Thread(target=set_variable, daemon=True, args=[text_to_command_queue_])
    thread_do_movement = threading.Thread(target=do_movement, daemon=True)
    # thread_click = threading.Thread(target=click, daemon=True)
    thread_get_voice_command = threading.Thread(target=get_voice_command, daemon=True, args=[text_to_command_queue_])

    # Start the threads
    thread_set_var.start()
    thread_do_movement.start()
    thread_get_voice_command.start()

    # Wait for the threads to finish
    thread_set_var.join()
    thread_do_movement.join()
    thread_get_voice_command.join()


from fastapi import FastAPI
from fastapi.middleware.cors import CORSMiddleware

from Globalizer.predict import k_clustering, radius_clustering, smart_clustering
app = FastAPI()

app.add_middleware(
    CORSMiddleware,
    allow_origins=["*"],  # Allows all origins
    allow_credentials=True,
    allow_methods=["*"],  # Allows all methods
    allow_headers=["*"],  # Allows all headers
)

@app.get("/")
def index():
    return {"ok": True}

@app.get("/predict")
def predict(country, threshold=None, n_centers=None, radius=None, percent=None):
    '''This function controls whether
    we use the Kmeans or the KmeansMinibatch. By default it's
    the Minibatch with 50km as threshold'''
    country = country.split(",")
    if n_centers != None:
        n_centers= int(n_centers)
        return k_clustering(country, n_centers)
    if threshold != None:
        threshold= float(threshold)
        return smart_clustering(country, threshold)
    if radius != None and percent != None:
        radius = int(radius)
        percent = int(percent)
        return radius_clustering(country, radius, percent)

## Check ##
# to check your code, run:
# python -m api.fast
if __name__=="__main__":
    #print(predict(country = "ABW", n_centers = 5))
    print(predict(country = "AUT", radius=60, percent=80))


import requests
from bs4 import BeautifulSoup
import re

news_main_page = "https://www.frankfurt-university.de/de/aktuelles/"
url = "https://www.frankfurt-university.de/de/newsmodule/details/mit-projekt-zero-waste-unter-bestplatzierten-beim-data-mining-cup-2023/"

def write_string_to_file(filename, text):
    with open(filename, "w") as file:
        file.write(text)

def remove_special_characters(text):
    return re.sub("[^a-zA-Z0-9 ]", "", text)

def get_all_page_links(url):
    response = requests.get(url)
    content = response.content
    soup = BeautifulSoup(content, "html.parser")

    page2_link = (
        "https://www.frankfurt-university.de"
        + soup.find("a", attrs={"title": "Zu Seite 2"})["href"]
    )
    page3_link = (
        "https://www.frankfurt-university.de"
        + soup.find("a", attrs={"title": "Zu Seite 3"})["href"]
    )

    return [url, page2_link, page3_link]

def get_all_news_links(urls):
    article_links = []

    for url in urls:
        response = requests.get(url)

        if response.status_code == 200:
            content = response.content

            soup = BeautifulSoup(content, "html.parser")

            # get article elementson this page
            article_div = soup.find("div", attrs={"class": "news-simple-list"})
            articles = article_div.find_all("article")

            for article in articles:
                # get article link
                arcticle_link = (
                    article.find("div", attrs={"class": "news-simple-list__text"})
                    .find("div", attrs={"class": "news-simple-list__header"})
                    .find("a", attrs={"class": "news-article-header__link"})
                )

                # add link to article_links list
                article_links.append(
                    "https://www.frankfurt-university.de" + arcticle_link["href"]
                )
        else:
            print("Failed to retrieve webpage.")

    return article_links

def scrape_website_and_save_document_to_file(url):
    response = requests.get(url)

    # Check the status of the request
    if response.status_code == 200:
        content = response.content

        soup = BeautifulSoup(content, "html.parser")

        # 1. headline
        headline = (
            soup.find("div", attrs={"class": "news-article-header"})
            .find_all("h1")[0]
            .text
        )

        # 2. content
        content_div = soup.find("div", attrs={"class": "news-text-wrap"})
        # Get all text within this div
        content_div_text = content_div.get_text()

        # 3. image captions
        try:
            image_captions_text = ""
            image_div = soup.find("div", attrs={"class": "news-img-wrap"})

            for image_caption in image_div.find_all("img"):
                image_captions_text += "\n" + image_caption.get("data-caption")
        except:
            image_captions_text = ""
            print(f"Article has no images: {url}")

        # deprecated
        # h2_elements = content_div.find_all("h2")
        # p_elements = content_div.find_all("p")
        # list_imtems = content_div.find_all("li")
        # text_elements = headline + h2_elements + p_elements
        # for text in text_elements:
        #     if len(document) == 0:
        #         document += text.text
        #     else:
        #         document += "\n" + text.text

        document = headline + "\n" + content_div_text + image_captions_text

        write_string_to_file(
            "documents/"
            + remove_special_characters(document.split("\n", 1)[0])
            + ".txt",
            document,
        )

    else:
        print("Failed to retrieve webpage.")

page_links = get_all_page_links(news_main_page)
news_links = get_all_news_links(page_links)

for news_link in news_links:
    scrape_website_and_save_document_to_file(news_link)


import json
import os
import time

import tvm
from tvm import relay
from tvm import autotvm
from tvm import rpc
from tvm.contrib import util
from tvm.contrib import graph_runtime as runtime
from tvm.contrib.debugger import debug_runtime
import numpy as np
import torch

from .get_model import get_model

def get_input_info(graph, params):
    """
    Get shape and dtype of the input tensors for a compiled model.
    """
    shape_dict = {}
    dtype_dict = {}
    graph = json.loads(graph)
    param_names = params.keys()
    for node_id in graph["arg_nodes"]:
        node = graph["nodes"][node_id]
        name = node["name"]
        if name not in param_names:
            shape_dict[name] = graph["attrs"]["shape"][1][node_id]
            dtype_dict[name] = graph["attrs"]["dltype"][1][node_id]
    return shape_dict, dtype_dict

def generate_tensor_data(shape, dtype):
    """
    Generate data to produce a tensor of given shape and dtype for TVM.
    """
    if "int8" in dtype:
        tensor = np.random.randint(128, size=shape, dtype=dtype)
    else:
        tensor = np.random.uniform(-1, 1, size=shape).astype(dtype)
    return tensor

def generate_pytorch_data(input_shapes, target_string, dtype):
    """
    Generate data to produce a tensor of given shape and dtype for PyTorch.
    """
    data = []
    for input_shape in input_shapes:
        if target_string == "cuda":
            if dtype == "int":
                data.append(torch.randint(0,10, input_shape[1], device="cuda"))
            else:
                data.append(torch.cuda.FloatTensor(input_shape[1]).normal_())
        else:
            data.append(torch.randn(input_shape[1]))
    return data

def make_inputs_dict(shape_dict, dtype_dict):
    """
    Make the inputs dictionary for a graph. Here, random input tensors
    for each input are produced.
    """
    inputs_dict = {}
    for input_name in shape_dict:
        shape = shape_dict[input_name]
        dtype = dtype_dict[input_name]
        data = generate_tensor_data(shape, dtype)
        inputs_dict[input_name] = data
    return inputs_dict

def extract_profile_data(times):
    """
    Provided a series of execution times calculate the mean, std, max and min.
    """
    mean_ts = np.mean(times)
    std_ts = np.std(times)
    max_ts = np.max(times)
    min_ts = np.min(times)

    header = "Execution time summary:\n{0:^10} {1:^10} {2:^10} {3:^10}".format(
        "mean (s)", "max (s)", "min (s)", "std (s)")
    stats = "{0:^10.5f} {1:^10.5f} {2:^10.5f} {3:^10.5f}".format(
        mean_ts, max_ts, min_ts, std_ts)

    return header, stats

def compile_model(mod, params, target_string, tuning_records=None, model_name=""):
    """
    Compile TVM model. Apply tuning records if they exist.
    """
    target = tvm.target.Target(target_string)
    target_host = "llvm"

    if tuning_records:
        if model_name:
            model_str = "_model=" + model_name + ".json"
            tuning_records = tuning_records[:-5]
            tuning_records += model_str
        if os.path.exists(tuning_records):
            print('applying tuning history...')
        with autotvm.apply_history_best(tuning_records):
            with tvm.transform.PassContext(opt_level=3):
                graph_module = relay.build(mod, target, params=params, target_host=target_host)
    else:
        with tvm.transform.PassContext(opt_level=3):
            graph_module = relay.build(mod, target, params=params, target_host=target_host)

    return graph_module.get_json(), graph_module.get_lib(), graph_module.get_params()

def run_model_tvm(graph, lib, params, run_settings, model_name, tuning_records=None):
    """
    Run TVM model. Apply tuning records if they exist.
    """
    profile = run_settings['profile']
    device = run_settings['device']
    repeat = run_settings['repeat']

    session = rpc.LocalSession()
    ctx = session.cpu() if device == "cpu" else session.gpu()
    is_tuned = True if tuning_records else False

    lib_name = "mod.so"
    temp = util.tempdir()
    lib_path = temp.relpath(lib_name)
    lib.export_library(lib_path)
    session.upload(lib_path)
    lib = session.load_module(lib_name)

    if profile:
        module = debug_runtime.create(graph, lib, ctx, dump_root=f"results/prof_{model_name}_tuned={is_tuned}")
    else:
        module = runtime.create(graph, lib, ctx)

    saved_params = relay.save_param_dict(params)
    module.load_params(saved_params)

    shape_dict, dtype_dict = get_input_info(graph, params)
    inputs_dict = make_inputs_dict(shape_dict, dtype_dict)
    module.set_input(**inputs_dict)

    if profile:
        module.run()

    timer = module.module.time_evaluator("run", ctx, 1, repeat=repeat)
    prof_result = timer()
    times = prof_result.results
    header, stats = extract_profile_data(times)

    filename = f'results/stat_table_{model_name}_tuned={is_tuned}'
    with open(filename, 'w') as f:
        print("%s\n%s\n" % (header, stats), filename, file=f)
    print("%s\n%s\n" % (header, stats))

def run_model_pytorch(trace, input_shapes, run_settings, model_name, target_string):
    """
    Run model using the standard pytorch runtime.
    """
    repeat = run_settings['repeat']

    # TODO extract datatype required from PyTorch model itself
    dtype = "int" if model_name == "bert" else "float"
    generated_data = generate_pytorch_data(input_shapes, target_string, dtype)

    # First run is always slower due to on-the-fly optimization, skip it.
    skip_first_run = True
    times = []
    for _ in range(repeat + 1):
        start = time.time()
        # Note: only a single input is currently supported
        trace(*generated_data)
        end = time.time() - start

        if skip_first_run:
            skip_first_run = False
            continue
        times.append(end)

    header, stats = extract_profile_data(times)

    filename = f'results/stat_table_{model_name}_pytorch'
    with open(filename, 'w') as f:
        print("%s\n%s\n" % (header, stats), filename, file=f)
    print("%s\n%s\n" % (header, stats))

def benchmark_models(data):
    """
    Compile and run a TVM model and benchmark it. Then run the same model using PyTorch
    and display statistics comparing the two.
    """
    target_string = data['target']
    tuning_records = data['autotuner_settings'].get('tuning_records') or ""
    run_settings = data['run_settings']

    for model in data['models']:
        for executor in ["tvm", "pytorch"]:
            trace, input_shapes = get_model(model['name'], model['type'])

            if executor == "tvm":
                mod, params = relay.frontend.from_pytorch(trace, input_shapes)

                print(f"Compiling TVM model {model['name']}")
                graph, lib, params = compile_model(mod, params, target_string, tuning_records, model['name'])

                print(f"Running TVM model {model['name']}")
                run_model_tvm(graph, lib, params, run_settings, model['name'], tuning_records)
            elif executor == "pytorch":
                run_model_pytorch(trace, input_shapes, run_settings, model['name'], target_string)
            else:
                raise ValueError(f"executor {executor} not supported.")