import os
import argparse
import json
import pickle
import random
from datetime import date

import numpy as np

import torch

from torch.utils.data import DataLoader, random_split, ConcatDataset
from torch.optim import SGD, Adam
from torch.nn.functional import binary_cross_entropy, pad, one_hot
from sklearn import metrics
from data_loaders.assist2009 import ASSIST2009
from data_loaders.assist2012 import ASSIST2012
from data_loaders.ednet01 import EdNet01
from data_loaders.csedm import CSEDM

# 모델 추가
from models.dkt_sc import DKT_FUSION as DKT_F
from models.dkt_rear import DKT as DKT_REAR
from models.dkt_front import DKT as DKT_FRONT
from models.dkt import DKT
from models.dkvmn import DKVMN
from models.dkvmn_text import SUBJ_DKVMN
from models.dkvmn_back import BACK_DKVMN
from models.sakt import SAKT
from models.sakt_front import SAKT as SAKT_FRONT
from models.sakt_rear import SAKT as SAKT_REAR
from models.saint import SAINT
from models.saint_front import SAINT as SAINT_FRONT
from models.saint_rear import SAINT as SAINT_REAR
from models.auto import AUTO
from models.mekt import MEKT
from models.dirt import DeepIRT
from models.qakt import QAKT
from models.akt import AKT

# 모델에 따른 train
from models.dkt_sc import dkt_train as dkf_train
from models.dkt_rear import dkt_train as dk_rear_train
from models.dkt_front import dkt_train as dk_front_train
from models.dkt import train_model as dkt_train
from models.auto import auto_train
from models.dkvmn_text import train_model as plus_train
from models.dkvmn_back import train_model as minus_train
from models.sakt import train_model as sakt_train
from models.sakt_front import sakt_train as sakt_front_train
from models.sakt_rear import sakt_train as sakt_rear_train
from models.dkvmn import train_model as dkvmn_train
from models.saint import train_model as saint_train
from models.saint_front import train_model as saint_front_train
from models.saint_rear import train_model as saint_rear_train
from models.akt import train_model as akt_train

# 모델에 따른 test
from models.dkt import test_model as dkt_test
from models.dkvmn import test_model as dkvmn_test
from models.sakt import test_model as sakt_test
from models.saint import test_model as saint_test
from models.akt import test_model as akt_test

from models.utils import collate_fn, collate_ednet, cal_acc_class, reset_weight

# Cross Validation
from sklearn.model_selection import KFold

# wandb
import wandb

# seed 고정
seed = 13
#deterministic = True

random.seed(seed)
np.random.seed(seed)
torch.manual_seed(seed)
torch.cuda.manual_seed_all(seed)
#if deterministic:
    #torch.backends.cudnn.deterministic = True
    #torch.backends.cudnn.benchmark = False

# Train function
def train_model(model, train_loader, valid_loader, num_q, num_epochs, opt, ckpt_path, mode=0, use_wandb=False):
    max_auc = 0
    loss_means = []
    aucs = []
    accs = []
    precisions = []
    recalls = []
    f1s = []

    for epoch in range(0, num_epochs):
        auc_mean = []
        loss_mean = []
        acc_mean = []
        model.train()
        for i, data in enumerate(train_loader, 0):
            # q_seqs, r_seqs, qshft_seqs, rshft_seqs, mask_seqs, bert_sentences, bert_sentence_types, bert_sentence_att_mask, proc_atshft_sentences
            q, r, qshft_seqs, rshft_seqs, m, bert_s, bert_t, bert_m, q2diff_seqs, pid_seqs, pidshift, hint_seqs = data

            # 현재 답안 예측
            inpt_q = q.long()
            pred_t = r
            if mode == 1: # 다음 답안 예측
                inpt_q = qshft_seqs.long()
                pred_t = rshft_seqs
            elif mode == 2: # 스코어 예측
                pred_t = pid_seqs
            elif mode == 3: # 다음 스코어 예측
                inpt_q = qshft_seqs.long()
                pred_t = pidshift

            # 현재까지의 입력을 받은 뒤 다음 문제 예측
            y, Mv = model(q.long(), r.long()) # sakt는 qshft_seqs.long() 추가
            # y = (y * one_hot(inpt_q, num_q)).sum(-1)

            opt.zero_grad()
            y = torch.masked_select(y, m)
            t = torch.masked_select(pred_t, m) # rshft 대신 pidshift
            h = torch.masked_select(hint_seqs, m)

            loss = binary_cross_entropy(y, t)
            loss.backward()
            opt.step()

            loss_mean.append(loss.detach().cpu().numpy())
            auc_mean.append(metrics.roc_auc_score(
                y_true=t.detach().cpu().numpy(), y_score=y.detach().cpu().numpy()
            ))
            bin_y = [1 if p >= 0.5 else 0 for p in y.detach().cpu().numpy()]
            acc_mean.append(metrics.accuracy_score(t.detach().cpu().numpy(), bin_y))

        loss_mean = np.mean(loss_mean)
        auc_mean = np.mean(auc_mean)
        acc_mean = np.mean(acc_mean)

        if(use_wandb != False):
            wandb.log(
            {
                "epoch": epoch,
                "train_auc": auc_mean,
                "train_acc": acc_mean,
                "train_loss": loss_mean
            })

        print(f"[Train] Epoch: {epoch}, AUC: {auc_mean}, acc: {acc_mean}, Loss Mean: {np.mean(loss_mean)}")

        with torch.no_grad():
            model.eval()
            auc_mean = []
            loss_mean = []
            acc_mean = []
            precision_mean = []
            recall_mean = []
            f1_mean = []

            best_loss = 10 ** 9
            patience_limit = 3
            patience_check = 0

            for i, data in enumerate(valid_loader):
                q, r, qshft_seqs, rshft_seqs, m, bert_s, bert_t, bert_m, q2diff_seqs, pid_seqs, pidshift, hint_seqs = data

                # 현재 답안 예측
                inpt_q = q.long()
                pred_t = r
                if mode == 1: # 다음 답안 예측
                    inpt_q = qshft_seqs.long()
                    pred_t = rshft_seqs
                elif mode == 2: # 스코어 예측
                    pred_t = pid_seqs
                elif mode == 3: # 다음 스코어 예측
                    inpt_q = qshft_seqs.long()
                    pred_t = pidshift

                y, Mv = model(q.long(), r.long()) # sakt는 qshft_seqs.long() 추가
                # y = (y * one_hot(inpt_q, num_q)).sum(-1)

                # y와 t 변수에 있는 행렬들에서 마스킹이 true로 된 값들만 불러옴
                q = torch.masked_select(inpt_q, m).detach().cpu()
                y = torch.masked_select(y, m).detach().cpu()
                t = torch.masked_select(pred_t, m).detach().cpu()
                h = torch.masked_select(hint_seqs, m).detach().cpu()

                auc = metrics.roc_auc_score(
                    y_true=t.numpy(), y_score=y.numpy()
                )
                auc_mean.append(auc)

                bin_y = [1 if p >= 0.5 else 0 for p in y.detach().cpu().numpy()]
                acc = metrics.accuracy_score(t.detach().cpu().numpy(), bin_y)
                precision = metrics.precision_score(t.numpy(), bin_y, average='binary')
                recall = metrics.recall_score(t.numpy(), bin_y, average='binary')
                f1 = metrics.f1_score(t.numpy(), bin_y, average='binary')
                q_accs, cnt = cal_acc_class(q.long(), t.long(), bin_y)

                acc_mean.append(acc)
                precision_mean.append(precision)
                recall_mean.append(recall)
                f1_mean.append(f1)

                loss = binary_cross_entropy(y, t)
                loss_mean.append(loss)

                if loss > best_loss:
                    patience_check += 1

                    if patience_check >= patience_limit:
                        break
                else:
                    best_loss = loss
                    patience_check = 0

                if auc > max_auc :
                    torch.save(
                        model.state_dict(),
                        os.path.join(
                            ckpt_path, "model.ckpt"
                        )
                    )
                    max_auc = auc
                    with open(os.path.join(ckpt_path, f"best_val_auc.pkl"), "wb") as f:
                        best_pef = {"seed": wandb.config.seed, \
                                    "dropout": wandb.config.dropout, \
                                    "lr": wandb.config.learning_rate, \
                                    'dim_s': {'values': [20, 50]}, \
                                    'size_m': {'values': [20, 50]}
                                    # "emb_size": wandb.config.emb_size, \
                                    # "hidden_size": wandb.config.hidden_size \
                                    }
                        pickle.dump(best_pef, f)

            loss_mean = np.mean(loss_mean)
            auc_mean = np.mean(auc_mean)
            acc_mean = np.mean(acc_mean)
            precision_mean = np.mean(precision_mean)
            recall_mean = np.mean(recall_mean)
            f1_mean = np.mean(f1_mean)

            if(use_wandb != False):
                wandb.log(
                {
                    "epoch": epoch,
                    "val_auc": auc_mean,
                    "val_acc": acc_mean,
                    "val_loss": loss_mean
                })

            aucs.append(auc_mean)
            loss_means.append(loss_mean)
            accs.append(acc_mean)
            precisions.append(precision_mean)
            recalls.append(recall_mean)
            f1s.append(f1_mean)
            print(f"[Valid] Epoch: {epoch} Result: AUC: {auc_mean}, ACC: {acc_mean}, loss: {loss_mean}")

        print(f"========== Finished Epoch: {epoch} ============")

    return aucs, loss_means, accs, q_accs, cnt, precisions, recalls, f1s

# Test function
def test_model(model, test_loader, num_q, ckpt_path, mode=0, use_wandb=False):

    if(use_wandb != False):
        wandb.init(group=f"test_{date.today().isoformat()}_{mode}", name=f"{mode}", reinit=True)

    # 실제 성능측정, mode 0은 현재 답안 예측, 1은 다음 답안 예측, 2는 스코어 예측
    model.load_state_dict(torch.load(os.path.join(ckpt_path, "model.ckpt")))
    loss_mean = []
    aucs = []
    accs = []
    precisions = []
    recalls = []
    f1s = []

    with torch.no_grad():
        for i, data in enumerate(test_loader, 0):
            q, r, qshft_seqs, rshft_seqs, m, bert_s, bert_t, bert_m, q2diff_seqs, pid_seqs, pidshift, hint_seqs = data

            # 현재 답안 예측
            inpt_q = q.long()
            pred_t = r
            if mode == 1: # 다음 답안 예측
                inpt_q = qshft_seqs.long()
                pred_t = rshft_seqs
            elif mode == 2: # 스코어 예측
                pred_t = pid_seqs
            elif mode == 3: # 다음 스코어 예측
                inpt_q = qshft_seqs.long()
                pred_t = pidshift

            model.eval()
            y, Mv = model(q.long(), r.long())
            # y = (y * one_hot(inpt_q, num_q)).sum(-1)

            # y와 t 변수에 있는 행렬들에서 마스킹이 true로 된 값들만 불러옴
            q = torch.masked_select(inpt_q, m).detach().cpu()
            y = torch.masked_select(y, m).detach().cpu()
            t = torch.masked_select(pred_t, m).detach().cpu()
            h = torch.masked_select(hint_seqs, m).detach().cpu()

            auc = metrics.roc_auc_score(
                y_true=t.numpy(), y_score=y.numpy()
            )
            bin_y = [1 if p >= 0.5 else 0 for p in y.detach().cpu().numpy()]
            acc = metrics.accuracy_score(t.detach().cpu().numpy(), bin_y)
            precision = metrics.precision_score(t.numpy(), bin_y, average='binary')
            recall = metrics.recall_score(t.numpy(), bin_y, average='binary')
            f1 = metrics.f1_score(t.numpy(), bin_y, average='binary')

            loss = binary_cross_entropy(y, t)

            print(f"[Test] number: {i}, AUC: {auc}, ACC: {acc}, loss: {loss}")

            aucs.append(auc)
            loss_mean.append(loss)
            accs.append(acc)
            q_accs, cnt = cal_acc_class(q.long(), t.long(), bin_y)
            precisions.append(precision)
            recalls.append(recall)
            f1s.append(f1)

        loss_means = np.mean(loss_mean) # 실제 로스 평균값을 구함
        auc_mean = np.mean(aucs)
        acc_mean = np.mean(accs)
        f1_mean = np.mean(f1s)

        if(use_wandb != False):
            wandb.log(
            {
                "test_auc": auc_mean,
                "test_acc": acc_mean,
                "test_loss": loss_mean
            })
            wandb.finish()

        print(f"[Total Test]: AUC: {auc_mean}, ACC: {acc_mean}, F1 Score: {f1_mean} ")

    return aucs, loss_means, accs, q_accs, cnt, precisions, recalls, f1s

# main program
def main(model_name, dataset_name, use_wandb):
    KAKAO_CKPTS = "/app/outputs/"
    # KAKAO_CKPTS = "./"
    if not os.path.isdir(f"{KAKAO_CKPTS}ckpts"): # original: ckpts
        os.mkdir(f"{KAKAO_CKPTS}ckpts")
    ckpt_path = os.path.join(f"{KAKAO_CKPTS}ckpts", model_name)
    if not os.path.isdir(ckpt_path):
        os.mkdir(ckpt_path)

    ckpt_path = os.path.join(ckpt_path, dataset_name)
    if not os.path.isdir(ckpt_path):
        os.mkdir(ckpt_path)

    with open("config.json") as f:
        config = json.load(f)
        model_config = config[model_name]
        train_config = config["train_config"]

    with open("wandb_config.json") as f:
        wandb_config = json.load(f)
        WANDB_API_KEY = wandb_config['wandb_api_key']

    batch_size = train_config["batch_size"]
    num_epochs = train_config["num_epochs"]
    train_ratio = train_config["train_ratio"]
    learning_rate = train_config["learning_rate"]
    optimizer = train_config["optimizer"] # can be sgd, adam
    seq_len = train_config["seq_len"] # 샘플링 할 갯수

    # 데이터셋 추가 가능
    collate_pt = collate_fn
    if dataset_name == "ASSIST2009":
        dataset = ASSIST2009(seq_len)
    elif dataset_name == "ASSIST2012":
        dataset = ASSIST2012(seq_len)
    elif dataset_name == "EDNET01":
        dataset = EdNet01(seq_len)
        collate_pt = collate_ednet
    elif dataset_name == "CSEDM":
        dataset = CSEDM(seq_len)

    if torch.cuda.is_available():
        device = "cuda"
    else:
        device = "cpu"

    # JSON으로 설정값 저장
    with open(os.path.join(ckpt_path, "model_config.json"), "w") as f:
        json.dump(model_config, f, indent=4)
    with open(os.path.join(ckpt_path, "train_config.json"), "w") as f:
        json.dump(train_config, f, indent=4)

    ## 가변 벡터이므로 **
    train_model = None
    test_model = None
    if model_name == "dkt":
        model = torch.nn.DataParallel(DKT(dataset.num_q, **model_config)).to(device)
        train_model = dkt_train
        test_model = dkt_test
    elif model_name == "dkf":
        model = torch.nn.DataParallel(DKT_F(dataset.num_q, **model_config)).to(device)
    elif model_name == "dkt-":
        model = torch.nn.DataParallel(DKT_FRONT(dataset.num_q, **model_config)).to(device)
    elif model_name == "dkt+":
        model = torch.nn.DataParallel(DKT_REAR(dataset.num_q, **model_config)).to(device)
    elif model_name == 'dkvmn':
        model = torch.nn.DataParallel(DKVMN(dataset.num_q, **model_config)).to(device)
        train_model = dkvmn_train
        test_model = dkvmn_test
    elif model_name == 'dkvmn+':
        model = torch.nn.DataParallel(SUBJ_DKVMN(dataset.num_q, num_qid=dataset.num_pid, **model_config)).to(device)
    elif model_name == 'dkvmn-':
        model = torch.nn.DataParallel(BACK_DKVMN(dataset.num_q, **model_config)).to(device)
    elif model_name == 'sakt':
        model = torch.nn.DataParallel(SAKT(dataset.num_q, **model_config)).to(device)
        train_model = sakt_train
        test_model = sakt_test
    elif model_name == 'sakt-':
        model = torch.nn.DataParallel(SAKT_FRONT(dataset.num_q, **model_config)).to(device)
    elif model_name == 'sakt+':
        model = torch.nn.DataParallel(SAKT_REAR(dataset.num_q, **model_config)).to(device)
    elif model_name == 'saint':
        model = torch.nn.DataParallel(SAINT(dataset.num_q, **model_config)).to(device)
        train_model = saint_train
        test_model = saint_test
    elif model_name == 'saint-':
        model = torch.nn.DataParallel(SAINT_FRONT(dataset.num_q, **model_config)).to(device)
    elif model_name == "saint+":
        model = torch.nn.DataParallel(SAINT_REAR(dataset.num_q, **model_config)).to(device)
    elif model_name == 'akt':
        model = torch.nn.DataParallel(AKT(n_question=dataset.num_q, n_pid=dataset.num_pid, **model_config)).to(device)
        train_model = akt_train
        test_model = akt_test
    elif model_name == "auto":
        model = torch.nn.DataParallel(AUTO(num_q=dataset.num_q, **model_config)).to(device)
    elif model_name == "mekt":
        model = MEKT(dataset.num_q, **model_config).to(device)
    elif model_name == "dirt":
        model = DeepIRT(dataset.num_q, dataset.num_u, **model_config).to(device)
    elif model_name == "qakt":
        model = QAKT(dataset.num_q, **model_config).to(device)

    else:
        print(f"The wrong model name was used...: {model_name}")
        return

    # 데이터셋 분할
    data_size = len(dataset)
    train_size = int(data_size * train_ratio)
    valid_size = int(data_size * ((1.0 - train_ratio) / 2.0))
    test_size = data_size - train_size - valid_size

    train_dataset, valid_dataset, test_dataset = random_split(
        dataset, [train_size, valid_size, test_size], generator=torch.Generator(device=device)
    )

    # 연결
    tv_dataset = ConcatDataset([train_dataset, valid_dataset])

    # pickle에 얼마만큼 분할했는지 저장
    if os.path.exists(os.path.join(dataset.dataset_dir, "train_indices.pkl")):
        with open(
            os.path.join(dataset.dataset_dir, "train_indices.pkl"), "rb"
        ) as f:
            train_dataset.indices = pickle.load(f)
        with open(
            os.path.join(dataset.dataset_dir, "valid_indicies.pkl"), "rb"
        ) as f:
            valid_dataset.indices = pickle.load(f)
        with open(
            os.path.join(dataset.dataset_dir, "test_indices.pkl"), "rb"
        ) as f:
            test_dataset.indices = pickle.load(f)
    else:
        with open(
            os.path.join(dataset.dataset_dir, "train_indices.pkl"), "wb"
        ) as f:
            pickle.dump(train_dataset.indices, f)
        with open(
            os.path.join(dataset.dataset_dir, "valid_indicies.pkl"), "wb"
        ) as f:
            pickle.dump(valid_dataset.indices, f)
        with open(
            os.path.join(dataset.dataset_dir, "test_indices.pkl"), "wb"
        ) as f:
            pickle.dump(test_dataset.indices, f)

    if optimizer == "sgd":
        opt = SGD(model.parameters(), learning_rate, momentum=0.9)
    elif optimizer == "adam":
        opt = Adam(model.parameters(), learning_rate)
    lr_scheduler = torch.optim.lr_scheduler.ExponentialLR(opt, gamma=0.5)
    opt.lr_scheduler = lr_scheduler

    # 현재꺼 예측, 다음꺼 예측, CSEDM 현재꺼 예측, 다음꺼 예측 이렇게 4개로 디자인
    pred_next = ["dkt", "dkt+", "dkt-", "sakt", "sakt+", "sakt-"]
    pred_now  = ["dkvmn", "dkvmn+", "dkvmn-", "akt", "saint", "saint+", "saint-"]

    mode = 0
    if model_name in pred_next:
        mode = 1
    elif model_name in pred_now and dataset_name == "CSEDM":
        mode = 2
    elif model_name in pred_next and dataset_name == "CSEDM":
        mode = 3

    # IIFE 즉시 실행 함수로 패킹해서 wandb로 넘겨줌
    def train_main():
        proj_name = f"{model_name}_{dataset_name}"
        num_epochs = train_config["num_epochs"]
        kfold = KFold(n_splits=5, shuffle=True)

        for fold, (train_ids, valid_ids) in enumerate(kfold.split(tv_dataset)):
            fold += 1
            print(f"========={fold}==========")
            model.apply(reset_weight)

            cv_name = f"{wandb.util.generate_id()}"
            run_name = f"{date.today().isoformat()}-{cv_name}-{fold:02}-runs"
            run = wandb.init(group=f"cv_{cv_name}_{fold}", name=run_name, reinit=True)

            seed = wandb.config.seed
            random.seed(seed)
            np.random.seed(seed)
            torch.manual_seed(seed)
            torch.cuda.manual_seed_all(seed)

            assert run is not None
            assert type(run) is wandb.sdk.wandb_run.Run
            wandb.summary["cv_fold"] = fold
            wandb.summary["num_cv_folds"] = kfold.n_splits
            wandb.summary["cv_random_state"] = kfold.random_state

            num_epochs = wandb.config.epochs
            opt.param_groups[0]['lr'] = wandb.config.learning_rate

            # 모델 파라미터
            # model.hidden_size = wandb.config.hidden_size
            model.dropout = wandb.config.dropout

            # Sample elements randomly from a given list of ids, no replacement.
            train_subsampler = torch.utils.data.SubsetRandomSampler(train_ids)
            valid_subsampler = torch.utils.data.SubsetRandomSampler(valid_ids)

            # Loader에 데이터 적재
            train_loader = DataLoader(
                tv_dataset, batch_size=batch_size,
                collate_fn=collate_pt, generator=torch.Generator(device=device),
                sampler=train_subsampler
            )
            valid_loader = DataLoader(
                tv_dataset, batch_size=batch_size,
                collate_fn=collate_pt, generator=torch.Generator(device=device),
                sampler=valid_subsampler
            )

            # 모델에서 미리 정의한 함수로 AUCS와 LOSS 계산
            # auc, loss_mean, acc, q_acc, q_cnt, precision, recall, f1 = \
            aucs, loss_means, accs, q_accs, cnt, precisions, recalls, f1s = train_model(
                model, train_loader, valid_loader, dataset.num_q, num_epochs, opt, ckpt_path, mode, use_wandb
            )

            with open(os.path.join(ckpt_path, f"{fold}_{seq_len}_aucs_{seed}.pkl"), "wb") as f:
                pickle.dump(aucs, f)
            with open(os.path.join(ckpt_path, f"{fold}_{seq_len}_loss_means_{seed}.pkl"), "wb") as f:
                pickle.dump(loss_means, f)
            with open(os.path.join(ckpt_path, f"{fold}_{seq_len}_accs_{seed}.pkl"), "wb") as f:
                pickle.dump(accs, f)
            with open(os.path.join(ckpt_path, f"{fold}_{seq_len}_q_accs_{seed}.pkl"), "wb") as f:
                pickle.dump(q_accs, f)
            with open(os.path.join(ckpt_path, f"{fold}_{seq_len}_q_cnts_{seed}.pkl"), "wb") as f:
                pickle.dump(cnt, f)
            # precisions, recalls, f1s
            with open(os.path.join(ckpt_path, f"{fold}_{seq_len}_precisions_{seed}.pkl"), "wb") as f:
                pickle.dump(precisions, f)
            with open(os.path.join(ckpt_path, f"{fold}_{seq_len}_recalls_{seed}.pkl"), "wb") as f:
                pickle.dump(recalls, f)
            with open(os.path.join(ckpt_path, f"{fold}_{seq_len}_f1s_{seed}.pkl"), "wb") as f:
                pickle.dump(f1s, f)

            wandb.finish()

            # DKT나 다른 모델 학습용
            # aucs, loss_means = model.train_model(train_loader, test_loader, num_epochs, opt, ckpt_path)

    if use_wandb == True:
        # wandb setting
        os.environ['WANDB_API_KEY'] = WANDB_API_KEY
        proj_name = f"{model_name}_{dataset_name}"

        # sweep config // optimization을 위한
        sweep_config = {
            'method': 'grid',
            'name': f'DKT-{model_name}',
            'metric': {
                'name': 'val_auc',
                'goal': 'maximize'
            },
            'parameters': {
                'epochs': {'values': [300]},
                'seed': {'values': [13, 42]},
                'dropout': {'values': [0, 0.05, 0.1, 0.15, 0.2, 0.25]},
                'learning_rate': {'values': [1e-3, 1e-4]},
                # 'dim_s': {'values': [20, 50]},
                # 'size_m': {'values': [20, 50]}
                # 'emb_size': {'values': [256, 512]},
                'hidden_size': {'values': [256, 512]}
            }
        }

        sweep_id = wandb.sweep(sweep=sweep_config, project=proj_name)
        wandb.agent(sweep_id, function=train_main, project=proj_name)
    else:
        train_loader = DataLoader(
            train_dataset, batch_size=batch_size,
            collate_fn=collate_pt, generator=torch.Generator(device=device)
        )
        valid_loader = DataLoader(
            valid_dataset, batch_size=batch_size,
            collate_fn=collate_pt, generator=torch.Generator(device=device)
        )
        train_model(
            model, train_loader, valid_loader, dataset.num_q, num_epochs, opt, ckpt_path, mode, use_wandb
        )

    # 마지막 테스트
    test_loader = DataLoader(
    test_dataset, batch_size=batch_size,
    collate_fn=collate_pt, generator=torch.Generator(device=device),
    )

    auc, loss_mean, acc, q_acc, q_cnt, precision, recall, f1 = \
    test_model(
        model, test_loader, dataset.num_q, ckpt_path, mode, use_wandb
    )

    with open(os.path.join(ckpt_path, f"{optimizer}_{seq_len}_aucs_{seed}.pkl"), "wb") as f:
        pickle.dump(auc, f)
    with open(os.path.join(ckpt_path, f"{optimizer}_{seq_len}_loss_means_{seed}.pkl"), "wb") as f:
        pickle.dump(loss_mean, f)
    with open(os.path.join(ckpt_path, f"{optimizer}_{seq_len}_accs_{seed}.pkl"), "wb") as f:
        pickle.dump(acc, f)
    with open(os.path.join(ckpt_path, f"{optimizer}_{seq_len}_q_accs_{seed}.pkl"), "wb") as f:
        pickle.dump(q_acc, f)
    with open(os.path.join(ckpt_path, f"{optimizer}_{seq_len}_q_cnts_{seed}.pkl"), "wb") as f:
        pickle.dump(q_cnt, f)

    # precisions, recalls, f1s
    with open(os.path.join(ckpt_path, f"{optimizer}_{seq_len}_precisions_{seed}.pkl"), "wb") as f:
        pickle.dump(precision, f)
    with open(os.path.join(ckpt_path, f"{optimizer}_{seq_len}_recalls_{seed}.pkl"), "wb") as f:
        pickle.dump(recall, f)
    with open(os.path.join(ckpt_path, f"{optimizer}_{seq_len}_f1s_{seed}.pkl"), "wb") as f:
        pickle.dump(f1, f)

# program main entry point
if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument(
        "--model_name",
        type=str,
        default="dkt",
        help="The name of the model to train. \
            The possible models are in [dkt]. \
            The default model is dkt."
    )
    parser.add_argument(
        '--dataset_name',
        type=str,
        default="ASSIST2009",
        help="The name of the dataset to use in training. \
            The possible datasets are in \
            [ASSIST2009]. \
            The default dataset is ASSIST2009."
    )

    # true_list 에 없으면 자동적으로 false가 됨.
    true_list = ['true', 'yes', '1', 't', 'y']

    parser.add_argument(
        '--use_wandb',
        type= lambda s: s.lower() in true_list,
        default=False,
        help="This option value is using wandb"
    )

    args = parser.parse_args()

    main(args.model_name, args.dataset_name, args.use_wandb)


import os
import json
import time
from PIL import Image
from typing import Dict
from collections import Counter
import random
# import cv2
import obonet
import networkx
import pandas as pd
import numpy as np
import plotly.express as px
import plotly.graph_objects as go
import matplotlib.pyplot as plt
import matplotlib.patches as mpatch
from Bio import SeqIO
from pyvis.network import Network

CAFA5 = "/Users/harrietobrien/Desktop/CAFA5"
TRAIN = "{home}/Train".format(home=CAFA5)

class CFG:
    gobasic = "{path}/go-basic.obo".format(path=TRAIN)
    sequences = "{path}/train_sequences.fasta".format(path=TRAIN)
    terms = "{path}/train_terms.tsv".format(path=TRAIN)
    taxonomy = "{path}/train_taxonomy.tsv".format(path=TRAIN)
    ia = "{path}/IA.txt".format(path=CAFA5)

class color:
    PURPLE = '\033[95m'
    CYAN = '\033[96m'
    DARKCYAN = '\033[36m'
    BLUE = '\033[94m'
    GREEN = '\033[92m'
    YELLOW = '\033[93m'
    RED = '\033[91m'
    BOLD = '\033[1m'
    UNDERLINE = '\033[4m'
    END = '\033[0m'

# obonet package to load this file
graph = obonet.read_obo(CFG.gobasic)
print(f"Number of nodes: {len(graph)}")
print(f"Number of edges: {graph.number_of_edges()}")
term = "GO:0034655"
# nucleobase-containing compound catabolic process node properties
print(graph.nodes[term])
print(type(graph))  # --> <class 'networkx.classes.multidigraph.MultiDiGraph'>
print(type(graph.nodes[term]))  # --> <class 'dict'>
# plt.plot_dag(graph, term, radius=1)
# plot_dag(graph, term, radius=1000)

# analyze  protein sequences from sequences file with Biopython package
print("Sequence example:\n\n", next(iter(SeqIO.parse(CFG.sequences, "fasta"))))
#  count the number of sequences
sequences = SeqIO.parse(CFG.sequences, "fasta")
num_sequences = sum(1 for seq in sequences)
print("Number of sequences:", num_sequences)

# plot the length distribution of the protein sequences
sequences = SeqIO.parse(CFG.sequences, "fasta")
# get the length of each sequence
lengths = [len(seq) for seq in sequences]
fig = px.histogram(x=lengths, nbins=1000, color_discrete_sequence=['goldenrod'])
fig.update_layout(
    title={
        'text': "Distribution of protein sequence lengths",
        'y': 0.95,
        'x': 0.5,
        'xanchor': 'center',
        'yanchor': 'top'
    },
    xaxis_title="Sequence length", yaxis_title="Count"
)
fig.show()
np.percentile(lengths, 99)

# calculate the amino acid composition of each protein sequence
records = SeqIO.parse(CFG.sequences, "fasta")
# create a list of all amino acids in the sequences
aa_list = [aa for record in records for aa in record.seq]
# count the frequency of each amino acid
aa_count = Counter(aa_list)
fig = px.bar(
    x=list(aa_count.values()), y=list(aa_count.keys()),
    color_discrete_sequence=['darkslateblue'],
    orientation='h', height=700
)
fig.update_layout(
    title={
        'text': "Amino Acid Composition",
        'y': 0.95,
        'x': 0.5,
        'xanchor': 'center',
        'yanchor': 'top'
    },
    xaxis_title="Frequency", yaxis_title="Amino Acid"
)
fig.show()
# Load the train terms dataframe
train_terms = pd.read_csv(CFG.terms, sep="\t")
train_terms.head()
train_terms.describe()
# TODO: Join train_terms_df.EntryID on train_taxonomy_df.EntryID
# Information accretion
limit = 10

with open(CFG.ia) as f:
    ia_weights = [x.replace("\n", "").split("\t") for x in f.readlines()]

ia_weights[:limit]


'''
This script continuously scans the 'New' posts on reddit.com/r/2007scape for submissions that are related
to the location of a dragon or lucky impling. If the submission's title or text contains at least one specific
keyword from a related set, an e-mail will be sent via gmail to the user specified in authentication.py.
Created by Dustin Kieler | /u/Chassy13 on July 3rd, 2018.
'''

import authentication
import configparser
import praw
import smtplib
from email.mime.text import MIMEText

# These keywords should all be lowercase only.
keywords = ['impling', 'dragon imp', 'drag imp', 'drg imp', 'dr imp', 'd imp', 'l imp', 'luck imp', 'lucky imp']

# Since we are just reading the New posts, we will create a Read-Only instance rather than
# using our own account or creating an account specifically for the bot.
# We do not need to use an infinite loop because the PRAW API will continuously grab the streamed submissions by itself.
def main():
    """Logs into Reddit in read-only mode and begins searching the 'New' posts of /r/2007scape
       for submissions related to dragon and lucky implings."""
    reddit = praw.Reddit(client_id="YOUR_APP_CLIENT_ID",
                         client_secret="YOUR_APP_CLIENT_SECRET",
                         user_agent="YOUR_UNIQUE_USER_AGENT_STRING")
    authentication.create_config()
    print("Now waiting patiently for implings...")
    for submission in reddit.subreddit('2007scape').stream.submissions():
        process_submission(submission)
    print("Submission loop has been terminated - restart needed.")

# We could compare the submission.created_utc time to time.time() if we wanted to only get posts that were created after
# we started running this script, but it's probably worth grabbing up to the last 25 submissions that may contain info
# about an impling because the submission may not have been seen yet.
def process_submission(submission):
    """Searches a submission title and text for keywords related to dragon and lucky implings.
       If a keyword is a match, calls to create and send an e-mail."""
    for keyword in keywords:
        if keyword in submission.title.lower() or keyword in submission.selftext.lower():
            email = create_email(submission.title, submission.selftext, submission.url)
            send_notification_email(email)

def send_notification_email(email):
    """Establishes the mail server and sends the e-mail to the address specified in the configuration."""
    server = smtplib.SMTP_SSL('smtp.gmail.com', 465)
    server.ehlo()
    server.login(email['From'], get_credential('Key'))
    server.sendmail(email['From'], email['To'], email.as_string())
    server.quit()
    print("E-mail sent!")

def create_email(sub_title, sub_text, sub_url):
    """Creates an e-mail containing the impling post's title and text.
       The TO and FROM fields are the e-mail address in the configuration."""
    msg = MIMEText("Title: " + sub_title + "\n" + "Text: " + sub_text + "\n" + "URL: " + sub_url)
    address = get_credential('Address')
    msg['Subject'] = 'POTENTIAL IMPLING ALERT'
    msg['From'] = address
    msg['To'] = address
    return msg

# There is probably a cleaner way to do this in the future if we ever wanted to retrieve more properties,
# such as opening the configparser just one time and then passing along the property and/or tag to allow for more
# general use.
def get_credential(tag):
    """Searches the [CREDENTIALS] in the configuration for a specific property (tag)."""
    config = configparser.ConfigParser()
    config.read('auth.ini')
    return config['CREDENTIALS'][tag]

if __name__ == '__main__':
    main()


from flask import current_app

from archivy.helpers import get_elastic_client

def add_to_index(model):
    """
    Adds dataobj to given index. If object of given id already exists, it will be updated.

    Params:

    - **index** - String of the ES Index. Archivy uses `dataobj` by default.
    - **model** - Instance of `archivy.models.Dataobj`, the object you want to index.
    """
    es = get_elastic_client()
    if not es:
        return
    payload = {}
    for field in model.__searchable__:
        payload[field] = getattr(model, field)
    es.index(index=current_app.config["SEARCH_CONF"]["index_name"], id=model.id, body=payload)
    return True

def remove_from_index(dataobj_id):
    """Removes object of given id"""
    es = get_elastic_client()
    if not es:
        return
    es.delete(index=current_app.config["SEARCH_CONF"]["index_name"], id=dataobj_id)

def query_index(query):
    """Returns search results for your given query"""
    es = get_elastic_client()
    if not es:
        return []
    search = es.search(
        index=current_app.config["SEARCH_CONF"]["index_name"],
        body={
            "query": {
                "multi_match": {
                    "query": query,
                    "fields": ["*"],
                    "analyzer": "rebuilt_standard"
                }
            },
            "highlight": {
                "fragment_size": 0,
                "fields": {
                    "content": {
                        "pre_tags": "==",
                        "post_tags": "==",
                    }
                }
            }
        }
    )

    hits = []
    for hit in search["hits"]["hits"]:
        formatted_hit = {"id": hit["_id"], "title": hit["_source"]["title"]}
        if "highlight" in hit:
            formatted_hit["highlight"] = hit["highlight"]["content"]
        hits.append(formatted_hit)

    return hits


from discord.ext import commands
from discord import activity

from secrets import token
from util.cogUtil import load_all_cogs, reload_all_cogs
from util.util import (quick_response, error_response)

from util.checks import is_owner

bot = commands.Bot('.')

@bot.event
async def on_ready():
    """On ready event to setup the bot"""
    await bot.change_presence(activity=activity.Game(name="Help: .help"))

    print('Online!')

@bot.command()
@commands.check(is_owner)
async def load(ctx: discord.ext.context, extension: str):
    """Load command for cogs"""
    bot.load_extension(f'cogs.{extension}')
    try:
        await quick_response(ctx.channel, f'{extension.title()} loaded.')
    except commands.ExtensionNotLoaded or commands.ExtensionNotFound:
        await error_response(ctx, 'Failed to load!')
        return

@bot.command()
@commands.check(is_owner)
async def unload(ctx, extension: str):
    """Unload command for cogs"""
    bot.unload_extension(f'cogs.{extension}')
    try:
        await quick_response(ctx.channel, f'{extension.title()} unloaded.')
    except commands.ExtensionNotLoaded or commands.ExtensionNotFound:
        await error_response(ctx, 'Failed to unload!')
        return

@bot.command()
@commands.check(is_owner)
async def reload(ctx, extension=None):
    """Reload command for cogs"""
    extensions_loaded = []
    if extension == "all" or extension is None:
        extensions_loaded = reload_all_cogs(bot)
    else:
        try:
            bot.reload_extension(f'cogs.{extension}')
            extensions_loaded.append(extension)
        except commands.ExtensionNotLoaded or commands.ExtensionNotFound:
            await error_response(ctx, 'Failed to reload!')
            return
    await quick_response(ctx.channel, f'{str.join(", ", extensions_loaded)} loaded!')

# load the cogs when the bot starts
load_all_cogs(bot)
# run the bot
bot.run(token)


from typing import Tuple, Optional, Sequence

import torch

from deep_opt import NeuralNetwork, Property
from nn_repair.counterexamples import CounterexampleGenerator, Counterexample

class FastGradientSignMethod(CounterexampleGenerator):
    """
    Generates counterexamples/adversarial examples by walking into the direction
    of the signs of the gradients until hitting the bounds.

    This attack has been introduced in [Goodfellow2015]_.
    Here it is generalized in a way to work for arbitrary properties
    using the satisfaction function.

    .. [Goodfellow2015] Goodfellow, Ian J. / Shlens, Jonathon / Szegedy, Christian
     Explaining and Harnessing Adversarial Examples
     2015
     ICLR (Poster)
    """

    def __init__(self):
        """
        Creates a new FastGradientSignMethod (FGSM) for generating counterexamples.
        """
        pass

    @property
    def name(self) -> str:
        return 'FGSM'

    def find_counterexample(self, target_network: NeuralNetwork, target_property: Property) \
            -> Tuple[Optional[Sequence[Counterexample]], str]:
        if target_property.input_constraint is not None:
            raise ValueError("FGSM can not handle Properties with additional input constraints.")

        # start at the center point of the property
        bounds = target_property.input_bounds(target_network)
        lower = torch.tensor([lower for lower, _ in bounds])
        upper = torch.tensor([upper for _, upper in bounds])

        x = (lower + (upper-lower)/2).clone().detach().requires_grad_()
        # network also accepts flat inputs (but satisfaction function requires a batch dimension)
        violation = target_property.satisfaction_function(x.unsqueeze(0), target_network)
        violation.backward(inputs=[x])
        gradient_sign = -x.grad.sign()

        cx_candidate = torch.where(gradient_sign < 0, lower, upper)
        # reset places where the gradient is zero
        cx_candidate = gradient_sign.abs() * cx_candidate + (1 - gradient_sign.abs()) * x

        _, network_outputs, satisfaction, is_sat = \
            target_property.full_witness(cx_candidate.unsqueeze(0), target_network)
        if not is_sat:
            cx = Counterexample(inputs=cx_candidate.detach().numpy(),
                                network_outputs=network_outputs,
                                property_satisfaction=satisfaction.item(),
                                property=target_property)
            return (cx, ), 'Violated'
        else:
            return (), 'Unknown'


from fabric.api import *
from fabric.contrib.files import exists
from fabric.contrib.project import *
from time import time
from datetime import datetime
from sys import exit

env.hosts = [
    '176.31.251.100'
]
env.user = 'root' # should be changed asap

def check_environment(env_name):
    existing_environments = ['staging', 'prod']
    if env_name not in existing_environments:
        puts('%s is not a valid environment' % env_name)
        puts('Allowed environments: %s' % ', '.join(existing_environments))
        exit(1)

def deploy(env_name):
    check_environment(env_name)

    deploy_timestamp = datetime.utcfromtimestamp(int(time())).strftime('%Y-%b-%d-at-%H:%M:%S')

    deployment_directory='/home/deploy/%s/releases/release-%s' % (env_name, deploy_timestamp)

    upload_code(deployment_directory)
    enable_project(env_name, deployment_directory)

def upload_code(deployment_directory):
    if not exists(deployment_directory):
       puts('creating deployment directory')
       run('mkdir -p %s' % deployment_directory)

    puts('Deploying project in %s' % deployment_directory)
    rsync_project(
        local_dir='./app/*',
        remote_dir=deployment_directory,
        default_opts='-azcrSh --stats',
        exclude=['.git', 'var', 'node_modules', 'vendor']
    )

def enable_project(env_name, deployment_directory):
    # Copy of the environment file
    run('cp /home/deploy/%s/shared/env %s/.env' % (env_name, deployment_directory))

    # warmup cache
    with cd(deployment_directory):
        run('time composer install --optimize-autoloader')
        run('chmod a+w var/ -R')
        if env_name == "staging":
            run('bin/console cache:clear --env=dev --no-debug')
            run('bin/console cache:warmup --env=dev --no-debug')
        else:
            run('bin/console cache:clear --env=prod --no-debug')
            run('bin/console cache:warmup --env=prod --no-debug')
        run('chmod a+w var/ -R')
        run('time bin/console doctrine:migrations:migrate --no-interaction')
        run('cp assets/static public/build/static -R')

    # Create symlink
    run('ln -sfn %s /var/www/pc-%s' % (deployment_directory, env_name))
    # Restart the necessary services
    run('service php7.3-fpm restart')
    run('service nginx restart')

def list_releases(env_name):
    check_environment(env_name)
    run('find /home/deploy/%s/releases -maxdepth 1 -printf "%%f\n"' % (env_name))

def set_release(env_name, release):
    # Create symlink
    deployment_directory='/home/deploy/%s/releases/%s' % (env_name, release)

    run('ln -sfn %s /var/www/pc-%s' % (deployment_directory, env_name))
    # Restart the necessary services
    run('service php7.3-fpm restart')
    run('service nginx restart')


from flask import Flask
from flask import request

app=Flask(__name__)

@app.route("/operBas",methods=["GET","POST"])
def operacion():
    if request.method == "POST":
        num1 = request.form.get("num1")
        num2 = request.form.get("num2")
        if request.form.get('button') == 'sum':
            return "
 La suma es: {}
".format(str(int(num1)+int(num2)))
        elif request.form.get('button') == 'res':
            return "
 La resta es: {}
".format(str(int(num1)-int(num2)))
        elif request.form.get('button') == 'mult':
            return "
 La multiplicacion es: {}
".format(str(int(num1)*int(num2)))
        elif request.form.get('button') == 'div':
            return "
 La division es: {}
".format(str(int(num1)/int(num2)))
    else:
        return'''
        <form action = "/operBas" method="POST">
        Ingrese los números para su operación

        N1:
        <input type="text" name="num1"/>

        N2:
        <input type="text" name="num2"/>

        Eligue la operacion

        Suma
        <input type="radio" name="button" value="sum">

        Resta
        <input type="radio" name="button" value="res">

        Multiplicacion
        <input type="radio" name="button" value="mult">

        Division
        <input type="radio" name="button" value="div">

        <button type="submit" name="Calcular">Calcular</button>
        </form>
        '''

if __name__ == "__main__":
    app.run(debug=True,port=3000)


"""Unit tests for antibody."""

import logging

import pandas as pd
import pytest
from pkg_resources import resource_filename

from sadie import antibody
from sadie.antibody import exception, genetable, segment

logger = logging.getLogger()

def fixture_file(file):
    """Helper method for test execution."""
    return resource_filename(__name__, "fixtures/{}".format(file))

def test_gene_table():
    """
    testing the gene table class
    """
    fixture_v_gene = (
        pd.read_csv(fixture_file("VSEGMENT.csv.gz"), index_col=0)
        .fillna("")
        .sort_values(["species", "full"])
        .reset_index(drop=True)
    )
    v_gene_table = genetable.VGeneTable()

    # Assert everything is equal between our fixture and test frame
    pd._testing.assert_frame_equal(fixture_v_gene, v_gene_table.gene_table)

    assert list(v_gene_table.available_species) == [
        "alpaca",
        "boar",
        "camel",
        "cat",
        "catfish",
        "cod",
        "cow",
        "crabmacaque",
        "dog",
        "dolphin",
        "ferret",
        "goat",
        "horse",
        "human",
        "junglefowl",
        "macaque",
        "mouse",
        "nhp",
        "night_monkey",
        "platypus",
        "rabbit",
        "rat",
        "salmon",
        "sharks",
        "sheep",
        "teleosts",
        "trout",
        "zebrafish",
    ]

    assert list(v_gene_table.get_available_genes("human", locus="IGL")) == [
        "IGLV(I)-68*01",
        "IGLV(I)-70*01",
        "IGLV(III)-59-1*01",
        "IGLV(IV)-66-1*01",
        "IGLV(V)-66*01",
        "IGLV1-36*01",
        "IGLV1-40*01",
        "IGLV1-41*01",
        "IGLV1-44*01",
        "IGLV1-47*01",
        "IGLV1-50*01",
        "IGLV1-51*01",
        "IGLV1-62*01",
        "IGLV10-54*01",
        "IGLV10-67*01",
        "IGLV11-55*01",
        "IGLV2-11*01",
        "IGLV2-14*01",
        "IGLV2-18*01",
        "IGLV2-23*01",
        "IGLV2-28*01",
        "IGLV2-33*01",
        "IGLV2-34*01",
        "IGLV2-5*01",
        "IGLV2-8*01",
        "IGLV3-1*01",
        "IGLV3-10*01",
        "IGLV3-12*01",
        "IGLV3-13*01",
        "IGLV3-16*01",
        "IGLV3-17*01",
        "IGLV3-19*01",
        "IGLV3-2*01",
        "IGLV3-21*01",
        "IGLV3-22*01",
        "IGLV3-24*01",
        "IGLV3-25*01",
        "IGLV3-26*01",
        "IGLV3-27*01",
        "IGLV3-29*01",
        "IGLV3-30*01",
        "IGLV3-31*01",
        "IGLV3-32*01",
        "IGLV3-6*01",
        "IGLV3-7*01",
        "IGLV3-9*01",
        "IGLV4-3*01",
        "IGLV4-60*01",
        "IGLV4-69*01",
        "IGLV5-37*01",
        "IGLV5-39*01",
        "IGLV5-45*01",
        "IGLV5-48*01",
        "IGLV5-52*01",
        "IGLV6-57*01",
        "IGLV7-35*01",
        "IGLV7-43*01",
        "IGLV7-46*01",
        "IGLV8-61*01",
        "IGLV8/OR8-1*01",
        "IGLV9-49*01",
    ]

    v323_human = v_gene_table.get_gene("IGHV3-23*01", "human")
    assert isinstance(v323_human, pd.Series)
    assert list(v323_human.index) == [
        "gene",
        "fwr1_nt",
        "cdr1_nt",
        "fwr2_nt",
        "cdr2_nt",
        "fwr3_nt",
        "cdr3_nt",
        "fwr1_aa",
        "cdr1_aa",
        "fwr2_aa",
        "cdr2_aa",
        "fwr3_aa",
        "cdr3_aa",
    ]
    with pytest.raises(exception.AmbiguousGene):
        v_gene_table.get_gene("IGHV3-23", "human")
        v_gene_table.get_gene("IGHV1-69", "human")
    with pytest.raises(exception.BadGene):
        v_gene_table.get_gene("IGHV3-700", "human")
        v_gene_table.get_gene("IGHZ3-33", "human")
    with pytest.raises(exception.NoSpecies):
        v_gene_table.get_gene("IGHV3-23", "robot")
        v_gene_table.get_gene("IGHV1-69", "elon")

    fixture_j_gene = (
        pd.read_csv(fixture_file("JSEGMENT.csv.gz"), index_col=0)
        .fillna("")
        .sort_values(["species", "full"])
        .reset_index(drop=True)
    )
    j_gene_table = genetable.JGeneTable()

    # Assert everything is equal between our fixture and test frame
    pd._testing.assert_frame_equal(fixture_j_gene, j_gene_table.gene_table)

    j6_human = j_gene_table.get_gene("IGHJ6*01", "human")
    assert isinstance(j6_human, pd.Series)
    with pytest.raises(exception.AmbiguousGene):
        j_gene_table.get_gene("IGHJ4", "human")
        j_gene_table.get_gene("IGHJ6", "human")
    with pytest.raises(exception.NoSpecies):
        j_gene_table.get_gene("IGHJ4", "robot")

    with pytest.raises(exception.BadGene):
        j_gene_table.get_gene("IGHJ12", "dog")

def test_v_gene():
    # test we can create v gene objects
    v_segments_table_manual = pd.read_csv(fixture_file("VSEGMENT.csv.gz"), index_col=0).fillna("")
    for gb_index, v_gene_df in v_segments_table_manual.groupby(["species", "gene"]):
        species = gb_index[0]
        v_gene = gb_index[1]
        if len(v_gene_df) == 1:
            lookup = v_gene_df.iloc[0]
            # we have a single allelic match
            cdr1_aa = lookup["cdr1_aa"]
            cdr2_aa = lookup["cdr2_aa"]
            cdr3_aa = lookup["cdr3_aa"]
            fwr1_aa = lookup["fwr1_aa"]
            fwr2_aa = lookup["fwr2_aa"]
            fwr3_aa = lookup["fwr3_aa"]
            #  cdr nt
            cdr1_nt = lookup["cdr1_nt"]
            cdr2_nt = lookup["cdr2_nt"]
            cdr3_nt = lookup["cdr3_nt"]
            fwr1_nt = lookup["fwr1_nt"]
            fwr2_nt = lookup["fwr2_nt"]
            fwr3_nt = lookup["fwr3_nt"]
            v_gene_object = antibody.VGene(v_gene, species)

            # Make sure we are getting the right subclass
            assert isinstance(v_gene_object, antibody.VGene)
            assert isinstance(v_gene_object.cdr1_aa, segment.CDR1AA)
            assert isinstance(v_gene_object.cdr2_aa, segment.CDR2AA)
            assert isinstance(v_gene_object.cdr3_aa, segment.CDR3AA)
            assert isinstance(v_gene_object.cdr1_nt, segment.CDR1NT)
            assert isinstance(v_gene_object.cdr2_nt, segment.CDR2NT)
            assert isinstance(v_gene_object.cdr3_nt, segment.CDR3NT)
            assert isinstance(v_gene_object.fwr1_aa, segment.FrameWork1AA)
            assert isinstance(v_gene_object.fwr2_aa, segment.FrameWork2AA)
            assert isinstance(v_gene_object.fwr3_aa, segment.FrameWork3AA)
            assert isinstance(v_gene_object.fwr1_nt, segment.FrameWork1NT)
            assert isinstance(v_gene_object.fwr2_nt, segment.FrameWork2NT)
            assert isinstance(v_gene_object.fwr3_nt, segment.FrameWork3NT)

            # assert we have the same objects strings
            assert v_gene_object.cdr1_aa == cdr1_aa
            assert v_gene_object.cdr2_aa == cdr2_aa
            assert v_gene_object.cdr3_aa == cdr3_aa
            assert v_gene_object.fwr1_aa == fwr1_aa
            assert v_gene_object.fwr2_aa == fwr2_aa
            assert v_gene_object.fwr3_aa == fwr3_aa
            assert v_gene_object.cdr1_nt == cdr1_nt
            assert v_gene_object.cdr2_nt == cdr2_nt
            assert v_gene_object.cdr3_nt == cdr3_nt
            assert v_gene_object.fwr1_nt == fwr1_nt
            assert v_gene_object.fwr2_nt == fwr2_nt
            assert v_gene_object.fwr3_nt == fwr3_nt
        else:
            # else test that we can raise an ambigous gene for those that have more than one allele
            with pytest.raises(exception.AmbiguousGene):
                v_gene_object = antibody.VGene(v_gene, species)
            for index in range(0, len(v_gene_df)):
                lookup = v_gene_df.iloc[index]
                # we have to use the full allel
                v_gene = lookup["full"]
                # we have a single allelic match
                cdr1_aa = lookup["cdr1_aa"]
                cdr2_aa = lookup["cdr2_aa"]
                cdr3_aa = lookup["cdr3_aa"]
                fwr1_aa = lookup["fwr1_aa"]
                fwr2_aa = lookup["fwr2_aa"]
                fwr3_aa = lookup["fwr3_aa"]
                # cdr nt
                cdr1_nt = lookup["cdr1_nt"]
                cdr2_nt = lookup["cdr2_nt"]
                cdr3_nt = lookup["cdr3_nt"]
                fwr1_nt = lookup["fwr1_nt"]
                fwr2_nt = lookup["fwr2_nt"]
                fwr3_nt = lookup["fwr3_nt"]
                v_gene_object = antibody.VGene(v_gene, species)
                # Make sure we are getting the right subclass
                assert isinstance(v_gene_object, antibody.VGene)
                assert isinstance(v_gene_object.cdr1_aa, segment.CDR1AA)
                assert isinstance(v_gene_object.cdr2_aa, segment.CDR2AA)
                assert isinstance(v_gene_object.cdr3_aa, segment.CDR3AA)
                assert isinstance(v_gene_object.cdr1_nt, segment.CDR1NT)
                assert isinstance(v_gene_object.cdr2_nt, segment.CDR2NT)
                assert isinstance(v_gene_object.cdr3_nt, segment.CDR3NT)
                assert isinstance(v_gene_object.fwr1_aa, segment.FrameWork1AA)
                assert isinstance(v_gene_object.fwr2_aa, segment.FrameWork2AA)
                assert isinstance(v_gene_object.fwr3_aa, segment.FrameWork3AA)
                assert isinstance(v_gene_object.fwr1_nt, segment.FrameWork1NT)
                assert isinstance(v_gene_object.fwr2_nt, segment.FrameWork2NT)
                assert isinstance(v_gene_object.fwr3_nt, segment.FrameWork3NT)

                # assert we have the same objects strings
                assert v_gene_object.cdr1_aa == cdr1_aa
                assert v_gene_object.cdr2_aa == cdr2_aa
                assert v_gene_object.cdr3_aa == cdr3_aa
                assert v_gene_object.fwr1_aa == fwr1_aa
                assert v_gene_object.fwr2_aa == fwr2_aa
                assert v_gene_object.fwr3_aa == fwr3_aa
                assert v_gene_object.cdr1_nt == cdr1_nt
                assert v_gene_object.cdr2_nt == cdr2_nt
                assert v_gene_object.cdr3_nt == cdr3_nt
                assert v_gene_object.fwr1_nt == fwr1_nt
                assert v_gene_object.fwr2_nt == fwr2_nt
                assert v_gene_object.fwr3_nt == fwr3_nt

def test_v_gene_numbering():
    v_gene_object = antibody.VGene("IGHV3-23*01", "human")
    assert v_gene_object.fwr1_aa.start_index == 0
    assert v_gene_object.fwr1_aa.start == 1
    assert v_gene_object.fwr1_aa.end == 25
    assert v_gene_object.fwr1_aa.end_index == 24

    assert v_gene_object.cdr1_aa.start_index == 25
    assert v_gene_object.cdr1_aa.start == 26
    assert v_gene_object.cdr1_aa.end == 33
    assert v_gene_object.cdr1_aa.end_index == 32

    assert v_gene_object.fwr2_aa.start_index == 33
    assert v_gene_object.fwr2_aa.start == 34
    assert v_gene_object.fwr2_aa.end == 50
    assert v_gene_object.fwr2_aa.end_index == 49

    assert v_gene_object.cdr2_aa.start_index == 50
    assert v_gene_object.cdr2_aa.start == 51
    assert v_gene_object.cdr2_aa.end == 58
    assert v_gene_object.cdr2_aa.end_index == 57

    assert v_gene_object.fwr3_aa.start_index == 58
    assert v_gene_object.fwr3_aa.start == 59
    assert v_gene_object.fwr3_aa.end == 96
    assert v_gene_object.fwr3_aa.end_index == 95

    assert v_gene_object.fwr1_nt.start_index == 0
    assert v_gene_object.fwr1_nt.start == 1
    assert v_gene_object.fwr1_nt.end == 75
    assert v_gene_object.fwr1_nt.end_index == 74

    assert v_gene_object.cdr1_nt.start_index == 75
    assert v_gene_object.cdr1_nt.start == 76
    assert v_gene_object.cdr1_nt.end == 99
    assert v_gene_object.cdr1_nt.end_index == 98

    assert v_gene_object.fwr2_nt.start_index == 99
    assert v_gene_object.fwr2_nt.start == 100
    assert v_gene_object.fwr2_nt.end == 150
    assert v_gene_object.fwr2_nt.end_index == 149

    assert v_gene_object.cdr2_nt.start_index == 150
    assert v_gene_object.cdr2_nt.start == 151
    assert v_gene_object.cdr2_nt.end == 174
    assert v_gene_object.cdr2_nt.end_index == 173

    assert v_gene_object.fwr3_nt.start_index == 174
    assert v_gene_object.fwr3_nt.start == 175
    assert v_gene_object.fwr3_nt.end == 288
    assert v_gene_object.fwr3_nt.end_index == 287

    rep = "<VGene>human->IGHV3-23*01\n        fwr1_nt-GAGGTGCAGCTGTTGGAGTCTGGGGGAGGCTTGGTACAGCCTGGGGGGTCCCTGAGACTCTCCTGTGCAGCCTCT\n        fwr1_aa-EVQLLESGGGLVQPGGSLRLSCAAS\n        cdr1_nt-GGATTCACCTTTAGCAGCTATGCC\n        cdr1_aa-GFTFSSYA\n        fwr2_nt-ATGAGCTGGGTCCGCCAGGCTCCAGGGAAGGGGCTGGAGTGGGTCTCAGCT\n        fwr2_aa-MSWVRQAPGKGLEWVSA\n        cdr2_nt-ATTAGTGGTAGTGGTGGTAGCACA\n        cdr2_aa-ISGSGGST\n        fwr3_nt-TACTACGCAGACTCCGTGAAGGGCCGGTTCACCATCTCCAGAGACAATTCCAAGAACACGCTGTATCTGCAAATGAACAGCCTGAGAGCCGAGGACACGGCCGTATATTACTGT\n        fwr3_aa-YYADSVKGRFTISRDNSKNTLYLQMNSLRAEDTAVYYC\n        cdr3_nt-GCGAAAGA\n        cdr3_aa-AK\n        "
    assert v_gene_object.__repr__() == rep

    # alternative numbering
    with pytest.raises(NotImplementedError):
        antibody.VGene("IGHV3-23*01", "human", "kabat")
    # assert v_gene_kabat.cdr1_aa == "SYAMS"
    # assert v_gene_kabat.cdr2_aa == "AISGSGGSTYYADSVKG"
    # assert v_gene_kabat.cdr3_aa == "K"

def test_j_gene():
    # test we can create j gene objects
    j_segments_table_manual = pd.read_csv(fixture_file("JSEGMENT.csv.gz"), index_col=0).fillna("")
    for gb_index, j_gene_df in j_segments_table_manual.groupby(["species", "gene"]):
        species = gb_index[0]
        j_gene = gb_index[1]
        if len(j_gene_df) == 1:
            lookup = j_gene_df.iloc[0]
            # we have a single allelic match
            cdr3_nt = lookup["cdr3_nt"]
            cdr3_aa = lookup["cdr3_aa"]

            # FW4
            fwr4_nt = lookup["fwr4_nt"]
            fwr4_aa = lookup["fwr4_aa"]

            # J Gene Object
            j_gene_object = antibody.JGene(j_gene, species)

            # Make sure we are getting the right subclass
            assert isinstance(j_gene_object, antibody.JGene)
            assert isinstance(j_gene_object.cdr3_nt, segment.CDR3NT)
            assert isinstance(j_gene_object.cdr3_aa, segment.CDR3AA)
            assert isinstance(j_gene_object.fwr4_nt, segment.FrameWork4NT)
            assert isinstance(j_gene_object.fwr4_aa, segment.FrameWork4AA)

            # assert we have the same objects strings
            assert j_gene_object.cdr3_nt == cdr3_nt
            assert j_gene_object.cdr3_aa == cdr3_aa
            assert j_gene_object.fwr4_nt == fwr4_nt
            assert j_gene_object.fwr4_aa == fwr4_aa
    j_gene_object = antibody.JGene("IGHJ6*01", "human")
    rep = "<JGene>human->IGHJ6*01\n        cdr3_nt-ATTACTACTACTACTACGGTATGGACGTC\n        cdr3_aa-YYYYYGMDV\n        fwr4_nt-TGGGGGCAAGGGACCACGGTCACCGTCTCCTCAG\n        fwr4_aa-WGQGTTVTVSS\n        "
    assert j_gene_object.__repr__() == rep


#https://github.com/llSourcell/how_to_deploy_a_keras_model_to_production/blob/master/model/load.py
#https://machinelearningmastery.com/save-load-keras-deep-learning-models/

import keras
from __future__ import print_function
from keras.callbacks import LambdaCallback
from keras.models import Sequential
from keras.callbacks import ModelCheckpoint
from keras.layers import Dense
from keras.layers import LSTM
from keras.optimizers import RMSprop
from keras.utils.data_utils import get_file
import matplotlib.pyplot as plt
import numpy as np
import h5py
import random
import sys
import io
from keras.models import model_from_json
from scipy.misc import imread, imresize,imshow
import os
#get file
path = get_file('romeo+juliet.txt', origin='http://www.awesomefilm.com/script/romeo+juliet.txt')
#reads file
with io.open(path, encoding='latin-1') as f:
 #makes everything lowercase
    text = f.read().lower()
#makes variables important to establish input shape
chars = sorted(list(set(text)))
char_indices = dict((c, i) for i, c in enumerate(chars))
indices_char = dict((i, c) for i, c in enumerate(chars))
maxlen = 40
step = 3
sentences = []
next_chars = []
for i in range(0, len(text) - maxlen, step):
    sentences.append(text[i: i + maxlen])
    next_chars.append(text[i + maxlen])
#load model
json_file = open('model.json', 'r')
loaded_model_json = json_file.read()
json_file.close()
loaded_model = model_from_json(loaded_model_json)
# load weights into new model
loaded_model.load_weights("model.h5")

def sample(preds, temperature=1.0):
    # helper function to sample an index from a probability array
    preds = np.asarray(preds).astype('float64')
    preds = np.log(preds) / temperature
    exp_preds = np.exp(preds)
    preds = exp_preds / np.sum(exp_preds)
    probas = np.random.multinomial(1, preds, 1)
    return np.argmax(probas)

start_index = random.randint(0, len(text) - maxlen - 1)
f= open("output.txt","w+") #makes output text file
for diversity in [0.2, 0.5, 1.0, 1.2]:
        print('----- diversity:', diversity)

        generated = ''
        sentence = text[start_index: start_index + maxlen]
        #sentence = "What did the viola say?"
        generated += sentence
        print('----- Generating with seed: "' + sentence + '"')
        sys.stdout.write(generated)

        for i in range(10000):
            x_pred = np.zeros((1, maxlen, len(chars)))
            for t, char in enumerate(sentence):
                x_pred[0, t, char_indices[char]] = 1.

            preds = loaded_model.predict(x_pred, verbose=0)[0]
            next_index = sample(preds, diversity)
            next_char = indices_char[next_index]

            generated += next_char
            sentence = sentence[1:] + next_char

            sys.stdout.write(next_char)
            sys.stdout.flush()
        print()


#selenium & firefox(cli)
import io
from selenium import webdriver
from selenium.webdriver.firefox.options import Options

sys.stdout = io.TextIOWrapper(sys.stdout.detach(), encoding = 'utf-8')
sys.stderr = io.TextIOWrapper(sys.stderr.detach(), encoding = 'utf-8')

firefox_options = Options()
firefox_options.add_argument("--headless")

driver = webdriver.Firefox(firefox_options=firefox_options,executable_path='D:/Atom_Workspace/section3/webdriver/firefox/geckodriver')
#driver.set_window_size(1920, 1280)

driver.implicitly_wait(5)
driver.get('https://google.com')
driver.save_screenshot("D:/Atom_Workspace/section3/website_ff1.png")

driver.implicitly_wait(5)
driver.get('https://www.daum.net')
driver.save_screenshot("D:/Atom_Workspace/section3/website_ff2.png")

driver.quit()


import re

import boto3
import pytest
from moto import mock_s3

from pylaprof import FS, S3, Profiler

from .sleepy import sleepy

def test_profiler_with_fs_storer(tmpcwd):
    """Check profiler with FS storer.

    We set a period of 0.01s and run `sleepy` for 0.105s, so we should get a report with
    10 hits of this function.
    """
    with Profiler(period=0.01, storer=FS(path=lambda: "report.txt")):
        sleepy(0.105)

    regex = re.compile(r".*;sleepy \(\/[^)]+\/sleepy\.py:28\) 10")
    with open("report.txt", "r") as fp:
        report = fp.read()
    for line in report.split("\n"):
        if regex.match(line):
            return  # Ok!
    else:
        pytest.fail("something is wrong with the report:\n" + report)

@mock_s3
def test_profiler_with_s3_storer():
    """Check profiler with S3 storer.

    We set a period of 0.01s and run `sleepy` for 0.105s, so we should get a report with
    10 hits of this function.
    """
    boto3.resource("s3").Bucket("pylaprof").create()

    with Profiler(period=0.01, storer=S3(key=lambda: "report.txt")):
        sleepy(0.105)

    regex = re.compile(r".*;sleepy \(\/[^)]+\/sleepy\.py:28\) 10")
    report = (
        boto3.resource("s3")
        .Object("pylaprof", "report.txt")
        .get()["Body"]
        .read()
        .decode("utf-8")
    )
    for line in report.split("\n"):
        if regex.match(line):
            return  # Ok!
    else:
        pytest.fail("something is wrong with the report:\n" + report)


import re
from itertools import islice

INPUT_LOC = "/Users/Alex/Desktop/Carnegie/15-421/Datasets/xfactor-database.txt"
OUTPUT_LOC = "/Users/Alex/Desktop/TestOut.txt"

def clean_Xfactor():
	#I removed every line before 10 | Nathan ... before running this

	myfile =  open(INPUT_LOC, "r")
	f2 = open(OUTPUT_LOC, "w")
	while True:
		#Pythons internal buffer is 4k
		#Lets assume average of 65 char lines
		#61 lines pulled in per read
		#Guess 3 pages given to python
		head = list(islice(myfile, 61*3))
		if not head:
			break

		for item in head:
			temp = item.split(" | ",14)
			if(len(temp) != 14):
				continue
			#id | firstname | lastname | dob | phone | email | auditiontype | numberofperformers | location | optin_1 | optin_2 | gender | zip | timestamp
			if not re.match(r"^[A-Za-z0-9\.\+_-]+@[A-Za-z0-9\._-]+\.[a-zA-Z]*$",temp[5]):
				temp[5] = "NULL"
			for x in range(1,14):
				if(temp[x] == "" or temp[x] == "0"):
					temp[x] = "NULL"

			if(temp[1] == "NULL" and temp[2] == "NULL" and temp[5] == "NULL"):
				continue

			final = ""
			for x in range(0,13):
				final = final + temp[x] + ":"
			final = final + temp[13]
			f2.write(final)

	myfile.close()
	f2.close()


import pyverilog.vparser.ast as vast
from pyverilog.ast_code_generator.codegen import ASTCodeGenerator as VGen

import sys
sys.setrecursionlimit(100000)

import logging

logger = logging.getLogger("ABG_VHandle")
logger.setLevel(logging.INFO)
logHandler = logging.StreamHandler()
logHandler.setFormatter(logging.Formatter(logging.BASIC_FORMAT, None))
logger.addHandler(logHandler)

class VHandle:
    def __init__(self, vnode_root, vnodes_pe, vnodes_fifo, vnodes_fifoInst):
        self.vnode_root      = vnode_root
        self.vnode_moduleDef = vnode_root.children()[0].children()[1]
        self.vnodes_pe       = vnodes_pe
        self.vnodes_fifo     = vnodes_fifo
        self.vnodes_fifoInst = vnodes_fifoInst
    def toCustomizedNames(self):
        logger.debug(f"Original Top Module Name: {self.vnode_moduleDef.name}")
        self.vnode_moduleDef.name += "_postABG"
        logger.debug(f"Modified Top Module Name: {self.vnode_moduleDef.name}")
        for vnode_fifoInst in self.vnodes_fifoInst:
            vnode_fifo = next(filter(lambda node : isinstance(node, vast.Instance), vnode_fifoInst.children()))
            assert(vnode_fifoInst.module == vnode_fifo.module)
            logger.debug(f"Original FIFO Module Name for Instance {vnode_fifo.name}: {vnode_fifo.module}")
            vnode_fifoInst.module = vnode_fifo.name + "_M"
            vnode_fifo.module = vnode_fifo.name + "_M"
            assert(vnode_fifoInst.module == vnode_fifo.module)
            logger.debug(f"Modified FIFO Module Name for Instance {vnode_fifo.name}: {vnode_fifo.module}")
    def toRTL(self):
        # may involve deep recursion...
        # due to impl. of pyverilog ASTCodeGenerator ...
        # which is a recursive DFS
        top_module_name = self.vnode_moduleDef.name
        rtl = VGen().visit(self.vnode_moduleDef)
        return top_module_name, rtl


from CodicePython.View.VistaDonazioni import VistaDonazioni
from CodicePython.View.VistaPagamenti import VistaPagamenti
from CodicePython.View.VistaRapportini import VistaRapportini
from CodicePython.View.VistaVisualizzaTurni import VistaVisualizzaTurni
from PyQt5.QtWidgets import QWidget, QGridLayout, QSizePolicy, QPushButton

class VistaAmministratore(QWidget):
    def __init__(self, parent=None):
        super(VistaAmministratore, self).__init__(parent)
        grid_layout = QGridLayout()
        grid_layout.addWidget(self.get_generic_button("Gestione Donazioni", self.go_handle_donazioni), 1, 0)
        grid_layout.addWidget(self.get_generic_button("Gestione Rapportini", self.go_handle_rapportini), 1, 1)
        grid_layout.addWidget(self.get_generic_button("Gestione Pagamenti", self.go_handle_payments), 0, 1)
        grid_layout.addWidget(self.get_generic_button("Gestione Turni", self.go_turni), 0, 0)
        self.setLayout(grid_layout)
        self.resize(400, 300)
        self.setWindowTitle("Amministratore")

    def get_generic_button(self, titolo, on_click):
        button = QPushButton(titolo)
        button.setSizePolicy(QSizePolicy.Expanding, QSizePolicy.Expanding)
        button.clicked.connect(on_click)
        return button

    def go_handle_donazioni(self):
        self.vista_donazioni = VistaDonazioni()
        self.vista_donazioni.show()

    def go_handle_rapportini(self):
        self.vista_rapportini = VistaRapportini()
        self.vista_rapportini.show()

    def go_handle_payments(self):
        self.vista_pagamenti = VistaPagamenti()
        self.vista_pagamenti.show()

    def go_turni(self):
        self.vista_turni = VistaVisualizzaTurni()
        self.vista_turni.show()


# https://www.acmicpc.net/problem/20055
# Implementation

from collections import deque
from sys import stdin

n, k = map(int, stdin.readline().split())
seq = deque(map(int, stdin.readline().split()))
robots = deque(False for _ in range(n))
zero = seq.count(0)
for t in range(1, 10**10):
    seq.appendleft(seq.pop())
    robots.pop()
    robots.appendleft(False)
    robots[-1] = False
    for i in reversed(range(n - 1)):
        if robots[i] and not robots[i + 1] and seq[i + 1] > 0:
            robots[i] = False
            robots[i + 1] = True
            seq[i + 1] -= 1
            if seq[i+1] == 0:
                zero += 1
    robots[-1] = False
    if seq[0] > 0:
        robots[0] = True
        seq[0] -= 1
        if seq[0] == 0:
            zero += 1
    if zero >= k:
        print(t)
        break


import datetime
import typing as t
from dateutil.parser import isoparse as dateutil_isoparse, parse as dateutil_parse
from nr.parsing.date import format_set, ISO_8601
from test.samples import SAMPLES

STDLIB_ISO_8601_DATETIME_FORMATS = [
  '%Y-%m-%dT%H:%M:%S.%f%z',
  '%Y-%m-%dT%H:%M:%S.%f',
  '%Y-%m-%dT%H:%M:%S%z',
  '%Y-%m-%dT%H:%M:%S',
  '%Y%m%dT%H%M%S%z',
  '%Y%m%dT%H%M%S',
  '%Y%m%d',
]

STDLIB_ISO_8601_DATETIME_FORMATS_REVERSED = list(reversed(STDLIB_ISO_8601_DATETIME_FORMATS))

ISO8601_DATETIME_SAMPLES = SAMPLES[SAMPLES.tags.apply(lambda x: 'iso8601' in x) &
  SAMPLES.parsed.apply(lambda x: isinstance(x, datetime.datetime))]

def vectorize(func, *args):
  for _idx, row in ISO8601_DATETIME_SAMPLES.iterrows():
    func(row.formatted, row.parsed, *args)

def parse_datetime_stdlib(s: str, expected: datetime.datetime, formats: t.List[str]) -> None:
  for fmt in formats:
    try:
      parsed = datetime.datetime.strptime(s, fmt)
      break
    except ValueError:
      pass
  else:
    raise ValueError(f'Unable to parse {s!r} using {formats!r}')
  assert parsed == expected

def parse_datetime_nr(s: str, expected: datetime.datetime, format: format_set) -> None:
  parsed = format.parse_datetime(s, True)
  assert parsed == expected

def parse_dateutil_parse(s: str, expected: datetime.datetime) -> None:
  parsed = dateutil_parse(s)
  assert parsed == expected

def parse_dateutil_isoparse(s: str, expected: datetime.datetime) -> None:
  parsed = dateutil_isoparse(s)
  assert parsed == expected

class DatetimeParsingSuite:
  """
  An example benchmark that times the performance of various kinds
  of iterating over dictionaries in Python.
  """

  def time_datetime_datetime_strptime(self):
    vectorize(parse_datetime_stdlib, STDLIB_ISO_8601_DATETIME_FORMATS)

  def time_datetime_datetime_strptime_reversed(self):
    vectorize(parse_datetime_stdlib, STDLIB_ISO_8601_DATETIME_FORMATS)

  def time_nr_parsing_date_ISO_8601_parse_datetime(self):
    vectorize(parse_datetime_nr, ISO_8601)

  def time_dateutil_parser_parse(self):
    vectorize(parse_dateutil_parse)

  def time_dateutil_parser_isoparse(self):
    vectorize(parse_dateutil_isoparse)


import datetime
import hashlib
import json
from urllib.parse import urlparse

import requests

class Blockchain:
    def __init__(self):
        self.chain = []
        self.transactions = []
        self.nodes = set()
        self.create_block(proof=1, previous_hash="0")

    @property
    def length(self):
        return len(self.chain)

    def create_block(self, proof, previous_hash):
        block = {
            "index": len(self.chain) + 1,
            "timestamp": str(datetime.datetime.now()),
            "proof": proof,  # Nonce
            "previous_hash": previous_hash,
            "transactions": self.transactions,
        }
        self.transactions = []
        self.chain.append(block)
        return block

    def get_previous_block(self):
        return self.chain[-1]

    def proof_of_work(self, previous_proof):
        new_proof = 1
        check_proof = False
        while check_proof is False:
            hash_operation = self.calculate_hash_operation(new_proof, previous_proof)
            if hash_operation[:4] == "0000":
                check_proof = True
            else:
                new_proof += 1
        return new_proof

    def hash(self, block):
        encoded_block = json.dumps(block, sort_keys=True).encode()
        return hashlib.sha256(encoded_block).hexdigest()

    def calculate_hash_operation(self, new_proof, previous_proof):
        calc = new_proof ** 2 - previous_proof ** 2
        return hashlib.sha256(str(calc).encode()).hexdigest()

    def is_chain_valid(self, chain):
        previous_block = chain[0]
        block_index = 1

        while len(chain) > block_index:
            block = chain[block_index]

            if not self._validate_block_hash(previous_block, block):
                return False

            if not self._validate_block_proof(previous_block, block):
                return False

            block_index += 1
            previous_block = block

        return True

    def add_transaction(self, sender, receiver, amount):
        self.transactions.append(
            {"sender": sender, "receiver": receiver, "amount": amount}
        )

        previous_block = self.get_previous_block()
        return previous_block["index"] + 1

    def add_node(self, address):
        url = urlparse(address)
        self.nodes.add(url.netloc)

    def replace_chain(self):
        network = self.nodes
        longest_chain = None
        max_length = self.length

        for node in network:
            response = requests.get(f"http://{node}/get_chain")

            if response.ok:
                data = response.json()
                if data["length"] > max_length and self.is_chain_valid(data["chain"]):
                    max_length = data["length"]
                    longest_chain = data["chain"]

        if longest_chain:
            self.chain = longest_chain
            return True
        return False

    def _validate_block_hash(self, prev, cur):
        return cur["previous_hash"] == self.hash(prev)

    def _validate_block_proof(self, prev, cur):
        previous_proof = prev["proof"]
        proof = cur["proof"]
        hash_operation = self.calculate_hash_operation(proof, previous_proof)
        return hash_operation[:4] == "0000"


import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.metrics import roc_curve, auc, precision_recall_curve, f1_score, confusion_matrix
from sklearn.metrics import average_precision_score, roc_auc_score
from sklearn.preprocessing import StandardScaler
from sklearn.model_selection import RandomizedSearchCV, GridSearchCV
from sklearn.ensemble import RandomForestClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.neural_network import MLPClassifier
from sklearn import svm
import lightgbm as lgb
import shap
import warnings
warnings.filterwarnings('ignore')

def shap_plot(model, data, center, modelname):
    if modelname == 'RF' or modelname == 'LGBM':
        explainer = shap.TreeExplainer(model)
        shap_values = explainer.shap_values(data, approximate=False, check_additivity=False)
    elif modelname == 'LR':
        masker = shap.maskers.Independent(data=data)
        explainer = shap.LinearExplainer(model, masker=masker)
        shap_values = explainer.shap_values(data)
    elif modelname == 'SVM' or modelname == 'MLP':
        explainer = shap.KernelExplainer(model.predict_proba, data)
        shap_values = explainer.shap_values(data)

    if isinstance(shap_values, list):
        shap.summary_plot(shap_values[1], data, max_display=10, show=False, plot_size=(12, 8))
    elif isinstance(shap_values, np.ndarray):
        shap.summary_plot(shap_values, data, max_display=10, show=False, plot_size=(12, 8))

    plt.savefig('plots/SHAP_{}_{}.png'.format(center, modelname))
    plt.close()

def plot_aucs(label, score, name):
    sns.set_style('whitegrid')
    fig, axs = plt.subplots(1, 2, figsize=(10.5, 4.5))
    axs = axs.ravel()

    precision, recall, _ = precision_recall_curve(label, score, pos_label=1)
    auprc = auc(recall, precision)
    fpr, tpr, _ = roc_curve(label, score, pos_label=1)
    auroc = auc(fpr, tpr)

    axs[0].plot(fpr, tpr, c='b', lw=2, label='AUROC: {:.3f}'.format(auroc))
    axs[0].set_xlabel('fpr', fontsize=15)
    axs[0].set_ylabel('tpr', fontsize=15)
    axs[0].plot([0, 1], [0, 1], linestyle='dashed')
    axs[0].set_xlim([0, 1.05])
    axs[0].set_ylim([0, 1.05])
    axs[0].legend()

    axs[1].plot(recall, precision, c='b', lw=2,
                label='AUPRC: {:.3f}'.format(auprc))
    axs[1].plot([0, 1], [np.mean(label), np.mean(label)], linestyle='dashed')
    axs[1].set_xlim([0, 1.05])
    axs[1].set_ylim([0, 1.05])
    axs[1].legend()
    axs[1].set_xlabel('recall', fontsize=15)
    axs[1].set_ylabel('precision', fontsize=15)

    plt.tight_layout()
    plt.savefig('plots/{}.png'.format(name), dpi=300)
    plt.close()

def plot_aucs_withUncertainty(data, name):
    labels = []
    scores = []
    aucs = []
    precisions, recalls, fprs, tprs, auprcs, aurocs = [], [], [], [], [], []
    mean_fpr = np.linspace(0, 1, 100)
    mean_recall = np.linspace(0, 1, 100)
    for j in range(len(data[0])):
        label = data[0][j].tolist()
        score = data[1][j]
        labels.extend(label)
        scores.extend(score)
        precision, recall, _ = precision_recall_curve(label, score, pos_label=1)
        precision, recall = precision[::-1], recall[::-1]
        auprc = auc(recall, precision)
        interp_precision = np.interp(mean_recall, recall, precision)
        precisions.append(interp_precision)
        auprcs.append(auprc)
        fpr, tpr, _ = roc_curve(label, score, pos_label=1)
        auroc = auc(fpr, tpr)
        interp_tpr = np.interp(mean_fpr, fpr, tpr)
        interp_tpr[0] = 0.0
        tprs.append(interp_tpr)
        aucs.append(auroc)
    mean_tpr = np.mean(tprs, axis=0)
    std_tpr = np.std(tprs, axis=0)
    tprs_upper = np.minimum(mean_tpr + std_tpr, 1)
    tprs_lower = np.maximum(mean_tpr - std_tpr, 0)
    precisions, recalls, _ = precision_recall_curve(labels, scores)
    auprc = auc(recalls, precisions)
    std_auprc = np.std(auprcs)
    std_precision = np.std(precisions, axis=0)
    fprs, tprs, _ = roc_curve(labels, scores, pos_label=1)
    auroc = auc(fprs, tprs)
    std_auroc = np.std(aucs)

    plt.figure(figsize=(10.5, 4.5))
    plt.subplot(121)
    plt.plot(mean_fpr, mean_tpr, color='r',
             label='AUROC {:.3f}'.format(auroc),
             lw=2, alpha=.8)
    plt.fill_between(mean_fpr, tprs_lower, tprs_upper, color='r', alpha=0.3, )
    plt.xlim([0, 1])
    plt.ylim([0, 1.05])
    plt.legend()
    plt.ylabel('tpr', fontsize=15)
    plt.xlabel('fpr', fontsize=15)
    plt.title('ROC', fontsize=15)
    plt.subplot(122)
    plt.plot(recalls, precisions, lw=2, c='r',
             label='AUPRC {:.3f}'.format(auprc))
    plt.fill_between(recalls, precisions + std_precision,
                     precisions - std_precision, alpha=0.3, linewidth=0, color='r')
    plt.plot([0, 1], [np.mean(labels), np.mean(labels)], color='navy', lw=2, linestyle='--')
    plt.xlabel('recall', fontsize=15)
    plt.ylabel('precision', fontsize=15)
    plt.xlim([0, 1])
    plt.ylim([0, 1.05])
    plt.legend()
    plt.title('PRC', fontsize=15)
    plt.savefig('plots/{}.png'.format(name), dpi=300)
    plt.close()
    return auprc, auroc, std_auprc, std_auroc

def get_label(data_day, daytime):
    data_day['MODS'] = data_day[['cons05.resp', 'cons05.cvs', 'cons05.cns',
          'cons05.ren', 'cons05.hep', 'cons05.hem']].sum(1)
    data_day['MODS'] = (data_day['MODS']>=2).astype(int)
    patients = []
    nb_pid = 0
    for pid, patient in data_day.groupby('sample.id'):
        nb_pid += 1
        if daytime in patient['daytime'].tolist() and patient[patient['daytime'] == daytime]['MODS'].values[0] == 1:
                patient['label_MODS_day{}'.format(daytime)] = 1
        if patient['death.delay'].values[0] <= daytime:
            patient.loc[:, 'label_death_day{}'.format(daytime)] = 1
        patients.append(patient)

    patients = pd.concat(patients)
    patients_bc = patients[patients['daytime'] == 0]
    nb_pid_bc = len(patients_bc)
    patients_bc['label'] = (patients_bc[['label_MODS_day{}'.format(daytime),
                                         'label_death_day{}'.format(daytime)]].sum(1)==0).astype(int)

    patients_bc_MODS = patients_bc[patients_bc['MODS'] == 1]
    nb_pid_bc_MODS = len(patients_bc_MODS)
    return patients_bc_MODS, nb_pid, nb_pid_bc, nb_pid_bc_MODS

def metrics_80recall(model, test, label, score):
    precisions, recalls, thresholds = precision_recall_curve(label, score)
    threshold = thresholds[np.abs(recalls - 0.8).argmin()]
    pred = (model.predict_proba(test)[:, 1] >= threshold).astype(int).flatten()
    test_table = pd.DataFrame(columns=['true', 'pred'])
    test_table['true'] = list(label)
    test_table['pred'] = list(pred)
    tn, fp, fn, tp = confusion_matrix(test_table['true'], test_table['pred']).ravel()
    precision = tp / (tp + fp)
    specificity = tn / (tn + fp)
    f1 = f1_score(test_table['true'], test_table['pred'])
    npv = tn / (tn + fn)
    accuracy = (tp + tn) / (tp + fp + tn + fn)
    recall = tp / (tp + fn)
    return [precision,  specificity, f1, npv, accuracy, recall]

def RF_grid(data, label, scoring):
    grid = {"n_estimators": np.arange(100, 2000, 200),
           'max_depth':[10,20,30,40,50,60],
           'max_leaf_nodes': [50, 100, 200],
           'min_samples_leaf':[1,2,4]}
    rf = RandomForestClassifier()
    rf_grid = RandomizedSearchCV(rf, grid, n_iter=50, cv=5, n_jobs=-1, scoring=scoring)
    rf_grid.fit(data, label)
    return rf_grid

def MLP_grid(data, label, scoring):
    grid = {
        'hidden_layer_sizes': [(25, 50, 25), (25, 50),
                               (25, 25), (50, 50),
                               (50, 100), (50, 50, 50),
                              (50, 100, 50), ],
        'activation': ['tanh', 'relu'],
        'solver': ['adam'],
        'alpha': [0.0001, 0.00001, 0.001],
        'learning_rate': ['constant', 'adaptive'],
        'learning_rate_init':[0.001, 0.0001],
        'max_iter':[500, 1000, 5000, 10000]
    }
    mlp = MLPClassifier()
    mlp_grid = RandomizedSearchCV(mlp, grid, n_iter=50, cv=5, n_jobs=-1, scoring=scoring)
    mlp_grid.fit(data, label)
    return mlp_grid

def SVM_grid(data, label, scoring):
    grid = {
        {"kernel": ["rbf"], "gamma": [1e-2, 1e-3, 1e-4], 'C':np.logspace(-4,4,15)},
        {"kernel": ["linear"], 'C':np.logspace(-4,4,15)},
        }
    model = svm.SVC(probability=True)
    svm_grid = RandomizedSearchCV(model, grid, n_iter=50, cv=5, n_jobs=-1, scoring=scoring)
    svm_grid.fit(data, label)
    return svm_grid

def LR_grid(data, label, scoring):
    clf = LogisticRegression(max_iter=15000)
    grid ={'penalty': ["l1",  'l2'],
           'C':np.logspace(-4,4,20),
           'solver':['lbfgs’', 'liblinear']}
    lr_grid = GridSearchCV(clf, grid, n_jobs=-1, cv=5, scoring=scoring)
    lr_grid.fit(data, label)
    return lr_grid

def LGBM_grid(data, label, scoring):
    clf = lgb.LGBMClassifier(
        random_state=42,
        objective='binary',
        is_unbalance=True
    )
    grid = {
        'learning_rate':[0.01, 0.005, 0.001],
        'boosting_type': ["gbdt", "dart"],
        'n_estimators': np.arange(100, 2000, 200),
        'max_depth': [10, 20, 30, 40, 50, 60],
        'subsample': [0.8, 0.9, 1]
    }
    gbm_grid = algorithm_pipeline(data, label, clf, grid, scoring)
    return gbm_grid

def algorithm_pipeline(train_x, train_y,
                model, param_grid, scoring):
    rs = RandomizedSearchCV(
        estimator=model,
        param_distributions=param_grid,
        n_iter=50,
        cv=5,
        n_jobs=-1,
        scoring=scoring
    )
    fitted_CV = rs.fit(train_x, train_y)
    return fitted_CV

def cross_validation(train_list, val_list, test_list, features_selected, features_to_scale, model):
    predictions_val = []
    labels_val = []
    predictions_test = []
    labels_test = []
    models = []
    scoring = 'roc_auc'
    scaler = StandardScaler()
    for i in range(len(train_list)):

        X_train = train_list[i][features_selected]
        X_train[features_to_scale] = scaler.fit_transform(X_train[features_to_scale])
        X_train[features_to_scale] = X_train[features_to_scale].fillna(0)
        y_train = train_list[i]['label']

        X_val = val_list[i][features_selected]
        X_val[features_to_scale] = scaler.transform(X_val[features_to_scale])
        X_val[features_to_scale] = X_val[features_to_scale].fillna(0)
        y_val = val_list[i]['label']

        X_test = test_list[i][features_selected]
        X_test[features_to_scale] = scaler.transform(X_test[features_to_scale])
        X_test[features_to_scale] = X_test[features_to_scale].fillna(0)
        y_test = test_list[i]['label']

        fitted_CV = model(X_train, y_train, scoring)
        pred_val = fitted_CV.best_estimator_.predict_proba(X_val)[:, 1]
        pred_test = fitted_CV.best_estimator_.predict_proba(X_test)[:, 1]

        predictions_val.extend(list(pred_val))
        labels_val.extend(list(y_val))
        predictions_test.extend(list(pred_test))
        labels_test.extend(list(y_test))
        models.append(fitted_CV)

    auprc_val = average_precision_score(labels_val, predictions_val)
    auroc_val = roc_auc_score(labels_val, predictions_val)
    auprc_test = average_precision_score(labels_test, predictions_test)
    auroc_test = roc_auc_score(labels_test, predictions_test)

    return auprc_val, auroc_val, auprc_test, auroc_test, models, predictions_test, labels_test

def enrichment_test(From, To, test_col, alpha=0.05):

    # test for enrichment/depletion in From set compared to To set.
    counts = {}
    draw_num = To.shape[0]
    # 100 random samplings
    instances = From[test_col].unique()
    num_test = instances.shape[0] * 100
    correction_num = From[test_col].unique().shape[0]

    for instance in instances:
        counts[instance] = []

    for k in range(num_test):
        draw = From.sample(draw_num)[test_col].value_counts() / draw_num
        for instance in instances:
            if instance in draw.index:
                counts[instance].append((draw)[instance])
            else:
                counts[instance].append(0)

    ratios_to = To[test_col].value_counts() / len(To)
    ratios_from = From[test_col].value_counts() / len(From)

    base = {}
    report = pd.DataFrame(columns=['Group', 'Instance', 'From', 'To', 'Type'])
    i = 1

    for instance in instances:
        if instance in ratios_to.index:
            base[instance] = 1 if (ratios_to)[instance] >= (ratios_from)[instance] else 0
        else:
            ratios_to[instance] = 0
            base[instance] = 0

        if base[instance] == 1 and np.sum(np.array(counts[instance]) >= (ratios_to)[instance]) <= (
                num_test * alpha / correction_num):
            print('Enrichment of {} in FP from {:.3f} to {:.3f}.'.format(
                instance, (ratios_from)[instance], (ratios_to)[instance]))
            report.loc[i] = [test_col, instance, (ratios_from)[instance],
                             (ratios_to)[instance], 'Enrichment']
            i += 1

        if base[instance] == 0 and np.sum(np.array(counts[instance]) <= (ratios_to)[instance]) <= (
                num_test * alpha / correction_num):
            print('Depletion of {} in FP from {:.3f} to {:.3f}.'.format(
                instance, (ratios_from)[instance], (ratios_to)[instance]))
            report.loc[i] = [test_col, instance, (ratios_from)[instance],
                             (ratios_to)[instance], 'Depletion']
            i += 1

    return report


import csv
import requests
from bs4 import BeautifulSoup as BS
from time import sleep
from random import randrange
import json
from selenium import webdriver
import selenium.common.exceptions as exc
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium_stealth import stealth

options = webdriver.ChromeOptions()
options.add_argument("start-maximized")

# options.add_argument("--headless")

options.add_experimental_option("excludeSwitches", ["enable-automation"])
options.add_experimental_option('useAutomationExtension', False)
driver = webdriver.Chrome(options=options)

stealth(driver,
        languages=["en-US", "en"],
        vendor="Google Inc.",
        platform="Win32",
        webgl_vendor="Intel Inc.",
        renderer="Intel Iris OpenGL Engine",
        fix_hairline=True,
        )

headers = {
    'Accept': '*/*',
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:120.0) Gecko/20100101 Firefox/120.0'
}

def get_artictles_urls(url):
    with requests.Session() as session:
        response = session.get(url=url, headers=headers)

        with open('index.html', 'w', encoding='utf-8') as file:
            file.write(response.text)

        articles_urls_list = []

    with requests.Session() as session:
        page = 0
        while True:
            page += 1
            response = session.get(f'https://announcements.bybit.com/en-US/?category=&page={page}', headers=headers)
            soup = BS(response.text, 'lxml')
            articles_urls = soup.find_all('a', class_='no-style')
            if (len(articles_urls)):
                for au in articles_urls:
                    art_url = au.get('href')
                    articles_urls_list.append(str(art_url))
            else:
                break

            # time.sleep(randrange(2, 5))

            print(f'Обработал {page}')

    with open('articles_urls.txt', 'w', encoding='utf-8') as file:
        for url in articles_urls_list:
            file.write(f'https://announcements.bybit.com{url}\n')

    print('Обработка законченна')
    return True

def get_data(file_path):
    with open(file_path) as file:
        urls_list = [line.strip() for line in file.readlines()]

    for url in urls_list:
        to_write = []
        driver.get(url)
        article_title = driver.find_element(By.CLASS_NAME, 'article-detail-title').text
        article_time = driver.find_element(By.CLASS_NAME, 'article-detail-date').text
        to_write.append(url)
        to_write.append(article_title)
        to_write.append(article_time)
        with open('data.csv', 'a', encoding='utf-8') as file:
            writer = csv.writer(file)
            writer.writerow(to_write)

def main():
    '''Получение всех ссылок(1 раз)'''

    #get_artictles_urls(url='https://announcements.bybit.com/en-US/?category=&page=1')

    '''Получение информации со всех ссылок'''

    if get_artictles_urls(url='https://announcements.bybit.com/en-US/?category=&page=1') is True:
        get_data('articles_urls.txt')

    '''Постоянный парсинг'''

    try:
        while True:
            if uniq_check(check()) is False:
                append_to_list(check())
                print('Добавлена новость')

            sleep(1)

    except Exception as ex:
        print(f'Что-то пошло не так {ex}')

def check():
    with requests.Session() as session:
        response = session.get(f'https://announcements.bybit.com/en-US/?category=&page=1', headers=headers)
        soup = BS(response.text, 'lxml')
        article_urls = soup.find('a', class_='no-style')
        art_url = article_urls.get('href')
        return 'https://announcements.bybit.com' + art_url

def uniq_check(url):
    with open('articles_urls.txt', 'r', encoding='utf-8') as file:
        for line in file:
            if url in line:
                return True
        else:
            return False

def append_to_list(url):
    with open('articles_urls.txt', 'a', encoding='utf-8') as file:
        file.write(f'https://announcements.bybit.com{url}\n')
    to_write = []
    driver.get(url)
    article_title = driver.find_element(By.CLASS_NAME, 'article-detail-title').text
    article_time = driver.find_element(By.CLASS_NAME, 'article-detail-date').text
    to_write.append(url)
    to_write.append(article_title)
    to_write.append(article_time)
    with open('data.csv', 'a', encoding='utf-8') as file:
        writer = csv.writer(file)
        writer.writerow(to_write)

if __name__ == '__main__':
    main()
    driver.quit()


# ! /usr/bin/env python

import json
import smbus

from time import sleep

class sonarSet:
    def __init__(self, config, status, robot, queue = None, bus = None, addr = None):
        self.config = config
        self.status = status
        self.robot = robot
        self.queue = queue

        self.bus = bus
        if bus is None:
            self.bus = int(config["ARDUINO_SONAR_SET_BUS"])

        self.addr = addr
        if addr is None:
            self.addr = int(config["ARDUINO_SONAR_SET_ADDR"], 16)

        self.i2c_interface = smbus.SMBus(self.bus)

    def _get_data(self, last_distance):
        try:
            sonar_data_from_arduino = self.i2c_interface.read_i2c_block_data(self.addr, 0, 32)
            sonar_data = ''
            for i in range(len(sonar_data_from_arduino)):
                sonar_data += chr(sonar_data_from_arduino[i])

            return str(sonar_data)
        except:
            if last_distance is not None:
                return '{},00.00,00.00,00.00, '.format(last_distance)

    def worker(self):
        status = self.status
        log_colission = self.config["LOG_COLISSION"].lower() in ('true', '1', 't')
        limit = int(self.config["LIMIT_COLISSION_IN_CM"])

        last_sonar_data = None
        sonar_keys = ("center", "centerRight", "back", "centerLeft")

        while True:
            try:
                sonar_data = self._get_data(last_sonar_data)
                sonar_data =  sonar_data.split(",")
                del(sonar_data[len(sonar_data)-1])

                sonar_data = dict(zip(sonar_keys, sonar_data))

                if float(sonar_data["center"]) <= limit and not status.colission_detected and status.direction == "Front":
                    self.robot.stop()
                    status.colission_detected = True
                    if log_colission:
                        status.colission_string = 'STOP CRASH {}'.format(sonar_data["center"])

                elif status.colission_detected and status.direction != "Front":
                    status.colission_detected = False
                    if log_colission:
                        status.colission_string ='RETURN {}'.format(sonar_data["center"])

                if self.queue is not None:
                    key = "distance_log"
                    msg = {
                        "type": "distanceLog",
                        "contents": sonar_data
                    }
                    self.queue.set(key, json.dumps(msg))

                if self.status is not None:
                    self.status.sonar_data = sonar_data

            except:
                #print("sonar failure")
                sonar_data = last_sonar_data

            sleep(0.10)


import cv2
import argparse
from image_align import SIFTAligner
from data_extraction import MorphDataExtractor

def extract_data(im, aligner, extractor):
    query = cv2.imread(im)
    aligned_sift = aligner.align(query)

    entities = extractor.extract(aligned_sift)

    return entities

def main(args):
    query_image_location = args.query
    template_locataion = args.template
    roi_json_file = args.roi

    # roi_data = "/Users/jaswantjonnada/Documents/TestRequestForm/roi_data/TRF-1_annotation.json"

    morph_extractor = MorphDataExtractor(roi_json_file)

    # lifecell_template_path = "/Users/jaswantjonnada/Documents/TestRequestForm/templates/TRF-1.png"

    sift_aligner = SIFTAligner(template_locataion)

    entities = extract_data(im=query_image_location,
                            aligner=sift_aligner,
                            extractor=morph_extractor)

    print(entities)

if __name__ == '__main__':

    parser = argparse.ArgumentParser("Parsing arguments for information extraction")

    parser.add_argument('--query', type=str,
                    help='Image from where the data needs to be extracted')

    parser.add_argument('--template', type=str,
                        help="The template document which is used for alignment")

    parser.add_argument("--roi", type=str,
                        help="json file that contains the roi data")

    args = parser.parse_args()

    main(args)