import gc
from collections import defaultdict

import cupy as cp
import pandas as pd
import torch
import torch.nn.functional as F
from cuml.metrics import pairwise_distances
from cuml.neighbors import NearestNeighbors
from torch.utils.data import DataLoader, Dataset, default_collate
from tqdm import tqdm
from transformers import AutoTokenizer, TrainerCallback

from utils import clean_text, f2_score, get_pos_score

LANGUAGE_TOKENS = [
    "<|lang_pnb|>",
    "<|lang_tr|>",
    "<|lang_ur|>",
    "<|lang_bn|>",
    "<|lang_hi|>",
    "<|lang_en|>",
    "<|lang_kn|>",
    "<|lang_km|>",
    "<|lang_zh|>",
    "<|lang_gu|>",
    "<|lang_ta|>",
    "<|lang_my|>",
    "<|lang_fr|>",
    "<|lang_swa|>",
    "<|lang_or|>",
    "<|lang_mul|>",
    "<|lang_fil|>",
    "<|lang_sw|>",
    "<|lang_es|>",
    "<|lang_pt|>",
    "<|lang_pl|>",
    "<|lang_ru|>",
    "<|lang_mr|>",
    "<|lang_it|>",
    "<|lang_ar|>",
    "<|lang_bg|>",
    "<|lang_te|>",
    "<|lang_as|>",
]

CATEGORY_TOKENS = [
    "<|category_supplemental|>",
    "<|category_aligned|>",
    "<|category_source|>",
]

LEVEL_TOKENS = [
    "<|level_0|>",
    "<|level_1|>",
    "<|level_2|>",
    "<|level_3|>",
    "<|level_4|>",
    "<|level_5|>",
    "<|level_6|>",
    "<|level_7|>",
    "<|level_8|>",
    "<|level_9|>",
    "<|level_10|>",
]

KIND_TOKENS = [
    "<|kind_document|>",
    "<|kind_video|>",
    "<|kind_html5|>",
    "<|kind_exercise|>",
    "<|kind_audio|>",
]

OTHER_TOKENS = [
    "<|topic|>",
    "<|content|>",
    "<s_title>",
    "</s_title>",
    "<s_description>",
    "</s_description>",
    "<s_text>",
    "</s_text>",
]

RELATION_TOKENS = [
    "<s_parent>",
    "</s_parent>",
    "<s_children>",
    "</s_children>",
]

def init_tokenizer(tokenizer_name):
    tokenizer = AutoTokenizer.from_pretrained(tokenizer_name)
    tokenizer.add_special_tokens(
        dict(
            additional_special_tokens=LANGUAGE_TOKENS
            + CATEGORY_TOKENS
            + LEVEL_TOKENS
            + KIND_TOKENS
            + OTHER_TOKENS
            + RELATION_TOKENS
        )
    )
    if "sentence-t5" in tokenizer_name:
        tokenizer.add_special_tokens({"sep_token": "<sep>"})
    return tokenizer

class LECRDataset(Dataset):
    def __init__(
        self,
        supervised_df,
        topic_df,
        content_df,
        topic_dict,
        content_dict,
        correlation_df,
        tokenizer_name="xlm-roberta-base",
        max_len=512,
        use_content_pair=False,
        is_training=False,
        use_augmentation=False,
        objective="siamese",
    ):
        self.tokenizer = init_tokenizer(tokenizer_name)
        self.max_len = max_len

        self.supervised_df = supervised_df.dropna()
        self.topic_df = topic_df
        self.content_df = content_df
        self.topic_dict, self.content_dict = topic_dict, content_dict
        self.correlation_df = correlation_df
        self.use_content_pair = use_content_pair
        self.is_training = is_training
        self.use_augmentation = use_augmentation
        self.objective = objective
        self.topic_texts, self.content_texts, self.labels = self.process_csv()

    def process_csv(self):
        # get text pairs
        topic_ids = self.supervised_df.topic_id.values
        content_ids = self.supervised_df.content_ids.values
        labels = list(self.supervised_df.target.values)

        topic_texts = []
        content_texts = []

        for topic_id in topic_ids:
            topic_texts.append(self.topic_dict[topic_id])

        for content_id in content_ids:
            content_texts.append(self.content_dict[content_id])

        set_topic_ids = set(topic_ids)
        use_all_pairs = (
            False  # use all pair, no need to be in the intersection of content_ids of topic ids
        )
        if self.use_content_pair:
            # todo: create content pairs from each topic
            content_to_topic = defaultdict(lambda: [])
            topic_to_content = defaultdict(lambda: [])

            pairs = set()

            for i, row in tqdm(self.correlation_df.iterrows()):
                content_list = row["content_ids"].split(" ")

                if row["topic_id"] not in set_topic_ids:
                    continue

                for content_id in content_list:
                    content_to_topic[content_id].append(row["topic_id"])
                    topic_to_content[row["topic_id"]].append(content_id)

                if len(content_list) <= 1:
                    continue

                if use_all_pairs:
                    for idx1 in range(len(content_list) - 1):
                        for idx2 in range(idx1 + 1, len(content_list)):
                            if (content_list[idx1], content_list[idx2],) not in pairs and (
                                content_list[idx2],
                                content_list[idx1],
                            ) not in pairs:
                                pairs.add((content_list[idx1], content_list[idx2]))

            if not use_all_pairs:
                for content_id, topics in tqdm(content_to_topic.items()):
                    intersection_contents = list(
                        set.intersection(*[set(topic_to_content[topic_id]) for topic_id in topics])
                    )
                    for idx1 in range(len(intersection_contents) - 1):
                        for idx2 in range(idx1 + 1, len(intersection_contents)):
                            if (
                                intersection_contents[idx1],
                                intersection_contents[idx2],
                            ) not in pairs and (
                                intersection_contents[idx2],
                                intersection_contents[idx1],
                            ) not in pairs:
                                pairs.add(
                                    (
                                        intersection_contents[idx1],
                                        intersection_contents[idx2],
                                    )
                                )

            for pair in pairs:
                topic_texts.append(self.content_dict[pair[0]])
                content_texts.append(self.content_dict[pair[1]])
                labels.append(1)

        return topic_texts, content_texts, labels

    def __len__(self):
        if self.is_training:
            return len(self.labels)
        else:
            return 1

    def augment(self, inputs):
        probability_matrix = torch.full(inputs["input_ids"].shape, 0.15)
        masked_indices = torch.bernoulli(probability_matrix).bool()
        indices_replaced = (
            torch.bernoulli(torch.full(inputs["input_ids"].shape, 0.8)).bool() & masked_indices
        )
        inputs["input_ids"][indices_replaced] = self.tokenizer.convert_tokens_to_ids(
            self.tokenizer.mask_token
        )
        inputs["input_ids"] *= inputs["attention_mask"]

        return inputs

    def __getitem__(self, idx):
        topic_text = self.topic_texts[idx]
        content_text = self.content_texts[idx]
        label = self.labels[idx]

        if self.objective == "siamese":
            # topic
            if isinstance(topic_text, tuple):
                topic_inputs = self.tokenizer.encode_plus(
                    topic_text[0],
                    topic_text[1],
                    return_tensors=None,
                    add_special_tokens=True,
                    max_length=self.max_len,
                    padding="max_length",
                    truncation=True,
                )
            else:
                topic_inputs = self.tokenizer.encode_plus(
                    topic_text,
                    return_tensors=None,
                    add_special_tokens=True,
                    max_length=self.max_len,
                    padding="max_length",
                    truncation=True,
                )
            for k, v in topic_inputs.items():
                topic_inputs[k] = torch.tensor(v, dtype=torch.long)

            # content
            content_inputs = self.tokenizer.encode_plus(
                content_text,
                return_tensors=None,
                add_special_tokens=True,
                max_length=self.max_len,
                padding="max_length",
                truncation=True,
            )
            for k, v in content_inputs.items():
                content_inputs[k] = torch.tensor(v, dtype=torch.long)

            if isinstance(topic_text, tuple):
                topic_text = topic_text[0] + topic_text[1]

            if self.is_training and self.use_augmentation:
                topic_inputs = self.augment(topic_inputs)
                content_inputs = self.augment(content_inputs)

            return topic_inputs, content_inputs, topic_inputs, label
        elif self.objective == "classification":
            combined_inputs = self.tokenizer.encode_plus(
                topic_text,
                content_text,
                return_tensors=None,
                add_special_tokens=True,
                max_length=self.max_len,
                padding="max_length",
                truncation=True,
            )
            for k, v in combined_inputs.items():
                combined_inputs[k] = torch.tensor(v, dtype=torch.long)

            if self.is_training and self.use_augmentation:
                combined_inputs = self.augment(combined_inputs)

            return combined_inputs, combined_inputs, combined_inputs, label
        else:
            raise ValueError("Only support siamese/classification for now.")

class InferenceDataset(Dataset):
    def __init__(self, texts, tokenizer_name="xlm-roberta-base", max_len=512):
        self.texts = texts

        self.tokenizer = init_tokenizer(tokenizer_name)
        self.max_len = max_len

    def __len__(self):
        return len(self.texts)

    def __getitem__(self, idx):
        text = self.texts[idx]

        # topic
        inputs = self.tokenizer.encode_plus(
            text,
            return_tensors=None,
            add_special_tokens=True,
            max_length=self.max_len,
            padding="max_length",
            truncation=True,
        )
        for k, v in inputs.items():
            inputs[k] = torch.tensor(v, dtype=torch.long)

        return inputs

def collate_fn(inputs):
    inputs = default_collate(inputs)
    mask_len = int(inputs["attention_mask"].sum(axis=1).max())
    for k, v in inputs.items():
        inputs[k] = inputs[k][:, :mask_len]

    return inputs

class DatasetUpdateCallback(TrainerCallback):
    """
    Trigger re-computing dataset

    A hack that modifies the train/val dataset, pointed by Trainer's dataloader

    0. Calculate new train/val topic/content embeddings, train KNN, get new top-k
    1. Calculate top-k max positive score, compare to current val best, if greater, continue to step 2, else do nothing
    2. Update supervised_df and update dataset:
        self.topic_texts, self.content_texts, self.labels = self.process_csv()
    """

    def __init__(
        self,
        trainer,
        train_topic_ids,
        val_topic_ids,
        topic_df,
        content_df,
        topic_dict,
        content_dict,
        correlation_df,
        tokenizer_name,
        max_len,
        best_score=0,
        top_k=50,
        use_translated=False,
        mix_translated=False,
        fold=0,
    ):
        super(DatasetUpdateCallback, self).__init__()
        self.trainer = trainer
        self.topic_df = topic_df
        self.content_df = content_df
        self.correlation_df = correlation_df
        self.best_score = best_score
        self.top_k = top_k
        self.use_translated = use_translated
        self.mix_translated = mix_translated
        self.fold = fold

        self.tokenizer = init_tokenizer(tokenizer_name)
        self.topic_dict, self.content_dict = topic_dict, content_dict

        train_topic_texts = [
            topic_dict[topic_id]
            for topic_id in self.topic_df.id.values
            if topic_id in train_topic_ids
        ]
        self.train_topic_ids = [
            topic_id for topic_id in self.topic_df.id.values if topic_id in train_topic_ids
        ]
        self.train_topic_languages = []
        for topic_id, topic_lang in zip(self.topic_df.id.values, self.topic_df.language.values):
            if topic_id in train_topic_ids:
                self.train_topic_languages.append(topic_lang)

        val_topic_texts = [
            topic_dict[topic_id]
            for topic_id in self.topic_df.id.values
            if topic_id in val_topic_ids
        ]
        self.val_topic_ids = [
            topic_id for topic_id in self.topic_df.id.values if topic_id in val_topic_ids
        ]

        content_texts = [
            content_dict[content_id]
            for content_id in self.content_df.id.values
            if content_id.startswith("c_")
        ]

        def inference_collate_fn(inputs):
            inputs = default_collate(inputs)
            mask_len = int(inputs["attention_mask"].sum(axis=1).max())
            for k, v in inputs.items():
                inputs[k] = inputs[k][:, :mask_len]

            return inputs

        train_topic_dataset = InferenceDataset(
            texts=train_topic_texts, tokenizer_name=tokenizer_name, max_len=max_len
        )
        self.train_topic_dataloader = DataLoader(
            train_topic_dataset,
            num_workers=self.trainer.args.dataloader_num_workers,
            batch_size=32,
            shuffle=False,
            collate_fn=inference_collate_fn,
        )

        val_topic_dataset = InferenceDataset(
            texts=val_topic_texts, tokenizer_name=tokenizer_name, max_len=max_len
        )
        self.val_topic_dataloader = DataLoader(
            val_topic_dataset,
            num_workers=self.trainer.args.dataloader_num_workers,
            batch_size=32,
            shuffle=False,
            collate_fn=inference_collate_fn,
        )

        content_dataset = InferenceDataset(
            texts=content_texts, tokenizer_name=tokenizer_name, max_len=max_len
        )
        self.content_dataloader = DataLoader(
            content_dataset,
            num_workers=self.trainer.args.dataloader_num_workers,
            batch_size=32,
            shuffle=False,
            collate_fn=inference_collate_fn,
        )

    def on_train_begin(self, args, state, control, **kwargs):
        self.on_epoch_end(args, state, control, **kwargs)

    def on_epoch_end(self, args, state, control, **kwargs):
        local_rank = args.local_rank if args.local_rank != -1 else 0
        with cp.cuda.Device(local_rank):
            torch.cuda.empty_cache()
            print("Callback on local_rank =", local_rank)
            self.trainer.model.eval()
            print("On Epoch Begin")
            topic_embs = []
            device = f"cuda:{local_rank}" if torch.cuda.is_available() else "cpu"

            with torch.no_grad():
                for inputs in tqdm(self.val_topic_dataloader):
                    for k, v in inputs.items():
                        inputs[k] = inputs[k].to(device)
                    out = self.trainer.model.feature(inputs)
                    topic_embs.extend(out.cpu().detach().numpy())

                content_embs = []
                # TODO: only use original content embeddings to avoid translation confusing
                for inputs in tqdm(self.content_dataloader):
                    for k, v in inputs.items():
                        inputs[k] = inputs[k].to(device)
                    out = self.trainer.model.feature(inputs)
                    content_embs.extend(out.cpu().detach().numpy())

            # Transfer predictions to gpu
            with cp.cuda.Device(local_rank):
                topic_embs_gpu = cp.array(topic_embs)
                content_embs_gpu = cp.array(content_embs)

            # Release memory
            torch.cuda.empty_cache()

            # KNN model
            content_idx_to_id = {}
            for idx, row in self.content_df.iterrows():
                content_idx_to_id[idx] = row.id

            print("Evaluating current score...")
            if self.use_translated:
                # get 500 nearest contents, then select top k contents that is in original contents, just approximate, can't check all
                original_indices = [  # indices of original contents in self.content_df
                    i
                    for i, emb in enumerate(content_embs)
                    if self.content_df.id.values[i].startswith("c_")
                ]
                # original_content_embs = [
                #     emb
                #     for i, emb in enumerate(content_embs)
                #     if self.content_df.id.values[i].startswith("c_")
                # ]
                # original_content_embs_gpu = cp.array(original_content_embs)
                original_content_embs_gpu = content_embs_gpu

                neighbors_model = NearestNeighbors(n_neighbors=500, metric="cosine")
                neighbors_model.fit(original_content_embs_gpu)

                indices = neighbors_model.kneighbors(topic_embs_gpu, return_distance=False)
                for selected_k in [5, 10, 20, 50, 100, 200]:
                    predictions = []
                    for k in tqdm(range(len(indices))):
                        pred = indices[k]
                        # original_contents = [self.content_df.loc[ind, "id"] for ind in pred.get() if self.content_df.loc[ind, "id"].startswith("c_")]
                        # original_contents = [content_idx_to_id[ind] for ind in pred.get() if content_idx_to_id[ind].startswith("c_")]
                        original_contents = [
                            content_idx_to_id[original_indices[ind]] for ind in pred.get()
                        ]
                        p = " ".join(original_contents[:selected_k])
                        predictions.append(p)

                    knn_preds = pd.DataFrame(
                        {"topic_id": self.val_topic_ids, "content_ids": predictions}
                    ).sort_values("topic_id")

                    gt = self.correlation_df[
                        self.correlation_df.topic_id.isin(self.val_topic_ids)
                    ].sort_values("topic_id")
                    score = get_pos_score(
                        gt["content_ids"],
                        knn_preds.sort_values("topic_id")["content_ids"],
                        selected_k,
                    )
                    print(
                        "Selecting",
                        selected_k,
                        "nearest contents",
                        "top-k score =",
                        f2_score(
                            gt["content_ids"],
                            knn_preds.sort_values("topic_id")["content_ids"],
                        ),
                        "max positive score =",
                        score,
                    )

                print("Training KNN model...")
                print("Generating KNN predictions with top_k =", self.top_k)
                neighbors_model = NearestNeighbors(n_neighbors=self.top_k, metric="cosine")
                neighbors_model.fit(original_content_embs_gpu)

                print("Generating embedding for validation topics")
                indices = neighbors_model.kneighbors(topic_embs_gpu, return_distance=False)
                predictions = []
                for k in tqdm(range(len(indices))):
                    pred = indices[k]
                    # original_contents = [self.content_df.loc[ind, "id"] for ind in pred.get() if self.content_df.loc[ind, "id"].startswith("c_")]
                    # original_contents = [content_idx_to_id[ind] for ind in pred.get() if content_idx_to_id[ind].startswith("c_")]
                    original_contents = [
                        content_idx_to_id[original_indices[ind]] for ind in pred.get()
                    ]
                    p = " ".join(original_contents[: self.top_k])
                    predictions.append(p)
            else:
                for selected_k in [5, 10, 20, 50, 100, 200]:
                    neighbors_model = NearestNeighbors(n_neighbors=selected_k, metric="cosine")
                    neighbors_model.fit(content_embs_gpu)

                    indices = neighbors_model.kneighbors(topic_embs_gpu, return_distance=False)
                    predictions = []
                    for k in tqdm(range(len(indices))):
                        pred = indices[k]
                        # p = " ".join([self.content_df.loc[ind, "id"] for ind in pred.get()])
                        p = " ".join([content_idx_to_id[ind] for ind in pred.get()])
                        predictions.append(p)

                    knn_preds = pd.DataFrame(
                        {"topic_id": self.val_topic_ids, "content_ids": predictions}
                    ).sort_values("topic_id")

                    gt = self.correlation_df[
                        self.correlation_df.topic_id.isin(self.val_topic_ids)
                    ].sort_values("topic_id")
                    score = get_pos_score(
                        gt["content_ids"],
                        knn_preds.sort_values("topic_id")["content_ids"],
                        selected_k,
                    )
                    print(
                        "Selecting",
                        selected_k,
                        "nearest contents",
                        "top-k score =",
                        f2_score(
                            gt["content_ids"],
                            knn_preds.sort_values("topic_id")["content_ids"],
                        ),
                        "max positive score =",
                        score,
                    )

                print("Training KNN model...")
                print("Generating KNN predictions with top_k =", self.top_k)
                neighbors_model = NearestNeighbors(n_neighbors=self.top_k, metric="cosine")
                neighbors_model.fit(content_embs_gpu)

                print("Generating embedding for validation topics")
                indices = neighbors_model.kneighbors(topic_embs_gpu, return_distance=False)
                predictions = []
                for k in tqdm(range(len(indices))):
                    pred = indices[k]
                    # p = " ".join([self.content_df.loc[ind, "id"] for ind in pred.get()])
                    p = " ".join([content_idx_to_id[ind] for ind in pred.get()])
                    predictions.append(p)

            knn_preds = pd.DataFrame(
                {"topic_id": self.val_topic_ids, "content_ids": predictions}
            ).sort_values("topic_id")

            score = get_pos_score(
                gt["content_ids"],
                knn_preds.sort_values("topic_id")["content_ids"],
                self.top_k,
            )
            print("Current Score:", score, "Best Score:", self.best_score)

            if score > self.best_score:
                self.best_score = score
                print("saving best model to data/ folder")
                # torch.save(self.trainer.model.state_dict(), f"data/siamese_model_{score}.pth")

            generate_new_dataset_every_epoch = True
            if generate_new_dataset_every_epoch or (score == self.best_score):
                # generate new pairs in dataset
                print("Building new validation supervised df")
                new_val_supervised_df = build_new_supervised_df(knn_preds, self.correlation_df)[
                    ["topic_id", "content_ids", "target"]
                ].sort_values(["topic_id", "content_ids"])
                if score == self.best_score:  # only save for the best checkpoint
                    print("saving new_val_supervised_df to data/ folder")
                    new_val_supervised_df.to_csv("data/new_val_supervised_df.csv")

                # get top-k for training set
                # TODO: only get original content neighbors for original topics
                print("Generating embedding for train topics")
                train_topic_embs = []

                with torch.no_grad():
                    for inputs in tqdm(self.train_topic_dataloader):
                        for k, v in inputs.items():
                            inputs[k] = inputs[k].to(device)
                        out = self.trainer.model.feature(inputs)
                        train_topic_embs.extend(out.cpu().detach().numpy())

                with cp.cuda.Device(local_rank):
                    train_topic_embs_gpu = cp.array(train_topic_embs)

                train_indices = neighbors_model.kneighbors(
                    train_topic_embs_gpu, return_distance=False
                )

                # if self.use_translated:
                #     topic_language_df = pd.DataFrame({
                #         "topic_id": self.train_topic_ids,
                #         "language": self.train_topic_languages
                #     })

                train_predictions = []
                for k in tqdm(range(len(train_indices))):
                    pred = train_indices[k]
                    # p = " ".join([self.content_df.loc[ind, "id"] for ind in pred.get()])
                    if self.use_translated:
                        p = " ".join(
                            [content_idx_to_id[original_indices[ind]] for ind in pred.get()]
                        )
                    else:
                        p = " ".join([content_idx_to_id[ind] for ind in pred.get()])

                    train_predictions.append(p)

                train_knn_preds = pd.DataFrame(
                    {
                        "topic_id": self.train_topic_ids,
                        "content_ids": train_predictions,
                        "language": self.train_topic_languages,
                    }
                ).sort_values("topic_id")

                print("Building new train supervised df")
                # if self.use_translated:
                #     count_dict = {
                #         "ar": 3701,
                #         "as": 167,
                #         "bg": 2867,
                #         "bn": 2176,
                #         "en": 36161,
                #         "es": 13910,
                #         "fil": 247,
                #         "fr": 3701,
                #         "gu": 2320,
                #         "hi": 1786,
                #         "it": 866,
                #         "km": 121,
                #         "kn": 119,
                #         "mr": 300,
                #         "mul": 4,
                #         "my": 135,
                #         "or": 70,
                #         "pl": 43,
                #         "pnb": 51,
                #         "pt": 4177,
                #         "ru": 34,
                #         "sw": 2860,
                #         "swa": 35,
                #         "ta": 60,
                #         "te": 93,
                #         "tr": 40,
                #         "ur": 66,
                #         "zh": 862,
                #     }

                #     times_positive_samples = 4

                #     # select all original topics and a part of translated topics
                #     translated_knn_preds = (
                #         train_knn_preds[~train_knn_preds.topic_id.str.startswith("t_")]
                #         .groupby("language")
                #         .apply(
                #             lambda x: x.sample(
                #                 n=count_dict[x["language"].iat[0]] * times_positive_samples,
                #                 replace=True,
                #             )
                #         )
                #         .reset_index(drop=True)
                #     )
                #     original_knn_preds = train_knn_preds[
                #         train_knn_preds.topic_id.str.startswith("t_")
                #     ]

                #     train_knn_preds = pd.concat([original_knn_preds, translated_knn_preds])

                new_train_supervised_df = build_new_supervised_df(
                    train_knn_preds, self.correlation_df
                )

                if self.use_translated:
                    # Only add positive cases in training set for translated topics
                    translated_supervised_df = new_train_supervised_df[
                        ~new_train_supervised_df.topic_id.str.startswith("t_")
                        & new_train_supervised_df.target
                        == 1
                    ].copy()

                    # Only original contents for original topics
                    original_supervised_df = new_train_supervised_df[
                        new_train_supervised_df.topic_id.str.startswith("t_")
                        & new_train_supervised_df.content_ids.str.startswith("c_")
                    ].copy()

                    # TODO: duplicate number of positive by using translated data
                    id_to_language = {}
                    for _, row in tqdm(self.topic_df.iterrows()):
                        id_to_language[row.id] = row.language

                    original_supervised_df["language"] = original_supervised_df["topic_id"].apply(
                        lambda x: id_to_language[x]
                    )
                    count_df = (
                        original_supervised_df[original_supervised_df.target == 1]
                        .groupby("language")
                        .size()
                        .reset_index(name="counts")
                    )

                    count_dict = {}
                    for _, row in count_df.iterrows():
                        count_dict[row.language] = row.counts

                    times_positive_samples = 3
                    translated_supervised_df["language"] = translated_supervised_df[
                        "topic_id"
                    ].apply(lambda x: id_to_language[x])
                    translated_supervised_df = (
                        translated_supervised_df.groupby("language")
                        .apply(
                            lambda x: x.sample(
                                n=count_dict[x["language"].iat[0]] * times_positive_samples,
                                replace=True,
                            )
                        )
                        .reset_index(drop=True)
                    )
                    original_supervised_df = original_supervised_df.drop(columns=["language"])
                    translated_supervised_df = translated_supervised_df.drop(columns=["language"])

                    new_train_supervised_df = pd.concat(
                        [translated_supervised_df, original_supervised_df]
                    )[["topic_id", "content_ids", "target"]].sort_values(
                        ["topic_id", "content_ids"]
                    )

                if score == self.best_score:  # only save for the best checkpoint
                    print("saving new_train_supervised_df to data/ folder")
                    new_train_supervised_df.to_csv("data/new_train_supervised_df.csv")

                # update train_dataset and val_dataset
                print("preprocess csv for train/validation topics, contents, labels")
                self.trainer.train_dataset.supervised_df = new_train_supervised_df.dropna()
                (
                    self.trainer.train_dataset.topic_texts,
                    self.trainer.train_dataset.content_texts,
                    self.trainer.train_dataset.labels,
                ) = self.trainer.train_dataset.process_csv()

                self.trainer.eval_dataset.supervised_df = new_val_supervised_df.dropna()
                (
                    self.trainer.eval_dataset.topic_texts,
                    self.trainer.eval_dataset.content_texts,
                    self.trainer.eval_dataset.labels,
                ) = self.trainer.eval_dataset.process_csv()
                print("Saving knn csvs ...")
                train_knn_preds.to_csv(f"data/train_knn_fold{self.fold}.csv")
                knn_preds.to_csv(f"data/val_knn_fold{self.fold}.csv")

                del (
                    train_topic_embs,
                    train_topic_embs_gpu,
                    train_knn_preds,
                    train_indices,
                    train_predictions,
                )
                gc.collect()

            del (
                topic_embs,
                content_embs,
                topic_embs_gpu,
                content_embs_gpu,
                knn_preds,
                indices,
                neighbors_model,
                predictions,
            )
            gc.collect()
            torch.cuda.empty_cache()
            if self.mix_translated:
                self.use_translated = not self.use_translated

def build_new_supervised_df(knn_df, correlations):
    # Create lists for training
    topics_ids = []
    content_ids = []
    targets = []

    # Iterate over each topic in df
    mapping = set()

    # get all class 1 in correlations
    topic_ids = set(knn_df.topic_id.values)
    filtered_correlations = correlations[correlations["topic_id"].isin(topic_ids)]
    for i, row in tqdm(filtered_correlations.iterrows()):
        if str(row["content_ids"]) and str(row["content_ids"]) != "nan":
            content_ids = str(row["content_ids"]).split(" ")
            for content_id in content_ids:
                mapping.add((row["topic_id"], content_id, 1))

    for i, row in tqdm(knn_df.iterrows()):
        if str(row["content_ids"]) and str(row["content_ids"]) != "nan":
            content_ids = str(row["content_ids"]).split(" ")
            for content_id in content_ids:
                if (
                    row["topic_id"],
                    content_id,
                    1,
                ) not in mapping:  # because mapping already contains all positive cases
                    mapping.add((row["topic_id"], content_id, 0))

    # Build training dataset
    mapping = list(mapping)
    new_df = pd.DataFrame(
        {
            "topic_id": [item[0] for item in mapping if item[1]],
            "content_ids": [item[1] for item in mapping if item[1]],
            "target": [item[2] for item in mapping if item[1]],
        }
    )
    # Release memory
    del topics_ids, content_ids
    gc.collect()
    return new_df

def collate_fn(batch):
    batch = default_collate(batch)

    topic_inputs, content_inputs, combined_inputs, labels = batch
    mask_len = int(topic_inputs["attention_mask"].sum(axis=1).max())
    for k, v in topic_inputs.items():
        topic_inputs[k] = topic_inputs[k][:, :mask_len]

    mask_len = int(content_inputs["attention_mask"].sum(axis=1).max())
    for k, v in content_inputs.items():
        content_inputs[k] = content_inputs[k][:, :mask_len]

    mask_len = int(combined_inputs["attention_mask"].sum(axis=1).max())
    for k, v in combined_inputs.items():
        combined_inputs[k] = combined_inputs[k][:, :mask_len]

    return {
        "topic_inputs": topic_inputs,
        "content_inputs": content_inputs,
        "combined_inputs": combined_inputs,
        "labels": labels,
    }


from typing import Optional

#se verifica daca treeul este symetric
class TreeNode:
     def __init__(self, val=0, left=None, right=None):
        self.val = val
        self.left = left
        self.right = right
class Solution:
    def isSymmetric(self, root: Optional[TreeNode]) -> bool:
        if not root or not root.left and not root.right:
            return True
        stack=[root.left,root.right]
        while stack:
            num_items=len(stack)
            for i in range(num_items//2):
                node1=stack.pop()
                node2=stack.pop()
                if node1 and not node2 or node2 and not node1:
                    return False
                elif not node1 and not node2:
                    continue
                elif node1.data!=node2.data:
                    return False
                else:
                    stack.append(node1.left)
                    stack.append(node2.right)
                    stack.append(node1.right)
                    stack.append(node2.left)

        return True
root=TreeNode()
root.data="root"
root.left=TreeNode()
root.left.data = "a"
root.right = TreeNode()
root.right.data = "a"
root.left.left=TreeNode()
root.left.left.data="a"
root.right.right=TreeNode()
root.right.right.data="a"

if Solution.isSymmetric(self=Solution,root=root):
    print("true")
else:
    print("false")


import math
from collections import defaultdict
import heapq
from itertools import permutations
from itertools import combinations
from itertools import combinations_with_replacement
from collections import Counter
import random

def test_case():
	n, a, b = list(map(int, input().split()))
	arr = list(map(int, input().split()))
	arr = [0, 0] + arr

	count = 0

	def find_par(node, x):
		res = 0
		while x > 0 and node > 0:
			node = arr[node]
			x -= 1

		return node

	for i in range(1, n + 1):
		for j in range(1, n + 1):
			vis = [0 for i in range(n + 1)]
			temp = 0
			node = i
			while node > 0:
				vis[node] = 1
				temp += 1
				node = find_par(node, a)
			node = j
			while node > 0:
				if vis[node] != 1:
					temp += 1
				node = find_par(node, b)
			count += temp

	print(count / (n ** 2))

def main():
	T = int(input())
	for i in range(1, T+1):
		print("Case #{}: ".format(i), end = "")
		test_case()

if __name__=="__main__":
	main()


import solid as sp
import solid.utils as spu

from frame.materials import tube
from frame.utils import entrypoint

from . import column_mount, instrument_panel, throttle, arm, arm_mount, wheel, wheel_mount
from .dimensions import column_diameter, column_length

def assembly():
    column = tube.volume(diameter=column_diameter, wall_thickness=2., length=column_length)

    return sp.union()(
        spu.up(column_length)(
            spu.up(wheel.plate_thickness / 2.)(
                sp.color('red')(wheel.volume()),
                spu.up(wheel.plate_thickness / 2.)(
                    sp.color('green')(instrument_panel.assembly())
                ),
                sp.translate((150, 80))(
                    sp.rotate((0., 60., 0.))(
                        sp.color('blue')(throttle.assembly())
                    )
                ),
            ),
            sp.rotate((0, 180, 0))(sp.color('cyan')(wheel_mount.volume())),
        ),
        sp.color('magenta')(column),
        spu.up(440.)(sp.color('purple')(column_mount.upper.assembly())),
        spu.up(60.)(sp.color('grey')(column_mount.lower.assembly())),
        sp.rotate((0, 0, 0))(
            sp.color('orange')(arm_mount.volume()),
            sp.color('pink')(spu.down(arm.thickness)(arm.volume())),
        ),
    )

if __name__ == '__main__':
    entrypoint.main(assembly())


"""
自动轨迹绘制
"""
import turtle as t
t.title("自动轨迹绘制")
t.setup(800, 600, 0, 0)
t.pencolor("red")
t.pensize(5)
#数据读取
datals = []
f = open("../resources/data.txt", encoding="utf-8")
for line in f:
    #去掉当前行末尾的换行符
    line = line.replace("\n", "")
    #用逗号分隔当前行，并把分割后得到的列表中的每个元素应用eval函数，
    # 这里的map就是起到每个元素作为参数传递到前面的函数中去
    #最后再把获取的值组成一个list追加到datals中
    datals.append(list(map(eval, line.split(","))))
f.close()
#自动绘制
for i in range(len(datals)):
    t.pencolor(datals[i][3], datals[i][4], datals[i][5])
    t.fd(datals[i][0])
    if datals[i][1]:
        t.right(datals[i][2])
    else:
        t.left(datals[i][2])
t.done()


class DjangoModelPermissionsWithRead(DjangoModelPermissions):
    perms_map = {
        'GET': ['%(app_label)s.view_%(model_name)s'],
        'OPTIONS': [],
        'HEAD': [],
        'POST': ['%(app_label)s.add_%(model_name)s'],
        'PUT': [],
        'PATCH': [],
        'DELETE': ['%(app_label)s.delete_%(model_name)s'],
    }


from django.contrib import admin
from django.urls import path, include
from django.http import HttpResponse

def homepage(request):
    return HttpResponse("you're in the home page, goto polls.")

urlpatterns = [
    path('admin/', admin.site.urls),
    path('', homepage),
    path('polls/', include('polls.urls')),
]


# -*- coding: utf-8 -*-
"""
Created on Tue Jun 23 11:31:08 2020

@author: dkafkes
"""

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt

df = pd.read_csv('master_stdev.csv', header = 0, skiprows = list(np.arange(1, 177)))
df.drop(columns = ['Filename'], inplace = True)
df = df.set_index('Unnamed: 0')
#%%

x = df['B:IMINER']
array, bins, patches = plt.hist(x, bins = 100)
plt.title("B:IMINER Standard Deviation Spread")
plt.xlabel("Average Standard Deviation")
plt.ylabel("Log(Files)")
plt.ylim(0.1, 1000)
plt.semilogy()
plt.show()


""" This module allows you to download public files from Google Drive and Dropbox """
import os
import requests
import zipfile
import logging
import patoolib
from bs4 import BeautifulSoup
import gdrivedl

# Define urls to filter by cloud service
GDRIVE_URL = 'drive.google.com'
DROPBOX_URL = 'dropbox.com'

def download_folder(url, output_folder, silent, filename=None):
    """Download Google Drive folders"""
    dl = gdrivedl.GDriveDL(quiet=silent, overwrite=False, mtimes=False)
    dl.process_url(url, output_folder, filename=None)

def download_file(url, output_folder, filename, silent):
    """ Download Google Drive files"""
    dl = gdrivedl.GDriveDL(quiet=silent, overwrite=False, mtimes=False)
    dl.process_url(url, output_folder, filename)

def gd_download(url, directory, quiet):
    """ Detects if url belongs to Google Drive folder or file and calls relavent function """
    if 'folder' in url:
        output = get_title(url)[:-15]
        output_path = directory + output
        logging.info(f"---> Downloading Google Drive folder to: {output_path}")
        download_folder(url, output_path, quiet)
        return True
    elif 'file' in url:
        temp_output = get_title(url)[:-15]
        output = temp_output.split('.', 1)[0]
        logging.info(f"---> Downloading Google Drive file to {directory + temp_output}")
        download_file(url, directory, temp_output, quiet)
        unzip(temp_output, output, directory)
        return True
    else:
        return False

def get_title(url):
    """ Gets file/folder title with requests library """
    reqs = requests.get(url)
    soup = BeautifulSoup(reqs.text, 'html.parser')
    for title in soup.find_all('title'):
        return title.get_text()

def compression_type(file_name):
    """ Detects file compression type """
    ext = os.path.splitext(file_name)[-1].lower()
    return ext

def unzip(zipped_file, unzipped_file, directory):
    """ Uncompresses files and then deletes compressed folder """
    if compression_type(zipped_file) == '.zip':
        zip_path = directory + zipped_file
        unzip_path = directory + unzipped_file
        logging.info(f"--> Extracting to: {unzip_path}")
        with zipfile.ZipFile(zip_path, 'r') as zip_ref:
                zip_ref.extractall(unzip_path)
                zip_ref.close()
        os.remove(zip_path)
    if compression_type(zipped_file) == '.rar':
        zip_path = directory + zipped_file
        unzip_path = directory + unzipped_file
        logging.info(f"---> Extracting to: {unzip_path}")
        patoolib.extract_archive(zip_path, outdir=directory)
        os.remove(zip_path)
    return

def db_download(url, directory):
    """ Downloads files from Dropbox URL """
    url = url[:-1] + '0'
    file_name = get_title(url)[:-21][10:]
    logging.info(f"Dropbox file name: {file_name}")
    suffix1 = file_name.endswith(".zip")
    suffix2 = file_name.endswith(".rar")
    dl_url = url[:-1] + '1'
    filepath = directory + file_name
    logging.info(f"Downloading dropbox file to: {filepath}")
    output = file_name[:-4]
    headers = {'user-agent': 'Wget/1.16 (linux-gnu)'}
    r = requests.get(dl_url, stream=True, headers=headers)
    if r.status_code == 200:
        with open(filepath, 'wb') as f:
            for chunk in r.iter_content(chunk_size=1024):
                if chunk:
                    f.write(chunk)
        if suffix1 or suffix2:
            unzip(file_name, output, directory)
        return True
    else:
        return False

def grab(url, output_path, quiet=True):
    """
        Detects if url belongs to Google Drive or a Dropbox url and calls the relevant method.
        You may change logging level by calling grab with quiet=False).
    """
    if(quiet==True):
        logging.basicConfig(format='%(asctime)s:%(levelname)s:%(message)s', level=logging.WARNING)

    else:
        logging.basicConfig(format='%(asctime)s:%(levelname)s:%(message)s', level=logging.INFO)

    if GDRIVE_URL in url:
        if (gd_download(url, output_path, quiet)):
            return True
        else:
            logging.warning(f"The Google Drive URL {url} is not supported")
            return False
    if DROPBOX_URL in url:
        if(db_download(url, output_path)):
            return True
        else:
            logging.warning(f"The Dropbox URL {url} is not supported")
            return False
    else:
        logging.warning(f"The URL {url} is not supported")
        return False


version = "0.8"

import os, io
import chardet

from functools import wraps
from tempfile import mkstemp, mkdtemp
from json import JSONEncoder as _JSONEncoder
from pathlib import Path
from collections import deque

from colorama import Fore as F

markdown = None

class LabelledTree (object) :
    def __init__ (self, label, children=[]) :
        self.label = str(label)
        self.children = list(children)
    def _print (self, out, prefix=None, last=True) :
        if prefix is None :
            out.write(f"{self.label}\n")
        elif last :
            out.write(f"{prefix}{F.WHITE}└─{F.RESET} {self.label}\n")
        else :
            out.write(f"{prefix}{F.WHITE}├─{F.RESET} {self.label}\n")
        for child in self.children :
            if prefix is None :
                child._print(out, "", child is self.children[-1])
            elif last :
                child._print(out, prefix + "   ", child is self.children[-1])
            else :
                child._print(out, prefix + f"{F.WHITE}│{F.RESET}  ", child is self.children[-1])
    def __str__ (self) :
        out = io.StringIO()
        self._print(out)
        return out.getvalue().rstrip()

class tree (dict) :
    def __getattr__ (self, key) :
        cls = self.__class__
        val = self.get(key, None)
        if isinstance(val, dict) and not isinstance(val, cls) :
            val = self[key] = tree(val)
        elif isinstance(val, list) :
            val = self[key] = [tree(v) if isinstance(v, dict) and not isinstance(v, cls)
                               else v for v in val]
        return val
    def __setattr__ (self, key, val) :
        if isinstance(val, dict) :
            val = self.__class__(val)
        self[key] = val

cwd = Path().absolute()

def new_path (type="file", **args) :
    if type == "file" :
        fd, path = mkstemp(**args)
        os.close(fd)
    elif type == "dir" :
        path = mkdtemp(**args)
    else :
        raise ValueError(f"unsupported path type {type!r}")
    return Path(path).absolute().relative_to(cwd)

encoding = tree(encoding="utf-8",
                errors="replace")

class JSONEncoder (_JSONEncoder) :
    def default (self, obj) :
        handler = getattr(obj, "__json__", None)
        if handler is None :
            return super().default(obj)
        else :
            return handler()

def cached_property (method) :
     @wraps(method)
     def wrapper (self) :
         name = method.__name__
         if not hasattr(self, "__cache") :
             self.__cache = {}
         if name not in self.__cache :
             self.__cache[name] = method(self)
         return self.__cache[name]
     @wraps(method)
     def delete (self) :
         self.__cache.pop(method.__name__, None)
     return property(wrapper, None, delete, method.__doc__)

def recode (path) :
    with open(path, "rb") as inf :
        raw = inf.read()
    try :
        enc = chardet.detect(raw)
        src = raw.decode(enc["encoding"], errors="replace")
    except :
        return
    with open(path, "w", **encoding) as out :
        out.write(src)
    return src

def md (text, inline=True) :
    # only load if necessary to speedup prog startup
    global markdown
    from markdown import markdown
    #
    try :
        html = markdown(str(text))
        if inline :
            html = html.replace("<p>", "").replace("</p>", "")
        return html.replace("§", " ")
    except :
        return text.replace("§", " ")

_esc = {c : f"\\{c}" for c in r"\`*_{}[]()#+-.!"}

def mdesc (text) :
    return str(text).translate(_esc)

def chmod_r (path) :
    q = deque([Path(path)])
    while q :
        sub = q.popleft()
        if sub.is_dir() :
            sub.chmod(sub.stat().st_mode | 0o750)
            q.extend(sub.iterdir())
        else :
            sub.chmod(sub.stat().st_mode | 0o640)


import pdb
import unittest
import json

from objbrowser import browse
import mock
from mock import patch

import music_server
from music_server import youtube_search
from music_server import config

class YoutubeSearchTestCase(unittest.TestCase):

    def setUp(self):
        pass
    def tearDown(self):
        pass

    def test_format_query(self):
        # given
        search_query = "simple_query"
        expected_result = "http://youtube.com/results?search_query=simple_query"
        # when
        query = youtube_search.format_query(search_query)
        # then
        self.assertEqual(query, expected_result)

    def test_format_query_with_space(self):
        # given
        search_query = "a b"
        expected_result = "http://youtube.com/results?search_query=a+b"
        # when
        query = youtube_search.format_query(search_query)
        # then
        self.assertEqual(query, expected_result)

    def test_format_with_plus(self):
        # given
        search_query = "a+b"
        expected_result = "http://youtube.com/results?search_query=a%2Bb"
        # when
        query = youtube_search.format_query(search_query)
        # then
        self.assertEqual(query, expected_result)

    def test_fetch_first_result_when_empty(self):
        self.assertRaises(TypeError, youtube_search.fetch_results, None)

    # empty list
    def test_fetch_first_result_when_no_result(self):
        # given
        html_content = "wrong html content"
        # when
        result = youtube_search.fetch_results(html_content)
        # then
        self.assertFalse(result, 'Result should be an empty list')

    def test_fetch_results(self):
        # given
        with open(config.test_resources_folder + 'youtube_search_pratos_osni.html', 'r') as myfile:
            html_content = myfile.read()
        with open(config.test_resources_folder + 'youtube_search_pratos_osni.json', 'r') as myfile2:
            expected_links = json.loads(myfile2.read())
        # when
        results = youtube_search.fetch_results(html_content)
        # then
        self.assertEqual(results, expected_links)

    @patch('music_server.youtube_search.get_html')
    def test_youtube_search(self, test_patch):
        # given
        with open(music_server.config.test_resources_folder + 'youtube_search_pratos_osni.html') as fh:
            mock_html = fh.read()
            test_patch.return_value = mock_html
        with open(config.test_resources_folder + 'youtube_search_pratos_osni.json', 'r') as myfile2:
            expected_links = json.loads(myfile2.read())
        # when
            results = youtube_search.YoutubeSearch("pratos osni").video_ids
        # then
        self.assertEqual(results, expected_links)

    def test_search_empty(self):
        # given
        search_query = ''
        # when
        results = youtube_search.YoutubeSearch(search_query)
        # then
        self.assertTrue(results)

    def test_search_none(self):
        # given
        search_query = None
        # when
        results = youtube_search.YoutubeSearch(search_query)
        # then
        self.assertTrue(results)

if __name__ == '__main__':
    unittest.main()


# my_file = open("data.txt")
# contents = my_file.read()
# print(contents)
# my_file.close()

# automaattisesti sulkee tiedoston lopussa
with open("data.txt") as my_file:
    contents = my_file.read()
    print(contents)

# read on moden default
with open("data.txt", mode="w") as my_file_again:
    my_file_again.write("Content changed to this.\n")

with open("data.txt") as my_file:
    contents = my_file.read()
    print(contents)

with open("data.txt", mode="a") as my_file_third_time:
    my_file_third_time.write("Append this line to content.")

with open("data.txt") as my_file:
    contents = my_file.read()
    print(contents)

with open("non_existent.txt", mode="w") as some_file:
    some_file.write("Write a line to a file that does not yet exist.")

with open("non_existent.txt") as my_file:
    contents = my_file.read()
    print(contents)


from .atari import Atari
from .obj3d import Obj3D
from torch.utils.data import DataLoader
from object_detector import CLIPort_Dataset

__all__ = ['get_dataset', 'get_dataloader']

def get_dataset(cfg, mode):
    assert mode in ['train', 'val', 'test']
    return CLIPort_Dataset(cfg.dataset_roots.TABLE, mode)

def get_dataloader(cfg, mode):
    assert mode in ['train', 'val', 'test']

    batch_size = getattr(cfg, mode).batch_size
    shuffle = True if mode == 'train' else False
    num_workers = getattr(cfg, mode).num_workers

    dataset = get_dataset(cfg, mode)
    dataloader = DataLoader(dataset, batch_size=batch_size, shuffle=shuffle, num_workers=num_workers)

    return dataloader


import random
karakterler= "qwertyuıopasdfghjklzxcvbnmiIQWERTYUOPASDFGHJKLZXCVBNM1234567890é!'^+%&/()=?_-*<>£#$½{[]}"
sifresayisi = int(input("olusturmak istediginiz sifre sayisini giriniz "))
for x in range(sifresayisi):
    sifre = ""
    for x in range(16):
        karakter = random.choice(karakterler)
        sifre = sifre + karakter
    print("Random Sifreniz : ", sifre)


# SOAl 3

# KAMUS
# x, y : int

# ALGORITMA

# membuat fungsi convert
def convert(code, TC):
    if code == 'F':
        hasil = ((9/5)*TC)+32
    elif code == 'R':
        hasil = (4/5)*TC
    else:
        hasil = TC + 273
    return f'{hasil} {code}'

# mengingput code dan besar suhu dalam celcius
code = input('kode konversi = ')
TC = float(input('Suhu (dalam celcius): '))

# panggil fungsi dan cetak
print(convert(code, TC))

Pengkom-TPB1/08 - Tugas Pengkom/PR/PR1/3.py


from icecream import ic
from stack import Stack
from datetime import datetime

def time_format():
    return f'{datetime.now().strftime("%m/%d/%Y, %I:%M:%S")}|> '

ic.configureOutput(prefix=time_format, includeContext=True)

def nextLargestElment(items):
    tempStack = Stack()
    returnStack = Stack()
    tempStack.push(items[0])
    print("Intial tempStack:", tempStack.getStack())
    print("-----------------------------------------")
    for current_item in items[1::]:
        print("current_item:", current_item)
        print("stack_top_item:", tempStack.peek())

        if tempStack.isEmpty() == False:
            stack_top_item = tempStack.pop()
            while stack_top_item < current_item:
                print(str(stack_top_item) + " -- " + str(current_item))
                returnStack.push(current_item)
                if tempStack.isEmpty():
                    break
                stack_top_item = tempStack.pop()
            if stack_top_item > current_item:
                tempStack.push(stack_top_item)
        tempStack.push(current_item)
        print("tempStack:", tempStack.getStack())
        print("-----------------------------------------")
    while tempStack.isEmpty() == False:
        element = tempStack.pop()
        returnStack.push(-1)
        next = -1
        print(str(element) + " -- " + str(next))
    return returnStack.getStack()

if __name__ == '__main__':
    # ic(nextLargestElment([int(item)
    #                       for item in input("Enter the list items : ").strip().split()]))

    ic(nextLargestElment([2, 6, 5, 4, 19]))


"""
создайте асинхронные функции для выполнения запросов к ресурсам (используйте aiohttp)
- доработайте модуль `jsonplaceholder_requests`:
    - установите значения в константы `USERS_DATA_URL` и `POSTS_DATA_URL` (ресурсы нужно взять отсюда https://jsonplaceholder.typicode.com/)
    - создайте асинхронные функции для выполнения запросов к данным ресурсам (используйте `aiohttp`)
    - рекомендуется добавить базовые функции для запросов, которые будут переиспользованы (например `fetch_json`)

"""
from aiohttp import ClientSession
import asyncio
# import logging
#
# DEFAULT_FORMAT = "%(asctime)s %(levelname)-8s [%(name)-8s] (%(filename)s:%(funcName)s:%(lineno)d) %(message)s"
#
# logging.basicConfig(format=DEFAULT_FORMAT, level=logging.DEBUG)
#
# log = logging.getLogger(__name__)

USERS_DATA_URL = "https://jsonplaceholder.typicode.com/users"
POSTS_DATA_URL = "https://jsonplaceholder.typicode.com/posts"

async def fetch_json(session: ClientSession, url: str):
    async with session.get(url) as response:
        return await response.json()

async def fetch_users():
    # log.info(f"Fetch users from {USERS_DATA_URL}")
    async with ClientSession() as session:
        json_data = await fetch_json(session, USERS_DATA_URL)
    # log.info(f"Fetch json from {USERS_DATA_URL}: {json_data}")
    return json_data

async def fetch_posts():
    # log.info(f"Fetch posts from {POSTS_DATA_URL}")
    async with ClientSession() as session:
        json_data = await fetch_json(session, POSTS_DATA_URL)
    # log.info(f"Fetch json from {POSTS_DATA_URL}: {json_data}")
    return json_data

# def main():
#     asyncio.run(fetch_users())
#     asyncio.run(fetch_posts())
#
#
# if __name__ == '__main__':
#     main()


import numpy as np
import pandas as pd
from sklearn.ensemble import RandomForestClassifier
from sklearn.decomposition import LatentDirichletAllocation
import os
from time import strftime

# Python 3.5

def load_data(filename):

    return np.loadtxt(filename, skiprows=1, delimiter=' ')

def save_predictions(X, model):

    filename = 'random_forest_' + strftime('%b%d%H%M%S') + '.csv'
    preds = model.predict(X).reshape((len(X), 1))
    ids = (np.arange(1, len(X) + 1)).reshape((len(X), 1))

    np.savetxt(
        os.path.join('predictions', filename),
        np.hstack((ids, preds)),
        fmt='%d',
        delimiter=',',
        header='Id,Prediction',
        comments=''
    )

def decompose(X, d, args={}):

    pca_model = LatentDirichletAllocation(n_components=d, **args)
    pca_model.fit(X)

    return pca_model

def train(X, y, args={}):

    model = RandomForestClassifier(**args)
    model.fit(X, y)

    return model

def test(X, y, model):

    return np.sum(model.predict(X) == y) / len(y)

train_raw = load_data('training_data.txt')
n_train = 10000
n_val = len(train_raw) - n_train

X_train, y_train = train_raw[:, 1:][:n_train], train_raw[:, 0][:n_train]
X_val, y_val = train_raw[:, 1:][n_train:], train_raw[:, 0][n_train:]

# reduce dimensions from 1000 to 200
pca_model = decompose(X_train, 10)
X_train_red = pca_model.transform(X_train)
X_val_red = pca_model.transform(X_val)

model = train(X_train_red, y_train, args={})

print('train / val split : %d / %d' % (n_train, n_val))
print('train acc :', test(X_train_red, y_train, model))
print('val acc :', test(X_val_red, y_val, model))

test_raw = load_data('test_data.txt')
X_test = test_raw[:, :]
X_test_red = pca_model.transform(X_test)

# save_predictions(X_test_red, model)

'''
<output>
train / val split : 10000 / 10000
train acc : 0.9892
val acc : 0.6557
'''


import json
import os
import gui
import wx

import addonHandler
import braille
import config
import controlTypes
import languageHandler

from .common import configDir

addonHandler.initTranslation()

CUR_LANG = languageHandler.getLanguage().split('_')[0]
PATH_JSON = os.path.join(configDir, f"roleLabels-{CUR_LANG}.json")

class SettingsDlg(gui.settingsDialogs.SettingsPanel):

	# Translators: title of a dialog.
	title = _("Role labels")

	roleLabels = {}

	def makeSettings(self, settingsSizer):
		self.roleLabels = roleLabels.copy()
		sHelper = gui.guiHelper.BoxSizerHelper(self, sizer=settingsSizer)

		self.toggleRoleLabels = sHelper.addItem(wx.CheckBox(self, label=_("Use custom braille &role labels")))
		self.toggleRoleLabels.SetValue(config.conf["brailleExtender"]["features"]["roleLabels"])
		self.toggleRoleLabels.Bind(wx.EVT_CHECKBOX, self.onToggleRoleLabels)
		self.categories = sHelper.addLabeledControl(_("Role cate&gory:"), wx.Choice, choices=[_("General"), _("Landmarks"), _("Positive states"), _("Negative states")])
		self.categories.Bind(wx.EVT_CHOICE, self.onCategories)
		self.categories.SetSelection(0)

		choices = []
		if hasattr(controlTypes, "roleLabels"):
			choices = [controlTypes.roleLabels[int(k)] for k in braille.roleLabels.keys()]

		self.labels = sHelper.addLabeledControl(_("&Role:"), wx.Choice, choices=choices)
		self.labels.Bind(wx.EVT_CHOICE, self.onLabels)

		self.label = sHelper.addLabeledControl(_("Braille &label"), wx.TextCtrl)
		self.label.Bind(wx.EVT_TEXT, self.onLabel)

		bHelper = gui.guiHelper.ButtonHelper(orientation=wx.HORIZONTAL)
		self.resetLabelBtn = bHelper.addButton(self, wx.NewId(), _("&Reset this role label"), wx.DefaultPosition)
		self.resetLabelBtn.Bind(wx.EVT_BUTTON, self.onResetLabelBtn)
		self.resetAllLabelsBtn = bHelper.addButton(self, wx.NewId(), _("Reset a&ll role labels"), wx.DefaultPosition)
		self.resetAllLabelsBtn.Bind(wx.EVT_BUTTON, self.onResetAllLabelsBtn)
		sHelper.addItem(bHelper)
		self.onToggleRoleLabels(None)
		self.onCategories(None)

	def onToggleRoleLabels(self, evt):
		l = [
			self.categories,
			self.labels,
			self.label,
			self.resetLabelBtn,
			self.resetAllLabelsBtn,
		]
		for e in l:
			if self.toggleRoleLabels.IsChecked():
				e.Enable()
			else:
				e.Disable()

	def onCategories(self, event):
		labels = []
		idCategory = self.categories.GetSelection()
		oldRoleLabels = hasattr(controlTypes, "roleLabels")
		if idCategory == 0:
			if oldRoleLabels:
				labels = [controlTypes.roleLabels[int(k)] for k in braille.roleLabels.keys()]
			else:
				labels = [role.displayString for role in braille.roleLabels.keys()]
		elif idCategory == 1:
			labels = list(braille.landmarkLabels.keys())
		elif idCategory == 2:
			if oldRoleLabels:
				labels = [controlTypes.stateLabels[k] for k in braille.positiveStateLabels.keys()]
			else:
				labels = [role.displayString for role in braille.positiveStateLabels.keys()]
		elif idCategory == 3:
			if oldRoleLabels:
				labels = [controlTypes.stateLabels[k] for k in braille.negativeStateLabels.keys()]
			else:
				labels = [role.displayString for role in braille.negativeStateLabels.keys()]
		for iLabel, label in enumerate(labels):
			idLabel = getIDFromIndexes(idCategory, iLabel)
			actualLabel = getLabelFromID(idCategory, idLabel)
			originalLabel = self.getOriginalLabel(idCategory, idLabel, actualLabel)
			labels[iLabel] += _(": %s") % actualLabel
			if actualLabel != originalLabel: labels[iLabel] += " (%s)" % originalLabel
		self.labels.SetItems(labels)
		if idCategory > -1 and idCategory < 4: self.labels.SetSelection(0)
		self.onLabels(None)

	def onLabels(self, event):
		idCategory = self.categories.GetSelection()
		idLabel = getIDFromIndexes(idCategory, self.labels.GetSelection())
		key = f"{idCategory}:{idLabel}"
		if key in self.roleLabels.keys(): self.label.SetValue(self.roleLabels[key])
		else: self.label.SetValue(self.getOriginalLabel(idCategory, idLabel))

	def onLabel(self, evt):
		idCategory = self.categories.GetSelection()
		iLabel = self.labels.GetSelection()
		idLabel = getIDFromIndexes(idCategory, iLabel)
		key = "%d:%s" % (idCategory, idLabel)
		label = self.label.GetValue()
		if idCategory >= 0 and iLabel >= 0:
			if self.getOriginalLabel(idCategory, idLabel, chr(4)) == label:
				if key in self.roleLabels.keys():
					self.roleLabels.pop(key)
			else: self.roleLabels[key] = label
			actualLabel = getLabelFromID(idCategory, idLabel)
			originalLabel = self.getOriginalLabel(idCategory, idLabel, actualLabel)
			if label != originalLabel: self.resetLabelBtn.Enable()
			else: self.resetLabelBtn.Disable()

	def onResetLabelBtn(self, event):
		idCategory = self.categories.GetSelection()
		iLabel = self.labels.GetSelection()
		idLabel = getIDFromIndexes(idCategory, iLabel)
		key = "%d:%s" % (idCategory, idLabel)
		actualLabel = getLabelFromID(idCategory, idLabel)
		originalLabel = self.getOriginalLabel(idCategory, idLabel, actualLabel)
		self.label.SetValue(originalLabel)
		self.onLabel(None)
		self.label.SetFocus()

	def onResetAllLabelsBtn(self, event):
		nbCustomizedLabels = len(self.roleLabels)
		if not nbCustomizedLabels:
			msg = _("You have no customized role labels.")
			res = gui.messageBox(msg, _("Reset role labels"),
			wx.OK|wx.ICON_INFORMATION)
			return
		msg = _("You have %d customized role labels defined. Do you want to reset all labels?") % nbCustomizedLabels
		flags = wx.YES|wx.NO|wx.ICON_INFORMATION
		res = gui.messageBox(msg, _("Reset role labels"), flags)
		if res == wx.YES:
			self.roleLabels = {}
			self.onCategories(None)

	def getOriginalLabel(self, idCategory, idLabel, defaultValue = ''):
		key = f"{idCategory}:{idLabel}"
		if key in backupRoleLabels.keys():
			return backupRoleLabels[key][1]
		return getLabelFromID(idCategory, idLabel)

	def postInit(self): self.toggleRoleLabels.SetFocus()

	def onSave(self):
		global roleLabels
		config.conf["brailleExtender"]["features"]["roleLabels"] = self.toggleRoleLabels.IsChecked()
		saveRoleLabels(self.roleLabels)
		discardRoleLabels()
		if config.conf["brailleExtender"]["features"]["roleLabels"]:
			loadRoleLabels()

backupRoleLabels = {}
roleLabels = {}

def getIDFromIndexes(idCategory, idLabel):
	oldRoleLabels = hasattr(controlTypes, "roleLabels")
	if not isinstance(idCategory, int):
		raise TypeError(f"Wrong type for idCategory ({idCategory})")
	if not isinstance(idLabel, int):
		raise TypeError(f"Wrong type for idLabel ({idLabel})")
	idRole = -1
	if idCategory == 0: idRole = list(braille.roleLabels.keys())[idLabel]
	elif idCategory == 1: idRole = list(braille.landmarkLabels.keys())[idLabel]
	elif idCategory == 2: idRole = list(braille.positiveStateLabels.keys())[idLabel]
	elif idCategory == 3: idRole = list(braille.negativeStateLabels.keys())[idLabel]
	else: raise ValueError(f"Wrong value for category ({idCategory})")
	if not oldRoleLabels and isinstance(idRole, (controlTypes.Role, controlTypes.State)):
		idRole = idRole.value
	return idRole

def getLabelFromID(idCategory, idLabel):
	if idCategory == 0: return braille.roleLabels[int(idLabel)]
	if idCategory == 1: return braille.landmarkLabels[idLabel]
	if idCategory == 2: return braille.positiveStateLabels[int(idLabel)]
	if idCategory == 3: return braille.negativeStateLabels[int(idLabel)]
	raise ValueError("Invalid value: %d" % idCategory)

def setLabelFromID(idCategory, idLabel, newLabel):
	if idCategory == 0: braille.roleLabels[int(idLabel)] = newLabel
	elif idCategory == 1: braille.landmarkLabels[idLabel] = newLabel
	elif idCategory == 2: braille.positiveStateLabels[int(idLabel)] = newLabel
	elif idCategory == 3: braille.negativeStateLabels[int(idLabel)] = newLabel
	else:
		raise ValueError(f"Unknown category {idCategory}")

def loadRoleLabels(roleLabels_=None):
	global backupRoleLabels, roleLabels
	roleLabels.clear()
	if roleLabels_:
		roleLabels.update(roleLabels_)
	elif "roleLabels" in config.conf["brailleExtender"] and config.conf["brailleExtender"]["roleLabels"].copy():
		roleLabels.update(config.conf["brailleExtender"]["roleLabels"].copy())
		saveRoleLabels(roleLabels)
		config.conf["brailleExtender"]["roleLabels"] = {}
	elif os.path.exists(PATH_JSON):
		f = open(PATH_JSON, "r", encoding="UTF-8")
		try:
			roleLabels.update(json.load(f))
		except json.decoder.JSONDecodeError:
			pass
		f.close()
	for k, v in roleLabels.items():
		idCategory, idRole = k.split(':')
		idCategory = int(idCategory)
		backupRoleLabels[k] = (v, getLabelFromID(idCategory, idRole))
		setLabelFromID(idCategory, idRole, v)

def saveRoleLabels(roleLabels_):
	f = open(PATH_JSON, 'w')
	json.dump(roleLabels_, f, ensure_ascii=False, indent=2)
	f.close()

def discardRoleLabels():
	global backupRoleLabels, roleLabels
	for k, v in backupRoleLabels.items():
		idCategory, idRole = k.split(':')
		idCategory = int(idCategory)
		setLabelFromID(idCategory, idRole, v[1])
	backupRoleLabels = {}
	roleLabels = {}


from demisto_sdk.commands.common.constants import CLASSIFIERS_DIR, PACKS_DIR
from demisto_sdk.commands.common.content.objects.pack_objects.abstract_pack_objects.json_content_object import \
    JSONContentObject
from demisto_sdk.commands.common.tools import src_root

TEST_DATA = src_root() / 'tests' / 'test_files'
TEST_CONTENT_REPO = TEST_DATA / 'content_slim'
TEST_JSON_NO_FROM_VERSION = TEST_CONTENT_REPO / PACKS_DIR / 'Sample01' / CLASSIFIERS_DIR / 'classifier-sample_new.json'

def test_to_version_no_from_version(datadir):
    from packaging.version import parse
    obj = JSONContentObject(TEST_JSON_NO_FROM_VERSION, "classifier")
    assert obj.from_version == parse("0.0.0")
    assert obj.to_version == parse("4.0.0")

class TestFileWithStem:
    def test_with_readme_change_log(self):
        obj = JSONContentObject(TEST_JSON_NO_FROM_VERSION, "classifier")

        assert obj.readme is not None
        assert obj.changelog is not None


#Faça um programa que ajude um jogador da MEGA SENA a criar palpites.O programa vai perguntar quantos jogos serão gerados e vai sortear 6 números entre 1 e 60 para cada jogo, cadastrando tudo em uma lista composta.
from random import randint
from time import sleep
jogos=int(input("Quantos jogos você deseja? "))
números_sorteados=[]
lista_jogos=[]
for c in range(jogos):
    while True:
        número=randint(1,60)
        if número not in números_sorteados:
            números_sorteados.append(número)
        if len(números_sorteados)==6:
            break
    lista_jogos.append(números_sorteados[:])
    números_sorteados.clear()
print("           MEGA SENA          ")
print("-"*33)
for c in range(jogos):
    print(f"{c+1}º jogo: {lista_jogos[c]}")
    sleep(1)
print("-"*33)
print("          BOA SORTE!!         ")


'''-Crear una subrutina llamada “Login”, que recibe un nombre de usuario y una contraseña y te devuelve Verdadero si el nombre de usuario es “admin” y la contraseña es “admin123*”. Además recibe el número de intentos que se ha intentado hacer login y si no se ha podido hacer login incremente este valor.'''

def login(usuario="",contra=""):
    intentos=0
    while(intentos < 4):
        usuario=input("Ingrese el usuario:")
        contra=input("Ingrese la contraseña:")
        if(usuario=='admin' and contra=='admin123'):
            print("Verdadero")
        else:
            print("Falso")
        intentos += 1

login()


import requests
import uuid

from datetime import datetime
import pandas as pd

# https://kcnew.ifrc.org/api/v1/forms find the kpi asset uid for forms here

#from settings import * #to import MYTOKEN and KPIASSETUID
##################
## RUN SETTINGS ##
##################

##https://kobonew.ifrc.org/token/?format=json
MYTOKEN = ""

#"kpi_asset_uid":
KPIASSETUID= ""

# https://kcnew.ifrc.org/api/v1/forms find the kpi asset uid

headers = {
    'Authorization': f'Token {MYTOKEN}',
    'Content-Type': 'application/json',
    'Accept': 'application/json'
}
now = datetime.now()
current_time = now.strftime("%Y-%b-%d %I:%M %p")

# Specify the path to your Excel file First Qtr 2022
file_path = 'data/ercs_base_wh_dummy.xlsx'

# Read the Excel file into a Pandas DataFrame
data_frame = pd.read_excel(file_path)

for index, row in data_frame.iterrows():
    # Access values in each column for the current row
    submission = {
        'meta': {
            'instanceID': f'uuid:{uuid.uuid4()}',
        },
        'Supplier_Donor':row['Supplier_Donor'],

        'Local_or_Foreign_Receival':row['Local_or_Foreign_Receival'],
        'Packing_List_Number':row['Packing_List_Number'],
        'Certificate_of_Origin':row['Certificate_of_Origin'],
        'Donation_Certificate':row['Donation_Certificate'],
        'Waybill_Number':row['Waybill_Number'],
        'Contract_Number':row['Contract_Number'],
        'Invoice_Number':row['Invoice_Number'],
        'Purchase_Requisition_Number':row['Purchase_Requisition_Number'],
        'Department_Name':row['Department_Name'],
        'Receiver':row['Receiver'],
        'Purchase_Order':row['Purchase_Order'],
        'Date_of_Reception':row['Date_of_Reception'].date().strftime("%Y-%m-%d"),
        'Items_Inspected_approved':row['Items_Inspected_approved'],
        'Received_By':row['Received_By'],
        'Received_On':row['Received_On'].date().strftime("%Y-%m-%d"),
        'Account_Number':row['Account_Number'],
        'Project_code':row['Project_code'],
        'Items':row['Items'],
        'Remark':row['Remark'],
        'EXPIRY_DATES':row['EXPIRY_DATES'].date().strftime("%Y-%m-%d"),
        'Vender_Manufacturer_No':row['Vender_Manufacturer_No'],
        'Vender_seiral_No':row['Vender_seiral_No'],
        'Unit_of_Measure':row['Unit_of_Measure'],
        'Quantity_Intial':row['Quantity_Intial'],
        'Unit_Price':row['Unit_Price'],
        'Currency_of_Purchase':row['Currency_of_Purchase'],

    }

    data_request = requests.post(
        f'https://kcnew.ifrc.org/api/v1/submissions',
        json={
            "id": f"{KPIASSETUID}",
            "submission": submission
        },
        headers=headers
    )


#! python3
# program to load current weather from api
# via cmd
# display for today and the next two days
# to run: currentWeather location
import json
import requests
import sys

if len(sys.argv) < 2:
    print('More argument pls')
    sys.exit()
location = ' '.join(sys.argv[1:])
key = ''
# download
url = 'http://api.openweathermap.org/data/2.5/forecast/daily?q=%s&cnt=3&APPID=%s' % (location, key)
print(url)
try:
    res = requests.get(url)
    res.raise_for_status()

    weatherData = json.loads(res.text)
    print(weatherData)
    w = weatherData['list']
    print('Current Weather', w)
except Exception as e:
    print(e)


from flask import Flask, request
import json
import socket
import urllib.request as urllib2
import re
from functools import wraps

application = Flask(__name__)

CONFIG = json.load(open("config.json", "r"))
API_KEYS = CONFIG["api_keys"]

def requires_auth_key(func):
    @wraps(func)
    def wrapplicationed(*args, **kwargs):
        api_key = request.form.get("api_key", None)
        if api_key not in API_KEYS:
            return "Unauthorized", 401
        else:
            if not API_KEYS[api_key]["enabled"]:
                return "Unauthorized", 401
        return func(*args, **kwargs)
    return wrapped

@application.route('/carbon/metrics', methods=["POST"])
@requires_auth_key
def post_metric():
    try:
        s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        s.connect((CONFIG["carbon"]["host"], int(CONFIG["carbon"]["port"])))
    except Exception as e:
        return "
Error: %s
" % e, 500
    else:
        data = request.form.get('data');
        if (data != None):
            data = re.findall("([\w\.]+\ [\S]+\ [\d]+)",request.form.get('data'), re.MULTILINE);
        else:
            data = request.form.getlist('data[]')

        sentCmd = 0
        for str in data:
            str = re.findall("([\w\.]+\ [\S]+\ [\d]+)",str);
            str = str[0]
            if (len(str) < 10):
                continue
            str += "\n"
            #print(("Send:"+str).encode('utf8'))
            s.send(b"%s" % str.encode('utf8'))
            sentCmd+=1
        s.close()
        if sentCmd < 1:
            return "NOTHING SENT TO SERVER. BAD FORMATED STRING/VAR?", 202
        return "OK", 200
    return "Unkown error", 500

@application.route('/carbon/events', methods=["POST"])
@requires_auth_key
def post_event():
    req = urllib2.Request('http://{host}:{port}/events'.format(**CONFIG["graphite"]),
        data=request.form.get('data').encode('utf8'), headers={'Content-type': 'application/json'})
    try:
        urllib2.urlopen(req)
    except Exception as e:
        return "
Error: %s
" % e, 500
    else:
        return "OK", 200

    return "Unkown error", 500

if __name__ == "__main__":
    application.run(debug=False, use_reloader=False, host="127.0.0.1", port=8081, threaded=True)


# update.py
import requests
import json
import tarfile

url = "https://ddragon.leagueoflegends.com/api/versions.json"
response = requests.get(url)
obj = response.json()

patch = str(obj[0])

zipUrl = "https://ddragon.leagueoflegends.com/cdn/dragontail-" + patch + ".tgz"
print(zipUrl)

data = requests.get(zipUrl)

with open("src/assets/prev-data/dragontail-" + patch + ".tgz", 'wb') as f:
    # opening the file in write mode
    f.write(data.content)

tgzFile = tarfile.open("src/assets/prev-data/dragontail-10.22.1.tgz", 'r')

print('Extracting one file...')
tgzFile.extractall('src/assets/prev-data/data-hold')

print('Extracting Done!')
tgzFile.close()


import sys, os, re
import unittest
from itertools import product as prod

from timeit import Timer
import time
import math

import logging

import numpy as np

from scipy.optimize import fmin, fmin_bfgs

from hydrodiy.stat.transform import BoxCox2
from hydrodiy.data.containers import Vector

from pygme.model import Model, ParameterCheckValueError
from pygme.calibration import Calibration, CalibParamsVector
from pygme.calibration import ObjFunSSE, ObjFunBCSSE, \
                                ObjFunKGE, ObjFunBiasBCSSE
from pygme.calibration import CalibrationExplorationError

from dummy import Dummy, CalibrationDummy, ObjFunSSEargs

BC = BoxCox2()

# Set logger
LOGGER = logging.getLogger('pygme.Calibration')
fmt='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
ft = logging.Formatter(fmt)
sh = logging.StreamHandler(sys.stdout)
sh.setFormatter(ft)
LOGGER.addHandler(sh)

class ObjFunTestCase(unittest.TestCase):

    def setUp(self):
        print('\t=> ObjFunTestCase')

        nval = 1000
        obs = np.random.uniform(0., 1, size=nval)
        idx = np.random.choice(np.arange(nval), nval//100)
        obs[idx] = np.nan
        self.obs = obs

        sim = np.random.uniform(0., 1, size=nval)
        idx = np.random.choice(np.arange(nval), nval//100)
        sim[idx] = np.nan
        self.sim = sim

    def test_print(self):
        of = ObjFunBCSSE(0.2)
        print(of)

        of = ObjFunSSE()
        print(of)

        of = ObjFunKGE()
        print(of)

    def test_SSE(self):
        obs, sim = self.obs, self.sim
        idx = (~np.isnan(obs)) & (~np.isnan(sim))

        of = ObjFunSSE()
        value = of.compute(obs[idx], sim[idx])
        err = self.obs-self.sim
        expected = np.nansum(err*err)
        self.assertTrue(np.allclose(value, expected))

        value = of.compute(obs, sim)
        self.assertTrue(np.isnan(value))

    def test_KGE(self):
        of = ObjFunKGE()
        obs, sim = self.obs, self.sim
        idx = (~np.isnan(obs)) & (~np.isnan(sim))

        value = of.compute(obs[idx], sim[idx])

        obsok, simok = obs[idx], sim[idx]
        bias = np.mean(simok)/np.mean(obsok)
        rstd = np.std(simok)/np.std(obsok)
        corr = np.corrcoef(obsok, simok)[0, 1]
        expected = 1-math.sqrt((1-bias)**2+(1-rstd)**2+(1-corr)**2)
        self.assertTrue(np.allclose(value, expected))

        value = of.compute(obs, sim)
        self.assertTrue(np.isnan(value))

    def test_BCSSE(self):
        ''' test the BCSSE objfun '''
        obs, sim = self.obs, self.sim
        idx = (~np.isnan(obs)) & (~np.isnan(sim))

        for lam, nu in prod([0.1, 0.5, 1., 2], [1e-4, 1e-2, 1]):
            of = ObjFunBCSSE(lam, nu)
            assert of.name == f"BCSSE{lam:0.1f}"

            value = of.compute(obs[idx], sim[idx])

            BC.lam = lam
            BC.nu = nu
            err = BC.forward(obs)-BC.forward(sim)
            expected = np.nansum(err*err)

            self.assertTrue(np.isclose(value, expected))

            value = of.compute(obs, sim)
            self.assertTrue(np.isnan(value))

    def test_BiasBCSSE(self):
        ''' test the BiasBCSSE objfun '''
        obs, sim = self.obs, self.sim
        idx = (~np.isnan(obs)) & (~np.isnan(sim))
        mo = obs[idx].mean()
        ms = sim[idx].mean()

        for lam, nu in prod([0.1, 0.5, 1., 2], [1e-4, 1e-2, 1]):
            of = ObjFunBiasBCSSE(lam, nu)
            assert of.name == f"BiasBCSSE{lam:0.1f}"

            value = of.compute(obs[idx], sim[idx])

            BC.lam = lam
            BC.nu = nu
            err = BC.forward(obs)-BC.forward(sim)

            expected = np.nansum(err*err)*(1+abs(ms-mo)/mo)
            self.assertTrue(np.isclose(value, expected))

            value = of.compute(obs, sim)
            self.assertTrue(np.isnan(value))

class CalibParamsVectorTestCase(unittest.TestCase):

    def setUp(self):
        print('\t=> CalibParamsVectorTestCase')

        config = Vector([])
        nval = 10
        params = Vector(['X{0}'.format(k) for k in range(1, nval+1)],
                    defaults=np.ones(nval), mins=np.zeros(nval), \
                    maxs=np.ones(nval)*5)
        states = Vector(['S{0}'.format(k) for k in range(1, 3)])
        self.model = Model('test', config, params, states, 2, 2)

    def test_default(self):
        ''' Test setting default values '''
        calparams = CalibParamsVector(self.model)

        self.assertTrue(np.all([s1==s2 for s1, s2 in zip(calparams.names, \
                                    self.model.params.names)]))

        self.assertTrue(np.allclose(calparams.defaults, \
                                    self.model.params.defaults))

    def test_errors_infinite(self):
        ''' Test errors for finite values in calibrated params '''
        nval = self.model.params.nval
        cp = Vector(['X{0}'.format(k) for k in range(1, nval+1)])
        try:
            calparams = CalibParamsVector(self.model, cp)
        except ValueError as err:
            self.assertTrue(str(err).startswith('Expected no infinite'))
        else:
            raise ValueError('Problem with error handling')

    def test_errors_funs(self):
        ''' Test errors related to trans2true and true2trans '''
        nval = self.model.params.nval
        cp = Vector(['X{0}'.format(k) for k in range(1, nval+1)])
        cp = Vector(['tX{0}'.format(k) for k in range(1, nval+1)],\
                    defaults=[0]*nval, mins=[-1]*nval, maxs=[1]*nval)

        fun1 = lambda x: 'string1'
        fun2 = lambda x: 'string2'
        try:
            calparams = CalibParamsVector(self.model, cp, fun1, fun2)
        except ValueError as err:
            self.assertTrue(str(err).startswith(\
                                'Problem with trans2true for'))
        else:
            raise ValueError('Problem with error handling')

        fun = lambda x: np.column_stack([x, x])
        try:
            calparams = CalibParamsVector(self.model, cp, fun, fun)
        except ValueError as err:
            self.assertTrue(str(err).startswith(\
                                'Problem with trans2true for'))
        else:
            raise ValueError('Problem with error handling')

    def test_identity(self):
        nval = self.model.params.nval
        cp = Vector(['tX{0}'.format(k) for k in range(1, nval+1)],\
                    defaults=[0]*nval, mins=[-1]*nval, maxs=[1]*nval)

        calparams = CalibParamsVector(self.model, cp)

        for i in range(10):
            val = np.random.uniform(0, 1, nval)
            calparams.values = val
            self.assertTrue(np.allclose(self.model.params.values, val))

            val = np.random.uniform(0, 1, nval)
            calparams.truevalues = val
            self.assertTrue(np.allclose(calparams.values, val))

    def test_common_transform(self):
        nval = self.model.params.nval
        cp = Vector(['tX{0}'.format(k) for k in range(1, nval+1)],\
                    defaults=[0]*nval, mins=[-1]*nval, maxs=[1]*nval)

        for i, trans in enumerate(['exp', 'sinh']):
            calparams = CalibParamsVector(self.model, cp, trans2true=trans)
            if i == 0:
                trans2true = np.exp
                true2trans = np.log
            elif i == 1:
                trans2true = np.sinh
                true2trans = np.arcsinh

            for i in range(10):
                val = np.random.uniform(0, 1, nval)
                calparams.values = val
                self.assertTrue(np.allclose(calparams.truevalues, \
                                                trans2true(val)))
                self.assertTrue(np.allclose(self.model.params.values, \
                                                trans2true(val)))

                val = np.random.uniform(math.exp(-1), 1, nval)
                calparams.truevalues = val
                self.assertTrue(np.allclose(calparams.values, \
                                                true2trans(val)))

    def test_fixed(self):
        nval = self.model.params.nval
        cp = Vector(['tX{0}'.format(k) for k in range(1, nval+1)],\
                    defaults=[0]*nval, mins=[-5]*nval, maxs=[5]*nval)

        # Choose a fixed value below the max value
        x1 = 4
        fixed = {'X1':x1}

        calparams = CalibParamsVector(self.model, cp, fixed=fixed)

        for i in range(10):
            val = np.random.uniform(0, 1, nval)
            calparams.values = val
            val2 = val.copy()
            val2[0] = x1
            self.assertTrue(np.allclose(self.model.params.values, val2))

            val = np.random.uniform(0, 1, nval)
            calparams.truevalues = val
            val2 = val.copy()
            val2[0] = x1
            self.assertTrue(np.allclose(calparams.truevalues, val2))
            self.assertTrue(np.allclose(calparams.values, val2))

class CalibrationTestCase(unittest.TestCase):

    def setUp(self):
        print('\t=> CalibrationTestCase')

        # Create random inputs
        inputs = np.random.exponential(1, (100, 2))

        # Allocate model
        dum = Dummy()
        dum.allocate(inputs, 2)

        # Run model to create a sudo obs
        params = dum.params.defaults+0.1
        dum.params.values = params
        dum.run()
        obs = dum.outputs[:, 0].copy()

        # Store calibration set up
        self.inputs = inputs
        self.params = params
        self.obs = obs
        self.ical = np.arange(10, obs.shape[0])

    def test_calibration_instance_print(self):
        ''' Test printing of calibration object '''
        calib = CalibrationDummy(warmup=10)
        calib.allocate(self.obs, self.inputs)
        str = '{0}'.format(calib)

    def test_calibration_errors(self):
        ''' Test calibration errors '''
        inputs = np.random.uniform(0, 1, (1000, 2))
        obs = np.random.uniform(0, 1, 1000)

        cp = Vector(['tX1', 'tX2'], mins=[-10]*2, maxs=[10]*2, \
                defaults=[1, 0])
        calparams = CalibParamsVector(Dummy(), cp, trans2true='exp')
        calib = Calibration(calparams)

        try:
            plib = calib.paramslib
        except ValueError as err:
            self.assertTrue(str(err).startswith(\
                    'Trying to access paramslib, but '))
        else:
            raise ValueError('Problem with error handling')

        try:
            calib.ical = obs==obs
        except ValueError as err:
            self.assertTrue(str(err).startswith('Trying to get obs, but '))
        else:
            raise ValueError('Problem with error handling')

    def test_explore(self):
        ''' Test explore function '''
        calib = CalibrationDummy(warmup=10)
        plib = np.random.uniform(-0.1, 0.1, size=(1000, 2)) \
                        + self.params[None, :]
        calib.paramslib = plib

        calib.allocate(self.obs, self.inputs)
        calib.ical = self.ical

        start, _, explo_ofun = calib.explore()
        self.assertTrue(np.allclose(start, self.params, rtol=0., atol=0.05))

    def test_explore_error(self):
        ''' Test calibration exploration error '''

        class ObjFunError(ObjFunSSE):
            ''' Sum of squared error objective function '''

            def __init__(self):
                super(ObjFunError, self).__init__()
                self.name = 'Error'

            def compute(self, obs, sim, **kwargs):
                of = super(ObjFunError, self).compute(obs, sim)
                if of < 1e-1:
                    # This is a stupid error generation
                    # we use it just for testing
                    raise ValueError('Error in exploration')
                return of

        calib = CalibrationDummy(warmup=10, objfun=ObjFunError())
        plib = np.random.uniform(-0.1, 0.1, size=(1000, 2)) \
                        + self.params[None, :]
        calib.paramslib = plib

        calib.allocate(self.obs, self.inputs)
        calib.ical = self.ical

        start, _, explo_ofun = calib.explore()
        # Check that no objective function is below 1e-1
        # because the objective function does not allow it
        self.assertTrue(np.all(explo_ofun > 1e-1))

        # Check that we trigger an error during exploration
        try:
            start, _, explo_ofun = calib.explore(raise_error=True)
        except CalibrationExplorationError as err:
            self.assertTrue(str(err).startswith('Error in explo'))
        else:
            raise ValueError('Problem with error handling')

    def test_explore_fit(self):
        ''' Test explore and fit functions '''
        calib = CalibrationDummy(warmup=10)
        calib.allocate(self.obs, self.inputs)
        calib.ical = self.ical

        start, _, _ = calib.explore()
        final, _, _ = calib.fit(iprint=10,
                                    maxfun=100000, ftol=1e-8)
        ck = np.allclose(calib.model.params.values, self.params, \
                            atol=1e-3, rtol=0.)
        self.assertTrue(ck)

    def test_fit_args(self):
        ''' Test passing arguments to objective function '''
        kwargs = {'lam':1.0, 'idx':np.arange(len(self.ical))}
        calib = CalibrationDummy(objfun=ObjFunSSEargs(), \
                    warmup=10, \
                    objfun_kwargs=kwargs)
        calib.allocate(self.obs, self.inputs)
        calib.ical = self.ical

        start, _, _ = calib.explore()
        final, _, _ = calib.fit(iprint=10,
                                    maxfun=100000, ftol=1e-8)
        ck = np.allclose(calib.model.params.values, self.params, \
                            atol=1e-3, rtol=0.)

        self.assertTrue(ck)

    def test_checkvalues(self):
        def fun(values):
            if values[1] < 0.5:
                raise ParameterCheckValueError

        calib = CalibrationDummy(warmup=10, checkvalues=fun)
        calib.allocate(self.obs, self.inputs)
        calib.ical = self.ical

        start, _, ofuns = calib.explore()

        idx = calib.paramslib[:, 1] < 0.5
        self.assertTrue(np.all(np.isinf(ofuns[idx])))

    def test_fixed(self):
        ''' Test calibration with fixed parameters '''
        # Test error
        fixed = {'X10':self.params[0]+3}
        try:
            calib = CalibrationDummy(warmup=10, fixed=fixed)
        except ValueError as err:
            self.assertTrue(str(err).startswith('Expected names '+\
                                'of fixed parameters'))
        else:
            raise ValueError('Problem with error handling')

        fixed = {'X1':self.params[0]+3}
        calib = CalibrationDummy(warmup=10, fixed=fixed)
        calib.allocate(self.obs, self.inputs)
        calib.ical = self.ical

        start, _, _ = calib.explore()
        final, _, _ = calib.fit(iprint=10,
                                    maxfun=100000, ftol=1e-8)

        self.assertEqual(fixed, calib.fixed)
        self.assertTrue(np.allclose(fixed['X1'], start[0]))
        self.assertTrue(np.allclose(fixed['X1'], final[0]))
        self.assertTrue(np.allclose(fixed['X1'], \
                            calib.model.params.values[0]))

    def test_workflow(self):
        ''' Test calibration workflow (i.e. explore+fit) '''
        calib = CalibrationDummy(warmup=10)

        # Check parameter are not close at the beginning
        ck = ~np.allclose(calib.model.params.values, self.params)
        self.assertTrue(ck)

        # Run calibration
        calib.workflow(self.obs, self.inputs, self.ical, iprint=0,
                maxfun=100000, ftol=1e-8)

        # Test parameters at the end
        ck = np.allclose(calib.model.params.values, self.params, \
                            atol=1e-5, rtol=0.)
        self.assertTrue(ck)

    def test_customised_objfun(self):
        ''' Test customised objective function '''
        # Define a customized objective function
        objfun = ObjFunBCSSE(lam=0.8, nu=1e-5)

        # Instanciate a new calib obj and applies objfun
        calib = CalibrationDummy(warmup=10, objfun=objfun)

        # Check parameter are not close at the beginning
        ck = ~np.allclose(calib.model.params.values, self.params)
        self.assertTrue(ck)

        # Run calibration
        calib.workflow(self.obs, self.inputs, self.ical, iprint=0,
                maxfun=100000, ftol=1e-8)

        # Test parameters at the end
        ck = np.allclose(calib.model.params.values, self.params, atol=1e-3)
        self.assertTrue(ck)

    def test_optimizers(self):
        ''' Test a range of optimizer from scipy '''
        calib = CalibrationDummy(objfun=ObjFunSSE(), \
                    warmup=10)
        calib.allocate(self.obs, self.inputs)
        calib.ical = self.ical
        start, _, _ = calib.explore()

        for iopt, opt in enumerate([fmin, fmin_bfgs]):
            if opt.__name__ in ['fmin', 'fmin_powell']:
                kwargs = dict(maxfun=100000, ftol=1e-8)
            else:
                kwargs = dict(maxiter=100000, gtol=1e-8)

            final, _, _ = calib.fit(start=start, iprint=10, optimizer=opt, \
                                            **kwargs)
            ck = np.allclose(calib.model.params.values, self.params, \
                    atol=5e-3, rtol=0.)
            if not ck:
                print(('Failing optimizer test {0} '+\
                    'expected params={1}, got {2}').format(\
                        iopt+1, \
                        ' '.join(list(np.round(\
                                self.params, 2).astype(str))), \
                        ' '.join(list(np.round(\
                                calib.model.params.values, 2).astype(str)))
                ))

            self.assertTrue(ck)

if __name__ == "__main__":
    unittest.main()