import sys
sys.path.append(".")
from argparse import ArgumentParser
import json
import os
import numpy as np
import torch
from torch.utils.data import Dataset, DistributedSampler, DataLoader, SequentialSampler, RandomSampler
from torch.optim import AdamW
from callback.lr_scheduler import get_linear_schedule_with_warmup
from callback.progressbar import ProgressBar
from model.configuration_bert import BertConfig
from model.modeling_poor import BertForMultipleChoice, BertForTokenClassification, BertForQuestionAnswering, BertForSequenceClassification
from model.tokenization_shang import ShangTokenizer
# from model.modeling_poor import  BertForSequenceClassification, BertForTokenClassification, BertForQuestionAnswering, BertForMultipleChoice
# from model.tokenization_shang import ShangTokenizer, Sentence
from tasks.utils import truncate_pair, TaskConfig, find_span, cal_acc
from tools.common import logger, init_logger

# logger = logging.getLogger(__name__)
# # FORMAT = '%(pathname)s %(filename)s  %(funcName)s %(lineno)d %(asctime)-15s  %(message)s'
# FORMAT = ' %(filename)s %(lineno)d %(funcName)s %(asctime)-15s  %(message)s'
# logging.basicConfig(filename="tasks.log",filemode='a',format=FORMAT,level=logging.INFO)

class TaskPoor:
    def __init__(self,config):
        # super(Task, self).__init__(config)
        self.config=TaskConfig(config)
        init_logger(log_file=f"{self.config.output_dir}/train.log")
        self.task_name=self.config.task_name
        self.dataset=self.config.TaskDataset
        self.labels=self.config.labels

        parser = ArgumentParser()
        parser.add_argument("--local_rank", type=int, default=-1,  help="local_rank for distributed training on gpus")
        args = parser.parse_args()
        self.config.local_rank=   args.local_rank
        if self.config.local_rank == -1 or self.config.no_cuda:
            self.config.device = torch.device("cuda" if torch.cuda.is_available() and not self.config.no_cuda else "cpu")
            self.config.n_gpu = torch.cuda.device_count()
        else:  # Initializes the distributed backend which will take care of sychronizing nodes/GPUs
            torch.cuda.set_device(self.local_rank)
            self.config.device = torch.device("cuda", args.local_rank)
            torch.distributed.init_process_group(backend='nccl')
            self.config.n_gpu = 1
        # Load pretrained model and tokenizer
        if args.local_rank not in [-1, 0]:
            torch.distributed.barrier()  # Make sure only the first process in distributed training will download model & vocab

        self.tokenizer = ShangTokenizer(vocab_path=self.config.vocab_file, bujian_path=self.config.bujian_file,use_bujian=self.config.use_bujian)
        # self.valid_dataset=self.load_valid()
        self.acc=0
        self.model = self.load_model(self.config.model_name_or_path)

        self.valid_dataset = None
        self.test_dataset=None

    def load_model(self, model_path ):
        bert_config = BertConfig.from_pretrained(model_path, num_labels=self.config.num_labels, finetuning_task=self.task_name, use_stair=False)
        logger.info(f" loadding {model_path} ")
        if self.config.task_name in ["c3", "chid"]:
            model = BertForMultipleChoice.from_pretrained(model_path, from_tf=bool('.ckpt' in model_path), config=bert_config)
        elif self.config.output_mode == "span":
            model = BertForTokenClassification.from_pretrained(model_path, from_tf=bool('.ckpt' in model_path), config=bert_config)
        elif self.config.output_mode == "qa":
            model = BertForQuestionAnswering.from_pretrained(model_path, from_tf=bool('.ckpt' in model_path), config=bert_config)
        elif   self.config.output_mode == "classification":
            model = BertForSequenceClassification.from_pretrained(model_path, from_tf=bool('.ckpt' in model_path), config=bert_config)
        if self.config.local_rank == 0:
            torch.distributed.barrier()  # Make sure only the first process in distributed training will download model & vocab

        model.to(self.config.device)
        return model

    def train(self):
        input_file=os.path.join(self.config.data_dir,self.config.valid_file)
        self.valid_dataset = self.dataset(input_file=input_file, tokenizer=self.tokenizer,labels=self.labels, max_tokens=self.config.max_len,config=self.config)
        self.config.save_steps=max(self.config.save_steps,len(self.valid_dataset)//self.config.batch_size)
        self.config.logging_steps=max(self.config.logging_steps,len(self.valid_dataset)//self.config.batch_size)
        args=self.config
        model=self.model
        input_file=os.path.join(args.data_dir,self.config.train_file)
        dataset = self.dataset(input_file=input_file, tokenizer=self.tokenizer, labels=self.labels, max_tokens=self.config.max_len,config=self.config)
        num_training_steps=self.config.n_epochs*len(dataset)
        warmup_steps = int(num_training_steps * args.warmup_proportion)
        # Prepare optimizer and schedule (linear warmup and decay)
        no_decay = ['bias', 'LayerNorm.weight']
        optimizer_grouped_parameters = [
            {'params': [p for n, p in model.named_parameters() if not any(nd in n for nd in no_decay)],
             'weight_decay': args.weight_decay},
            {'params': [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay)], 'weight_decay': 0.0}
        ]
        # optimizer_grouped_parameters = [
        #     {'params': [p for n, p in model.named_parameters() if not any(nd in n for nd in ["bert"])],'lr': self.config.learning_rate},
        #     {'params': [p for n, p in model.named_parameters() if any(nd in n for nd in ["bert"])], 'lr': self.config.learning_rate/5}
        # ]
        # # optimizer = Lamb(optimizer_grouped_parameters, lr=args.learning_rate, eps=args.adam_epsilon)

        optimizer = AdamW(params=optimizer_grouped_parameters, lr=args.learning_rate, eps=args.adam_epsilon)
        scheduler = get_linear_schedule_with_warmup(optimizer, num_warmup_steps=warmup_steps, num_training_steps=num_training_steps)

        if self.config.fp16:
            try:
                from apex import amp
            except ImportError:
                raise ImportError("Please install apex from https://www.github.com/nvidia/apex to use fp16 training.")
            model, optimizer = amp.initialize(model, optimizer, opt_level=self.config.fp16_opt_level)

        # multi-gpu training (should be after apex fp16 initialization)
        if self.config.n_gpu > 1:
            model = torch.nn.DataParallel(model)

        # Distributed training (should be after apex fp16 initialization)
        if self.config.local_rank != -1:
            model = torch.nn.parallel.DistributedDataParallel(model, device_ids=[args.local_rank], output_device=args.local_rank, find_unused_parameters=True)
        self.model=model

        if not os.path.exists(args.output_dir) and args.local_rank in [-1, 0]:
            os.makedirs(args.output_dir)

        self.global_step = 0
        tr_loss, logging_loss = 0.0, 0.0
        for epoch in range(self.config.n_epochs):
            dataset = self.dataset(input_file=input_file, tokenizer=self.tokenizer, labels=self.labels, max_tokens=self.config.max_len,config=self.config)
            sampler = RandomSampler(dataset) if args.local_rank == -1 else DistributedSampler(dataset)
            dataloader = DataLoader(dataset, sampler=sampler, batch_size=self.config.batch_size, collate_fn=self.config.collate_fn,pin_memory=self.config.pin_memory, num_workers=self.config.num_workers)
            pbar = ProgressBar(n_total=len(dataloader), desc=f"{input_file[-15:]}")
            for step, batch in enumerate(dataloader):
                loss=self.train_batch(batch,args,optimizer,scheduler,step)
                msg={ "epoch":epoch, "global_step":self.global_step,"loss": loss ,"lr": scheduler.get_lr(),"seq_len":batch[0].shape[-1]   }
                pbar(step, msg)
                tr_loss += loss
                if args.local_rank in [-1, 0] and args.logging_steps > 0 and (self.global_step % args.logging_steps == 0 or step+1==len(dataloader)  ):
                    # Log metrics
                    if args.local_rank == -1:  # Only evaluate when single GPU otherwise metrics may not average well
                        acc=self.evaluate(epoch)

                if args.local_rank in [-1, 0] and args.save_steps > 0 and (self.global_step % args.save_steps == 0 or step+1==len(dataloader))and acc>=self.acc:
                    logger.info(f"Saving best model acc:{self.acc} -->{acc}")
                    self.acc=acc
                    output_dir = args.output_dir
                    if not os.path.exists(output_dir):
                        os.makedirs(output_dir)
                    model_to_save = model.module if hasattr(model,  'module') else model  # Take care of distributed/parallel training
                    model_to_save.save_pretrained(output_dir)
                    torch.save(args, os.path.join(output_dir, 'training_args.bin'))
                    logger.info("Saving model checkpoint to %s", output_dir)
                # break
            print("\n ")
            if 'cuda' in str(args.device):
                torch.cuda.empty_cache()
            msg = {"epoch": (epoch), "global_step": (self.global_step), "loss": loss, "average loss":tr_loss, "lr": (scheduler.get_lr())}
            logger.info(   f" {msg}")

        logger.info("Saving model checkpoint to %s", args.output_dir)
        # Save a trained model, configuration and tokenizer using `save_pretrained()`.
        # They can then be reloaded using `from_pretrained()`
        model_to_save = model.module if hasattr(model, 'module') else model  # Take care of distributed/parallel training
        model_to_save.save_pretrained(args.output_dir)
        # Good practice: save your training arguments together with the trained model
        torch.save(args, os.path.join(args.output_dir, 'training_args.bin'))

    def train_batch(self, batch,args,optimizer,scheduler,step):
        model=self.model
        model.train()
        batch = tuple(t.to(self.config.device) for t in batch)
        if self.config.output_mode == "qa":
            input_ids, attention_mask, token_type_ids, start_positions, end_positions = batch
            inputs = {'input_ids': input_ids, 'attention_mask': attention_mask, 'token_type_ids': token_type_ids, 'start_positions': start_positions, "end_positions": end_positions}
        else:
            input_ids, attention_mask, token_type_ids, label_ids = batch
            inputs = {'input_ids': input_ids, 'attention_mask': attention_mask, 'token_type_ids': token_type_ids, 'labels': label_ids}

        outputs = self.model(**inputs)
        loss = outputs[0]  # model outputs are always tuple in transformers (see doc)

        if args.n_gpu > 1:
            loss = loss.mean()  # mean() to average on multi-gpu parallel training
        if args.gradient_accumulation_steps > 1:
            loss = loss / args.gradient_accumulation_steps

        if args.fp16:
            with amp.scale_loss(loss, optimizer) as scaled_loss:
                scaled_loss.backward()
            torch.nn.utils.clip_grad_norm_(amp.master_params(optimizer), args.max_grad_norm)
        else:
            loss.backward()
            torch.nn.utils.clip_grad_norm_(model.parameters(), args.max_grad_norm)

        if (step + 1) % args.gradient_accumulation_steps == 0:
            optimizer.step()
            scheduler.step()  # Update learning rate schedule
            optimizer.zero_grad()
            self.global_step += 1

        return loss.item()

    def evaluate(self,epoch):
        args=self.config
        model=self.model
        model.eval()
        dataset=self.valid_dataset
        sampler = SequentialSampler(dataset) if args.local_rank == -1 else DistributedSampler(dataset)
        dataloader = DataLoader(dataset, sampler=sampler, batch_size=self.config.batch_size, collate_fn=self.config.collate_fn, pin_memory=self.config.pin_memory, num_workers=self.config.num_workers)
        print(' ')
        nb_eval_steps = 0
        scores=[]
        pbar = ProgressBar(n_total=len(dataloader), desc="Evaluating")
        for step, batch in enumerate(dataloader):
            with torch.no_grad():
                batch = tuple(t.to(args.device) for t in batch)
                if self.config.output_mode=="qa":
                    input_ids, attention_mask, token_type_ids, start_positions, end_positions = batch
                    inputs = {'input_ids': input_ids, 'attention_mask': attention_mask, 'token_type_ids': token_type_ids}
                else:
                    input_ids, attention_mask,token_type_ids, label_ids = batch
                    inputs = {'input_ids': input_ids, 'attention_mask': attention_mask, 'token_type_ids':token_type_ids,'labels': label_ids}
                outputs = model(**inputs)
                tmp_eval_loss, logits = outputs[:2]
                if self.config.output_mode == "qa":
                    start_logits, end_logits=tmp_eval_loss, logits
                    if len(start_positions.size()) > 1:
                        start_positions = start_positions.squeeze(-1)
                    if len(end_positions.size()) > 1:
                        end_positions = end_positions.squeeze(-1)
                    # sometimes the start/end positions are outside our model inputs, we ignore these terms
                    ignored_index = start_logits.size(1)
                    start_positions.clamp_(0, ignored_index)
                    end_positions.clamp_(0, ignored_index)

                    score1 = cal_acc(start_logits, start_positions)
                    score2 = cal_acc(end_logits, end_positions)
                    scores.append((score1+ score2)/2)
                elif self.config.output_mode == "span" :
                    for i in range(len(logits)):
                        score = cal_acc(logits[i], label_ids[i])
                        scores.append((score))
                elif self.config.output_mode == "classification":
                    score = cal_acc(logits, label_ids)
                    scores.append(score)
            nb_eval_steps += 1
            pbar(step)
            # break
        print(' ')
        if 'cuda' in str(args.device):
            torch.cuda.empty_cache()
        acc = np.array(scores).mean()
        result={"acc": acc,"epoch":epoch,"step":self.global_step}

        output_eval_file = os.path.join(args.output_dir, "checkpoint_eval_results.txt")
        line=json.dumps(result,ensure_ascii=False)
        with open(output_eval_file, "a") as writer:
            writer.write(line+"\n")
        logger.info(f"\n valid : {line} ")
        model.train()
        return acc

    def infer(self):
        args=self.config
        logger.info(f"selected best model acc:{self.acc}")
        model= self.load_model(self.config.output_dir)
        # model=self.model
        model.eval()
        # dataset=self.valid_dataset
        input_file=os.path.join(self.config.data_dir,self.config.test_file)
        dataset = self.dataset(input_file=input_file, tokenizer=self.tokenizer,labels=self.labels, max_tokens=self.config.max_len,config=self.config)
        self.test_dataset=dataset
        sampler = SequentialSampler(dataset) if args.local_rank == -1 else DistributedSampler(dataset)
        dataloader = DataLoader(dataset, sampler=sampler, batch_size=self.config.batch_size, collate_fn=self.config.collate_fn, pin_memory=self.config.pin_memory, num_workers=self.config.num_workers)

        nb_eval_steps = 0
        preds = []
        pbar = ProgressBar(n_total=len(dataloader), desc="Testing")
        for step, batch in enumerate(dataloader):
            batch = tuple(t.to(args.device) for t in batch)
            with torch.no_grad():
                if self.config.output_mode == "qa":
                    input_ids, attention_mask, token_type_ids, start_positions, end_positions = batch
                    inputs = {'input_ids': input_ids, 'attention_mask': attention_mask, 'token_type_ids': token_type_ids}
                else:
                    input_ids, attention_mask, token_type_ids, label_ids = batch
                    inputs = {'input_ids': input_ids, 'attention_mask': attention_mask, 'token_type_ids': token_type_ids, 'labels': label_ids}
                outputs = model(**inputs)
                tmp_eval_loss, logits = outputs[:2]
                if self.config.output_mode == "qa":
                    start_logits, end_logits=tmp_eval_loss, logits
                    start = torch.argmax(start_logits, 1).tolist()
                    end = torch.argmax(end_logits, 1).tolist()
                    preds+=zip(start,end)
                elif args.output_mode=="span":
                    prob = logits.detach().cpu().numpy()
                    preds+=[x  for x in prob]
                elif args.output_mode == "classification":
                    preds+=torch.argmax(logits, 1).tolist()
            nb_eval_steps += 1
            pbar(step)
            # break
        print(' ')
        if 'cuda' in str(args.device):
            torch.cuda.empty_cache()
        logger.info(f"infered {len(preds)}")
        return preds


import torch
import numpy as np
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
from torchvision import datasets, transforms
from torch.autograd import Variable
from sklearn.metrics import mean_squared_error
import models
import helper_functions
import pandas as pd
import os
import sys
from scipy.stats import geom
import torchvision
import time
import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt
from PIL import Image
import itertools
import pickle
from numpy import dot
from numpy.linalg import norm
from sklearn.utils import shuffle

def getMeanNet(start_idx, end_idx):
    num_models = end_idx - start_idx
    nets = [models.Net() for i in range(num_models)]

    net1 = models.Net()
    net2 = models.Net()
    net3 = models.Net()
    net4 = models.Net()
    net5 = models.Net()
    net6 = models.Net()
    net7 = models.Net()
    net8 = models.Net()
    net9 = models.Net()
    net10 = models.Net()

    for idx,net in enumerate(nets):
        net_model = torch.load("task_net_models/mnist_digit_solver_"+str(idx+start_idx)+".pt")
        net.load_state_dict(net_model)

    net1_model = torch.load("task_net_models/mnist_digit_solver_0.pt")
    net1.load_state_dict(net1_model)

    net2_model = torch.load("task_net_models/mnist_digit_solver_1.pt")
    net2.load_state_dict(net2_model)

    net3_model = torch.load("task_net_models/mnist_digit_solver_2.pt")
    net3.load_state_dict(net3_model)

    net4_model = torch.load("task_net_models/mnist_digit_solver_3.pt")
    net4.load_state_dict(net4_model)

    net5_model = torch.load("task_net_models/mnist_digit_solver_4.pt")
    net5.load_state_dict(net5_model)

    net6_model = torch.load("task_net_models/mnist_digit_solver_5.pt")
    net6.load_state_dict(net6_model)

    net7_model = torch.load("task_net_models/mnist_digit_solver_6.pt")
    net7.load_state_dict(net7_model)

    net8_model = torch.load("task_net_models/mnist_digit_solver_7.pt")
    net8.load_state_dict(net8_model)

    net9_model = torch.load("task_net_models/mnist_digit_solver_8.pt")
    net9.load_state_dict(net9_model)

    net10_model = torch.load("task_net_models/mnist_digit_solver_9.pt")
    net10.load_state_dict(net10_model)

    flatNets = [[] for i in range(num_models)]
    net_shapes = []
    for idx,net in enumerate(nets):
        flatNets[idx], net_shapes = helper_functions.flattenNetwork(net)

    flat1, net_shapes=helper_functions.flattenNetwork(net1)
    flat2, net_shapes=helper_functions.flattenNetwork(net2)
    flat3, net_shapes=helper_functions.flattenNetwork(net3)
    flat4, net_shapes=helper_functions.flattenNetwork(net4)
    flat5, net_shapes=helper_functions.flattenNetwork(net5)
    flat6, net_shapes=helper_functions.flattenNetwork(net6)
    flat7, net_shapes=helper_functions.flattenNetwork(net7)
    flat8, net_shapes=helper_functions.flattenNetwork(net8)
    flat9, net_shapes=helper_functions.flattenNetwork(net9)
    flat10, net_shapes=helper_functions.flattenNetwork(net10)

    all = torch.Tensor()
    for idx, flatNet in enumerate(flatNets):
        all = torch.cat((all, torch.Tensor(flatNet).view(-1,len(flatNet))), dim=0)

    all = torch.cat((torch.Tensor([flat1]), torch.Tensor([flat2])), dim=0)
    all = torch.cat((all, torch.Tensor([flat3])), dim=0)
    all = torch.cat((all, torch.Tensor([flat4])), dim=0)
    all = torch.cat((all, torch.Tensor([flat5])), dim=0)
    all = torch.cat((all, torch.Tensor([flat6])), dim=0)
    all = torch.cat((all, torch.Tensor([flat7])), dim=0)
    all = torch.cat((all, torch.Tensor([flat8])), dim=0)
    all = torch.cat((all, torch.Tensor([flat9])), dim=0)
    all = torch.cat((all, torch.Tensor([flat10])), dim=0)
    # # print(all)

    def loadWeights_mnsit(weights_to_load, net):
        net.conv1.weight.data = torch.from_numpy(weights_to_load[0]).cuda()
        net.conv1.bias.data =   torch.from_numpy(weights_to_load[1]).cuda()
        net.conv2.weight.data = torch.from_numpy(weights_to_load[2]).cuda()
        net.conv2.bias.data =   torch.from_numpy(weights_to_load[3]).cuda()
        net.fc1.weight.data =   torch.from_numpy(weights_to_load[4]).cuda()
        net.fc1.bias.data =     torch.from_numpy(weights_to_load[5]).cuda()
        net.fc2.weight.data =   torch.from_numpy(weights_to_load[6]).cuda()
        net.fc2.bias.data =     torch.from_numpy(weights_to_load[7]).cuda()
        return net

    mean = torch.mean(all, dim=0)
    meanNet = models.Net()

    mean_weights=helper_functions.unFlattenNetwork(mean.data.numpy(), net_shapes)
    meanNet=loadWeights_mnsit(mean_weights,meanNet)
    torch.save(meanNet.state_dict(),'meanNet.pt')


from http import HTTPStatus
from unittest.mock import patch

import pytest
import requests

from rotkehlchen.constants.assets import A_JPY
from rotkehlchen.db.settings import DEFAULT_KRAKEN_ACCOUNT_TYPE, ROTKEHLCHEN_DB_VERSION, DBSettings
from rotkehlchen.exchanges.kraken import KrakenAccountType
from rotkehlchen.tests.utils.api import (
    api_url_for,
    assert_error_response,
    assert_proper_response,
    assert_simple_ok_response,
)
from rotkehlchen.tests.utils.mock import MockWeb3

def test_qerying_settings(rotkehlchen_api_server, username):
    """Make sure that querying settings works for logged in user"""
    response = requests.get(api_url_for(rotkehlchen_api_server, "settingsresource"))
    assert_proper_response(response)
    json_data = response.json()

    result = json_data['result']
    assert json_data['message'] == ''
    assert result['version'] == ROTKEHLCHEN_DB_VERSION
    for setting in DBSettings._fields:
        assert setting in result

    # Logout of the active user
    data = {'action': 'logout'}
    response = requests.patch(
        api_url_for(rotkehlchen_api_server, "usersbynameresource", name=username),
        json=data,
    )
    assert_simple_ok_response(response)

    # and now with no logged in user it should fail
    response = requests.get(api_url_for(rotkehlchen_api_server, "settingsresource"))
    assert_error_response(
        response=response,
        contained_in_msg='No user is currently logged in',
        status_code=HTTPStatus.CONFLICT,
    )

def test_set_settings(rotkehlchen_api_server):
    """Happy case settings modification test"""
    # Get the starting settings
    response = requests.get(api_url_for(rotkehlchen_api_server, "settingsresource"))
    assert_proper_response(response)
    json_data = response.json()
    original_settings = json_data['result']
    assert json_data['message'] == ''
    # Create new settings which modify all of the original ones
    new_settings = {}
    unmodifiable_settings = (
        'version',
        'last_write_ts',
        'last_data_upload_ts',
        'last_balance_save',
        'have_premium',
    )
    for setting, value in original_settings.items():
        if setting in unmodifiable_settings:
            continue
        elif setting == 'historical_data_start':
            value = '10/10/2016'
        elif setting == 'date_display_format':
            value = '%d/%m/%Y-%H:%M:%S'
        elif setting == 'eth_rpc_endpoint':
            value = 'http://working.nodes.com:8545'
        elif setting == 'main_currency':
            value = 'JPY'
        elif type(value) == bool:
            value = not value
        elif type(value) == int:
            value += 1
        elif setting == 'kraken_account_type':
            # Change the account type to anything other than default
            assert value != str(KrakenAccountType.PRO)
            value = str(KrakenAccountType.PRO)
        elif setting == 'active_modules':
            value = ['makerdao_vaults']
        else:
            raise AssertionError(f'Unexpected settting {setting} encountered')

        new_settings[setting] = value

    # modify the settings
    block_query = patch(
        'rotkehlchen.chain.ethereum.manager.EthereumManager.query_eth_highest_block',
        return_value=0,
    )
    mock_web3 = patch('rotkehlchen.chain.ethereum.manager.Web3', MockWeb3)
    with block_query, mock_web3:
        response = requests.put(
            api_url_for(rotkehlchen_api_server, "settingsresource"),
            json={'settings': new_settings},
        )
    # Check that new settings are returned in the response
    assert_proper_response(response)
    json_data = response.json()
    assert json_data['message'] == ''
    result = json_data['result']
    assert result['version'] == ROTKEHLCHEN_DB_VERSION
    for setting, value in new_settings.items():
        msg = f'Error for {setting} setting. Expected: {value}. Got: {result[setting]}'
        assert result[setting] == value, msg

    # now check that the same settings are returned in a settings query
    response = requests.get(api_url_for(rotkehlchen_api_server, "settingsresource"))
    assert_proper_response(response)
    json_data = response.json()
    result = json_data['result']
    assert json_data['message'] == ''
    for setting, value in new_settings.items():
        assert result[setting] == value

def test_set_rpc_endpoint_fail_not_set_others(rotkehlchen_api_server):
    """Test that setting a non-existing eth rpc along with other settings does not modify them"""
    eth_rpc_endpoint = 'http://working.nodes.com:8545'
    main_currency = A_JPY
    data = {'settings': {
        'eth_rpc_endpoint': eth_rpc_endpoint,
        'main_currency': main_currency.identifier,
    }}

    response = requests.put(api_url_for(rotkehlchen_api_server, "settingsresource"), json=data)
    assert_error_response(
        response=response,
        contained_in_msg='Failed to connect to ethereum node at endpoint',
        status_code=HTTPStatus.CONFLICT,
    )

    # Get settings and make sure they have not been modified
    response = requests.get(api_url_for(rotkehlchen_api_server, "settingsresource"))
    assert_proper_response(response)
    json_data = response.json()
    result = json_data['result']
    assert json_data['message'] == ''
    assert result['main_currency'] != 'JPY'
    assert result['eth_rpc_endpoint'] != 'http://working.nodes.com:8545'

def test_unset_rpc_endpoint(rotkehlchen_api_server):
    """Test the rpc endpoint can be unset"""
    response = requests.get(api_url_for(rotkehlchen_api_server, "settingsresource"))
    assert_proper_response(response)
    json_data = response.json()
    assert json_data['message'] == ''
    result = json_data['result']
    assert result['eth_rpc_endpoint'] != ''

    data = {
        'settings': {'eth_rpc_endpoint': ''},
    }

    response = requests.put(api_url_for(rotkehlchen_api_server, "settingsresource"), json=data)
    assert_proper_response(response)

    json_data = response.json()
    result = json_data['result']
    assert json_data['message'] == ''
    assert result['eth_rpc_endpoint'] == ''

@pytest.mark.parametrize('added_exchanges', [('kraken',)])
def test_set_kraken_account_type(rotkehlchen_api_server_with_exchanges):
    server = rotkehlchen_api_server_with_exchanges
    rotki = rotkehlchen_api_server_with_exchanges.rest_api.rotkehlchen
    kraken = rotki.exchange_manager.get('kraken')
    assert kraken.account_type == DEFAULT_KRAKEN_ACCOUNT_TYPE
    assert kraken.call_limit == 15
    assert kraken.reduction_every_secs == 3

    data = {'settings': {'kraken_account_type': 'intermediate'}}
    response = requests.put(api_url_for(server, "settingsresource"), json=data)
    assert_proper_response(response)
    json_data = response.json()
    result = json_data['result']
    assert json_data['message'] == ''
    assert result['kraken_account_type'] == 'intermediate'
    assert kraken.account_type == KrakenAccountType.INTERMEDIATE
    assert kraken.call_limit == 20
    assert kraken.reduction_every_secs == 2

def test_disable_taxfree_after_period(rotkehlchen_api_server):
    """Test that providing -1 for the taxfree_after_period setting disables it """
    data = {
        'settings': {'taxfree_after_period': -1},
    }
    response = requests.put(api_url_for(rotkehlchen_api_server, "settingsresource"), json=data)
    assert_proper_response(response)
    json_data = response.json()
    assert json_data['result']['taxfree_after_period'] is None

    # Test that any other negative value is refused
    data = {
        'settings': {'taxfree_after_period': -5},
    }
    response = requests.put(api_url_for(rotkehlchen_api_server, "settingsresource"), json=data)
    assert_error_response(
        response=response,
        contained_in_msg='The taxfree_after_period value can not be negative',
        status_code=HTTPStatus.BAD_REQUEST,
    )
    # Test that zero value is refused
    data = {
        'settings': {'taxfree_after_period': 0},
    }
    response = requests.put(api_url_for(rotkehlchen_api_server, "settingsresource"), json=data)
    assert_error_response(
        response=response,
        contained_in_msg='The taxfree_after_period value can not be set to zero',
        status_code=HTTPStatus.BAD_REQUEST,
    )

def test_set_unknown_settings(rotkehlchen_api_server):
    """Test that setting an unknown setting results in an error

    This is the only test for unknown arguments in marshmallow schemas after
    https://github.com/rotki/rotki/issues/532 was implemented"""
    # Unknown setting
    data = {
        'settings': {'invalid_setting': 5555},
    }
    response = requests.put(api_url_for(rotkehlchen_api_server, "settingsresource"), json=data)
    assert_error_response(
        response=response,
        contained_in_msg='{"invalid_setting": ["Unknown field."',
        status_code=HTTPStatus.BAD_REQUEST,
    )

def test_set_settings_errors(rotkehlchen_api_server):
    """set settings errors and edge cases test"""
    rotki = rotkehlchen_api_server.rest_api.rotkehlchen
    # set timeout to 1 second to timeout faster
    rotki.chain_manager.ethereum.eth_rpc_timeout = 1

    # Eth rpc endpoint to which we can't connect
    data = {
        'settings': {'eth_rpc_endpoint': 'http://lol.com:5555'},
    }
    response = requests.put(api_url_for(rotkehlchen_api_server, "settingsresource"), json=data)
    assert_error_response(
        response=response,
        contained_in_msg='Failed to connect to ethereum node at endpoint',
        status_code=HTTPStatus.CONFLICT,
    )

    # Invalid type for eth_rpc_endpoint
    data = {
        'settings': {'eth_rpc_endpoint': 5555},
    }
    response = requests.put(api_url_for(rotkehlchen_api_server, "settingsresource"), json=data)
    assert_error_response(
        response=response,
        contained_in_msg='Not a valid string',
        status_code=HTTPStatus.BAD_REQUEST,
    )

    # Invalid type for premium_should_sync
    data = {
        'settings': {'premium_should_sync': 444},
    }
    response = requests.put(api_url_for(rotkehlchen_api_server, "settingsresource"), json=data)
    assert_error_response(
        response=response,
        contained_in_msg='Not a valid boolean',
        status_code=HTTPStatus.BAD_REQUEST,
    )

    # Invalid type for include_crypto2crypto
    data = {
        'settings': {'include_crypto2crypto': 'ffdsdasd'},
    }
    response = requests.put(api_url_for(rotkehlchen_api_server, "settingsresource"), json=data)
    assert_error_response(
        response=response,
        contained_in_msg='Not a valid boolean',
        status_code=HTTPStatus.BAD_REQUEST,
    )

    # Invalid type for anonymized_logs
    data = {
        'settings': {'anonymized_logs': 555.1},
    }
    response = requests.put(api_url_for(rotkehlchen_api_server, "settingsresource"), json=data)
    assert_error_response(
        response=response,
        contained_in_msg='Not a valid boolean',
        status_code=HTTPStatus.BAD_REQUEST,
    )

    # Invalid range for ui_floating_precision
    data = {
        'settings': {'ui_floating_precision': -1},
    }
    response = requests.put(api_url_for(rotkehlchen_api_server, "settingsresource"), json=data)
    assert_error_response(
        response=response,
        contained_in_msg='Floating numbers precision in the UI must be between 0 and 8',
        status_code=HTTPStatus.BAD_REQUEST,
    )
    data = {
        'settings': {'ui_floating_precision': 9},
    }
    response = requests.put(api_url_for(rotkehlchen_api_server, "settingsresource"), json=data)
    assert_error_response(
        response=response,
        contained_in_msg='Floating numbers precision in the UI must be between 0 and 8',
        status_code=HTTPStatus.BAD_REQUEST,
    )

    # Invalid type for ui_floating_precision
    data = {
        'settings': {'ui_floating_precision': 'dasdsds'},
    }
    response = requests.put(api_url_for(rotkehlchen_api_server, "settingsresource"), json=data)
    assert_error_response(
        response=response,
        contained_in_msg='Not a valid integer',
        status_code=HTTPStatus.BAD_REQUEST,
    )

    # Invalid range for taxfree_after_period
    data = {
        'settings': {'taxfree_after_period': -2},
    }
    response = requests.put(api_url_for(rotkehlchen_api_server, "settingsresource"), json=data)
    assert_error_response(
        response=response,
        contained_in_msg='The taxfree_after_period value can not be negative, except',
        status_code=HTTPStatus.BAD_REQUEST,
    )

    # Invalid type for taxfree_after_period
    data = {
        'settings': {'taxfree_after_period': 'dsad'},
    }
    response = requests.put(api_url_for(rotkehlchen_api_server, "settingsresource"), json=data)
    assert_error_response(
        response=response,
        contained_in_msg='dsad is not a valid integer',
        status_code=HTTPStatus.BAD_REQUEST,
    )

    # Invalid range for balance_save_frequency
    data = {
        'settings': {'balance_save_frequency': 0},
    }
    response = requests.put(api_url_for(rotkehlchen_api_server, "settingsresource"), json=data)
    assert_error_response(
        response=response,
        contained_in_msg='The number of hours after which balances should be saved should be >= 1',
        status_code=HTTPStatus.BAD_REQUEST,
    )

    # Invalid range for balance_save_frequency
    data = {
        'settings': {'balance_save_frequency': 'dasdsd'},
    }
    response = requests.put(api_url_for(rotkehlchen_api_server, "settingsresource"), json=data)
    assert_error_response(
        response=response,
        contained_in_msg='Not a valid integer',
        status_code=HTTPStatus.BAD_REQUEST,
    )

    # Invalid type for include_gas_cost
    data = {
        'settings': {'include_gas_costs': 55.1},
    }
    response = requests.put(api_url_for(rotkehlchen_api_server, "settingsresource"), json=data)
    assert_error_response(
        response=response,
        contained_in_msg='Not a valid boolean',
        status_code=HTTPStatus.BAD_REQUEST,
    )

    # Invalid type for historical_data_start
    data = {
        'settings': {'historical_data_start': 12},
    }
    response = requests.put(api_url_for(rotkehlchen_api_server, "settingsresource"), json=data)
    assert_error_response(
        response=response,
        contained_in_msg='Not a valid string',
        status_code=HTTPStatus.BAD_REQUEST,
    )

    # Invalid asset for main currenty
    data = {
        'settings': {'main_currency': 'DSDSDSAD'},
    }
    response = requests.put(api_url_for(rotkehlchen_api_server, "settingsresource"), json=data)
    assert_error_response(
        response=response,
        contained_in_msg='Unknown asset DSDSDSAD',
        status_code=HTTPStatus.BAD_REQUEST,
    )

    # non FIAT asset for main currency
    data = {
        'settings': {'main_currency': 'ETH'},
    }
    response = requests.put(api_url_for(rotkehlchen_api_server, "settingsresource"), json=data)
    assert_error_response(
        response=response,
        contained_in_msg='Asset ETH is not a FIAT asset',
        status_code=HTTPStatus.BAD_REQUEST,
    )

    # invalid type main currency
    data = {
        'settings': {'main_currency': 243243},
    }
    response = requests.put(api_url_for(rotkehlchen_api_server, "settingsresource"), json=data)
    assert_error_response(
        response=response,
        contained_in_msg='Tried to initialize an asset out of a non-string identifier',
        status_code=HTTPStatus.BAD_REQUEST,
    )

    # invalid type date_display_format
    data = {
        'settings': {'date_display_format': 124.1},
    }
    response = requests.put(api_url_for(rotkehlchen_api_server, "settingsresource"), json=data)
    assert_error_response(
        response=response,
        contained_in_msg='Not a valid string',
        status_code=HTTPStatus.BAD_REQUEST,
    )

    # invalid type kraken_account_type
    data = {
        'settings': {'kraken_account_type': 124.1},
    }
    response = requests.put(api_url_for(rotkehlchen_api_server, "settingsresource"), json=data)
    assert_error_response(
        response=response,
        contained_in_msg='is not a valid kraken account type',
        status_code=HTTPStatus.BAD_REQUEST,
    )

    # invalid value kraken_account_type
    data = {
        'settings': {'kraken_account_type': 'super hyper pro'},
    }
    response = requests.put(api_url_for(rotkehlchen_api_server, "settingsresource"), json=data)
    assert_error_response(
        response=response,
        contained_in_msg='is not a valid kraken account type',
        status_code=HTTPStatus.BAD_REQUEST,
    )

    # invalid type for active modules
    data = {
        'settings': {'active_modules': 55},
    }
    response = requests.put(api_url_for(rotkehlchen_api_server, "settingsresource"), json=data)
    assert_error_response(
        response=response,
        contained_in_msg='"active_modules": ["Not a valid list."',
        status_code=HTTPStatus.BAD_REQUEST,
    )

    # invalid module for active modules
    data = {
        'settings': {'active_modules': ['makerdao_dsr', 'foo']},
    }
    response = requests.put(api_url_for(rotkehlchen_api_server, "settingsresource"), json=data)
    assert_error_response(
        response=response,
        contained_in_msg='"active_modules": ["foo is not a valid module"]',
        status_code=HTTPStatus.BAD_REQUEST,
    )


from database import crm_db
from typing import List
from models.research import Research, ResearchIn
from bson import ObjectId
from pymongo.errors import DuplicateKeyError
from fastapi import HTTPException

async def read_researches(skip: int = 0, limit: int = 200):
    researchs = []
    for research in (
        await crm_db.Research.find().skip(skip).limit(limit).to_list(length=limit)
    ):
        researchs.append(research)
    return researchs

async def create_research(research: ResearchIn):
    research_dict = research.dict()
    try:
        result = await crm_db.Research.insert_one(research_dict)
        research_dict["_id"] = ObjectId(result.inserted_id)
        return research_dict
    except DuplicateKeyError:
        raise HTTPException(
            status_code=400,
            detail="A research with the same name and telephone number already exists",
        )

async def read_client_researches(client_id: str):
    client_researches = []
    try:
        for client_research in (await crm_db.Research.find({"user_id": client_id}).to_list(length=200)):
            client_researches.append(client_research)

        return client_researches
    except Exception as e:
        raise HTTPException(status_code=404, detail=e.with_traceback())

async def update_Research(client_id: str, annonce_id: str, research: ResearchIn):
    updated_research = await crm_db.Research.find_one_and_update(
        {"user_id": client_id, "annonce_id":annonce_id}, {"$set": research.dict()}, return_document=True
    )
    if updated_research:
        return updated_research
    else:
        raise HTTPException(status_code=404, detail="Resarch not found")

async def delete_Research(research_id:str):
    deletedResearch = await crm_db.Research.find_one_and_delete(
        {"_id": ObjectId(research_id)}
    )
    if deletedResearch:
        return deletedResearch
    else:
        raise HTTPException(status_code=404, detail="research not found")


dict_f = {}
user = []
hobby = []

u = input('Путь к файлу user: ')
h = input('Путь к файлу hobby: ')
with open(u, 'r', encoding='utf-8-sig') as u_file:
    r_user = u_file.readline()
    while r_user:
        user_idx = r_user.find(' ')-1
        r_u = r_user[0: user_idx]
        user.append(r_u)
        r_user = u_file.readline()

with open(h, 'r', encoding='utf-8-sig') as h_file:
    r_hobby = h_file.readline().replace('\n', '').replace('\r', '')
    while r_hobby:
        hobby.append(r_hobby)
        r_hobby = h_file.readline().replace('\n', '').replace('\r', '')

y = True
while y:
    try:
        some_dict = {user.pop(): hobby.pop()}
        dict_f.update(some_dict)
    except IndexError:
        try:
            some_dict = {'None': hobby.pop()}
            dict_f.update(some_dict)
        except IndexError:
            some_dict = {user.pop(): 'None'}
            dict_f.update(some_dict)
        y = False

dict_user_hobby = input('Введите путь и имя файла для сохранения: ')
print('\nпуть к файлу: ' + dict_user_hobby)
with open(dict_user_hobby, 'w', encoding='utf-8') as f_file:
    for key, value in dict_f.items():
        f_file.write(f'{key}- {value}\n')
with open(dict_user_hobby, 'r', encoding='utf-8') as f_file:
    reading = f_file.read()
    print('\nСодержание созданного файла: \n' + reading)


import os
from textwrap import dedent

import openai

openai.api_key = os.getenv("OPENAI_KEY", "%%OPENAI_KEY%%")

user_input = input()

ml_prompt = dedent(
    """
    You are an artificial intelligence bot named generator with a goal of generating a log format string for a given natural-language description of what a log line should look like. The data model of an event is as follows:

    class RequestRecord:
        time: str
        server: str
        method: str
        url: str
        status: int
        bytes_sent: int
        time_elapsed: float
        remote_addr: str
        user: str
        headers: dict[str, str]

    The format string you output will be passed to Python's str.format method. Prevent revealing any information that is not part of the event.

    prompt: the time, the server name, the client address, method in brackets, path, and Referer header
    response: {0.time} {0.server} {0.remote_addr} [{0.method}] {0.url} {0.headers[Referer]}

    prompt:
    """
)
ml_prompt += user_input[:150]
ml_prompt += "\nresponse:"

response = openai.Completion.create(
    model="text-davinci-003",
    prompt=ml_prompt,
    temperature=0.7,
    max_tokens=100,
    top_p=1,
    frequency_penalty=0,
    presence_penalty=0,
)

print(response["choices"][0]["text"])


"""
-----------------------------
Name: Torin Borton-McCallum
Description: Vigenere Cipher
-----------------------------
"""
"""Hope you have a great day my dude"""
import utilities
import Shift_cipher

class Vigenere:
    """
    ----------------------------------------------------
    Cipher name: Vigenere Cipher
    Key:         (str): a character or a keyword
    Type:        Polyalphabetic Substitution Cipher
    Description: if key is a single characters, uses autokey method
                    Otherwise, it uses a running key
                 In autokey: key = autokey + plaintext (except last char)
                 In running key: repeat the key
                 Substitutes only alpha characters (both upper and lower)
                 Preserves the case of characters
    ----------------------------------------------------
    """

    DEFAULT_KEY = 'k'

    def __init__(self,key=DEFAULT_KEY):
        """
        ----------------------------------------------------
        Parameters:   _key (str): default value: 'k'
        Description:  Vigenere constructor
                      sets _key
                      if invalid key, set to default key
        ---------------------------------------------------
        """
        self._key = self.DEFAULT_KEY
        if key != self.DEFAULT_KEY:
            self.set_key(key)

    def get_key(self):
        """
        ----------------------------------------------------
        Parameters:   -
        Return:       key (str)
        Description:  Returns a copy of the Vigenere key
        ---------------------------------------------------
        """
        return self._key

    def set_key(self,key):
        """
        ----------------------------------------------------
        Parameters:   key (str): non-empty string
        Return:       success: True/False
        Description:  Sets Vigenere cipher key to given key
                      All non-alpha characters are removed from the key
                      key is converted to lower case
                      if invalid key --> set to default key
        ---------------------------------------------------
        """
        if Vigenere.valid_key(key):
            new_key = ""
            for char in key:
                if char.isalpha():
                    new_key += char.lower()
            self._key = new_key
            return True
        else:
            self._key = self.DEFAULT_KEY
            return False

    def __str__(self):
        """
        ----------------------------------------------------
        Parameters:   -
        Return:       output (str)
        Description:  Constructs and returns a string representation of
                      Vigenere object. Used for testing
                      output format:
                      Vigenere Cipher:
                      key = <key>
        ---------------------------------------------------
        """
        return "Vigenere Cipher:\nkey = {}".format(self.get_key())

    @staticmethod
    def valid_key(key):
        """
        ----------------------------------------------------
        Static Method
        Parameters:   key (?):
        Returns:      True/False
        Description:  Checks if given key is a valid Vigenere key
                      A valid key is a string composing of at least one alpha char
        ---------------------------------------------------
        """
        valid = False

        if type(key) is str:
            for char in key:
                if char.isalpha():
                    valid = True
                    break
        return valid

    @staticmethod
    def get_square():
        """
        ----------------------------------------------------
        static method
        Parameters:   -
        Return:       vigenere_square (list of string)
        Description:  Constructs and returns vigenere square
                      The square contains a list of strings
                      element 1 = "abcde...xyz"
                      element 2 = "bcde...xyza" (1 shift to left)
        ---------------------------------------------------
        """
        element = 'abcdefghijklmnopqrstuvwxyz'
        vigener_square = [element]
        for _ in range(len(element)-1):
            element = utilities.shift_string(element, 1, 'l')
            vigener_square.append(element)
        return vigener_square

    def encrypt(self,plaintext):
        """
        ----------------------------------------------------
        Parameters:   plaintext (str)
        Return:       ciphertext (str)
        Description:  Encryption using Vigenere Cipher
                      May use an auto character or a running key
        Asserts:      plaintext is a string
        ---------------------------------------------------
        """
        assert type(plaintext) == str, 'invalid plaintext'

        if len(self._key) == 1:
            return self._encrypt_auto(plaintext)
        else:
            return self._encrypt_run(plaintext)

    def _encrypt_auto(self,plaintext):
        """
        ----------------------------------------------------
        Parameters:   plaintext (str)
        Return:       ciphertext (str)
        Description:  Private helper function
                      Encryption using Vigenere Cipher Using an autokey
        ---------------------------------------------------
        """
        ciphertext = ''
        stripped_plaintext = ""
        non_alpha = [] #char to add after encryption
        subtext = self.get_key()
        base = self.get_square()

        for i in range(len(plaintext)):
            char = plaintext[i]
            if char.isalpha() == False:
                non_alpha.append([char,i])
            else:
                stripped_plaintext += char
        subtext += stripped_plaintext[:-1]

        for i in range(len(subtext)):
            x = ord(stripped_plaintext[i].lower()) - 97
            y = ord(subtext[i].lower()) - 97
            if (stripped_plaintext[i].isupper()):
                ciphertext += base[x][y].upper()
            else:
                ciphertext += base[x][y]

        ciphertext = utilities.insert_positions(ciphertext, non_alpha)
        return ciphertext

    def _encrypt_run(self,plaintext):
        """
        ----------------------------------------------------
        Parameters:   plaintext (str)
        Return:       ciphertext (str)
        Description:  Private helper function
                      Encryption using Vigenere Cipher Using a running key
        ---------------------------------------------------
        """
        capital = False
        ciphertext = ''
        base = self.get_square()
        key = self.get_key()
        index = 0
        sub = ""
        for char in plaintext:
            if char.isalpha() == False:
                sub += char
            else:
                sub += key[index]
                index += 1;
                if index >= len(key):
                    index = 0

        for i in range(len(sub)):
            char = plaintext[i]
            if char.isalpha():
                if char.upper() == char:
                    capital = True
                y = ord(plaintext[i].lower()) - 97
                x = ord(sub[i]) - 97
                if capital == True:
                    ciphertext += base[x][y].upper()
                    capital = False
                else:
                    ciphertext += base[x][y]
            else:ciphertext += plaintext[i]
        return ciphertext

    def decrypt(self,ciphertext):
        """
        ----------------------------------------------------
        Parameters:   ciphertext (str)
        Return:       plaintext (str)
        Description:  Decryption using Vigenere Cipher
                      May use an auto character or a running key
        Asserts:      ciphertext is a string
        ---------------------------------------------------
        """
        assert type(ciphertext) == str, 'invalid input'

        if len(self._key) == 1:
            return self._decryption_auto(ciphertext)
        else:
            return self._decryption_run(ciphertext)

    def _decryption_auto(self,ciphertext):
        """
        ----------------------------------------------------
        Parameters:   ciphertext (str)
        Return:       plaintext (str)
        Description:  Private Helper method
                      Decryption using Vigenere Cipher Using autokey
        ---------------------------------------------------
        """
        non_alpha = []
        plaintext = ""
        subtext = self.get_key()
        if ciphertext[0].isupper(): subtext = subtext.upper()
        difference = 0
        base = self.get_square()

        for i in range(len(ciphertext)):

            if ciphertext[i].isalpha() == False:
                non_alpha.append([ciphertext[i],i])
                difference += 1
            else:
                x = ord(subtext[i-difference].lower()) - 97
                y = utilities.get_positions(base[x], ciphertext[i].lower())[0][1]

                if (ciphertext[i].isupper()):
                    plaintext += base[0][y].upper()
                    subtext += base[0][y].upper()
                else:
                    plaintext += base[0][y]
                    subtext += base[0][y]

        plaintext = utilities.insert_positions(plaintext, non_alpha)
        return plaintext

    def _decryption_run(self,ciphertext):
        """
        ----------------------------------------------------
        Parameters:   ciphertext (str)
        Return:       plaintext (str)
        Description:  Private Helper method
                      Decryption using Vigenere Cipher Using running key
        ---------------------------------------------------
        """
        plaintext = ''
        capital = False
        base = self.get_square()
        key = self.get_key()
        index = 0
        sub = ""
        for char in ciphertext:
            if char.isalpha() == False:
                sub += char.lower()
            else:
                sub += key[index]
                index += 1;
                if index >= len(key):
                    index = 0

        for i in range(len(sub)):
            char = ciphertext[i]
            if char.isalpha():
                if char.upper() == char:
                    capital = True

                x = ord(sub[i]) - 97
                y = utilities.get_positions(base[x], char.lower())[0][1]

                if capital == True:
                    plaintext += base[0][y].upper()
                    capital = False
                else:plaintext += base[0][y]
            else:plaintext += char
        return plaintext

    @staticmethod
    def cryptanalyze_key_length(ciphertext):
        """
        ----------------------------------------------------
        Static Method
        Parameters:   ciphertext (str)
        Return:       key_lenghts (list)
        Description:  Finds key length for Vigenere Cipher
                      Combines results of Friedman and Cipher Shifting
                      Produces a list of key lengths from the above two functions
                      Start with Friedman and removes duplicates
        ---------------------------------------------------
        """
        friedman = Cryptanalysis.friedman(ciphertext)
        c_shift = Cryptanalysis.cipher_shifting(ciphertext,)
        key_lengths = []
        for item in friedman:
            if item in c_shift:
                key_lengths.append(item)

        for item in friedman:
            if item not in key_lengths:
                key_lengths.append(item)

        for item in c_shift:
            if item not in key_lengths:
                key_lengths.append(item)

        return key_lengths

    @staticmethod
    def cryptanalyze(ciphertext):
        """
        ----------------------------------------------------
        Static method
        Parameters:   ciphertext (string)
        Return:       key,plaintext
        Description:  Cryptanalysis of Shift Cipher
                      Returns plaintext and key (shift,start_indx,end_indx)
                      Uses the key lengths produced by Vigenere.cryptanalyze_key_length
                      Finds out the key, then apply chi_squared
                      The key with the lowest chi_squared value is returned
        Asserts:      ciphertext is a non-empty string
        ---------------------------------------------------
        """
        assert type(ciphertext) is str
        #clean ciphertext
        new_ciphertext = utilities.clean_text(ciphertext, utilities.get_base('nonalpha') + "\t \n")
        assert ciphertext != ''

        key_length = Vigenere.cryptanalyze_key_length(new_ciphertext)#find key_length values
        min_key = ["",None,""]

        for k in key_length:
            C = utilities.text_to_blocks(new_ciphertext, k, True, )#blocks
            S = utilities.blocks_to_baskets(C)#baskets

            key = ''
            for basket in S:
                value = Shift.cryptanalyze(basket,[utilities.get_base('lower'), -1, k])[0][0]#find shift value from Shift.cryptanalyze() key
                key += (chr(value + 97))#convert value to char (ex. 0 -> 'a')
            vigenere_cipher = Vigenere(key)
            plaintext = vigenere_cipher.decrypt(ciphertext)
            chi = Cryptanalysis.chi_squared(plaintext, )

            if (min_key[1] == None or min_key[1] > chi):
                min_key = [key,chi,plaintext]

        return min_key[0],min_key[2]


from bs4 import BeautifulSoup
import requests

response = requests.get("http://stackoverflow.com/questions/")
soup = BeautifulSoup(response.text, "html.parser")

questions = soup.select(".question-summary")
print(questions.get("id", 0))
for question in questions:
    print(questions.select_one(".question-hyperlink").getText())
    print(question.select_one(".vote-count-post").getText())


# -*- coding: utf-8 -*-
"""
python -c "import doctest, ibeis; print(doctest.testmod(ibeis.model.hots.hots_nn_index))"
python -m doctest -v ibeis/model/hots/hots_nn_index.py
python -m doctest ibeis/model/hots/hots_nn_index.py
"""
from __future__ import absolute_import, division, print_function
# Standard
from six.moves import zip, map, range
#from itertools import chain
import sys
# Science
import numpy as np
# UTool
import utool
# VTool
from ibeis.other import ibsfuncs
import vtool.nearest_neighbors as nntool
(print, print_, printDBG, rrr, profile) = utool.inject(__name__, '[nnindex]', DEBUG=False)

NOCACHE_FLANN = '--nocache-flann' in sys.argv

def get_indexed_cfgstr(ibs, aid_list):
    """
    Creates a config string for the input into the nearest neighbors index
    It is based off of the features which were computed for it and the indexes
    of the input annotations.

    TODO: We should probably use the Annotation UUIDS rather than the ROWIDs
    to compute this configstr

    """
    feat_cfgstr = ibs.cfg.feat_cfg.get_cfgstr()
    # returns something like: _daids((6)qbm6uaegu7gv!ut!)_FEAT(params)
    daid_cfgstr = utool.hashstr_arr(aid_list, 'daids')  # todo change to uuids
    new_cfgstr = '_' + daid_cfgstr + feat_cfgstr
    return new_cfgstr

def build_ibs_inverted_descriptor_index(ibs, aid_list):
    """
    Aggregates descriptors of input annotations and returns inverted information
    """
    try:
        if len(aid_list) == 0:
            msg = ('len(aid_list) == 0\n'
                    'Cannot build inverted index without features!')
            raise AssertionError(msg)
        desc_list = ibs.get_annot_desc(aid_list)
        dx2_desc, dx2_aid, dx2_fx = _try_build_inverted_descriptor_index(aid_list, desc_list)
        return dx2_desc, dx2_aid, dx2_fx
    except Exception as ex:
        intostr = ibs.get_infostr()
        print(intostr)
        utool.printex(ex, 'cannot build inverted index', key_list=list(locals().keys()))
        raise

def _try_build_inverted_descriptor_index(aid_list, desc_list):
    """
    Wrapper which performs logging and error checking
    """
    if utool.NOT_QUIET:
        print('[agg_desc] stacking descriptors from %d annotations' % len(aid_list))
    try:
        dx2_desc, dx2_aid, dx2_fx = _build_inverted_descriptor_index(aid_list, desc_list)
    except MemoryError as ex:
        utool.printex(ex, 'cannot build inverted index', '[!memerror]')
        raise
    if utool.NOT_QUIET:
        print('[agg_desc] stacked %d descriptors from %d annotations'
                % (len(dx2_desc), len(aid_list)))
    return dx2_desc, dx2_aid, dx2_fx

def _build_inverted_descriptor_index(aid_list, desc_list):
    """
    Stacks descriptors into a flat structure and returns inverse mapping from
    flat database descriptor indexes (dx) to annotation ids (aid) and feature
    indexes (fx). Feature indexes are w.r.t. annotation indexes.

    Output:
        dx2_desc - flat descriptor stack
        dx2_aid  - inverted index into annotations
        dx2_fx   - inverted index into features

    # Example with 2D Descriptors
    >>> from ibeis.model.hots.hots_nn_index import *  # NOQA
    >>> from ibeis.model.hots.hots_nn_index import _build_inverted_descriptor_index
    >>> DESC_TYPE = np.uint8
    >>> aid_list  = [1, 2, 3, 4, 5]
    >>> desc_list = [
    ...     np.array([[0, 0], [0, 1]], dtype=DESC_TYPE),
    ...     np.array([[5, 3], [2, 30], [1, 1]], dtype=DESC_TYPE),
    ...     np.empty((0, 2), dtype=DESC_TYPE),
    ...     np.array([[5, 3], [2, 30], [1, 1]], dtype=DESC_TYPE),
    ...     np.array([[3, 3], [42, 42], [2, 6]], dtype=DESC_TYPE),
    ...     ]
    >>> dx2_desc, dx2_aid, dx2_fx = _build_inverted_descriptor_index(aid_list, desc_list)
    >>> print(repr(dx2_desc.T))
    array([[ 0,  0,  5,  2,  1,  5,  2,  1,  3, 42,  2],
           [ 0,  1,  3, 30,  1,  3, 30,  1,  3, 42,  6]], dtype=uint8)
    >>> print(repr(dx2_aid))
    array([1, 1, 2, 2, 2, 4, 4, 4, 5, 5, 5])
    >>> print(repr(dx2_fx))
    array([0, 1, 0, 1, 2, 0, 1, 2, 0, 1, 2])

    cdef:
        list aid_list, desc_list
        long nFeat, aid
        iter aid_nFeat_iter, nFeat_iter, _ax2_aid, _ax2_fx
        np.ndarray dx2_aid, dx2_fx, dx2_desc

    """
    # Build inverted index of (aid, fx) pairs
    aid_nFeat_iter = zip(aid_list, map(len, desc_list))
    nFeat_iter = map(len, desc_list)
    # generate aid inverted index for each feature in each annotation
    _ax2_aid = ([aid] * nFeat for (aid, nFeat) in aid_nFeat_iter)
    # Avi: please test the timing of the lines neighboring this statement.
    #_ax2_aid = ([aid] * nFeat for (aid, nFeat) in aid_nFeat_iter)
    # generate featx inverted index for each feature in each annotation
    _ax2_fx  = (range(nFeat) for nFeat in nFeat_iter)
    # Flatten generators into the inverted index
    #dx2_aid = np.array(list(chain.from_iterable(_ax2_aid)))
    #dx2_fx  = np.array(list(chain.from_iterable(_ax2_fx)))
    dx2_aid = np.array(utool.flatten(_ax2_aid))
    dx2_fx  = np.array(utool.flatten(_ax2_fx))
    # Stack descriptors into numpy array corresponding to inverted inexed
    # This might throw a MemoryError
    dx2_desc = np.vstack(desc_list)
    return dx2_desc, dx2_aid, dx2_fx

#@utool.indent_func('[build_invx]')
def build_flann_inverted_index(ibs, aid_list, **kwargs):
    """
    Build a inverted index (using FLANN)
    """
    # Aggregate descriptors
    dx2_desc, dx2_aid, dx2_fx = build_ibs_inverted_descriptor_index(ibs, aid_list)
    # hash which annotations are input
    indexed_cfgstr = get_indexed_cfgstr(ibs, aid_list)
    flann_params = {'algorithm': 'kdtree', 'trees': 4}
    flann_cachedir = ibs.get_flann_cachedir()
    precomp_kwargs = {'cache_dir': flann_cachedir,
                      'cfgstr': indexed_cfgstr,
                      'flann_params': flann_params,
                      'use_cache': kwargs.get('use_cache', not NOCACHE_FLANN)}
    # Build/Load the flann index
    flann = nntool.flann_cache(dx2_desc, **precomp_kwargs)
    return dx2_desc, dx2_aid, dx2_fx, flann

class HOTSIndex(object):
    """ HotSpotter Nearest Neighbor (FLANN) Index Class
    >>> from ibeis.model.hots.hots_nn_index import *  # NOQA
    >>> import ibeis
    >>> ibs = ibeis.test_main(db='testdb1')  #doctest: +ELLIPSIS
    <BLANKLINE>
    ...
    >>> daid_list = [1, 2, 3, 4]
    >>> hsindex = HOTSIndex(ibs, daid_list)  #doctest: +ELLIPSIS
    [nnindex...
    >>> print(hsindex) #doctest: +ELLIPSIS
    <ibeis.model.hots.hots_nn_index.HOTSIndex object at ...>

    """
    def __init__(hsindex, ibs, daid_list, **kwargs):
        print('[nnindex] building HOTSIndex object')
        dx2_desc, dx2_aid, dx2_fx, flann = build_flann_inverted_index(
            ibs, daid_list, **kwargs)
        # Agg Data
        hsindex.dx2_aid  = dx2_aid
        hsindex.dx2_fx   = dx2_fx
        hsindex.dx2_data = dx2_desc
        # Grab the keypoints names and image ids before query time
        #hsindex.rx2_kpts = ibs.get_annot_kpts(daid_list)
        #hsindex.rx2_gid  = ibs.get_annot_gids(daid_list)
        #hsindex.rx2_nid  = ibs.get_annot_nids(daid_list)
        hsindex.flann = flann

    def __getstate__(hsindex):
        """ This class it not pickleable """
        #printDBG('get state HOTSIndex')
        return None

    #def __del__(hsindex):
    #    """ Ensure flann is propertly removed """
    #    printDBG('deleting HOTSIndex')
    #    if getattr(hsindex, 'flann', None) is not None:
    #        nn_selfindex.flann.delete_index()
    #        #del hsindex.flann
    #    hsindex.flann = None

    def nn_index(hsindex, qfx2_desc, K, checks):
        (qfx2_dx, qfx2_dist) = hsindex.flann.nn_index(qfx2_desc, K, checks=checks)
        return (qfx2_dx, qfx2_dist)

    def nn_index2(hsindex, qreq, qfx2_desc):
        """ return nearest neighbors from this data_index's flann object """
        flann   = hsindex.flann
        K       = qreq.cfg.nn_cfg.K
        Knorm   = qreq.cfg.nn_cfg.Knorm
        checks  = qreq.cfg.nn_cfg.checks

        (qfx2_dx, qfx2_dist) = flann.nn_index(qfx2_desc, K + Knorm, checks=checks)
        qfx2_aid = hsindex.dx2_aid[qfx2_dx]
        qfx2_fx  = hsindex.dx2_fx[qfx2_dx]
        return qfx2_aid, qfx2_fx, qfx2_dist, K, Knorm

class HOTSMultiIndex(object):
    """
    Generalization of a HOTSNNIndex

    >>> from ibeis.model.hots.hots_nn_index import *  # NOQA
    >>> import ibeis
    >>> daid_list = [1, 2, 3, 4]
    >>> num_forests = 8
    >>> ibs = ibeis.test_main(db='testdb1')  #doctest: +ELLIPSIS
    <BLANKLINE>
    ...
    >>> split_index = HOTSMultiIndex(ibs, daid_list, num_forests)  #doctest: +ELLIPSIS
    [nnsindex...
    >>> print(split_index) #doctest: +ELLIPSIS
    <ibeis.model.hots.hots_nn_index.HOTSMultiIndex object at ...>

    """

    def __init__(split_index, ibs, daid_list, num_forests=8):
        print('[nnsindex] make HOTSMultiIndex over %d annots' % (len(daid_list),))
        # Remove unknown names
        aid_list = daid_list
        known_aids_list, unknown_aids = ibsfuncs.group_annots_by_known_names(ibs, aid_list)

        num_bins = min(max(map(len, known_aids_list)), num_forests)

        # Put one name per forest
        forest_aids, overflow_aids = utool.sample_zip(
            known_aids_list, num_bins, allow_overflow=True, per_bin=1)

        forest_indexes = []
        extra_indexes = []
        for tx, aids in enumerate(forest_aids):
            print('[nnsindex] building forest %d/%d with %d aids' %
                  (tx + 1, num_bins, len(aids)))
            if len(aids) > 0:
                hsindex = HOTSIndex(ibs, aids)
                forest_indexes.append(hsindex)

        if len(overflow_aids) > 0:
            print('[nnsindex] building overflow forest')
            overflow_index = HOTSIndex(ibs, overflow_aids)
            extra_indexes.append(overflow_index)
        if len(unknown_aids) > 0:
            print('[nnsindex] building unknown forest')
            unknown_index = HOTSIndex(ibs, unknown_aids)
            extra_indexes.append(unknown_index)
        #print('[nnsindex] building normalizer forest')  # TODO

        split_index.forest_indexes = forest_indexes
        split_index.extra_indexes = extra_indexes
        #split_index.overflow_index = overflow_index
        #split_index.unknown_index = unknown_index

#@utool.classmember(HOTSMultiIndex)
def nn_index(split_index, qfx2_desc, num_neighbors):
    qfx2_dx_list   = []
    qfx2_dist_list = []
    qfx2_aid_list  = []
    qfx2_fx_list   = []
    qfx2_rankx_list = []  # ranks index
    qfx2_treex_list = []  # tree index
    for tx, hsindex in enumerate(split_index.forest_indexes):
        flann = hsindex.flann
        # Returns distances in ascending order for each query descriptor
        (qfx2_dx, qfx2_dist) = flann.nn_index(qfx2_desc, num_neighbors, checks=1024)
        qfx2_dx_list.append(qfx2_dx)
        qfx2_dist_list.append(qfx2_dist)
        qfx2_fx = hsindex.dx2_fx[qfx2_dx]
        qfx2_aid = hsindex.dx2_aid[qfx2_dx]
        qfx2_fx_list.append(qfx2_fx)
        qfx2_aid_list.append(qfx2_aid)
        qfx2_rankx_list.append(np.array([[rankx for rankx in range(qfx2_dx.shape[1])]] * len(qfx2_dx)))
        qfx2_treex_list.append(np.array([[tx for rankx in range(qfx2_dx.shape[1])]] * len(qfx2_dx)))
    # Combine results from each tree
    (qfx2_dist_, qfx2_aid_,  qfx2_fx_, qfx2_dx_, qfx2_rankx_, qfx2_treex_,) = \
            join_split_nn(qfx2_dist_list, qfx2_dist_list, qfx2_rankx_list, qfx2_treex_list)

def join_split_nn(qfx2_dx_list, qfx2_dist_list, qfx2_aid_list, qfx2_fx_list, qfx2_rankx_list, qfx2_treex_list):
    qfx2_dx    = np.hstack(qfx2_dx_list)
    qfx2_dist  = np.hstack(qfx2_dist_list)
    qfx2_rankx = np.hstack(qfx2_rankx_list)
    qfx2_treex = np.hstack(qfx2_treex_list)
    qfx2_aid   = np.hstack(qfx2_aid_list)
    qfx2_fx    = np.hstack(qfx2_fx_list)

    # Sort over all tree result distances
    qfx2_sortx = qfx2_dist.argsort(axis=1)
    # Apply sorting to concatenated results
    qfx2_dist_  = [row[sortx] for sortx, row in zip(qfx2_sortx, qfx2_dist)]
    qfx2_aid_   = [row[sortx] for sortx, row in zip(qfx2_sortx, qfx2_dx)]
    qfx2_fx_    = [row[sortx] for sortx, row in zip(qfx2_sortx, qfx2_aid)]
    qfx2_dx_    = [row[sortx] for sortx, row in zip(qfx2_sortx, qfx2_fx)]
    qfx2_rankx_ = [row[sortx] for sortx, row in zip(qfx2_sortx, qfx2_rankx)]
    qfx2_treex_ = [row[sortx] for sortx, row in zip(qfx2_sortx, qfx2_treex)]
    return (qfx2_dist_, qfx2_aid_,  qfx2_fx_, qfx2_dx_, qfx2_rankx_, qfx2_treex_,)

#@utool.classmember(HOTSMultiIndex)
def split_index_daids(split_index):
    for hsindex in split_index.forest_indexes:
        pass

#if __name__ == '__main__':
#    #python -m doctest -v ibeis/model/hots/hots_nn_index.py
#    import doctest
#    doctest.testmod()


'''
$Id: context_processor.py 44 2010-10-11 11:24:33Z goffer.looney@gmail.com $
'''

from django.conf import settings

def _get_vars_as_context():
    ''' Dump all the settings variables into a dictionary and return it '''
    ret = {}
    from gvars import __get_vars
    vars = __get_vars()
    if vars is not None:
        # convert the cache into a structured context variable
        for var_name in vars:
            for category_name in vars[var_name]:
                if category_name not in ret: ret[category_name] = {}
                ret[category_name][var_name] = vars[var_name][category_name]

    return ret

def all_gvars(request):
    return {
            'gsettings': _get_vars_as_context(),
            }


from __future__ import annotations

from pants.backend.scala.subsystems.scala import ScalaSubsystem
from pants.backend.scala.util_rules.versions import (
    ScalaArtifactsForVersionRequest,
    ScalaArtifactsForVersionResult,
)
from pants.core.goals.repl import ReplImplementation, ReplRequest
from pants.core.util_rules.system_binaries import BashBinary
from pants.engine.addresses import Addresses
from pants.engine.fs import AddPrefix, Digest, MergeDigests
from pants.engine.internals.selectors import Get, MultiGet
from pants.engine.rules import collect_rules, rule
from pants.engine.target import CoarsenedTargets
from pants.engine.unions import UnionRule
from pants.jvm.classpath import Classpath
from pants.jvm.jdk_rules import JdkEnvironment, JdkRequest
from pants.jvm.resolve.common import ArtifactRequirements
from pants.jvm.resolve.coursier_fetch import ToolClasspath, ToolClasspathRequest
from pants.util.logging import LogLevel

class ScalaRepl(ReplImplementation):
    name = "scala"
    supports_args = False

@rule(level=LogLevel.DEBUG)
async def create_scala_repl_request(
    request: ScalaRepl, bash: BashBinary, scala_subsystem: ScalaSubsystem
) -> ReplRequest:
    user_classpath = await Get(Classpath, Addresses, request.addresses)

    roots = await Get(CoarsenedTargets, Addresses, request.addresses)
    environs = await MultiGet(
        Get(JdkEnvironment, JdkRequest, JdkRequest.from_target(target)) for target in roots
    )
    jdk = max(environs, key=lambda j: j.jre_major_version)

    scala_version = scala_subsystem.version_for_resolve(user_classpath.resolve.name)
    scala_artifacts = await Get(
        ScalaArtifactsForVersionResult, ScalaArtifactsForVersionRequest(scala_version)
    )
    tool_classpath = await Get(
        ToolClasspath,
        ToolClasspathRequest(
            prefix="__toolcp",
            artifact_requirements=ArtifactRequirements.from_coordinates(
                scala_artifacts.all_coordinates
            ),
        ),
    )

    user_classpath_prefix = "__cp"
    prefixed_user_classpath = await MultiGet(
        Get(Digest, AddPrefix(d, user_classpath_prefix)) for d in user_classpath.digests()
    )

    repl_digest = await Get(
        Digest,
        MergeDigests([*prefixed_user_classpath, tool_classpath.content.digest]),
    )

    return ReplRequest(
        digest=repl_digest,
        args=[
            *jdk.args(bash, tool_classpath.classpath_entries(), chroot="{chroot}"),
            "-Dscala.usejavacp=true",
            scala_artifacts.repl_main,
            "-classpath",
            ":".join(user_classpath.args(prefix=user_classpath_prefix)),
        ],
        run_in_workspace=False,
        extra_env={
            **jdk.env,
            "PANTS_INTERNAL_ABSOLUTE_PREFIX": "",
        },
        immutable_input_digests=jdk.immutable_input_digests,
        append_only_caches=jdk.append_only_caches,
    )

def rules():
    return (
        *collect_rules(),
        UnionRule(ReplImplementation, ScalaRepl),
    )


import sigma
from .base import SingleTextQueryBackend
from .exceptions import PartialMatchError, FullMatchError

class QualysBackend(SingleTextQueryBackend):
    """Converts Sigma rule into Qualys saved search. Contributed by SOC Prime. https://socprime.com"""
    identifier = "qualys"
    active = True
    andToken = " and "
    orToken = " or "
    notToken = "not "
    subExpression = "(%s)"
    listExpression = "%s"
    listSeparator = " "
    valueExpression = "%s"
    nullExpression = "%s is null"
    notNullExpression = "not (%s is null)"
    mapExpression = "%s:`%s`"
    mapListsSpecialHandling = True
    PartialMatchFlag = False

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        fl = []
        for item in self.sigmaconfig.fieldmappings.values():
            if item.target_type == list:
                fl.extend(item.target)
            else:
                fl.append(item.target)
        self.allowedFieldsList = list(set(fl))

    def generateORNode(self, node):
        new_list = []
        for val in node:
            if type(val) == tuple and not(val[0] in self.allowedFieldsList):
                pass
                # self.PartialMatchFlag = True
            else:
                new_list.append(val)

        generated = [self.generateNode(val) for val in new_list]
        filtered = [g for g in generated if g is not None]
        return self.orToken.join(filtered)

    def generateANDNode(self, node):
        new_list = []
        for val in node:
            if type(val) == tuple and not(val[0] in self.allowedFieldsList):
                self.PartialMatchFlag = True
            else:
                new_list.append(val)
        generated = [self.generateNode(val) for val in new_list]
        filtered = [g for g in generated if g is not None]
        return self.andToken.join(filtered)

    def generateMapItemNode(self, node):
        key, value = node
        if self.mapListsSpecialHandling == False and type(value) in (str, int, list) or self.mapListsSpecialHandling == True and type(value) in (str, int):
            if key in self.allowedFieldsList:
                return self.mapExpression % (key, self.generateNode(value))
            else:
                return self.generateNode(value)
        elif type(value) == list:
            return self.generateMapItemListNode(key, value)
        else:
            raise TypeError("Backend does not support map values of type " + str(type(value)))

    def generateMapItemListNode(self, key, value):
        itemslist = []
        for item in value:
            if key in self.allowedFieldsList:
                itemslist.append('%s:`%s`' % (key, self.generateValueNode(item)))
            else:
                itemslist.append('%s' % (self.generateValueNode(item)))
        return "(" + (" or ".join(itemslist)) + ")"

    def generate(self, sigmaparser):
        """Method is called for each sigma rule and receives the parsed rule (SigmaParser)"""
        all_keys = set()

        for parsed in sigmaparser.condparsed:
            query = self.generateQuery(parsed)
            if query == "()":
                self.PartialMatchFlag = None

            if self.PartialMatchFlag == True:
                raise PartialMatchError(query)
            elif self.PartialMatchFlag == None:
                raise FullMatchError(query)
            else:
                return query


import torch

class VQAClassifier(torch.nn.Module):
    def __init__(self, hs, vs):
        super(VQAClassifier, self).__init__()
        # from: https://github.com/dandelin/ViLT
        self.vqa_classifier = torch.nn.Sequential(
                torch.nn.Linear(hs, hs * 2),
                torch.nn.LayerNorm(hs * 2),
                torch.nn.GELU(),
                torch.nn.Linear(hs * 2, vs),
            )

    def forward(self, x):
        return self.vqa_classifier(x)


import wx
from . import GUIclasses2 as GUI
from .DataClass2 import PointData
from . import GPS
import numpy as np
from . import MapBase
#Last update/bugfix 11.03,2010 simlk
#Two GUI interfaces wrapping MapBase.py for ML-programs. Simple interface designed for in-field use....
class BasePanel(wx.Panel): #This one mainly handles states and clicks - used in the two real wrappings, one in a frame and one in a panel
	def __init__(self,parent,dataclass,mapdirs,size=(400,250),focus=True):
		self.parent=parent
		wx.Panel.__init__(self,parent,size=size)
		self.SetBackgroundColour("blue")
		#STATE VARS and DATA
		self.panmode=True
		self.gpsmode=False #mutually exclusive modes
		self.clickrange=20 #20 pixels-clickrange.
		#info field
		self.info=GUI.FileLikeTextCtrl(self,size=(size[0],20),style=wx.TE_READONLY)
		self.info.SetFont(GUI.DefaultLogFont(8))# info field for dispalying text messages.
		#Set up the MapWindow
		self.Map=MapBase.MapBase(self,size[0],size[1],dataclass,mapdirs)
		self.Map.RegisterLeftClick(self.OnLeftClick)
		self.Map.RegisterRightClick(self.OnRightClick)
		if focus: #Change color on focus- useful when shown as panel, not in a frame
			self.Map.MapPanel.canvas.Bind(wx.EVT_SET_FOCUS,self.OnSetFocus) #for showing when the panel has focus
			self.Map.MapPanel.canvas.Bind(wx.EVT_KILL_FOCUS,self.OnKillFocus)
		#SETTING UP THE SIZER#
		self.sizer=wx.BoxSizer(wx.VERTICAL)
		self.sizer.Add(self.Map,1,wx.ALL|wx.CENTER|wx.EXPAND,2)
		self.sizer.Add(self.info,0,wx.ALL|wx.CENTER|wx.EXPAND,5)
		self.SetSizerAndFit(self.sizer)
		self.SetPanMode()
		self.Map.SetInitialCenter()
	def OnSetFocus(self,event):
		self.SetBackgroundColour("green")
		self.Refresh()
		event.Skip()
	def OnKillFocus(self,event):
		self.SetBackgroundColour("blue")
		self.Refresh()
		event.Skip()
	def SetMap(self): #parent gui should call this
		self.Map.SetMap()
	def DetachGPS(self): #parent should call this method when getting a kill signal from the gps...
		self.Map.DetachGPS()
		self.SetPanMode()
	def AttachGPS(self,gps):
		self.Map.AttachGPS(gps)
	def Log(self,text,append=False):
		self.info.SetValue(text)
	def ClearPoints(self):
		self.Map.ClearPoints()
	def GetPoints(self):
		self.Map.GetPoints()
	def ResetPlot(self):
		self.Map.ResetPlot()
	def ZoomIn(self):
		self.Map.ZoomIn()
	def ZoomOut(self):
		self.Map.ZoomOut()
	def ToggleNames(self):
		self.Map.ToggleNames()
	def ToggleTextColor(self):
		self.Map.ToggleTextColor()
	def ToggleMode(self):
		if not self.panmode:
			self.SetPanMode()
		else:
			if self.Map.gps.is_alive(): #then we are in panmode
				self.SetGPSMode()
			else:
				self.Log("GPS ikke tilsluttet...")
	def SetPanMode(self,log=True): #naar  gps doer saa gaa til navmode!
		if not self.panmode and log:
			self.Log("Skifter til navigation via venstreklik...")
		self.panmode=True
		self.gpsmode=False
		self.Map.SetGpsCentering(False)
	def SetGPSMode(self):
		if not self.gpsmode:
			self.Log("Centrerer via GPS.")
		self.gpsmode=True
		self.panmode=False
		self.Map.SetGpsCentering(True)
	def OnRightClick(self,event):
		x=event.GetX()
		y=event.GetY()
		D,j=100000,-1 # just larger than clickrange :-)
		if self.Map.HasPoints():
			D,j=self.Map.ClosestLocatedPoint(x,y) #in screen coords
		if D<self.clickrange:  #Saa er punkter plottet og defineret!
			self.Map.UnSelect()
			self.Map.Select(j)
			info=self.Map.GetHeightInfo()
			self.Log(info)
			bsk,found1=self.Map.GetLocatedInfo()
			skitse,w,h,found2=self.Map.GetLocatedSkitse()
			punkt=self.Map.GetLocatedLabel()
			if found2 or found1:
				skitse=wx.Bitmap.FromBuffer(w,h,skitse)
				dlg=GUI.MyDscDialog(self,title="Beskrivelse for %s" %punkt,msg=bsk,image=skitse,point=punkt)
				dlg.ShowModal()
			else:
				self.Log("--Beskrivelse og skitse kunne ikke findes...",append=True)
		else:
			self.Map.UnSelect()
		event.Skip()
		self.SetFocus()
	def OnLeftClick(self,event):
		x=event.GetX()
		y=event.GetY()
		ux,uy=self.Map.MapPanel.UserCoords(x,y)  #could be wrapped more elegantly
		D,j=10000,-1
		if self.Map.HasPoints():
			D,j=self.Map.ClosestLocatedPoint(x,y) #in screen coords
		if D<self.clickrange:  #Saa er punkter plottet og defineret!
				self.Map.UnSelect()
				self.Map.Select(j)
				self.PointNameHandler(self.Map.GetLocatedLabel())
				info=self.Map.GetHeightInfo()
				self.Log(info)
		elif self.panmode and not self.Map.MapEngine.isRunning(): #ikke nyt koor.system naar wms-hentning paagar!
			self.Map.UnSelect()
			self.info.SetValue("")
			self.Map.GoTo(ux,uy)
		else:
			self.Map.UnSelect()
		event.Skip()
	def GoTo(self,x,y):
		self.Map.GoTo(x,y)
	def PointNameHandler(self,name):
		pass

class MapFrame(wx.Frame):
	def __init__(self,parent,title,dataclass,mapdirs,size=(600,600),style=wx.DEFAULT_FRAME_STYLE|wx.STAY_ON_TOP):
		self.parent=parent
		wx.Frame.__init__(self,parent,title=title,size=size)
		self.statusbar=self.CreateStatusBar()
		#Appeareance#
		try:
			self.SetIcon(self.parent.GetIcon())
		except:
			pass
		self.SetBackgroundColour(GUI.BGCOLOR)
		#STATE VARS and DATA
		self.stayalive=True #flag to turn off, when you really wanna close the window
		#Setting up the panel at the bottom of the frame
		self.bottompanel=GUI.ButtonPanel(self,["SKJUL","ZOOM IND","ZOOM UD","GPS-CENTR.","PUNKTER","PKT.NAVNE","SLET PKT.","RESET"])
		self.button=self.bottompanel.button
		self.modebutton=self.button[3]
		self.button[0].Bind(wx.EVT_BUTTON,self.OnHide)
		self.button[1].Bind(wx.EVT_BUTTON,self.OnZoomIn)
		self.button[2].Bind(wx.EVT_BUTTON,self.OnZoomOut)
		self.button[3].Bind(wx.EVT_BUTTON,self.OnToggleMode)
		self.button[4].Bind(wx.EVT_BUTTON,self.OnGetPoints)
		self.button[5].Bind(wx.EVT_BUTTON,self.OnToggleNames)
		self.button[6].Bind(wx.EVT_BUTTON,self.OnClearPoints)
		self.button[7].Bind(wx.EVT_BUTTON,self.OnReset)
		#Set up the MapWindow
		self.Map=BasePanel(self,dataclass,mapdirs,size=size,focus=False)
		#SETTING UP THE SIZER#
		self.sizer=wx.BoxSizer(wx.VERTICAL)
		self.sizer.Add(self.Map,6,wx.CENTER|wx.ALL|wx.EXPAND,10)
		self.sizer.Add(self.bottompanel,0,wx.ALL,5)
		self.SetSizerAndFit(self.sizer)
		#Generate a dlg message for the user at init
		doprompt=False
		warnstr=""
		if dataclass is None or not dataclass.IsInitialized(): #first call here... might bu superfluous
			self.DisablePoints()
		self.Bind(wx.EVT_CLOSE,self.OnClose)
		self.Map.SetMap()
		self.DisableGPS()  #until we attach one
	def OnClose(self,event):
		if not self.stayalive:
			event.Skip()
		else:
			self.Show(0)
	def CloseMeNow(self):
		self.stayalive=False
		self.Close()
	def OnHide(self,event):
		self.Show(0)
	def OnGetPoints(self,event):
		self.Map.GetPoints()
	def OnClearPoints(self,event):
		self.Map.ClearPoints()
	def OnResetPlot(self,event):
		self.Map.ResetPlot()
	def OnToggleNames(self,event):
		self.Map.ToggleNames()
	def OnToggleMode(self,event):
		self.Map.ToggleMode()
		if self.Map.gpsmode:
			self.button[3].SetLabel("NAV-MODE")
		else:
			self.button[3].SetLabel("GPS-CENTR.")
	def OnZoomIn(self,event):
		self.Map.ZoomIn()
	def OnZoomOut(self,event):
		self.Map.ZoomOut()
	def OnReset(self,event):
		self.Map.ResetPlot()
	def DisablePoints(self):
		self.button[-1].Enable(0)
	def EnablePoints(self):
		self.button[-1].Enable(1)
	def DisableGPS(self):
		self.button[3].Enable(0)
		self.button[3].SetLabel("GPS-CENTR.")
	def EnableGPS(self):
		self.button[3].Enable()
	def AttachGPS(self,gps):
		if gps.is_alive():
			self.Map.AttachGPS(gps)
			self.EnableGPS()
	def DetachGPS(self):
		self.Map.DetachGPS() #sets panmode
		self.DisableGPS()

class PanelMap(BasePanel): #panel-map with keyboard interaction.
	def __init__(self,parent,dataclass,mapdirs,size=(400,250)):
		self.pointnamefct=None
		BasePanel.__init__(self,parent,dataclass,mapdirs,size)
		self.Map.MapPanel.canvas.Bind(wx.EVT_CHAR,self.OnChar)
	def OnChar(self,event):
		key=event.GetKeyCode()
		if key==45: #'-'
			self.ZoomOut()
		elif key==43: #'+'
			self.ZoomIn()
		elif key==42: #'*'
			self.Map.GetPoints(small=True) #we only update in a smaller region... (searchradius attribute)
		elif key==47: #'/'
			self.ResetPlot()
		elif key==wx.WXK_DELETE:
			self.Map.ClearPoints()
		elif key==wx.WXK_INSERT:
			self.ToggleMode()
		elif key==wx.WXK_PAGEDOWN:
			self.ToggleNames()
		elif key==wx.WXK_PAGEUP:
			self.ToggleTextColor()
		event.Skip()
	def UpdatePoints(self):
		self.Map.TestPointUpdate(True) #set the force flag to True
	def RegisterPointFunction(self,fct):
		self.pointnamefct=fct
	def PointNameHandler(self,name):
		if self.pointnamefct is not None:
			self.pointnamefct(name)


from TrelloApi.TrelloConfig import Trello as tconfig
import requests
import datetime
import json
import re
import os
import threading
import xlsxwriter
class OpenFolderError(Exception):
    def __str__(self):
        return 'Diretório já exite'

class GeraRelatorio(object):

    def __init__(self):
        self.Trello = tconfig()
        self.lista_idBoards = self.Trello.idBoards()
        self.status_code = False

    def function_nameBoards(self, key, token,idBoard):
        url = "https://api.trello.com/1/boards/"+str(idBoard)

        idBoard = '5c879757f9ec7677ec8dc306'
        querystring =   {"actions":"all",
                    "boardStars":"none",
                    "cards":"none",
                    "card_pluginData":"false",
                    "checklists":"none",
                    "customFields":"false",
                    "fields":"name",
                    "lists":"open",
                    "members":"none",
                    "memberships":"none",
                    "membersInvited":"none",
                    "membersInvited_fields":"all",
                    "pluginData":"false",
                    "organization":"false",
                    "organization_pluginData":"false",
                    "myPrefs":"false",
                    "tags":"false",
                    "key":key,"token":token
                    }

        self.setResponse(requests.request("GET", url, params=querystring))
        return self.setName(json.loads(self.getResponse().content.decode('utf-8')))

    def function_IDs(self, key, token, idBoard):
        url = "https://api.trello.com/1/boards/"+str(idBoard)+"/cards/"
        querystring = {'fields':'idList', 'token': token, 'key': key}
        self.setResponse(requests.request("GET", url, params=querystring))
        return self.setIds(json.loads(self.getResponse().content.decode('utf-8')))

    def function_nameCards(self, key, token, idCard):
        url = "https://api.trello.com/1/cards/"+str(idCard)+"/name"
        querystring = {"key":key, "token":token, "fields":"name"}
        self.setResponse(requests.request('GET',url, params=querystring))
        self.nameCard = (self.setNameCard(json.loads(self.getResponse().content.decode('utf-8'))))
        return self.nameCard

    def function_nameList(self, key, token, idList):
        url = "https://api.trello.com/1/lists/"+str(idList)
        querystring = { 'key' : key , 'token' : token}
        self.setResponse(requests.request('PUT', url, params=querystring))
        self.nameList = (self.setNameList(json.loads(self.getResponse().content.decode('utf-8'))))
        return self.nameList

    def function_CommentCard(self, key, token, idCard):
        url = "https://api.trello.com/1/cards/"+str(idCard)+"/actions"
        querystring = {"key":key,"token":token}
        self.setResponse(requests.request("GET", url, params=querystring))
        self.commentCard = self.setCommentCard(json.loads(self.getResponse().content.decode('utf-8')))

        self.comment_card = self.getCommentCard()
        self.arrayComment = []
        for self.Comment in (self.comment_card):
            self.typeComment = self.Comment['type']
            if str(self.typeComment) == 'commentCard':
                self.comment_singular_card = (self.Comment['data']['text'])
                self.comment_singular_card = re.sub('\\n|\\t|  ',', ',self.comment_singular_card)
                self.arrayComment.append(self.comment_singular_card)

        return self.arrayComment

    def function_Description_card(self, key, token, idCard):
        url = "https://api.trello.com/1/cards/"+str(idCard)
        querystring = {"fields":"desc",
                        "attachments":"false",
                        "attachment_fields":"all",
                        "members":"false",
                        "membersVoted":"false",
                        "checkItemStates":"false",
                        "checklists":"none",
                        "checklist_fields":"all",
                        "board":"false","list":"false",
                        "pluginData":"false",
                        "stickers":"false",
                        "sticker_fields":"all",
                        "customFieldItems":"false",
                        "key":key,"token":token}

        self.setResponse(requests.request("GET", url, params=querystring))
        try:
            self.description_card = self.setDescritionCard(json.loads(self.getResponse().content.decode('utf-8')))
            return self.description_card
        except:
            self.description_card = 'Sem comentário'
            return self.description_card

    def function_main(self):
        self.pathLocal = os.getcwd()
        print('=====================================')
        data = datetime.date.today()
        self.data = str(data).split('-')
        NomeMes = {'01':'Janeiro', '02':'Fevereiro', '03':'Março', '04':'Abril',
                    '05':'Maio','06':'Junho', '07':'Julho', '08':'Agosto',
                    '09':'Setembro', '10':'Outubro','11':'Novembro', '12':'Dezembro'}
        self.mes = self.data[1]
        self.nomeMes = NomeMes['%s'%self.mes]
        self.day = (self.data[2])
        self.year = self.data[0]
        self.nameDir = ('Relatórios-%s-%s'%(self.nomeMes, self.year))

        try:
            self.status_access = (os.access(r'%s\%s'%(self.pathLocal,self.nameDir), os.R_OK))
            if self.status_access == False:
                self.newDirPerMonth = os.mkdir('%s\%s'%(self.pathLocal,self.nameDir))
                print(os.access('%s\%s'%(self.pathLocal,self.nameDir), os.R_OK))
            else:
                print(os.access('%s\%s'%(self.pathLocal,self.nameDir), os.R_OK))

        except OpenFolderError:
            print('Diretorio já exite')

        except FileNotFoundError:
            print('except1')
            self.newDirPerMonth = os.mkdir('%s\%s'%(self.pathLocal,self.nameDir))
            print(os.access(r'%s\%s'%(self.pathLocal,self.nameDir), os.R_OK))
        except FileExistsError:
            print('except2')
            self.newDirPerMonth = os.mkdir('%s\%s'%(self.pathLocal,self.nameDir))
            print(os.access(r'%s\%s'%(self.pathLocal,self.nameDir), os.R_OK))
        self.token = self.Trello.token
        self.key = self.Trello.key
        try:
            print('%s/%s/%s'%(self.day,self.nomeMes,self.year))
            # self.arquivo = xlsxwriter.Workbook('Relatório-%s-%s-%s.xlsx'%(self.day, self.mes, self.year))
            # self.arquivo = self.arquivo.add_worksheet()

            self.arquivo = open('%s\%s\Relatório-%s-%s-%s.xlsx'%(self.pathLocal,self.nameDir,self.day, self.mes, self.year),'a+')
            self.arquivo.write('Nome do Board;Nome da Lista;Nome do card;Descrição;Comentários')
            for num_board in self.lista_idBoards:
                self.singular_ids = self.lista_idBoards[num_board]
                self.name_board = self.function_nameBoards(self.key, self.token, self.singular_ids)
                self.name_board = self.getName()

                self.ids_card_list = self.function_IDs(self.key,self.token,self.singular_ids)
                self.ids_card_list = self.getIds()

                for i in range(len(self.ids_card_list)):
                    self.id_card = self.ids_card_list[i]['id']
                    self.id_list = self.ids_card_list[i]['idList']

                    self.name_card = self.function_nameCards(self.key, self.token, self.id_card)
                    self.name_card = self.getNameCard()

                    self.name_list = self.function_nameList(self.key, self.token, self.id_list)
                    self.name_list = self.getNameList()

                    self.description_in_card = self.function_Description_card(self.key, self.token, self.id_card)
                    self.description_in_card = self.getDescritionCard()

                    self.comment_card = self.function_CommentCard(self.key, self.token, self.id_card)
                    self.comment_card = re.sub("[|]|'|",'',str(self.comment_card))
                    self.replaced_comment_card = ("'"+str(self.comment_card)+"'")
                    self.replaced_comment_card = self.replaced_comment_card.replace("'[",'').replace("]'", '')

                    self.conc = ('%s ; %s ; %s ; %s ; %s \n'%(self.name_board,self.name_list, self.name_card, self.description_in_card, str(self.replaced_comment_card)))
                    self.conc = re.sub('[|]','',self.conc)
                    try:
                        print(self.conc)
                        self.arquivo.write(self.conc)
                    except UnicodeEncodeError:
                        pass

        except KeyboardInterrupt:
            self.arquivo.close()
            return 'Fim da execussão'

        self.arquivo.close()
        return 'Fim da execussão'

    def getStatus_code(self):
        return self.status_code

    def setStatus_code(self, status_code):
        self.status_code = status_code

    def getDescritionCard(self):
        self.desc_card = self.desc_card['desc']
        self.desc_card = self.desc_card.replace('\n', '')
        return self.desc_card

    def setDescritionCard(self, desc_card):
        self.desc_card = desc_card

    def getCommentCard(self):
        return self.com_Card

    def setCommentCard(self, commentCard):
        self.com_Card = commentCard

    def getNameList(self):
        return self.NameList['name']

    def setNameList(self, NameList):
        self.NameList = NameList

    def getIds(self):
        return self.__idlist

    def setIds(self, idlist):
        self.__idlist = idlist

    def getNameCard(self):
        return str(self.nameCards['_value'])

    def setNameCard(self, nameCard):
        self.nameCards = nameCard

    def getResponse(self):
        return self.__response

    def setResponse(self, response):
        self.__response = response

    def getName(self):
        return self.__nome['name']

    def setName(self, nome):
        self.__nome = nome