import math
import sys
import time
import torch
import torchvision
import pandas as pd
import networkx
from networkx.algorithms.components.connected import connected_components

import utils
from coco_utils import get_coco_api_from_dataset
from coco_eval import CocoEvaluator

def train_one_epoch(model, optimizer, data_loader, device, epoch, print_freq):
    model.train()
    metric_logger = utils.MetricLogger(delimiter="  ")
    metric_logger.add_meter('lr', utils.SmoothedValue(window_size=1, fmt='{value:.6f}'))
    header = 'Epoch: [{}]'.format(epoch)

    lr_scheduler = None
    if epoch == 0:
        warmup_factor = 1. / 1000
        warmup_iters = min(1000, len(data_loader) - 1)
        lr_scheduler = utils.warmup_lr_scheduler(optimizer, warmup_iters, warmup_factor)

    for images, targets, _ in metric_logger.log_every(data_loader, print_freq, header):
        images = list(image.to(device) for image in images)
        targets = [{k: v.to(device) for k, v in t.items()} for t in targets]
        loss_dict = model(images, targets)
        losses = sum(loss for loss in loss_dict.values())
        # reduce losses over all GPUs for logging purposes
        loss_dict_reduced = utils.reduce_dict(loss_dict)
        losses_reduced = sum(loss for loss in loss_dict_reduced.values())
        loss_value = losses_reduced.item()
        if not math.isfinite(loss_value):
            print("Loss is {}, stopping training".format(loss_value))
            sys.exit(1)

        optimizer.zero_grad()
        losses.backward()
        optimizer.step()

        if lr_scheduler is not None:
            lr_scheduler.step()
        metric_logger.update(loss=losses_reduced, **loss_dict_reduced)
        metric_logger.update(lr=optimizer.param_groups[0]["lr"])
    return loss_dict_reduced, losses_reduced

def _get_iou_types(model):
    model_without_ddp = model
    if isinstance(model, torch.nn.parallel.DistributedDataParallel):
        model_without_ddp = model.module
    iou_types = ["bbox"]
    if isinstance(model_without_ddp, torchvision.models.detection.MaskRCNN):
        iou_types.append("segm")
    if isinstance(model_without_ddp, torchvision.models.detection.KeypointRCNN):
        iou_types.append("keypoints")
    return iou_types

@torch.no_grad()
def evaluate(model, data_loader, device):
    n_threads = torch.get_num_threads()
    # FIXME remove this and make paste_masks_in_image run on the GPU
    torch.set_num_threads(n_threads)
    model.eval()
    metric_logger = utils.MetricLogger(delimiter="  ")
    header = 'Test:'

    coco = get_coco_api_from_dataset(data_loader.dataset)
    iou_types = _get_iou_types(model)
    coco_evaluator = CocoEvaluator(coco, iou_types)

    for images, targets, _ in metric_logger.log_every(data_loader, 100, header):
        images = list(img.to(device) for img in images)
        targets = [{k: v.to(device) for k, v in t.items()} for t in targets]
        if device.type == 'cuda':
            torch.cuda.synchronize()
        model_time = time.time()
        outputs = model(images)

        outputs = [{k: v.to(device) for k, v in t.items()} for t in outputs]
        model_time = time.time() - model_time

        res = {target["image_id"].item(): output for target, output in zip(targets, outputs)}
        evaluator_time = time.time()
        coco_evaluator.update(res)
        evaluator_time = time.time() - evaluator_time
        metric_logger.update(model_time=model_time, evaluator_time=evaluator_time)

    # gather the stats from all processes
    metric_logger.synchronize_between_processes()
    print("Averaged stats:", metric_logger)
    coco_evaluator.synchronize_between_processes()

    # accumulate predictions from all images
    coco_evaluator.accumulate()
    coco_evaluator.summarize()
    torch.set_num_threads(n_threads)
    return coco_evaluator

def _area(a, b):  # returns None if rectangles don't intersect
    '''
    return intersect area of two boxes if overlaping otherwise return None
    '''
    dx = min(a.x1, b.x1) - max(a.x0, b.x0)
    dy = min(a.y1, b.y1) - max(a.y0, b.y0)
    if (dx>=0) and (dy>=0):
        return dx*dy

def _to_graph(l):
    G = networkx.Graph()
    for part in l:
        # each sublist is a bunch of nodes
        G.add_nodes_from(part)
        # it also imlies a number of edges:
        G.add_edges_from(_to_edges(part))
    return G

def _to_edges(l):
    """
        treat `l` as a Graph and returns it's edges
        to_edges(['a','b','c','d']) -> [(a,b), (b,c),(c,d)]
    """
    it = iter(l)
    last = next(it)
    for current in it:
        yield last, current
        last = current

def idx_cleanboxes(boxes, scores, second_cutoff=0.83):
    '''
    boxes: 2d npy containing all boxes in the image
    socres: 1d npy containing score for each box in boxes
    '''
    df = pd.DataFrame(boxes)
    df.columns = ['x0', 'y0', 'x1', 'y1']
    df['score'] = scores

    clusters = []
    for box1 in df.itertuples():
        cluster = {box1.Index}
        df2 = df.iloc[box1.Index+1:, :]
        for box2 in df2.itertuples():
            if _area(box1, box2):
                cluster.add(box2.Index)
        clusters.append(cluster)
    G = _to_graph(clusters)

    final_index = []
    for c in connected_components(G):
        c = list(c)
        scores = df.loc[c, 'score']
        cdas = scores[scores>=second_cutoff]
        cdas_idx = cdas.index.to_list()
        if len(cdas_idx)>0:
            final_index.extend(cdas_idx)
        else:
            final_index.append(scores.idxmax())
    return final_index


"""
    1.拿到主页源代码，找到iframe的链接
    2.拿到iframe链接的页面源码，获取m3u8链接
    3.下载m3u8文件(2层)，过程有些繁琐
    4.对文件进行检索，拿到小视频的链接，进行异步下载
    5.解密合并小的文件切片还原为大的MP4文件仅从播放
"""

import requests
import re
import asyncio
import aiohttp
import aiofiles
from bs4 import BeautifulSoup
from Crypto.Cipher import AES  #pycryptodome
# from Crypto import AES
import os

def get_iframe_src(url):
    iframe_resp = requests.get(url)
    main_page = BeautifulSoup(iframe_resp.text, "html.parser")
    src = main_page.find("iframe").get("src")
    print(src)
    # return "https://boba.52kuyun.com/share/xfPs9NPHvYGhNzFp"  # 得到了iframe的src

def get_first_m3u8_url(url):
    resp = requests.get(url)
    # print(resp.text)
    obj = re.compile(r'var main = "(?P<m3u8_url>.*?)"', re.S)
    m3u8_url = obj.search(resp.text).group("m3u8_url")
    # print(m3u8_url)
    return m3u8_url

# 第一层的m3u8文件，找ajax发送的请求，第一词发送的一般在第一条，然后找和resource文件下的源代码匹配
def get_first_m3u8_url(url):
    first_m3u8_resp = requests.get(url)
    obj = re.compile(r'var main="(?P<m3u8_url>.*?)', re.S)
    m3u8_url = obj.search(first_m3u8_resp.text).group("m3u8_url")
    return m3u8_url

# 下载m3u8文件，可复用
def download_m3u8_file(url, name):
    resp =requests.get(url)
    with open(name, mode='wb') as f:
        f.write(resp.content)

async def downlaod_ts(url, name, session):
    async with session.get(url) as resp:
        async with aiofiles.open(f'vide2/{name}',mode='wb') as f:
            await f.write(await resp.content.read())
    print('下载完成！')

# 创建一部下载任务
async def aio_download(second_m3u8_url_download):
    tasks = []
    async with aiohttp.ClientSession() as session:
        async with aiofiles.open("越狱第一季第一集.txt", mode='r', encoding='utf-8') as f:
            async for line in f:
                if line.startswith("#"):
                    continue
                line = line.strip()
                ts_url = second_m3u8_url_download + line
                task = asyncio.create_task(downlaod_ts(ts_url, line, session))
                tasks.append(task)
            await asyncio.wait(tasks)

def get_key(key_url):
    resp = requests.get(key_url)
    return resp.text

async def dec_ts(name, key):
    aes = AES.new(key=key, IV=b"0000000000000000", mode=AES.MODE_CBC)
    async with aiofiles.open(f"video2/{name}", mode='rb') as f1, \
        aiofiles.open(f"video2/temp_{name}", mode='wb') as f2:

        bs = await f1.read()
        await f2.write(aes.decrypt(bs))
    print(f"{name} 处理完成")

async def aio_dec(key):
    tasks = []
    async with aiofiles.open("越狱第一季第一集2.txt", mode='r', encoding='utf-8') as f:
        async for line in f:
            if line.startswith("#"):
                continue
            line = line.strip()
            task = asyncio.create_task(dec_ts(line, key))
            tasks.append(task)
        await asyncio.wait(tasks)

def merge_ts():
    "windows: copy /b 1.ts+2.ts+3.ts xxx.mp4"
    lst = []
    with open("越狱第一季第一集2.txt", mode='r', encoding='utf-8') as f:
        for line in f:
            if line.startswith("#"):
                continue
            line = line.strip()
            lst.append(f"video2/temp_{line}")
    s = "+".join(lst)
    os.system(f"copy /b {s} movie.mp4")
    print("下载终于完成了")

def main(url):
    # 1.iframe的src
    iframe_url = get_iframe_src(url)
    # 2.第一层的m3u8文件，通过source里面的页面源代码，搜索m3u8然后就在script标签里面找到了该地址
    #这一步操作是为了获取到第一层的m3u8文件的下载链接地址的
    first_m3u8_url = get_first_m3u8_url(iframe_url)
    # iframe的真正下载地址，需要拼接字符串。 split
    # 第二层的m3u8文件，怎么找到呢，看第一层爬下来的m3u8文件，有个链接，然后，
    # 看下面发送的ajax请求，比对一下，找到域名(就是第一次请求iframe的网址的域名
    iframe_domain = iframe_url.split("/share")[0] #就是iframe的url
    first_m3u8_url = iframe_domain + first_m3u8_url #拼接成功了
    download_m3u8_file(first_m3u8_url, "越狱第一季第一集.txt")

    # 下载好了之后就去读取第一个m3u8文件
    with open("越狱第一季第一集.txt", mode='r', encoding='utf-8') as f:
        for line in f:
            if line.startswith("#"):
                continue
            else:
                line = line.strip() # hls/index.m3u8
                # 准备拼接第二层m3u8的下载路径
                # https://boba.52kuyun.com/20170906/Moh2l9zV/index.m3u8?sign=548ae366a075f0f9e7c76af215aa18e1 这是第一层m3u8文件
                # https://boba.52kuyun.com/20170906/Moh2l9zV/ + hls/index.m3u8
                # https://boba.52kuyun.com/20170906/Moh2l9zV/hls/index.m3u8  第二次m3u8下载路径
                second_m3u8_url = first_m3u8_url.split("/index.m3u8")[0] + line
                download_m3u8_file(second_m3u8_url, "越狱第一季第一集2.txt")
                print('总的m3u8文件下载完成！')
                # cFN8o3436000.ts 第二个m3u8文件的路径
    # https://boba.52kuyun.com/20170906/Moh2l9zV/hls/cFN8o3436000.ts  ts下载路径

    second_m3u8_url_download = second_m3u8_url.replace("index.m3u8", "")

#     异步协程
    asyncio.run(aio_download(second_m3u8_url_download))

    # 这里还是要看xhr请求
    key_url = second_m3u8_url_download + "key.key" #偷懒写法

    key = get_key(key_url)

    # 5.2 解密
    asyncio.run(aio_dec(key))

    # 6. 合并ts文件为mp4文件
    merge_ts()

if __name__ == '__main__':
    url = "https://www.91kanju.com/vod-play/5104-1-1.html"
    main(url)


from epipack.symbolic_epi_models import get_temporal_interpolation, SymbolicEpiModel
import sympy
from epipack.plottools import plot
from bfmplot import pl
import numpy as np

data = np.array([
    (0.0,  2.00),
    (0.75, 2.68),
    (1.5,  3.00),
    (2.25, 2.78),
    (3.0,  2.14),
    (3.75, 1.43),
    (4.5,  1.02),
    (5.25, 1.14),
    (6.0,  1.72),
])

times, rates = data[:,0], data[:,1]

f = get_temporal_interpolation(times, rates, interpolation_degree=1)

S, I, R, t, rho = sympy.symbols("S I R t rho")

model = SymbolicEpiModel([S,I,R])
model.set_processes([
        (S, I, f, I, I),
        (I, rho, R),
    ])\
    .set_initial_conditions({S:0.99,I:0.01})\
    .set_parameter_values({rho:1})

t = np.linspace(0,6,1000)

result = model.integrate(t)
ax = plot(t,result)
ax.legend()
ax.get_figure().savefig('interp_SIR_symbolic.png',dpi=300)
pl.show()


from collections import deque, defaultdict

def main():
    n, k = map(int, input().split())
    visited = defaultdict(int)
    q = deque([[n, 0]])
    res = 1
    while q:
        now, cnt = q.popleft()
        if now==k and visited[k]:
            res += 1
        if now<0 or now>100000 or visited[now]:
            continue
        visited[now] = cnt
        q.append([now+1, cnt+1])
        q.append([now-1, cnt+1])
        q.append([now*2, cnt+1])
    print(visited[k])
    print(res)

def first():
    n, k = map(int, input().split())
    visited = defaultdict(int)
    q = deque([[n, 0]])
    res, minval = 0, False
    while q:
        now, cnt = q.popleft()
        if now == k:
            if minval and cnt == minval:
                res += 1
            elif not minval:
                minval = cnt
                res += 1
        if now<0 or now>100000 or visited[now]>30 or minval and cnt>=minval:
            continue
        visited[now] += 1
        q.append([now+1, cnt+1])
        q.append([now-1, cnt+1])
        q.append([now*2, cnt+1])
    print(minval)
    print(res)

if __name__ == "__main__":
    main()


#!/usr/bin/env python3.7

"""Deploy certificate to Halon"""

import base64
import json
from datetime import datetime, timezone

import requests

# read configuration
with open("halon-deploy-cert.json") as input_file:
    config = json.load(input_file)

timestamp = datetime.now(tz=timezone.utc).strftime("%Y-%m-%d %H:%M:%S%z")
halon_url = f"https://{config['hostname']}/api/1.0.0"

# read cert & key
with open(config["certfile"]) as cert_file:
    pki_certdata = cert_file.read()
with open(config["keyfile"]) as key_file:
    pki_keydata = key_file.read()

# construct new PKI parameters
pki_key = f"pki__{config['pki_key']}"
pki_params = [
    f"{config['certname']} (certbot {timestamp})",
    "x509+privatekey",
    base64.b64encode((pki_certdata + pki_keydata).encode()).decode(),
]

# create session for Halon
session = requests.Session()
session.auth = (config["username"], config["password"])
if "cacert" in config:
    session.verify = config["cacert"]

# fetch configuration from Halon
response = session.get(f"{halon_url}/config/revisions/HEAD")
response.raise_for_status()

# digest current configuration
last_halon_config = response.json()
last_halon_revision = last_halon_config["id"]

# prepare new configuration
next_halon_config = {}
next_halon_revision = last_halon_revision + 1
next_halon_config["message"] = f"certbot {timestamp}"
next_halon_config["config"] = []

# update new configuration
for c in last_halon_config["config"]:
    paramset = c.copy()
    if paramset["name"] == pki_key:
        paramset["params"] = pki_params
    next_halon_config["config"].append(paramset)

# deploy configuration
if config.get("deploy", False):
    response = session.post(
        f"{halon_url}/config/revisions/{next_halon_revision}", json=next_halon_config
    )
    response.raise_for_status()
else:
    print(json.dumps(next_halon_config, indent=4))


from typing import List

class Solution:
    def findMaxLength(self, nums: List[int]) -> int:
        max_count = 0
        d = dict()
        count = 0
        for i, n in enumerate(nums):
            if n == 1:
                count += 1
            else:
                count -= 1
            if count == 0:
                max_count = i + 1
            if count in d:
                max_count = max(i - d.get(count), max_count)
            else:
                d[count] = i
        return max_count


import torch
import numpy as np
import time
import math

class BaseModule(torch.nn.Module):
    def __init__(self):
        super().__init__()

    def forward(self, x):
        return x

class Agent0(torch.nn.Module):
    def __init__(self,res=100):
        super().__init__()
        self.res = res
        self.grid = torch.nn.Parameter(torch.zeros((res,res)))

    def forward(self, x):

        pos_x = (x[:,0]*(self.res-1)).floor().long()
        pos_y = (x[:,1]*(self.res-1)).floor().long()
        value = self.grid[pos_x,pos_y]
        action = torch.rand((len(x),4), device=x.device)*2-1

        #print(torch.stack([pos_x, pos_y,value],1))
        return action, value[...,None]

class Agent1(torch.nn.Module):
    def __init__(self,in_features, hidden_dim):
        super().__init__()
        self.l1 = torch.nn.Linear(in_features, hidden_dim)
        self.l2 = torch.nn.Linear(hidden_dim, hidden_dim)
        self.action = torch.nn.Linear(hidden_dim, 2*2)
        self.value = torch.nn.Linear(hidden_dim, 1)

    def forward(self, x):
        x = torch.relu(self.l1(x))
        x = torch.relu(self.l2(x))
        action = self.action(x)
        value = self.value(x)
        return action, value

class Agent2(torch.nn.Module):
    def __init__(self,in_features, hidden_dim):
        super().__init__()
        self.l1 = torch.nn.Linear(in_features, hidden_dim)
        self.l2 = torch.nn.Linear(hidden_dim, hidden_dim)
        self.l3 = torch.nn.Linear(hidden_dim, hidden_dim)
        self.l4 = torch.nn.Linear(hidden_dim, hidden_dim)
        self.action = torch.nn.Linear(hidden_dim, 2*2)
        self.value = torch.nn.Linear(hidden_dim, 1)

    def forward(self, x):
        x = torch.relu(self.l1(x))
        x = torch.relu(self.l2(x))
        xr = x
        x = torch.relu(self.l3(x))
        x = torch.relu(self.l4(x)) + xr
        action = self.action(x)
        value = self.value(x)
        return action, value

class Agent3(torch.nn.Module):
    def __init__(self,in_features, hidden_dim):
        super().__init__()
        self.n = 3
        self.l1 = torch.nn.Linear(in_features*(self.n*2+1), hidden_dim)
        self.l2 = torch.nn.Linear(hidden_dim, hidden_dim)
        self.action = torch.nn.Linear(hidden_dim, 2*2)
        self.value = torch.nn.Linear(hidden_dim, 1)
        self.K = torch.pow(1.5, torch.arange(6))

    def forward(self, x):
        e=torch.empty(x.shape+(self.n*2,), device=x.device)
        for i in range(self.n):
            e[:,:,2*i+0]=torch.sin(x*self.K[i]+i)
            e[:,:,2*i+1]=torch.cos(x*self.K[i]+i)
        x = torch.concat([x,e.reshape(x.shape[0],-1)],-1)
        x = torch.relu(self.l1(x))
        x = torch.relu(self.l2(x))
        action = self.action(x)
        value = self.value(x)
        return action, value

class Agent4(torch.nn.Module):
    def __init__(self,in_features, hidden_dim, levels=16, table_size=2048, channels=2, growith_factor = 1.3):
        super().__init__()
        self.l1 = torch.nn.Linear(in_features+levels*channels, hidden_dim)
        self.l2 = torch.nn.Linear(hidden_dim, hidden_dim)
        self.action = torch.nn.Linear(hidden_dim, 2*2)
        self.value = torch.nn.Linear(hidden_dim, 1)

        self.levels = levels
        self.table_size = table_size
        self.channels = channels
        self.T = torch.nn.Parameter(torch.randn(levels, table_size, channels))
        self.K = torch.nn.Parameter(torch.pow(growith_factor, torch.arange(levels)[:,None]), requires_grad=False)

        self.PRIMES = [2654436881, 5915587277, 1500450271, 3267000013, 5754853343,
         4093082899, 9576890767, 3628273133, 2860486313, 5463458053,
         3367900313, 5654500741, 5654500763, 5654500771, 5654500783,
         5654500801, 5654500811, 5654500861, 5654500879, 5654500889,
         5654500897, 5654500927, 5654500961, 5654500981, 5654500993,
         9999999967, 7654487179, 7654489553, 7654495087, 7654486423,
         7654488209, 8654487029, 8654489771, 8654494517, 8654495341]

    def forward(self, x):

        self.K = self.K.to(x.device)
        p = (x.cumsum(-1)[:,None]*self.K)
        c = p.frac().sort(-1)[0]
        R = (p.frac()[...,None]>=c[...,None,:]).float()
        R = torch.concat([R,torch.zeros(*R.shape[:-1],1, device=x.device)],-1)
        c = torch.concat([c,torch.ones(*c.shape[:-1],1, device=x.device)],-1)
        A = R + p.floor()[...,None]
        A -= torch.concat([torch.zeros_like(A[...,-1:,:]),A[...,:-1,:]],-2)
        c -= torch.concat([torch.zeros_like(c[...,-1:]),c[...,:-1]],-1)

        cidx = A[:,:,0].long() * self.PRIMES[0]
        for i in range(1, A.shape[2]):
            cidx = cidx ^ (A[:,:,i].int() * self.PRIMES[i])

        feats=[]
        for i in range(self.levels):
            t=self.T[i]
            feats.append(t[cidx[:,i] % self.table_size])
        f = torch.stack(feats,1)
        f = (f*c[...,None]/self.K[...,None]).sum(2)

        x = torch.concat([x, f.reshape(x.shape[0],-1)],-1)

        x = torch.relu(self.l1(x))
        x = torch.relu(self.l2(x))
        action = self.action(x)
        value = self.value(x)
        return action, value

config={
	"optimizer": {
		"otype": "Adam",
		"learning_rate": 1e-2,
		"beta1": 0.9,
		"beta2": 0.99,
		"epsilon": 1e-15,
		"l2_reg": 1e-6
	},
	"encoding": {
		"otype": "HashGrid",
		"n_levels": 16,
		"n_features_per_level": 2,
		"log2_hashmap_size": 15,
		"base_resolution": 4,
		"per_level_scale": 1.3
	},
	"network": {
		"otype": "FullyFusedMLP",
		"activation": "ReLU",
		"output_activation": "None",
		"n_neurons": 64,
		"n_hidden_layers": 2
	}
}

import tinycudann as tcnn

class Agent5(torch.nn.Module):
    def __init__(self, in_coordinates, hidden_dim=64, levels=16, table_bit_size=15, channels=2, growith_factor = 1.25, base_res=1):
        super().__init__()

        self.in_coordinates=in_coordinates

        config__encoding = {
            "otype": "HashGrid",
            "n_levels": levels,
            "n_features_per_level": channels,
            "log2_hashmap_size": table_bit_size,
            "base_resolution": base_res,
            "per_level_scale": growith_factor
        }
        config_nn = {
            "otype": "FullyFusedMLP",
            "activation": "ReLU",
            "output_activation": "None",
            "n_neurons": hidden_dim,
            "n_hidden_layers": 2
	    }

        self.encodings = torch.nn.ParameterList()
        for _ in range(in_coordinates):
            self.encodings.append(tcnn.Encoding(2, config__encoding))

        self.nn = tcnn.Network(in_coordinates * levels * channels, 5, config_nn)

    def forward(self, x):

        x = x.reshape(x.shape[0], self.in_coordinates, 2)

        enc = []
        for i in range(self.in_coordinates):
            enc.append( self.encodings[i](x[:,i]) )
        x = torch.concat(enc,-1)
        x = self.nn(x)

        return x[:,:-1], x[:,-1:]

class Agent6(torch.nn.Module):
    def __init__(self,in_features, hidden_dim, heads=64, heads_dim=2, levels=5):
        super().__init__()

        self.heads = heads
        self.in_features = in_features
        self.hidden_dim = hidden_dim
        self.heads_dim = heads_dim
        self.levels= levels
        self.groups = 1

        self.l1 = torch.nn.Linear(in_features, hidden_dim)

        self.L_residual = torch.nn.ParameterList()
        for i in range(levels):
            lr1 = torch.nn.Linear(hidden_dim, hidden_dim)
            pr1 = torch.nn.Linear(hidden_dim, heads*heads_dim)
            lr2 = torch.nn.Linear((hidden_dim + heads)//self.groups, hidden_dim//self.groups)
            self.L_residual.append(torch.nn.ParameterList([lr1,pr1,lr2]))

        self.action = torch.nn.Linear(hidden_dim, 2*2)
        self.value = torch.nn.Linear(hidden_dim, 255)

    def forward(self, x):

        x = self.l1(x)

        for i in range(self.levels):
            lr1,pr1,lr2 = self.L_residual[i]
            xr = 2

            p1 = pr1(x).reshape(-1,self.heads, self.heads_dim)
            d1 = torch.norm(p1, dim=-1)
            x = torch.relu(lr1(x))
            x = torch.concat([x, d1],-1)

            x = x.reshape(-1,(self.hidden_dim + self.heads)//self.groups)
            x = torch.relu(lr2(x)).reshape(-1,self.hidden_dim)
            x = x + xr

        action = self.action(x)
        value = self.value(x)
        return action, value

class Agent7(torch.nn.Module):
    def __init__(self,in_features, hidden_dim):
        super().__init__()

        self.in_features = in_features
        self.hidden_dim = hidden_dim

        self.l1 = torch.nn.Linear(in_features, hidden_dim)

        self.l2 = torch.nn.Linear(hidden_dim*2, hidden_dim)

        self.l3 = torch.nn.Linear(hidden_dim*2, hidden_dim)

        self.action = torch.nn.Linear(hidden_dim*2, 2*2)
        self.value = torch.nn.Linear(hidden_dim*2, 1)

    def forward(self, x):
        x = self.l1(x)
        x = torch.concat([x.relu(), x.square()],-1)

        x = self.l2(x)
        x = torch.concat([x.relu(), x.square()],-1)

        x = self.l3(x)
        x = torch.concat([x.relu(), x.square()],-1)

        action = self.action(x)
        value = self.value(x)
        return action, value

class Agent8(torch.nn.Module):
    def __init__(self,in_features, hidden_dim):
        super().__init__()
        self.l1 = torch.nn.Linear(in_features, hidden_dim)
        self.l2 = torch.nn.Linear(hidden_dim, hidden_dim)
        self.action = torch.nn.Linear(hidden_dim, 2*2)
        self.value = torch.nn.Linear(hidden_dim, 1)

    def forward(self, x):
        x = self.l1(x).relu().square()
        x = self.l2(x).relu().square()
        action = self.action(x)
        value = self.value(x)
        return action, value

class Agent9(torch.nn.Module):
    def __init__(self,in_features, hidden_dim):
        super().__init__()
        self.l1 = torch.nn.Linear(in_features, hidden_dim)
        self.l2 = torch.nn.Linear(hidden_dim, hidden_dim)
        self.action = torch.nn.Linear(hidden_dim, 2*2)
        self.value = torch.nn.Linear(hidden_dim, 1)

    def forward(self, x):
        x = torch.exp(-self.l1(x).square())
        x = torch.exp(-self.l2(x).square())
        action = self.action(x)
        value = self.value(x)
        return action, value

class Agent10(torch.nn.Module):
    def __init__(self, in_points, hidden_dim):
        super().__init__()
        self.in_points = in_points
        self.n = 0
        self.l1 = torch.nn.Linear((in_points*2+in_points*(in_points-1))*(self.n*2+1), hidden_dim)
        self.l2 = torch.nn.Linear(hidden_dim, hidden_dim)
        self.action = torch.nn.Linear(hidden_dim, 2*2)
        self.value = torch.nn.Linear(hidden_dim, 255)
        self.K = torch.pow(1.5, torch.arange(6))

    def forward(self, x):

        S = x.shape

        xs = x.reshape(S[0], self.in_points, 2)
        c = torch.cdist(xs,xs)
        c = c.reshape(S[0], -1)[:,1:].reshape(S[0], self.in_points-1, self.in_points+1)[:,:,:-1].reshape(S[0], -1)

        x = torch.concat([x,c],-1)

        e=torch.empty(x.shape+(self.n*2,), device=x.device)
        for i in range(self.n):
            e[:,:,2*i+0]=torch.sin(x*self.K[i]+i)
            e[:,:,2*i+1]=torch.cos(x*self.K[i]+i)
        x = torch.concat([x,e.reshape(x.shape[0],-1)],-1)
        x = torch.relu(self.l1(x))
        x = torch.relu(self.l2(x))
        action = self.action(x)
        value = self.value(x)
        return action, value

class Agent11(torch.nn.Module):
    def __init__(self, in_points, hidden_dim):
        super().__init__()
        self.n = 2
        self.in_points = in_points
        self.t_L = torch.nn.TransformerEncoderLayer(d_model=2*(1+self.n*2), nhead=1+self.n*2, dim_feedforward=hidden_dim)
        self.T = torch.nn.TransformerEncoder(self.t_L, 2)
        self.action = torch.nn.Linear(in_points*2*(1+self.n*2), 4)
        self.value = torch.nn.Linear(in_points*2*(1+self.n*2), 1)
        self.K = torch.pow(1.5, torch.arange(6))

    def forward(self, x):
        e=torch.empty(x.shape+(self.n*2,), device=x.device)
        for i in range(self.n):
            e[:,:,2*i+0]=torch.sin(x*self.K[i]+i)
            e[:,:,2*i+1]=torch.cos(x*self.K[i]+i)
        x = torch.concat([x.reshape(x.shape[0], self.in_points, 2),e.reshape(x.shape[0], self.in_points, -1)], -1)
        x = self.T(x)
        x = x.reshape(x.shape[0], -1)
        action = self.action(x)
        value = self.value(x)
        return action, value

class Agent12(torch.nn.Module):
    def __init__(self, in_features, heads=64, heads_dim=2):
        super().__init__()

        self.heads = heads
        self.in_features = in_features
        self.heads_dim = heads_dim

        self.p1 = torch.nn.Linear(in_features, heads*heads_dim)
        self.p2 = torch.nn.Linear(heads*(heads_dim+1), heads*heads_dim)
        self.p3 = torch.nn.Linear(heads*(heads_dim+1), heads*heads_dim)

        self.action = torch.nn.Linear(heads*(heads_dim+1), 2*2)
        self.value = torch.nn.Linear(heads*(heads_dim+1), 255)

    def forward(self, x):
        p1 = self.p1(x)
        d1 = torch.norm(p1.reshape(-1,self.heads, self.heads_dim), dim=-1)
        x = torch.relu(p1)
        x = torch.concat([x, d1],-1)

        p2 = self.p2(x)
        d2 = torch.norm(p2.reshape(-1,self.heads, self.heads_dim), dim=-1)
        x = torch.relu(p2)
        x = torch.concat([x, d2],-1)

        p3 = self.p3(x)
        d3 = torch.norm(p3.reshape(-1,self.heads, self.heads_dim), dim=-1)
        x = torch.relu(p3)
        x = torch.concat([x, d3],-1)

        action = self.action(x)
        value = self.value(x)
        return action, value

class Agent13(torch.nn.Module):
    def __init__(self, in_points, hidden_dim, levels=3):
        super().__init__()
        self.in_points = in_points

        self.l1 = torch.nn.Linear((in_points*2), hidden_dim)
        self.p1 = torch.nn.Parameter(torch.randn((hidden_dim, in_points*2)))
        self.l2 = torch.nn.Linear(hidden_dim*2, hidden_dim)
        self.p2 = torch.nn.Parameter(torch.randn((hidden_dim, hidden_dim*2)))
        self.action = torch.nn.Linear(hidden_dim*2, 2*2)
        self.value = torch.nn.Linear(hidden_dim*2, 255)
        self.eps = 1e-10

    def forward(self, x):

        l = self.l1(x).relu()
        p = (self.p1.exp()/x.shape[-1] @ x.square()[...,None] + self.eps).sqrt()[...,0]
        x = torch.concat([l,p],-1)

        l = self.l2(x).relu()
        p = (self.p2.exp()/x.shape[-1] @ x.square()[...,None] + self.eps).sqrt()[...,0]
        x = torch.concat([l,p],-1)

        action = self.action(x)
        value = self.value(x)
        return action, value

class Agent14(torch.nn.Module):
    def __init__(self, in_points, hidden_dim):
        super().__init__()
        self.in_points = in_points
        in_features = in_points*2
        self.l1 = torch.nn.Linear(in_features + in_features**2 + in_points*(in_points-1), hidden_dim)
        #self.l1 = torch.nn.Linear(in_features, hidden_dim)

        self.l2 = torch.nn.Linear(hidden_dim, hidden_dim)
        self.l3 = torch.nn.Linear(hidden_dim, hidden_dim)

        self.l4 = torch.nn.Linear(hidden_dim, hidden_dim)
        self.l5 = torch.nn.Linear(hidden_dim, hidden_dim)

        self.action = torch.nn.Linear(hidden_dim, 25)
        self.value = torch.nn.Linear(hidden_dim, 255)
        with torch.no_grad():
            #self.action.weight[:] *= 0.1
            #self.action.bias[:] = 1/hidden_dim
            self.value.weight[:] = 1
            self.value.bias[:]= 0
            pass

    def forward(self, x):

        S = x.shape

        x2 = x[:,None]*x[:,:,None]/2

        xs = x.reshape(S[0], self.in_points, 2)
        c = torch.cdist(xs,xs)
        c = c.reshape(S[0], -1)[:,1:].reshape(S[0], self.in_points-1, self.in_points+1)[:,:,:-1].reshape(S[0], -1)

        x = torch.concat([x,x2.flatten(1),c],1)#,x3.flatten(1)
        x = torch.relu(self.l1(x))

        xr = x
        x = torch.relu(self.l2(x))
        x = torch.relu(self.l3(x)) + xr

        xr = x
        x = torch.relu(self.l4(x))
        x = torch.relu(self.l5(x)) + xr

        action = self.action(x)
        value = self.value(x)
        return action, value


import requests
from ..constant import CHARACTER_URL, PERMANENT_CHARACTER_URL

def getPermanentCharacter():
    return requests.get(PERMANENT_CHARACTER_URL).json()["permanent"]

def categoryCharacterInStar():
    originalData = requests.get(CHARACTER_URL).json()["data"]["list"][0]["list"]
    level_5 = {}
    level_4 = {}
    for eachCharacter in originalData:
        if "开拓者" not in eachCharacter["title"]:
            if "五星" in eachCharacter["ext"]:
                level_5.update({eachCharacter["title"]: eachCharacter["content_id"]})
            elif "四星" in eachCharacter["ext"]:
                level_4.update({eachCharacter["title"]: eachCharacter["content_id"]})
    return {"5": level_5, "4": level_4}


import os, sys
import re
import string
import nltk
import nltk.corpus
import time
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import tensorflow as tf
import seaborn as sn
from keras import Sequential
from keras.initializers import Constant
from keras.layers import Embedding, Conv1D, MaxPooling1D, LSTM, Dense, Dropout, Bidirectional, GRU, Concatenate
from keras.regularizers import l2
from keras.preprocessing import sequence
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
from keras.optimizers import Adadelta

def loadEnsembleData(datafile):
    x=[]
    labels=[]
    with open (datafile, "r") as f:
        for line in f:
            x.append([float(line.split(",")[0]),float(line.split(",")[1]),float(line.split(",")[2])])
            labels.append(int(line.split(",")[3]))
    return np.asarray(x), np.asarray(labels)

def train_ensemble(x, labels):
    model=Sequential()
#    model.add(Dense(3, activation="relu"))
    model.add(Dense(1, activation="relu"))
    model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])

#previous results datafile
datafile="all_results.txt"
testdatafile="test_results.txt"
X_train, Y_train = loadData(datafile)
x_test, y_test = loadData(testdatafile)
model=train_ensemble(X_train,Y_train)
start = time.time()
history = model.fit(X_train, y_train, epochs=100, verbose=2,validation_data=(X_test, y_test), batch_size=20)
stop = time.time()
print(f"Overall training time: {stop - start} s")

# evaluate the model
loss, accuracy = model.evaluate(X_train, y_train, verbose=False)
print("Training Accuracy: {:.4f}".format(accuracy))
loss, accuracy = model.evaluate(X_test, y_test, verbose=False)
print("Test Accuracy: {:.4f}".format(accuracy))

########## plot accuracy and loss
plt.style.use('ggplot')
acc = history.history['accuracy']
val_acc = history.history['val_accuracy']
loss = history.history['loss']
val_loss = history.history['val_loss']
x = range(1, len(acc) + 1)
plt.figure(figsize=(12, 5))
plt.subplot(1, 2, 1)
plt.plot(x, acc, 'b', label='Training acc')
plt.plot(x, val_acc, 'r', label='Test acc')
plt.ylabel('Accuracy', fontsize = 14)
plt.xlabel('Epoch', fontsize = 14)
plt.title('Training and test accuracy')
plt.legend()
plt.subplot(1, 2, 2)
plt.plot(x, loss, 'b', label='Training loss')
plt.plot(x, val_loss, 'r', label='Test loss')
plt.ylabel('Loss', fontsize = 14)
plt.xlabel('Epoch', fontsize = 14)
plt.title('Training and test loss')
plt.legend()

# In[4]:

# training confusion matrix
y_pred = (np.round(model.predict(X_train))).astype(int)
test_cm = confusion_matrix(y_train, y_pred)
sn.heatmap(test_cm, annot=True, annot_kws={"size": 12}, cmap=plt.cm.Blues)
plt.title('Training confusion matrix')
plt.ylabel('True label')
plt.xlabel('Predicted label')
plt.show()

# test confusion matrix
y_pred = (np.round(model.predict(X_test))).astype(int)
test_cm = confusion_matrix(y_test, y_pred)
sn.heatmap(test_cm, annot=True, annot_kws={"size": 12}, cmap=plt.cm.Blues)
plt.title('Test confusion matrix')
plt.ylabel('True label')
plt.xlabel('Predicted label')
plt.show()


import tkinter as tk

import frames
from auto_sender import Auto_Sender
from data import Data

class Gui(tk.Tk):
    WIDTH: int = 1000
    HEIGHT: int = 420

    def __init__(self, *args, **kwargs) -> None:
        super().__init__(*args, **kwargs)

        # Data and Automation class
        self.data: Data = Data()
        self.auto_sender: Auto_Sender = Auto_Sender(self.data)

        # creating base window
        self.geometry(f"{Gui.WIDTH}x{Gui.HEIGHT}")
        self.title("WhatsApp Auto Sender")

        # creating frames
        self.current_frame: tk.Frame
        self.create_frames()

    def create_frames(self):
        self.frames: dict[frames.Names, tk.Frame] = dict()
        self.current_frame = self.frames[frames.Names.Main_Menu] = frames.Main_Menu(self, width= Gui.WIDTH, height= Gui.HEIGHT)
        self.frames[frames.Names.Settings] = frames.Settings(self)
        self.frames[frames.Names.Help] = frames.Help(self)

        self.current_frame.pack(expand= True, fill= tk.BOTH)

    def change_frame(self, frame_name: frames.Names):
        self.current_frame.pack_forget()
        self.current_frame = self.frames[frame_name]
        self.current_frame.pack(expand= True, fill= tk.BOTH)

    def run(self):
        self.mainloop()


import networkx as nx
import pickle
import matplotlib.pyplot as plt
from tpg.extensions import getRootTeamGraph
from tpg.extensions import getFullGraph

with open('saved-model-sgp.pkl', 'rb') as f:
    trainer = pickle.load(f)

#nodes, edges = getFullGraph(trainer)
nodes, edges = getRootTeamGraph(trainer.getBestAgent(tasks=['Assault-v0-18000']).team)

graph = nx.MultiDiGraph()
graph.add_nodes_from(nodes)
graph.add_edges_from(edges)

nodeColors = []
edgeColors = []
edgeWeights = []
labels = {}
for node in nodes:
    if node[0] == 'R':
        nodeColors.append('red')
        labels[node] = ''
    elif node[0] == 'T':
        nodeColors.append('lightcoral')
        labels[node] = ''
    else:
        nodeColors.append('yellow')
        labels[node] = node.split(':')[1]

for edge in edges:
    if edge[1][0] == 'T':
        edgeWeights.append(1.5)
    else:
        edgeWeights.append(1)
    edgeColors.append('grey')

pos = nx.spring_layout(graph, k=.5, iterations=20)
nx.draw(graph, pos, node_color=nodeColors,
        edge_color=edgeColors, width=edgeWeights)
nx.draw_networkx_labels(graph, pos, labels)
plt.show()


"""
In this module lives the type tree.
"""
from collections import defaultdict
from typing import Any, DefaultDict, Dict, List, Optional, Tuple, Type, Union

import dask.dataframe as dd
import numpy as np
import pandas as pd
from ..clean import validate_country, validate_lat_long
from ..errors import UnreachableError

CATEGORICAL_NUMPY_DTYPES = [bool, object]
CATEGORICAL_PANDAS_DTYPES = [pd.CategoricalDtype, pd.PeriodDtype]
CATEGORICAL_DTYPES = CATEGORICAL_NUMPY_DTYPES + CATEGORICAL_PANDAS_DTYPES

STRING_PANDAS_DTYPES = [pd.StringDtype]
STRING_DTYPES = STRING_PANDAS_DTYPES

NUMERICAL_NUMPY_DTYPES = [np.number]
NUMERICAL_DTYPES = NUMERICAL_NUMPY_DTYPES

DATETIME_NUMPY_DTYPES = [np.datetime64]
DATETIME_PANDAS_DTYPES = [pd.DatetimeTZDtype]
DATETIME_DTYPES = DATETIME_NUMPY_DTYPES + DATETIME_PANDAS_DTYPES

NULL_VALUES = {
    float("NaN"),
    "#N/A",
    "#N/A N/A",
    "#NA",
    "-1.#IND",
    "-1.#QNAN",
    "-NaN",
    "-nan",
    "1.#IND",
    "1.#QNAN",
    "<NA>",
    "N/A",
    "NA",
    "NULL",
    "NaN",
    "n/a",
    "nan",
    "null",
    "",
}

class DType:
    """
    Root of Type Tree
    """

############## Syntactic DTypes ##############
class Categorical(DType):
    """
    Type Categorical
    """

class Nominal(Categorical):
    """
    Type Nominal, Subtype of Categorical
    """

class Ordinal(Categorical):
    """
    Type Ordinal, Subtype of Categorical
    """

class Numerical(DType):
    """
    Type Numerical
    """

class Continuous(Numerical):
    """
    Type Continuous, Subtype of Numerical
    """

class Discrete(Numerical):
    """
    Type Discrete, Subtype of Numerical
    """

############## Semantic DTypes ##############

class DateTime(Numerical):
    """
    Type DateTime, Subtype of Numerical
    """

class Text(Nominal):
    """
    Type Text, Subtype of Nominal
    """

class GeoGraphy(Categorical):
    """
    Type GeoGraphy, Subtype of Categorical
    """

class GeoPoint(DType):
    """
    Type GeoPoint
    """

class LatLong(GeoPoint):
    """
    Type LatLong, Tuple
    """

    def __init__(self, lat_col: str, long_col: str) -> None:
        self.lat = lat_col
        self.long = long_col

############## End of the Type Tree ##############

DTypeOrStr = Union[DType, Type[DType], str, None]
DTypeDict = Union[Dict[str, Union[DType, Type[DType], str]], None]
DTypeDef = Union[Dict[str, Union[DType, Type[DType], str]], DType, Type[DType], None]

def detect_dtype(
    col: dd.Series,
    known_dtype: Optional[DTypeDef] = None,
    detect_small_distinct: bool = True,
) -> DType:
    """
    Given a column, detect its type or transform its type according to users' specification

    Parameters
    ----------
    col: dask.datafram.Series
        A dataframe column
    known_dtype: Optional[Union[Dict[str, Union[DType, str]], DType]], default None
        A dictionary or single DType given by users to specify the types for designated columns or
        all columns. E.g.  known_dtype = {"a": Continuous, "b": "Nominal"} or
        known_dtype = {"a": Continuous(), "b": "nominal"} or
        known_dtype = Continuous() or known_dtype = "Continuous" or known_dtype = Continuous()
    detect_small_distinct: bool, default True
        Whether to detect numerical columns with small distinct values as categorical column.
    """
    if not known_dtype:
        return detect_without_known(col, detect_small_distinct=detect_small_distinct)

    if isinstance(known_dtype, dict):
        if col.name in known_dtype:
            dtype = normalize_dtype(known_dtype[col.name])
            return map_dtype(dtype)

    elif isinstance(normalize_dtype(known_dtype), DType):
        return map_dtype(normalize_dtype(known_dtype))

    return detect_without_known(col, detect_small_distinct=detect_small_distinct)

def map_dtype(dtype: DType) -> DType:
    """
    Currently, we want to keep our Type System flattened.
    We will map Categorical() to Nominal() and Numerical() to Continuous()
    """
    if (
        isinstance(dtype, Categorical) is True
        and isinstance(dtype, Ordinal) is False
        and isinstance(dtype, Nominal) is False
    ):
        return Nominal()
    elif (
        isinstance(dtype, Numerical) is True
        and isinstance(dtype, Continuous) is False
        and isinstance(dtype, Discrete) is False
    ):
        return Continuous()
    else:
        return dtype

def detect_without_known(col: dd.Series, detect_small_distinct: bool) -> DType:
    # pylint: disable=too-many-return-statements
    """
    This function detects dtypes of column when users didn't specify.
    """
    if is_nominal(col.dtype):
        if is_geography(col):
            return GeoGraphy()
        if is_geopoint(col):
            return GeoPoint()
        else:
            return Nominal()

    elif is_continuous(col.dtype):
        if detect_small_distinct:
            # detect as categorical if distinct value is small
            nuniques = col.nunique_approx().compute()
            if nuniques < 10:
                return Nominal()
            else:
                return Continuous()
        else:
            return Continuous()

    elif is_datetime(col.dtype):
        return DateTime()
    else:
        raise UnreachableError

def is_dtype(dtype1: Any, dtype2: DType) -> bool:
    """
    This function detects if dtype2 is dtype1.
    """
    return isinstance(dtype1, dtype2.__class__)

def normalize_dtype(dtype_repr: Any) -> DType:
    """
    This function normalizes a dtype repr.
    """
    normalized: DType
    str_dic = {
        "Categorical": Categorical,
        "Ordinal": Ordinal,
        "Nominal": Nominal,
        "Numerical": Numerical,
        "Continuous": Continuous,
        "Discrete": Discrete,
        "DateTime": DateTime,
        "Text": Text,
    }
    for str_dtype, dtype in str_dic.items():
        if isinstance(dtype_repr, str):
            if dtype_repr.lower() == str_dtype.lower():
                normalized = dtype()
                break

        elif isinstance(dtype_repr, dtype):
            normalized = dtype_repr
            break

        elif dtype_repr == dtype:
            normalized = dtype()
            break

    return normalized

def is_nominal(dtype: Any) -> bool:
    """
    Given a type, return if that type is a nominal type
    """

    if is_continuous(dtype) or is_datetime(dtype):
        return False

    if isinstance(dtype, np.dtype):
        dtype = dtype.type

        return any(issubclass(dtype, c) for c in CATEGORICAL_NUMPY_DTYPES)
    else:
        return any(isinstance(dtype, c) for c in CATEGORICAL_PANDAS_DTYPES)

def is_geography(col: dd.Series) -> bool:
    """
    Given a column, return if its type is a geography type
    """
    geo = col.compute()[:100]
    geo_ratio: float = np.sum(validate_country(geo)) / geo.shape[0]
    return geo_ratio > 0.8

def is_geopoint(col: dd.Series) -> bool:
    """
    Given a column, return if its type is a geopoint type
    """
    lat_long = pd.Series(col.compute()[:100], dtype="string")
    lat_long_ratio: float = np.sum(validate_lat_long(lat_long)) / lat_long.shape[0]
    return lat_long_ratio > 0.8

def is_continuous(dtype: Any) -> bool:
    """
    Given a type, return if that type is a continuous type
    """
    dtype = dtype.type
    return any(issubclass(dtype, c) for c in NUMERICAL_NUMPY_DTYPES)

def is_datetime(dtype: Any) -> bool:
    """
    Given a type, return if that type is a datetime type
    """
    if isinstance(dtype, np.dtype):
        dtype = dtype.type
        return any(issubclass(dtype, c) for c in DATETIME_NUMPY_DTYPES)
    else:
        return any(isinstance(dtype, c) for c in DATETIME_PANDAS_DTYPES)

def is_pandas_categorical(dtype: Any) -> bool:
    """
    Detect if a dtype is categorical and from pandas.
    """
    return any(isinstance(dtype, c) for c in CATEGORICAL_PANDAS_DTYPES)

def drop_null(
    var: Union[dd.Series, pd.DataFrame, dd.DataFrame]
) -> Union[pd.Series, dd.Series, pd.DataFrame, dd.DataFrame]:
    """
    Drop the null values (specified in NULL_VALUES) from a series or DataFrame
    """

    if isinstance(var, (pd.Series, dd.Series)):
        if is_datetime(var.dtype):
            return var.dropna()
        return var[~var.isin(NULL_VALUES)]

    elif isinstance(var, (pd.DataFrame, dd.DataFrame)):
        df = var
        for values in df.columns:
            if is_datetime(df[values].dtype):
                df = df.dropna(subset=[values])
            else:
                df = df[~df[values].isin(NULL_VALUES)]
        return df

    raise ValueError("Input should be a Pandas/Dask Dataframe or Series")

def get_dtype_cnts_and_num_cols(
    df: dd.DataFrame,
    dtype: Union[Dict[str, Union[DType, Type[DType], str]], DType, Type[DType], None],
) -> Tuple[Dict[str, int], List[str]]:
    """
    Get the count of each dtype in a dataframe
    """
    dtype_cnts: DefaultDict[str, int] = defaultdict(int)
    num_cols: List[str] = []
    for col in df.columns:
        col_dtype = detect_dtype(df[col], dtype)
        if is_dtype(col_dtype, Nominal()):
            dtype_cnts["Categorical"] += 1
        elif is_dtype(col_dtype, Continuous()):
            dtype_cnts["Numerical"] += 1
            num_cols.append(col)
        elif is_dtype(col_dtype, DateTime()):
            dtype_cnts["DateTime"] += 1
        elif is_dtype(col_dtype, GeoGraphy()):
            dtype_cnts["GeoGraphy"] += 1
        elif is_dtype(col_dtype, GeoPoint()):
            dtype_cnts["GeoPoint"] += 1
        else:
            raise NotImplementedError
    return dtype_cnts, num_cols


import logging
import sys
import traceback

def debug_log(func):
    """Обеспечивает логирование функций используемых в программе"""
    def wrap(*args, **kwargs):
        """Функция-обертка"""
        logger_type = 'server' if 'server.py' in sys.argv[0] else 'client'
        logger = logging.getLogger(logger_type)
        func_name = f'Вызов функции {func.__name__}.'
        func_module = f'Из модуля {func.__module__}.'
        func_caller = f'Из из функции {traceback.format_stack()[0].strip().split()[-1]}.'
        func_params = f'Параметры: {args}, {kwargs}'
        logger.debug('%s %s %s %s' % (func_name, func_module, func_caller, func_params))
        result = func(*args, **kwargs)
        return result
    return wrap


'''
    This file contains a class that defines the api for retrieving bonds data from investing.com
'''

import investpy
import tkinter.messagebox as mb
import requests
import json

class BondsAPI:
    def __init__(self, bond=None, country=None, from_date=None, to_date=None):
        self.bond = bond
        self.country = country
        self.from_date = from_date
        self.from_to = to_date

    # get bonds countries
    def getCountries(self):
        countries = investpy.get_bond_countries()
        return countries

    # function to get historical data
    def getBondData(self):
        try:
            data = investpy.get_bond_historical_data(
                bond=self.bond, from_date=self.from_date, to_date=self.to_date, as_json=True)
            return data
        except (ValueError, FileNotFoundError, IOError, RuntimeError, ConnectionError):
            if ValueError:
                mb.showinfo(
                    'Error', 'Check parameters for required data. There may be an error or some values missing.')
            elif FileNotFoundError:
                mb.showinfo(
                    'Error', 'File containing required data not found.')
            elif IOError:
                mb.showinfo(
                    'Error', 'File containing required data not found.')
            elif RuntimeError:
                mb.showinfo(
                    'Error', 'Country or Stock does not match any existing data entries.')
            elif ConnectionError:
                mb.showinfo(
                    'Error', 'Could not establish connection to Investing.com')
            else:
                mb.showinfo('Error', 'Unknown error occured.')

    # function to get recent data
    def getBondRecentData(self):
        try:
            data = investpy.get_bond_recent_data(bond=self.bond, as_json=True)
            return data
        except (ValueError, FileNotFoundError, IOError, RuntimeError, ConnectionError):
            if ValueError:
                mb.showinfo(
                    'Error', 'Check parameters for required data. There may be an error or some values missing.')
            elif FileNotFoundError:
                mb.showinfo(
                    'Error', 'File containing required data not found.')
            elif IOError:
                mb.showinfo(
                    'Error', 'File containing required data not found.')
            elif RuntimeError:
                mb.showinfo(
                    'Error', 'Country or Stock does not match any existing data entries.')
            elif ConnectionError:
                mb.showinfo(
                    'Error', 'Could not establish connection to Investing.com')
            else:
                mb.showinfo('Error', 'Unknown error occured.')

    # function bonds dict
    # return dict
    def getBondsDict(self, country=None, columns=None):
        try:
            bonds = investpy.get_bonds_dict(country=country, columns=columns)
            return bonds
        except(ValueError, FileNotFoundError, IOError, RuntimeError):
            if ValueError:
                mb.showinfo(
                    'Error', 'Check parameters for required data. There may be an error or some values missing.')
            elif FileNotFoundError:
                mb.showinfo(
                    'Error', 'File containing required data not found.')
            elif IOError:
                mb.showinfo(
                    'Error', 'File containing required data not found.')
            elif RuntimeError:
                mb.showinfo(
                    'Error', 'Country or Stock does not match any existing data entries.')

    # defining setters
    def setBond(self, bond):
        self.bond = bond

    def setCountry(self, country):
        self.country = country

    def setFromDate(self, from_date):
        self.from_date = from_date

    def setToDate(self, to_date):
        self.to_date = to_date

    # defining getters
    def getName(self):
        return self.bond

    def getCountry(self):
        if self.country == None:
            return ""
        else:
            return self.country

    def getFromDate(self):
        return self.from_date

    def getToDate(self):
        return self.to_date


import json
from boto3.dynamodb.conditions import Key

from lib import ncr_util, authz
from lib.logger import logger
from lib.lambda_decorator.decorator import api_decorator
from lib.lambda_decorator.scan_id_decorator import get_scan_id_decorator
from lib.lambda_decorator.email_decorator import email_decorator
from lib.dynamodb import ncr_table, requirements_table

@api_decorator
@email_decorator
@get_scan_id_decorator
def ncr_handler(event, context):
    scan_id = event['scanId']
    user = event['userRecord']

    multivalue_querystring_parameters = event.get('multiValueQueryStringParameters') or {}
    querystring_parameters = event.get('queryStringParameters') or {}
    account_ids = multivalue_querystring_parameters.get('accountId', [])
    requirement_id = querystring_parameters.get('requirementId', False)
    logger.debug('Account Ids: %s', json.dumps(account_ids, default=str))
    logger.debug('Requirement ID: %s', json.dumps(requirement_id, default=str))
    authz.require_can_read_account(user, account_ids)

    # get requirements
    if requirement_id:
        requirements = {
            requirement_id: requirements_table.get(requirement_id)
        }
    else:
        all_requirements = requirements_table.scan_all()
        requirements = {}
        for requirement in all_requirements:
            requirements[requirement['requirementId']] = requirement
    logger.debug('Requirements: %s', json.dumps(requirements, default=str))

    ncr_records, to_parse = [], []
    for account_id in account_ids:
        if isinstance(requirement_id, str):
            to_parse = ncr_table.query_all(
                IndexName='by-scanId',
                KeyConditionExpression=Key('scanId').eq(scan_id) &
                Key('rqrmntId_accntId').eq(
                    '{}#{}'.format(requirement_id, account_id)
                )
            )
        else:
            to_parse = ncr_table.query_all(
                KeyConditionExpression=Key('scanId').eq(scan_id) &
                Key('accntId_rsrceId_rqrmntId').begins_with(account_id)
            )
        logger.debug('To parse: %s', json.dumps(to_parse, default=str))

        for item in to_parse:
            ncr = prepare_allowed_actions_output(initialize_output(scan_id, item), item, user, account_id, requirements[item['requirementId']])
            ncr_records.append(prepare_resource_output(ncr, item))

    return {'ncrRecords': ncr_records}

##########
##-PREP-##
##########
def initialize_output(scan_id, resource):
    """
    Method to create the initial dict based on scan_id and resource.

    :param scan_id: String representing the scan id for this run.
    :param resource: a dict representing the resource to configure the id from.

    :returns dict: A dict with the ncrId only combining the scan_id and ncr unique identifier.
    """
    return {'ncrId': ncr_table.create_ncr_id(resource)}

def prepare_allowed_actions_output(output, resource, user, account, requirement):
    """
    Method to build the allowedActions section of the output.

    :param output: The output to append the allowedActions to.
    :param resource: a dict representing the resource to configure the id from.
    :param user: a dict representing the user which the dict is being built for.
    :param account: a string representing the account.
    :param requirement: a dict representing the requirement.

    :returns dict: A dict combining the output parameter passed in and the generated allowedActions dict.
    """
    output['allowedActions'] = ncr_util.get_allowed_actions(user, account, requirement, resource.get('exclusion', {}))
    return output

def prepare_resource_output(output, resource):
    """
    Method to build the resource section of the output.

    :param output: The output to append the resource to.
    :param resource: a dict representing the resource to configure the id from.

    :returns dict: A dict combining the output parameter passed in and the generated resource dict.
    """
    new_resource = dict(resource)
    if 'scanId' in new_resource:
        del new_resource['scanId']
    if 'accntId_rsrceId_rqrmntId' in new_resource:
        del new_resource['accntId_rsrceId_rqrmntId']
    if 'rqrmntId_accntId' in new_resource:
        del new_resource['rqrmntId_accntId']

    output['resource'] = new_resource
    return output


from constructs import Construct
import os
import aws_cdk.aws_s3 as s3
import aws_cdk.aws_stepfunctions as sfn
import aws_cdk.aws_lambda_event_sources as eventsources
from aws_cdk import (CfnOutput, RemovalPolicy, Stack, Duration)
import amazon_textract_idp_cdk_constructs as tcdk

class AnalyzeExpenseStack(Stack):

    def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None:
        super().__init__(
            scope,
            construct_id,
            description=
            "IDP CDK constructs sample for Textract AnalyzeExpense (SO9217)",
            **kwargs)

        s3_expense_prefix = "expense-uploads"
        s3_output_prefix = "textract-output"

        # BEWARE! This is a demo/POC setup, remote the auto_delete_objects=True and
        document_bucket = s3.Bucket(
            self,
            "SchademCdkIdpStackPaystubW2",
            auto_delete_objects=True,
            removal_policy=RemovalPolicy.DESTROY,
            block_public_access=s3.BlockPublicAccess.BLOCK_ALL)

        # get the event source that will be used later to trigger the executions
        s3_event_source = eventsources.S3EventSource(
            document_bucket,
            events=[s3.EventType.OBJECT_CREATED],
            filters=[s3.NotificationKeyFilter(prefix=s3_expense_prefix)])

        # EXPENSE
        decider_task_expense = tcdk.TextractPOCDecider(
            self,
            f"ExpenseDecider",
        )

        textract_sync_task_expense = tcdk.TextractGenericSyncSfnTask(
            self,
            "TextractSyncExpense",
            s3_output_bucket=document_bucket.bucket_name,
            s3_output_prefix=s3_output_prefix,
            textract_api="EXPENSE",
            integration_pattern=sfn.IntegrationPattern.WAIT_FOR_TASK_TOKEN,
            lambda_log_level="DEBUG",
            timeout=Duration.hours(24),
            input=sfn.TaskInput.from_object({
                "Token":
                sfn.JsonPath.task_token,
                "ExecutionId":
                sfn.JsonPath.string_at('$$.Execution.Id'),
                "Payload":
                sfn.JsonPath.entire_payload,
            }),
            result_path="$.textract_result")

        workflow_chain_expense = sfn.Chain \
            .start(decider_task_expense) \
            .next(textract_sync_task_expense)

        state_machine = sfn.StateMachine(self,
                                         f'Expense',
                                         definition=workflow_chain_expense)

        # The StartThrottle triggers based on event_source (in this case S3 OBJECT_CREATED)
        # and handles all the complexity of making sure the limits or bottlenecks are not exceeded
        tcdk.SFExecutionsStartThrottle(
            self,
            "ExecutionThrottle",
            state_machine_arn=state_machine.state_machine_arn,
            executions_concurrency_threshold=550,
            sqs_batch=10,
            lambda_log_level="ERROR",
            event_source=[s3_event_source])

        # OUTPUT
        CfnOutput(
            self,
            "ExpenseDocumentUploadLocation",
            value=f"s3://{document_bucket.bucket_name}/{s3_expense_prefix}/")
        current_region = Stack.of(self).region
        CfnOutput(
            self,
            'StepFunctionFlowLink',
            value=
            f"https://{current_region}.console.aws.amazon.com/states/home?region={current_region}#/statemachines/view/{state_machine.state_machine_arn}"
        )


from ast import List
from fastapi import APIRouter, HTTPException, status
from fastapi.encoders import jsonable_encoder
from config.database import Session
from fastapi.responses import JSONResponse
from schemas.director import Director
from service.director import DirectorService
from models.director import Director as DirectorModels

director_router = APIRouter()

@director_router.get('/director', tags=['director'], status_code=status.HTTP_200_OK)
def get_director() -> Director:
    db = Session()
    result = DirectorService(db).get_director()
    if not result:
        raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=f"No hay directores registrados")
    return JSONResponse(content=jsonable_encoder(result), status_code=200)

@director_router.get('/director_first_name', tags=['director'], status_code=status.HTTP_200_OK)
def get_director_first_name(first_name: str):
    db = Session()
    result = DirectorService(db).get_director_first_name(first_name)
    if not result:
        raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=f"No registran directores con el nombre "+ str(first_name))
    return JSONResponse(content=jsonable_encoder(result), status_code=200)

@director_router.get('/director_for_id', tags=['director'], status_code=status.HTTP_200_OK)
def get_movies_for_id(id: int):
    db = Session()
    result = DirectorService(db).get_movies_for_id(id)
    if not result:
        raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=f"No registran directores con el id "+ str(id))
    return JSONResponse(content=jsonable_encoder(result), status_code=status.HTTP_200_OK)

@director_router.get('/director_for_last_name', tags=['director'], status_code=status.HTTP_200_OK)
def get_movies_for_last_name(last_name: str):
    db = Session()
    result = DirectorService(db).get_movies_for_last_name(last_name)
    if not result:
        raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=f"No registran directores con el apellido "+ str(last_name))
    return JSONResponse(content=jsonable_encoder(result), status_code=status.HTTP_200_OK)

@director_router.post('/director', tags=['director'], status_code=200, response_model=dict)
def create_director(director:Director):
    db = Session()
    DirectorService(db).create_director(director)
    return JSONResponse(status_code=status.HTTP_200_OK,content={"message": "Se ha registrado el director"})

@director_router.put('/director{id}',tags=['director'])
def update_director(id:int,director:Director):
    db =  Session()
    result: DirectorModels = db.query(DirectorModels).filter(DirectorModels.dir_id == id)
    final = result.first()
    if not final:
        raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=f"El id "+ str(id)+" no existe")
    DirectorService(db).update_director(id,director)
    return JSONResponse(status_code=status.HTTP_200_OK,content={"message":"Se ha modificado el director con id:"+ str(id)})

@director_router.delete('/director/{id}', tags=['director'], status_code=200)
def delete_director(id: int):
    db = Session()
    result: DirectorModels = db.query(DirectorModels).filter(DirectorModels.dir_id == id)
    final = result.first()
    if not final:
        raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=f"El id "+ str(id)+" no existe")
    DirectorService(db).delete_director(id)
    return JSONResponse(status_code=status.HTTP_200_OK, content={"message": "Se ha eliminado el director con id:"+ str(id)})


#!/usr/bin/env python
# -*- coding: UTF-8 -*-
'''
@Project :Awesome-Uplift-Model
@File    :example_2.py
@Author  :JackHCC
@Date    :2022/12/3 17:47
@Desc    :

'''
import pandas as pd
import numpy as np
import statsmodels.formula.api as smf
import matplotlib.pyplot as plt

np.random.seed(10)

# Ten groups with ten periods each
id = pd.DataFrame({'id': range(0, 10), 'key': 1})
t = pd.DataFrame({'t': range(1, 11), 'key': 1})
d = id.merge(t, on='key')
# Add an event in period 6 with a one-period effect
d['Y'] = np.random.normal(0, 1, 100) + 1 * (d['t'] == 6)

# Estimate our model using time 5 as reference
m = smf.ols('Y~C(t, Treatment(reference = 5))', data=d)

# Fit with SEs clustered at the group level
m = m.fit(cov_type='cluster', cov_kwds={'groups': d['id']})

# Get coefficients and CIs
# The original table will have an intercept up top
# But we'll overwrite it with our 5 reference
p = pd.DataFrame({'t': [5, 1, 2, 3, 4, 6, 7, 8, 9, 10],
                  'b': m.params, 'se': m.bse})
# And add our period-5 zero
p.iloc[0] = [5, 0, 0]

# Sort for plotting
p = p.sort_values('t')
# and make CIs by scaling the standard error
p['ci'] = 1.96 * p['se']

# Plot the estimates as connected lines with error bars
plt.errorbar(x='t', y='b',
             yerr='ci', data=p)
# Add a horizontal line at 0
plt.axhline(0, linestyle='dashed')
# And a vertical line at the treatment time
plt.axvline(5)
plt.show()


import configparser
import csv
from essearcher import ESSearcher
from solrsearcher import SolrSearcher

class Evaluator:
    query_to_positions = {}
    fields = ["title", "tags", "entities", "website", "type", "content_main", "content_additional"]
    weights_to_score = {}
    searcher = None
    statistics = None

    def __init__(self):
        with open('queries.csv', encoding='utf-8') as csvfile:
            model_reader = csv.reader(csvfile, delimiter=',')
            for row in model_reader:
                results = row[1].split(sep=":")
                self.query_to_positions[row[0]] = Evaluator.Expected(doc_id=results[0],
                                                                     expected=results[1][3:],
                                                                     adequate=results[2][3:])
        self.statistics = Evaluator.Statistics()
        config = configparser.ConfigParser()
        config.read('properties.ini')
        searcher = config['DEFAULT']['searcher']
        if searcher == 'ES':
            self.searcher = ESSearcher()
        elif searcher == 'Solr':
            self.searcher = SolrSearcher()
        else:
            raise ValueError('Illegal searcher value')

    def quality(self, weights=None, verbose=None):
        if verbose:
            print(str(self.statistics))
        self.statistics.total += 1

        cache_key = self.key(weights)
        if cache_key in self.weights_to_score:
            self.statistics.cache_retrievals += 1
            return self.weights_to_score[cache_key]
        else:
            self.statistics.cached_values += 1

        score = 0

        for key, value in self.query_to_positions.items():
            results_ids = self.searcher.search(key, self.fields, weights)

            if len(results_ids) == 0:
                continue

            if value.doc_id in results_ids[:value.expected_in_top]:
                score += 1
            elif value.doc_id in results_ids[:value.adequate_in_top]:
                score += 0.5
            else:
                continue

        quality = round(score / len(self.query_to_positions) * 100, 1)

        if verbose:
            print("Search quality is [%(q)d%%] for %(w)s:" % {'q': quality, 'w': weights})

        self.weights_to_score[cache_key] = quality

        return quality

    @staticmethod
    def key(weights):
        return tuple(weights)

    def getCacheLength(self):
        return len(self.weights_to_score)

    class Statistics:
        total = 0
        cache_retrievals = 0
        cached_values = 0

        def __str__(self) -> str:
            return "Total evaluations: " + str(self.total) + ", " + \
                   "cached values: " + str(self.cached_values) + \
                   ", retrievals from cache: " + str(self.cache_retrievals) + "."

    class Expected:
        doc_id = -1
        expected_in_top = -1
        adequate_in_top = -1

        def __init__(self, doc_id, expected, adequate):
            self.doc_id = doc_id
            self.expected_in_top = int(expected)
            self.adequate_in_top = int(adequate)

        def __str__(self):
            return "id: " + self.doc_id + ", expected in top: " + str(self.expected_in_top) + \
                   ", adequate in top: " + str(self.adequate_in_top)

        def __repr__(self):
            return self.__str__()

def main():
    evaluator = Evaluator()
    evaluator.quality([5, 3, 1, 2, 2, 1, 0.5])

if __name__ == '__main__':
    main()


import logging
import time
import os
import sys

# 日志管理
logger = logging.getLogger('logger')
formatter = logging.Formatter('%(asctime)s %(levelname)-8s: %(message)s \n')
try:
    file_handler = logging.FileHandler(
        'logs/{}.log'.format(time.strftime('%Y-%m-%d_%H:%M:%S', time.localtime(time.time()))))
except:
    os.mkdir('logs')
    file_handler = logging.FileHandler(
        'logs/{}.log'.format(time.strftime('%Y-%m-%d_%H:%M:%S', time.localtime(time.time()))))
file_handler.setFormatter(formatter)

console_handler = logging.StreamHandler(sys.stdout)
console_handler.formatter = formatter

logger.addHandler(file_handler)
logger.addHandler(console_handler)
logger.setLevel(logging.INFO)


import time
from datetime import datetime
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import json
from os import path
import re as regex
from .Xpaths import Xpaths

if path.dirname(__file__) == 'main': relativePath = path.join(path.dirname(__file__), '../../../../archives/')
else: relativePath = path.join(path.dirname(__file__), '../../archives/')

class CovidTracker:
    def __init__(self, headless, source = 'https://www.worldometers.info/coronavirus/'):
        self.option = Options()
        self.option.headless = True if headless else False
        self.driver = webdriver.Chrome(executable_path=r'./chromedriver.exe', chrome_options=self.option)
        self.source = source

    def goTo(self, newURL):
        try:
            self.driver.get(newURL)
            _ = WebDriverWait(self.driver, 10).until(EC.presence_of_element_located((By.TAG_NAME, "body")))
        except Exception:
            raise Exception('The page did not load')

    def getOffline(self):
        self.driver.quit()

    def getInfo(self):
        if regex.search('who', self.source, regex.IGNORECASE): entireObject = self.getWhoInfo()
        else: entireObject = self.getOmetersInfo()
        return entireObject

    def getOmetersInfo(self):
        finalObject = {}
        countriesURL = []

        totalCases = self.driver.find_element_by_xpath(Xpaths.ometersInfo['totalCasesXpath']).get_attribute('innerHTML')
        recoveredCases = self.driver.find_element_by_xpath(Xpaths.ometersInfo['recoveredCasesXpath']).get_attribute('innerHTML')
        deaths = self.driver.find_element_by_xpath(Xpaths.ometersInfo['deathsXpath']).get_attribute('innerHTML')

        mainInfos = {
            'totalCases': totalCases.strip(),
            'recoveredCases': recoveredCases.strip(),
            'deaths': deaths.strip()
        }
        finalObject.update(mainInfos)

        for countries in range(5):
            countriesURL.append(self.driver.find_element_by_xpath(Xpaths.ometersInfo[f'{countries+1}Xpath']).get_attribute('href'))

        for countries in range(5):
            self.goTo(countriesURL[countries])
            countryName = self.driver.find_element_by_xpath(Xpaths.ometersInfo['countryNameXpath']).get_attribute('innerHTML')
            countryName = countryName[121:len(countryName)].strip()
            totalCases = self.driver.find_element_by_xpath(Xpaths.ometersInfo['totalCasesXpath']).get_attribute('innerHTML')
            recoveredCases = self.driver.find_element_by_xpath(Xpaths.ometersInfo['recoveredCasesXpath']).get_attribute('innerHTML')
            deaths = self.driver.find_element_by_xpath(Xpaths.ometersInfo['deathsXpath']).get_attribute('innerHTML')

            unitInfos = {
                f'{countryName}CountryName': countryName,
                f'{countryName}TotalCases': totalCases.strip(),
                f'{countryName}RecoveredCases': recoveredCases.strip(),
                f'{countryName}Deaths': deaths.strip()
            }
            finalObject.update(unitInfos)

        return finalObject

    def getWhoInfo(self):
        countryName = self.driver.find_element_by_xpath(Xpaths.whoInfo['countryNameXpath']).get_attribute('innerHTML')
        newCases = self.driver.find_element_by_xpath(Xpaths.whoInfo['newCasesXpath']).get_attribute('innerHTML')
        confirmedCases = self.driver.find_element_by_xpath(Xpaths.whoInfo['confirmedCasesXpath']).get_attribute('innerHTML')
        deaths = self.driver.find_element_by_xpath(Xpaths.whoInfo['deathsXpath']).get_attribute('innerHTML')

        mainInfos = {
            'countryName': countryName.strip(),
            'newCases': newCases.strip(),
            'confirmedCases': confirmedCases.strip(),
            'deaths': deaths.strip()
        }

        return mainInfos

    def generateText(self, countryInfo, standard = True):
        dividedInfo = []
        counter = 0
        date = datetime.now()
        if standard:
            infosTxt = f'Para o dia {date.day}/{date.month}/{date.year},\n\no status de COVID-19 para: Mundo\n\nCasos Totais: {countryInfo["totalCases"]}\nCasos Recuperados: {countryInfo["recoveredCases"]}\nMortos: {countryInfo["deaths"]}'
            countryInfo.pop('totalCases')
            countryInfo.pop('recoveredCases')
            countryInfo.pop('deaths')
            for item in countryInfo.values():
                dividedInfo.append(item)
                counter += 1
                if counter >= 4:
                    infosTxt += f'\n\n\nO status de COVID-19 para: {dividedInfo[0]}\n\nCasos Totais: {dividedInfo[1]}\nCasos Recuperados: {dividedInfo[2]}\nMortos: {dividedInfo[3]}'
                    counter = 0
                    dividedInfo.clear()
        else:
            for item in countryInfo.values(): dividedInfo.append(item)
            infosTxt = f'Para o dia {date.day}/{date.month}/{date.year},\n\no status de COVID-19 para: {dividedInfo[0]}\n\nNovos Casos: {dividedInfo[1]}\nCasos Confirmados: {dividedInfo[2]}\nMortos: {dividedInfo[3]}'
        return infosTxt

    def handleInfos(self, infos):
        infosJson = json.dumps(infos)
        if regex.search('who', self.source, regex.IGNORECASE):
            text = self.generateText(infos, standard = False)
            with open(relativePath + f'{infos["countryName"]}Stats.json', 'w+') as stats:
                stats.writelines(infosJson)
            with open(relativePath + f'{infos["countryName"]}Stats.txt', 'w+') as stats:
                stats.writelines(text)
        else:
            text = self.generateText(infos)
            with open(relativePath + 'GeneralStats.json', 'w+') as stats:
                stats.writelines(infosJson)
            with open(relativePath + 'GeneralStats.txt', 'w+') as stats:
                stats.writelines(text)

    def fetch(self):
        self.goTo(self.source)
        self.handleInfos(self.getInfo())
        self.getOffline()


#!/usr/bin/env python3

# Developer : Hamdy Abou El Anein
# hamdy.aea@protonmail.com

# This Python3 software print in the terminal the next rockets launch. The data is updated at every run.

from urllib.request import Request, urlopen
import json

def main():
    req = Request("https://astrometry.ch/rockets.json", headers={'User-Agent': 'Mozilla/5.0'})
    data = urlopen(req,timeout=10).read()
    Rocketlaunch = json.loads(data.decode())
    for i in Rocketlaunch["results"]:
        try:
            spacecraft = i["rocket"]["configuration"]["full_name"]
        except:
            spacecraft = ""
        try:
            type = i["launch_service_provider"]["type"]
        except:
            type = ""
        try:
            time = i["net"]
        except:
            time = ""
        try:
            company = i["launch_service_provider"]["name"]
        except:
            company = ""
        try:
            pic = i["image"]
        except:
            pic = ""
        try:
            mission1 = i["mission"]["name"]
        except:
            mission1 = ""
        try:
            description = i["mission"]["description"]
        except:
            description = ""
        try:
            type = i["mission"]["type"]
        except:
            type = ""
        try:
            orbit = i["mission"]["orbit"]["name"]
        except:
            orbit = ""
        print(spacecraft)
        print(type)
        print(time)
        print(company)
        print(pic)
        print(mission1)
        print(description)
        print(type)
        print(orbit)
        print("\n")

    num = Rocketlaunch["count"]
    print("Number of Rockets")
    print(num)

main()


from common.views import index, CreateUserProfile
from django.urls import path
from allauth.account.views import login, logout, signup

app_name = 'common'
urlpatterns = [
    path('', index, name='index'),
    path('login/', login, name='login'),
    path('logout/', logout, name='logout'),
    path('register/', signup, name='register'),
    path('profile-create/', CreateUserProfile.as_view(), name='profile-create'),
]


from django.conf.urls.defaults import *
from django.views.generic.simple import direct_to_template

from registration.views import register

urlpatterns = patterns('',
    url(r'^register/$', register,
        {'backend': 'editkit.users.registration_backend.SaplingBackend'},
        name='registration_register'),
    url(r'^register/closed/$', direct_to_template,
        {'template': 'registration/registration_closed.html'},
        name='registration_disallowed'),
    (r'', include('registration.auth_urls')),
)


"""
In the previous exercise we created a dictionary of famous scientists’ birthdays. In this
exercise, modify your program from Part 1 to load the birthday dictionary from a JSON file on
disk, rather than having the dictionary defined in the program.
"""

import json

def main():

	with open("birthdays.json", "r") as f:
		birthdays = json.load(f)
	print(birthdays)

if __name__ == '__main__':
	main()


import torch
from torch.utils.data import DataLoader, random_split
import pytorch_lightning as pl

from transformers import BertTokenizerFast, EncoderDecoderModel, BertForQuestionAnswering, BertTokenizer, BartForConditionalGeneration

from dataset import QADataset

default = {'dataset': './PTT_QA.csv', 'train_size': 0.2, 'model_name': 'ckiplab/bert-base-chinese', 'freeze_bert': False,
           'num_workers': 4, 'batch_size': 16, 'lr': 1e-4, 'dropout': 0.2, 'seq_len': 64}

class chatBot(pl.LightningModule):
    def __init__(self, config=default):
        super(chatBot, self).__init__()
        self.config = config
        self.seq_len = self.config['seq_len']
        model = self.config['model_name']
        tokenizer = BertTokenizerFast.from_pretrained(model)
        tokenization = lambda s: tokenizer(
                        list(s),
                        add_special_tokens = True, # Add '[CLS]' and '[SEP]'
                        max_length = self.seq_len,
                        truncation = True,
                        padding = 'max_length',
                        return_attention_mask = True,
                        return_tensors = 'pt').to(self.config['device'])
        self.map_f = lambda Q, A: (tokenization(Q), tokenization(A))
        self.transformer = BertForQuestionAnswering.from_pretrained(model).to(self.config['device'])
        # self.attention = nn.MultiheadAttention(embed_dim=len(self.tokenizer), num_heads=1, dropout=self.config['dropout'], batch_first=True)
        if self.config['freeze_bert']:
            for param in self.transformer.bert.parameters():
                param.requires_grad = False

    def forward(self, x):
        Q, A = self.map_f(x['Q'], x['A'])
        output = self.transformer(input_ids=Q['input_ids'], attention_mask=Q['attention_mask'], labels=A['input_ids'])
        del Q, A
        loss, logits = output.loss, output.logits
        # output contains many attributes, use encoder_last_hidden_state or encoder_hidden_state to get encoder output
        return {'seq_loss': loss, 'logits': logits}

    def setup(self, stage=None):
        """
        setup dataset for each machine
        """
        dataset = QADataset(self.config['dataset'])
        dataset_length = len(dataset)
        train_length = int(self.config['train_size'] * dataset_length)
        self.train_dataset, self.val_dataset = random_split(dataset, [train_length, dataset_length - train_length])
        del dataset

    def train_dataloader(self):
        return DataLoader(self.train_dataset,
                          shuffle=True,
                          num_workers=self.config['num_workers'],
                          batch_size=self.config['batch_size'],
                          pin_memory=True)

    def val_dataloader(self):
        return DataLoader(self.val_dataset,
                          shuffle=False,
                          num_workers=self.config['num_workers'],
                          batch_size=self.config['batch_size'],
                          pin_memory=True)

    def configure_optimizers(self):
        optimizer = torch.optim.AdamW(filter(lambda p: p.requires_grad, self.parameters()), lr=self.config['lr'])
        return optimizer

    def training_step(self, batch, batch_idx):
        outputs = self(batch)

        loss = outputs['seq_loss']
        self.log('train_loss', loss)

        return loss

    def validation_step(self, batch, batch_idx):
        outputs = self(batch)

        loss = outputs['seq_loss']
        log = {'val_loss': loss}

        return log

    def validation_epoch_end(self, outputs):
        mean_loss = torch.stack([x['val_loss'] for x in outputs]).mean()

        self.log('val/loss', mean_loss, prog_bar=True)
        mean_loss = torch.stack([x['val_loss'] for x in outputs]).mean()

        self.log('val/loss', mean_loss, prog_bar=True)


from enum import Enum

class IntervalKucoin(Enum):
    MIN1 = "1"
    MIN5 = "5"
    MIN15 = "15"
    H1 = "60"
    H2 = "120"
    H4 = "240"
    H8 = "480"
    H12 = "720"
    D1 = "1440"
    W1 = "10080"

class IntervalBinance(Enum):
    MIN1 = "1m"
    MIN5 = "5m"
    MIN15 = "15m"
    H1 = "1h"
    H2 = "2h"
    H4 = "4h"
    H8 = "8h"
    H12 = "12h"
    D1 = "1d"
    W1 = "1w"

class Stock(Enum):
    BINANCE = 'Binance'
    KUCOIN = 'Kucoin'

interval_dict_binance = {
    "1m": 1,
    "5m": 5,
    "15m": 15,
    "1h": 60,
    "2h": 120,
    "4h": 240,
    "8h": 480,
    "12h": 720,
    "1d": 1440,
    "1w": 10080
}

# Kucoin API symbols to understandable dev-symbols
symbols_dict_kucoin = {
    "BTC": "XBTUSDTM",
    "ETH": "ETHUSDTM",
    "BCH": "BCHUSDTM",
    "BSV": "BSVUSDTM",
    "LINK": "LINKUSDTM",
    "UNI": "UNIUSDTM",
    "YFI": "YFIUSDTM",
    "EOS": "EOSUSDTM",
    "DOT": "DOTUSDTM",
    "FIL": "FILUSDTM",
    "ADA": "ADAUSDTM",
    "XRP": "XRPUSDTM",
    "LTC": "LTCUSDTM",
    "TRX": "TRXUSDTM",
    "GRT": "GRTUSDTM",
    "SUSHI": "SUSHIUSDTM",
    "XLM": "XLMUSDTM",
    "1INCH": "1INCHUSDTM",
    "ZEC": "ZECUSDTM",
    "DASH": "DASHUSDTM",
    "AAVE": "AAVEUSDTM",
    "KSM": "KSMUSDTM",
    "DOGE": "DOGEUSDTM",
    "VET": "VETUSDTM",
    "BNB": "BNBUSDTM",
    "SXP": "SXPUSDTM",
    "SOL": "SOLUSDTM",
    "IOST": "IOSTUSDTM",
    "CRV": "CRVUSDTM",
    "ALGO": "ALGOUSDTM",
    "AVAX": "AVAXUSDTM",
    "FTM": "FTMUSDTM",
    "MATIC": "MATICUSDTM",
    "THETA": "THETAUSDTM",
    "ATOM": "ATOMUSDTM",
    "CHZ": "CHZUSDTM",
    "ENJ": "ENJUSDTM",
    "MANA": "MANAUSDTM",
    "DENT": "DENTUSDTM",
    "OCEAN": "OCEANUSDTM",
    "BAT": "BATUSDTM",
    "XEM": "XEMUSDTM",
    "QTUM": "QTUMUSDTM",
    "XTZ": "XTZUSDTM",
    "SNX": "SNXUSDTM",
    "NEO": "NEOUSDTM",
    "ONT": "ONTUSDTM",
    "XMR": "XMRUSDTM",
    "COMP": "COMPUSDTM",
    "ETC": "ETCUSDTM",
    "WAVES": "WAVESUSDTM",
    "BAND": "BANDUSDTM",
    "MKR": "MKRUSDTM",
    "RVN": "RVNUSDTM",
    "DGB": "DGBUSDTM",
    "SHIB": "SHIBUSDTM",
    "ICP": "ICPUSDTM",
    "DYDX": "DYDXUSDTM",
    "AXS": "AXSUSDTM",
    "HBAR": "HBARUSDTM",
    "EGLD": "EGLDUSDTM",
    "ALICE": "ALICEUSDTM",
    "YGG": "YGGUSDTM",
    "NEAR": "NEARUSDTM",
    "SAND": "SANDUSDTM",
    "C98": "C98USDTM",
    "ONE": "ONEUSDTM",
    "VRA": "VRAUSDTM",
    "GALA": "GALAUSDTM",
    "TLM": "TLMUSDTM",
    "CHR": "CHRUSDTM",
    "LRC": "LRCUSDTM",
    "FLOW": "FLOWUSDTM",
    "RNDR": "RNDRUSDTM",
    "IOTX": "IOTXUSDTM",
    "CRO": "CROUSDTM",
    "WAXP": "WAXPUSDTM",
    "PEOPLE": "PEOPLEUSDTM",
    "OMG": "OMGUSDTM",
    "LINA": "LINAUSDTM",
    "IMX": "IMXUSDTM",
    "NFT": "NFTUSDTM",
    "CELR": "CELRUSDTM",
    "ENS": "ENSUSDTM",
    "CELO": "CELOUSDTM",
    "CTSI": "CTSIUSDTM",
    "SLP": "SLPUSDTM",
    "ARPA": "ARPAUSDTM",
    "KNC": "KNCUSDTM",
    "API3": "API3USDTM",
    "ROSE": "ROSEUSDTM",
    "AGLD": "AGLDUSDTM",
    "APE": "APEUSDTM",
    "JASMY": "JASMYUSDTM",
    "ZIL": "ZILUSDTM",
    "GMT": "GMTUSDTM",
    "RUNE": "RUNEUSDTM",
    "LOOKS": "LOOKSUSDTM",
    "AUDIO": "AUDIOUSDTM",
    "KDA": "KDAUSDTM",
    "KAVA": "KAVAUSDTM",
    "BAL": "BALUSDTM",
    "GAL": "GALUSDTM",
    "LUNA": "LUNAUSDTM",
    "LUNC": "LUNCUSDTM",
    "OP": "OPUSDTM",
    "XCN": "XCNUSDTM",
    "UNFI": "UNFIUSDTM",
    "LIT": "LITUSDTM",
    "DUSK": "DUSKUSDTM",
    "STORJ": "STORJUSDTM",
    "RSR": "RSRUSDTM",
    "JST": "JSTUSDTM",
    "OGN": "OGNUSDTM",
    "TRB": "TRBUSDTM",
    "PERP": "PERPUSDTM",
    "KLAY": "KLAYUSDTM",
    "ANKR": "ANKRUSDTM",
    "LDO": "LDOUSDTM",
    "WOO": "WOOUSDTM",
    "REN": "RENUSDTM",
    "CVC": "CVCUSDTM",
    "INJ": "INJUSDTM",
    "APT": "APTUSDTM",
    "MASK": "MASKUSDTM",
    "REEF": "REEFUSDTM",
    "TON": "TONUSDTM",
    "MAGIC": "MAGICUSDTM",
    "CFX": "CFXUSDTM",
    "AGIX": "AGIXUSDTM",
    "FXS": "FXSUSDTM",
    "FET": "FETUSDTM",
    "AR": "ARUSDTM",
    "GMX": "GMXUSDTM",
    "BLUR": "BLURUSDTM",
    "ASTR": "ASTRUSDTM",
    "HIGH": "HIGHUSDTM",
    "ACH": "ACHUSDTM",
    "STX": "STXUSDTM",
    "COCOS": "COCOSUSDTM",
    "SSV": "SSVUSDTM",
    "FLOKI": "FLOKIUSDTM",
    "CKB": "CKBUSDTM",
    "TRU": "TRUUSDTM",
    "QNT": "QNTUSDTM"
}

# Binance API symbols to understandable dev-symbols
symbols_dict_binance = {
    "BTC": "BTCUSDT",
    "ETH": "ETHUSDT",
    "MATIC": "MATICUSDT",
}