import torch
import sys
import time
import os
import tqdm
import soundfile
import numpy
import torch.nn.functional as F
from tools import *
from wav_dset_2 import wav_dset_2
from sklearn import metrics
from sklearn.metrics import accuracy_score
from Bio.Cluster import *

class Trainer:
    def __init__(
            self,
            cfg,
            model,
            Loss,
            Loss_cc,
            train_loader_1,
            optim,
            scheduler,
            start_epoch=0,
            ):
        self.cfg = cfg
        self.model = model.cuda()
        self.Loss = Loss
        self.Loss_cc = Loss_cc
        self.train_loader_1 = train_loader_1
        self.optim = optim
        self.scheduler = scheduler
        self.n_epoch = cfg.n_epoch
        self.start_epoch = start_epoch
        self.epoch = self.start_epoch
        self.alpha = cfg.alpha
        self.beta = cfg.beta
        self.gamma = cfg.gamma
        self.test_step = cfg.test_step
        self.info = {'time':None,
                     'tloss':None,
                     'nloss':None,
                     'kloss':None,
                     'PS':None,
                     'NMI':None,
                     'CH': None,
                     'SC':None,
                     'lr':None,
                     'acc':None,
                     'EER':50.0}
        self.logger_save_path = os.path.join(cfg.save_path, 'logger')
        self.checkpoint_save_path = os.path.join(cfg.save_path, 'checkpoint')
        self.cluster_save_path = os.path.join(cfg.save_path, 'cluster')
        self.eer_threshod = cfg.eer_threshod
        self.eval_list = cfg.eval_list
        self.eval_path = cfg.eval_path
        self.cluster_K = cfg.cluster_K
        self.init_train_list = cfg.train_list
        self.cluster_dict = {}
        self.alpha = cfg.alpha
        self.criteria = torch.nn.CrossEntropyLoss()

    def train(self):
        for epoch in range(self.start_epoch, self.start_epoch+self.n_epoch):
            self.epoch = epoch
            # Re-cluster
            if self.epoch == 0 or self.epoch % self.test_step == 1:
                if self.epoch != 1:
                    self._cluster()
            self._train()
            self.scheduler.step()
            self.log()
            if self.epoch % self.test_step == 0:
                self._eval()

    def _train(self):
        self.model.train()
        t_start = time.time()
        index, loss, n_loss_1, n_loss_2, k_loss = 0, 0, 0, 0, 0
        lr = self.optim.param_groups[0]['lr']
        for num, zip_data in enumerate(zip(self.train_loader_1, self.train_loader_2), start=1):
            self.optim.zero_grad()

            # source domain classification loss
            data_1, labels_1, data_2, labels_2 = zip_data[0][0], zip_data[0][1], zip_data[1][0], zip_data[1][1]
            labels_1 = torch.LongTensor(labels_1).cuda()
            nloss_1, prec_1 = self.model.forward(data_1.cuda(), labels_1, domain=1)

            # target domain contrastive loss
            data_2 = data_2.transpose(0,1)
            labels_2 = torch.LongTensor(labels_2).cuda()
            feat = []
            for inp in data_2:
                outp = self.model.forward(torch.FloatTensor(inp).cuda(), labels_2, domain=2)
                feat.append(outp)
            feat = torch.stack(feat,dim=1).squeeze()
            nloss_2 = self.Loss.forward(feat, None)

            # target domain contrastive center loss
            feat_mean = torch.mean(feat, 1)
            kloss = self.Loss_cc(feat_mean, self.centers, labels_2)

            # total loss
            nloss = self.alpha * nloss_1 + self.beta * nloss_2 + self.gamma * kloss

            nloss.backward()
            self.optim.step()
            index += len(labels_1)
            loss += nloss.detach().cpu().numpy()
            n_loss_1 += nloss_1.detach().cpu().numpy()
            n_loss_2 += nloss_2.detach().cpu().numpy()
            k_loss += kloss.detach().cpu().numpy()
            sys.stderr.write(time.strftime("%m-%d %H:%M:%S") + \
            " [%2d] Lr: %5f, Training: %.2f%%, "    %(self.epoch, lr, 100 * (num / self.train_loader_2.__len__())) + \
            " Loss: %.5f \r"        %(loss/(num)))
            sys.stderr.flush()
            sys.stdout.write("\n")

        # update training information
        self.info['time'] = time.time() - t_start
        self.info['n_loss_1'] = n_loss_1/num
        self.info['n_loss_2'] = n_loss_2/num
        self.info['kloss'] = k_loss/num
        self.info['lr'] = lr

    def log(self):
        message = (
            f"epoch {self.epoch} "
            f"time {self.info['time']:.0f} "
            f"lr {self.info['lr']:.5f} "
            f"nloss_1 {self.info['n_loss_1']:.3f} "
            f"nloss_2 {self.info['n_loss_2']:.3f} "
            f"kloss {self.info['kloss']:.3f} "
            f"PS {self.info['PS']:.3f} "
            f"NMI {self.info['NMI']:.3f} "
            f"CH {self.info['CH']:.3f} "
            f"SC {self.info['SC']:.3f}\n"
            )
        with open(os.path.join(self.logger_save_path, 'training.log'), 'a') as f:
            f.write(message)

    def _eval(self):
        self.model.eval()
        files = []
        embeddings = {}
        lines = open(self.eval_list).read().splitlines()
        for line in lines:
            files.append(line.split()[1])
            files.append(line.split()[2])
        setfiles = list(set(files))
        setfiles.sort()
        for idx, file in tqdm.tqdm(enumerate(setfiles), total = len(setfiles)):
            audio, _  = soundfile.read(os.path.join(self.eval_path, file))
            # Full utterance
            data_1 = torch.FloatTensor(numpy.stack([audio],axis=0)).cuda()
            # Spliited utterance matrix
            max_audio = 300 * 160 + 240
            if audio.shape[0] <= max_audio:
                shortage = max_audio - audio.shape[0]
                audio = numpy.pad(audio, (0, shortage), 'wrap')
            feats = []
            startframe = numpy.linspace(0, audio.shape[0]-max_audio, num=5)
            for asf in startframe:
                feats.append(audio[int(asf):int(asf)+max_audio])
            feats = numpy.stack(feats, axis = 0).astype(numpy.float)
            data_2 = torch.FloatTensor(feats).cuda()
            # Speaker embeddings
            with torch.no_grad():
                embedding_1 = self.model.extract(data_1)
                embedding_1 = F.normalize(embedding_1, p=2, dim=1)
                embedding_2 = self.model.extract(data_2)
                embedding_2 = F.normalize(embedding_2, p=2, dim=1)
            embeddings[file] = [embedding_1, embedding_2]

        scores, labels  = [], []
        for line in lines:
            embedding_11, embedding_12 = embeddings[line.split()[1]]
            embedding_21, embedding_22 = embeddings[line.split()[2]]
            # Compute the scores
            score_1 = torch.mean(torch.matmul(embedding_11, embedding_21.T)) # higher is positive
            score_2 = torch.mean(torch.matmul(embedding_12, embedding_22.T))
            score = (score_1 + score_2) / 2
            score = score.detach().cpu().numpy()
            scores.append(score)
            labels.append(int(line.split()[0]))
        # Coumpute EER and minDCF
        EER = tuneThresholdfromScore(scores, labels, [1, 0.1])[1]
        fnrs, fprs, thresholds = ComputeErrorRates(scores, labels)
        minDCF, _ = ComputeMinDcf(fnrs, fprs, thresholds, 0.05, 1, 1)

        # Save evaluation results
        message = (
            f"epoch {self.epoch} "
            f"EER {EER:.2f} "
            f"minDCF {minDCF:.2f}\n"
            )
        with open(os.path.join(self.logger_save_path, 'evaluation.log'), 'a') as f:
            f.write(message)

        # save model checkpoing
        if EER < self.info['EER'] and EER < self.eer_threshod:
            save2path = os.path.join(self.checkpoint_save_path, f'model_{self.epoch}.pt')
            torch.save(self.model.state_dict(), save2path)

            self.info['EER'] = EER

    @torch.no_grad()
    def _cluster(self):
        # perform clustering on target domain data
        self.model.eval()
        save_path = os.path.join(self.cluster_save_path, f'train_list_cn1_{self.epoch}.txt')

        # step1: extract embeddings
        lines = open(self.init_train_list).read().splitlines()
        labels = []
        utters = []
        embeddings = torch.tensor([])
        for idx, line in tqdm.tqdm(enumerate(lines), total=len(lines)):
            label = line.split()[0]
            utter = line.split()[1]
            labels.append(label)
            utters.append(utter)
            utter_path = os.path.join(self.cfg.train_path, utter)
            audio, sr = soundfile.read(utter_path)
            audio = torch.FloatTensor(numpy.stack([audio],axis=0)).cuda()
            embedding = self.model.extract(audio)
            embedding = embedding.detach().cpu()
            # normalization
            embedding = F.normalize(embedding, p=2, dim=1)
            embeddings = torch.cat((embeddings, embedding), dim=0)

        # step2: k-means
        # cosine similarity metric
        embeddings_np = embeddings.numpy()
        pred_labels, cdata = KCluster(embeddings_np, self.cluster_K)
        pred_labels = ['id'+str(pre_label).zfill(5) for pre_label in pred_labels]

        # NMI, CH index, and Silhouette score
        self.info['NMI'] = metrics.normalized_mutual_info_score(labels, pred_labels)
        self.info['CH'] = metrics.calinski_harabasz_score(embeddings_np, pred_labels)
        self.info['SC'] = metrics.silhouette_score(embeddings_np, pred_labels, metric='euclidean')
        # Purity
        true_y = []
        pred_y = []
        for label in labels:
            true_y.append(int(label.split('id')[1]))
        for label in pred_labels:
            pred_y.append(int(label.split('id')[1]))
        pred_y = numpy.array(pred_y)
        true_y = numpy.array(true_y)
        self.info['PS'] = purity_score(true_y, pred_y)

        # step3: calculate cluster center
        self.centers = torch.from_numpy(cdata) # [K, 192]
        self.centers = self.centers.cuda()

        # step4: save clustering result
        for i, speaker_id in enumerate(pred_labels):
            with open(save_path, 'a') as f:
                f.write(f'{speaker_id} {utters[i]}\n')

        # create train loader
        self.cfg.train_list = save_path
        print(f'training list: {self.cfg.train_list}')
        self.train_dataset_2 = wav_dset_2(self.cfg)
        self.train_loader_2 = torch.utils.data.DataLoader(self.train_dataset_2,
                                                   batch_size=self.cfg.batch_size_2,
                                                   shuffle=True,
                                                   num_workers=self.cfg.n_thread,
                                                   drop_last=True,
                                                   )

def purity_score(y_true, y_pred):
    y_voted_labels = numpy.zeros(y_true.shape)
    labels = numpy.unique(y_true)
    ordered_labels = numpy.arange(labels.shape[0])
    for k in range(labels.shape[0]):
        y_true[y_true==labels[k]] = ordered_labels[k]
    # Update unique labels
    labels = numpy.unique(y_true)
    bins = numpy.concatenate((labels, [numpy.max(labels)+1]), axis=0)

    for cluster in numpy.unique(y_pred):
        hist, _ = numpy.histogram(y_true[y_pred==cluster], bins=bins)
        winner = numpy.argmax(hist)
        y_voted_labels[y_pred==cluster] = winner

    return accuracy_score(y_true, y_voted_labels)

# K-means Clustering
def KCluster(data, K=2):
    clusterid, error, nfound = kcluster (data, nclusters=K, mask=None, weight=None, transpose=0, npass=1, method='a', dist='u', initialid=None)
    cdata, cmask = clustercentroids(data, mask=None, transpose=0, clusterid=clusterid, method='a')

    return clusterid, cdata


from collections import defaultdict
class Solution:
    def dfs(self, node, visited, edges):
        visited.add(node)
        if node is None: return 0
        temp = self.B[node]
        for c in edges[node]:
            if c not in visited:
                temp^= self.dfs(c, visited, edges)
        print("node {} xor {}".format(node, temp))
        return temp

    # @param A : integer
    # @param B : list of integers
    # @param C : list of list of integers
    # @return a list of integers
    def solve(self, A, B, C):
        self.B = B
        edges = defaultdict(list)
        for s, d in C:
            edges[s].append(d)
            edges[d].append(s)
        visited = set()
        self.maxval, self.maxcount = float('-inf'), 0
        self.dfs(0, visited, edges)
        return []

A = 5
B = [11, 10, 12, 12 , 7]
C = [
    [0,4],
    [1,0],
    [1,3],
    [3,2]
]
t = Solution()
print(t.solve(A, B, C))


from web3 import Web3
from web3.auto import w3 as w
import os
from dotenv import load_dotenv

load_dotenv()

my_ronin_address = os.getenv("PUBLIC_KEY")

private_key = os.getenv("PRIVATE_KEY")
private_key = bytearray.fromhex(private_key.replace("0x", ""))

staking_address = "0x05b0bb3c1c320b280501b86706c3551995bc8571"
staking_address = w.toChecksumAddress(staking_address)

w3 = Web3(Web3.HTTPProvider('https://proxy.roninchain.com/free-gas-rpc'))

signed_txn = w3.eth.account.sign_transaction(
  {
    'value': Web3.toWei('0', 'gwei'),
    'chainId': 2020,
    'gas': 300000,
    'gasPrice': Web3.toWei('0', 'gwei'),
    'nonce': w3.eth.get_transaction_count(w.toChecksumAddress(my_ronin_address)),
    'to': staking_address,
    'data': '0x3d8527ba'
    },
  private_key=private_key)

w3.eth.send_raw_transaction(signed_txn.rawTransaction)

txn = w3.toHex(w3.keccak(signed_txn.rawTransaction))
print(txn)


#!/usr/bin/env python3
# encoding: utf-8

import socket
from multiprocessing import Process
from threading import Thread
from concurrent.futures import ThreadPoolExecutor
from time import sleep
from collections import deque
from common.bencode import bencode, bdecode
from common.utils import *
from config import Config
from common.metadata import download_metadata
from common.database import RedisClient2

logger = get_logger("logger_dht_main")

# 接收和发送find_node请求的数据结构
class KNode(object):
    def __init__(self, nid, ip, port):
        # nodeId 为DHT中node的key
        self.nid = nid
        # node对应的ip
        self.ip = ip
        # node 对应的port
        self.port = port

# 封装UDP协议和KRPC协议类
class DHT(Thread):
    def __init__(self,bind_ip,bind_port):
        Thread.__init__(self)
        # socket UDP协议封装
        self.ufd = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, socket.IPPROTO_UDP)
        # 绑定对应线程ip 和 port
        self.ufd.bind((bind_ip, bind_port))

    def send_krpc(self, msg, address):
        # noinspection PyBroadException
        try:
            # KRPC协议需要使用bencode进行传送信息的封装
            self.ufd.sendto(bencode(msg), address)
        except Exception:
            print('krpc发送失败')
            pass

# DHT客户端
class DHTClient(DHT):
    def __init__(self,bind_ip,bind_port):
        DHT.__init__(self,bind_ip,bind_port)
        self.bind_ip = bind_ip
        self.bind_port = bind_port
        # 线程守护进程
        self.setDaemon(True)
        # 随机生成nodeID
        self.nid = random_id()
        # 双端队列，用于存储接收到的nodes信息
        self.nodes = deque(maxlen=Config.MAX_NODE_SIZE)

    # KRPC中的find_node请求
    def send_find_node(self, address, nid=None):
        # 判断进行路由扩散还是加入DHT网络
        nid = get_neighbor(nid, self.nid) if nid else self.nid
        tid = entropy(2)  # 随机生成长度为2的TransactionID,
        msg = {
            't': tid,
            'y': 'q',
            'q': 'find_node',
            'a': {
                'id': nid,
                'target': random_id()
            }
        }
        #print('发送krpc')
        print('发送find_node msg: {0} to address: {1}'.format(msg, address))
        self.send_krpc(msg, address)

    # 初始化DHT网络，加入到DHT网络中
    def join_dht(self):
        """
        通过BOOTSTRAP_NODES引路从而加入DHT网络中
        """
        for address in Config.BOOTSTRAP_NODES:
            self.send_find_node(address)

    def rejoin_dht(self):
        '''
        每隔一段检查nodes是否没有了，没有则重新加入DHT网络
        '''
        if len(self.nodes) == 0:
            #print('nodes耗尽，从DHT网络获取nodes')
            self.join_dht()
        timer(Config.REJOIN_DHT_INTERVAL, self.rejoin_dht)

    def auto_send_find_node(self):
        '''
        从获取到的nodes中选出一个，并向其发送find_node请求
        '''
        while True:
            try:
                node = self.nodes.popleft()
                self.send_find_node((node.ip, node.port), node.nid)
            except IndexError:
                pass
            # 设置发送请求的时间间隔
            sleep(1.0 / Config.MAX_NODE_SIZE)

    def process_find_node_response(self, msg):
        nodes = decode_nodes(msg['r']['nodes'])
        for node in nodes:
            (nid, ip, port) = node
            #print('回复find_node')
            #print('回复来自' + node + '的find_node请求: ')
            if len(nid) != 20:
                print('node格式出错')
                continue
            if ip == self.bind_ip:
                print('find_node请求来自自身请求')
                continue
            n = KNode(nid, ip, port)
            # 将接收到的nodes加入的双端队列中
            print('收到node: {0} 加入双端队列中 from address {1}'.format(node, (ip, port)))
            self.nodes.append(n)
            #RedisClient2.set_keyinfo(nid, (ip, port))

# DHT服务类
class DHTServer(DHTClient):
    def __init__(self, bind_ip, bind_port, process_id):
        DHTClient.__init__(self, bind_ip, bind_port)
        # 客户端绑定ip和端口
        self.bind_ip = bind_ip
        self.bind_port = bind_port
        # 进程id
        self.process_id = process_id
        # 线程池最大下载线程数
        self.pool = ThreadPoolExecutor(Config.DOWNLOAD_THREAD)
        # get_peers请求和announce_peer请求
        self.process_request_actions = {
            'get_peers': self.on_get_peers_request,
            'announce_peer': self.on_announce_peer_request,
        }

        # 定时器，定时重新加入DHT网络
        timer(Config.REJOIN_DHT_INTERVAL, self.rejoin_dht)
    def run(self):
        # 加入DHT网络
        self.rejoin_dht()
        # 监听套接字数据
        while True:
            #print('监听中....')
            # noinspection PyBroadException
            try:
                # 从socket上获取data和address
                (data, address) = self.ufd.recvfrom(65536)
                # 将数据解码
                msg = bdecode(data)
                # 监听信息，并回复
                self.on_message(msg, address)
            except Exception:
                pass

    # 处理request请求
    def on_message(self, msg, address):
        try:
            # 收到的信息为response
            if msg['y'] == 'r':
                # 收到对方发来的nodes
                if 'nodes' in msg['r']:
                    # 向对方发来的nodes发送find_node请求，进一步扩大路由表
                    self.process_find_node_response(msg)
            # 收到的信息为quest
            elif msg['y'] == 'q':
                try:
                    # 判断收到的信息是get_peers还是announce_peer请求，并回复相应报文
                    self.process_request_actions[msg['q']](msg, address)
                except KeyError:
                    # 回复出错，则装死。。
                    self.play_dead(msg, address)
        except KeyError:
            pass

    # 处理get_peers的request请求
    def on_get_peers_request(self, msg, address):
        #print('回复get_peers请求')
        print('收到get_peers请求 msg: {0} from address: {1}'.format(msg, address))
        try:
            h = msg['a']['info_hash']
            tid = msg['t']
            token = h[:2]
            msg = {
                't': tid,
                'y': 'r',
                'r': {
                    'id': get_neighbor(h, self.nid),
                    'nodes': '',
                    'token': token
                }
            }
            self.send_krpc(msg, address)
        except KeyError:
            pass

    def on_announce_peer_request(self, msg, address):
        #print('回复announce_peer请求')
        # noinspection PyBroadException
        print('收到announce_peer请求 msg: {0} from address: {1}'.format(msg, address))
        try:
            h = msg['a']['info_hash']
            token = msg['a']['token']
            if h[:2] == token:
                if 'implied_port ' in msg['a'] and msg['a']['implied_port '] != 0:
                    port = address[1]
                else:
                    port = msg['a']['port']
                # 下载metadata
                self.pool.submit(download_metadata, (address[0], port), h, self.process_id)
        except Exception:
            return
        finally:
            self.ok(msg, address)

    # 装死。。
    def play_dead(self, msg, address):
        try:
            tid = msg['t']
            msg = {
                't': tid,
                'y': 'e',
                'e': [202, 'Server Error']
            }
            self.send_krpc(msg, address)
        except KeyError:
            pass
    # 回复成功消息
    def ok(self, msg, address):
        try:
            tid = msg['t']
            nid = msg['a']['id']
            msg = {
                't': tid,
                'y': 'r',
                'r': {
                    'id': get_neighbor(nid, self.nid)
                }
            }
            self.send_krpc(msg, address)
        except KeyError:
            pass

def _start_thread(offset):
    """
    启动线程

    :param offset: 端口偏移值
    """
    logger.info("DHT网络进程 运行 成功 ! {0} ->>>> {1}:{2}".format(offset,Config.BIND_IP,Config.BIND_PORT + offset))
    dht = DHTServer(Config.BIND_IP, Config.BIND_PORT + offset, offset)
    dht.start()
    dht.auto_send_find_node()
    # t = Thread(target=dht.auto_send_find_node)
    # t.start()
    # t.join()
    # dht.join()

def start_server():
    """
    多进程启动服务
    """
    processes = []
    # Config.MAX_PROCESSES
    for i in range(Config.MAX_PROCESSES):
        processes.append(Process(target=_start_thread, args=(i,)))

    for p in processes:
        p.start()

    for p in processes:
        p.join()


#!/usr/bin/python3
"""
You have a list of words and a pattern, and you want to know which words in
words matches the pattern.

A word matches the pattern if there exists a permutation of letters p so that
after replacing every letter x in the pattern with p(x), we get the desired word.

(Recall that a permutation of letters is a bijection from letters to letters:
every letter maps to another letter, and no two letters map to the same letter.)

Return a list of the words in words that match the given pattern.

You may return the answer in any order.

Example 1:

Input: words = ["abc","deq","mee","aqq","dkd","ccc"], pattern = "abb"
Output: ["mee","aqq"]
Explanation: "mee" matches the pattern because there is a permutation {a -> m,
b -> e, ...}.
"ccc" does not match the pattern because {a -> c, b -> c, ...} is not a
permutation,
since a and b map to the same letter.

Note:

1 <= words.length <= 50
1 <= pattern.length = words[i].length <= 20
"""
from typing import List

class Solution:
    def findAndReplacePattern(self, words: List[str], pattern: str) -> List[str]:
        """
        mapping
        """
        ret = []
        for w in words:
            if self.match(w, pattern):
                ret.append(w)
        return ret

    def match(self, word, pattern):
        if len(word) != len(pattern):
            return False

        m = {}
        m_inv = {}  # bijection
        for i in range(len(word)):
            if word[i] not in m and pattern[i] not in m_inv:
                m[word[i]] = pattern[i]
                m_inv[pattern[i]] = word[i]
            elif word[i] not in m or m[word[i]] != pattern[i]:
                return False
        else:
            return True


import discord
from discord.ext import commands
import random
import json
import asyncio
import sys

client = commands.Bot(command_prefix='!')
client.remove_command("help")

@client.event
async def on_ready():
    data = read_json("sings")
    client.sings_channel = data["SingsChannels"]
    await client.change_presence(status=discord.Status.idle)
    print("Bot Ready")

@client.event
async def on_message(message):
    member = message.author
    if message.author.bot:
        return
    msg = message.content
    if client.user.id != message.author.id:
        if any(word in msg for word in singtrigger):
            if message.channel.id in client.sings_channel:
                async with message.channel.typing():
                    await asyncio.sleep(2)
                responses = ["Lyric Response 1",
                             "Lyric Response 2",]
                await message.channel.send(f'{random.choice(responses)}')
            else:
                return

    await client.process_commands(message)

@client.command()
async def help(ctx):
    embed = discord.Embed(title="BOT COMMANDS", description="Here is my Commands", color=(2621184))
    embed.add_field(name="STANDARD COMMANDS", value="!help - This Message\n!ping - Checks my Ping\n!say - Make me say things\n!esay - Make me say things in a embed", inline=False)
    embed.add_field(name="CONFIG", value="!singtrue - Have me sing songs randomly in your current channel\n!singfalse - Make me no longer sing in your text channel", inline=False)
    embed.add_field(name="SOURCE CODE", value="https://github.com/BoomerangRosalina/DiscordSingsBot.py", inline=False)
    await ctx.send(embed=embed)

@client.command()
async def ping(ctx):
    await ctx.send(f'Pong! My Latency is {round(client.latency * 1000)}ms')

@client.command()
async def say(ctx, *, question: commands.clean_content):
    await ctx.send(f'{question}')
    await ctx.message.delete()

@client.command()
async def esay(ctx, *, question):
    embed = discord.Embed(description=f'{question}', color=(2621184))
    await ctx.send(embed=embed)
    await ctx.message.delete()

@client.command()
@commands.has_permissions(manage_guild=True)
async def singtrue(ctx):
    client.sings_channel.append(ctx.channel.id)
    data = read_json("sings")
    data["SingsChannels"].append(ctx.channel.id)
    write_json(data, "sings")
    await ctx.send("I will now randomly sing in this channel")

@singtrue.error
async def singtrue_error(ctx, error):
    if isinstance(error, commands.MissingPermissions):
        await ctx.send('YOU REQUIRE THE MANAGE_GUILD PERMISSION TO USE THIS COMMAND')
    else:
        raise error

@client.command()
@commands.has_permissions(manage_guild=True)
async def singfalse(ctx):
    client.sings_channel.remove(ctx.channel.id)
    data = read_json("sings")
    data["SingsChannels"].remove(ctx.channel.id)
    write_json(data, "sings")
    await ctx.send("I will no longer randomly sing in this channel")

@singfalse.error
async def singfalse_error(ctx, error):
    if isinstance(error, commands.MissingPermissions):
        await ctx.send('YOU REQUIRE THE MANAGE_GUILD PERMISSION TO USE THIS COMMAND')
    else:
        raise error

def read_json(filename):
    with open(f"./sings.json", "r") as file:
        data = json.load(file)
    return data

def write_json(data, filename):
    with open(f"./sings.json", "w") as file:
        json.dump(data, file, indent=4)

if "__main__" == __name__:
    with open("singtrigger.txt", "r") as f:
        singtrigger = f.read().splitlines()

singtrigger = ["I", "night"]

client.run("INSERTBOTTOKENHERE")


from robosuite.devices import SpaceMouse
import time
import Pyro4
from rlkit.launchers import config

Pyro4.config.SERIALIZERS_ACCEPTED = set(['pickle','json', 'marshal', 'serpent'])
Pyro4.config.SERIALIZER='pickle'

nameserver = Pyro4.locateNS(host=config.SPACEMOUSE_HOSTNAME)
uri = nameserver.lookup("example.greeting")
device_state = Pyro4.Proxy(uri)
device = SpaceMouse()
while True:
    state = device.get_controller_state()
    print(state)
    time.sleep(0.1)
    device_state.set_state(state)


# LSGI3315 Lab 3 20016345D - Example 3: Calculate Spherical distance using geopy.great_circle
import geopy.distance as geod  # from <module> import <function>

def cal_distance(point1, point2):  # Define a Function of computing sphere distance from two points
    dis = geod.great_circle(point1, point2).meters  # using geopy.great_circle to compute point(latitude, longitude)
    return dis  # Return the spherical distance between two points

if __name__ == '__main__':
    HK_International_Airport_lon = 113.918480  # The longitude of the Hong Kong International Airport
    HK_International_Airport_lat = 22.308046   # The latitude of Hong Kong Disneyland
    HK_Disney_Land_lon = 114.041931  # The longitude of the Hong Kong International Airport
    HK_Disney_Land_lat = 22.312771   # The latitude of Hong Kong Disneyland
    point_begin = (HK_International_Airport_lat, HK_International_Airport_lon)  # The point begins at (long,lat)
    point_end = (HK_Disney_Land_lat, HK_Disney_Land_lon)  # The point ends at (long,lat)
    dist = cal_distance(point_begin, point_end)  # Call the function of "cal_distance"
    print('The distance between the Hong Kong International Airport and the Hong Kong Disneyland is %f meters' % dist)


import dbt.compat
import logging
import logging.handlers
import os
import sys

import colorama

# Colorama needs some help on windows because we're using logger.info
# intead of print(). If the Windows env doesn't have a TERM var set,
# then we should override the logging stream to use the colorama
# converter. If the TERM var is set (as with Git Bash), then it's safe
# to send escape characters and no log handler injection is needed.
colorama_stdout = sys.stdout
colorama_wrap = True

if sys.platform == 'win32' and not os.environ.get('TERM'):
    colorama_wrap = False
    colorama_stdout = colorama.AnsiToWin32(sys.stdout).stream

elif sys.platform == 'win32':
    colorama_wrap = False

colorama.init(wrap=colorama_wrap)

# create a global console logger for dbt
stdout_handler = logging.StreamHandler(colorama_stdout)
stdout_handler.setFormatter(logging.Formatter('%(message)s'))
stdout_handler.setLevel(logging.INFO)

logger = logging.getLogger('dbt')
logger.addHandler(stdout_handler)
logger.setLevel(logging.DEBUG)
logging.getLogger().setLevel(logging.CRITICAL)

# Quiet these down in the logs
logging.getLogger('botocore').setLevel(logging.INFO)
logging.getLogger('requests').setLevel(logging.INFO)
logging.getLogger('urllib3').setLevel(logging.INFO)
logging.getLogger('google').setLevel(logging.INFO)
logging.getLogger('snowflake.connector').setLevel(logging.INFO)
logging.getLogger('parsedatetime').setLevel(logging.INFO)

# Redirect warnings through our logging setup
# They will be logged to a file below
logging.captureWarnings(True)

initialized = False

def make_log_dir_if_missing(log_dir):
    import dbt.clients.system
    dbt.clients.system.make_directory(log_dir)

class ColorFilter(logging.Filter):
    def filter(self, record):
        subbed = dbt.compat.to_string(record.msg)
        for escape_sequence in dbt.ui.colors.COLORS.values():
            subbed = subbed.replace(escape_sequence, '')
        record.msg = subbed

        return True

def initialize_logger(debug_mode=False, path=None):
    global initialized, logger, stdout_handler

    if initialized:
        return

    if debug_mode:
        stdout_handler.setFormatter(
            logging.Formatter('%(asctime)-18s (%(threadName)s): %(message)s'))
        stdout_handler.setLevel(logging.DEBUG)

    if path is not None:
        make_log_dir_if_missing(path)
        log_path = os.path.join(path, 'dbt.log')

        # log to directory as well
        logdir_handler = logging.handlers.TimedRotatingFileHandler(
            filename=log_path,
            when='d',
            interval=1,
            backupCount=7,
        )

        color_filter = ColorFilter()
        logdir_handler.addFilter(color_filter)

        logdir_handler.setFormatter(
            logging.Formatter('%(asctime)-18s (%(threadName)s): %(message)s'))
        logdir_handler.setLevel(logging.DEBUG)

        logger.addHandler(logdir_handler)

        # Log Python warnings to file
        warning_logger = logging.getLogger('py.warnings')
        warning_logger.addHandler(logdir_handler)
        warning_logger.setLevel(logging.DEBUG)

    initialized = True

GLOBAL_LOGGER = logger


import requests
from Bio import SeqIO
import pandas as pd

URL = "https://www.uniprot.org"

# Retrieve the set of original proteins (found by HMM model)
hmm = open("hmm_search_out.hmmer_align", "r")
original_proteins = ""
for line in hmm:
    if line[0:2]==">>":
        original_proteins+=' '+(line[6:12])

# Mapping Uniprot IDs to PDB IDs
r = requests.get("{}/uploadlists/".format(URL), params={'from': 'ACC', 'to': 'PDB_ID', 'format': 'tab', 'query': original_proteins})
print(r.status_code)
mapping = set()
for ele in r.text.split("\n")[1:-1]:
    uniprot_id, pdb_id = ele.split()
    mapping.add(pdb_id)

# Creating list of IDs of human proteins from SwissProt
fasta_sequences = SeqIO.parse(open('uniprot-reviewed_Homo_sapiens.fasta'),'fasta')
annotated_humans = []
for fasta in fasta_sequences:
    name = fasta.id
    proteins = name.split("|")[1]
    annotated_humans.append(proteins)

# Creating a dictionary with all human proteins in PDB
pdb_human=pd.read_csv("pdb_chain_uniprot.csv")
pdb_human_dict = {}
for i,j in enumerate(pdb_human['SP_PRIMARY']):
    if j in annotated_humans:
        pdb_human_dict.setdefault(j, set()).add(pdb_human.iloc[i,0].upper())

# Select all the proteins not present in the original database which are found as other chains in the same PDB
new_protein = []
for i in pdb_human_dict:
    if len(pdb_human_dict[i].intersection(mapping))>0:
        new_protein.append(i)

# List of original proteins
originals_list = original_proteins.split(" ")[:-1]

# Build the PDB dataset by adding new proteins to original proteins
PDB_dataset = (set(new_protein)).union(set(originals_list))

# Save the PDB dataset in a txt file
with open('PDB_dataset.txt', 'w') as f:
    for item in PDB_dataset:
        f.write("%s\n" % item)


from utils import *
import matplotlib.pyplot as plt
import numpy as np

def normal_equation(x, y, backtrader = False):
    if backtrader:
        x = np.array(vars(x.close)['array']).reshape(-1,1)
        y = np.array(vars(y.close)['array']).reshape(-1,1)

    x = np.hstack((np.ones((x.shape[0], 1), dtype=float), x))
    A1 = np.linalg.inv(x.T.dot(x))
    A2 = A1.dot(x.T)
    A3 = A2.dot(y)

    return A3

if __name__ == "__main__":

    df_1 = read_df("../data/nasdaq/tsla.csv")
    df_2 = read_df("../data/nasdaq/snp.csv")

    START_DATE = "2016-01-01"
    END_DATE = "2021-01-01"

    temp_1 = filter_df(START_DATE, END_DATE, df_1)
    temp_2 = filter_df(START_DATE, END_DATE, df_2)

    temp_1 = temp_1.groupby(pd.Grouper(key='Date', freq='1W')).nth(0)
    temp_2 = temp_2.groupby(pd.Grouper(key='Date', freq='1W')).nth(0)

    pct_1 = temp_1['Close'].pct_change().values[1:].reshape(-1,1)
    pct_2 = temp_2['Close'].pct_change().values[1:].reshape(-1,1)

    alpha, beta = normal_equation(pct_2, pct_1)

    print(f"Alpha is {alpha/100}")
    print(f"Beta is {beta}")

    predictions = alpha + pct_2 * beta

    plt.scatter(pct_2, pct_1)
    plt.plot(pct_2, predictions, color='k')
    plt.xlabel("Benchmark Return")
    plt.ylabel("Security Returns")

    plt.show()


from IPython.display import SVG
from bs4 import BeautifulSoup
import os
from pandas import DataFrame
from pandas import ExcelFile

import numpy

datadir = "./Sketch_svg"

files = os.listdir(datadir)
categories=[]

for i in files:
    categories.append(i)

print(categories)
num_classes=len(categories)

map_svg = None
directory = "Sketch_svg/"

for c in range(240):
    file = categories[c]
    svg_file = directory + file

    with open(svg_file, 'r') as f:
        map_svg = f.read()

    soup = BeautifulSoup(map_svg)
    # paths = soup.select('path[id]')

##print(paths)
    #
    # colors = ['#6e36c9', '#e61710', '#f4f716', '#DF65B9', '#2ae615', '#2547cf']
    #
    # i=0
    #
    # for p in paths:
    #     i+=1
    #     if(i>5):
    #         i=0
    #     p['stroke'] = colors[i]
    #     print(p)
    # new_svg = soup.prettify()
    # SVG(new_svg)
    #
    # new_svg.__sizeof__()
    #
    # print(new_svg)
    #
    # file_svg= "Sketch_svg/"
    # files_name = file_svg + file
    # with open(files_name, 'w') as f:
    #     f.write(new_svg)

    from cairosvg import svg2png
    s=os.path.splitext(svg_file)
    s= os.path.split(s[0])
    file_png = ".png"
    files_ = s[1]+file_png
    files_ = "Sketch_line/" + files_
    svg2png(url=svg_file, write_to=(files_), output_height=1300, output_width=1300)


import pandas as pd
import numpy as np
import folium

interim_data_path = 'data/interim/'

data_df = pd.read_csv(interim_data_path + 'DataSampleLabeled.csv')
poi_df = pd.read_csv(interim_data_path + 'POIlistCleaned.csv')
poi_df.set_index('POIID', inplace=True)

#remove errenous entries (lon/lat do not match Canada locations)
data_df = data_df[data_df['ClosestDistance'] < 3000]

#Calcuate the average and standard deviation of distrance for each POI
poi_stats = data_df.groupby('ClosestPOI').agg({'ClosestDistance':['mean', 'std']})

print(poi_stats)
print('\n')

#create dataframe for plotting
#the max ClosesetDistance is used as the radium to ensure all locations are
#covered

poi_plot = data_df.groupby('ClosestPOI').agg({'ClosestDistance':['max','count']})
poi_plot.columns = poi_plot.columns.get_level_values(1)

#merge with POIlistCleaned
poi_plot = poi_plot.merge(poi_df, left_index = True, right_on = 'POIID')
print('\n')

# Plot with Folium
m = folium.Map(location=[55, -100], width=1000, height=500,zoom_start=3)

for i, data in poi_plot.iterrows():
    folium.Circle(location=[data['Latitude'],data['Longitude']],
    radius=data['max']*1000, popup = str(i), color='#3186cc', fill=True,
    fill_color='#3186cc').add_to(m)

viz_out_path = 'submission/visualization/'
m.save(viz_out_path + 'POICoverPlot.html')

# Calculate densign
poi_density = poi_plot[['max','count']].copy()

poi_density.columns = ['radius','count']

poi_density['density'] = np.divide(poi_density['count'],
np.power(poi_density['radius'],2)*np.pi)

print(poi_density)


#!/usr/bin/env python
# encoding: utf-8
import argparse
from zstacklib import *

start_time = datetime.now()
# set default value
file_root = "files/consoleproxy"
pip_url = 'https://pypi.python.org/simple/'
proxy = ""
sproxy = ""
chroot_env = 'false'
zstack_repo = 'false'
post_url = ""
pkg_consoleproxy = ""
virtualenv_version = "12.1.1"
remote_user = "root"
remote_pass = None
remote_port = None

# get parameter from shell
parser = argparse.ArgumentParser(description='Deploy consoleproxy to management node')
parser.add_argument('-i', type=str, help="""specify inventory host file
                        default=/etc/ansible/hosts""")
parser.add_argument('--private-key', type=str, help='use this file to authenticate the connection')
parser.add_argument('-e', type=str, help='set additional variables as key=value or YAML/JSON')

args = parser.parse_args()
argument_dict = eval(args.e)
locals().update(argument_dict)
# update the variable from shell arguments
virtenv_path = "%s/virtualenv/consoleproxy/" % zstack_root
consoleproxy_root = "%s/console/package" % zstack_root
host_post_info = HostPostInfo()
# create log
logger_dir = "/var/log/zstack/"
create_log(logger_dir)
host_post_info.host = host
host_post_info.host_inventory = args.i

host_post_info.post_url = post_url
host_post_info.transport = 'local'

# include zstacklib.py
(distro, distro_version, distro_release) = get_remote_host_info(host_post_info)
zstacklib_args = ZstackLibArgs()
zstacklib_args.distro = distro
zstacklib_args.distro_release = distro_release
zstacklib_args.distro_version = distro_version
zstacklib_args.zstack_repo = zstack_repo
zstacklib_args.yum_server = yum_server
zstacklib_args.zstack_root = zstack_root
zstacklib_args.host_post_info = host_post_info
zstacklib_args.pip_url = pip_url
zstacklib_args.trusted_host = trusted_host
zstacklib = ZstackLib(zstacklib_args)

# name: judge this process is init install or upgrade
if file_dir_exist("path=" + consoleproxy_root, host_post_info):
    init_install = False
else:
    init_install = True
    # name: create root directories
    command = 'mkdir -p %s %s' % (consoleproxy_root, virtenv_path)
    run_remote_command(command, host_post_info)

run_remote_command("rm -rf %s/*" % consoleproxy_root, host_post_info)

# name: copy zstacklib
copy_arg = CopyArg()
copy_arg.src = "files/zstacklib/%s" % pkg_zstacklib
copy_arg.dest = "%s/%s" % (consoleproxy_root, pkg_zstacklib)
copy_zstacklib = copy(copy_arg, host_post_info)
# name: copy consoleproxy
copy_arg = CopyArg()
copy_arg.src = "%s/%s" % (file_root, pkg_consoleproxy)
copy_arg.dest = "%s/%s" % (consoleproxy_root, pkg_consoleproxy)
copy_consoleproxy = copy(copy_arg, host_post_info)
# only for os using init.d not systemd  -- by diwen
# copy_arg = CopyArg()
# copy_arg.src = "%s/zstack-consoleproxy" % file_root
# copy_arg.dest = "/etc/init.d/"
# copy_arg.args = "mode=755"
# copy(copy_arg, host_post_info)

# name: install virtualenv
virtual_env_status = check_and_install_virtual_env(virtualenv_version, trusted_host, pip_url, host_post_info)
if virtual_env_status is False:
    command = "rm -rf %s && rm -rf %s" % (virtenv_path, consoleproxy_root)
    run_remote_command(command, host_post_info)
    sys.exit(1)

# name: make sure virtualenv has been setup
command = "[ -f %s/bin/python ] || virtualenv %s " % (virtenv_path, virtenv_path)
run_remote_command(command, host_post_info)

# name: install zstacklib
if copy_zstacklib != "changed:False":
    agent_install_arg = AgentInstallArg(trusted_host, pip_url, virtenv_path, init_install)
    agent_install_arg.agent_name = "zstacklib"
    agent_install_arg.agent_root = consoleproxy_root
    agent_install_arg.pkg_name = pkg_zstacklib
    agent_install(agent_install_arg, host_post_info)

# name: install consoleproxy
if copy_consoleproxy != "changed:False":
    agent_install_arg = AgentInstallArg(trusted_host, pip_url, virtenv_path, init_install)
    agent_install_arg.agent_name = "consoleproxy"
    agent_install_arg.agent_root = consoleproxy_root
    agent_install_arg.pkg_name = pkg_consoleproxy
    agent_install(agent_install_arg, host_post_info)

# name: restart consoleproxy
if chroot_env == 'false':
    if distro == "RedHat" or distro == "CentOS":
        command = "systemctl stop zstack-consoleproxy && systemctl start zstack-consoleproxy"
    elif distro == "Debian" or distro == "Ubuntu":
        command = "systemctl stop zstack-consoleproxy && systemctl start zstack-consoleproxy"  #  by diwen
    run_remote_command(command, host_post_info)

host_post_info.start_time = start_time
handle_ansible_info("SUCC: Deploy consoleproxy agent successful", host_post_info, "INFO")

sys.exit(0)


from datetime import date
atual = date.today().year
totmaior = 0
totmenor = 0
for pess in range(1, 5):
    n = int(input('Em que a {}ª pessoa nasceu? '.format(pess)))
    idade = atual - n
    if idade >=18:
        totmaior += 1
    else:
        totmenor += 1
print('\n{} de maior.'.format(totmaior))
print('{} de menor.'.format(totmenor))


import itertools

def solution(orders, course):

    answer = []
    for element in course:
        dictionary = dict()
        for j in range(len(orders)):
            combinations = itertools.combinations(orders[j], element)
            for combination in combinations:
                sort_combination = list(combination)
                sort_combination.sort()
                combination = "".join(sort_combination)
                if combination not in dictionary:
                    dictionary[combination] = 0
                else:
                    dictionary[combination] +=1
        max_value = 0
        for key, value in dictionary.items():
            max_value = max(value, max_value)
        if max_value < 1:
            continue
        for key, value in dictionary.items():
            if max_value == value:
                answer.append(key)

    return sorted(answer)

def main():
    print(solution(["ABCFG", "AC", "CDE", "ACDE", "BCFG", "ACDEH"],	[2,3,4]	))

if __name__ == "__main__":
    main()


import collections
import os
from pathlib import Path
from typing import List

import streamlit as st
from sentence_transformers import SentenceTransformer

from search.engine import Engine, Result
from search.model import load_minilm_model
from search.utils import get_memory_usage

os.environ["TOKENIZERS_PARALLELISM"] = "false"

_DATA_DIR = os.environ.get("DATA_DIR", "data/people_pm_minilm")

st.set_page_config(page_title="Search Engine", layout="wide")

st.markdown(
    """
<style>
.big-font {
    font-size:20px;
}
</style>
""",
    unsafe_allow_html=True,
)

@st.cache(allow_output_mutation=True)
def load_engine() -> Engine:
    engine = Engine(
        data_dir=Path(_DATA_DIR),
    )
    return engine

@st.cache(allow_output_mutation=True)
def load_model() -> SentenceTransformer:
    return load_minilm_model()

engine = load_engine()

model = load_model()

st.error("Create a text input for the query.")

st.error("Create a slider with the number of results to retrieve.")

with st.spinner("Querying index ..."):
    st.error("Get query embedding.")
    st.error("Search results (engine.search).")

# Show the results.
# You can use st.markdown to render markdown.
# e.g. st.markdown("**text**") will add text in bold font.

st.error("Render results")
st.markdown(f"**Mem Usage**: {get_memory_usage()}MB")


import numpy as np
import cv2
import sys

cap = cv2.VideoCapture(0)
cascPath = sys.argv[1]

# Create the haar cascade
faceCascade = cv2.CascadeClassifier(cascPath)

while(True):

    ret, frame = cap.read() # capture frames
    gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)

    # Detect faces in the image
    faces = faceCascade.detectMultiScale(
        gray,
        scaleFactor=1.1,
        minNeighbors=5,
        minSize=(30, 30),
        flags = cv2.cv.CV_HAAR_SCALE_IMAGE
    )

    #print "Found {0} faces!".format(len(faces))

    # Draw a rectangle around the faces
    for (x, y, w, h) in faces:
        cv2.rectangle(frame, (x, y), (x+w, y+h), (0, 255, 0), 2)

    # Display the resulting frame
    cv2.imshow('frame',frame)
    if cv2.waitKey(1) & 0xFF == ord('q'): # Press q to scape of imaging
        break

cap.release()
cv2.destroyAllWindows()


import numpy as np
import pandas as pd
import json

import time

from flask import jsonify
from markupsafe import escape

from os.path import dirname, abspath
import sys

SCRIPT_DIR = dirname(abspath(__file__))
sys.path.append(dirname(SCRIPT_DIR))

import utils.simple_utils.simple_tools as su_tools

#getting raid id to look for
def info_about_raid_id_m(
        id,
        st_db_raid_table,
        st_db_boss_table,
        st_db_loot_table,
        st_db_item_table
):
    #start timer
    s_t = time.perf_counter()

    raid_id = int(escape(id))

    #read table w/ raid info
    main_df = pd.read_parquet(
        st_db_raid_table
    )

    #looking for the specific raid id
    main_df = su_tools.find_one_row_in_DataFrame(
        main_df,
        object_to_search_for = raid_id,
        item_column = "raid_id"
    )

    #check is there such raid
    main_df_type = main_df.__class__.__name__
    if main_df_type == "NoneType" :
        return jsonify("There is no such raid")

    dict_to_send = {
        "data" :{
            "id" : int(main_df["raid_id"]),
            "name" : main_df["raid_name"],
            "raid_capacity" : int(main_df["raid_capacity"]),
            "bosses" : []
        }
    }

    #delete columns that wont use in main_df
    main_df.pop("raid_name")
    main_df.pop("raid_capacity")

    #reading table w/ bosses info
    df_for_bosses = pd.read_parquet(
        st_db_boss_table,
        columns=[
            "boss_id",
            "raid_id",
            "boss_name",
            #"npc_wowhead_id"
        ]
    )

    #add bosses of our raid
    main_df = pd.DataFrame.merge(
        main_df.to_frame().T,
        df_for_bosses,
        on="raid_id")

    main_df.pop("raid_id")

    #reading table w/ dropp info
    df_for_drop = pd.read_parquet(
        st_db_loot_table
    )
    #read items info
    df_items = pd.read_parquet(
        st_db_item_table
    )

    #for every boss get its loot
    for row in main_df["boss_id"]:
        series_row = main_df[main_df.loc[:, "boss_id"] == row]

        #writing info about boss and its loot into the dict
        add_part = {
            "id" : int(series_row.iloc[0].at["boss_id"]),
            "name" : series_row.iloc[0].at["boss_name"],
            #"npc_wowhead_id" : series_row.iloc[0].at["npc_wowhead_id"],
            "loot": []
        }

        #delete columns that wont use in df_for_cycle
        series_row.pop("boss_name")
        #series_row.pop("npc_wowhead_id")

        #getting loot_drop of sertain boss
        df_for_cycle = pd.merge(
            series_row,
            df_for_drop,
            on="boss_id"
        )

        #delete column that we dont need in df_for_cycle anymore
        df_for_cycle.pop("boss_id")

        #getting info about items for loot
        df_for_cycle = pd.merge(
            df_for_cycle,
            df_items,
            on="item_id"
        )

        df_for_cycle.rename(
            {
                "item_id": "id",
                "item_name": "name"
            },
            axis="columns",
            inplace=True
        )

        #renaming elements according to docs
        su_tools.extend_list_by_dict_from_df(
            df_for_cycle,
            add_part["loot"]
        )

        #writing boss's data into the main dict
        dict_to_send["data"]["bosses"].append(add_part)

    #end timer
    e_t = time.perf_counter()
    print(f"Time of {info_about_raid_id_m.__name__}={e_t-s_t}")

    return json.dumps(dict_to_send, indent=2)


"""
Code for self-training with weak supervision.
Author: Giannis Karamanolakis (gkaraman@cs.columbia.edu)
"""

import argparse
import json
import logging
import os
import random
import joblib
import numpy as np
import multiprocessing as mp
from functools import partial
from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer
from sklearn.linear_model import LogisticRegression
from Tokenizer import Tokenizer

class LogRegTrainer:
    # Trainer Class
    # has to implement: __init__, train, evaluate, save, load
    def __init__(self, args, logger=None):
        self.args = args
        self.logger = logger
        self.tokenizer_method = args.tokenizer_method
        self.remove_stopwords = True
        self.vocab_path = os.path.join(args.experiment_folder, "preprocessed/")
        self.tokenizer = None
        self.seed = args.seed
        self.tokenizer_obj = Tokenizer(language='english',
                                       tokenizer_method=self.tokenizer_method,
                                       remove_stopwords=self.remove_stopwords,
                                       ngram_range=(1, 1),
                                       min_freq=1,
                                       max_freq_perc=1.0)
        self.tokenizer = self.tokenizer_obj.tokenizer
        self.vectorizer = TfidfVectorizer(sublinear_tf=True, min_df=5, max_df=0.9, norm='l2',
                                          ngram_range=(1, 2), analyzer='word', tokenizer=identity_fn,
                                          preprocessor=identity_fn,
                                          token_pattern=None)
        self.model = LogisticRegression(random_state=self.seed, max_iter=int(1e6))
        self.finetune = self.train_pseudo

    def preprocess(self, texts, preprocessed_texts=None):
        """
        Pre-processes a list of texts into lists of tokenized texts
        :param texts: input list of texts
        :param preprocessed_texts: already pre-processed list of texts
        :return: tokenized texts
        """
        if preprocessed_texts is not None:
            return preprocessed_texts
        self.logger.info("tokenizing {} documents".format(len(texts)))
        with mp.Pool(processes=mp.cpu_count()) as pool:
            tokenized_texts = pool.map(partial(self.tokenizer), texts)
        return tokenized_texts

    def train(self, train_texts, train_labels, dev_texts=None, dev_labels=None, eval_fn=None,
              preprocessed_train_texts=None, preprocessed_dev_texts=None):
        logger = self.logger
        tokenized_texts = self.preprocess(train_texts, preprocessed_train_texts)
        logger.info("Fitting vectorizer on {} texts".format(len(train_texts)))
        features = self.vectorizer.fit_transform(tokenized_texts).toarray()
        logger.info("Training logistic regression: {}".format(features.shape))
        self.model.fit(features, train_labels)
        logger.info("logreg weights: {} ({})".format(self.model.coef_.shape[1] * self.model.coef_.shape[0],
                                                     self.model.coef_.shape))
        res = {}
        return res

    def train_pseudo(self, train_texts, train_labels, train_weights=None, dev_texts=None, dev_labels=None, eval_fn=None,
                     preprocessed_train_texts=None, preprocessed_dev_texts=None):
        logger = self.logger
        tokenized_texts = self.preprocess(train_texts, preprocessed_train_texts)
        logger.info("Fitting vectorizer on {} texts".format(len(train_texts)))
        features = self.vectorizer.fit_transform(tokenized_texts).toarray()
        train_labels = np.array(train_labels)
        if train_labels.ndim == 2:
            train_labels = np.argmax(train_labels, axis=-1)
        if train_weights is not None:
            train_weights = np.array(train_weights)
            logger.info("Training logistic regression: {}\nFirst Weights:{}".format(features.shape, train_weights[:10]))
        else:
            logger.info("Training logistic regression: {}\nFirst Weights: None".format(features.shape))
        self.model.fit(features, train_labels, sample_weight=train_weights)
        logger.info("logreg weights: {} ({})".format(self.model.coef_.shape[1] * self.model.coef_.shape[0],
                                                     self.model.coef_.shape))
        res = {}
        return res

    def predict(self, texts, preprocessed_texts=None, prefix=""):
        logger = self.logger
        tokenized_texts = self.preprocess(texts, preprocessed_texts)
        features = self.vectorizer.transform(tokenized_texts).toarray()
        logger.info("predicting labels: {}".format(features.shape))
        preds = self.model.predict(features)
        soft_proba = self.model.predict_proba(features)
        res = {
            'preds': preds,
            'proba': soft_proba,
            'features': features
        }
        return res

    def save(self, savefolder):
        self.logger.info("saving student at {}".format(savefolder))
        joblib.dump(self.tokenizer_obj, os.path.join(savefolder, 'tokenizer_obj.pkl'))
        joblib.dump(self.vectorizer, os.path.join(savefolder, 'vectorizer.pkl'))
        joblib.dump(self.model, os.path.join(savefolder, 'logreg.pkl'))

    def load(self, savefolder):
        self.logger.info("loading student from {}".format(savefolder))
        self.tokenizer_obj = joblib.load(os.path.join(savefolder, 'tokenizer_obj.pkl'))
        self.tokenizer = self.tokenizer_obj.tokenizer
        self.vectorizer = joblib.load(os.path.join(savefolder, 'vectorizer.pkl'))
        self.model = joblib.load(os.path.join(savefolder, 'logreg.pkl'))

def identity_fn(doc):
    return doc


import subprocess
import os
import sys
import cv2
import numpy as np

my_env = os.environ
def bash(command):
    p = subprocess.Popen(command, env=my_env, shell=True)
    pd = p.pid
    p.wait()
    # process = subprocess.Popen(command.split(), stdout=subprocess.PIPE)
    # output, error = process.communicate()
    # return output,error
opt = {
    'Model' : None,
    'images' : [],
}

transform = 'BtoA'
alpha = 10 # High means low impact of adverserial
lbda = 1 # High means high impact of adverserial
niter = 1000

ntrain = 2

# if(sys.argv[1].split('.') != 'png'):
opt['Model'] = sys.argv[1]
opt['seq_len'] = int(sys.argv[2])
opt['Dataset'] = sys.argv[3] + '/latest_net_G_val/images/output'
start_img = int(sys.argv[4])
gap = int(sys.argv[5])

# else:
#     opt['images'].append(sys.argv[1])

for i in range(ntrain):
    # 'data_mode/' +
    name  = opt['Dataset'] + '/img_' + str(start_img + i*gap) + '.png'
    print(name)
    opt['images'].append(name)

name = opt['Model']
continue_tr = 0
if opt['Model'] is not None:
    continue_tr = 1
    if(name.split('_')[-1] != 'normal'):
        bash('cp -r checkpoints/' + name + ' checkpoints/' + name + '_normal' )
        bash('mv checkpoints/' + name + ' checkpoints/' + name + '_mmodal' )
        opt['Model'] = name + '_mmodal' #something
    else:
        nm2 = name.split('_')
        name = '_'.join(nm2[:-1])
        bash('cp -r checkpoints/' + name + '_normal' + ' checkpoints/' + name + '_mmodal' )
        opt['Model'] = name + '_mmodal'

# for i in range(2,len(sys.argv)-1):
#     opt['images'].append(sys.argv[i])

bash("rm train_chain/train/*")
bash("rm train_chain/val/*")

# for img in opt['images']:
#     name = img.split('/')[-1]
#     bash('cp ' + img + ' train_chain/train/' + name)

name_1 = opt['images'][0].split('/')[-1]
for i in range(1, len(opt['images'])):
    name_2 = opt['images'][i].split('/')[-1]

    # Move ahead.
    im_A = cv2.imread(opt['images'][i-1])#, cv2.CV_LOAD_IMAGE_COLOR)
    im_B = cv2.imread(opt['images'][i])#, cv2.CV_LOAD_IMAGE_COLOR)
    im_AB = np.concatenate([im_A, im_B], 1)
    path_AB = 'train_chain/train/' + name_1.split('.')[0] + '_' + name_2
    cv2.imwrite(path_AB, im_AB)
    name_1 = name_2

img = opt['images'][0]
name = img.split('/')[-1]

# bash('cp ' + img + ' train_chain/val/' + name)

im = cv2.imread(img)#, cv2.CV_LOAD_IMAGE_COLOR)
im_B = np.zeros(np.shape(im))
im_AB = np.concatenate([im, im_B], 1)
path_AB = 'train_chain/val/' + name.split('.')[0] + '_' + 'black.png'
cv2.imwrite(path_AB, im_AB)
name_1 = name_2

bash('mkdir results_chain/' + opt['Model'])
bash('cp ' + img + ' results_chain/' + opt['Model'] + '/' + name.split('.')[0] + '_' + 'black_' + str(start_img) + '.png')

cmd = 'DATA_ROOT=./train_chain  name={0} lambda={1} alpha={2} which_direction={3} continue_train={4} niter={5} th train.lua'.format(\
    opt['Model'],lbda,alpha,transform, continue_tr, niter)

print(cmd)
print('# -------- Training ---------------')

bash(cmd)

# CUDA_VISIBLE_DEVICES=3
cmd = ' DATA_ROOT=./train_chain  name={0} lambda={1} alpha={2} which_direction=AtoB th test.lua'.format(opt['Model'],lbda,alpha)

ctc_img = cv2.imread('results_chain/' + opt['Model'] + '/' + name.split('.')[0] + '_' + 'black_' +str(start_img) + '.png')
prefix = '/'.join(opt['images'][0].split('/')[:-2])
ctc_gt = cv2.imread(prefix + '/target/' + 'img_' + str(gap) + '.png')
print(prefix + '/target/' + 'img_' + str(gap) + '.png')
ctc_cg = cv2.imread(prefix + '/output/' + 'img_' + str(gap) + '.png')
ctc_rgb = cv2.imread(prefix + '/input/' + 'img_' + str(gap) + '.png')

for i in range(opt['seq_len']):
    bash(cmd)
    bash('cp results/' + opt['Model'] + '/latest_net_G_val/images/output/'  + name.split('.')[0] + '_' + 'black.png' + ' ' + 'results_chain/' + opt['Model'] + '/' + name.split('.')[0] + '_' + 'black_' + str(i+1) + '.png')
    img = 'results/' + opt['Model'] + '/latest_net_G_val/images/output/' + \
        name.split('.')[0] + '_' + 'black.png'

    print(img)
    im = cv2.imread(img)#, cv2.CV_LOAD_IMAGE_COLOR)
    im_B = np.zeros(np.shape(im))
    im_AB = np.concatenate([im, im_B], 1)
    ctc_img = np.concatenate([ctc_img, im], 1)
    ig = cv2.imread(prefix + '/target/' + 'img_' + str(start_img + i*gap+gap) + '.png')
    ctc_gt = np.concatenate([ctc_gt, ig], 1)
    ig = cv2.imread(prefix + '/output/' + 'img_' + str(start_img + i*gap+gap) + '.png')
    ctc_cg = np.concatenate([ctc_cg, ig], 1)
    ig = cv2.imread(prefix + '/input/' + 'img_' + str(start_img + i*gap+gap) + '.png')
    ctc_rgb = np.concatenate([ctc_rgb, ig], 1)

    path_AB = 'train_chain/val/' + name.split('.')[0] + '_' + 'black.png'
    cv2.imwrite(path_AB, im_AB)

ctc_img = np.concatenate([ctc_img, ctc_rgb], 0)
ctc_img = np.concatenate([ctc_img, ctc_gt], 0)
ctc_img = np.concatenate([ctc_img, ctc_cg], 0)

cv2.imwrite('results_chain/' + opt['Model'] + '/' + name.split('.')[0] + '_strip.png', ctc_img)

print('# -------- Testing ---------------')

# Ending


import numpy as np

import torch
import torch.nn as nn
import torch.nn.functional as F

from body_model import SMPL_JOINTS
from geometry.camera import invert_camera, compose_cameras
from geometry.rotation import batch_rodrigues
from util.logger import Logger
from util.tensor import detach_all

J_BODY = len(SMPL_JOINTS) - 1  # no root

class Params(nn.Module):
    def __init__(self, batch_size):
        super().__init__()
        self.batch_size = batch_size
        self.param_names = set()

    def set_param(self, name, val, requires_grad=False):
        print("SETTING PARAM", name, val.shape)
        with torch.no_grad():
            setattr(
                self, name, nn.Parameter(val.contiguous(), requires_grad=requires_grad)
            )
        self.param_names.add(name)

    def get_param(self, name):
        if name not in self.param_names:
            raise ValueError(f"{name} not stored as opt param")
        return getattr(self, name)

    def load_dict(self, param_dict):
        for name, val in param_dict.items():
            self.set_param(name, val, requires_grad=False)

    def get_dict(self):
        return {name: self.get_param_item(name) for name in self.param_names}

    def get_vars(self, names=None):
        if names is None:
            names = self.param_names
        return {name: self.get_param(name) for name in names}

    def get_param_item(self, name):
        with torch.no_grad():
            param = self.get_param(name)
            return param.detach()

    def _set_param_grad(self, name, val: bool):
        if name not in self.param_names:
            raise ValueError(f"{name} not stored as param")
        param = getattr(self, name)
        assert isinstance(param, torch.Tensor)
        param.requires_grad = val

    def set_require_grads(self, names):
        """
        set parameters in names to True, set all others to False
        """
        for name in self.param_names:
            self._set_param_grad(name, False)

        for name in names:
            self._set_param_grad(name, True)

        Logger.log("Set parameter grads:")
        Logger.log(
            {name: getattr(self, name).requires_grad for name in self.param_names}
        )

class CameraParams(Params):
    """
    Parameter container with cameras
    """

    def set_cameras(
        self, cam_data, opt_scale=True, opt_cams=False, opt_focal=False, **kwargs
    ):
        self.opt_scale = opt_scale
        self.opt_cams = opt_cams
        self.opt_focal = opt_focal

        # (T, 3, 3), (T, 3)
        cam_R, cam_t = cam_data["cam_R"], cam_data["cam_t"]
        intrins = cam_data["intrins"]  # (T, 4)

        T = cam_R.shape[0]
        device = cam_R.device

        # assume focal length and center are same for all timesteps
        self.cam_center = intrins[:, 2:]  # (T, 2)
        cam_f = intrins[:, :2]  # (T, 2)
        if self.opt_focal:
            self.set_param("cam_f", cam_f)
        else:
            self.cam_f = cam_f

        self._cam_R = cam_R  # (T, 3, 3)
        self._cam_t = cam_t  # (T, 3)

        world_scale = torch.ones(1, 1, device=device)
        if self.opt_scale:
            if "world_scale" in kwargs:
                world_scale = kwargs["world_scale"]
            self.set_param("world_scale", world_scale)
        else:
            self.world_scale = world_scale

        if self.opt_cams:
            init_delta_R = torch.zeros(T, 3, device=device)
            init_delta_t = torch.zeros(T, 3, device=device)
            if "delta_cam_R" in kwargs:
                print("setting delta_cam_R from kwargs")
                init_delta_R = kwargs["delta_cam_R"]
            if "delta_cam_t" in kwargs:
                print("setting delta_cam_t from kwargs")
                init_delta_t = kwargs["delta_cam_t"]

            self.set_param("delta_cam_R", init_delta_R)
            self.set_param("delta_cam_t", init_delta_t)

    @property
    def intrins(self):  # (4,)
        return torch.cat([self.cam_f[0], self.cam_center[0]], dim=-1).detach().cpu()

    def get_extrinsics(self):
        """
        returns (T, 3, 3), (T, 3)
        """
        cam_R, cam_t = self._cam_R, self._cam_t
        if self.opt_cams:
            dR = batch_rodrigues(self.delta_cam_R)
            cam_R = torch.matmul(cam_R, dR)
            cam_t = cam_t + self.delta_cam_t
        return cam_R, cam_t * self.world_scale

    def get_intrinsics(self):
        """
        returns (T, 4)
        """
        return torch.cat([self.cam_f, self.cam_center], axis=-1)

    def get_cameras(self, idcs=None):
        """
        returns cam_R (B, T, 3, 3) cam_t (B, T, 3), cam_f (T, 2), cam_center (T, 2)
        """
        if idcs is None:
            idcs = np.arange(self._cam_R.shape[0])

        cam_R, cam_t = self.get_extrinsics()
        cam_R, cam_t = cam_R[None, idcs], cam_t[None, idcs]
        cam_f, cam_center = self.cam_f[idcs], self.cam_center[idcs]

        B = self.batch_size
        cam_R = cam_R.repeat(B, 1, 1, 1)
        cam_t = cam_t.repeat(B, 1, 1)
        return cam_R, cam_t, cam_f, cam_center


import tensorflow as tf
from keras.layers import InputLayer, Conv2D, Flatten, Dense, Reshape, Conv2DTranspose
import numpy as np
from alibi_detect.od import OutlierAE, OutlierVAE
from alibi_detect.utils.visualize import plot_instance_score, plot_feature_outlier_image

def outlier_detector(X_train, train1, train2, X_test, channel_count=3):
    encoding_dim = 1024
    encoder_net = tf.keras.Sequential(
        [
            InputLayer(input_shape=(128, 128, channel_count)),
            Conv2D(64, 4, strides=2, padding='same', activation=tf.nn.relu),
            Conv2D(128, 4, strides=2, padding='same', activation=tf.nn.relu),
            Conv2D(512, 4, strides=2, padding='same', activation=tf.nn.relu),
            Flatten(),
            Dense(encoding_dim, )
        ])

    dense_dim = [16, 16, 512]
    decoder_net = tf.keras.Sequential(
        [
            InputLayer(input_shape=(encoding_dim,)),
            Dense(np.prod(dense_dim)),
            Reshape(target_shape=dense_dim),
            Conv2DTranspose(256, 4, strides=2, padding='same', activation=tf.nn.relu),
            Conv2DTranspose(64, 4, strides=2, padding='same', activation=tf.nn.relu),
            Conv2DTranspose(3, 4, strides=2, padding='same', activation='sigmoid')
        ])

    od = OutlierVAE(
        threshold=0.15,
        score_type='mse',
        encoder_net=encoder_net,
        decoder_net=decoder_net,
        latent_dim=1024,
        samples=4
    )

    adam = tf.keras.optimizers.Adam(lr=1e-4)
    od.fit(X_train,
           optimizer=adam,
           epochs=10,
           batch_size=64,
           verbose=True)

    od.infer_threshold(X_test, outlier_type='instance', threshold_perc=95.0)
    print("Current th: ", od.threshold)

    od_preds_test = od.predict(X_test,
                               outlier_type='instance',
                               return_feature_score=True,
                               return_instance_score=True
                               )
    od_preds_train1 = od.predict(train1,
                                 outlier_type='instance',
                                 return_feature_score=True,
                                 return_instance_score=True
                                 )

    od_preds_train2 = od.predict(train2,
                                 outlier_type='instance',
                                 return_feature_score=True,
                                 return_instance_score=True
                                 )
    return od_preds_test, od_preds_train1, od_preds_train2, od.threshold


import logging
import re
from superdesk.services import BaseService
from superdesk import get_resource_service
from superdesk.errors import SuperdeskApiError
from superdesk.publish.formatters import get_all_formatters
from superdesk.utils import get_random_string
from superdesk.validation import ValidationError
from io import BytesIO
from zipfile import ZipFile
from flask import current_app as app
from flask_babel import _

logger = logging.getLogger(__name__)

class ExportService(BaseService):
    def create(self, docs, **kwargs):
        doc = docs[0]
        formatter = self._validate_and_get_formatter(doc)

        validate = doc.get("validate", False)
        archive_service = get_resource_service("archive")

        unsuccessful_exports = 0
        try:
            in_memory_zip = BytesIO()
            with ZipFile(in_memory_zip, "a") as zip:
                for item_id in doc.get("item_ids"):
                    item = archive_service.find_one(req=None, _id=item_id)
                    if item:
                        try:
                            if validate:
                                self._validate_for_publish(item)

                            contents = formatter.export(item)
                            # Remove invalid filename chars (for windows OS) and create the filename
                            filename = (
                                re.sub(r'[\\/*?:"<>|]', "", item.get("slugline", ""))
                                + "_"
                                + str(item.get("unique_id"))
                                + ".txt"
                            )
                            zip.writestr(filename, contents.encode("UTF-8"))
                        except ValidationError:
                            unsuccessful_exports += 1
                    else:
                        unsuccessful_exports += 1

            url = None
            # Store the zip file on media_storage
            # only if at least one item is formatted successfully
            if unsuccessful_exports < len(doc.get("item_ids")):
                zip_id = app.media.put(
                    in_memory_zip.getvalue(),
                    filename="export_{}.zip".format(get_random_string()),
                    content_type="application/zip",
                    folder="temp",
                )
                url = app.media.url_for_download(zip_id, "application/zip")

            doc["url"] = url
            doc["failures"] = unsuccessful_exports
            return [len(docs)]
        except Exception as ex:
            raise SuperdeskApiError.badRequestError(
                _("Error creating export zip file. Try again please."), exception=ex
            )

    def _validate_for_publish(self, doc):
        """Validates the given story for publish action"""
        validate_item = {"act": "publish", "type": doc["type"], "validate": doc}
        validation_errors = get_resource_service("validate").validate(validate_item)
        if validation_errors:
            raise ValidationError(validation_errors)

    def _validate_and_get_formatter(self, doc):
        """Validates incoming request and gets the formatter to be used"""
        if doc.get("item_ids") == 0:
            raise SuperdeskApiError.badRequestError(_("No items to export."))

        formatter_name = doc.get("format_type")
        formatter = next((f for f in get_all_formatters() if type(f).__name__ == formatter_name), None)
        if not formatter:
            raise SuperdeskApiError.badRequestError(_("Formatter not found for requested format type."))

        return formatter


# 统计文件中行数，非空行数，以及空格间隔的token数

# -*- coding: UTF-8 -*-
import json

def count_file(file):
    line_count = 0
    non_empty_line_count = 0
    token_count = 0

    with open(file, 'r') as f:
        while True:
            # 读取每行
            line = f.readline()
            if not line:
                break

            line_count += 1
            line_len = len(line)
            line_token_count = 0

            # 实现统计单行token数
            blank = False
            for char in line:
                if char in [' ', '\t', '\b']:
                    blank = True
                else:
                    if blank:
                        line_token_count += 1
                    blank = False

            token_count += line_token_count
            if line_token_count > 0:
                non_empty_line_count += 1

    return {
        'file': file,
        'line_count': line_count,
        'line_token_count': token_count,
        'non_empty_line_count': non_empty_line_count
    }

if __name__ == '__main__':
    ret = count_file('count_file.py')
    print('行数：', ret['line_count'])
    print('非空行：', ret['non_empty_line_count'])
    print('非空词数：', ret['line_token_count'])
    with open('/tmp/count.json', 'w') as f:
        f.write(json.dumps(ret, indent=2, ensure_ascii=False))


import argparse
import xarray as xr
from .grid import make_grid_SG

if __name__ == '__main__':
    parser = argparse.ArgumentParser(
        description='Append grid-box corners to file')
    parser.add_argument('filein')
    parser.add_argument(
        '--sg_params', metavar='P', type=float, nargs=3,
        default=[1.0, 170.0, -90.0],
        help='input grid stretching parameters (stretch-factor, target longitude, target latitude)')
    args = parser.parse_args()

    ds = xr.open_dataset(args.filein)

    csgrid, csgrid_list = make_grid_SG(
        ds.dims['Ydim'],
        stretch_factor=args.sg_params[0],
        target_lon=args.sg_params[1],
        target_lat=args.sg_params[2]
    )

    ds = ds.assign_coords(
        {'corner_lons': xr.DataArray(
            csgrid['lon_b'],
            dims=['nf', 'YCdim', 'XCdim']),
         'corner_lats': xr.DataArray(
             csgrid['lat_b'],
             dims=['nf', 'YCdim', 'XCdim'])})
    ds.load()
    ds.close()
    ds.to_netcdf(args.filein)

    print(ds)


import pandas as pd
import matplotlib
import seaborn
from sklearn import svm

matplotlib.use('Agg')

if __name__ == "__main__":
    data = pd.read_csv(\
        './datasets/_f6284c13db83a3074c2b987f714f24f5_svm-data.csv',
        names=['target', 'A', 'B'])

    S = svm.SVC(C=100000, random_state=241, kernel='linear')
    svm_model = S.fit(data[['A', 'B']], data['target'])
#    f = open('./svm_result.txt', 'wt')
#    res = ''
#    tmp = svm_model.support_
#    tmp.sort()
#    for i in tmp:
#        res = res + str(i+1) + ','
#    res = res.strip('\n')
#    f.write(res)
#    f.close()
    with open('./svm_result.txt', 'w') as res_f:
        res = ''
        for i in svm_model.support_:
            res = res + str(i+1) + ','
        res = res.strip(',')
        res_f.write(res)

    plt = seaborn.regplot(data['A'], data['B'])
    plt.figure.savefig('./svm.png')


import logging
import serial
from Adafruit_IO import Client

logging.basicConfig(
    format=' %(levelname)s - %(asctime)s - %(message)s ', level=logging.INFO)

ser = serial.Serial('/dev/ttyUSB0')

def getPm25():
    data = []
    for index in range(0, 10):
        datum = ser.read()
        data.append(datum)
        pmtwofive = int.from_bytes(
            b''.join(data[2:4]), byteorder='little') / 10
        logging.info("PM 2.5:\t %s", pmtwofive)
    return pmtwofive

def getPm10():
    data = []
    for index in range(0, 10):
        datum = ser.read()
        data.append(datum)

        pmten = int.from_bytes(b''.join(data[4:6]), byteorder='little') / 10
        logging.info("PM 10:\t %s", pmten)

    return pmten


from __future__ import unicode_literals
import json
import pygame

import pyglet
from pygame import mixer, event, init

import youtube_dl

class MyLogger(object):
    def debug(self, msg):
        pass
    def warning(self, msg):
        pass
    def error(self, msg):
        print(msg)

def my_hook(d):
    if d['status'] == 'finished':
        print('Done downloading, now converting ...')

ydl_opts = {
    'outtmpl': 'C:\\Users\\ADMIN\\Desktop\\c4t\\Sesson2\\Sesson12\\finalhack\\%(title)s.%(ext)s',
    'default_search': 'ytsearch5',
    'format': 'bestaudio/best',
    'postprocessors': [{
        'key': 'FFmpegExtractAudio',
        'preferredcodec': 'mp3',
        'preferredquality': '192',
    }],
    'logger': MyLogger(),
    'progress_hooks': [my_hook],
}
from youtube_dl import YoutubeDL

ydl = YoutubeDL(ydl_opts)

print("Welcome to music app!")

while True:
    with open('C:\\Users\\ADMIN\\Desktop\\c4t\\Sesson2\\Sesson12\\finalhack\\database.json') as json_files:
        current_database = json.load(json_files)
    def songlist():
        if len(current_database) == 0:
            print("You don't have any song!")
        else:
            for i in range(len(current_database)):
                print(f"{i+1}. {current_database[i]['title']}")

    print("""Pick one of these options:
    1. Show all songs
    2. Show details of a song
    3. Play a song
    4. Search and download songs
    5. Exit:""")
    while True:
        selection = int(input(">>>"))
        if selection in [1,2,3,4,5]:
            break
        else:
            print('Invalid Command !!')

    if selection == 1 :
        songlist()

    elif selection == 2:
        songlist()
        song = int(input("Enter the number of the song you want to see detail: "))
        while True:
            if song in range(1, len(current_database)+1):
                break
            else:
                print("Invalid song!")
        for k, v in current_database[song-1].items():
            print(f'{k}: {v}')

    elif selection == 3:

        songlist()
        song = int(input("Which song do you want to hear? >>> "))

        mixer.init()
        mixer.music.load(f"""{current_database[song-1]['title']}.mp3""".encode('utf8'))

        mixer.music.play()

        while True:
            choose = input("Play/ Pause/ Stop ? >> ")
            if choose.upper() == "PLAY":
                mixer.music.unpause()
            elif choose.upper() == "PAUSE":
                mixer.music.pause()
            elif choose.upper() == "STOP":
                mixer.music.stop()
                break

    elif selection == 4:
        keyword = input('Enter the name of the song you want to search >>> ')
        search_result = ydl.extract_info(keyword, False)
        for i in range(5):
            print(f"""{i+1}. {search_result['entries'][i]["title"]}""")

        ask = input("Which song do you want to choose? (Enter a number or enter n to out the download!) ")
        while True:
            if ask.isdigit():
                if int(ask) in [1,2,3,4,5]:
                    with youtube_dl.YoutubeDL(ydl_opts) as ydl:
                        ydl.download([search_result['entries'][int(ask) - 1]['webpage_url']])
                    print("Your song have been downloaded")
                    new_data = {
                        'title': search_result['entries'][int(ask)-1]['title'],
                        'creator': search_result['entries'][int(ask)-1]['creator'],
                        'artist': search_result['entries'][int(ask)-1]['artist'],
                        'track': search_result['entries'][int(ask)-1]['track']
                    }
                    data = []
                    data.extend(current_database)
                    data.append(new_data)

                    with open('C:\\Users\\ADMIN\\Desktop\\c4t\\Sesson2\\Sesson12\\finalhack\\database.json', 'w') as outfile:
                        json.dump(data, outfile)
                    break
                else:
                    print("Invalid number! Enter the number again!")
            elif str(ask).upper() == "N":
                break
            else:
                print("")
                print("Invalid command !!!")

    elif selection == 5:
        print('See you later!!!')
        break


import copy
import logging

from .preprocess import Preprocess

LOG = logging.getLogger(__name__)

class MinSize(Preprocess):
    """Convert annotations below a size to crowd annotations."""

    def __init__(self, min_side=1.0):
        self.min_side = min_side

    def __call__(self, image, anns, meta):
        anns = copy.deepcopy(anns)
        for ann in anns:
            if ann['bbox'][2] > self.min_side \
               and ann['bbox'][3] > self.min_side:
                continue
            ann['iscrowd'] = True

        return image, anns, meta


import logging
import time
from pathlib import Path
from typing import Iterable, Set, Tuple, Dict, Sequence

import numpy as np

from quant.datamodel import TypedSpan, get_dataset_files, DatasetType, read_annotation, read_text, collect_categories
from quant.model import SpanNERModel
from quant.utils import invert

class Solution:

    model_path = Path('onnx/main.pkl')
    exclude_filenames = {
        '165459_text', '176167_text', '178485_text', '192238_text',
        '193267_text', '193946_text', '194112_text', '2021',
        '202294_text', '2031', '209438_text', '209731_text', '546860_text'
    }

    @classmethod
    def predict(cls, texts: Sequence[str]) -> Iterable[Set[Tuple[TypedSpan]]]:
        model = SpanNERModel.load(cls.model_path)
        return model.predict(list(texts))

    @classmethod
    def evaluate(cls):
        text_files, annotation_files = get_dataset_files(Path('data'), DatasetType.TEST, exclude_filenames=cls.exclude_filenames)
        test_categories = collect_categories(annotation_files)

        ground_truth = list(map(read_annotation, annotation_files))
        texts = list(map(read_text, text_files))

        model = SpanNERModel.load(cls.model_path)
        model_predictions = list(model.predict(texts))

        print(f'Computing metrics for {test_categories} ({len(test_categories)}/{len(model.category_mapping) - 1})')

        n_categories = len(test_categories)
        category_id_mapping = dict(enumerate(test_categories))
        category_mapping = invert(category_id_mapping)

        def group_by_category(entities: Set[TypedSpan]) -> Dict[str, Set[Tuple[int, int]]]:
            groups = {cat: set() for cat in test_categories}
            for entity in entities:
                groups[entity.type].add((entity.start, entity.end))
            return groups

        true_positives = np.zeros(n_categories, dtype=int)
        false_positives = np.zeros(n_categories, dtype=int)
        false_negatives = np.zeros(n_categories, dtype=int)

        for true_entities, predicted_entities in zip(ground_truth, model_predictions):
            true_groups = group_by_category(true_entities)
            predicted_groups = group_by_category(predicted_entities)

            for category in test_categories:
                category_id = category_mapping[category]
                true_set = true_groups[category]
                predicted_set = predicted_groups[category]

                true_positives[category_id] += len(true_set.intersection(predicted_set))
                false_positives[category_id] += len(predicted_set.difference(true_set))
                false_negatives[category_id] += len(true_set.difference(predicted_set))

        category_f1 = true_positives / (true_positives + (false_positives + false_negatives) / 2)
        for category_id, f1 in enumerate(category_f1):
            category = category_id_mapping[category_id]
            print(f'{category}: {f1 * 100:.2f}%')

        f1_macro = category_f1.mean()
        print(f'macro F1: {f1_macro * 100:.2f}%')

        return f1_macro

if __name__ == '__main__':
    logging.basicConfig(level=logging.INFO)
    start_time = time.time()
    Solution.evaluate()
    end_time = time.time()

    print(f'Test time: {end_time - start_time:.4f}s')


import numpy as np
import matplotlib.pyplot as plt
import Tools.general
from copy import deepcopy
from random import sample

class hyperParameters:
    def __init__(self):
        self.mean = np.array([])
        self.cov  = np.array([])
        self.lik  = np.array([])

def randperm(k):
    # return a random permutation of range(k)
    z = range(k)
    y = []
    ii = 0
    while z and ii < 2*k:
        n = sample(z,1)[0]
        y.append(n)
        z.remove(n)
        ii += 1
    return y

def plotter(xs,ym,ys2,x,y,axisvals=None,file=None):
    xss  = np.reshape(xs,(xs.shape[0],))
    ymm  = np.reshape(ym,(ym.shape[0],))
    ys22 = np.reshape(ys2,(ys2.shape[0],))
    plt.plot(xs, ym, 'g-', x, y, 'r+', linewidth = 3.0, markersize = 10.0)
    plt.fill_between(xss,ymm + 2.*np.sqrt(ys22), ymm - 2.*np.sqrt(ys22), facecolor=[0.,1.0,0.0,0.8],linewidths=0.0)
    plt.grid()
    if axisvals:
        plt.axis(axisvals)
    plt.xlabel('input x')
    plt.ylabel('output y')

    if file and isinstance(file,str):
        plt.savefig(file)
    plt.show()

def FITCplotter(u,xs,ym,ys2,x,y,axisvals=None,file=None):
    xss  = np.reshape(xs,(xs.shape[0],))
    ymm  = np.reshape(ym,(ym.shape[0],))
    ys22 = np.reshape(ys2,(ys2.shape[0],))
    plt.plot(xs, ym, 'g-', x, y, 'r+', linewidth = 3.0, markersize = 10.0)
    plt.fill_between(xss,ymm + 2.*np.sqrt(ys22), ymm - 2.*np.sqrt(ys22), facecolor=[0.,1.0,0.0,0.8],linewidths=0.0)
    plt.grid()
    if axisvals:
        plt.axis(axisvals)
    plt.xlabel('input x')
    plt.ylabel('output y')
    plt.plot(u,np.ones_like(u),'kx',markersize=12)
    if file and isinstance(file,str):
        plt.savefig(file)
    plt.show()

def convert_to_array(hyp):
    y = np.concatenate((np.concatenate((hyp.mean, hyp.cov),axis=0),hyp.lik),axis=0)
    return y

def convert_to_class(x,hyp):
    y = deepcopy(hyp)
    Lm = len(hyp.mean)
    Lc = len(hyp.cov)
    y.mean = x[:Lm]
    y.cov  = x[Lm:(Lm+Lc)]
    y.lik  = x[(Lm+Lc):]
    return y

def unique(x):
    # First flatten x
    y = [item for sublist in x for item in sublist]
    if isinstance(x,np.ndarray):
        n,D = x.shape
        assert(D == 1)
        y = np.array( list(set(x[:,0])) )
        y = np.reshape(y, (len(y),1))
    else:
        y = list(set(y))
    return y

def flatten(l, ltypes=(list, tuple)):
    ltype = type(l)
    l = list(l)
    i = 0
    while i < len(l):
        while isinstance(l[i], ltypes):
            if not l[i]:
                l.pop(i)
                i -= 1
                break
            else:
                l[i:i + 1] = l[i]
        i += 1
    return ltype(l)

def checkParameters(func,hyp,D):
    valt = flatten(Tools.general.feval(func))
    val = 0
    if isinstance(valt,list):
        for v in valt:
            if isinstance(v,str):
                val += eval(v)
            else:
                val += v
    else:
        val = valt
    res = (val == len(hyp))
    return res


# -*- coding: utf-8 -*-
#Pregnant women1 with COVID-19, United States, January 22, 2020 - October 25, 2021
import pandas as pd
import matplotlib.pyplot as plt
import numpy as np
from matplotlib.ticker import MaxNLocator

#import csv file
df  = pd.read_csv("cases_of_covid19_among_pregnant_women_by_week_of_diagnosis_download.csv")
df1 = pd.read_csv("pregnancy_data_download.csv")
df2 = pd.read_csv("Covid-Vaccine-among-prengnant.csv")
#graph1 ********
dateDiag = df.iloc[2:92,0]
numCase  = df.iloc[2:92,1]

fig,axes = plt.subplots(1,1)

plt.bar(dateDiag, numCase)
plt.xticks(rotation=45)
axes.xaxis.set_major_locator(MaxNLocator(23))
plt.xlabel('week of Diagnosis')
plt.ylabel('Number of cases')
plt.title('Cases of COVID-19 among Pregnant Women by Week of Diagnosis*')
plt.savefig('by Week of Diagnosis.png', dpi=300, bbox_inches='tight')
plt.show()
#*****************************************
age        = df1.iloc[5:14,0]
numAgeCase = df1.iloc[5:14,1]

plt.bar(age, numAgeCase)
plt.xticks(rotation=45)
plt.xlabel('Age Group')
plt.ylabel('Number of Cases')
plt.title('Pregnant women with COVID-19 by age, United States')
plt.savefig('Pregnant women by age.png', dpi=300, bbox_inches='tight')
plt.show()
#***************************************
Race    = df1.iloc[16:21,0]
numRace = df1.iloc[16:21,1]

plt.bar(Race, numRace)
plt.xticks(rotation=45)
plt.xlabel('Race / Ethnicity')
plt.ylabel('Number of Cases')
plt.title('Pregnant women with COVID-19 by race/ethnicity, United States')
plt.savefig('By raceEthnicity', dpi=300, bbox_inches='tight')
plt.show()
#**************************************
Hospital = df1.iloc[30:33,0]
numHos   = df1.iloc[30:33,1]

plt.bar(Hospital, numHos)
plt.xlabel('Hospitalization Status')
plt.ylabel('Number of Cases')
plt.title('Pregnant women with COVID-19 by Hospitalization Status, United States')
plt.savefig('Hospitalization Status.png', dpi=300, bbox_inches='tight')
plt.show()
#***************************Vaccine in prengnancy 14 Dec 2020 - 30 Oct 2021
prior = [[0]]*34
priorAndDuring = [[0]]*34
During = [[0]]*34
dateVaccine = [[0]]*34
CDCVaccineData1 = df2.iloc[0:102]
n = 0
a = 0
b = 0
for i in range(0,102,3):
    prior[n]       = (CDCVaccineData1.iloc[i,4])
    dateVaccine[n] = CDCVaccineData1.iloc[i,0]
    n = n+1
for i in range(1,102,3):
    priorAndDuring[a] = CDCVaccineData1.iloc[i,4]
    a = a+1
for i in range(2,102,3):
    During[b] = CDCVaccineData1.iloc[i,4]
    b = b+1

prior1 = np.array(prior)
priorAndDuring1 = np.array(priorAndDuring)
During1 = np.array(During)

plt.bar(dateVaccine, prior1)
plt.bar(dateVaccine, priorAndDuring1, bottom = prior1)
plt.bar(dateVaccine, During1, bottom = prior1 + priorAndDuring1)
plt.xticks(rotation=45, fontsize = 7)
plt.xlabel('Date')
plt.ylabel('Vaccination Coverage(%)')
plt.title('Percent of Pregnant People Aged 18-49 Years \n Fully Vaccinated Covid-19,United States')
plt.legend(['Prior to Pregnancy','Prior to and During Pregnancy','During Pregnancy'])
plt.savefig('FullyVaccinatedCovid-19.png', dpi=300, bbox_inches='tight')
plt.show()
#************************************


import sys
assert sys.version_info >= (3, 5) # make sure we have Python 3.5+
import json
from pyspark.sql import SparkSession, functions, types
from pyspark.sql.functions import col, length

# spark-submit data_join.py output_b output_r output_u output_join

business_schema = types.StructType([
types.StructField('business_star_rating', types.DoubleType()),
types.StructField('business_id', types.StringType()),
types.StructField('categories', types.StringType())
])

review_schema = types.StructType([
types.StructField('review_star_rating', types.DoubleType()),
types.StructField('user_id', types.StringType()),
types.StructField('business_id', types.StringType()),
types.StructField('review_id', types.StringType()),
types.StructField('length_of_review_text', types.IntegerType()),
types.StructField('review_text', types.StringType())
])

user_schema = types.StructType([
types.StructField('review_count', types.IntegerType()),
types.StructField('average_user_rating', types.DoubleType()),
types.StructField('user_id', types.StringType())
])

def main(inp1,inp2,inp3, output):

    df_b = spark.read.csv(inp1, schema=business_schema)
    df_b.printSchema()
    df_b.createOrReplaceTempView('df_b')

    df_r = spark.read.csv(inp2, schema=review_schema)
    df_r.printSchema()
    df_r.createOrReplaceTempView('df_r')

    df_u = spark.read.csv(inp3, schema=user_schema)
    df_u.printSchema()
    df_u.createOrReplaceTempView('df_u')

    joined_data = spark.sql("""SELECT df_r.user_id,df_u.review_count, average_user_rating,business_star_rating, length_of_review_text,categories,\
                            review_star_rating, review_id,df_b.business_id,review_text from df_u inner join df_r on df_r.user_id=df_u.user_id inner join df_b \
                            on df_r.business_id=df_b.business_id""")
    joined_data.write.csv(output, mode='overwrite')

if __name__ == '__main__':
    inp1 = sys.argv[1]
    inp2 = sys.argv[2]
    inp3 = sys.argv[3]
    output = sys.argv[4]
    spark = SparkSession.builder.appName('BD Project').getOrCreate()
    assert spark.version >= '3.0' # make sure we have Spark 3.0+
    spark.sparkContext.setLogLevel('WARN')
    sc = spark.sparkContext
    main(inp1,inp2,inp3,output)


"""
coding_schedule.py
Input: Current weekday, derived from the datetime package within Python
 (called on line 14).
Output: A string that states the present day and declares whether that
day is a programming day or a day off.

"""

import datetime as dt

# variable defined as accurate, present weekday taken from python datetime
todays_day = dt.datetime.weekday(dt.datetime.now())

def convert_to_pronoun(day):
	"""Take input "day" as a datatype integer, and convert to datatype string"""
	if day == 0:
		return "Monday"
	if day == 1:
		return "Tuesday"
	if day == 2:
		return "Wednesday"
	if day == 3:
		return "Thursday"
	if day == 4:
		return "Friday"
	if day == 5:
		return "Saturday"
	if day == 6:
		return "Sunday"

# If/Else statement utilizing boolean variable to determine if present
# weekday is a coding day or not. Returns string.
if todays_day == 0 or 1 or 2 or 3 or 4 or 5:
	print("Today is " + convert_to_pronoun(todays_day) + \
		" and you should program!")
else:
	print("Today is Sunday and you have the day off!")


import click
from achilles.model import AchillesModel
from achilles.utils import get_dataset_dim

@click.option(
    "--gpu",
    "-g",
    metavar="",
    default=None,
    required=False,
    show_default=True,
    help="SET CUDA_VISIBLE_DEVICES to train model on specific" " GPU (e.g. 0 or 0,1)",
)
@click.option(
    "--gpus",
    metavar="",
    default=1,
    required=False,
    show_default=True,
    help="Build the model for distributed training across multiple GPUs",
)
@click.option(
    "--threads",
    "-t",
    metavar="",
    default=2,
    required=False,
    show_default=True,
    help="Feed batches into training function using multiple processes",
)
@click.option(
    "--batch_size",
    "-b",
    metavar="",
    default=200,
    required=False,
    show_default=True,
    help="Batch size for training, major determinant of RAM used on GPU",
)
@click.option(
    "--epochs",
    "-e",
    metavar="",
    default=100,
    required=False,
    show_default=True,
    help="Number of epochs to train model for",
)
@click.option(
    "--loss_function",
    "--loss",
    metavar="",
    default="binary_crossentropy",
    required=False,
    show_default=True,
    help="Compile model with loss function for training",
)
@click.option(
    "--optimizer",
    metavar="",
    default="adam",
    required=False,
    show_default=True,
    help="Compile model with optimizer for training",
)
@click.option(
    "--recurrent_dropout",
    metavar="",
    default=0.2,
    required=False,
    show_default=True,
    help="Internal dropout applied to LSTM layers",
)
@click.option(
    "--dropout",
    metavar="",
    default=0.2,
    required=False,
    show_default=True,
    help="Dropout applied to LSTM layers",
)
@click.option(
    "--bidirectional",
    metavar="",
    is_flag=True,
    required=False,
    show_default=True,
    help="Bidirectional LSTM",
)
@click.option(
    "--units",
    metavar="",
    default=200,
    required=False,
    show_default=True,
    help="Number of units per LSTMs",
)
@click.option(
    "--channels",
    metavar="",
    default=256,
    required=False,
    show_default=True,
    help="Number channels per Residual Block",
)
@click.option(
    "--lstm",
    metavar="",
    default=1,
    required=False,
    show_default=True,
    help="Number of stacked LSTMs connected to Residual Blocks",
)
@click.option(
    "--residual_block",
    metavar="",
    default=1,
    required=False,
    show_default=True,
    help="Number of stacked ResidualBlocks in initial layers",
)
@click.option(
    "--activation",
    "-a",
    metavar="",
    default="softmax",
    required=False,
    show_default=True,
    help="Activation function applied to final fully connected " "classification layer",
)
@click.option(
    "--verbose",
    "-v",
    metavar="",
    is_flag=True,
    required=False,
    show_default=True,
    help="Show training progress output and model architecture in Keras",
)
@click.option(
    "--load",
    "-l",
    metavar="",
    default="",
    required=False,
    show_default=True,
    help="Trained model weights from Keras, HDF5 to"
    " continue training, or re-train model",
)
@click.option(
    "--outdir",
    "-o",
    metavar="",
    default="training_model",
    required=True,
    show_default=True,
    help="Output directory",
)
@click.option(
    "--run_id",
    "-i",
    metavar="",
    default="model",
    required=True,
    show_default=True,
    help="Training run ID",
)
@click.option(
    "--file",
    "-f",
    metavar="",
    default=None,
    required=True,
    show_default=True,
    help="Input training / validation HDF5 dataset",
)
@click.command()
def train(
    file,
    run_id,
    outdir,
    load,
    verbose,
    activation,
    residual_block,
    lstm,
    channels,
    units,
    bidirectional,
    dropout,
    optimizer,
    loss_function,
    epochs,
    batch_size,
    threads,
    gpus,
    gpu,
):
    """Train neural network classifiers in Achilles"""

    achilles = AchillesModel(data_file=file)

    window_size = get_dataset_dim(file)[2]  # 4D tensors

    if load:
        achilles.load_model(load)
    else:
        achilles.build(
            window_size=window_size,
            activation=activation,
            gpus=gpus,
            nb_residual_block=residual_block,
            nb_channels=channels,
            nb_rnn=lstm,
            rnn_units=units,
            dropout=dropout,
            bidirectional=bidirectional,
        )

        # Compile model with loss function and optimizer
        achilles.compile(optimizer=optimizer, loss=loss_function)

    achilles.train(
        epochs=epochs,
        batch_size=batch_size,
        workers=threads,
        run_id=run_id,
        outdir=outdir,
        verbose=verbose,
        gpu=gpu,
    )