## @package Server
#  Module server
import os
import socket
import threading
import sys
import configparser
import re
import Log

#   Code retour
#   INFO
USERLIST_ENABLE = 300
USERLIST_DISABLE = 301
HAS_JOIN = 302
HAS_LEFT = 303
NEW_MSG = 304
NAME_CHANGED = 305
NEW_PM = 306
ASKING_FOR_PM = 307
PRIVATE_DISCU_ACCEPTED_FROM = 308
PRIVATE_DISCU_REFUSED_FROM = 309
IS_NOW_ENABLE = 310
IS_NOW_DISABLE = 311
HAS_ASKED_FILE = 312
CAN_SEND_FILE = 313
HAS_REJECT_FILE = 314

#   SUCCESS
SUCC_CHANNEL_JOINED = 200
SUCC_CHANNEL_QUIT = 201
SUCC_MESSAGE_SENDED = 202
SUCC_NICKNAME_CHANGED = 203
SUCC_PM_SENDED = 205
SUCCESSFUL_ASKED_CONV = 206
SUCCESSFUL_ACCEPTED_CONV = 207
SUCCESSFUL_REFUSED_CONV = 208
SUCC_ENABLED = 209
SUCC_DISABLED = 210
SUCC_PMFILE = 211
SUCC_ACCEPTED_FILE = 212
SUCC_REFUSED_FILE = 213

#    ERROR
ERR_NICKNAME_ALREADY_USED = 400
ERR_NO_NICKNAME = 401
ERR_CONV_NOT_ALLOWED = 402
DEST_NOT_FOUND = 403
ERR_ALREADY_ASKED_FOR_PM = 404
ERR_NO_INVIT_TO_CONV_FOUND = 405
ERR_UNKNOWN_ACCEPTED_FILE = 406
COMMAND_NOT_FOUND = 407
ERR_INVALID_NICKNAME = 408
ERR_INTERNAL_SERVER_ERROR = 409
ERR_NOT_DISABLED = 410
ERR_NOT_ENABLED = 411

##
#   Load Configuration and Start the Server
def main():
    # Global vars
    global usersConnected, log, sock
    global askPM, validatePM
    global askFT
    usersConnected = {}
    askPM = []
    validatePM = []
    askFT = []

    # Config
    config = configparser.ConfigParser()
    if not os.path.isfile("dncServer.conf"):
        config['NETWORK'] = {'port': '2222'}
        config['LOG'] = {'logDirectory': 'log'}
        with open('dncServer.conf', 'w') as configfile:
            config.write(configfile)
    config.read("dncServer.conf")
    log = Log.Log(config["LOG"]["logdirectory"])
    log.printL("Configuration Load", Log.lvl.INFO)
    log.printL("Server start", Log.lvl.INFO)

    #Init socket serv
    sock = socket.socket()
    sock.bind(("", int(config["NETWORK"]["port"])))
    sock.listen(5)
    log.printL("Server Listen on port {}".format(config["NETWORK"]["port"]), Log.lvl.INFO)

    try:
        while True:
            #Connection client
            connection, client_address = sock.accept()
            usersConnected[connection] = [client_address, None, True]  # (ip,port) pseudo status
            threading.Thread(target=handle_connection, args=(connection, client_address)).start()
    except KeyboardInterrupt:
        # Disable to received more requests on socket
        for con, value in usersConnected.items():
            con.shutdown(socket.SHUT_RD)
    finally:
        #Wait for threads finish
        log.printL("Wait for threads ending", Log.lvl.INFO)
        for t in threading.enumerate():
            if t != threading.main_thread():
                t.join()
        sock.close()
        log.printL("Server shutdown", Log.lvl.INFO)
        sys.exit(0)

##
#   Handle a connection from a client.
#   Wait for request from the client
#   @param connection the socket descriptor of the connection
#   @param client_adress ("ip", port) of the connection
def handle_connection(connection, client_address):
    try:
        log.printL("Connection from IP -> {}".format(client_address), Log.lvl.INFO)
        while True:
            data = connection.recv(4096)
            if data:
                log.printL("Request from IP -> {}"
                           " {}".format(client_address, data.decode()), Log.lvl.INFO)
                threading.Thread(target=handle_request, args=(connection, data.decode())).start()
            else:
                break
    except Exception as e:
        log.printL("Handle connection fail : ".format(str(e)), Log.lvl.FAIL)
    finally:
        quit_user(connection)

##
#   Handle a request.
#   @param connection the socket descriptor of the request sender
#   @param data the request to handle in String
def handle_request(connection, data):
    try:
        array_data = data.split(" ")

        ### Command for user with nickname ###
        if usersConnected[connection][1] is not None:
            ### No command -> new message ###
            if not array_data[0][0] == "/" and usersConnected[connection][2]:
                connection.sendall("{}".format(SUCC_MESSAGE_SENDED).encode())
                broadcast_message(connection, "{} {} {} ".format(NEW_MSG, usersConnected[connection][1], data))
                return
            else:
                ### Command for user enable & disable ###
                if array_data[0] == "/name":
                    change_name(connection, array_data[1])
                    return
                if array_data[0] == "/userlist":
                    user_list_active(connection)
                    return
                if array_data[0] == "/userlistaway":
                    user_list_away(connection)
                    return
                if array_data[0] == "/enable":
                    enable_user(connection)
                    return
                if array_data[0] == "/disable":
                    disable_user(connection)
                    return
                if array_data[0] == "/quit":
                    connection.shutdown(socket.SHUT_RD)
                    return

                ### Command available for enable only ###
                if not usersConnected[connection][2]:
                    connection.sendall("{}".format(ERR_CONV_NOT_ALLOWED).encode())
                    return
                else:
                    if array_data[0] == "/askpm":
                        ask_private_message(connection, array_data[1])
                        return
                    if array_data[0] == "/acceptpm":
                        accept_private_message(connection, array_data[1])
                        return
                    if array_data[0] == "/rejectpm":
                        reject_private_message(connection, array_data[1])
                        return
                    if array_data[0] == "/pm":
                        private_message(connection, array_data[1], " ".join(array_data[2:]).strip())
                        return
                    if array_data[0] == "/pmfile":
                        ask_file(connection, array_data[1], " ".join(array_data[2:]).strip())
                        return
                    if array_data[0] == "/acceptfile":
                        accept_file(connection, array_data[1], " ".join(array_data[3:]).strip(), array_data[2])
                        return
                    if array_data[0] == "/rejectfile":
                        reject_file(connection, array_data[1], " ".join(array_data[2:]).strip())
                        return
            connection.sendall("{}".format(COMMAND_NOT_FOUND).encode())
        else:
            ### Command for user without nickname ###
            if array_data[0] == "/newname":
                new_name(connection, array_data[1])
                return
            if array_data[0] == "/quit":
                connection.shutdown(socket.SHUT_RD)
                return
            connection.sendall("{}".format(ERR_NO_NICKNAME).encode())
    except IndexError:
        log.printL("Parameter missing in the request", Log.lvl.WARNING)
        connection.sendall("{}".format(COMMAND_NOT_FOUND).encode())
    except Exception as e:
        log.printL("Handle request fail : {}".format(str(e)), Log.lvl.FAIL)
        connection.sendall("{}".format(ERR_INTERNAL_SERVER_ERROR).encode())

##
#   Broadcast a message to all the users connected except to the sender of the request
#   @param connection the socket descriptor of the request sender
#   @param message message to broadcast (String)
def broadcast_message(connection, message):
    #log.printL("User Connected : {}".format(usersConnected), Log.lvl.DEBUG)
    for con, value in usersConnected.items():
        # value 1 : pseudo value 2 : status (enable/disable)
        if value[1] is not None and con != connection and value[2]:
            try:
                con.sendall(message.encode())
            except Exception as e:
                log.printL(str(e), Log.lvl.FAIL)
        log.printL("Broadcast : {}".format(message), Log.lvl.INFO)

##
#   Send the list of enable user
#   @param connection the socket descriptor of the target
def user_list_active(connection):
    l = "{} ".format(USERLIST_ENABLE)
    for con, value in usersConnected.items():
        if value[1] is not None and value[2]:
            l += value[1] + " "
    connection.sendall(l[:-1].encode())
    log.printL("Send to {} : {}".format(usersConnected[connection][0], l[:-1]), Log.lvl.INFO)

##
#   Send the list of disable user
#   @param connection the socket descriptor of the target
def user_list_away(connection):
    l = "{} ".format(USERLIST_DISABLE)
    for con, value in usersConnected.items():
        if value[1] is not None and not value[2]:
            l += value[1] + " "
    connection.sendall(l[:-1].encode())
    log.printL("Send to {} : {}".format(usersConnected[connection][0], l[:-1]), Log.lvl.INFO)

##
#   Change the nickname of the user
#   @param connection the socket descriptor of the target
#   @param pseudo new nickname for the user (String)
def change_name(connection, pseudo):
    if not re.match("^\w{3,15}$", pseudo):
        connection.sendall("{}".format(ERR_INVALID_NICKNAME).encode())
        log.printL("Send to {} : {}".format(usersConnected[connection][0],
                                            ERR_INVALID_NICKNAME), Log.lvl.INFO)
    elif get_connection_by_pseudo(pseudo) is not None:
        connection.sendall("{}".format(ERR_NICKNAME_ALREADY_USED).encode())
        log.printL("Send to {} : {}".format(usersConnected[connection][0],
                                            ERR_NICKNAME_ALREADY_USED), Log.lvl.INFO)
    else:
        broadcast_message(connection, "{} {} {}".format(NAME_CHANGED, usersConnected[connection][1], pseudo))
        connection.sendall("{}".format(SUCC_NICKNAME_CHANGED).encode())
        log.printL("Send to {} : {}".format(usersConnected[connection][0],
                                            SUCC_NICKNAME_CHANGED), Log.lvl.INFO)
        usersConnected[connection][1] = pseudo

##
#   Affect the nickname of the user for the first time
#   @param connection the socket descriptor of the target
#   @param pseudo nickname for the user (String)
def new_name(connection, pseudo):
    if not re.match("^\w{3,15}$", pseudo):
        connection.sendall("{}".format(ERR_INVALID_NICKNAME).encode())
        log.printL("Send to {} : {}".format(usersConnected[connection][0],
                                            ERR_INVALID_NICKNAME), Log.lvl.INFO)
    elif get_connection_by_pseudo(pseudo) is not None:
        connection.sendall("{}".format(ERR_NICKNAME_ALREADY_USED).encode())
        log.printL("Send to {} : {}".format(usersConnected[connection][0],
                                            ERR_NICKNAME_ALREADY_USED), Log.lvl.INFO)
    else:
        broadcast_message(connection, "{} {} ".format(HAS_JOIN, pseudo))
        connection.sendall("{}".format(SUCC_CHANNEL_JOINED).encode())
        log.printL("Send to {} : {}".format(usersConnected[connection][0],
                                            SUCC_CHANNEL_JOINED), Log.lvl.INFO)
        usersConnected[connection][1] = pseudo

##
#   Ask for a private discussion between the sender of the request and the pseudo
#   @param connection the socket descriptor of the sender
#   @param pseudo the pseudo of target of the demand
def ask_private_message(connection, pseudo):
    c = get_connection_by_pseudo(pseudo)
    if c is None:
        connection.sendall("{}".format(DEST_NOT_FOUND).encode())
        log.printL("Send to {} : {}".format(usersConnected[connection][0],
                                            DEST_NOT_FOUND), Log.lvl.INFO)
    else:
        pm = (connection, c)
        if pm in askPM:
            connection.sendall("{}".format(ERR_ALREADY_ASKED_FOR_PM).encode())
            log.printL("Send to {} : {}".format(usersConnected[connection][0],
                                                ERR_ALREADY_ASKED_FOR_PM), Log.lvl.INFO)
        else:
            askPM.append(pm)
            #log.printL("askPm {}".format(askPM), Log.lvl.DEBUG)
            c.sendall("{} {}".format(ASKING_FOR_PM, usersConnected[connection][1]).encode())
            log.printL("Send to {} : {} {}".format(usersConnected[c][0], ASKING_FOR_PM,
                                                   usersConnected[connection][1]), Log.lvl.INFO)
            connection.sendall("{}".format(SUCCESSFUL_ASKED_CONV).encode())
            log.printL("Send to {} : {}".format(usersConnected[connection][0],
                                                SUCCESSFUL_ASKED_CONV), Log.lvl.INFO)

##
#   Accept a private discussion
#   @param connection the socket descriptor of the person who accept the private discussion
#   @param pseudo the pseudo of the person who asked for a private discussion
def accept_private_message(connection, pseudo):
    #log.printL("askPm {}".format(askPM), Log.lvl.DEBUG)
    c = get_connection_by_pseudo(pseudo)
    if c is None:
        connection.sendall("{}".format(DEST_NOT_FOUND).encode())
        log.printL("Send to {} : {}".format(usersConnected[connection][0],
                                            DEST_NOT_FOUND), Log.lvl.INFO)
    else:
        pm = (c, connection)
        if pm not in askPM:
            connection.sendall("{}".format(ERR_NO_INVIT_TO_CONV_FOUND).encode())
            log.printL("Send to {} : {}".format(usersConnected[connection][0],
                                                ERR_NO_INVIT_TO_CONV_FOUND), Log.lvl.INFO)
        else:
            askPM.remove(pm)
            validatePM.append(pm)
            connection.sendall("{}".format(SUCCESSFUL_ACCEPTED_CONV).encode())
            log.printL("Send to {} : {}".format(usersConnected[connection][0],
                                                SUCCESSFUL_ACCEPTED_CONV), Log.lvl.INFO)
            c.sendall("{} {}".format(PRIVATE_DISCU_ACCEPTED_FROM, usersConnected[connection][1]).encode())
            log.printL("Send to {} : {} {}".format(usersConnected[c][0], PRIVATE_DISCU_ACCEPTED_FROM,
                                                   usersConnected[connection][1]), Log.lvl.INFO)

##
#   Reject a private discussion
#   @param connection the socket descriptor of the person who reject private discussion
#   @param pseudo the pseudo of the person who asked for a private discussion
def reject_private_message(connection, pseudo):
    c = get_connection_by_pseudo(pseudo)
    if c is None:
        connection.sendall("{}".format(DEST_NOT_FOUND).encode())
        log.printL("Send to {} : {}".format(usersConnected[connection][0],
                                            DEST_NOT_FOUND), Log.lvl.INFO)
    else:
        pm = (c, connection)
        pmr = (connection, c)
        if pm not in askPM:
            if pm in validatePM:
                validatePM.remove(pm)
                connection.sendall("{}".format(SUCCESSFUL_REFUSED_CONV).encode())
                log.printL("Send to {} : {}".format(usersConnected[connection][0],
                                                    SUCCESSFUL_REFUSED_CONV), Log.lvl.INFO)
                c.sendall("{} {}".format(PRIVATE_DISCU_REFUSED_FROM, usersConnected[connection][1]).encode())
                log.printL("Send to {} : {} {}".format(usersConnected[c][0], PRIVATE_DISCU_REFUSED_FROM,
                                                       usersConnected[connection][1]), Log.lvl.INFO)
            elif pmr in validatePM:
                validatePM.remove(pmr)
                connection.sendall("{}".format(SUCCESSFUL_REFUSED_CONV).encode())
                log.printL("Send to {} : {}".format(usersConnected[connection][0],
                                                    SUCCESSFUL_REFUSED_CONV), Log.lvl.INFO)
                c.sendall("{} {}".format(PRIVATE_DISCU_REFUSED_FROM, usersConnected[connection][1]).encode())
                log.printL("Send to {} : {} {}".format(usersConnected[c][0], PRIVATE_DISCU_REFUSED_FROM,
                                                       usersConnected[connection][1]), Log.lvl.INFO)
            else:
                connection.sendall("{}".format(ERR_NO_INVIT_TO_CONV_FOUND).encode())
                log.printL("Send to {} : {}".format(usersConnected[connection][0],
                                                    ERR_NO_INVIT_TO_CONV_FOUND), Log.lvl.INFO)
        else:
            askPM.remove(pm)
            connection.sendall("{}".format(SUCCESSFUL_REFUSED_CONV).encode())
            log.printL("Send to {} : {}".format(usersConnected[connection][0],
                                                SUCCESSFUL_REFUSED_CONV), Log.lvl.INFO)
            c.sendall("{} {}".format(PRIVATE_DISCU_REFUSED_FROM, usersConnected[connection][1]).encode())
            log.printL("Send to {} : {} {}".format(usersConnected[c][0], PRIVATE_DISCU_REFUSED_FROM,
                                                   usersConnected[connection][1]), Log.lvl.INFO)

##
#   Send a private message if a private discussion had been accepted
#   @param connection the soccket descriptor of the sender
#   @param pseudo the pseudo of the private message target
#   @param msg the message to send
def private_message(connection, pseudo, msg):
    c = get_connection_by_pseudo(pseudo)
    if c is None:
        connection.sendall("{}".format(DEST_NOT_FOUND).encode())
        log.printL("Send to {} : {}".format(usersConnected[connection][0],
                                            DEST_NOT_FOUND), Log.lvl.INFO)
    else:
        pm = (connection, c)
        pmr = (c, connection)
        if pm not in validatePM and pmr not in validatePM:
            connection.sendall("{}".format(ERR_CONV_NOT_ALLOWED).encode())
            log.printL("Send to {} : {}".format(usersConnected[connection][0],
                                                ERR_CONV_NOT_ALLOWED), Log.lvl.INFO)
        else:
            c.sendall("{} {} {}".format(NEW_PM, usersConnected[connection][1], msg).encode())
            log.printL("Send to {} : {} {} {}".format(usersConnected[c][0], NEW_PM,
                                                      usersConnected[connection][1], msg), Log.lvl.INFO)
            connection.sendall("{}".format(SUCC_PM_SENDED).encode())
            log.printL("Send to {} : {}".format(usersConnected[connection][0],
                                                SUCC_PM_SENDED), Log.lvl.INFO)

##
#   Ask for a file transfer between the sender of the request and the pseudo
#   @param connection the socket descriptor of the sender
#   @param pseudo the pseudo of target of the demand
def ask_file(connection, pseudo, file):
    c = get_connection_by_pseudo(pseudo)
    if c is None:
        connection.sendall("{}".format(DEST_NOT_FOUND).encode())
        log.printL("Send to {} : {}".format(usersConnected[connection][0],
                                            DEST_NOT_FOUND), Log.lvl.INFO)
    else:
        f = (connection, c, file)
        if f in askFT:
            connection.sendall("{}".format(ERR_ALREADY_ASKED_FOR_PM).encode())
            log.printL("Send to {} : {}".format(usersConnected[connection][0],
                                                ERR_ALREADY_ASKED_FOR_PM), Log.lvl.INFO)
        else:
            askFT.append(f)
            log.printL("askFT {}".format(askFT), Log.lvl.DEBUG)
            c.sendall("{} {} {}".format(HAS_ASKED_FILE, usersConnected[connection][1], file).encode())
            log.printL("Send to {} : {} {} {}".format(usersConnected[c][0], HAS_ASKED_FILE,
                                                      usersConnected[connection][1], file), Log.lvl.INFO)
            connection.sendall("{}".format(SUCC_PMFILE).encode())
            log.printL("Send to {} : {}".format(usersConnected[connection][0],
                                                SUCC_PMFILE), Log.lvl.INFO)

##
#   Accept a file transfer
#   @param connection the socket descriptor of the person who accept a file transfer
#   @param pseudo the pseudo of the person who asked for a file transfer
def accept_file(connection, pseudo, file, port):
    #log.printL("askFT {}".format(askFT), Log.lvl.DEBUG)
    c = get_connection_by_pseudo(pseudo)
    if c is None:
        connection.sendall("{}".format(DEST_NOT_FOUND).encode())
        log.printL("Send to {} : {}".format(usersConnected[connection][0],
                                            DEST_NOT_FOUND), Log.lvl.INFO)
    else:
        f = (c, connection, file)
        if f not in askFT:
            connection.sendall("{}".format(ERR_UNKNOWN_ACCEPTED_FILE).encode())
            log.printL("Send to {} : {}".format(usersConnected[connection][0],
                                                ERR_UNKNOWN_ACCEPTED_FILE), Log.lvl.INFO)
        else:
            askFT.remove(f)
            connection.sendall("{} {}".format(SUCC_ACCEPTED_FILE, usersConnected[c][0][0]).encode())
            log.printL("Send to {} : {}".format(usersConnected[connection][0],
                                                SUCC_ACCEPTED_FILE), Log.lvl.INFO)
            c.sendall("{} {} {} {} {}".format(CAN_SEND_FILE, pseudo, usersConnected[connection][0][0],
                                              port, file).encode())
            log.printL("Send to {} : {} {} {} {} {}".format(usersConnected[c][0], CAN_SEND_FILE, pseudo,
                                                            usersConnected[connection][0][0], port,
                                                            file), Log.lvl.INFO)

##
#   Reject a file transfer
#   @param connection the socket descriptor of the person who reject a file transfer
#   @param pseudo the pseudo of the person who asked for a file transfer
def reject_file(connection, pseudo, file):
    c = get_connection_by_pseudo(pseudo)
    if c is None:
        connection.sendall("{}".format(DEST_NOT_FOUND).encode())
        log.printL("Send to {} : {}".format(usersConnected[connection][0],
                                            DEST_NOT_FOUND), Log.lvl.INFO)
    else:
        f = (c, connection, file)
        if f not in askFT:
            connection.sendall("{}".format(ERR_UNKNOWN_ACCEPTED_FILE).encode())
            log.printL("Send to {} : {}".format(usersConnected[connection][0],
                                                ERR_UNKNOWN_ACCEPTED_FILE), Log.lvl.INFO)
        else:
            askPM.remove(f)
            connection.sendall("{}".format(SUCC_REFUSED_FILE).encode())
            log.printL("Send to {} : {}".format(usersConnected[connection][0],
                                                SUCC_REFUSED_FILE), Log.lvl.INFO)
            c.sendall("{} {} {}".format(HAS_REJECT_FILE, pseudo, file).encode())
            log.printL("Send to {} : {} {} {}".format(usersConnected[c][0], HAS_REJECT_FILE,
                                                      pseudo, file), Log.lvl.INFO)

##
#   Enable user
#   @param connection the socket descriptor of the person to enable
def enable_user(connection):
    if not usersConnected[connection][2]:
        usersConnected[connection][2] = True
        connection.sendall("{}".format(SUCC_ENABLED).encode())
        log.printL("Send to {} : {}".format(usersConnected[connection][0],
                                            SUCC_ENABLED), Log.lvl.INFO)
        broadcast_message(connection, "{} {}".format(IS_NOW_ENABLE, usersConnected[connection][1]))
    else:
        connection.sendall("{}".format(ERR_NOT_DISABLED).encode())
        log.printL("Send to {} : {}".format(usersConnected[connection][0],
                                            ERR_NOT_DISABLED), Log.lvl.INFO)

##
#   Disable user
#   @param connection the socket descriptor of the person to disable
def disable_user(connection):
    if usersConnected[connection][2]:
        usersConnected[connection][2] = False
        connection.sendall("{}".format(SUCC_DISABLED).encode())
        log.printL("Send to {} : {}".format(usersConnected[connection][0],
                                            SUCC_DISABLED), Log.lvl.INFO)
        broadcast_message(connection, "{} {}".format(IS_NOW_DISABLE, usersConnected[connection][1]))
    else:
        connection.sendall("{}".format(ERR_NOT_ENABLED).encode())
        log.printL("Send to {} : {}".format(usersConnected[connection][0],
                                            ERR_NOT_ENABLED), Log.lvl.INFO)

##
#   Disconnect user
#   @param connection the socket descriptor of the person to disconnect
def quit_user(connection):
    try:
        connection.sendall("{}".format(SUCC_CHANNEL_QUIT).encode())
        log.printL("Send to {} : {}".format(usersConnected[connection][0],
                                            SUCC_CHANNEL_QUIT), Log.lvl.INFO)
    except OSError:  # Client close the socket in this side not properly
        log.printL("Client IP -> {} close connection not properly"
                   "".format(usersConnected[connection][0]), Log.lvl.WARNING)
    connection.close()
    log.printL("Disconnection from IP -> {}".format(usersConnected[connection][0]), Log.lvl.INFO)
    pseudo = usersConnected[connection][1]
    usersConnected.pop(connection)
    broadcast_message(connection, "{} {}".format(HAS_LEFT, pseudo))

##
#   Get the socket descriptor by a pseudo
#   @param pseudo pseudo
#   @return the socket descriptor of the pseudo or None
def get_connection_by_pseudo(pseudo):
    for con, value in usersConnected.items():
        if value[1] == pseudo:
            return con
    return None


from new_kNN import *
import pandas as pd
import numpy as np

def accuracy(predictions, outcomes):
    return 100*np.mean(predictions==outcomes)

data =pd.read_csv("wine.csv")

#print(data.head(5))
numeric_data = data.drop("color",axis=1)

import sklearn.preprocessing as sp
scaled_data = sp.scale(numeric_data)
numeric_data =pd.DataFrame(scaled_data,columns=numeric_data.columns)

import sklearn.decomposition as sd
pca = sd.PCA(n_components=2)
#principal_components = pca.fit_transform(numeric_data)
principal_components=pca.fit(numeric_data).transform(numeric_data)

import matplotlib.pyplot as plt
from matplotlib.colors import ListedColormap
observation_colormap = ListedColormap(['red', 'blue'])
x = principal_components[:,0]
y = principal_components[:,1]

plt.title("Principal Components of Wine")
plt.scatter(x, y, alpha = 0.2,
    c = data['high_quality'], cmap = observation_colormap, edgecolors = 'none')
plt.xlim(-8, 8); plt.ylim(-8, 8)
plt.xlabel("Component 1"); plt.ylabel("Component 2")
plt.show()

predictors = np.array(numeric_data)
outcomes = np.array(data["high_quality"])

my_predictions = np.array([knn_predict(p,predictors,outcomes,k=5) for p in predictors])
percentage = accuracy(my_predictions,data.high_quality)
print(percentage)

"""
from sklearn.neighbors import KNeighborsClassifier
knn=KNeighborsClassifier(n_neighbors=3001)
knn.fit(predictors,outcomes)
scikit_prediction=knn.predict(predictors)
percentage = accuracy(scikit_prediction,data.high_quality)
print(percentage)
"""


import torch
import torch.nn as nn
from mmcv.cnn import ConvModule

from mmseg.models.decode_heads.decode_head import BaseDecodeHead
from mmseg.registry import MODELS

class UpBlock(nn.Module):
    """Upsample Block with two consecutive convolution layers."""

    def __init__(self, in_channels, out_channels, guidance_channels):
        super().__init__()
        self.up = nn.ConvTranspose2d(
            in_channels,
            in_channels - guidance_channels,
            kernel_size=2,
            stride=2)
        self.conv1 = ConvModule(
            in_channels,
            out_channels,
            3,
            padding=1,
            bias=False,
            norm_cfg=dict(type='GN', num_groups=out_channels // 16))
        self.conv2 = ConvModule(
            out_channels,
            out_channels,
            3,
            padding=1,
            bias=False,
            norm_cfg=dict(type='GN', num_groups=out_channels // 16))

    def forward(self, x, guidance=None):
        """Forward function with visual guidance."""
        x = self.up(x)
        if guidance is not None:
            T = x.size(0) // guidance.size(0)
            # guidance = repeat(guidance, "B C H W -> (B T) C H W", T=T)
            guidance = guidance.repeat(T, 1, 1, 1)
            x = torch.cat([x, guidance], dim=1)
        x = self.conv1(x)

        return self.conv2(x)

@MODELS.register_module()
class CATSegHead(BaseDecodeHead):
    """CATSeg Head.

    This segmentation head is the mmseg implementation of
    `CAT-Seg <https://arxiv.org/abs/2303.11797>`_.

    Args:
        embed_dims (int): The number of input dimensions.
        decoder_dims (list): The number of decoder dimensions.
        decoder_guidance_proj_dims (list): The number of appearance
            guidance dimensions.
        init_cfg
    """

    def __init__(self,
                 embed_dims=128,
                 decoder_dims=(64, 32),
                 decoder_guidance_dims=(256, 128),
                 decoder_guidance_proj_dims=(32, 16),
                 **kwargs):
        super().__init__(**kwargs)
        self.decoder_guidance_projection = nn.ModuleList([
            nn.Sequential(
                nn.Conv2d(
                    dec_dims,
                    dec_dims_proj,
                    kernel_size=3,
                    stride=1,
                    padding=1),
                nn.ReLU(),
            ) for dec_dims, dec_dims_proj in zip(decoder_guidance_dims,
                                                 decoder_guidance_proj_dims)
        ]) if decoder_guidance_dims[0] > 0 else None

        self.decoder1 = UpBlock(embed_dims, decoder_dims[0],
                                decoder_guidance_proj_dims[0])
        self.decoder2 = UpBlock(decoder_dims[0], decoder_dims[1],
                                decoder_guidance_proj_dims[1])
        self.conv_seg = nn.Conv2d(
            decoder_dims[1], 1, kernel_size=3, stride=1, padding=1)

    def forward(self, inputs):
        """Forward function.

        Args:
            inputs (dict): Input features including the following features,
                corr_embed: aggregated correlation embeddings.
                appearance_feats: decoder appearance feature guidance.
        """
        # decoder guidance projection
        if self.decoder_guidance_projection is not None:
            projected_decoder_guidance = [
                proj(g) for proj, g in zip(self.decoder_guidance_projection,
                                           inputs['appearance_feats'])
            ]

        # decoder layers
        B = inputs['corr_embed'].size(0)
        corr_embed = inputs['corr_embed'].transpose(1, 2).flatten(0, 1)
        corr_embed = self.decoder1(corr_embed, projected_decoder_guidance[0])
        corr_embed = self.decoder2(corr_embed, projected_decoder_guidance[1])

        output = self.cls_seg(corr_embed)

        # rearrange the output to (B, T, H, W)
        H_ori, W_ori = output.shape[-2:]
        output = output.reshape(B, -1, H_ori, W_ori)
        return output


import os
import pandas as pd
import datetime
from Functions import combine_tcrex

# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
# Combine into one file TCRex predictions for every individual in split dataset
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
folder = './data/splitDS_IMSEQ'
# specify path to where TCRex output files are stored
directory = f'{folder}/TCRex_results'
# where to save processed TCRex output files
if not os.path.exists(f'{folder}/TCRex_processed'):
    os.makedirs(f'{folder}/TCRex_processed')
    print(f'Processed TCRex files will be saved in {folder}/TCRex_processed')
directory_new = f'{folder}/TCRex_processed'

# from CovidTCRS in PhD_2020
# list of all IDs of patients (folder names in the TCRex_results folder)
id_list = [one_id for one_id in os.listdir(directory)
           if os.path.isdir(os.path.join(directory, one_id))]

# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
# add seq counts and frequencies to tcrex output files
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
for sample in id_list:
    # tcrex output (predictions) file
    print(str(datetime.datetime.now()) +
          f':\nStarted combining TCRex predictions with '
          f'sequencing counts data for {sample}')
    file_tcrex = f'{directory}/{sample}/filtered_results.csv'
    tcrex = pd.read_csv(file_tcrex, sep='\t', header=6)
    tcrex['sample_id'] = sample

    # sequencing files with counts
    file_seq = f'{folder}/TCRex_format_meta_TRB/{sample}.txt'
    seq = pd.read_csv(file_seq, sep='\t', index_col=0)
    seq.drop(columns=['d', 'VEnd', 'DStart', 'DEnd', 'JStart'], inplace=True)
    # Total number of sequenced TCRs
    seq['total_count'] = seq['count'].sum()
    # Total number of different TCRs (V-CDR3-J)
    seq['unique_TCRs'] = len(
        seq[['TRBV_gene', 'CDR3_beta', 'TRBJ_gene']].drop_duplicates()[
            'CDR3_beta'])
    # Total number of different CDR3s
    seq['unique_CDR3s'] = len(seq['CDR3_beta'].unique())

    # change V-J gene notation to IMGT (TCRex also has the same, IMGT)
    # remove white spaces V-J genes
    seq['TRBV_gene'] = seq['TRBV_gene'].apply(str).str.replace(' ', '')
    seq['TRBJ_gene'] = seq['TRBJ_gene'].apply(str).str.replace(' ', '')

    # Convert V-J genes to IMGT format
    # seq['TRBV_gene'] = seq['TRBV_gene'].str.replace('\*00.+$', '', regex=True)
    # change TRBV/J1- into TRBV/J01-
    seq.loc[seq['TRBV_gene'].str.contains(pat='TRBV[1-9]-',
                                          regex=True), 'TRBV_gene'] = \
        seq.loc[seq['TRBV_gene'].str.contains(pat='TRBV[1-9]-',
                                              regex=True), 'TRBV_gene'].str.replace(
            'TRBV', 'TRBV0',
            regex=False)
    seq.loc[seq['TRBJ_gene'].str.contains(pat='TRBJ[1-9]-',
                                          regex=True), 'TRBJ_gene'] = \
        seq.loc[seq['TRBJ_gene'].str.contains(pat='TRBJ[1-9]-',
                                              regex=True), 'TRBJ_gene'].str.replace(
            'TRBJ', 'TRBJ0',
            regex=False)
    # change TRBV/J1 into TRBV/J01 (no - )
    seq.loc[seq['TRBV_gene'].str.contains(pat='TRBV[1-9]$',
                                          regex=True), 'TRBV_gene'] = \
        seq.loc[seq['TRBV_gene'].str.contains(pat='TRBV[1-9]$',
                                              regex=True), 'TRBV_gene'].str.replace(
            'TRBV', 'TRBV0',
            regex=False)
    seq.loc[seq['TRBJ_gene'].str.contains(pat='TRBJ[1-9]$',
                                          regex=True), 'TRBJ_gene'] = \
        seq.loc[seq['TRBJ_gene'].str.contains(pat='TRBJ[1-9]$',
                                              regex=True), 'TRBJ_gene'].str.replace(
            'TRBJ', 'TRBJ0',
            regex=False)
    # change -1 into -01
    seq.loc[seq['TRBV_gene'].str.contains(pat='-[1-9]$',
                                          regex=True), 'TRBV_gene'] = \
        seq.loc[seq['TRBV_gene'].str.contains(pat='-[1-9]$',
                                              regex=True), 'TRBV_gene'].str.replace(
            '-', '-0',
            regex=False)
    seq.loc[seq['TRBJ_gene'].str.contains(pat='-[1-9]$',
                                          regex=True), 'TRBJ_gene'] = \
        seq.loc[seq['TRBJ_gene'].str.contains(pat='-[1-9]$',
                                              regex=True), 'TRBJ_gene'].str.replace(
            '-', '-0',
            regex=False)

    # combine tcrex and sequencing files
    tcrex_seq = pd.merge(tcrex, seq, how="left",
                         on=['TRBV_gene', 'CDR3_beta', 'TRBJ_gene'])
    tcrex_seq.drop_duplicates(inplace=True)

    # number of different nt sequences per CDR3 with the same V-J genes, epitope, etc.
    nt_seq = tcrex_seq.groupby(
        ['TRBV_gene', 'CDR3_beta', 'TRBJ_gene', 'epitope', 'pathology',
         'score', 'bpr'],
        as_index=False, sort=False)['count'].count().rename(
        columns={'count': 'N_nt_seq'})
    # combined counts for CDR3 with the same V-J genes, epitope, etc. but different nt sequences
    new_count = tcrex_seq.groupby(
        ['TRBV_gene', 'CDR3_beta', 'TRBJ_gene', 'epitope', 'pathology',
         'score', 'bpr'],
        as_index=False, sort=False)['count'].sum()
    # combined frequencies for CDR3 with the same V-J genes, epitope, etc. but different nt sequences
    new_freq = tcrex_seq.groupby(
        ['TRBV_gene', 'CDR3_beta', 'TRBJ_gene', 'epitope', 'pathology',
         'score', 'bpr'],
        as_index=False, sort=False)['freq'].sum()

    # step-by-step merging to update count and freq columns and substitude cdr3nt with N_nt_seq
    interim = pd.merge(nt_seq, new_count, how='inner',
                       on=['TRBV_gene', 'CDR3_beta', 'TRBJ_gene', 'epitope',
                           'pathology',
                           'score', 'bpr'])
    interim = pd.merge(interim, new_freq, how='inner',
                       on=['TRBV_gene', 'CDR3_beta', 'TRBJ_gene', 'epitope',
                           'pathology',
                           'score', 'bpr'])
    tcrex_seq = pd.merge(tcrex_seq[
                             ['TRBV_gene', 'CDR3_beta', 'TRBJ_gene', 'epitope',
                              'pathology', 'score', 'bpr',
                              'sample_id', 'total_count', 'unique_TCRs',
                              'unique_CDR3s']], interim, how='inner',
                         on=['TRBV_gene', 'CDR3_beta', 'TRBJ_gene', 'epitope',
                             'pathology', 'score', 'bpr'])

    tcrex_seq.drop_duplicates(inplace=True)
    tcrex_seq.sort_values(by=['bpr', 'pathology', 'epitope', 'CDR3_beta'],
                          inplace=True)
    tcrex_seq.to_csv(f'{directory}/{sample}/results_with_seq_counts.csv',
                     index=False)
    print(f"{sample} files have been processed!")

# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
# combine individual sample files into one; add metadata; keep only viral TCRs
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
# metadata file
meta = pd.read_csv(folder + '/metadata_TRB_10umi_filter_MiXCR_to_VDJtools.txt',
                   sep='\t')
meta = meta[
    ['sample_id', 'patient_id', 'T-cell_type', 'Repeat', 's_id', 'Gene',
     'study_patient_id', 'disease_severity', 'time']]

tcrex_data = combine_tcrex(folder, id_list, 'results_with_seq_counts.csv')
# All predicted epitopes: ['SARS-CoV-2', 'HIV', 'EBV', 'CMV', 'HCV', 'DENV1', 'Melanoma', 'Influenza',
# 'DENV2', 'DENV3/4', 'HTLV1', 'HSV2', 'Multiple Myeloma', 'YellowFeverVirus']

tcrex_viral = tcrex_data.loc[~((tcrex_data.pathology == 'Melanoma') | (
            tcrex_data.pathology == 'Multiple Myeloma'))]
tcrex_viral = tcrex_viral.sort_values(
    by=['sample_id', 'bpr', 'pathology', 'epitope', 'CDR3_beta'])

# add metadata
tcrex_viral_meta = pd.merge(tcrex_viral, meta, on='sample_id')
tcrex_viral_meta = tcrex_viral_meta.sort_values(
    by=['study_patient_id', 'patient_id', 's_id', 'Repeat',
        'bpr', 'pathology', 'epitope', 'CDR3_beta'])

# convert time from d10 (str) to day: 10 (int)
tcrex_viral_meta['time'] = tcrex_viral_meta['time'].str.replace(pat='d',
                                                                repl='',
                                                                regex=False).map(
    lambda x: int(x))
tcrex_viral_meta.rename(columns={'time': 'day'}, inplace=True)

# creating week column
tcrex_viral_meta.loc[tcrex_viral_meta['day'] <= 7, 'week'] = 1
tcrex_viral_meta.loc[(tcrex_viral_meta['day'] > 7) & (
            tcrex_viral_meta['day'] <= 14), 'week'] = 2
tcrex_viral_meta.loc[(tcrex_viral_meta['day'] > 14) & (
            tcrex_viral_meta['day'] <= 21), 'week'] = 3
tcrex_viral_meta.loc[(tcrex_viral_meta['day'] > 21) & (
            tcrex_viral_meta['day'] <= 28), 'week'] = 4
tcrex_viral_meta['week'] = tcrex_viral_meta['week'].map(lambda x: int(x))

# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
# MERGING (combined or intersected) REPEATS TOGETHER; PATIENTS (TP and cell type) are still SEPARATE
# ? update frequencies to the median ones when merging (adj.Freq and adj.Total_counts as an alternative)
# ? add CDR3 diversity column
# ? add N_unique V, J genes
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
# new patient id will contain patient_id and cell type only (united/intersected repeats -> no s_id)
new_pat_id = []
for patient in tcrex_viral_meta['patient_id'].unique().tolist():
    for cell in ['cd4', 'cd8']:
        new_pat_id.append(f'{str(patient)}_{cell}')

# combine 3 sequencing repeats into one TCRex output file per patient
# TP and cell type are still separate samples
all_combined = pd.DataFrame()
all_intersect2 = pd.DataFrame()  # occurs at least twice, in two repeats
for new_patient in new_pat_id:
    # to drop columns 'sample_id', 'Repeat', 's_id', 'Gene'
    df1 = tcrex_viral_meta.loc[
        tcrex_viral_meta['sample_id'].str.startswith(pat=(new_patient + '_1')),
        ['TRBV_gene', 'CDR3_beta', 'TRBJ_gene', 'epitope', 'pathology',
         'score', 'bpr',
         'total_count', 'unique_TCRs', 'unique_CDR3s', 'N_nt_seq', 'count',
         'freq',
         'patient_id', 'T-cell_type', 'study_patient_id', 'disease_severity',
         'day', 'week']]
    df2 = tcrex_viral_meta.loc[
        tcrex_viral_meta['sample_id'].str.startswith(pat=(new_patient + '_2')),
        ['TRBV_gene', 'CDR3_beta', 'TRBJ_gene', 'epitope', 'pathology',
         'score', 'bpr',
         'total_count', 'unique_TCRs', 'unique_CDR3s', 'N_nt_seq', 'count',
         'freq',
         'patient_id', 'T-cell_type', 'study_patient_id', 'disease_severity',
         'day', 'week']]
    df3 = tcrex_viral_meta.loc[
        tcrex_viral_meta['sample_id'].str.startswith(pat=(new_patient + '_3')),
        ['TRBV_gene', 'CDR3_beta', 'TRBJ_gene', 'epitope', 'pathology',
         'score', 'bpr',
         'total_count', 'unique_TCRs', 'unique_CDR3s', 'N_nt_seq', 'count',
         'freq',
         'patient_id', 'T-cell_type', 'study_patient_id', 'disease_severity',
         'day', 'week']]
    # merge all entries of all three repeats (filter later)
    df_combined = pd.concat([df1, df2, df3], ignore_index=True, sort=False)
    df_combined['new_id'] = new_patient

    # keep CDR3s that occur at least in 2 repeats out of 3
    # (many same CDR3s have different predicted specificity and V/J genes)
    # all repeated CDR3s
    # confusing keep parameter, but every repeat gets assigned True
    df_intersect2 = df_combined[
        df_combined.duplicated(subset=['CDR3_beta'], keep=False)].copy()
    df_intersect2.sort_values('CDR3_beta', inplace=True)
    try:
        # V-J-CDR3 that occur at least in two repeats
        df_intersect2 = pd.concat(g for _, g in df_intersect2.groupby(
            ['TRBV_gene', 'CDR3_beta', 'TRBJ_gene'])
                                  if len(g['total_count'].unique()) > 1)
        df_intersect2.sort_values(
            by=['total_count', 'bpr', 'pathology', 'CDR3_beta'],
            inplace=True)  # ascending
        # keep only the last one, with the highest counts (gets assigned False)
        df_intersect2 = df_intersect2[~df_intersect2.duplicated(
            subset=['TRBV_gene', 'CDR3_beta', 'TRBJ_gene',
                    'epitope'], keep='last')]
        df_intersect2.sort_values(by=['bpr', 'pathology', 'CDR3_beta'],
                                  inplace=True)
    except ValueError as err:
        print(
            f"{err}: {new_patient} does not have sequences with the same V-J-CDR3 occurring in at least two repeats. "
            f"Looking for the same CDR3 sequences only (disregarding V-J genes.)")
        try:
            # CDR3 that occur at least in two repeats
            df_intersect2 = pd.concat(g for _, g in df_intersect2.groupby(
               'CDR3_beta') if len(g['total_count'].unique()) > 1)
            df_intersect2.sort_values(
                by=['total_count', 'bpr', 'pathology', 'CDR3_beta'],
                inplace=True)  # ascending
            # keep only the last one, with the highest counts (gets assigned False)
            df_intersect2 = df_intersect2[~df_intersect2.duplicated(
                subset=['TRBV_gene', 'CDR3_beta', 'TRBJ_gene',
                        'epitope'], keep='last')]
            df_intersect2.sort_values(by=['bpr', 'pathology', 'CDR3_beta'],
                                      inplace=True)
        except ValueError as err2:
            print(
                f"{err2}: {new_patient} does not have sequences with the same V-CDR3-J or just CDR3 occurring "
                f"in at least two repeats. Skipping to the next file.")
        else:
            print(
                f"File with the same CDR3 only occurring in at least two repeats has been created successfully "
                f"for {new_patient}.")
    else:
        print(
            f"File with the same V-CDR3-J occurring in at least two repeats has been created successfully "
            f"for {new_patient}.")
    # assemble all patients together
    all_intersect2 = all_intersect2.append(df_intersect2, ignore_index=True,
                                           sort=False)
all_intersect2.to_csv(f'{directory_new}/viralTCRexPreds_withCounts_splitDS_intersected2x_meta.tsv',
                      sep='\t', index=False)
# all_intersected_double_viral_meta


"""Attachments."""

from __future__ import annotations

from logging import getLogger
from pathlib import Path

logger = getLogger(__name__)

filetypes = {
    "画像": [
        "jpg",
        "jpeg",
        "png",
        "aping",
        "gif",
        "tiff",
        "bmp",
        "webp",
    ],
    "動画": [
        "mp4",
        "webm",
        "avi",
        "mov",
    ],
    "音声": [
        "mp3",
        "m4a",
        "wav",
        "flac",
    ],
}

def judge_filetype(filename: str) -> str:
    ext = Path(filename).suffix
    # ドットを削除
    ext = ext.replace(".", "")
    for key in filetypes:
        if ext.lower() in filetypes[key]:
            return key
    return "ファイル"

def extract_filetypes(filenames: list[str]) -> str:
    """ファイル名のリストから読み上げ内容を取得.

    「画像,音声を送信しました」
    """
    logger.debug("receved files: %s", str(filenames))
    result = set()
    for f in filenames:
        t = judge_filetype(f)
        if t:
            result.add(t)
    logger.debug(result)
    if result:
        return ",".join(result) + "を送信しました。"
    return ""


import os
import time

import torch
from logzero import logger

import tools.utils as ut
from tools import utils
from trains import params
from trains.train import Trainer

def main():
    # 训练集
    train_dataset = ut.get_dataset(params.dataset, noise=True)
    # 测试集
    test_dataset = ut.get_dataset(params.dataset, train=False)

    # 初始化模块
    feature_extractor = params.feature_extractor_dict[params.dataset]
    label_predictor = params.label_predictor_dict[params.dataset]
    domain_classifier = params.domain_predictor_dict[params.dataset]

    # 初始化优化器
    optimizer = torch.optim.SGD([{'params': feature_extractor.parameters()},
                                 {'params': label_predictor[0].parameters()},
                                 {'params': label_predictor[1].parameters()},
                                 {'params': label_predictor[2].parameters()},
                                 {'params': domain_classifier.parameters()}],
                                lr=params.learning_rate,
                                momentum=0.9,
                                weight_decay=0.0001)

    # 加载训练参数
    save_path = params.net_save_path
    if not os.path.exists(params.save_dir):
        os.mkdir(params.save_dir)
    if not os.path.exists(save_path):
        os.mkdir(save_path)
    if os.path.exists(save_path + "/fe.pth"):
        feature_extractor.load_state_dict(torch.load(save_path + "/fe.pth"))
    if os.path.exists(save_path + "/dc.pth"):
        domain_classifier.load_state_dict(torch.load(save_path + "/dc.pth"))
    if os.path.exists(save_path + "/lp1.pth"):
        label_predictor[0].load_state_dict(torch.load(save_path + "/lp1.pth"))
        label_predictor[1].load_state_dict(torch.load(save_path + "/lp2.pth"))
        label_predictor[2].load_state_dict(torch.load(save_path + "/lp3.pth"))

    # 初始化Trainer
    trainer = Trainer(feature_extractor, label_predictor, domain_classifier, optimizer)

    # 日志存储
    utils.log_save(params.save_dir)
    start_time = time.time()
    logger.info("initial_model")
    # 初始化训练
    for epoch in range(params.initial_epochs):
        trainer.train(epoch=epoch, dataset=train_dataset)
        # 验证
        trainer.test(dataset=test_dataset)
        # 保存日志到文件
        utils.log_save(params.save_dir, start_time=start_time, limit=3600)
        start_time = time.time()
    # 用未标记的数据集更新模型
    # trainer.update(initial_dataset=train_dataset, unlabeled_dataset=test_dataset, test_dataset=test_dataset)

if __name__ == '__main__':
    main()


import requests
import openpyxl
from openpyxl import Workbook

File_Path="D:\Programming\Application Security\coe-application-security\Data Files\Data.xlsx"
Excel_Location= "D:\Programming\Application Security\coe-application-security\DataFiles\API.xlsx"
Excel_Sheet_Name = 'Sheet1'
ModuleName = "Technicalskills_POST"

def read_specific_cell_values(Excel_Location, Excel_Sheet_Name):
    try:
        Excel_WorkBook = openpyxl.load_workbook(Excel_Location)
        for SheetName in Excel_WorkBook.worksheets:
            if SheetName.title == Excel_Sheet_Name:
                print("Sheet Found")
                ActiveWorkSheet = Excel_WorkBook[SheetName.title]
                print("Title of Sheet = " + ActiveWorkSheet.title)
                # Read Cell Value
                print(ActiveWorkSheet['A2'].value)
                print(ActiveWorkSheet['B2'].value)
                break
            else:
                print("Sheet Not Found")
    except:
        print("Failed for execute")

def read_located_cell_values(Excel_Location, Excel_Sheet_Name):
    Excel_WorkBook = openpyxl.load_workbook(Excel_Location)
    for SheetName in Excel_WorkBook.worksheets:
        if SheetName.title == Excel_Sheet_Name:
            print("Sheet Found")
            ActiveWorkSheet = Excel_WorkBook[SheetName.title]
            print("Title of Sheet = " + ActiveWorkSheet.title)
            Expected_Cell_Value = SheetName.cell(2, 1)
            print(Expected_Cell_Value.value)

def read_refernce_cell_values(Excel_Location, Excel_Sheet_Name):
    Excel_WorkBook = openpyxl.load_workbook(Excel_Location)
    for SheetName in Excel_WorkBook.worksheets:
        if SheetName.title == Excel_Sheet_Name:
            print("Sheet Found")
            ActiveWorkSheet = Excel_WorkBook[SheetName.title]
            print("Title of Sheet = " + ActiveWorkSheet.title)
            Expected_Cell_Value = SheetName.cell(column=1, row=2)
            print(Expected_Cell_Value.value)

def read_number_of_rows_and_columns(Excel_Location, Excel_Sheet_Name):
    Excel_WorkBook = openpyxl.load_workbook(Excel_Location)
    for SheetName in Excel_WorkBook.worksheets:
        if SheetName.title == Excel_Sheet_Name:
            print("Sheet Found")
            ActiveWorkSheet = Excel_WorkBook[SheetName.title]
            print("Title of Sheet = " + ActiveWorkSheet.title)
    rows = SheetName.max_row
    columns = SheetName.max_column
    print("Total Rows are = " + str(rows))
    print("Total Columns are = " + str(columns))

def read_first_row_values(Excel_Location, Excel_Sheet_Name):
    Excel_WorkBook = openpyxl.load_workbook(Excel_Location)
    for SheetName in Excel_WorkBook.worksheets:
        if SheetName.title == Excel_Sheet_Name:
            print("Worksheet Found = " + SheetName.title)
            ActiveWorkSheet = Excel_WorkBook[SheetName.title]
            print("Title of Sheet = " + ActiveWorkSheet.title)

            RowLength = SheetName.max_row
            ColumnLength = SheetName.max_column
            print("Number of Payloads for " + str(SheetName.title) + " = " + str(RowLength - 1))

            for i in range(2, RowLength + 1):
                RowContents = SheetName.cell(row=i, column=1)
                print("Row "+ str(RowContents.row-1) + " = "+ RowContents.value, end="" + "\n")
                return RowContents.value


"""Base for interacting with the Censys Assets API."""
import re
from typing import Any, Dict, Iterator, List, Optional

from ..api import CensysAsmAPI
from censys.common.exceptions import CensysInvalidColorException

HEX_REGEX = re.compile(r"^#(?:[0-9a-fA-F]{3}){1,2}$")

class Assets(CensysAsmAPI):
    """Assets API class."""

    def __init__(self, asset_type: str, *args, **kwargs):
        """Inits Assets.

        Args:
            asset_type (str): Type of asset to interact with.
            *args: Variable length argument list.
            **kwargs: Arbitrary keyword arguments.
        """
        CensysAsmAPI.__init__(self, *args, **kwargs)
        self.base_path = f"assets/{asset_type}"

    def get_assets(
        self,
        page_number: int = 1,
        page_size: Optional[int] = None,
        tag: Optional[List[str]] = None,
        tag_operator: Optional[str] = None,
        source: Optional[List[str]] = None,
    ) -> Iterator[dict]:
        """Requests assets data.

        Args:
            page_number (int): Optional; Page number to begin at when searching.
            page_size (int): Optional; Page size for retrieving assets.
            tag (list): Optional; List of tags to search for.
            tag_operator (str): Optional; Operator to use when searching for tags.
            source (list): Optional; List of sources to search for.

        Yields:
            dict: The assets result returned.
        """
        args: Dict[str, Any] = {}
        if tag:
            args["tag"] = tag
        if tag_operator:
            args["tagOperator"] = tag_operator
        if source:
            args["source"] = source
        yield from self._get_page(
            self.base_path, page_number=page_number, page_size=page_size, args=args
        )

    def get_asset_by_id(self, asset_id: str) -> dict:
        """Requests asset data by ID.

        Args:
            asset_id (str): Requested asset ID.

        Returns:
            dict: Asset search result.
        """
        path = f"{self.base_path}/{asset_id}"

        return self._get(path)

    def get_comments(
        self,
        asset_id: str,
        page_number: int = 1,
        page_size: Optional[int] = None,
    ) -> Iterator[dict]:
        """Requests comments on a specified asset.

        Args:
            asset_id (str): Asset ID for requested comments.
            page_number (int): Optional; Page number to begin at when searching.
            page_size (int): Optional; Page size for retrieving comments.

        Returns:
            generator: Comment search results.
        """
        path = f"{self.base_path}/{asset_id}/comments"

        return self._get_page(
            path, page_number=page_number, page_size=page_size, keyword="comments"
        )

    def get_comment_by_id(self, asset_id: str, comment_id: int) -> dict:
        """Requests a comment on a specified asset by comment ID.

        Args:
            asset_id (str): Asset ID for requested comments.
            comment_id (int): Requested comment ID.

        Returns:
            dict: Comment search result.
        """
        path = f"{self.base_path}/{asset_id}/comments/{comment_id}"

        return self._get(path)

    def add_comment(self, asset_id: str, comment: str) -> dict:
        """Adds a comment to a specified asset on the ASM platform.

        Args:
            asset_id (str): Asset ID to add comment to.
            comment (str): New comment text.

        Returns:
            dict: Added comment results.
        """
        path = f"{self.base_path}/{asset_id}/comments"
        data = {"markdown": str(comment)}

        return self._post(path, data=data)

    def delete_comment(self, asset_id: str, comment_id: int) -> dict:
        """Deletes a comment from a specified asset on the ASM platform by comment ID.

        Args:
            asset_id (str): Asset ID to delete comment from.
            comment_id (int): Comment ID to delete.

        Returns:
            dict: Deleted comment results.
        """
        path = f"{self.base_path}/{asset_id}/comments/{comment_id}"

        return self._delete(path)

    def add_tag(self, asset_id: str, name: str, color: Optional[str] = None) -> dict:
        """Adds a tag to a specified asset on the ASM platform.

        Args:
            asset_id (str): Asset ID to add tag to.
            name (str): New tag name.
            color (str): Optional; New tag color (hex).

        Returns:
            dict: Added tag results.
        """
        path = f"{self.base_path}/{asset_id}/tags"
        data = format_tag(name, color)

        return self._post(path, data=data)

    def delete_tag(self, asset_id: str, name: str) -> dict:
        """Deletes a tag from a specified asset on the ASM platform by tag name.

        Args:
            asset_id (str): Asset ID to delete tag from.
            name (str): Tag name to delete.

        Returns:
            dict: Deleted tag results.
        """
        path = f"{self.base_path}/{asset_id}/tags/{name}"

        return self._delete(path)

def format_tag(name: str, color: Optional[str] = None) -> dict:
    """Formats tag name and color request data.

    Args:
        name (str): Tag name.
        color (str): Optional; Tag color (hex).

    Raises:
        CensysInvalidColorException: Raised if color is not a valid hex color.

    Returns:
        dict: Formatted tag request data.
    """
    if color:
        if not HEX_REGEX.match(color):
            raise CensysInvalidColorException(10037, f"{color} is not a valid color.")
        return {"name": str(name), "color": str(color)}

    return {"name": str(name)}


# benodigde libraries
import atexit
import configparser
from pyVim import connect
from pyVmomi import vmodl
from pyVim.connect import Disconnect
import humanize
import sys

# config file
config = configparser.ConfigParser()
config.read('config.ini')
config.sections()

# gebruik printen in .txt file
f = open("gebruik.txt", 'w')
sys.stdout = f

MBFACTOR = float(1 << 20)

printVM = True
printDatastore = True
printHost = True

# opvragen data van vSphere
def head():
    try:
        si = connect.ConnectNoSSL(host= (config['NETLAB'].get('host')), port= 443,
        user= (config['NETLAB'].get('username')), pwd= (config['NETLAB'].get('password')))

        atexit.register(Disconnect, si)
        content = si.RetrieveContent()

        for datacenter in content.rootFolder.childEntity:
            print ("Datacenter: " + datacenter.name)

            if printDatastore:
                datastores = datacenter.datastore
                for ds in datastores:
                    data_store(ds)

            if printHost:
                if hasattr(datacenter.vmFolder, 'childEntity'):
                    hostFolder = datacenter.hostFolder
                    computeResourceList = hostFolder.childEntity

                    for computeResource in computeResourceList:
                        data_res(computeResource)

    except vmodl.MethodFault as e:
        print ("Fout:Controlleer je verbinding met de Cisco VPN en je inloggegevens van Vcenter!", e)

# data van de data store
def data_store(datastore):
    try:
        summary = datastore.summary
        capacity = summary.capacity
        freeSpace = summary.freeSpace
        uncommittedSpace = summary.uncommitted
        freeSpacePercentage = (float(freeSpace) / capacity) * 100

        print ("\nDatastore name: ", summary.name)
        print ("Capacity: ", humanize.naturalsize(capacity, binary=True))

        if uncommittedSpace is not None:
            provisionedSpace = (capacity - freeSpace) + uncommittedSpace

            print ("Provisioned space: ", humanize.naturalsize(provisionedSpace, binary=True))
            print ("Free space: ", humanize.naturalsize(freeSpace, binary=True))
            print ("Free space percentage: " + str(freeSpacePercentage) + "%")

    except Exception:
        print ("De summary kan niet geopend worden!: ", datastore.name)

        pass

# data gebruik resources
def data_res(computeResource):
    try:
        hostList = computeResource.host
        for host in hostList:
            data_host(host)

    except Exception:
        print ("Informatie hierover kan niet opgehaald worden!:", computeResource.name)

        pass

# data van de host
def data_host(host):
    try:
        summary = host.summary
        stats = summary.quickStats
        hardware = host.hardware
        cpuCapacityMhz = (host.hardware.cpuInfo.hz * host.hardware.cpuInfo.numCpuCores) / 1000 / 1000
        cpuUsage = stats.overallCpuUsage
        memoryCapacity = hardware.memorySize
        memoryCapacityInMB = hardware.memorySize/MBFACTOR
        memoryUsage = stats.overallMemoryUsage
        freeMemoryPercentage = 100 - ((float(memoryUsage) / memoryCapacityInMB) * 100)

        print ("--------------------------------------------------")
        print ("Host name: ", host.name)
        print ("Host CPU capacity: ", cpuCapacityMhz, " Mhz")
        print ("Host CPU usage: ", cpuUsage)
        print ("Host memory capacity: ", humanize.naturalsize(memoryCapacity, binary=True))
        print ("Host memory usage: ", memoryUsage / 1024, "GB")
        print ("Free memory percentage: " + str(freeMemoryPercentage) + "%")
        print ("--------------------------------------------------")

    except Exception:
        print ("De host kan niet gepingd worden!:", host.name)

        pass

if __name__ == "__main__":

    head()
    f.close()


import cv2

"""
디렉토리를 python/opencv로 설정하고 실행
검정 배경 이미지에
도형 그리기
"""

img = cv2.imread('./img/black.png',cv2.IMREAD_COLOR)

# 직선 그리기
img = cv2.line(img,(50,50),(150,150),(0,255,0),10)

# 직사각형 그리기
# (x1, y1) , (x2,y2) , BGR , thickness
img = cv2.rectangle(img, (100,100),(300,400),(255,0,0),3)

# 원 그리기
img = cv2.circle(img,(300,400),100,(0,0,255),2)

# 텍스트
cv2.putText(img,'Test Text',(10,100),cv2.FONT_HERSHEY_SIMPLEX,4,(255,255,255))

cv2.imshow('black',img)

cv2.imwrite('./result/new_black.png',img)

cv2.waitKey(10000)


# coding: utf-8
import sys
sys.path.append('/root/ZX/SMRCToolkit')
from sogou_mrc.data.vocabulary import Vocabulary
from sogou_mrc.dataset.squad import SquadReader, SquadEvaluator
from sogou_mrc.dataset.cmrc import CMRCReader,CMRCEvaluator
from sogou_mrc.model.bidaf import BiDAF
import tensorflow as tf
import logging
from sogou_mrc.data.batch_generator import BatchGenerator
# import argparse
# from os import environ
# parser.add_argument('-g','--gpu',nargs=1,choices=[0,1],type=int,metavar='',help="Run single-gpu version.Choose the GPU from:{!s}".format([0,1]))
# args = parser.parse_args()
# if args.gpu:
#   environ["CUDA_VISIBLE_DEVICES"] = str(args.gpu[0])
#   mode_='single-gpu'

tf.logging.set_verbosity(tf.logging.ERROR)
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')

data_folder = '/root/ZX/SMRCToolkit/cmrc_data_folder/'
embedding_folder = '/root/ZX/SMRCToolkit/embedding_folder/'
train_file = data_folder+"cmrc2018_train.json"
dev_file = data_folder+"cmrc2018_dev.json"

reader = CMRCReader()

train_data = reader.read(train_file)
eval_data = reader.read(dev_file)
evaluator = CMRCEvaluator(dev_file)

vocab = Vocabulary(do_lowercase=False)
vocab.build_vocab(train_data + eval_data, min_word_count=3, min_char_count=10)
word_embedding = vocab.make_word_embedding(embedding_folder)
train_batch_generator = BatchGenerator(vocab, train_data, batch_size=32, training=True)
eval_batch_generator = BatchGenerator(vocab, eval_data, batch_size=60)

model = BiDAF(vocab, pretrained_word_embedding=word_embedding,word_embedding_size=300)
model.compile(tf.train.AdamOptimizer, 0.001)
model.train_and_evaluate(train_batch_generator, eval_batch_generator, evaluator, epochs=50, eposides=2)


"""Permutation test function as described in CellPhoneDB 2.0."""
from __future__ import annotations

from abc import ABC
from collections import namedtuple
from functools import partial
from itertools import product
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Iterable, Literal, Mapping, Sequence, Union

import numpy as np
import pandas as pd
from anndata import AnnData
from numba import njit, prange  # noqa: F401
from scanpy import logging as logg
from scipy.sparse import csc_matrix
from spatialdata import SpatialData

from squidpy._constants._constants import ComplexPolicy, CorrAxis
from squidpy._constants._pkg_constants import Key
from squidpy._docs import d, inject_docs
from squidpy._utils import NDArrayA, Signal, SigQueue, _get_n_cores, parallelize
from squidpy.gr._utils import (
    _assert_categorical_obs,
    _assert_positive,
    _check_tuple_needles,
    _create_sparse_df,
    _genesymbols,
    _save_data,
)

__all__ = ["ligrec", "PermutationTest"]

StrSeq = Sequence[str]
SeqTuple = Sequence[tuple[str, str]]
Interaction_t = Union[pd.DataFrame, Mapping[str, StrSeq], StrSeq, tuple[StrSeq, StrSeq], SeqTuple]
Cluster_t = Union[StrSeq, tuple[StrSeq, StrSeq], SeqTuple]

SOURCE = "source"
TARGET = "target"

TempResult = namedtuple("TempResult", ["means", "pvalues"])

_template = """
@njit(parallel={parallel}, cache=False, fastmath=False)
def _test_{n_cls}_{ret_means}_{parallel}(
    interactions: NDArrayA,  # [np.uint32],
    interaction_clusters: NDArrayA,  # [np.uint32],
    data: NDArrayA,  # [np.float64],
    clustering: NDArrayA,  # [np.uint32],
    mean: NDArrayA,  # [np.float64],
    mask: NDArrayA,  # [np.bool_],
    res: NDArrayA,  # [np.float64],
    {args}
) -> None:

    {init}
    {loop}
    {finalize}

    for i in prange(len(interactions)):
        rec, lig = interactions[i]
        for j in prange(len(interaction_clusters)):
            c1, c2 = interaction_clusters[j]
            m1, m2 = mean[rec, c1], mean[lig, c2]

            if np.isnan(res[i, j]):
                continue

            if m1 > 0 and m2 > 0:
                {set_means}
                if mask[rec, c1] and mask[lig, c2]:
                    # both rec, lig are sufficiently expressed in c1, c2
                    res[i, j] += (groups[c1, rec] + groups[c2, lig]) > (m1 + m2)
                else:
                    res[i, j] = np.nan
            else:
                # res_means is initialized with 0s
                res[i, j] = np.nan
"""

def _create_template(n_cls: int, return_means: bool = False, parallel: bool = True) -> str:
    if n_cls <= 0:
        raise ValueError(f"Expected number of clusters to be positive, found `{n_cls}`.")

    rng = range(n_cls)
    init = "".join(
        f"""
    g{i} = np.zeros((data.shape[1],), dtype=np.float64); s{i} = 0"""
        for i in rng
    )

    loop_body = """
        if cl == 0:
            g0 += data[row]
            s0 += 1"""
    loop_body = loop_body + "".join(
        f"""
        elif cl == {i}:
            g{i} += data[row]
            s{i} += 1"""
        for i in range(1, n_cls)
    )
    loop = f"""
    for row in prange(data.shape[0]):
        cl = clustering[row]
        {loop_body}
        else:
            assert False, "Unhandled case."
    """
    finalize = ", ".join(f"g{i} / s{i}" for i in rng)
    finalize = f"groups = np.stack(({finalize}))"

    if return_means:
        args = "res_means: NDArrayA,  # [np.float64]"
        set_means = "res_means[i, j] = (m1 + m2) / 2.0"
    else:
        args = set_means = ""

    return _template.format(
        n_cls=n_cls,
        parallel=bool(parallel),
        ret_means=int(return_means),
        args=args,
        init=init,
        loop=loop,
        finalize=finalize,
        set_means=set_means,
    )

def _fdr_correct(
    pvals: pd.DataFrame,
    corr_method: str,
    corr_axis: Literal["interactions", "clusters"] | CorrAxis,
    alpha: float = 0.05,
) -> pd.DataFrame:
    """Correct p-values for FDR along specific axis in ``pvals``."""
    from pandas.core.arrays.sparse import SparseArray
    from statsmodels.stats.multitest import multipletests

    def fdr(pvals: pd.Series) -> SparseArray:
        _, qvals, _, _ = multipletests(
            np.nan_to_num(pvals.values, copy=True, nan=1.0),
            method=corr_method,
            alpha=alpha,
            is_sorted=False,
            returnsorted=False,
        )
        qvals[np.isnan(pvals.values)] = np.nan

        return SparseArray(qvals, dtype=qvals.dtype, fill_value=np.nan)

    corr_axis = CorrAxis(corr_axis)

    if corr_axis == CorrAxis.CLUSTERS:
        # clusters are in columns
        pvals = pvals.apply(fdr)
    elif corr_axis == CorrAxis.INTERACTIONS:
        pvals = pvals.T.apply(fdr).T
    else:
        raise NotImplementedError(f"FDR correction for `{corr_axis}` is not implemented.")

    return pvals

@d.get_full_description(base="PT")
@d.get_sections(base="PT", sections=["Parameters"])
@d.dedent
class PermutationTestABC(ABC):
    """
    Class for receptor-ligand interaction testing.

    The expected workflow is::

        pt = PermutationTest(adata).prepare()
        res = pt.test("clusters")

    Parameters
    ----------
    %(adata)s
    use_raw
        Whether to access :attr:`anndata.AnnData.raw`.
    """

    def __init__(self, adata: AnnData, use_raw: bool = True):
        if not isinstance(adata, AnnData):
            raise TypeError(f"Expected `adata` to be of type `anndata.AnnData`, found `{type(adata).__name__}`.")
        if not adata.n_obs:
            raise ValueError("No cells are in `adata.obs_names`.")
        if not adata.n_vars:
            raise ValueError("No genes are in `adata.var_names`.")

        self._adata = adata
        if use_raw:
            if adata.raw is None:
                raise AttributeError("No `.raw` attribute found. Try specifying `use_raw=False`.")
            if adata.raw.n_obs != adata.n_obs:
                raise ValueError(f"Expected `{adata.n_obs}` cells in `.raw` object, found `{adata.raw.n_obs}`.")
            adata = adata.raw

        self._data = pd.DataFrame.sparse.from_spmatrix(
            csc_matrix(adata.X), index=adata.obs_names, columns=adata.var_names
        )

        self._interactions: pd.DataFrame | None = None
        self._filtered_data: pd.DataFrame | None = None

    @d.get_full_description(base="PT_prepare")
    @d.get_sections(base="PT_prepare", sections=["Parameters", "Returns"])
    @inject_docs(src=SOURCE, tgt=TARGET, cp=ComplexPolicy)
    def prepare(
        self, interactions: Interaction_t, complex_policy: Literal["min", "all"] | ComplexPolicy = ComplexPolicy.MIN.v
    ) -> PermutationTestABC:
        """
        Prepare self for running the permutation test.

        Parameters
        ----------
        interactions
            Interaction to test. The type can be one of:

                - :class:`pandas.DataFrame` - must contain at least 2 columns named `{src!r}` and `{tgt!r}`.
                - :class:`dict` - dictionary with at least 2 keys named `{src!r}` and `{tgt!r}`.
                - :class:`typing.Sequence` - Either a sequence of :class:`str`, in which case all combinations are
                  produced, or a sequence of :class:`tuple` of 2 :class:`str` or a :class:`tuple` of 2 sequences.

            If `None`, the interactions are extracted from :mod:`omnipath`. Protein complexes can be specified by
            delimiting the components with `'_'`, such as `'alpha_beta_gamma'`.
        complex_policy
            Policy on how to handle complexes. Valid options are:

                - `{cp.MIN.s!r}` - select gene with the minimum average expression. This is the same as in
                  :cite:`cellphonedb`.
                - `{cp.ALL.s!r}` - select all possible combinations between `{src!r}` and `{tgt!r}` complexes.

        Returns
        -------
        Sets the following attributes and returns :attr:`self`:

            - :attr:`interactions` - filtered interactions whose `{src!r}` and `{tgt!r}` are both in the data.
        """
        complex_policy = ComplexPolicy(complex_policy)

        if isinstance(interactions, Mapping):
            interactions = pd.DataFrame(interactions)

        if isinstance(interactions, pd.DataFrame):
            if SOURCE not in interactions.columns:
                raise KeyError(f"Column `{SOURCE!r}` is not in `interactions`.")
            if TARGET not in interactions.columns:
                raise KeyError(f"Column `{TARGET!r}` is not in `interactions`.")

            self._interactions = interactions.copy()
        elif isinstance(interactions, Iterable):
            interactions = tuple(interactions)
            if not len(interactions):
                raise ValueError("No interactions were specified.")

            if isinstance(interactions[0], str):
                interactions = list(product(interactions, repeat=2))
            elif len(interactions) == 2:
                interactions = tuple(zip(*interactions))

            if not all(len(i) == 2 for i in interactions):
                raise ValueError("Not all interactions are of length `2`.")

            self._interactions = pd.DataFrame(interactions, columns=[SOURCE, TARGET])
        else:
            raise TypeError(
                f"Expected either a `pandas.DataFrame`, `dict` or `iterable`, found `{type(interactions).__name__}`"
            )
        if TYPE_CHECKING:
            assert isinstance(self.interactions, pd.DataFrame)

        if self.interactions.empty:
            raise ValueError("The interactions are empty")

        # first uppercase, then drop duplicates
        self._data.columns = self._data.columns.str.upper()
        self.interactions[SOURCE] = self.interactions[SOURCE].str.upper()
        self.interactions[TARGET] = self.interactions[TARGET].str.upper()

        logg.debug("DEBUG: Removing duplicate interactions")
        self.interactions.dropna(subset=(SOURCE, TARGET), inplace=True, how="any")
        self.interactions.drop_duplicates(subset=(SOURCE, TARGET), inplace=True, keep="first")

        logg.debug("DEBUG: Removing duplicate genes in the data")
        n_genes_prior = self._data.shape[1]
        self._data = self._data.loc[:, ~self._data.columns.duplicated()]
        if self._data.shape[1] != n_genes_prior:
            logg.warning(f"Removed `{n_genes_prior - self._data.shape[1]}` duplicate gene(s)")

        self._filter_interactions_complexes(complex_policy)
        self._filter_interactions_by_genes()
        self._trim_data()

        # this is necessary because of complexes
        self.interactions.drop_duplicates(subset=(SOURCE, TARGET), inplace=True, keep="first")

        return self

    @d.get_full_description(base="PT_test")
    @d.get_sections(base="PT_test", sections=["Parameters"])
    @d.dedent
    @inject_docs(src=SOURCE, tgt=TARGET, fa=CorrAxis)
    def test(
        self,
        cluster_key: str,
        clusters: Cluster_t | None = None,
        n_perms: int = 1000,
        threshold: float = 0.01,
        seed: int | None = None,
        corr_method: str | None = None,
        corr_axis: Literal["interactions", "clusters"] | CorrAxis = CorrAxis.INTERACTIONS.v,
        alpha: float = 0.05,
        copy: bool = False,
        key_added: str | None = None,
        numba_parallel: bool | None = None,
        **kwargs: Any,
    ) -> Mapping[str, pd.DataFrame] | None:
        """
        Perform the permutation test as described in :cite:`cellphonedb`.

        Parameters
        ----------
        %(cluster_key)s
        clusters
            Clusters from :attr:`anndata.AnnData.obs` ``['{{cluster_key}}']``. Can be specified either as a sequence
            of :class:`tuple` or just a sequence of cluster names, in which case all combinations considered.
        %(n_perms)s
        threshold
            Do not perform permutation test if any of the interacting components is being expressed
            in less than ``threshold`` percent of cells within a given cluster.
        %(seed)s
        %(corr_method)s
        corr_axis
            Axis over which to perform the FDR correction. Only used when ``corr_method != None``. Valid options are:

                - `{fa.INTERACTIONS.s!r}` - correct interactions by performing FDR correction across the clusters.
                - `{fa.CLUSTERS.s!r}` - correct clusters by performing FDR correction across the interactions.
        alpha
            Significance level for FDR correction. Only used when ``corr_method != None``.
        %(copy)s
        key_added
            Key in :attr:`anndata.AnnData.uns` where the result is stored if ``copy = False``.
            If `None`, ``'{{cluster_key}}_ligrec'`` will be used.
        %(numba_parallel)s
        %(parallelize)s

        Returns
        -------
        %(ligrec_test_returns)s
        """
        _assert_positive(n_perms, name="n_perms")
        _assert_categorical_obs(self._adata, key=cluster_key)

        if corr_method is not None:
            corr_axis = CorrAxis(corr_axis)
        if TYPE_CHECKING:
            assert isinstance(corr_axis, CorrAxis)

        if len(self._adata.obs[cluster_key].cat.categories) <= 1:
            raise ValueError(
                f"Expected at least `2` clusters, found `{len(self._adata.obs[cluster_key].cat.categories)}`."
            )
        if TYPE_CHECKING:
            assert isinstance(self.interactions, pd.DataFrame)
            assert isinstance(self._filtered_data, pd.DataFrame)

        interactions = self.interactions[[SOURCE, TARGET]]
        self._filtered_data["clusters"] = self._adata.obs.copy()[cluster_key].astype("string").astype("category").values

        if clusters is None:
            clusters = list(map(str, self._adata.obs[cluster_key].cat.categories))
        if all(isinstance(c, str) for c in clusters):
            clusters = list(product(clusters, repeat=2))  # type: ignore[assignment]
        clusters = sorted(
            _check_tuple_needles(
                clusters,  # type: ignore[arg-type]
                self._filtered_data["clusters"].cat.categories,
                msg="Invalid cluster `{0!r}`.",
                reraise=True,
            )
        )
        clusters_flat = list({c for cs in clusters for c in cs})

        data = self._filtered_data.loc[np.isin(self._filtered_data["clusters"], clusters_flat), :]
        data["clusters"] = data["clusters"].cat.remove_unused_categories()
        cat = data["clusters"].cat

        cluster_mapper = dict(zip(cat.categories, range(len(cat.categories))))
        gene_mapper = dict(zip(data.columns[:-1], range(len(data.columns) - 1)))  # -1 for 'clusters'

        data.columns = [gene_mapper[c] if c != "clusters" else c for c in data.columns]
        clusters_ = np.array([[cluster_mapper[c1], cluster_mapper[c2]] for c1, c2 in clusters], dtype=np.uint32)

        data["clusters"] = cat.rename_categories(cluster_mapper)
        # much faster than applymap (tested on 1M interactions)
        interactions_ = np.vectorize(lambda g: gene_mapper[g])(interactions.values)

        n_jobs = _get_n_cores(kwargs.pop("n_jobs", None))
        start = logg.info(
            f"Running `{n_perms}` permutations on `{len(interactions)}` interactions "
            f"and `{len(clusters)}` cluster combinations using `{n_jobs}` core(s)"
        )
        res = _analysis(
            data,
            interactions_,
            clusters_,
            threshold=threshold,
            n_perms=n_perms,
            seed=seed,
            n_jobs=n_jobs,
            numba_parallel=numba_parallel,
            **kwargs,
        )
        res = {
            "means": _create_sparse_df(
                res.means,
                index=pd.MultiIndex.from_frame(interactions, names=[SOURCE, TARGET]),
                columns=pd.MultiIndex.from_tuples(clusters, names=["cluster_1", "cluster_2"]),
                fill_value=0,
            ),
            "pvalues": _create_sparse_df(
                res.pvalues,
                index=pd.MultiIndex.from_frame(interactions, names=[SOURCE, TARGET]),
                columns=pd.MultiIndex.from_tuples(clusters, names=["cluster_1", "cluster_2"]),
                fill_value=np.nan,
            ),
            "metadata": self.interactions[self.interactions.columns.difference([SOURCE, TARGET])],
        }
        res["metadata"].index = res["means"].index.copy()

        if TYPE_CHECKING:
            assert isinstance(res, dict)

        if corr_method is not None:
            logg.info(
                f"Performing FDR correction across the `{corr_axis.v}` "
                f"using method `{corr_method}` at level `{alpha}`"
            )
            res["pvalues"] = _fdr_correct(res["pvalues"], corr_method, corr_axis, alpha=alpha)

        if copy:
            logg.info("Finish", time=start)
            return res

        _save_data(self._adata, attr="uns", key=Key.uns.ligrec(cluster_key, key_added), data=res, time=start)

    def _trim_data(self) -> None:
        """Subset genes :attr:`_data` to those present in interactions."""
        if TYPE_CHECKING:
            assert isinstance(self._data, pd.DataFrame)
            assert isinstance(self.interactions, pd.DataFrame)

        logg.debug("DEBUG: Removing genes not in any interaction")
        self._filtered_data = self._data.loc[:, list(set(self.interactions[SOURCE]) | set(self.interactions[TARGET]))]

    def _filter_interactions_by_genes(self) -> None:
        """Subset :attr:`interactions` to only those for which we have the data."""
        if TYPE_CHECKING:
            assert isinstance(self.interactions, pd.DataFrame)

        logg.debug("DEBUG: Removing interactions with no genes in the data")
        self._interactions = self.interactions[
            self.interactions[SOURCE].isin(self._data.columns) & self.interactions[TARGET].isin(self._data.columns)
        ]

        if self.interactions.empty:
            raise ValueError("After filtering by genes, no interactions remain.")

    @inject_docs(src=SOURCE, tgt=TARGET, cp=ComplexPolicy)
    def _filter_interactions_complexes(self, complex_policy: ComplexPolicy) -> None:
        """
        Filter the :attr:`interactions` by extracting genes from complexes.

        Parameters
        ----------
        complex_policy
            Policy on how to handle complexes. Valid options are:

                - `{cp.MIN.s!r}` - select gene with the minimum average expression. This is the same as in
                  :cite:`cellphonedb`.
                - `{cp.ALL.s!r}` - select all possible combinations between `{src!r}` and `{tgt!r}` complexes.

        Returns
        -------
        Nothing, just updates the following fields:

            - :attr:`interactions` - filtered interactions whose `{src!r}` and `{tgt!r}` are both in the data.

        Note that for ``complex_policy={cp.ALL.s!r}``, all pairwise comparisons within a complex are created,
        but no filtering happens at this stage - genes not present in the data are filtered at a later stage.
        """

        def find_min_gene_in_complex(_complex: str | None) -> str | None:
            # TODO(michalk8): how can this happen?
            if _complex is None:
                return None
            if "_" not in _complex:
                return _complex
            complexes = [c for c in _complex.split("_") if c in self._data.columns]
            if not len(complexes):
                return None
            if len(complexes) == 1:
                return complexes[0]

            df = self._data[complexes].mean()
            try:
                return str(df.index[df.argmin()])
            except ValueError as e:
                if "attempt to get argmin of an empty sequence" in str(e):
                    return str(df.index[0])
                else:
                    logg.error(e)

        if TYPE_CHECKING:
            assert isinstance(self._interactions, pd.DataFrame)
            assert isinstance(self.interactions, pd.DataFrame)

        if complex_policy == ComplexPolicy.MIN:
            logg.debug("DEBUG: Selecting genes from complexes based on minimum average expression")
            self.interactions[SOURCE] = self.interactions[SOURCE].apply(find_min_gene_in_complex)
            self.interactions[TARGET] = self.interactions[TARGET].apply(find_min_gene_in_complex)
        elif complex_policy == ComplexPolicy.ALL:
            logg.debug("DEBUG: Creating all gene combinations within complexes")
            src = self.interactions.pop(SOURCE).apply(lambda s: str(s).split("_")).explode()
            src.name = SOURCE
            tgt = self.interactions.pop(TARGET).apply(lambda s: str(s).split("_")).explode()
            tgt.name = TARGET

            self._interactions = pd.merge(self.interactions, src, how="left", left_index=True, right_index=True)
            self._interactions = pd.merge(self.interactions, tgt, how="left", left_index=True, right_index=True)
        else:
            raise NotImplementedError(f"Complex policy {complex_policy!r} is not implemented.")

    @property
    def interactions(self) -> pd.DataFrame | None:
        """The interactions."""  # noqa: D401
        return self._interactions

    def __repr__(self) -> str:
        return (
            f"<{self.__class__.__name__}"
            f"[n_interaction={len(self.interactions) if self.interactions is not None else None}]>"
        )

    def __str__(self) -> str:
        return repr(self)

@d.dedent
class PermutationTest(PermutationTestABC):
    """
    %(PT.full_desc)s

    Parameters
    ----------
    %(PT.parameters)s
    """  # noqa: D400

    @d.get_sections(base="PT_prepare_full", sections=["Parameters"])
    @d.dedent
    def prepare(
        self,
        interactions: Interaction_t | None = None,
        complex_policy: Literal["min", "all"] = ComplexPolicy.MIN.v,
        interactions_params: Mapping[str, Any] = MappingProxyType({}),
        transmitter_params: Mapping[str, Any] = MappingProxyType({"categories": "ligand"}),
        receiver_params: Mapping[str, Any] = MappingProxyType({"categories": "receptor"}),
        **_: Any,
    ) -> PermutationTest:
        """
        %(PT_prepare.full_desc)s

        Parameters
        ----------
        %(PT_prepare.parameters)s
        interactions_params
            Keyword arguments for :func:`omnipath.interactions.import_intercell_network` defining the interactions.
            These datasets from :cite:`omnipath` are used by default: `omnipath`, `pathwayextra`, `kinaseextra` and
            `ligrecextra`.
        transmitter_params
            Keyword arguments for :func:`omnipath.interactions.import_intercell_network` defining the transmitter
            side of intercellular connections.
        receiver_params
            Keyword arguments for :func:`omnipath.interactions.import_intercell_network` defining the receiver
            side of intercellular connections.

        Returns
        -------
        %(PT_prepare.returns)s
        """  # noqa: D400
        if interactions is None:
            from omnipath.interactions import import_intercell_network

            start = logg.info("Fetching interactions from `omnipath`")
            interactions = import_intercell_network(
                interactions_params=interactions_params,
                transmitter_params=transmitter_params,
                receiver_params=receiver_params,
            )
            if TYPE_CHECKING:
                assert isinstance(interactions, pd.DataFrame)

            logg.info(f"Fetched `{len(interactions)}` interactions\n    Finish", time=start)

            # we don't really care about these
            if SOURCE in interactions.columns:
                interactions.pop(SOURCE)
            if TARGET in interactions.columns:
                interactions.pop(TARGET)
            interactions.rename(
                columns={"genesymbol_intercell_source": SOURCE, "genesymbol_intercell_target": TARGET}, inplace=True
            )

            interactions[SOURCE] = interactions[SOURCE].str.replace("^COMPLEX:", "", regex=True)
            interactions[TARGET] = interactions[TARGET].str.replace("^COMPLEX:", "", regex=True)

        _ = super().prepare(interactions, complex_policy=complex_policy)
        return self

@d.dedent
def ligrec(
    adata: AnnData | SpatialData,
    cluster_key: str,
    interactions: Interaction_t | None = None,
    complex_policy: Literal["min", "all"] = ComplexPolicy.MIN.v,
    threshold: float = 0.01,
    corr_method: str | None = None,
    corr_axis: Literal["interactions", "clusters"] = CorrAxis.CLUSTERS.v,
    use_raw: bool = True,
    copy: bool = False,
    key_added: str | None = None,
    gene_symbols: str | None = None,
    **kwargs: Any,
) -> Mapping[str, pd.DataFrame] | None:
    """
    %(PT_test.full_desc)s

    Parameters
    ----------
    %(PT.parameters)s
    %(PT_prepare_full.parameters)s
    %(PT_test.parameters)s
    gene_symbols
        Key in :attr:`anndata.AnnData.var` to use instead of :attr:`anndata.AnnData.var_names`.

    Returns
    -------
    %(ligrec_test_returns)s
    """  # noqa: D400
    if isinstance(adata, SpatialData):
        adata = adata.table
    with _genesymbols(adata, key=gene_symbols, use_raw=use_raw, make_unique=False):
        return (  # type: ignore[no-any-return]
            PermutationTest(adata, use_raw=use_raw)
            .prepare(interactions, complex_policy=complex_policy, **kwargs)
            .test(
                cluster_key=cluster_key,
                threshold=threshold,
                corr_method=corr_method,
                corr_axis=corr_axis,
                copy=copy,
                key_added=key_added,
                **kwargs,
            )
        )

@d.dedent
def _analysis(
    data: pd.DataFrame,
    interactions: NDArrayA,
    interaction_clusters: NDArrayA,
    threshold: float = 0.1,
    n_perms: int = 1000,
    seed: int | None = None,
    n_jobs: int = 1,
    numba_parallel: bool | None = None,
    **kwargs: Any,
) -> TempResult:
    """
    Run the analysis as described in :cite:`cellphonedb`.

    This function runs the mean, percent and shuffled analysis.

    Parameters
    ----------
    data
        Array of shape `(n_cells, n_genes)`.
    interactions
        Array of shape `(n_interactions, 2)`.
    interaction_clusters
        Array of shape `(n_interaction_clusters, 2)`.
    threshold
        Percentage threshold for removing lowly expressed genes in clusters.
    %(n_perms)s
    %(seed)s
    n_jobs
        Number of parallel jobs to launch.
    numba_parallel
        Whether to use :class:`numba.prange` or not. If `None`, it's determined automatically.
    kwargs
        Keyword arguments for :func:`squidpy._utils.parallelize`, such as ``n_jobs`` or ``backend``.

    Returns
    -------
    Tuple of the following format:

        - `'means'` - array of shape `(n_interactions, n_interaction_clusters)` containing the means.
        - `'pvalues'` - array of shape `(n_interactions, n_interaction_clusters)` containing the p-values.
    """

    def extractor(res: Sequence[TempResult]) -> TempResult:
        assert len(res) == n_jobs, f"Expected to find `{n_jobs}` results, found `{len(res)}`."

        meanss: list[NDArrayA] = [r.means for r in res if r.means is not None]
        assert len(meanss) == 1, f"Only `1` job should've calculated the means, but found `{len(meanss)}`."
        means = meanss[0]
        if TYPE_CHECKING:
            assert isinstance(means, np.ndarray)

        pvalues = np.sum([r.pvalues for r in res if r.pvalues is not None], axis=0) / float(n_perms)
        assert means.shape == pvalues.shape, f"Means and p-values differ in shape: `{means.shape}`, `{pvalues.shape}`."

        return TempResult(means=means, pvalues=pvalues)

    groups = data.groupby("clusters")
    clustering = np.array(data["clusters"].values, dtype=np.int32)

    mean = groups.mean().values.T  # (n_genes, n_clusters)
    mask = groups.apply(lambda c: ((c > 0).sum() / len(c)) >= threshold).values.T  # (n_genes, n_clusters)
    # (n_cells, n_genes)
    data = np.array(data[data.columns.difference(["clusters"])].values, dtype=np.float64, order="C")
    # all 3 should be C contiguous

    return parallelize(  # type: ignore[no-any-return]
        _analysis_helper,
        np.arange(n_perms, dtype=np.int32).tolist(),
        n_jobs=n_jobs,
        unit="permutation",
        extractor=extractor,
        **kwargs,
    )(
        data,
        mean,
        mask,
        interactions,
        interaction_clusters=interaction_clusters,
        clustering=clustering,
        seed=seed,
        numba_parallel=numba_parallel,
    )

def _analysis_helper(
    perms: NDArrayA,
    data: NDArrayA,
    mean: NDArrayA,
    mask: NDArrayA,
    interactions: NDArrayA,
    interaction_clusters: NDArrayA,
    clustering: NDArrayA,
    seed: int | None = None,
    numba_parallel: bool | None = None,
    queue: SigQueue | None = None,
) -> TempResult:
    """
    Run the results of mean, percent and shuffled analysis.

    Parameters
    ----------
    perms
        Permutation indices. Only used to set the ``seed``.
    data
        Array of shape `(n_cells, n_genes)`.
    mean
        Array of shape `(n_genes, n_clusters)` representing mean expression per cluster.
    mask
        Array of shape `(n_genes, n_clusters)` containing `True` if the a gene within a cluster is
        expressed at least in ``threshold`` percentage of cells.
    interactions
        Array of shape `(n_interactions, 2)`.
    interaction_clusters
        Array of shape `(n_interaction_clusters, 2)`.
    clustering
        Array of shape `(n_cells,)` containing the original clustering.
    seed
        Random seed for :class:`numpy.random.RandomState`.
    numba_parallel
        Whether to use :class:`numba.prange` or not. If `None`, it's determined automatically.
    queue
        Signalling queue to update progress bar.

    Returns
    -------
    Tuple of the following format:

        - `'means'` - array of shape `(n_interactions, n_interaction_clusters)` containing the true test
          statistic. It is `None` if ``min(perms)!=0`` so that only 1 worker calculates it.
        - `'pvalues'` - array of shape `(n_interactions, n_interaction_clusters)`  containing `np.sum(T0 > T)`
          where `T0` is the test statistic under null hypothesis and `T` is the true test statistic.
    """
    rs = np.random.RandomState(None if seed is None else perms[0] + seed)

    clustering = clustering.copy()
    n_cls = mean.shape[1]
    return_means = np.min(perms) == 0

    # ideally, these would be both sparse array, but there is no numba impl. (sparse.COO is read-only and very limited)
    # keep it f64, because we're setting NaN
    res = np.zeros((len(interactions), len(interaction_clusters)), dtype=np.float64)
    numba_parallel = (
        (np.prod(res.shape) >= 2**20 or clustering.shape[0] >= 2**15) if numba_parallel is None else numba_parallel  # type: ignore[assignment]
    )

    fn_key = f"_test_{n_cls}_{int(return_means)}_{bool(numba_parallel)}"
    if fn_key not in globals():
        exec(
            compile(_create_template(n_cls, return_means=return_means, parallel=numba_parallel), "", "exec"), globals()  # type: ignore[arg-type]
        )
    _test = globals()[fn_key]

    if return_means:
        res_means: NDArrayA | None = np.zeros((len(interactions), len(interaction_clusters)), dtype=np.float64)
        test = partial(_test, res_means=res_means)
    else:
        res_means = None
        test = _test

    for _ in perms:
        rs.shuffle(clustering)
        test(interactions, interaction_clusters, data, clustering, mean, mask, res=res)

        if queue is not None:
            queue.put(Signal.UPDATE)

    if queue is not None:
        queue.put(Signal.FINISH)

    return TempResult(means=res_means, pvalues=res)


"""
CLI tool for uploading to zenodo
"""
import logging
import os
from functools import partial
from pathlib import Path

import click
from dotenv import load_dotenv
from openscm_zenodo.zenodo import create_new_zenodo_version, get_bucket_id, upload_file
from tqdm.contrib.concurrent import thread_map

DEPOSITION_ID_OLD: str = "7972657"
"""
Deposition ID of our Zenodo archive. This can be any version.
"""

ZENODO_URL: str = "zenodo.org"
"""Zenodo URL to upload to"""

ZENODO_FILE_NAME: str = "zenodo.json"
"""Name of the file in which the Zenodo metadata is written"""

openscm_zenodo_logger = logging.getLogger("openscm_zenodo")
openscm_zenodo_logger.setLevel(logging.INFO)

logFormatter = logging.Formatter(
    "%(levelname)s - %(asctime)s %(name)s %(threadName)s (%(module)s:%(funcName)s:%(lineno)d):  %(message)s",
    datefmt="%Y-%m-%d %H:%M:%S",
)
stdoutHandler = logging.StreamHandler()
stdoutHandler.setFormatter(logFormatter)

openscm_zenodo_logger.addHandler(stdoutHandler)

@click.command()
@click.argument(
    "bundle_path",
    type=click.Path(exists=True, file_okay=False, resolve_path=True, path_type=Path),
)
def upload_bundle(bundle_path: Path) -> None:
    """
    Upload bundle to zenodo

    Parameters
    ----------
    bundle_path
        Path to bundle
    """
    load_dotenv()

    new_version_deposition_id = create_new_zenodo_version(
        deposition_id=DEPOSITION_ID_OLD,
        zenodo_url=ZENODO_URL,
        token=os.environ["ZENODO_TOKEN"],
        deposit_metadata=bundle_path / ZENODO_FILE_NAME,
    )

    click.echo(f"Deposition ID to use: {new_version_deposition_id}")

    bucket_id = get_bucket_id(
        deposition_id=new_version_deposition_id,
        zenodo_url=ZENODO_URL,
        token=os.environ["ZENODO_TOKEN"],
    )

    click.echo(f"Uploading to bucket: {bucket_id}")

    files_to_upload = []
    for dir_path, _, files in os.walk(bundle_path):
        # Ignore interim outputs
        if "interim" in dir_path:
            continue

        for file in files:
            # Ignore unexecuted notebooks for upload
            if "unexecuted" in file:
                continue

            files_to_upload.append(Path(dir_path) / file)

    click.echo(f"{len(files_to_upload)} files to upload")

    thread_map(
        partial(
            upload_file,
            bucket=bucket_id,
            root_dir=str(bundle_path),
            zenodo_url=ZENODO_URL,
            token=os.environ["ZENODO_TOKEN"],
        ),
        (str(file) for file in files_to_upload),
        desc="File uploads",
        total=len(files_to_upload),
    )

if __name__ == "__main__":
    upload_bundle()