from collections import namedtuple
import abc
import datetime
import bisect
import logging
import pandas as pd
import numpy as np
from functools import partial
from pymongo import MongoClient
from sqlalchemy import create_engine, Table, Column, MetaData
from sqlalchemy_utils import database_exists, create_database
from dateutil.relativedelta import relativedelta
from data import (products, nrbys, months, years, fields, name_map, train_years, test_years, verify_years)
from lib.utils import ( contract_month_map, Credentials, SummStats, Singleton, Visitor,
                        US_Federal_Calendar, bmth_us, bday_us, bus_day_add)
from lib import GSCI_params
from data import GSCIData
import lib
import db

# Parameters, in order
# DB = num closes before first roll close that we start accumulating position
# acc_method = position accumulation method:
#        { 'linear' : proportional accumulation
#          'bullet' : immediate accumulation }
# DA = num closes after last roll close that we start liquidating position
# liq_method = position liquidation method:
#        { 'linear' : proportional accumulation
#          'bullet' : immediate accumulation }
# DBE = num days before expiry we must be flat

GSCI_data     = GSCIData()
GSCI_comp     = GSCI_data.GSCI_comp
GSCI_products = GSCI_data.products

ParamList     = GSCI_params.GSCIParamList
POS_COLS = ['Date', 'Prod1', 'Month1', 'Month2', 'Offset','Settle1', 'Settle2', 'Position', 'Settle12', '_merge']

def roll_spread_months(product, month):
    ''' Returns front month, back month as contract letter
    for the product roll during the give calendar month
    letter.

    offset      == 1 ===> front month occurs in current calendar year, back
                          month appears in following year
    year_offset == 1 ===> front month and back month occur in following calendar
                          year
    '''

    front_month = str(month)
    back_month  = str(1+(month%12))
    GSCI_comp_row = GSCI_comp.loc[GSCI_comp['Product'] == product]

    offset, year_offset = 0,0

    months = ( front_month, back_month)
    front_month_str, back_month_str = [ GSCI_comp_row[month].iloc[0] for month in months ]

    offset = 1 if contract_month_map[front_month_str] > contract_month_map[back_month_str] else 0
    year_offset = 1 if contract_month_map[front_month_str] < month else 0

    return front_month_str, back_month_str, offset, year_offset

def roll_dates_by_month(yr, mth):
    ''' roll dates by month, with rule specifiation
    '''

    yr, mth = int(yr), int(mth)
    dts = [dtime.date() for
           dtime in pd.date_range(datetime.date(yr, mth, 1) + relativedelta(days=-2) + bmth_us,
                                  periods=9,
                                  freq=bday_us)]

    return dts[4], dts[-1]

def create_roll_dates(days_offset, roll_date, establish=True):
    ''' Create roll dates, with rule specification
    '''
    _dates = list()

    if establish:
        last_date = roll_date

        while days_offset > 0:
            last_date -= bday_us
            days_offset -= 1
            _dates.append(last_date)
        _dates.reverse()
        df_dates = pd.DataFrame({'Date' : [dt.date() for dt in _dates] })
    else:
        first_date = roll_date

        while days_offset > 0:
            days_offset -= 1
            _dates.append(first_date)
            first_date += bday_us

        df_dates = pd.DataFrame( {'Date' : [dt.date() if not type(dt).__name__ == 'date' else
                                            dt for dt in _dates] })

    return df_dates

def create_pos( dates, price, lots, foll_year, product_dols, dols_per_tick, establishing):
    ''' Create position dataframe
    '''

    if establishing:
        pos_df = pd.DataFrame({ 'Position' : np.linspace(0,      -1*lots, 1+dates.shape[0])[1:] })
    else:
        pos_df = pd.DataFrame({ 'Position' : np.linspace(lots,   0,       1+dates.shape[0])[:-1]})
    pos_df = pd.concat([dates, pos_df], axis=1)

    price_pos = pd.merge(pos_df, price[foll_year], how='left', indicator=True)[POS_COLS]

    price_pos = price_pos.dropna(how='any')

    # Recalculate weights for new size, if necessary
    if price_pos.shape[0] != dates.shape[0]:
        if establishing:
            price_pos['Position'] = np.linspace(0,    -1*lots, 1+price_pos.shape[0])[1:]
        else:
            price_pos['Position'] = np.linspace(lots, 0,       1+price_pos.shape[0])[:-1]

    price_pos['Dols']          = product_dols
    price_pos['Price_mult']    = dols_per_tick
    price_pos['SubStrategy']   = 'EST' if establishing else 'LIQ'
    price_pos.rename(columns   = { 'Prod1' : 'Prod' }, inplace=True)

    return price_pos

class Backtester(Visitor):
    def __init__(self, product, param_obj=GSCI_params.GSCIParamList[0]):
        self._product     = product
        self._exchange    = name_map['exch_map'][self._product]
        self._credentials = Credentials()
        self._metrics_map = {'drawupK'   : partial(lib.max_drawup_levels,   mult=1e-3),
                             'drawdnK'   : partial(lib.max_drawdown_levels, mult=1e-3),
                             'meanretK'  : partial(lib.meanret_levels,      mult=1e-3),
                             'sharpe'    : lib.sharpe_levels,
                             'updnrat'   : lib.uprat_levels,
                             'nonnegrat' : lib.nonnegrat_levels,
                             'freq'      : lib.freq_levels,
                             'ampl'      : lib.ampl_levels}
        self._SpreadObj   = lib.Spread(product)
        self._db_name     = '_'.join(['STIR', self._exchange, self._product, 'SUMM'])
        self._paramObj    = param_obj
        self._params      = param_obj.params
        self._props       = param_obj.props
        self._props_name  = param_obj.props_name

        credentials       = self._credentials.DB_creds(db_override=self._db_name)
        self._DBConn      = self._DB_conn(*credentials)

    @staticmethod
    def _DB_conn(user, passwd, db):
        conn = create_engine('mysql+mysqldb://{}:{}@localhost/{}'.format(user,passwd,db))
        if not database_exists(conn.url):
            create_database(conn.url)
        return conn

    def __repr__(self):
        r = '_'.join([ self._product,
                       self._exchange,
                       str( self._params.DB ),
                       self._params.acc_method,
                       str( self._params.DA ),
                       self._params.liq_method ] )
        if self._props:
            r += '_{}'.format(self._props_name)
        return r

    def visit(self, element):
        pass

    def backtest_helper(self, total_dollars=1e6):

        if self._product not in GSCI_products:
            return None, None, None, None

        portfolio_results = pd.DataFrame()

        RPDW = GSCI_comp.loc[GSCI_comp['Product'] == self._product]['2017_RPDW'].iloc[0]
        days_before, est_method, days_after, liq_method, days_before_expiration = self._params

        S = lib.Spread(self._product)

        for month_num in range(1,13):
            front_month, back_month, offset, year_offset = roll_spread_months(self._product, month_num)

            if front_month == back_month and offset == 0:
                log_msg = 'Front month == back month for month number: {}'.format(month_num)
                logging.warn(log_msg)
                continue

            try:
                price = S._create_study(front_month, back_month, offset, train_only=False)
            except OperationError:
                logging.warn('Sql connection down for (month, year): ({}, {})'.format( month_num, year))

            all_years = sorted(price.keys())

            for year in all_years:
                curr_year,foll_year = year, str(int(year)+year_offset)
                if int(foll_year) > max([int(yr) for yr in all_years]):
                    log_msg = 'Spread window for (year, month number): ({}, {}) is beyond test period; skipping'.format(curr_year, month_num)
                    logging.warn(log_msg)
                    continue

                if not isinstance(price[year], pd.DataFrame):
                    log_msg = 'Price data for year {} missing'.format(year)
                    logging.warn(log_msg)
                    continue

                start_roll_date, end_roll_date = roll_dates_by_month(curr_year, month_num)

                # We will use an approximation to the weight of each component in the GSCI
                # The documents state that
                # IDW_d = Sum_c{CPW_D^d * DCRP_d^c)
                #
                # where
                # c = the designated contract
                # d = the S&P GSCI business day on which the calculation is made
                # DCRP = the Daily Contract Reference Price
                #
                # We will use the settlement price of the front month on the first day of
                # the establishing leg of the strategy in the above formula to calculate
                # holdings according to the percentage given in the RPDW column.

                est_dates_df  = create_roll_dates(days_before, start_roll_date, establish=True)
                liq_dates_df  = create_roll_dates(days_after, end_roll_date, establish=False)

                # Establishing position: calculation of number of units
                try:
                    DCRP_ref_date = est_dates_df['Date'].get_values()[0]
                    DCRP = float(price[foll_year][price[foll_year]['Date'] == DCRP_ref_date]['Open1'].get_values()[0])
                except KeyError:
                    logging.warn(
                        'year {} not contained in price matrix - needed for (month,year): ([], {})calculation'.format( foll_year,month_num, year))
                    continue
                except Exception as e:
                    days_back = 1
                    while days_back < 10:
                        try:
                            DCRP_ref_date = bus_day_add(est_dates_df['Date'].get_values()[0], -days_back)
                            DCRP = float(price[foll_year][price[foll_year]['Date'] == DCRP_ref_date]['Open1'].get_values()[0])
                            days_back += 1
                            break
                        except IndexError as e:
                            days_back += 1

                # Round to nearest integer
                # lots              = round(total_dollars * (RPDW / 100) / float(name_map['lotsize_map'][self._product])
                #                           / DCRP / float(name_map['mult_map'][self._product]), 0)

                # Don't
                lots              = total_dollars * (RPDW / 100) / float(name_map['lotsize_map'][self._product]) / \
                                    DCRP / float(name_map['mult_map'][self._product])

                product_dols      = total_dollars * (RPDW / 100)
                dols_per_tick     = float(name_map['lotsize_map'][self._product]) * float(name_map['mult_map'][self._product])

                # Establishing position
                est_price_pos = create_pos(est_dates_df, price, lots, foll_year, product_dols, dols_per_tick, True)
                est_price_pos = SummStats(est_price_pos).PL()

                # Liquidating position
                if self._props.get('liq_if_est_money', False):
                    est_levels   = pd.Series(est_price_pos['PL']).astype('float')
                    est_metrics  = pd.DataFrame({k:[v(est_levels)] for k,v in self._metrics_map.items()})
                    # No EST positions for this strategy
                    est_price_pos = None
                    liq_price_pos = None

                    if est_metrics.loc[0].sharpe > self._props.get('liq_cutoff'):
                        liq_price_pos = create_pos(liq_dates_df, price, lots, foll_year, product_dols, dols_per_tick, False)
                        liq_price_pos = SummStats(liq_price_pos).PL()
                else:
                    liq_price_pos = create_pos(liq_dates_df, price, lots, foll_year, product_dols, dols_per_tick, False)
                    liq_price_pos = SummStats(liq_price_pos).PL()

                portfolio_results = pd.concat([portfolio_results, est_price_pos, liq_price_pos])

        portfolio_results['RPDW'] = RPDW
        portfolio_results['Strategy'] = self.__repr__()

        portfolio_results = portfolio_results.sort_values('Date')
        portfolio_results = portfolio_results.dropna(how='any', subset=['Date'])

        levels  = pd.Series(portfolio_results['PL']).astype('float')

        if not levels.empty:

            summ_table_name = '_'.join(['GSCI_strat_summ', self.__repr__()])
            portfolio_results.to_sql(con=self._DBConn, name=summ_table_name, if_exists='replace', index=False)
            logging.info('Wrote to table {}'.format( summ_table_name ))

            metrics  = pd.DataFrame({k:[v(levels)] for k,v in self._metrics_map.items()})
            metrics_table_name = '_'.join(['GSCI_strat_metrics', self.__repr__()])
            metrics.to_sql(con=self._DBConn, name=metrics_table_name, if_exists='replace', index=False)
            logging.info('Wrote to table {}'.format( metrics_table_name ))

            return portfolio_results, metrics, summ_table_name, metrics_table_name

        return None, None, None, None

def GSCI_backtest(products=products):

    for product in products:
        for ind,param_obj in enumerate(ParamList):
            print('Product: {} ParamObj: {}'.format( product, ind ))
            B = Backtester(product, param_obj=param_obj)
            B.backtest_helper()


import logging
import math
from datetime import datetime
from pathlib import Path

import numpy as np
import torch
import yaml
from tqdm import tqdm

def float_samples_to_int16(y):
  """Convert floating-point numpy array of audio samples to int16."""
  if not issubclass(y.dtype.type, np.floating):
    raise ValueError('input samples not floating-point')
  return (y * np.iinfo(np.int16).max).astype(np.int16)

def int16_samples_to_float32(y):
  """Convert int16 numpy array of audio samples to float32."""
  if y.dtype != np.int16:
    raise ValueError('input samples not int16')
  return y.astype(np.float32) / np.iinfo(np.int16).max

class TqdmLoggingHandler(logging.Handler):
    def __init__(self, level=logging.NOTSET):
        super().__init__(level)

    def emit(self, record):
        try:
            msg = self.format(record)
            tqdm.write(msg)
            self.flush()
        except:
            self.handleError(record)

def create_logging(logs_dir, filemode):
    """Create log objective.

    Args:
      logs_dir (Path obj): logs directory
      filenmode: open file mode
    """
    logs_dir.mkdir(parents=True, exist_ok=True)

    i1 = 0

    while logs_dir.joinpath('{:04d}.log'.format(i1)).is_file():
        i1 += 1

    logs_path = logs_dir.joinpath('{:04d}.log'.format(i1))
    logging.basicConfig(
        level=logging.DEBUG,
        format='%(filename)s[line:%(lineno)d] %(levelname)s %(message)s',
        # format='%(asctime)s %(filename)s[line:%(lineno)d] %(levelname)s %(message)s',
        datefmt='%a, %d %b %Y %H:%M:%S',
        filename=logs_path,
        filemode=filemode)

    # Print to console
    console = logging.StreamHandler()
    console.setLevel(logging.INFO)
    formatter = logging.Formatter('%(name)-12s: %(levelname)-8s %(message)s')
    console.setFormatter(formatter)
    # logging.getLogger('').addHandler(console)
    logging.getLogger('').addHandler(TqdmLoggingHandler())

    dt_string = datetime.now().strftime('%a, %d %b %Y %H:%M:%S')
    logging.info(dt_string)
    logging.info('')

    return logging

def convert_ordinal(n):
    """Convert a number to a ordinal number

    """
    ordinal = lambda n: "%d%s" % (n,"tsnrhtdd"[(math.floor(n/10)%10!=1)*(n%10<4)*n%10::4])
    return ordinal(n)

def move_model_to_gpu(model, cuda):
    """Move model to GPU

    """
    # TODO: change DataParallel to DistributedDataParallel
    model = torch.nn.DataParallel(model)
    if cuda:
        logging.info('Utilize GPUs for computation')
        logging.info('Number of GPU available: {}\n'.format(torch.cuda.device_count()))
        model.cuda()
    else:
        logging.info('Utilize CPU for computation')
    return model

def count_parameters(model):
    """Count model parameters

    """
    params_num = sum(p.numel() for p in model.parameters() if p.requires_grad)
    logging.info('Total number of parameters: {}\n'.format(params_num))

def print_metrics(logging, writer, values_dict, it, set_type='train'):
    """Print losses and metrics, and write it to tensorboard

    Args:
      logging: logging
      writer: tensorboard writer
      values_dict: losses or metrics
      it: iter
      set_type: 'train' | 'valid' | 'test'
    """
    out_str = ''
    if set_type == 'train':
        out_str += 'Train: '
    elif set_type == 'valid':
        out_str += 'valid: '

    for key, value in values_dict.items():
        out_str += '{}: {:.3f},  '.format(key, value)
        writer.add_scalar('{}/{}'.format(set_type, key), value, it)
    logging.info(out_str)


# This is a basic example from discordpy's website, I plan
# on changing up some things to see what it does, and adding comments.
# I think I should be able to do this project fully because
# As much as people say the documentation sucks for
# discordpy, I think it is alright. Just not too technical though.

# This example requires the 'message_content' intent.

import discord  # Import the discord library

# Initialise client with intents (like permissions but not quite)
intents = discord.Intents.default() # Get defaults
intents.message_content = True      # Declare intent for message content
client = discord.Client(intents=intents)    # Init bot client

# Set the client's event listeners to do certain actions asynchronously
@client.event
async def on_ready():
    print(f'We have logged in as {client.user}')

@client.event
async def on_message(message):
    """
    Interact with Discord's API such that when a message is sent
    it ignores it if the author of the message is itself with a guard
    clause, then checks if the content of the message has a command.
    If the message starts with a command, the bot sends a message.
    """
    if message.author == client.user:
        return

    if message.content.startswith('$hello'):
        await message.channel.send('Hello!')

# Run bot with discord login token
client.run('your token here')


from collections import deque

def simplifyPath(path):
    folders = path.split('/')
    pathQ = deque()
    for folder in folders:
        pathQ.append(folder)

    simplifiedPath = deque()

    while len(pathQ) > 0:
        nextFolder = pathQ.popleft()
        if nextFolder == '..':
            if len(simplifiedPath) > 0:
                simplifiedPath.pop()
        elif nextFolder == '.' or nextFolder == '':
            pass
        else:
            simplifiedPath.append(nextFolder)

    out = '/'
    for i in simplifiedPath:
        out += i + '/'

    if len(out) > 1:
        return out[:-1]
    else:
        return out


from jukebox.models import Song, JukeBox, JukeUser, Stats, PlaylistMembership, Playlist
from django.contrib.auth.models import User
u = User.objects.all()[0]
# JukeUser.objects.all()[0]
user = JukeUser(user=u,firstname="Sir",lastname="Dummy")
user.save()
jb = JukeBox(name="DummyJuke",owner=user,is_public=True)
playlist = Playlist()
playlist.save()
stats = Stats()
stats.save()
jb.djs.add(user)
jb.playlist = playlist
jb.stats = stats
jb.save()


from multiprocessing import Semaphore
from typing import Callable

class H2O:
    def __init__(self):
        # set hydrogen semaphore to 0
        self.hSem = Semaphore(0)
        # set oxygen semaphore to 2, required to call oxygen once
        self.oSem = Semaphore(2)
        # using a mutual exclusion semaphore so that oxygen can acquire two counts of oSem
        self.mutex = Semaphore(1)

    # releaseHydrogen() outputs "H"

    def hydrogen(self, releaseHydrogen: Callable[[], None]) -> None:
        # calling hyrogen aquires one hydrogen semaphore, and releases one oxygen semaphore
        self.hSem.acquire()
        releaseHydrogen()
        self.oSem.release()

    # releaseOxygen() outputs "O"

    def oxygen(self, releaseOxygen: Callable[[], None]) -> None:
        # calling oxygen requires two hydrogen semaphores, as the oxygen to hydrogen ratio is 1:2
        # acquire and release mutual exclusion semaphore
        with self.mutex:
            self.oSem.acquire()
            self.oSem.acquire()
        releaseOxygen()
        # release two hydrogen semaphores, (release(n) in python 3.9)
        self.hSem.release()
        self.hSem.release()


import tensorflow as tf

import argparse
import os
import time
from tqdm import tqdm
import cv2
import sys

sys.path.append("./models")
sys.path.append("./data_utils")
sys.path.append("./losses")

from untied_model import deepfuse_triple_untied
from tied_model import deepfuse_triple_tied
from tied_model_double_encoder import deepfuse_triple_2encoder
from tied_model_triple_encoder import deepfuse_triple_3encoder
from refine_unet import refine_net
from PWCNet import pwc_net

from logCompressor import log_compressor
from transform_utils import *
from skimage.transform import resize
from tf_warp import backward_warp

parser = argparse.ArgumentParser()
parser.add_argument('--source_dir', default='./data_samples/test_data/', help='link to list of training patches')
parser.add_argument('--fusion_model', default='triple_encoder', help='tied|untied|double_encoder|triple_encoder')
parser.add_argument('--fusion_ckpt', default='./checkpoints/pretrained_triple_encoder/model170001.ckpt', help='path to fusion ckpt')
parser.add_argument('--refine_ckpt', default='./checkpoints/pretrained_refine/model55.ckpt', help='path to refine ckpt')
parser.add_argument('--flow_ckpt', default='./checkpoints/pretrained_pwc/pwcnet.ckpt-595000', help='path to flow ckpt')
parser.add_argument('--ref_label', default=2, help = 'which image to use as reference')
parser.add_argument('--hdr_channels', default=1, help = 'weather to concatenate hdr along channels')
parser.add_argument('--gpu', default=0, help = 'which gpu to use')

def refine_dark(im1, im2, im12, im21, image_shape, batch_size=1, reuse=False, fl=None):
    """
    Called to generate darker version of reference image
    Args:
        im1: Darker Under-exposed Image
        im2: Brighter reference image
        im21: im2 mapped to exposure of im1
        im12 : im1 mapped to exposure of im2
        fl: Optional flow if computed offline or using some other algorithm
    Returns:
        output_im: corrected im21
    """
    inp1 = tf.expand_dims(im2, 1)
    inp2 = tf.expand_dims(im12, 1)
    forward_vals = tf.concat([inp1, inp2], axis =1)
    backward_vals = tf.concat([inp2, inp1], axis = 1)
    if fl is not None:
        flow = fl
    else:
        pwc_inp = tf.concat([forward_vals, backward_vals], axis=0)
        flow, _ = pwc_net(pwc_inp)
    warped_im1 = backward_warp(im1, flow[:batch_size])
    warped_im1 = tf.reshape(warped_im1, tf.concat([image_shape,[3]], -1))
#    forward_occlusion, backward_occlusion = occlusion(flow[:batch_size], flow[batch_size:])
    refine_input = tf.concat([warped_im1, im21, flow[:batch_size]], axis=-1)
    output_map = refine_net(refine_input, name='dark_refine', reuse=reuse)
    output_im = output_map*im21+(1.-output_map)*warped_im1
    return output_im

def refine_bright(im1, im2, im21, image_shape, batch_size=1, reuse=False, fl=None):
    """
    Called to generate brighter version of reference image
    Args:
        im1: Bright Over-exposed Image
        im2: Darker reference image
        im21: im2 mapped to exposure of im1
        fl: Optional flow if computed offline or using some other algorithm
    Returns:
        output_im: corrected im21
    """
    inp1 = tf.expand_dims(im21, 1)
    inp2 = tf.expand_dims(im1, 1)
    forward_vals = tf.concat([inp1, inp2], axis =1)
    backward_vals = tf.concat([inp2, inp1], axis = 1)
    if fl is not None:
        flow = fl
    else:
        pwc_inp = tf.concat([forward_vals, backward_vals], axis=0)
        flow, _ = pwc_net(pwc_inp)
    warped_im1 = backward_warp(im1, flow[:batch_size])
    warped_im1 = tf.reshape(warped_im1, tf.concat([image_shape,[3]], -1))
#    forward_occlusion, backward_occlusion = occlusion(flow[:batch_size], flow[batch_size:])
    refine_input = tf.concat([warped_im1, im21, flow[:batch_size]], axis=-1)
    output_map = refine_net(refine_input, name='dark_refine', reuse=reuse)
    output_im = output_map*im21+(1.-output_map)*warped_im1
    return output_im

def log_com(inp):
   return (tf.log(1+(5000.0*inp))/tf.log(1+5000.0))

def infer(source_dir, ref_label, hdr_channels, fusion_model, fusion_ckpt, refine_ckpt, flow_ckpt):
    im1 = tf.placeholder(tf.float32,shape=[1,None,None,3])
    im2 = tf.placeholder(tf.float32, shape=[1,None,None,3])
    im3 = tf.placeholder(tf.float32, shape=[1,None,None,3])
    im23 = tf.placeholder(tf.float32, shape=[1,None,None,3])
    im12 = tf.placeholder(tf.float32, shape=[1,None,None,3])
    im21 = tf.placeholder(tf.float32, shape=[1,None,None,3])
    im13 = tf.placeholder(tf.float32, shape=[1,None,None,3])
    image_dims = tf.placeholder(tf.int32, shape=[3])
    ev_bias = tf.placeholder(tf.float32, shape=[1])

    if ref_label==2:
        im1_ref = refine_dark(im1, im2, im12, im21, image_dims)
        im2_ref = im2
        im3_ref = refine_bright(im3, im2, im23, image_dims, reuse=True)
    if ref_label==1:
        im1_ref = im1
        im2_ref = refine_bright(im2, im1, im12, image_dims)
        im3_ref = refine_bright(im3, im1, im13, image_dims, reuse=True)

    if hdr_channels:
        im1_ref_hdr = tf_ldr_to_hdr(im1_ref, 2**(ev_bias*0))
        im2_ref_hdr = tf_ldr_to_hdr(im2_ref, 2**(ev_bias*1))
        im3_ref_hdr = tf_ldr_to_hdr(im3_ref, 2**(ev_bias*2))
        inp1 = tf.concat([im1_ref, im1_ref_hdr], axis=-1)
        inp2 = tf.concat([im2_ref, im2_ref_hdr], axis=-1)
        inp3 = tf.concat([im3_ref, im3_ref_hdr], axis=-1)
    else:
        inp1 = im1_ref
        inp2 = im2_ref
        inp3 = im3_ref

    if fusion_model == 'tied':
        final_hdr = deepfuse_triple_tied(inp1, inp2, inp3)
    elif fusion_model == 'untied':
        final_hdr = deepfuse_triple_untied(inp1, inp2, inp3)
    elif fusion_model == 'double_encoder':
        final_hdr = deepfuse_triple_2encoder(inp1, inp2, inp3)
    elif fusion_model == 'triple_encoder':
        final_hdr = deepfuse_triple_3encoder(inp1, inp2, inp3)
    else:
        print('Choose either of tied or untied model')
        return 0

    with tf.Session() as sess:
        sess.run(tf.global_variables_initializer())
        pwc_saver = tf.train.Saver([v for v in tf.all_variables() if "pwcnet" in v.name])
        pwc_saver.restore(sess, flow_ckpt)
        pwc_saver = tf.train.Saver([v for v in tf.all_variables() if "refine" in v.name])
        pwc_saver.restore(sess, refine_ckpt)
        saver = tf.train.Saver([v for v in tf.all_variables() if "DeepFuse"  in v.name])
        saver.restore(sess, fusion_ckpt)
        for path in os.listdir(source_dir):
            print('Generating HDR for ', path)
            img1 = np.expand_dims((cv2.imread(source_dir+path+'/1.tif', -1)[:,:,::-1].astype(np.float32))/65535.0, 0)
            img2 = np.expand_dims((cv2.imread(source_dir+path+'/2.tif', -1)[:,:,::-1].astype(np.float32))/65535.0, 0)
            img3= np.expand_dims((cv2.imread(source_dir+path+'/3.tif', -1)[:,:,::-1].astype(np.float32))/65535.0, 0)
            padding_x = 0
            padding_y = 0
            if (img1.shape[1])%64 != 0:
                padding_x = int(64*((img1.shape[1]//64)+1)-(img1.shape[1]))
            if (img1.shape[2])%64 != 0:
                padding_y = int(64*((img1.shape[2]//64)+1)-(img1.shape[2]))

            img1 = np.pad(img1, [[0,0],[padding_x//2, padding_x-(padding_x//2)], [padding_y//2, padding_y-(padding_y//2)], [0,0]], mode='reflect')
            img2 = np.pad(img2, [[0,0],[padding_x//2, padding_x-(padding_x//2)], [padding_y//2, padding_y-(padding_y//2)], [0,0]], mode='reflect')
            img3 = np.pad(img3, [[0,0],[padding_x//2, padding_x-(padding_x//2)], [padding_y//2, padding_y-(padding_y//2)], [0,0]], mode='reflect')

            with open(source_dir+path+'/exposure.txt') as f:
                lines = f.readlines()
                ev0 = float(lines[0][:-1])
                ev1 = float(lines[1][:-1])
                ev2 = float(lines[2][:-1])

            img12 = hdr_to_ldr(ldr_to_hdr(img1, 2**ev0), 2**ev1)
            img21 = hdr_to_ldr(ldr_to_hdr(img2, 2**ev1), 2**ev0)
            img23 = hdr_to_ldr(ldr_to_hdr(img2, 2**ev1), 2**ev2)
            img13 = hdr_to_ldr(ldr_to_hdr(img1, 2**ev0), 2**ev2)

            img1_ref, img2_ref, img3_ref, hdr_im = sess.run([im1_ref, im2_ref, im3_ref, final_hdr], feed_dict = {im1:img1, im2:img2,im3:img3,
                                                                                im12:img12, im21:img21, im23:img23,
                                                                                ev_bias: [ev1], image_dims:img1.shape[:-1]})
#            img1_ref = (img1_ref[0,padding_x//2:-1*(padding_x-(padding_x//2)),int(padding_y/2):-1*(padding_y-(padding_y//2)),:]*65535.0).astype(np.uint16)
#            img2_ref = (img2_ref[0,padding_x//2:-1*(padding_x-(padding_x//2)),int(padding_y/2):-1*(padding_y-(padding_y//2)),:]*65535.0).astype(np.uint16)
#            img3_ref = (img3_ref[0,padding_x//2:-1*(padding_x-(padding_x//2)),int(padding_y/2):-1*(padding_y-(padding_y//2)),:]*65535.0).astype(np.uint16)
            radiance_writer(source_dir+path+'/output.hdr', hdr_im[0,padding_x//2:-1*(padding_x-(padding_x//2)),int(padding_y/2):-1*(padding_y-(padding_y//2)),:])

if __name__ == '__main__':
    opts = parser.parse_args()
    os.environ["CUDA_VISIBLE_DEVICES"] = str(opts.gpu)
    infer(source_dir = opts.source_dir,
          ref_label = opts.ref_label,
          hdr_channels = opts.hdr_channels,
          fusion_ckpt = opts.fusion_ckpt,
          fusion_model = opts.fusion_model,
          refine_ckpt = opts.refine_ckpt,
          flow_ckpt = opts.flow_ckpt)


# ID - 10984191
# DEPARTMENT - BIOMEDICAL ENGINEERING

import qrcode
import PySimpleGUI as sg

sg.theme('BlueMono')
font = ('Monaco', 18)

qr_image=[sg.Image('',key='-QRCODE-',size=(250,250),background_color='grey')]

layout=[
    [sg.Text('Enter URL:')],
    [sg.Input('',key='-URL-',justification='center')],
    [sg.Button('Create',key='-submit-',expand_x=True,button_color='red')],
    [sg.Column([qr_image],justification='center')],
]

wind = sg.Window('QRCODE Generator App',layout, font=font)

while True:
    event,values = wind.read()
    if event==sg.WIN_CLOSED:
        break
    elif event == '-submit-':
        url = values['-URL-']
        if url:
            image= qrcode.make(url)
            image.save('qr.png')
            wind['-QRCODE-'].update('qr.png')
wind.close()


import sqlite3

from flask import Flask
from opencensus.ext.flask.flask_middleware import FlaskMiddleware

from opentelemetry import trace
from opentelemetry.exporter.jaeger.thrift import JaegerExporter
from opentelemetry.instrumentation.sqlite3 import SQLite3Instrumentor
from opentelemetry.sdk.resources import Resource
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace.export import BatchSpanProcessor
from opentelemetry.shim.opencensus import install_shim

DB = "example.db"

# Set up OpenTelemetry
tracer_provider = TracerProvider(
    resource=Resource(
        {
            "service.name": "opencensus-shim-example-flask",
        }
    )
)
trace.set_tracer_provider(tracer_provider)

# Configure OTel to export traces to Jaeger
tracer_provider.add_span_processor(
    BatchSpanProcessor(
        JaegerExporter(
            agent_host_name="localhost",
            agent_port=6831,
        )
    )
)
tracer = tracer_provider.get_tracer(__name__)

# Install the shim to start bridging spans from OpenCensus to OpenTelemetry
install_shim()

# Instrument sqlite3 library
SQLite3Instrumentor().instrument()

# Setup Flask with OpenCensus instrumentation
app = Flask(__name__)
FlaskMiddleware(app)

# Setup the application database
def setup_db():
    with sqlite3.connect(DB) as con:
        cur = con.cursor()
        cur.execute(
            """
            CREATE TABLE IF NOT EXISTS movie(
                title,
                year,
                PRIMARY KEY(title, year)
            )
            """
        )
        cur.execute(
            """
            INSERT OR IGNORE INTO movie(title, year) VALUES
                ('Mission Telemetry', 2000),
                ('Observing the World', 2010),
                ('The Tracer', 1999),
                ('The Instrument', 2020)
            """
        )

setup_db()

@app.route("/")
def hello_world():
    lines = []
    with tracer.start_as_current_span("query movies from db"), sqlite3.connect(
        DB
    ) as con:
        cur = con.cursor()
        for title, year in cur.execute("SELECT title, year from movie"):
            lines.append(f"
{title} is from the year {year}
")

    with tracer.start_as_current_span("build response html"):
        html = f"
{''.join(lines)}
"

    return html


import pandas as pd
import nltk
import os
from output_folder import get_output_file, get_output_folder

STEMMING = False
LEMMING =  True

OUTPUT_PATH = os.getenv('OUTPUT_PATH')
OUT_FOLDER = get_output_folder('so_data/')
PROCESSED_CSV = get_output_file('processed/so_questions.csv')

# Utility functions
w_tokenizer = nltk.tokenize.WhitespaceTokenizer()
lemmatizer = nltk.stem.WordNetLemmatizer()
stemmer = nltk.stem.snowball.SnowballStemmer("english")

def lemmatize_text(text):
    return [lemmatizer.lemmatize(w) for w in w_tokenizer.tokenize(text)]

def stem_text(text):
    return [stemmer.stem(w) for w in w_tokenizer.tokenize(text)]

def df_to_file(_index, _row, _fieldname):
    issue_name = str(int(_row['Id']))
    issue_body = str(_row[_fieldname])

    if STEMMING:
        issue_body = ' '.join(stem_text(issue_body))
    if LEMMING:
        issue_body = ' '.join(lemmatize_text(issue_body))
    with open(OUT_FOLDER + issue_name + '.txt', 'a+') as issue_file:
        issue_file.write(issue_body + '\n')

def export_to_mallet():
  # load stack overflow questions
  so = pd.read_csv(PROCESSED_CSV)

  # output one question per file as mallet requires (title only)
  for index, row in so.iterrows():
    df_to_file(index, row, 'Title')

  # output comments one question per file as mallet requires (body only)
  for index, row in so.iterrows():
    df_to_file(index, row, 'Body')

if __name__ == '__main__':
  export_to_mallet()


# _*_ coding:utf-8 _*_

import unittest
import json

from unittest.mock import Mock

from 测试用例.接口自动化.接口自动化_V2.接口测试.public_method import create_code
from 测试用例.接口自动化.接口自动化_V2.接口测试.测试套件.生产工单下发 import workorder_changestatus
from 测试用例.接口自动化.接口自动化_V2.接口管理.物流管理.物料需求 import MaterialRequirement
from 测试用例.接口自动化.接口自动化_V2.接口管理.物流管理.物流单 import Logistics
from 测试用例.接口自动化.接口自动化_V2.接口管理.物流管理.物流单报工 import LogBillReport
from public import params

class TestMaterialeRequirementWareHouse(unittest.TestCase):
    """物料需求分配仓库测试类"""

    def setUp(self):
        self.mr = MaterialRequirement()
        self.lg = Logistics()
        self.lbr = LogBillReport()

        self.mr_create_data = {
            "uda2": "小米仓",
            "demandType": "WAREHOUSE",
            "scheduleDate": "2018-07-15 10:51:11",
            "factoryStationRef": {
                "stationName": "CPU"
            },

            "code": "test",
            "delivery": "params.bmFactoryWorkStationCPUGid",     #
            "sourceType": "CREATE",
            "reqWarehouseGidRef": {
                "name": "小米仓"      #
            },
            "demandGid": params.bmWarehouseGid,
            "planDate": "2018-07-01 10:51:11",
            "smBusiUnitGidRef": {
                "busiUnitName": "小米科技"
            },
            "imeLogisticsMaterialRequirementDetailList": [
                {
                    "originalQty": "8",
                    "bmMeasurementUnitGidRef": {
                        "gid": params.bmMeasurementUnitGid
                    },
                    "factoryStationRef": {
                        "stationName": "CPU"
                    },
                    "factoryWorkStationGid": params.bmFactoryWorkStationCPUGid,
                    "demandQty": "8",
                    "materialRef": {
                        "code": "Q11",
                        "name": "处理器",
                        "bmMeasurementUnitGidRef": {
                            "name": "件"
                        }
                    },
                    "supplyWarehouseGid": params.bmWarehouseGid,
                    "warehouseRef": {
                        "name": "小米仓"
                    },
                    "eventPayload": {},
                    "materialGid": params.MaterielCPUGid
                }
            ],
            "smBusiUnitGid": params.busiUnitGid,
            "uda1": "Q1"
        }

        self.mr_mauwh_data = {
            "gids": [],
            "warehouseGid": params.bmWarehouseGid2
        }

        self.mr_query_data = {
            "query": {
                "query": [
                    {
                        "field": "code",
                        "type": "eq",
                        "value": "TB002420180427",
                        "operator": "and"
                    }
                ]
            }
        }

    def test_mrwh_manuallyWarehouse_1(self):
        """
        物料需求手工分配仓库
        :return:
        """
        code = create_code()
        self.mr_create_data['code'] = code
        self.mr.materialmrequirement_add = Mock(side_effect=self.mr.materialmrequirement_add)
        resp_mr_add = self.mr.materialmrequirement_add(self.mr_create_data)

        mr_gid_list = []
        mr_gid = resp_mr_add.get('data')
        mr_gid_list.append(mr_gid)

        self.mr_mauwh_data['gids'] = mr_gid_list
        self.mr.materialmrequirement_manuallyWarehouse = Mock(side_effect=self.mr.materialmrequirement_manuallyWarehouse)
        resp_mr_mauwh = self.mr.materialmrequirement_manuallyWarehouse(self.mr_mauwh_data)

        self.assertEqual(1, len(resp_mr_mauwh.get('data')))

    def test_mrwh_autoWarehouse_1(self):
        """
        物料需求按条件自动分配仓库
        :return:
        """
        code = create_code()
        self.mr_create_data['code'] = code
        self.mr.materialmrequirement_add = Mock(side_effect=self.mr.materialmrequirement_add)
        resp_mr_add = self.mr.materialmrequirement_add(self.mr_create_data)

        mr_gid = resp_mr_add.get('data')

        self.mr_query_data['query']['query'][0]['value'] = code
        self.mr.materialmrequirement_autoWarehouse = Mock(side_effect=self.mr.materialmrequirement_autoWarehouse)
        resp_mr_auwh = self.mr.materialmrequirement_autoWarehouse(self.mr_query_data)

        # 验证返回中的物料需求GID存在且为1
        self.assertEqual(1, len(resp_mr_auwh.get('data')))

    def test_mrwh_autoWarehouseBy_1(self):
        """
        物料需求按条件自动分配仓库
        :return:
        """
        code = create_code()
        self.mr_create_data['code'] = code
        self.mr.materialmrequirement_add = Mock(side_effect=self.mr.materialmrequirement_add)
        resp_mr_add = self.mr.materialmrequirement_add(self.mr_create_data)

        mr_gid_list = []
        mr_gid = resp_mr_add.get('data')
        mr_gid_list.append(mr_gid)

        self.mr_mauwh_data['gids'] = mr_gid_list
        self.mr.materialmrequirement_autoWarehouseByGid = Mock(
            side_effect=self.mr.materialmrequirement_autoWarehouseByGid)
        resp_mr_aubwh = self.mr.materialmrequirement_autoWarehouseByGid(mr_gid_list)

        self.assertEqual(1, len(resp_mr_aubwh.get('data')))

if __name__ == '__main__':
    unittest.main()

测试用例.接口自动化.接口自动化_V2.接口管理.物流管理.物料需求.MaterialRequirement

测试用例.接口自动化.接口自动化_V2.接口管理.物流管理.物流单.Logistics

测试用例.接口自动化.接口自动化_V2.接口管理.物流管理.物流单报工.LogBillReport


from treq import get

def done(response):
    print(response.code)
    reactor.stop()

def eb(e):
    print(e)
    reactor.stop()

get("https://httpbin.org/get").addCallback(done).addErrback(eb)

from twisted.internet import reactor

reactor.run()


import boto3
from fastapi import UploadFile, HTTPException, status
from typing import IO, List
import io

from promy_api.infra.env import settings
from promy_api.infra.custom_type import FileType

def get_s3obj():
    return boto3.client(
        's3', aws_access_key_id=settings.aws_access_key_id, aws_secret_access_key=settings.aws_secret_access_key
    )

def get_s3url(key: str) -> str:
    return f"https://{settings.s3_bucket}.s3.{settings.s3_region}.amazonaws.com/{key}"

def upload_image(file: UploadFile, file_type: FileType, id: int) -> str:
    upload_key = f"{file_type}/{id}.{file.filename.split('.')[-1]}"
    s3 = get_s3obj()
    try:
        s3.upload_fileobj(file.file, Bucket=settings.s3_bucket, Key=upload_key, ExtraArgs={'ACL': 'public-read'})
        return get_s3url(upload_key)
    except Exception as e:
        raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=f"S3에 업로드하는데 실패했습니다")

def delete_image(file_type: FileType, file_name: str) -> None:
    s3 = get_s3obj()
    try:
        s3.delete_object(Bucket=settings.s3_bucket, Key=f"{file_type}/{file_name}")
    except Exception as e:
        raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=f"S3 오브젝트를 삭제하는데 실패했습니다")


import logging
from pathlib import Path
from typing import Any, Dict, Set, Union

from constants import DASH,PACKAGE_NAME,Strategy
from calculator import Calculator
from s3id_result import S3IDResultMatch, S3IDResultMismatch

logging.basicConfig(level=logging.DEBUG)

log = logging.getLogger(f"{PACKAGE_NAME}.{__name__}")

class Comparator(Calculator):
    @classmethod
    def run(
        cls,
        etag: str,
        local_file_path: Path,
        threshold_in_bytes: int,
        partition_set_in_bytes: Set[int],
    ) -> Dict[str, Union[int, str]]:
        """
        Iteratively create ETag values over a range of chunk sizes in hopes of matching 'etag'
        and return a summary result of a match or non-match.
        """
        result: Dict[str, Any] = Comparator(
            etag,
            local_file_path,
            threshold_in_bytes,
            partition_set_in_bytes,
        ).summary()
        if result["match"]:
            log.info(f"S3 object etag matched local file: '{local_file_path}'")
            return result
        else:
            log.info(
                f"S3 object etag did NOT match local file: '{local_file_path}' with any chunk sizes"
            )
            return result

    def __init__(
        self,
        etag: str,
        local_file_path: Path,
        threshold_in_bytes: int,
        partition_set_in_bytes: Set[int],
    ) -> None:
        if not etag:
            raise ValueError("'etag' cannot be blank.")

        self.etag: str = etag
        super().__init__(local_file_path, (Strategy.MULTI_PART if DASH in self.etag else Strategy.SINGLE_PART), threshold_in_bytes)

        if not isinstance(partition_set_in_bytes, set) or not all(
            isinstance(x, int) for x in partition_set_in_bytes
        ):
            raise ValueError("'partition_set_in_bytes' must be a set of integers")

        self.partition_set_in_bytes = partition_set_in_bytes

    def summary(self) -> Dict[str, Union[int, str]]:
        """
        Calculate an ETag from the local file contents.  If a matching ETag has been
        found to the object_etag, return:
        { "match": True, "partition_set_in_bytes": <partition_set_in_bytes> }

        If no match is found, return: { "match": False }
        """
        for partition_in_bytes in sorted(self.partition_set_in_bytes):
            result = self.calculate(partition_in_bytes)
            signature = result["signature"]
            log.debug(
                f"Calculated ETag: {signature} with chunk size (bytes): {partition_in_bytes}"
            )
            if self.etag.replace('"', "") == signature.replace('"', ""):
                return S3IDResultMatch(
                    signature, result["strategy"], partition_in_bytes
                ).summary()

        return S3IDResultMismatch(signature).summary()


from fast_class1 import NMS_1d, Openvideo, SlidingWindow, load_pre_trained_model,\
                        get_class_labels, I3DVideoStackAppend\

import facenet
import time
import cv2
import numpy as np
import tensorflow as tf
sess = tf.Session()
import i3d
import sys

#sys.path.append('D:/GitHub/darknet')
#import darknet
from PIL import Image
from utils import utils
from socket import *
from yolo import YOLO
from deep_sort import preprocessing
from deep_sort import nn_matching
from deep_sort.detection import Detection
from deep_sort.tracker import Tracker
from tools import generate_detections as gdet
#from deep_sort.detection import Detection as ddet

#'''tcpip parameter'''
#serverName = '127.0.0.1'
#serverPort = 50007

'''deepsort parameter'''
yolo = YOLO()
# Definition of the parameters
max_cosine_distance = 0.3
nn_budget = None
nms_max_overlap = 1.0

# deep_sort
model_filename = 'model_data/mars-small128.pb'
encoder = gdet.create_box_encoder(model_filename,batch_size=1)
metric = nn_matching.NearestNeighborDistanceMetric("cosine", max_cosine_distance, nn_budget)
tracker = Tracker(metric)

'''Initial parameter'''
SAMPLE_FRAME = 16
SLIDING_WINDOW_LAYER = 1
MAXIMUM_FRAME = ( SAMPLE_FRAME * 2**(SLIDING_WINDOW_LAYER-1) )
MODE = 'video' # camera or video
out = cv2.VideoWriter('0318demo.avi',cv2.VideoWriter_fourcc(*'XVID'), 30.0, (1920,1080))
_SAMPLE_VIDEO_FRAMES = 16
_IMAGE_SIZE = 224
NUM_CLASSES = 12
_BATCH_SIZE = 1
FPS = 0.0
STEP = 4
zoomrate = 2.25
totalresult = []
ALLVideoStack = []
Picked_BBox = []
people = {}
colors=[(255,0,0),(0,255,0),(0,0,255),(255,0,255),(128,0,0),(0,128,0),(0,0,128),
        (128,0,128),(128,128,0),(0,128,128),(128,128,128),
        (128,0,255),(128,255,0),(0,128,255),(255,128,0),(255,0,128),
        (255,128,128),(128,255,128),(128,128,255)]

classes_name = 'ClassDay2.0.list'
_CHECKPOINT_PATHS = {'rgb_Day': 'models/rgb_model-318300'}

'''Definition of tensorflow'''
FLAGS = tf.flags.FLAGS
tf.flags.DEFINE_string('eval_type', 'rgb', 'rgb, flow, or joint')
tf.flags.DEFINE_boolean('imagenet_pretrained', True, '')
tf.logging.set_verbosity(tf.logging.INFO)
eval_type = FLAGS.eval_type
imagenet_pretrained = FLAGS.imagenet_pretrained

'''Placeholder of input images'''
labels_placeholder = tf.placeholder(tf.float32, [_BATCH_SIZE, NUM_CLASSES])
keep_prob = tf.placeholder(tf.float32)

'''Build the proposal network with i3d'''
# Proposal RGB input has 3 channels.
proposal_input = tf.placeholder(tf.float32,
    shape=(_BATCH_SIZE, _SAMPLE_VIDEO_FRAMES, _IMAGE_SIZE, _IMAGE_SIZE, 3))

with tf.variable_scope('RGB'):
  proposal_model = i3d.InceptionI3d( NUM_CLASSES, spatial_squeeze=True, final_endpoint='Logits' )
  proposal_logits, _ = proposal_model( proposal_input, is_training=True, dropout_keep_prob=keep_prob )

proposal_variable_map = {}
proposal_saver_savedata = {}
for variable in tf.global_variables():
  if variable.name.split('/')[0] == 'RGB':
     proposal_variable_map[variable.name.replace(':0', '')] = variable
proposal_saver_savedata =  tf.train.Saver(var_list=proposal_variable_map, reshape=True)

model_logits = proposal_logits

'''Tensorflow output definition'''
model_predictions = tf.nn.softmax(model_logits)
output_class = tf.argmax(model_predictions, 1)

'''Initialize the tensorflow variavles'''
init = tf.group(tf.global_variables_initializer(), tf.local_variables_initializer())

'''Get the class labels'''
answer_names = get_class_labels(classes_name)

#class_path='config/coco.names'
#classes = utils.load_classes(class_path)
frame_num = 0
cv2.namedWindow('Real-time Multiple Person Action Recognition System', cv2.WINDOW_NORMAL)
cv2.resizeWindow('Real-time Multiple Person Action Recognition System', 640, 480)
sess.run(init)
load_pre_trained_model(sess,  _CHECKPOINT_PATHS['rgb_Day'], proposal_saver_savedata)

with Openvideo(filename = r'0713original.mp4', mode = MODE) as vc:

    while(vc.isOpened()):

        rval , original_frame = vc.read()
        if rval == True:
            frame_num = frame_num + 1
            '''Timer start'''
    #           out.write(original_frame)

#            original_frame = original_frame[:,240:1680]
            fframe = cv2.resize(original_frame,(640,480))
#            frame = fframe.copy()
#            frame = infer(frame)
            ti1 = time.time()
            '''yolov3 + deepsort'''
            image = Image.fromarray(fframe[...,::-1]) #bgr to rgb
            boxs = yolo.detect_image(image)
            features = encoder(fframe,boxs)
            detections = [Detection(bbox, 1.0, feature) for bbox, feature in zip(boxs, features)]
            boxes = np.array([d.tlwh for d in detections])
            scores = np.array([d.confidence for d in detections])
            indices = preprocessing.non_max_suppression(boxes, nms_max_overlap, scores)
            detections = [detections[i] for i in indices]
            tracker.predict()
            tracker.update(detections)

            for track in tracker.tracks:
                if not track.is_confirmed() or track.time_since_update > 1:
                    continue
                bbox = track.to_tlbr()
                #bbox ----->  (int(bbox[0]), int(bbox[1])), (int(bbox[2]), int(bbox[3])
                '對應座標'
                bbox = np.maximum(bbox, 0)
                ID = track.track_id
                x1 = int(bbox[0])
                y1 = int(bbox[1])
                box_h = int(bbox[3]-y1)
                box_w = int(bbox[2]-x1)
                '1080原圖切人物'
                crop_person = original_frame[int(y1*zoomrate):int((y1+box_h)*zoomrate),
                                             int(x1*zoomrate):int((x1+box_w)*zoomrate)]

                'new ID ??'
                try:
                    'old ID'
                    personname = people[ID]
                except KeyError:
                    'have new person run facenet'
                    personname = facenet.infer(crop_person)
                    if personname!=None:
                        people[ID] = str(personname[0])

                if( frame_num%STEP == 0):

                    '''zoom in ?'''
                    if box_h > 160:

                        '人物夠大直接做模糊'
                        back_frame = cv2.GaussianBlur(fframe, (0, 0), sigmaX=15, sigmaY=15)
                        _back_frame = back_frame.copy()
                        crop_person_s = fframe[y1:y1+box_h,x1:x1+box_w]
                        _back_frame[y1:y1+box_h,x1:x1+box_w] = crop_person_s

                    else:
                        '人物太小'
                        '找人物中心點位置'

                        center_x = int((x1+ 0.5*box_w)*zoomrate)
                        center_y = int((y1+ 0.5*box_h)*zoomrate)

                        '切出人物'
#                        crop_person_l = original_frame[int(y1*zoomrate):int((y1+box_h)*zoomrate),int(x1*zoomrate):int((x1+box_w)*zoomrate)]

                        '以人物為中心切出480*640背景'
                        cropy1 = center_y-240
                        cropy2 = cropy1+480
                        cropx1 = center_x-320
                        cropx2 = cropx1+640

                        '若超出邊界則對齊邊界'
                        if cropx1 < 0:
                            cropx1 = 0
                            cropx2 = 640

                        if cropy1 < 0:
                            cropy1 = 0
                            cropy2 = 480

                        if cropy2 > 1080:
                            cropy1 = 600
                            cropy2 = 1080

                        if cropx2 > 1440:
                            cropx1 = 800
                            cropx2 = 1440

                        '模糊背景'
                        ori_blur_frame = cv2.GaussianBlur(original_frame, (0, 0), sigmaX=15)
                        '人物貼到模糊背景'
                        ori_blur_frame[int(y1*zoomrate):int((y1+box_h)*zoomrate),int(x1*zoomrate):int((x1+box_w)*zoomrate)] = crop_person
                        '切出zoomin後辨識畫面'
                        zin_blur_frame = ori_blur_frame[cropy1:cropy2,cropx1:cropx2]

                        '儲存'
                        _back_frame = zin_blur_frame

                    '''Stack the video'''
                    try:
                        I3DVideoStackAppend(ALLVideoStack[ID].append, _back_frame)
                    except IndexError:
                        for addspc in range (ID-len(ALLVideoStack)+1):
                            ALLVideoStack.append([])
                        I3DVideoStackAppend(ALLVideoStack[ID].append, _back_frame)

                    if( len(ALLVideoStack[ID]) > MAXIMUM_FRAME ):
                        ALLVideoStack[ID].pop(0)

                    '''Sliding window segmentation'''
                    with SlidingWindow(ALLVideoStack[ID], LayersOfScale=SLIDING_WINDOW_LAYER) as SlidingSegment:

                        rgb_buffer = SlidingSegment.OutputStack

                        if SlidingSegment.State:

                            for slidinglayer in range(SlidingSegment.State):

                                class_out = sess.run([model_predictions], feed_dict={proposal_input: rgb_buffer[slidinglayer], keep_prob: 1})
                                a=list(class_out[0][0])
                                current_label = int(a.index(max(a)))
#                                print(current_label)
                                BBox_data = [(frame_num-((slidinglayer+1)*STEP*SAMPLE_FRAME)), frame_num, max(a),current_label]
                            try:
                                totalresult[ID].append(BBox_data)
                            except IndexError:
                                for ept in range (ID-len(totalresult)+1):
                                    totalresult.append([])
                                totalresult[ID].append(BBox_data)

                            '1D NMS'
                            if( len(totalresult[ID]) == 7):
                                try:
                                    Picked_BBox[ID] = NMS_1d(totalresult[ID])
                                except IndexError:
                                    for apt in range (ID-len(Picked_BBox)+1):
                                        Picked_BBox.append([])
                                    Picked_BBox[ID] = NMS_1d(totalresult[ID])
                                totalresult[ID].clear()

#            'tcpoutput = [ID, action_index, Ix, Iy, ...x5...]'
#            tcpoutput = [0]*20
#            outputnumber = 0
            for track in tracker.tracks:
                if not track.is_confirmed() or track.time_since_update > 1:
                    continue
                bbox = track.to_tlbr()
                #bbox ----->  (int(bbox[0]), int(bbox[1])), (int(bbox[2]), int(bbox[3])

                '對應座標'
                bbox = np.maximum(bbox, 0)
                ID = track.track_id
                x1 = int(bbox[0])
                y1 = int(bbox[1])
                box_h = int(bbox[3]-y1)
                box_w = int(bbox[2]-x1)

                try:
                    NMS_result_class = answer_names[Picked_BBox[ID][-1][3]]
#                    tcpoutput[outputnumber*4:outputnumber*4+4] = [ID, Picked_BBox[ID][-1][3], int((x1+(box_w/2))/3), int((y1+box_h)/3)]
#                    outputnumber = outputnumber+1
                except IndexError:
                    NMS_result_class = 'Background'

                color = colors[int(ID) % len(colors)]
                cv2.rectangle(fframe, (x1, y1), (x1+box_w, y1+box_h), color, 4)

                try:
                    personname = people[ID]
                except KeyError:
                    personname = 'Unknown'

                cv2.rectangle(fframe, (x1-2, y1-22), (x1+50 +(len(str(ID))+len(personname))*15, y1), color, -1)
                cv2.rectangle(fframe, (x1-2, y1-49), (x1 +len(NMS_result_class)*16, y1-27), color, -1)
                cv2.putText(fframe,'ID:'+str(ID)+' '+str(personname), (x1,y1-4),cv2.FONT_HERSHEY_COMPLEX_SMALL, 1, (255, 255, 255), 1, cv2.LINE_AA)
                cv2.putText(fframe, NMS_result_class, (x1,y1-31),cv2.FONT_HERSHEY_COMPLEX_SMALL, 1, (255, 255, 255), 1, cv2.LINE_AA)
            '''Timer end'''
            FPS = 1/(time.time()-ti1)
            cv2.putText(fframe, 'FPS:' + '{:5.1f}'.format(FPS), (20, 20),cv2.FONT_HERSHEY_COMPLEX_SMALL, 1, (0, 0, 255), 1, cv2.LINE_AA)

    #                print(tcpoutput)
    #                clientSocket =socket(AF_INET,SOCK_STREAM)
    #                clientSocket.connect((serverName,serverPort))
    #                sentence = tcpoutput
    #                clientSocket.send(bytes(sentence))

            cv2.imshow('Real-time Multiple Person Action Recognition System', fframe)
    #                out.write(frame)
            key = cv2.waitKey(1) & 0xFF
            if key == ord('q'):
                break
        else:
            break

out.release()
cv2.destroyAllWindows()


import pytest

from aiohttp.test_utils import TestClient
from pytest_mock import MockerFixture

from ahriman.models.build_status import BuildStatusEnum
from ahriman.models.package import Package
from ahriman.models.user_access import UserAccess
from ahriman.web.views.v1.status.packages import (PackagesView)

async def test_get_permission() -> None:
    """
    must return correct permission for the request
    """
    for method in ("GET",):
        request = pytest.helpers.request("", "", method)
        assert await PackagesView.get_permission(request) == UserAccess.Read
    for method in ("POST",):
        request = pytest.helpers.request("", "", method)
        assert await PackagesView.get_permission(request) == UserAccess.Full

def test_routes() -> None:
    """
    must return correct routes
    """
    assert PackagesView.ROUTES == ["/api/v1/packages"]

async def test_get(client: TestClient, package_ahriman: Package, package_python_schedule: Package) -> None:
    """
    must return status for all packages
    """
    await client.post(f"/api/v1/packages/{package_ahriman.base}",
                      json={"status": BuildStatusEnum.Success.value, "package": package_ahriman.view()})
    await client.post(f"/api/v1/packages/{package_python_schedule.base}",
                      json={"status": BuildStatusEnum.Success.value, "package": package_python_schedule.view()})
    response_schema = pytest.helpers.schema_response(PackagesView.get)

    response = await client.get("/api/v1/packages")
    assert response.status == 200
    json = await response.json()
    assert not response_schema.validate(json, many=True)

    packages = [Package.from_json(item["package"]) for item in json]
    assert packages
    assert {package.base for package in packages} == {package_ahriman.base, package_python_schedule.base}

async def test_get_with_pagination(client: TestClient, package_ahriman: Package,
                                   package_python_schedule: Package) -> None:
    """
    must return paginated status for packages
    """
    await client.post(f"/api/v1/packages/{package_ahriman.base}",
                      json={"status": BuildStatusEnum.Success.value, "package": package_ahriman.view()})
    await client.post(f"/api/v1/packages/{package_python_schedule.base}",
                      json={"status": BuildStatusEnum.Success.value, "package": package_python_schedule.view()})
    request_schema = pytest.helpers.schema_request(PackagesView.get, location="querystring")
    response_schema = pytest.helpers.schema_response(PackagesView.get)

    payload = {"limit": 1, "offset": 1}
    assert not request_schema.validate(payload)
    response = await client.get("/api/v1/packages", params=payload)
    assert response.status == 200
    json = await response.json()
    assert not response_schema.validate(json, many=True)

    packages = [Package.from_json(item["package"]) for item in json]
    assert packages
    assert {package.base for package in packages} == {package_python_schedule.base}

async def test_post(client: TestClient, mocker: MockerFixture) -> None:
    """
    must be able to reload packages
    """
    load_mock = mocker.patch("ahriman.core.status.watcher.Watcher.load")

    response = await client.post("/api/v1/packages")
    assert response.status == 204
    load_mock.assert_called_once_with()


from torch import nn
import torch

class BaseLay1(nn.Module):
    def __init__(self, in_channels):
        super(BaseLay1, self).__init__()
        self.lay = nn.Sequential(
            nn.Conv2d(in_channels, in_channels, kernel_size=3, padding=1, stride=1),
            nn.ReLU(),
            nn.Conv2d(in_channels, in_channels, kernel_size=3, padding=1, stride=1),
            nn.ReLU(),
            nn.Conv2d(in_channels, in_channels, kernel_size=3, padding=1, stride=1),
            nn.ReLU()
        )

    def forward(self, intputs):
        return self.lay(intputs) + intputs

class BaseLay2(nn.Module):
    def __init__(self, in_channels):
        super(BaseLay2, self).__init__()
        self.lay = nn.Sequential(
            nn.Conv2d(in_channels, in_channels, kernel_size=3, stride=1, padding=1),
            nn.ReLU(),
            nn.Conv2d(in_channels, in_channels, kernel_size=3, stride=1, padding=1),
            nn.ReLU(),
            nn.Conv2d(in_channels, in_channels, kernel_size=1, stride=1, padding=0),
            nn.ReLU()
        )

    def forward(self, intputs):
        return self.lay(intputs) + intputs

class Model_v1(nn.Module):
    def __init__(self, in_channels, targets):
        super(Model_v1, self).__init__()
        base_lay1 = BaseLay1(128)
        base_lay2 = BaseLay2(128)
        self.x1_lay1 = nn.Sequential(
            nn.Conv2d(in_channels, 128, kernel_size=3, padding=(1, 0), stride=1),
            nn.ReLU(),
            nn.Conv2d(128, 128, kernel_size=3, padding=1, stride=1),
            nn.ReLU(),
            nn.Conv2d(128, 128, kernel_size=1, padding=0, stride=1),
            nn.ReLU()
        )

        lays = []
        for i in range(4):
            lays.append(base_lay1)
        self.x1_lay2 = nn.Sequential(*lays)

        self.x2_lay1 = nn.Sequential(
            nn.Conv2d(in_channels, 128, kernel_size=3, padding=1, stride=1),
            nn.ReLU(),
            nn.Conv2d(128, 128, kernel_size=3, padding=1, stride=1),
            nn.ReLU(),
            nn.Conv2d(128, 128, kernel_size=1, padding=0, stride=1),
            nn.ReLU()
        )

        lays = []
        for i in range(4):
            lays.append(base_lay2)
        self.x2_lay2 = nn.Sequential(*lays)

        lays = []
        for i in range(8):
            lays.append(base_lay1)
        self.lay = nn.Sequential(*lays)
        self.flatten = nn.Flatten()
        self.cf = nn.Sequential(
            nn.Linear(4 * 7 * 128, 1024),
            nn.ReLU(),
            nn.Linear(1024, targets)
        )

    def cul(self, x1, x2):
        x1 = self.x1_lay1(x1)
        x1 = self.x1_lay2(x1)
        x2 = self.x2_lay1(x2)
        x2 = self.x2_lay2(x2)
        # print(x1.shape, x2.shape)
        x = torch.cat((x1, x2), dim=2)
        x = self.lay(x)
        x = self.flatten(x)
        x = self.cf(x)
        return x

    def forward(self, x1, x2):
        return self.cul(x1, x2)

    def predict(self, x1, x2, mask):
        x = self.cul(x1, x2)
        for idx, item in enumerate(mask):
            x[idx][item == 0] = -100.0
        return x

if __name__ == '__main__':
    x1 = torch.randint(2, (2, 71, 3, 9), dtype=torch.float)
    x2 = torch.randint(2, (2, 71, 1, 7), dtype=torch.float)
    mask = torch.randint(2, (1, 35))
    model = Model_v1(71, 35)
    # print(model)
    print(model.predict(x1, x2, mask).argmax(1))


# -*- coding: utf-8 -*-
import os
from Qt.QtWidgets import *
from Qt.QtCore import *
from Qt.QtGui import *
import maya.cmds as mc
from miraLibs.pipeLibs import pipeFile, pipeMira
from miraLibs.pyLibs import join_path

def get_proxy_dir():
    obj = pipeFile.PathDetails.parse_path()
    project = obj.project
    asset_name = obj.asset_name
    root_dir = pipeMira.get_root_dir(project)
    proxy_dir = join_path.join_path2(root_dir, project, "assets", "proxy", asset_name, "static")
    return proxy_dir

def export_ass():
    proxy_dir = get_proxy_dir()
    meshes = mc.ls(type="mesh", long=1)
    progress_dialog = QProgressDialog("Ass Exporting...", 'Cancel', 0, len(meshes))
    progress_dialog.setWindowModality(Qt.WindowModal)
    progress_dialog.setMinimumWidth(350)
    progress_dialog.show()
    for index, mesh in enumerate(meshes):
        progress_dialog.setValue(index)
        if progress_dialog.wasCanceled():
            break
        mc.select(mesh, r=1)
        new_name = mesh.replace("|", "@")
        mc.file(os.path.join(proxy_dir, "%s.ass" % new_name),
                options="-mask 8;-lightLinks 0;-boundingBox;-shadowLinks 0",
                f=1, type="ASS Export", shader=0, exportSelected=1)

if __name__ == "__main__":
    export_ass()


from functools import partial
import pickle
from graphein.protein.config import ProteinGraphConfig
from graphein.protein.graphs import construct_graph
from graphein.protein.features.sequence import compute_esm_embedding
from graphein.protein.features.sequence.utils import (
    subset_by_node_feature_value,
)
from graphein.protein.features.nodes.amino_acid import (
    amino_acid_one_hot,
    meiler_embedding,
    expasy_protein_scale,
    hydrogen_bond_acceptor,
    hydrogen_bond_donor,
)
from graphein.protein.edges.distance import (
    add_distance_threshold,
    add_peptide_bonds,
    add_hydrophobic_interactions,
    add_hydrogen_bond_interactions,
    add_disulfide_interactions,
    add_ionic_interactions,
    add_aromatic_interactions,
    add_aromatic_sulphur_interactions,
    add_cation_pi_interactions,
)
from graphein.ml import GraphFormatConvertor
import graphein
import networkx as nx
import numpy as np
import torch
import freesasa
graphein.verbose(enabled=False)

def load_protein_as_graph(pdb_file, output_file=None):
    """
    Loads a protein chain PDB file as a Graphein networkx graph.

    Parameters
    ----------
    pdb_file : str
        Path to PDB file.
    output_file : str, optional
        Path to output file to save graph as pickle, by default None

    Returns
    -------
    nx.Graph
    """
    config = ProteinGraphConfig(
        node_metadata_functions= [
            amino_acid_one_hot,
            meiler_embedding,
            expasy_protein_scale,
            hydrogen_bond_acceptor,
            hydrogen_bond_donor,
        ],
        edge_construction_functions=[       # List of functions to call to construct edges.
            add_peptide_bonds,
            add_hydrophobic_interactions,
            add_aromatic_interactions,
            add_disulfide_interactions,
            add_ionic_interactions,
            add_hydrogen_bond_interactions,
            add_aromatic_sulphur_interactions,
            add_cation_pi_interactions,
            partial(add_distance_threshold, long_interaction_threshold=5, threshold=10.),
        ],
        graph_metadata_functions=[
                             esm_residue_embedding,
                             partial(solvent_accessible_surface_area, pdb_file=pdb_file)
                             ]
    )

    graph = construct_graph(config=config, path=pdb_file, verbose=False)
    if output_file is not None:
        with open(output_file, "wb") as f:
            pickle.dump(graph, f)
    return graph

def graphein_to_pytorch_graph(graphein_graph, node_attr_columns: list, edge_attr_columns: list, edge_kinds: set, labels):
    """
    Converts a Graphein graph to a pytorch-geometric Data object.
    """
    if edge_attr_columns is None:
        edge_attr_columns = []
    columns = [
            "chain_id",
            "coords",
            "edge_index",
            "kind",
            "node_id",
            "residue_number",
        ] + node_attr_columns + edge_attr_columns
    convertor = GraphFormatConvertor(src_format="nx", dst_format="pyg", columns=columns, verbose = None)
    data = convertor(graphein_graph)
    data_dict= data.to_dict()
    x_data = []
    for x in node_attr_columns:
        if data_dict[x].ndim == 1:
            x_data.append(torch.atleast_2d(data_dict[x]).T)
        else:
            x_data.append(torch.atleast_2d(data_dict[x]))
    data.x = torch.hstack(x_data).float()
    if len(edge_attr_columns) > 0:
        edge_attr_data = []
        for x in edge_attr_columns:
            if data_dict[x].ndim == 1:
                edge_attr_data.append(torch.atleast_2d(data_dict[x]).T)
            else:
                edge_attr_data.append(torch.atleast_2d(data_dict[x]))
        data.edge_attr = torch.hstack(edge_attr_data).float()
    # prune edge_index and edge_attr based on edge_kinds
    edge_index = []
    edge_attr = []
    for i, kind in enumerate(data_dict["kind"]):
        if len(kind.intersection(edge_kinds)) > 0:
            edge_index.append([data.edge_index[0, i], data.edge_index[1, i]])
            if len(edge_attr_columns) > 0:
                edge_attr.append(data.edge_attr[i])
    data.edge_index = torch.tensor(edge_index).T.long()
    if len(edge_attr_columns) > 0:
        data.edge_attr = torch.tensor(edge_attr).float()
    data.pos = data.coords.float()
    data.y = torch.zeros(data.num_nodes)
    for i, res_num in enumerate(data.residue_number):
        if f"{data.chain_id[i]}_{res_num.item()}" in labels:
            data.y[i] = 1
    return data

def solvent_accessible_surface_area(G: nx.Graph, pdb_file) -> nx.Graph:
    #Add SASA as a node feature using freesasa
    structure = freesasa.Structure(str(pdb_file))
    result = freesasa.calc(structure)

    for node, data in G.nodes(data=True):
        chain_id = data['chain_id']
        residue_number = str(data['residue_number'])

        # Calculate the SASA for each node/residue
        G.nodes[node]['sasa'] = 0
        G.nodes[node]['rel_sasa'] = 0
        try:
            G.nodes[node]['sasa'] = result.residueAreas()[chain_id][residue_number].total
            G.nodes[node]['rel_sasa'] = result.residueAreas()[chain_id][residue_number].relativeTotal
        except KeyError:
            pass
    return G

def esm_residue_embedding(
    G: nx.Graph,
    model_name: str = "esm1b_t33_650M_UR50S",
    output_layer: int = 33,
) -> nx.Graph:
    """
    Computes ESM residue embeddings from a protein sequence and adds the to the
    graph.

        *Biological Structure and Function Emerge from Scaling Unsupervised*
        *Learning to 250 Million Protein Sequences* (2019)
        Rives, Alexander and Meier, Joshua and Sercu, Tom and Goyal, Siddharth
        and Lin, Zeming and Liu, Jason and Guo,
        Demi and Ott, Myle and Zitnick, C. Lawrence and Ma, Jerry and Fergus,
        Rob

        *Transformer protein language models are unsupervised structure learners*
        (2020) Rao, Roshan M and Meier, Joshua and Sercu, Tom and Ovchinnikov,
        Sergey and Rives, Alexander

    **Pre-trained models**

    =========                     ====== ====== ======= ============= =========
    Full Name                     layers params Dataset Embedding Dim Model URL
    =========                     ====== ====== ======= ============= =========
    ESM-1b esm1b_t33_650M_UR50S   33    650M   UR50/S     1280        https://dl.fbaipublicfiles.com/fair-esm/models/esm1b_t33_650M_UR50S.pt
    ESM1-main esm1_t34_670M_UR50S 34    670M   UR50/S     1280        https://dl.fbaipublicfiles.com/fair-esm/models/esm1_t34_670M_UR50S.pt
    esm1_t34_670M_UR50D           34    670M   UR50/D     1280        https://dl.fbaipublicfiles.com/fair-esm/models/esm1_t34_670M_UR50D.pt
    esm1_t34_670M_UR100           34    670M   UR100      1280        https://dl.fbaipublicfiles.com/fair-esm/models/esm1_t34_670M_UR100.pt
    esm1_t12_85M_UR50S            12    85M    UR50/S     768         https://dl.fbaipublicfiles.com/fair-esm/models/esm1_t12_85M_UR50S.pt
    esm1_t6_43M_UR50S             6     43M    UR50/S     768         https://dl.fbaipublicfiles.com/fair-esm/models/esm1_t6_43M_UR50S.pt
    =========                     ====== ====== ======= ============= =========

    :param G: ``nx.Graph`` to add esm embedding to.
    :type G: nx.Graph
    :param model_name: Name of pre-trained model to use.
    :type model_name: str
    :param output_layer: index of output layer in pre-trained model.
    :type output_layer: int
    :return: ``nx.Graph`` with esm embedding feature added to nodes.
    :rtype: nx.Graph
    """

    for chain in G.graph["chain_ids"]:
        sequence = G.graph[f"sequence_{chain}"]
        embeddings = []
        for i in range(0, len(sequence), 1024): #TODO: improve chunking using overlap
            partial_embedding = compute_esm_embedding(
                sequence[i : i + 1022],
                representation="residue",
                model_name=model_name,
                output_layer=output_layer,
            )
            embeddings.append(partial_embedding[0, 1:-1])
        embedding = np.concatenate(embeddings, axis=0)
        assert len(embedding) == len(sequence)
        subgraph = subset_by_node_feature_value(G, "chain_id", chain)

        for i, (n, d) in enumerate(subgraph.nodes(data=True)):
            G.nodes[n]["esm_embedding"] = embedding[i]

    return G


import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns

metrics = pd.read_csv('metrics.csv')
metrics.set_index('Trait', inplace=True)
# print(metrics)

traits = ['O', 'C', 'E', 'A', 'N']
Y = ['Image','Audio']
Z = ['False', 'True']

metrics_dict = {}
for y in Y:
	metrics_dict[y] = {}
	for z in Z:
		metrics_dict[y][z] = {}
		for trait in traits:
				metrics_dict[y][z][trait]= metrics.loc[trait, str(y)+' ' + z]

print(metrics_dict)

sns.set()
for y in metrics_dict.keys():
	count= 0
	fig, axes = plt.subplots(nrows=1, ncols=1)
	# fig.suptitle("Audio Metrics for Y = "+str(y))
	# for z in metrics_dict[y].keys():
		# print(y,z,metrics_dict[y][z])
	df = pd.DataFrame(index=['Debiased Model', 'Traditional (Biased) Model'], columns=traits)
	df.loc['Debiased Model', :] = metrics_dict[y]['False']
	df.loc['Traditional (Biased) Model',:] = metrics_dict[y]['True']
	print(df)
		# df_T = df.transpose()
		# df_T = df_T.iloc[::-1]
	df.plot.barh()
	plt.show()
	# a = df.plot.barh(ax=axes[count], color = ['r','g'], legend=False)
	# axes[count].axvline(x=1)
	# 	# if z=='E':
	# 	# 	axes[count].set_title('Ethnicity')
	# 	# else:
	# 	# 	axes[count].set_title('Gender')
	# count+=1
	# for container in a.containers:
	# 		# print(container)
	# 	a.bar_label(container, fmt='%.2f')

	# lines = []
	# labels = []

	# for ax in fig.axes:
	#     Line, Label = ax.get_legend_handles_labels()
	#     lines.extend(Line)
	#     labels.extend(Label)

	# fig.legend(lines[0:3], labels[0:3], loc='upper right')
	# plt.show()


import numpy as np # linear algebra
import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)
#from sklearn.linear_model import LogisticRegression
#import matplotlib.pyplot as plt
from sklearn.metrics import explained_variance_score, mean_absolute_error, mean_squared_error, mean_squared_log_error
#from sklearn.linear_model import LinearRegression
from sklearn import linear_model
from sklearn.utils import shuffle
from sklearn.preprocessing import StandardScaler

import itertools
#from sklearn.model_selection import GridSearchCV

#https://www.kaggle.com/lpkirwin/fivethirtyeight-elo-ratings/
#https://en.wikipedia.org/wiki/Elo_rating_system
#https://fivethirtyeight.com/features/how-we-calculate-nba-elo-ratings/

def elo_pred(elo1, elo2):
    '''Calculates expected value E of team'''
    return(1. / (10. ** (-(elo1 - elo2) / 400.) + 1.))

def expected_margin(elo_diff):
    '''Calculates Expected Margin of Victory (see footnote 2 of 538 article)'''
    return((7.5 + 0.006 * elo_diff))

def elo_update(w_elo, l_elo, margin, K=20.):
    elo_diff = w_elo - l_elo
    pred = elo_pred(w_elo, l_elo)
    mult = ((margin + 3.) ** 0.8) / expected_margin(elo_diff)
    update = K * mult * (1 - pred)
    return(pred, update)

def elo_back_update(w_elo, l_elo, margin, K=20.):
    elo_diff = w_elo - l_elo
    pred = elo_pred(w_elo, l_elo)
    mult = ((margin + 3.) ** 0.8) / expected_margin(elo_diff)
    update = -K * mult * (1-pred)
    return(pred, update)

def final_elo_per_season(df, team_id):
    d = df.copy()
    d = d.loc[(d.WTeamID == team_id) | (d.LTeamID == team_id), :]
    d.sort_values(['Season'], inplace=True)
    d.drop_duplicates(['Season'], keep='last', inplace=True)
    w_mask = d.WTeamID == team_id
    l_mask = d.LTeamID == team_id
    d['SeasonElo'] = None
    d.loc[w_mask, 'SeasonElo'] = d.loc[w_mask, 'w_elo']
    d.loc[l_mask, 'SeasonElo'] = d.loc[l_mask, 'l_elo']
    out = pd.DataFrame({
        'TeamID': team_id,
        'Season': d.Season,
        'SeasonElo': d.SeasonElo
    })
    return(out)

def elos_per_season(df, team_id):
    d = df.copy()
    d = d.loc[(d.WTeamID == team_id) | (d.LTeamID == team_id), :]
    d.sort_values(['Season'], inplace=True)
    #d.drop_duplicates(['Season'], keep='last', inplace=True)
    w_mask = d.WTeamID == team_id
    l_mask = d.LTeamID == team_id
    d['SeasonElo'] = None
    d.loc[w_mask, 'SeasonElo'] = d.loc[w_mask, 'w_elo']
    d.loc[l_mask, 'SeasonElo'] = d.loc[l_mask, 'l_elo']
    out = pd.DataFrame({
        'GameID': d.GameID,
        'TeamID': team_id,
        'Season': d.Season,
        'SeasonElo': d.SeasonElo
    })
    return(out)

def season_elos(rs, K=20., HOME_ADVANTAGE=100.):
    '''
    Inputs:
    rs = dataframe of regular season results with columns: W/LTeamID, W/LScore
    K = K parameter that measures the volatility of the model. High K means elo ratings change more after one game and vice versa.
    HOME_ADVANTAGE = measures the value of home court advantage. The higher this parameter, the stronger home court advantage is.
    Thus at a high value, if a road team beats a home team, then more elo points are transferred to the road team.

    Outputs:
    preds = The estimated predictions of results based on the logistic curve on the 538 site.
    season_elos = the elo ratings at the end of the season for each team
    '''
    team_ids = set(rs.WTeamID).union(set(rs.LTeamID))
    # This dictionary will be used as a lookup for current
    # scores while the algorithm is iterating through each game
    elo_dict = dict(zip(list(team_ids), [1500] * len(team_ids)))
    # Elo updates will be scaled based on the margin of victory
    rs['margin'] = rs.WScore - rs.LScore
    # I'm going to iterate over the games dataframe using
    # index numbers, so want to check that nothing is out
    # of order before I do that.
    assert np.all(rs.index.values == np.array(range(rs.shape[0]))), "Index is out of order."
    preds = []
    w_elo = []
    l_elo = []

    # Loop over all rows of the games dataframe
    for row in rs.itertuples():

        # Get key data from current row
        w = row.WTeamID
        l = row.LTeamID
        margin = row.margin
        wloc = row.WLoc

        # Does either team get a home-court advantage?
        w_ad, l_ad, = 0., 0.
        if wloc == "H":
            w_ad += HOME_ADVANTAGE
        elif wloc == "A":
            l_ad += HOME_ADVANTAGE

        # Get elo updates as a result of the game
        pred, update = elo_update(elo_dict[w] + w_ad,
                                  elo_dict[l] + l_ad,
                                  margin,
                                 K)
        elo_dict[w] += update
        elo_dict[l] -= update

        # Save prediction and new Elos for each round
        preds.append(pred)
        w_elo.append(elo_dict[w])
        l_elo.append(elo_dict[l])

    rs['w_elo'] = w_elo
    rs['l_elo'] = l_elo
    df_list = [final_elo_per_season(rs, id) for id in team_ids]
    season_elos = pd.concat(df_list)
    return season_elos, preds

def elo_history(rs, K=20., HOME_ADVANTAGE=100.):

    team_ids = set(rs.WTeamID).union(set(rs.LTeamID))
    # This dictionary will be used as a lookup for current
    # scores while the algorithm is iterating through each game
    elo_dict = dict(zip(list(team_ids), [1500] * len(team_ids)))
    # Elo updates will be scaled based on the margin of victory
    rs['margin'] = rs.WScore - rs.LScore
    # I'm going to iterate over the games dataframe using
    # index numbers, so want to check that nothing is out
    # of order before I do that.
    assert np.all(rs.index.values == np.array(range(rs.shape[0]))), "Index is out of order."
    preds = []
    w_elo = []
    l_elo = []

    # Loop over all rows of the games dataframe
    for row in rs.itertuples():

        # Get key data from current row
        w = row.WTeamID
        l = row.LTeamID
        margin = row.margin
        wloc = row.WLoc
        game_id = row.GameID

        # Does either team get a home-court advantage?
        w_ad, l_ad, = 0., 0.
        if wloc == "H":
            w_ad += HOME_ADVANTAGE
        elif wloc == "A":
            l_ad += HOME_ADVANTAGE

        # Get elo updates as a result of the game
        pred, update = elo_update(elo_dict[w] + w_ad,
                                  elo_dict[l] + l_ad,
                                  margin,
                                 K)
        elo_dict[w] += update
        elo_dict[l] -= update

        # Save prediction and new Elos for each round
        preds.append(pred)
        w_elo.append(elo_dict[w])
        l_elo.append(elo_dict[l])

    rs['w_elo'] = w_elo
    rs['l_elo'] = l_elo
    df_list = [elos_per_season(rs, team_id) for team_id in team_ids]
    season_elos = pd.concat(df_list)
    return season_elos

def evaluate():
    pass

def training_form(elos):
    df_winelos = elos.rename(columns={'TeamID':'WTeamID', 'SeasonElo':'WElo'})
    df_losselos = elos.rename(columns={'TeamID':'LTeamID', 'SeasonElo':'LElo'})
    df_dummy = pd.merge(left=df_tour, right=df_winelos, how='left', on=['Season', 'WTeamID'])
    df_concat = pd.merge(left=df_dummy, right=df_losselos, on=['Season', 'LTeamID'])
    df_concat['EloDiff'] = df_concat.WElo - df_concat.LElo
    return df_concat

def winloss_features(df_teams, df_feature, feature_cols, merge_on=['Season'], team_id='TeamID'):
    '''
    Takes input:
    df_teams: a dataframe with columns that specify the winning and losing team, identified by arg team_id
    df_feature: a dataframe with features (e.g. stats) that each team has.
    feature_cols: the columns from df_features to extract
    merge_on: the common columns from df_teams and df_features to merge to. Default is 'Season'
    team_id: The name of the column that contains the team identifier for both df_teams and df_features. Default is 'TeamID'.
    Outputs:
    df_concat: a dataframe containing the original info from df_teams with columns that contain the features for each team
    from df_feature
    '''

    # Separate teams by win/loss
    w_team_id = 'W'+team_id
    l_team_id = 'L'+team_id

    # Prepare columns for feature dataframe
    w_cols = ['W{}'.format(feature) for feature in feature_cols]
    l_cols = ['L{}'.format(feature) for feature in feature_cols]
    diff_cols = ['{}Diff'.format(feature) for feature in feature_cols]

    # Create a dict with key = feature, value = Wfeature or Lfeature
    w_dict = dict(zip(feature_cols, w_cols))
    w_dict[team_id] = w_team_id

    l_dict = dict(zip(feature_cols, l_cols))
    l_dict[team_id] = l_team_id

    df_wins = df_feature.rename(columns=w_dict)
    df_losses = df_feature.rename(columns=l_dict)

    # Prepare common columns to merge on
    w_merge_on = merge_on + [w_team_id]
    l_merge_on = merge_on + [l_team_id]

    # Merge the dataframes that contain the winning team's features with the losing team's features
    df_dummy = pd.merge(left=df_teams, right=df_wins, how='left', on=w_merge_on)
    df_concat = pd.merge(left=df_dummy, right=df_losses, on=l_merge_on)
    df_concat = df_concat.reindex(columns=list(df_concat.columns)+diff_cols)
    df_concat[diff_cols] = df_concat[w_cols].values - df_concat[l_cols].values

    return df_concat

def mirror_df(df, x, y):
    df_pos = df.copy()
    df_pos[x] = df[x]
    df_pos[y] = df[y]

    df_neg = df.copy()
    df_neg[x] = -df[x]
    df_neg[y] = -df[y]

    df_mirrored = pd.concat((df_pos, df_neg))

    return df_mirrored

def elo_tune(rs, tourney, Ks, HCAs, x_col='SeasonEloDiff', y_col='WMargin', season_cutoff=2017):
    params = list(itertools.product(Ks, HCAs))
    n_params = len(params)
    scores_all = []
    coefs = []
    for i in range(n_params):
        K, HCA = params[i]
        param_string = "K{}HCA{}".format(K,HCA)
        ### Calculate elos
        elos, rs_preds = season_elos(rs, K, HCA)
        #elos.to_csv(elo_out_dir+"SeasonElos{}.csv".format(param_string))

        ### Preprocess elos into [[X][y], [-X],[-y]] form
        elos_tourney = winloss_features(df_teams=tourney, df_feature=elos, feature_cols=['SeasonElo'])

        ### Mirror dfs
        elos_tourney = mirror_df(elos_tourney, x=x_col, y=y_col).reset_index(drop=True)
        #elos_tourney.to_csv(elo_out_dir+"EloTrain{}.csv".format(param_string))

        ### Evaluate
        scores, coef = elo_lr(elos_tourney, x_col, y_col)
        scores_all.append(scores)
        coefs.append(coef)

    return list(zip(params, scores_all)), list(zip(params, coefs))

def elo_lr(elos_tourney, x, y, season_cutoff=2017, metrics=[mean_squared_error, explained_variance_score]):
    lr = lm.LinearRegression()
    scores = []
    X_train = elos_tourney.loc[elos_tourney.Season<season_cutoff, x].values.reshape(-1,1)
    X_train = StandardScaler().fit_transform(X_train)
    X_test = elos_tourney.loc[elos_tourney.Season>=season_cutoff, x].values.reshape(-1,1)
    X_test = StandardScaler().fit_transform(X_test)
    y_train = elos_tourney.loc[elos_tourney.Season<season_cutoff, y].values.reshape(-1,1)
    y_train = StandardScaler().fit_transform(y_train)
    y_true = elos_tourney.loc[elos_tourney.Season>=season_cutoff, y].values.reshape(-1,1)
    y_true = StandardScaler().fit_transform(y_true)

    lr.fit(X_train,y_train)
    y_pred = lr.predict(X_test)
    coef = lr.coef_

    for metric in metrics:
        scores.append(metric(y_true, y_pred))

    return scores, coef


from collections import defaultdict
from typing import List, Dict

def merge_dicts(lst_of_dicts: List[Dict], keys_to_merge: List[str]) -> dict:
    res = defaultdict(list)
    for dictionary in lst_of_dicts:
        for key in keys_to_merge:
            res[key] += dictionary[key]
    return res


import json
import os
import traceback
from collections import UserDict
from functools import partial

from configobj import ConfigObj
from yaml import load, dump
from yaml import Loader, Dumper

def format_json(content):
    if isinstance(content, (str, bytes)):
        try:
            content = json.loads(content)
        except:
            return content

    result = json.dumps(content, sort_keys=True, indent=4, separators=(',', ': ')). \
        encode('latin-1').decode('unicode_escape')

    return result

def get_root_dir(flag_file_name, cur_dir):
    if os.path.isfile(cur_dir):
        cur_dir = os.path.dirname(cur_dir)
    if flag_file_name in os.listdir(cur_dir):
        return cur_dir

    up_dir = os.path.dirname(cur_dir)
    if up_dir == cur_dir:
        return None
    return get_root_dir(flag_file_name, up_dir)

class WDict(UserDict):
    """
    It's hard to name！！！
    注意：正常使用key中不能再出现 "."，否则会被拆分掉，如a['a.b.c']等于a['a']['b']['c']
    示例：> d = {'a': {'b': 'c': 1}}
         > print(WDict(**d)['a.b.c']) # -> 1
    """

    def __getitem__(self, item: str):
        key_list = item.split('.')
        content = self.data
        for key in key_list:
            key = int(key) if str.isdigit(key) else key
            try:
                content = content[key]
            except (KeyError, IndexError):
                return None
        return content

class DDict:
    """
    It's hard to name！！！
    示例: > d = {'a': {'b': 'c': 1}}
         > print(d.a.b.c) # 1
    """

    def __init__(self, d):
        self.__d = d

    def __getattr__(self, item):
        try:
            return DDict(self.__d[item])
        except TypeError:
            raise ValueError('找不到%s配置，请检查' % item)

    def __str__(self):
        return str(self.__d)

def merge_dict(d1: dict, d2: dict):
    """
    merge d2 to d1
    d2有d1没有，则新增
    d1、d2同时有，但不全是字典，则替换
    d1、d2同时有，但都是字典，则递归调用合并函数
    """
    for k in d2:
        if k in d1:
            if isinstance(d1[k], dict) and isinstance(d2[k], dict):
                merge_dict(d1[k], d2[k])
                continue
        d1[k] = d2[k]

class ConfigFileParser:
    def __init__(self, suffix, file_path):
        if not isinstance(file_path, str):
            raise ValueError('file_path error, is not a valid str')

        if not os.path.isfile(file_path):
            raise ValueError('file_path error, is not a valid file path')

        self.file_path = file_path
        self.content = None
        self.parser = self.get_parser(suffix)

        try:
            with open(self.file_path, encoding='utf-8') as f:
                self.content = self.parser(f) or {}
        except Exception:
            traceback.print_exc()
            raise ValueError('%s is not a valid %s file' % (file_path, suffix))

    @staticmethod
    def get_parser(suffix):
        if suffix == 'yaml':
            return partial(load, Loader=Loader)
        elif suffix == 'json':
            return json.load
        elif suffix == 'ini':
            return ConfigObj
        else:
            raise ValueError('不支持的配置文件类型')

    def as_dict(self):
        return self.content


from ...common.utils import Singleton, Mkdir, Touch
from ...common import config
import json

class Table(Singleton):
    def __init__(self, filename:str) -> None:
        self.__dir = config.make_data_path('work_table')
        Mkdir(self.__dir)
        self.__path = self.__dir + '/' + filename
        Touch(self.__path, '{}')
        self.__data = dict()

        self.__load_from_db()

    def __load_from_db(self):
        with open(self.__path, 'r') as file:
            self.__data = json.loads(file.read())

    def __write_to_db(self):
        with open(self.__path, 'w') as file:
            file.write(json.dumps(self.__data))

    def add(self, group_id:str, obj:str) -> bool:
        "Return True if successfully add a new obj, False if exists obj for the group"
        if group_id in self.__data.keys():
            return False

        self.__data[group_id] = obj
        self.__write_to_db()
        return True

    def delete(self, group_id:str) -> bool:
        "Return True if successfully delete obj for the group, False if not exists obj"
        if group_id not in self.__data.keys():
            return False

        del self.__data[group_id]
        self.__write_to_db()
        return True

    def get(self, group_id:str) -> str:
        if group_id not in self.__data.keys():
            return ""

        return self.__data[group_id]

    def update(self, group_id:str, obj:str) -> bool:
        "Return True if successfully update obj for the group, False if not exists obj"
        if group_id not in self.__data.keys():
            return False

        self.__data[group_id] = obj
        self.__write_to_db()
        return True


import asyncio
from message import Message

@asyncio.coroutine
def run(message, matches, chat_id, step):
    return [Message(chat_id).set_text(
        "Hi :-) \n*Bot Search Music!*\n✔Fast and accurate✔\n\n_/Music <Artist name>_\n*Creator* = @LullSky", parse_mode="markdown")]

plugin = {
    "name": "About",
    "desc": "Get Information About The Developer!",
    "usage": ["/about"],
    "run": run,
    "sudo": False,
    "patterns": ["^[/]start$"]
}


import socket
from _thread import *
import pandas as pd
import numpy as np
import argparse
import requests
from requests.auth import HTTPBasicAuth
from datetime import datetime
from ml.LinearRegression import DistributedLinearRegression

parser = argparse.ArgumentParser()

parser.add_argument("--email", type=str)
parser.add_argument("--password", type=str)
parser.add_argument("--exp_name", type=str)
parser.add_argument("--exp_id", type=int)
parser.add_argument("--desc", type=str)

args = parser.parse_args()
dateformat = r"%d-%m-%Y %H:%M:%S"

body = {
    "device_type": 1,
    "id": args.exp_id,
    "name": args.exp_name,
    "description": args.desc,
    "active": 1,
    "start": datetime.now().strftime(dateformat)
}

url = "http://172.18.132.255:5000/experiments/commit"
response = requests.post(
    url,
    json = body,
    auth = HTTPBasicAuth(
        args.email, args.password
    )
)

if response.content != b'1':
    print("Unsuccessful !")
    print("Run the script again !")
    exit()

def recvall(sock):
    BUFF_SIZE = 4096 # 4 KiB
    data = b''
    while True:
        part = sock.recv(BUFF_SIZE)
        data += part
        if len(part) < BUFF_SIZE:
            # either 0 or end of data
            break
    return data

ServerSideSocket = socket.socket()
host = '127.0.0.1'
port = 2004
connectport = [12345, 12346]  #, 12346, 12347
nworkers = len(connectport)
ThreadCount = 0

try:
    ServerSideSocket.bind((host, port))
except socket.error as e:
    print(str(e))

print('Socket is listening..')
ServerSideSocket.listen(5)

# data = pd.read_csv("master/cleaned.csv").to_json(orient = 'table', index = False)
df = pd.read_csv("cleaned_normalised.csv")
df_shuffled = df.sample(frac=1)
df_splits = np.array_split(df_shuffled, nworkers)
df_splits = [
    df_split.to_json(
        orient = 'table',
         index = False
    ) for df_split in df_splits]

thetas = []
completed = [nworkers]

def multi_threaded_client(connection, data, thetas, completed):
    connection.sendall(data.encode())
    print("Distributed")
    data = recvall(connection).decode()
    print("received")
    thetas.append(data)
    connection.close()
    completed[0] -= 1

i = 1
while(i <= nworkers):
    conn, addr = ServerSideSocket.accept()
    print(addr)
    if addr[1] not in connectport:
        conn.close()
        i -= 1
    else:
        # do what ever you want here
        print('Connected to: ' + addr[0] + ':' + str(addr[1]))
        start_new_thread(multi_threaded_client, (conn, df_splits[i-1], thetas, completed))
        ThreadCount += 1
        print('Thread Number: ' + str(ThreadCount))
    i += 1

while(completed[0] != 0):
    pass

lr = DistributedLinearRegression()

def literal_eval(lst):
    lst = lst[1:len(lst)-1]    # Removes '[', ']'
    lst = list(map(float, lst.split()))
    return lst

thetas = np.array(list(map(literal_eval, thetas)))
print(thetas)

lr.from_weights(thetas)
theta = lr.get_weights()
print(theta)

print("Closing server")
ServerSideSocket.close()

body["active"] = 0
body["end"] = datetime.now().strftime(dateformat)

response = requests.post(
    url,
    json = body,
    auth = HTTPBasicAuth(
        args.email, args.password
    )
)

print("Execution successful !")