import MySQLdb
from MySQLdb import _mysql
import pybedtools
import re

default_columns = ['chrom', 'chromStart', 'chromEnd']

def ucsc_connect(build):
    """
    Open connection to UCSC MySQL database for a specified reference
    """
    conv = { MySQLdb.FIELD_TYPE.LONG: int }
    db = _mysql.connect(host='genome-mysql.cse.ucsc.edu', user='genome',
                        passwd='', db=build, conv=conv)
    return db

def table_exists(db, table):
    """
    Check if a table exists
    """
    db.query('SELECT count(*) FROM information_schema.TABLES WHERE TABLE_NAME = "{0}"'.format(table))
    check = int(db.store_result().fetch_row(how=0, maxrows=0)[0][0])
    if check == 0:
        return False
    else:
        return True

def _check_hg_compliance(feature, bedpe=False):
    """
    Convert chroms in a bedtool to hg18/hg19 nomenclature
    """

    if 'chr' not in feature.chrom:
        feature.chrom = 'chr' + feature.chrom
    if bedpe and 'chr' not in feature[3]:
        feature[3] = 'chr' + feature[3]

    return feature

def _check_grc_compliance(feature, bedpe=False):
    """
    Convert chroms in a bedtool to GRC nomenclature
    """

    if 'chr' in feature.chrom:
        feature.chrom = str(feature.chrom).replace('chr', '')
    if bedpe and 'chr' in feature[3]:
        feature[3] = str(feature[3]).replace('chr', '')

    return feature

def collapse_query_regions(bins):
    """
    Collapse bins to minimal ucsc query intervals
    """

    query_ranges = bins.cut(range(3)).sort().merge()
    query_ranges = query_ranges.each(_check_hg_compliance).saveas()
    return query_ranges

def constrain_query_regions(query, columns, query_ranges):
    """
    Add regional restrictions to a UCSC query SELECT statement
    """

    def _write_single_constraint(interval):
        c = '(`{0}` = "{1}" AND `{2}` <= {3} AND `{4}` >= {5})'
        c = c.format(columns[0], interval.chrom,
                     columns[1], interval.stop,
                     columns[2], interval.start)
        return c

    constraint = ''
    cidx = 0
    for interval in query_ranges:
        newcons = _write_single_constraint(interval)
        if cidx == 0:
            constraint = constraint + newcons
        else:
            constraint = ' '.join([constraint, 'OR', newcons])
        cidx += 1

    if ' WHERE ' in query:
        query = '{0} AND ( {1} )'.format(query, constraint)
    else:
        query = ' '.join([query, 'WHERE', constraint])

    return query

def parse_table_arg(track):
    """
    Parse table & column input
    """

    if ':' in track:
        table = track.split(':')[0]
        track_opts = track.split(':')[1].split(',')

        columns = [i for i in track_opts if not any(s in i for s in '! = < >'.split())]
        if len(columns) < 3:
            columns = default_columns + columns

        conditions = [i for i in track_opts if any(s in i for s in '! = < >'.split())]
        if len(conditions) == 0:
            conditions = None

    else:
        table = track
        columns = default_columns
        conditions = None

    return table, columns, conditions

def format_conditions(conditions):
    """
    Format conditions into SQL-compliant syntax
    """

    def _form_cond(cond):
        terms = re.sub('[!=<>]', ' ', cond).split()
        col = '`{0}`'.format(terms[0])
        query = terms[1]
        if not query.isdigit():
            query = '"{0}"'.format(query)
        comp = ''.join([c for c in cond if c in set('!=><')])
        form_cond = ' '.join([col, comp, query])
        return form_cond

    form_conds = [_form_cond(cond) for cond in conditions]

    return form_conds

def compose_query(table, db, oformat, query_ranges=None,
                  columns=default_columns, conditions=None,
                  print_query=False):
    """
    Format UCSC query dependent on desired output format
    """

    # Query dependent on desired output behavior
    if oformat == 'bed':

        query = 'SELECT ' + ', '.join(['`{0}`'.format(c) for c in columns])

        query = query + ' from `{0}`'.format(table)

        # Add other conditions, as optioned
        if conditions is not None:
            form_conds = ' AND '.join(format_conditions(conditions))
            query = '{0} WHERE {1}'.format(query, form_conds)

        # Constraint query region, if optioned
        if query_ranges is not None:
            query = constrain_query_regions(query, columns, query_ranges)

    else:
        query = 'SELECT * from `{0}`'.format(table)

    # Useful for debugging purposes
    if print_query:
        print(query)

    return query

def query_table(table, db, oformat='bed', query_ranges=None,
                columns=default_columns, conditions=None):
    """
    Download a single table from UCSC database
    """

    # Query database
    query = compose_query(table, db, oformat, query_ranges, columns, conditions)
    db.query(query)

    # Format output
    if oformat == 'bed':
        result = pybedtools.BedTool(db.store_result().fetch_row(how=0, maxrows=0))

    elif oformat == 'bigwig':
        result = dict(db.store_result().fetch_row(how=1, maxrows=0)[0])
        result = 'http://hgdownload.soe.ucsc.edu' + result['fileName'].decode('utf-8')

    else:
        result = db.store_result().fetch_row(how=1, maxrows=0)

    return result

def query_ucsc(bins, track, columns, conditions, db, action, query_ranges):
    """
    Master function for handling UCSC queries
    """

    elig_bed_actions = 'count count-unique count-pairs any-overlap ' + \
                       'any-pairwise-overlap coverage pairwise-coverage'
    bedpe = 'pairwise' in action

    # Get raw data
    if 'map-' in action:
        if columns is default_columns:
            result = query_table(track, db, 'bigwig', query_ranges)
        else:
            result = query_table(track, db, 'bed', query_ranges,
                                 columns, conditions)
            # Coerce to GRC nomenclature, if necessary
            if 'chr' not in bins[0].chrom:
                result = result.each(_check_grc_compliance, bedpe=bedpe)

    elif action in elig_bed_actions.split():
        result = query_table(track, db, 'bed', query_ranges,
                             columns, conditions)
        # Coerce to GRC nomenclature, if necessary
        if 'chr' not in bins[0].chrom:
            result = result.each(_check_grc_compliance, bedpe=bedpe).saveas()

    else:
        from sys import exit
        exit('ERROR: Unknown action "{0}" in ucsc.query_ucsc'.format(action))

    return result

def subquery_ucsc(bins, track, columns, conditions, db, action, query_ranges,
                  contig):
    """
    UCSC query restricted to a single contig
    """

    sub_ranges = query_ranges.filter(lambda f: f.chrom == contig).saveas()
    result = query_ucsc(bins, track, columns, conditions, db, action, sub_ranges)
    if isinstance(result, pybedtools.BedTool):
        result = result.saveas()

    return result


import sys
sys.stdin = open("실패율_input.txt")

from collections import Counter

N = int(input())
stages = list(map(int, input().split()))

answer = []
result = []
stageCnt = Counter(stages)
# print(stageCnt)
for stage in range(1, N+1):
    playCnt = 0
    clear = 0
    # print('---------------')
    for key in stageCnt.keys():
        # print(key)
        if key == stage:
            playCnt += stageCnt[key]
        elif key > stage:
            clear += stageCnt[key]
    if (playCnt + clear) != 0:
        result.append([stage, playCnt / (playCnt + clear)])
    else:
        result.append([stage, 0])
# print(result)
result.sort(key=lambda x: x[1], reverse=True)
for i in range(len(result)):
    answer.append(result[i][0])
print(answer)


import pygame
import os
from network import Network
from player import Player

pygame.font.init()
pygame.mixer.init()

WHITE = (255, 255, 255)
BLACK = (0, 0, 0)
RED = (255, 0, 0)
YELLOW = (255, 255, 0)

WIDTH, HEIGHT = 900, 500
MAX_BULLETS = 3
FPS = 60
WIN = pygame.display.set_mode((WIDTH, HEIGHT))
pygame.display.set_caption("First Game!")

SPACESHIP_WIDTH, SPACESHIP_HEIGHT = 55, 40
SPACE = pygame.transform.scale(pygame.image.load(
        os.path.join('Assets', 'space.png')), (WIDTH, HEIGHT))

WIDTH, HEIGHT = 900, 500
BORDER = pygame.Rect(WIDTH//2 - 5, 0, 10, HEIGHT)

#BULLET_HIT_SOUND = pygame.mixer.Sound('Assets/Grenade+1.mp3')
#BULLET_FIRE_SOUND = pygame.mixer.Sound('Assets/Gun+Silencer.mp3')

HEALTH_FONT = pygame.font.SysFont('comicsans', 40)
WINNER_FONT = pygame.font.SysFont('comicsans', 100)

players=[Player(SPACESHIP_WIDTH,SPACESHIP_HEIGHT,'spaceship_yellow.png',90,10,[],1,100,300,0),Player(SPACESHIP_WIDTH,SPACESHIP_HEIGHT,'spaceship_red.png',270,10,[],2,700,300,1)]

def read_pos(str):
    str = str.split(",")
    return int(str[0]), int(str[1]) ,int(str[2]),int(str[3]), int(str[4])

def make_pos(tup):
    return str(tup[0]) + "," + str(tup[1]) + "," + str(tup[2]) + "," + str(tup[3]) + "," + str(tup[4])

def bullets_read(str):
    str=str.split(",")
    return str

def bullets_make(tup):
    str_ans=""
    for i in range(len(tup)):
        str_ans+=str(tup[i]) + ","
    return str_ans

def main():
    clock = pygame.time.Clock()
    run = True
    n=Network()
    player_number=n.getP()
    player1=players[int(player_number)]
    player2=players[int(not int(player_number))]
    while run:
        clock.tick(FPS)
        if int(player_number)==0:
            p2Pos = read_pos(n.send(make_pos((player_number,player1.x, player1.y,player1.health,player2.health))))
        else:
            p2Pos = read_pos(n.send(make_pos((player_number,player1.x, player1.y,player2.health,player1.health))))

        player2.x = p2Pos[1]
        player2.y = p2Pos[2]
        if int(player_number)==0:
            player1.health=min(player1.health,p2Pos[3])
            player2.health=min(player2.health,p2Pos[4])
        else:
            player1.health=min(player1.health,p2Pos[4])
            player2.health=min(player2.health,p2Pos[3])
        player2.update()
        for event in pygame.event.get():
            if event.type == pygame.QUIT:
                run = False
                pygame.quit()
                exit()

            if event.type == pygame.KEYDOWN:
                if event.key == pygame.K_LCTRL and len(player1.bullets) < MAX_BULLETS:
                    if(not int(player_number)):
                        bullet = pygame.Rect(
                            player1.x + player1.width, player1.y + player1.height//2 - 2, 10, 5)
                    else:
                        bullet = pygame.Rect(
                            player1.x, player1.y + player1.height//2 - 2, 10, 5)
                    player1.bullets.append(bullet)
                    #BULLET_FIRE_SOUND.play()

            if event.type == player1.HIT:
                player2.health-=1
                #BULLET_HIT_SOUND.play()

        winner_text = ""
        if player1.health <= 0:
            if(not int(player_number)):
                winner_text = "Player 1 Wins!"
            else:
                winner_text = "Player 2 Wins!"

        if player2.health <= 0:
            if(not int(player_number)):
                winner_text = "Player 1 Wins!"
            else:
                winner_text = "Player 2 Wins!"

        if winner_text != "":
            player1.draw_window(SPACE,WIN,player2,player_number)
            Player.draw_winner(WIN,winner_text)
            return 1

        keys_pressed = pygame.key.get_pressed()
        player1.handle_bullets(player2)
        player1.handle_movement(keys_pressed)
        player1.draw_window(SPACE,WIN,player2,player_number)

main()


import logging
import unittest

from pythoncommons.file_utils import FileUtils

from yarndevtools.common.shared_command_utils import CommandType
from yarndevtools.commands.review_branch_creator import ReviewBranchCreator
from yarndevtools.constants import TRUNK, ORIGIN_TRUNK
from tests.test_utilities import TestUtilities, Object
from yarndevtools.yarn_dev_tools import YarnDevTools

LOG = logging.getLogger(__name__)
COMMIT_MSG_TEMPLATE = "patch file: {file}"
PATCH_FILENAME = "YARN-12345.001.patch"
REVIEW_BRANCH = "review-YARN-12345"
YARN_TEST_BRANCH = "YARNTEST-12345"
REMOTE_BASE_BRANCH = ORIGIN_TRUNK
BASE_BRANCH = TRUNK

class TestReviewBranchCreator(unittest.TestCase):
    utils = None
    repo = None
    log_dir = None
    sandbox_hadoop_repo_path = None

    @classmethod
    def setUpClass(cls):
        cls.utils = TestUtilities(cls, YARN_TEST_BRANCH)
        cls.utils.setUpClass(CommandType.CREATE_REVIEW_BRANCH)
        cls.utils.pull_to_trunk()
        cls.repo = cls.utils.repo
        cls.repo_wrapper = cls.utils.repo_wrapper
        cls.saved_patches_dir = cls.utils.saved_patches_dir
        cls.dummy_patches_dir = cls.utils.dummy_patches_dir

    @classmethod
    def tearDownClass(cls) -> None:
        TestUtilities.tearDownClass(cls.__name__, command_type=CommandType.CREATE_REVIEW_BRANCH)

    def setUp(self):
        self.utils.reset_and_checkout_existing_branch(BASE_BRANCH, pull=False)
        self.repo_wrapper.remove_branches_with_prefix(REVIEW_BRANCH, checkout_before_remove=TRUNK)

    def cleanup_and_checkout_branch(self):
        self.utils.cleanup_and_checkout_test_branch(pull=False)
        self.assertEqual(YARN_TEST_BRANCH, str(self.repo.head.ref))

    def test_with_not_existing_patch(self):
        args = Object()
        args.patch_file = FileUtils.join_path("tmp", "blablabla")
        review_branch_creator = ReviewBranchCreator(args, self.repo_wrapper, BASE_BRANCH, REMOTE_BASE_BRANCH)
        self.assertRaises(ValueError, review_branch_creator.run)

    def test_with_oddly_named_patch(self):
        patch_file = FileUtils.join_path(self.dummy_patches_dir, "testpatch1.patch")
        FileUtils.create_files(patch_file)
        args = Object()
        args.patch_file = patch_file

        review_branch_creator = ReviewBranchCreator(args, self.repo_wrapper, BASE_BRANCH, REMOTE_BASE_BRANCH)
        self.assertRaises(ValueError, review_branch_creator.run)

    def test_with_bad_patch_content(self):
        patch_file = FileUtils.join_path(self.dummy_patches_dir, PATCH_FILENAME)
        FileUtils.save_to_file(patch_file, "dummycontents")
        args = Object()
        args.patch_file = patch_file

        review_branch_creator = ReviewBranchCreator(args, self.repo_wrapper, BASE_BRANCH, REMOTE_BASE_BRANCH)
        self.assertRaises(ValueError, review_branch_creator.run)

    def test_with_normal_patch(self):
        patch_file = FileUtils.join_path(self.dummy_patches_dir, PATCH_FILENAME)
        self.utils.add_file_changes_and_save_to_patch(self, patch_file)
        args = Object()
        args.patch_file = patch_file

        review_branch_creator = ReviewBranchCreator(args, self.repo_wrapper, BASE_BRANCH, REMOTE_BASE_BRANCH)
        review_branch_creator.run()

        self.assertTrue(REVIEW_BRANCH in self.repo.heads, f"Review branch does not exist: {REVIEW_BRANCH}")
        self.utils.verify_commit_message_of_branch(REVIEW_BRANCH, COMMIT_MSG_TEMPLATE.format(file=patch_file))

    def test_with_normal_patch_two_consecutive_branches(self):
        patch_file = FileUtils.join_path(self.dummy_patches_dir, PATCH_FILENAME)
        self.utils.add_file_changes_and_save_to_patch(self, patch_file)
        args = Object()
        args.patch_file = patch_file

        review_branch_creator = ReviewBranchCreator(args, self.repo_wrapper, BASE_BRANCH, REMOTE_BASE_BRANCH)
        review_branch_creator.run()
        review_branch_creator.run()

        branch_2 = REVIEW_BRANCH + "-2"
        self.assertTrue(REVIEW_BRANCH in self.repo.heads, f"Review branch does not exist: {REVIEW_BRANCH}")
        self.assertTrue(branch_2 in self.repo.heads, f"Review branch does not exist: {branch_2}")
        self.utils.verify_commit_message_of_branch(REVIEW_BRANCH, COMMIT_MSG_TEMPLATE.format(file=patch_file))
        self.utils.verify_commit_message_of_branch(branch_2, COMMIT_MSG_TEMPLATE.format(file=patch_file))

    def test_with_normal_patch_from_yarn_dev_tools(self):
        self.cleanup_and_checkout_branch()
        self.utils.add_some_file_changes(commit=False)

        self.utils.set_env_vars(self.utils.sandbox_repo_path, self.utils.sandbox_repo_path)
        yarn_dev_tools = YarnDevTools()
        yarn_dev_tools.upstream_repo = self.repo_wrapper

        args = Object()
        patch_file = FileUtils.join_path(self.dummy_patches_dir, PATCH_FILENAME)
        self.utils.add_file_changes_and_save_to_patch(self, patch_file)
        args.patch_file = patch_file
        ReviewBranchCreator.execute(args)

        self.assertTrue(REVIEW_BRANCH in self.repo.heads, f"Review branch does not exist: {REVIEW_BRANCH}")
        self.utils.verify_commit_message_of_branch(REVIEW_BRANCH, COMMIT_MSG_TEMPLATE.format(file=patch_file))


import cv2
import glob
import os
import numpy as np
from keras.utils.np_utils import to_categorical
import scipy.misc
import matplotlib.pyplot as plt
import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)
from tqdm import tqdm
import pickle
from skimage import io
import tables
import numpy as np

LABEL_PATH = '/mnt/DataStorage/NIH/Data_Entry_2017.csv'
LABEL_BBOX_PATH = '/mnt/DataStorage/NIH/BBox_List_2017.csv'

def load_hp_data_hd5_tf(data_path, image_size, label_path, labels_size, hd5_name):
    df_train = pd.read_csv(label_path)

    targets_count = np.zeros(labels_size)

    data_model = tables.open_file(hd5_name, mode='w')
    atom = tables.Float64Atom()
    X = data_model.create_earray(data_model.root, 'data', atom, (0, image_size, image_size, 4))
    y = data_model.create_earray(data_model.root, 'tag', atom, (0, labels_size))
    idx = 0
    for row in tqdm(df_train.values[:]):
        f = row[0]
        tags = row[1]

        tag_files = [data_path+'/{}'.format(f)+'_blue.png', data_path+'/{}'.format(f)+'_green.png', data_path+'/{}'.format(f)+'_blue.png', data_path+'/{}'.format(f)+'_yellow.png']

        if not os.path.exists(tag_files[0]):
            continue

        new_image = np.zeros((0, image_size, image_size))
        #print(file_name)
        for file in tag_files:
            idx += 1

            img = cv2.imread(file, cv2.IMREAD_GRAYSCALE)
            #print('print img shape = ', img.shape)
            #img = cv2.resize(img, (image_size, image_size))

                #print('show origin image')
                #io.imshow(img)
                #io.show()
                #print('show flip image')
                #io.imshow(img_f)
                #io.show()

            img = img / 255.
            img = np.expand_dims(img, axis=0)
            new_image = np.vstack((new_image, img))
        targets = np.zeros(labels_size)
        targets = targets.reshape(1, len(targets))
        for t in tags.split(' '):
            targets[0][int(t)] = 1
            targets_count[int(t)] += 1
        #print('new_image shape = ', new_image.shape)
        new_image = np.transpose(new_image, (1,2,0))
        new_image = np.expand_dims(new_image, axis=0)
        X.append(new_image)
        y.append(targets)

    data_model.close()
    print('tags count = ', targets_count)

def unison_shuffled_copies(a, b):
    assert len(a) == len(b)
    p = np.random.permutation(len(a))
    return p


from typing import Optional

import torch.nn as nn
from torch import Tensor

from openspeech.modules.positional_encoding import RelPositionalEncoding
from openspeech.modules.relative_multi_head_attention import RelativeMultiHeadAttention

class MultiHeadedSelfAttentionModule(nn.Module):
    r"""
    Conformer employ multi-headed self-attention (MHSA) while integrating an important technique from Transformer-XL,
    the relative sinusoidal positional encoding scheme. The relative positional encoding allows the self-attention
    module to generalize better on different input length and the resulting encoders is more robust to the variance of
    the utterance length. Conformer use prenorm residual units with dropout which helps training
    and regularizing deeper models.

    Args:
        d_model (int): The dimension of model
        num_heads (int): The number of attention heads.
        dropout_p (float): probability of dropout

    Inputs: inputs, mask
        - **inputs** (batch, time, dim): Tensor containing input vector
        - **mask** (batch, 1, time2) or (batch, time1, time2): Tensor containing indices to be masked

    Returns:
        - **outputs** (batch, time, dim): Tensor produces by relative multi headed self attention module.
    """

    def __init__(
        self,
        d_model: int,
        num_heads: int,
        dropout_p: float = 0.1,
    ) -> None:
        super(MultiHeadedSelfAttentionModule, self).__init__()
        self.positional_encoding = RelPositionalEncoding(d_model)
        self.layer_norm = nn.LayerNorm(d_model)
        self.attention = RelativeMultiHeadAttention(d_model, num_heads, dropout_p)
        self.dropout = nn.Dropout(p=dropout_p)

    def forward(self, inputs: Tensor, mask: Optional[Tensor] = None) -> Tensor:
        r"""
        Forward propagate of conformer's multi-headed self attention module.

        Inputs: inputs, mask
            - **inputs** (batch, time, dim): Tensor containing input vector
            - **mask** (batch, 1, time2) or (batch, time1, time2): Tensor containing indices to be masked

        Returns:
            - **outputs** (batch, time, dim): Tensor produces by relative multi headed self attention module.
        """
        batch_size = inputs.size(0)
        pos_embedding = self.positional_encoding(inputs)
        pos_embedding = pos_embedding.repeat(batch_size, 1, 1)

        inputs = self.layer_norm(inputs)
        outputs = self.attention(inputs, inputs, inputs, pos_embedding=pos_embedding, mask=mask)

        return self.dropout(outputs)


import pyotp
import qrcode

# This Should be a Secure Key and different for each user
otp = pyotp.TOTP("KRUGS4ZAKNUG65LMMQQGEZJAMEQFGZLDOVZGKICLMV4Q====")
print(otp.now())

def register():
    auth_string = otp.provisioning_uri(name="Test User", issuer_name="My test System", )
    # print(auth_string)
    img = qrcode.make(auth_string)
    img.show()
    while True:
        val = input("Enter current code to complete registration: ")
        if otp.now() == val:
            print("Registration Complete")
            break
        else:
            print("Wrong OTP please try again...")

def login():
    val = input("Enter your Google Authenticator to Login: ")
    if otp.now() == val:
        print("Login Successful")
    else:
        print("Login Failed please try again...")

register()
while True:
    login()


# from core.Optimizers import CholeskyCMAES
# from core.CNN_scorers import TorchScorer
# from core.GAN_utils import upconvGAN

import os
import re
from os.path import join
from glob import glob
from time import time
from easydict import EasyDict
from tqdm import tqdm
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
import pickle as pkl
import matplotlib as mpl
mpl.rcParams['pdf.fonttype'] = 42
mpl.rcParams['ps.fonttype'] = 42
mpl.rcParams['axes.spines.right'] = False
mpl.rcParams['axes.spines.top'] = False
pd.options.display.max_columns = 10
pd.options.display.max_colwidth = 200
figdir = r"E:\OneDrive - Harvard University\GECCO2022\Figures\TuneGeometry"
#%%
datadir = r"E:\OneDrive - Washington University in St. Louis\HessTune\AlexNet"
data = np.load(join(datadir, "fc8_1_(1_1).npz"))
#%%
for x in data:
    print(x, data[x].shape)
#%%
plt.semilogy(sorted(np.abs(data["eigvals_u"])))
plt.semilogy(sorted(np.abs(data["eigvals"])))
plt.show()
#%% Visualize and plot the eigenvalues
unitfnlist = ["conv1_0_(27_27).npz",
            "conv2_0_(13_13).npz",
            "conv3_0_(6_6).npz",
            "conv4_0_(6_6).npz",
            "conv5_0_(6_6).npz",
            "fc6_0_(1_1).npz",
            "fc7_0_(1_1).npz",
            "fc8_0_(1_1).npz",]

def plot_CNN_spect_comparison(maxnorm=True, colorcyc=plt.rcParams['axes.prop_cycle'].by_key()['color']):
    figh = plt.figure(figsize=[6, 6])
    for k, unitfn in enumerate(unitfnlist):
        match = re.findall("(.*)_(\d*)_\((\d*)_(\d*)\).npz", unitfn, )
        layer, channum, pos = match[0][0], int(match[0][1]), (int(match[0][2]), int(match[0][3]))
        for chi in range(5):
            unitfn = "%s_%d_(%d_%d).npz" % (layer, chi, pos[0], pos[1])
            data = np.load(join(datadir, unitfn))
            eigvals = data["eigvals"].copy()
            normalizer = np.abs(eigvals).max() if maxnorm else np.array(1.0)
            plt.semilogy(sorted(np.abs(eigvals))[::-1] / normalizer,
                         label=layer if chi == 0 else None,
                         lw=2, alpha=0.5, color=colorcyc[k])
    if maxnorm:
        figh.gca().set_ylim(1E-9, 1.5)
    else:
        figh.gca().set_ylim(1E-9, 10)

    plt.ylabel("abs Hessian eigenvalue"+"(norm by max)" if maxnorm else "",fontsize=12)
    plt.xlabel("eigen rank",fontsize=12)
    plt.title("Hessian of Landscape Compared across CNN Hierarchy\nEvaluated at peak of act max",fontsize=14)
    plt.legend()
    plt.savefig(join(figdir, "alexnet_layers_peak_hess_spect_cmp%s%s.png"%("_maxnorm" if maxnorm else "", "")))
    plt.savefig(join(figdir, "alexnet_layers_peak_hess_spect_cmp%s%s.pdf"%("_maxnorm" if maxnorm else "", "")))
    plt.show()
    return figh

cmap = plt.get_cmap("hsv")
colorcyc = [cmap(i/(8)) for i in range(8)]
plot_CNN_spect_comparison(maxnorm=True, colorcyc=colorcyc)
plot_CNN_spect_comparison(maxnorm=False, colorcyc=colorcyc)
#%%
def plot_CNN_spect_comparison_randinit(maxnorm=True, colorcyc=plt.rcParams['axes.prop_cycle'].by_key()['color']):
    figh = plt.figure(figsize=[6, 6])
    for k, unitfn in enumerate(unitfnlist):
        match = re.findall("(.*)_(\d*)_\((\d*)_(\d*)\).npz", unitfn, )
        layer, channum, pos = match[0][0], int(match[0][1]), (int(match[0][2]), int(match[0][3]))
        for chi in range(5):
            unitfn = "%s_%d_(%d_%d).npz" % (layer, chi, pos[0], pos[1])
            data = np.load(join(datadir, unitfn))
            eigvals = data["eigvals_u"].copy()
            normalizer = np.abs(eigvals).max() if maxnorm else np.array(1.0)
            plt.semilogy(sorted(np.abs(eigvals))[::-1] / normalizer,
                         label=layer if chi == 0 else None,
                         lw=2, alpha=0.5, color=colorcyc[k])
    if maxnorm:
        figh.gca().set_ylim(3E-8, 1.5)
    else:
        figh.gca().set_ylim(1E-8, 10)

    plt.ylabel("abs Hessian eigenvalue"+"(norm by max)" if maxnorm else "",fontsize=12)
    plt.xlabel("eigen rank",fontsize=12)
    plt.title("Hessian of Landscape Compared across CNN Hierarchy\nEvaluated at random vector",fontsize=14)
    plt.legend()
    plt.savefig(join(figdir, "alexnet_layers_rnd_hess_spect_cmp%s%s.png"%("_maxnorm" if maxnorm else "", "")))
    plt.savefig(join(figdir, "alexnet_layers_rnd_hess_spect_cmp%s%s.pdf"%("_maxnorm" if maxnorm else "", "")))
    plt.show()
    return figh
#%%
plot_CNN_spect_comparison_randinit(maxnorm=True, colorcyc=colorcyc)
plot_CNN_spect_comparison_randinit(maxnorm=False, colorcyc=colorcyc)


# -*- ecoding: utf-8 -*-
# @ModuleName: 波形图生成
# @Function:
# @Author: Jack Chen
import os, glob
import cv2 as cv
import numpy as np
import matplotlib
import matplotlib.pyplot as plt
import pandas as pd

def draw_waveform(dir):
    for file in os.listdir(dir):
        file1 = dir + '\\' + file
        (filepath, tempfilename) = os.path.split(file)
        file_name = tempfilename + '.png'
        for file2 in os.listdir(file1):
            file3 = file1 + '\\' + file2
            file4 = glob.glob(os.path.join(file3, "*.png"))
            for file_name in file4:
                (filepath1, tempfilename1) = os.path.split(file_name)
                img = cv.imread(file_name)
                GrayImage = cv.cvtColor(img, cv.COLOR_BGR2GRAY)
                img = np.array(GrayImage)
                height, width = img.shape
                img_y = []
                for col in range(1, width + 1):
                    a = 0
                    b = 0
                    for row in range(height):
                        val = img[row][col - 1]
                        if (val) == 0:
                            a = a + 1
                        else:
                            b = b + 1
                    img_y.append(b)
                img_x = list(range(1, width + 1))

                matplotlib.rcParams['font.sans-serif'] = ['SimHei']  # 用黑体显示中文
                # "r" 表示红色，ms用来设置*的大小
                plt.figure(figsize=(12.8, 6.4))
                plt.plot(img_x, img_y, "black", marker='.', ms=8)
                plt.xticks(rotation=45)
                plt.xlabel("列数")
                plt.ylabel("白色像素个数")
                plt.title("图像每列白色像素数")

                poly_save = 'pix_img'
                isExists = os.path.exists(poly_save)
                if not isExists:
                    os.makedirs(poly_save)
                save_name = poly_save + '\\' + tempfilename + '_' + str(file2) + '_' + tempfilename1
                plt.savefig(save_name)
                plt.close()

                data_save_csv = 'pix_csv'
                isExists = os.path.exists(data_save_csv)
                if not isExists:
                    os.makedirs(data_save_csv)
                pix_name = os.path.splitext(tempfilename1)[0]
                # save csv
                chss = [img_x, img_y]
                chss = np.transpose(chss)
                save_name_1 = data_save_csv + '\\' + tempfilename + '_' + str(file2) + '_' + pix_name + '.csv'
                mess = pd.DataFrame(chss, columns=None)
                mess.to_csv(save_name_1, index=False, header=False)

if __name__ == '__main__':
    dir = 'canny_core'
    draw_waveform(dir)


import Person_Model
from collections import defaultdict
class Viewer:
    def __init__(self,person):
        self.person = person
    def printAll(self):
        self.printGraveInformation()
    def printGraveInformation(self):
        print(self.person.options)
        numberNoters = defaultdict(lambda: 'th')
        numberNoters['1'], numberNoters['2'], numberNoters['3'] = 'st','nd','rd'

        if len(self.person.names["First"]) > 1: # Only one first name can be assigned
            print("Type which first name to include (type in a number):\n",
                  *[str(self.person.names["First"].index(item)+1)+"."+item+"\n" for item in self.person.names["First"]])
            self.person.idealNames["First"] = self.person.names["First"][int(input("Enter a number from 1 to "+str(len(self.person.names["First"]))+": "))-1]
            print()
        else: # if only one, just use that one
            self.person.idealNames["First"] = self.person.names["First"][0]

        if "Middle" not in self.person.names:
            self.person.idealNames["Middle"] = []
        elif len(self.person.names["Middle"]) > 1: # Multiple middle names can be assigned, but which ones should?
            print("Type which middles names to include (type in a single or string of numbers):\n",
                  *[str(self.person.names["Middle"].index(item)+1)+"."+item+"\n" for item in self.person.names["Middle"]])
            self.person.idealNames["Middle"] = [self.person.names["Middle"][index-1]
                                                for index in list(map(int,input("Enter a sequence of number including any from 1 to "+str(len(self.person.names["Middle"]))+
                                                                                " (separate with spaces): ").split()))]
            print()
        elif "Middle" in self.person.names: # if only one, just use that one
            self.person.idealNames["Middle"] = self.person.names["Middle"]

        if len(self.person.names["Last"]) > 1: # Only one last name can be assigned
            print("Type which last name to include (type in a number):\n",
                  *[str(self.person.names["Last"].index(item)+1)+"."+item+"\n" for item in self.person.names["Last"]])
            self.person.idealNames["Last"] = self.person.names["Last"][int(input("Enter a number from 1 to "+str(len(self.person.names["Last"]))+": "))-1]
        else:
            self.person.idealNames["Last"] = self.person.names["Last"][0]
        print()

        # Every month's abreviation mapped to their full word
        convertMonth = {"JAN":"January","FEB":"February","MAR":"March","APR":"April",
                        "JUN":"June","JUL":"July","AUG":"August","SEP":"September",
                        "OCT":"October","NOV":"November","DEC":"December"}
        if len(self.person.dates["Birth"]) == 3:
            monthB = self.person.dates["Birth"][1]
            # makes months into their full form if not already
            if monthB.upper() in convertMonth:
                monthB = convertMonth[monthB.upper()]
        if len(self.person.dates["Death"]) == 3:
            monthD = self.person.dates["Death"][1]
            if monthD.upper() in convertMonth:
                monthD = convertMonth[monthD.upper()]

        #try:
        print(self.person.idealNames["First"], # first name
              *self.person.idealNames["Middle"], # middle names
              self.person.idealNames["Last"],end=" ") # last name

        if len(self.person.dates["Birth"]) == 1:
            print("was born in the year",self.person.dates["Birth"][0])
        elif len(self.person.dates["Birth"]) == 3:
            birthDay = self.person.dates["Birth"][0]
            # date notation determined in options.txt
            if self.person.options["Dates"].lower() == "british":
                print("was born the",birthDay+(numberNoters[birthDay[-1]] if not (birthDay[0] == "1" and len(birthDay) ==2) else "th"),
                      "of",monthB+",",self.person.dates["Birth"][2],end="") # birth date
            else:
                print("was born",monthB,birthDay+",",self.person.dates["Birth"][2],end="") # birth date

##        except:
##            print("Current knowledge of birth of this individual is unknown.")

        if "BirthPlace" in self.person.events:
            # Prints grave citation and birth place w/ citation if the birthplace is given
            print('<!--\n\n---><ref name="Grave">'+self.person.citation+'</ref><!--\n\n-->',"in",self.person.events["BirthPlace"][0]+
                  '.<!--\n\n--><ref name="I'+str(self.person.events["BirthPlace"][2])[1:]+'">'+ # citation
                  self.person.events["BirthPlace"][1]+'</ref><!--\n\n-->')

        else: # Prints grave citation
            print('.<!--\n\n---><ref name="Grave">'+self.person.citation+'</ref><!--\n\n-->',end=" ")

        if len(self.person.dates["Death"]) == 1:
            print(self.person.idealNames["First"],"passed away in the year",
                  self.person.dates["Death"][0],"and",end=" ") # death date
        if len(self.person.dates["Death"]) == 3:
            deathDay = self.person.dates["Death"][0]

            if self.person.options["Dates"].lower() == "british":
                print(self.person.idealNames["First"],"passed away the",
                      deathDay+(numberNoters[deathDay[-1]] if not (deathDay[0] == "1" and len(deathDay) == 2) else "th"),"of",monthD+",",
                      self.person.dates["Death"][2],"and",end=" ") # death date
            else:
                print(self.person.idealNames["First"],"passed away", monthD, deathDay+",",
                      self.person.dates["Death"][2],"and",end=" ")
        else:
            print("he" if self.person.gender == "Male" else "she",end=" ")

        # Prints cemetery
        location = ''.join([self.person.cemetery[area]+", " if self.person.cemetery[area] != '' else '' for area in ["City","County","State"]])[:-2]
        print("was buried in",self.person.cemetery["Cemetery"],"in",location+'.<!--\n\n--><ref name="Grave"/>')

if __name__ == '__main__':
    person = Person_Model.Person()

    person.addName("First","John")
    person.addName("First","Bob")
    person.addName("Middle","Jacob")
    person.addName("Middle","George")
    person.addName("Last","JingleheimerSchmit")

    person.gender = "Male"

    person.citation = "Austin Lowery. First Hand Recollection. 13 Nov 2019."

    person.addDate("Birth",["APR","12","1970"])
    person.addDate("Death",["DEC","2","2013"])

    person.addCemetery("Cemetery","New Hope Cemetery")
    person.addCemetery("City","Charlotte")
    person.addCemetery("County","Mecklenburg")
    person.addCemetery("State","North Carolina")

    view = Viewer(person)
    view.printAll()


from flask import Flask, render_template, request, url_for
from threading import Thread, Lock
import logging
import sys
cli = sys.modules['flask.cli']
cli.show_server_banner = lambda *x: None

import os , stat
mtx = Lock()

app = Flask(__name__)

@app.route('/')
def root():
    return render_template('tm.html')

def run_web():
    logging.getLogger('werkzeug').disabled = True

    app.run( debug=False, host="0.0.0.0", port=7100)

if __name__ == "__main__":
    app.run()


'''
# LC 33. Search in Rotated Sorted Array
Medium

Given an integer array nums sorted in ascending order, and an integer target.

Suppose that nums is rotated at some pivot unknown to you beforehand (i.e., [0,1,2,4,5,6,7] might become [4,5,6,7,0,1,2]).

You should search for target in nums and if you found return its index, otherwise return -1.

Example 1:

Input: nums = [4,5,6,7,0,1,2], target = 0
Output: 4

Example 2:

Input: nums = [4,5,6,7,0,1,2], target = 3
Output: -1

Example 3:

Input: nums = [1], target = 0
Output: -1

Attempted solve at September 3rd 2020, 17:12. Found solution at 17:20.
Then pomo ended.

Started up pomo again at 1730 and ended at 1802.
Trying to find the index transformation from non-pivoted array to pivoted array

Then tried again until 1832. Cannot get the find_pivot function correct.
Circle back to this

Used help from
153. Find Minimum in Rotated Sorted Array. Off by ones KILL me
'''

from typing import List

class Solution:
    def find_pivot(self, nums: List[int]) -> int:
        '''
        Returns the index of the pivot element
        A pivot element a[i]
        must be strictly bigger than the element on the right
        that is, a[i] > a[i+1]

        How do we find the pivot element?
        We start at the middle element.
        If the middle element is the pivot element, we are done.

        If the middle element lies before the pivot element,
        then nums[mid] > nums[last].
        And the pivot element must lie between the middle element
        and the last element.

        If the middle element lies

        '''
        l = 0
        r = len(nums) - 1

        '''
        if len(nums) == 1:
            return 0
        if len(nums) == 2:
            if nums[1] < nums[0]:
                return 0
            else:
                return 1

        while l < r:
            mid = (l + r) // 2
            if nums[mid] > nums[mid+1]:
                # This is the pivot element
                return mid
            elif nums[mid] > nums[-1]:  # Before the pivot
                # If element lies before the pivot,
                # then pivot must lie between mid and end
                l = mid + 1
            else:  # nums[mid] <= nums[-1] # After the pivot
                # If this element lies after the pivot
                r = mid
        '''

        while l < r:
            mid = (l + r) // 2
            if nums[mid] > nums[r]:
                l = mid + 1
            else:
                r = mid

        return l

    def unpivot(self, nums: List[int], i: int) -> List[int]:
        # pass
        if len(nums) == 1:
            return nums
        a = nums[i:] + nums[:i]
        return a

    def binary_search(self, nums: List[int], n: int) -> int:
        lo = 0
        hi = len(nums)

        while lo < hi:
            mid = (lo + hi) // 2
            if nums[mid] < n:  # too small
                lo = mid + 1
            elif nums[mid] > n:
                hi = mid
            else:
                return mid
        return -1

    def search(self, nums: List[int], target: int) -> int:
        pivot_point = self.find_pivot(nums)
        sorted_nums = self.unpivot(nums, pivot_point)
        # print(sorted_nums)
        sorted_i = self.binary_search(sorted_nums, target)
        # Then do a simple transformation from sorted i to i
        # The binary search gives you the position in the sorted array
        # And you know exactly where the pivot point is
        # We know the pivot point in the sorted array.
        # print(nums, sorted_nums)

        sorted_pivot_point = len(nums) - pivot_point - 1
        # print(sorted_i)
        if sorted_i == -1:
            return -1
        if sorted_i <= sorted_pivot_point:
            # This element is small and before the pivot element
            # in the sorted list.
            return pivot_point + sorted_i
        else:
            return sorted_i - sorted_pivot_point - 1

def sol(nums, target):
    s = Solution()
    print(nums, target, ': ', s.search(nums, target))

sol([4, 5, 6, 7, 0, 1, 2], 0)  # 4 , 0
sol([4, 5, 6, 7, 0, 1, 2], 3)  # -1
sol([6, 7, 0, 1, 2, 3, 4, 5], 3)  # 3, pivot_elem = 1, +1 = 2, --> 5
sol([6, 7, 0, 1, 2, 3, 4, 5], 6)  # 6, pivot_elem = 1, +1 = 2, --> 0
sol([6, 7, 0, 1, 2, 3, 4, 5], 7)  # 7, 1, --> 1
sol([1], 0)  # -1
sol([1, 3], 1)  # 0
sol([3, 1], 1)  # 1
sol([3, 1], 0)  # -1
sol([3, 1, 2], 1)  # 1
sol([3, 1, 2], 2)  # 2
sol([3, 1, 2], 3)  # 0
sol([2, 3, 1], 3)  # 1
sol([1, 2, 3], 3)  # 2
sol([1, 2, 3], 1)  # 0
sol([2, 3, 4, 5, 1], 1)


# bfs
import sys
from collections import deque
input = sys.stdin.readline

# 양 #
# 풀 .

dx = [1, -1, 0, 0]
dy = [0, 0, 1, -1]

def bfs(x, y, h, w, visited):
    q = deque([(x, y)])
    visited[x][y] = True

    while q:
        ex, ey = q.pop()

        for i in range (4):
            nx = ex + dx[i]
            ny = ey + dy[i]

            if 0<=nx<h and 0<=ny<w:
                if visited[nx][ny] == False and grid[nx][ny] == '#':
                    visited[nx][ny] = True
                    q.append((nx, ny))

t = int(input())
for _ in range (t):
    h, w = map(int, input().split())

    grid = []

    for _ in range (h):
        grid.append(list(map(str, input().rstrip())))

    visited = [[False for _ in range (w)] for _ in range (h)]
    count = 0

    for i in range (h):
        for j in range (w):
            if visited[i][j] == False:
                if grid[i][j] == '#':
                    bfs(i, j, h, w, visited)
                    count += 1

    print(count)


import cv2
import time
import numpy as np

fourcc = cv2.VideoWriter_fourcc(*"xvid")
outputfile = cv2.VideoWriter("output.avi")

capture = cv2.VideoCapture(0)
time.sleep(2)
background = 0

for i in range(60):
    ret,background = capture.read()

background = np.flip(background,axis = 1)

while(capture.isopen()):
    ret,image = capture.read()
    if not ret:
        break

    image = np.flip(image,axis = 1)

    hsv = cv2.cvtcolor(image,cv2.COLOR_BGR2HSV)
    lowerred = np.array([0,120,50])
    upperred = np.array([0,100,150])
    mask1 = cv2.inrange(hsv,lowerred,upperred)

    lowerred = np.array([120,120,70])
    upperred = np.array([180,255.255])
    mask2 = cv2.inrange(hsv,lowerred,upperred)

    mask = mask1 + mask2

    mask_1 = cv2.morphologyEx(mask_1, cv2.MORPH_OPEN, np.ones((3, 3), np.uint8))
    mask_1 = cv2.morphologyEx(mask_1, cv2.MORPH_DILATE, np.ones((3, 3), np.uint8))

    mask2 = cv2.bitwise_not(mask1)

capture.realase()
cv2.destroyallwindows()


#!/usr/bin/python
import re
import sys

word_input = sys.argv[12]
word_output = sys.argv[14]
concepts_number = int(sys.argv[2])
num_results_reqd = int(sys.argv[4])
directory_location = sys.argv[6]
document_input = sys.argv[8]
query_input = sys.argv[16]
document_output = sys.argv[10]
query_output = sys.argv[18]

# num_results_reqd=10
# concepts_number = 200

import os
current_directory = os.getcwd()
doc_location = current_directory+'/'+directory_location
os.chdir(doc_location)

pattern = re.compile(r'\W+')

from collections import Counter

# curr_file_words.remove('')
dictionary_list = [];
set_of_words = set()
Title_index={}
index_Title={}

file_up_limit=5000;

for file_num in range(1,file_up_limit+1):
	file_str = str(file_num) + ".txt"
	reading_file = open(file_str,'r')
	title=reading_file.readline().rstrip()
	curr_file_str=reading_file.read()
	reading_file.close()
	Title_index[title]=file_num
	index_Title[file_num]=title
	curr_file_words = pattern.split(curr_file_str.lower().strip(' ').rstrip('\n'))
	set_of_words=set_of_words.union(set(curr_file_words))
	dictionary_list.append(dict(Counter(curr_file_words) ))
	# print file_num

# TODO->convert to lower case

# print(len(dictionary_list))

os.chdir(current_directory)

list_of_words = list(set_of_words);

unique_words_num=len(list_of_words)

words_index={};

for i in range(0,unique_words_num):
	words_index[list_of_words[i]]=i

list_doc_col=[];
list_word_row=[];
list_count_data=[];

# print("Started making the 3 lists")

for doc_num in range (0,file_up_limit):
	for current_word in dictionary_list[doc_num]:
		list_doc_col.append(doc_num)
		list_word_row.append(words_index[current_word])
		list_count_data.append(dictionary_list[doc_num][current_word])

# print("Made the 3 lists")

import numpy as np
import scipy.sparse as sp
from scipy import spatial
sparse_doc_word_matrix=sp.csc_matrix((np.array(list_count_data), (np.array(list_word_row), np.array(list_doc_col) ) ), shape = (unique_words_num, file_up_limit)  )

# sp.coo_matrix.asfptype(sparse_doc_word_matrix)

from scipy.sparse.linalg import svds
term_row_matrix , strength_matrix , doc_col_matrix = svds(sparse_doc_word_matrix.astype(float), concepts_number , which = 'LM')

doc_row_matrix= np.dot(np.transpose(doc_col_matrix),np.diag(strength_matrix))
# Now a query is given and we need to find out the Documents!!

# print("Started Query IO")

multiply1= np.dot( np.array(np.array(term_row_matrix)),np.linalg.inv(np.diag(strength_matrix))  )

query_input_list = [(line.rstrip('\n')).strip(' ') for line in open(query_input)]

write_query_results = open(query_output,"a")

for query_input_index in range(0,len(query_input_list)):
	query_string = query_input_list[query_input_index]
	query_words=pattern.split(query_string.lower())
	query_dictionary=dict(Counter(query_words) );
	query_list_doc_col=[];
	query_list_word_row=[];
	query_list_count_data=[];
	for current_word in query_dictionary:
		query_list_doc_col.append(0)
		query_list_word_row.append(words_index[current_word])
		query_list_count_data.append(query_dictionary[current_word])
	query_sparse_doc_word_matrix=sp.csc_matrix((np.array(query_list_count_data), (np.array(query_list_word_row), np.array(query_list_doc_col) ) ), shape = (unique_words_num, 1)  )
	query_sparse_doc_word_matrix=query_sparse_doc_word_matrix.todense();
	Final_query_vector=np.dot(  np.transpose(query_sparse_doc_word_matrix),multiply1)
	# doc_row_matrix= doc_col_matrix.transpose()
	# doc_query=1
	# doc_query_index=doc_query-1
	doc_dot_product_list=np.array([])
	doc_query_row=Final_query_vector.transpose();
	for other_doc_row in doc_row_matrix:
		doc_dot_product_list=np.append(doc_dot_product_list,1 - spatial.distance.cosine(doc_query_row,other_doc_row))
	unsorted_answer_indices = np.argpartition(  doc_dot_product_list , -1*(num_results_reqd))[-1*(num_results_reqd):]
	sorted_answer_indices= unsorted_answer_indices[np.argsort(doc_dot_product_list[unsorted_answer_indices])]
	final_doc_answer_list=[]
	for i in range(0,num_results_reqd):
		write_query_results.write(index_Title[sorted_answer_indices[num_results_reqd-i-1]+1]+";"+"\t")
		# final_words_answer_list.append(sorted_answer_indices[num_results_reqd-i-1]+1)
		# print (sorted_answer_indices[num_results_reqd-i-1]+1)
	write_query_results.write('\n')

write_query_results.close()

# print("Ended Query IO")

# Given a word, then finding its best k words
# print("Started Word IO")

compare_term_matrix=np.dot(term_row_matrix,np.diag(strength_matrix))

word_input_list = [(line.rstrip('\n')).strip(' ') for line in open(word_input)]

write_word_results= open(word_output,"a")

# print "size is ",len(word_input_list)

for word_input_index in range(0,len(word_input_list)):
	# print "Doing for index",word_input_index
	word_query=word_input_list[word_input_index]
	word_query_index=words_index[word_query]
	word_dot_product_list=[0 for x in xrange(unique_words_num)]
	word_query_row=compare_term_matrix[word_query_index];
	# print "step1"
	for i in xrange(unique_words_num):
		word_dot_product_list[i]=(1 - spatial.distance.cosine(word_query_row,compare_term_matrix[i]))
	# print "step2"
	# print len(word_dot_product_list)
	# print len(word_dot_product_list[0])
	unsorted_answer_indices = np.argpartition(  word_dot_product_list , -1*(num_results_reqd))[-1*(num_results_reqd):]
	# print "WORD DOT",word_dot_product_list
	# print "UNSORTED INDICES",unsorted_answer_indices
	# print "step3"
	# sorted_answer_indices= unsorted_answer_indices[np.argsort(word_dot_product_list[unsorted_answer_indices])]
	# final_words_answer_list=[]
	# print "step4"
	for i in range(0,num_results_reqd):
		write_word_results.write(list_of_words[unsorted_answer_indices[num_results_reqd-1-i]]+";"+"\t")
		# final_words_answer_list.append(list_of_words[sorted_answer_indices[num_results_reqd-1-i]])
		# print list_of_words[sorted_answer_indices[num_results_reqd-1-i]]
	# print "step5"
	write_word_results.write('\n')

write_word_results.close()

# print("Ended Word IO")

# Given a doc then finding similar documents

# print("Started DOC IO")

doc_input_list = [(line.rstrip('\n')).strip(' ') for line in open(document_input)]

write_doc_results= open(document_output,"a")

# print "Size is",len(doc_input_list)

for doc_input_index in range(0,len(doc_input_list)):
	# print(doc_input_index)
	doc_query=Title_index[doc_input_list[doc_input_index]]
	doc_query_index=doc_query-1
	doc_dot_product_list=np.array([])
	doc_query_row=doc_row_matrix[doc_query_index];
	for other_doc_row in doc_row_matrix:
		doc_dot_product_list=np.append(doc_dot_product_list,1 - spatial.distance.cosine(doc_query_row,other_doc_row))
	unsorted_answer_indices = np.argpartition(  doc_dot_product_list , -1*(num_results_reqd))[-1*(num_results_reqd):]
	sorted_answer_indices= unsorted_answer_indices[np.argsort(doc_dot_product_list[unsorted_answer_indices])]
	final_doc_answer_list=[]
	for i in range(0,num_results_reqd):
		write_doc_results.write(index_Title[sorted_answer_indices[num_results_reqd-i-1]+1]+";"+"\t")
		# final_words_answer_list.append(sorted_answer_indices[num_results_reqd-i-1]+1)
		# print (sorted_answer_indices[num_results_reqd-i-1]+1)
	write_doc_results.write('\n')

write_doc_results.close()

# print("Ended DOC IO")


import os
import re
import json
import getpass
import requests
import pyblish.api

class CelactionSubmitDeadline(pyblish.api.InstancePlugin):
    """Submit CelAction2D scene to Deadline

    Renders are submitted to a Deadline Web Service.

    """

    label = "Submit CelAction to Deadline"
    order = pyblish.api.IntegratorOrder + 0.1
    hosts = ["celaction"]
    families = ["render.farm"]

    deadline_department = ""
    deadline_priority = 50
    deadline_pool = ""
    deadline_pool_secondary = ""
    deadline_group = ""
    deadline_chunk_size = 1
    deadline_job_delay = "00:00:08:00"

    def process(self, instance):

        context = instance.context

        # get default deadline webservice url from deadline module
        deadline_url = instance.context.data["defaultDeadline"]
        # if custom one is set in instance, use that
        if instance.data.get("deadlineUrl"):
            deadline_url = instance.data.get("deadlineUrl")
        assert deadline_url, "Requires Deadline Webservice URL"

        self.deadline_url = "{}/api/jobs".format(deadline_url)
        self._comment = instance.data["comment"]
        self._deadline_user = context.data.get(
            "deadlineUser", getpass.getuser())
        self._frame_start = int(instance.data["frameStart"])
        self._frame_end = int(instance.data["frameEnd"])

        # get output path
        render_path = instance.data['path']
        script_path = context.data["currentFile"]

        response = self.payload_submit(instance,
                                       script_path,
                                       render_path
                                       )
        # Store output dir for unified publisher (filesequence)
        instance.data["deadlineSubmissionJob"] = response.json()

        instance.data["outputDir"] = os.path.dirname(
            render_path).replace("\\", "/")

        instance.data["publishJobState"] = "Suspended"

        # adding 2d render specific family for version identification in Loader
        instance.data["families"] = ["render2d"]

    def payload_submit(self,
                       instance,
                       script_path,
                       render_path
                       ):
        resolution_width = instance.data["resolutionWidth"]
        resolution_height = instance.data["resolutionHeight"]
        render_dir = os.path.normpath(os.path.dirname(render_path))
        render_path = os.path.normpath(render_path)
        script_name = os.path.basename(script_path)

        for item in instance.context:
            if "workfile" in item.data["family"]:
                msg = "Workfile (scene) must be published along"
                assert item.data["publish"] is True, msg

                template_data = item.data.get("anatomyData")
                rep = item.data.get("representations")[0].get("name")
                template_data["representation"] = rep
                template_data["ext"] = rep
                template_data["comment"] = None
                anatomy_filled = instance.context.data["anatomy"].format(
                    template_data)
                template_filled = anatomy_filled["publish"]["path"]
                script_path = os.path.normpath(template_filled)

                self.log.info(
                    "Using published scene for render {}".format(script_path)
                )

        jobname = "%s - %s" % (script_name, instance.name)

        output_filename_0 = self.preview_fname(render_path)

        try:
            # Ensure render folder exists
            os.makedirs(render_dir)
        except OSError:
            pass

        # define chunk and priority
        chunk_size = instance.context.data.get("chunk")
        if not chunk_size:
            chunk_size = self.deadline_chunk_size

        # search for %02d pattern in name, and padding number
        search_results = re.search(r"(%0)(\d)(d)[._]", render_path).groups()
        split_patern = "".join(search_results)
        padding_number = int(search_results[1])

        args = [
            f"<QUOTE>{script_path}<QUOTE>",
            "-a",
            "-16",
            "-s <STARTFRAME>",
            "-e <ENDFRAME>",
            f"-d <QUOTE>{render_dir}<QUOTE>",
            f"-x {resolution_width}",
            f"-y {resolution_height}",
            f"-r <QUOTE>{render_path.replace(split_patern, '')}<QUOTE>",
            f"-= AbsoluteFrameNumber=on -= PadDigits={padding_number}",
            "-= ClearAttachment=on",
        ]

        payload = {
            "JobInfo": {
                # Job name, as seen in Monitor
                "Name": jobname,

                # plugin definition
                "Plugin": "CelAction",

                # Top-level group name
                "BatchName": script_name,

                # Arbitrary username, for visualisation in Monitor
                "UserName": self._deadline_user,

                "Department": self.deadline_department,
                "Priority": self.deadline_priority,

                "Group": self.deadline_group,
                "Pool": self.deadline_pool,
                "SecondaryPool": self.deadline_pool_secondary,
                "ChunkSize": chunk_size,

                "Frames": f"{self._frame_start}-{self._frame_end}",
                "Comment": self._comment,

                # Optional, enable double-click to preview rendered
                # frames from Deadline Monitor
                "OutputFilename0": output_filename_0.replace("\\", "/"),

                # # Asset dependency to wait for at least
                # the scene file to sync.
                # "AssetDependency0": script_path
                "ScheduledType": "Once",
                "JobDelay": self.deadline_job_delay
            },
            "PluginInfo": {
                # Input
                "SceneFile": script_path,

                # Output directory
                "OutputFilePath": render_dir.replace("\\", "/"),

                # Plugin attributes
                "StartupDirectory": "",
                "Arguments": " ".join(args),

                # Resolve relative references
                "ProjectPath": script_path,
                "AWSAssetFile0": render_path,
            },

            # Mandatory for Deadline, may be empty
            "AuxFiles": []
        }

        plugin = payload["JobInfo"]["Plugin"]
        self.log.debug("using render plugin : {}".format(plugin))

        self.log.debug("Submitting..")
        self.log.debug(json.dumps(payload, indent=4, sort_keys=True))

        # adding expectied files to instance.data
        self.expected_files(instance, render_path)
        self.log.debug("__ expectedFiles: `{}`".format(
            instance.data["expectedFiles"]))

        response = requests.post(self.deadline_url, json=payload)

        if not response.ok:
            self.log.error(
                "Submission failed! [{}] {}".format(
                    response.status_code, response.content))
            self.log.debug(payload)
            raise SystemExit(response.text)

        return response

    def preflight_check(self, instance):
        """Ensure the startFrame, endFrame and byFrameStep are integers"""

        for key in ("frameStart", "frameEnd"):
            value = instance.data[key]

            if int(value) == value:
                continue

            self.log.warning(
                "%f=%d was rounded off to nearest integer"
                % (value, int(value))
            )

    def preview_fname(self, path):
        """Return output file path with #### for padding.

        Deadline requires the path to be formatted with # in place of numbers.
        For example `/path/to/render.####.png`

        Args:
            path (str): path to rendered images

        Returns:
            str

        """
        self.log.debug("_ path: `{}`".format(path))
        if "%" in path:
            search_results = re.search(r"[._](%0)(\d)(d)[._]", path).groups()
            split_patern = "".join(search_results)
            split_path = path.split(split_patern)
            hashes = "#" * int(search_results[1])
            return "".join([split_path[0], hashes, split_path[-1]])

        self.log.debug("_ path: `{}`".format(path))
        return path

    def expected_files(self, instance, filepath):
        """ Create expected files in instance data
        """
        if not instance.data.get("expectedFiles"):
            instance.data["expectedFiles"] = []

        dirpath = os.path.dirname(filepath)
        filename = os.path.basename(filepath)

        if "#" in filename:
            pparts = filename.split("#")
            padding = "%0{}d".format(len(pparts) - 1)
            filename = pparts[0] + padding + pparts[-1]

        if "%" not in filename:
            instance.data["expectedFiles"].append(filepath)
            return

        for i in range(self._frame_start, (self._frame_end + 1)):
            instance.data["expectedFiles"].append(
                os.path.join(dirpath, (filename % i)).replace("\\", "/")
            )


# -*- coding: utf-8 -*-
import scrapy
from pymongo import MongoClient
from DAzhaopin.items import JobdescItem
import logging

class JobdescSpider(scrapy.Spider):
    name = 'jobdesc'
    allowed_domains = ['lagou.com']
    db = MongoClient('mongodb://localhost:32768')['zhaopin']
    logging = logging.getLogger(__name__)

    def start_requests(self):
        idset = set([p['positionId'] for p in self.db['jobs'].find({})])
        pidset = set([p['positionId'] for p in self.db['lagou'].find({})])
        posid = pidset.difference(idset)

        url = 'https://www.lagou.com/jobs/%s.html'
        count = len(posid)
        for n, value in enumerate(posid):
            yield scrapy.Request(url % value, meta={'positionId': value, 'count':(n+1, count)})

    def parse(self, response):
        self.logging.info('正在处理%s'%response.url)

        sub = response.xpath('//dl[@class="job_detail"]')
        item = JobdescItem()
        job_detail = sub.xpath('./dd/div[@class="job-detail"]//text()').re('\S+')
        job_addr = sub.xpath('//div[@class="work_addr"]//text()').re('\S+')[:-1]
        if job_addr and job_detail:
            self.logging.info('处理进度%d / %d' % response.meta['count'])
            item['positionId'] = response.meta['positionId']
            item['job_detail'] = job_detail
            item['job_addr'] = job_addr
            yield item
        else:
            self.logging.info('重新处理%s'%response.meta['positionId'])
            yield scrapy.Request('https://www.lagou.com/jobs/%s.html'%response.meta['positionId'], dont_filter=True, meta=response.meta)


# -*- coding: utf-8 -*-

import os
import telnetlib

from log.logger import logger

from fg.error import (
    Error,
    ErrorCode,
    ErrorMsg
)
from fg.return_tools import (
    return_error,
    return_success
)
from fg.constants import (
    ACTION_V2V_NAS_PARSE_VMS_FROM_NAS,
    ACTION_V2V_NAS_CHECK_NAS_CONNECTIVITY
)
from fg.resource_control.nas import (
    interface,
    vmware_img
)
from fg.uutils.utils_common import (
    is_contains_chinese,
    order_list_and_paginate
)

def handle_parse_vms_from_nas(req):
    """从NAS服务器中解析虚拟机信息"""
    logger.debug('handle parse vms from nas start, {}'.format(req))
    url = req.get("image_addr")
    offset = req.get("offset", 0)
    limit = req.get("limit", 10)
    sort_key = req.get("sort_key", "template")
    reverse = req.get("reverse", False)

    # 中文检查
    is_cn = is_contains_chinese(url)
    if is_cn:
        return return_error(req,
                            Error(ErrorCode.ERR_CODE_NAS_INVAILD_NFS_URL_ADDRESS.value,
                                  ErrorMsg.ERR_MSG_NAS_INVAILD_NFS_URL_ADDRESS.value))

    # 初始化客户端
    nfs_client = interface.NFSClient(url)
    if not nfs_client.mount():
        logger.error("nfs url check connectivity failed, url: %s" % url)
        return return_error(req,
                            Error(ErrorCode.ERR_CODE_NAS_NFS_URL_NOT_CONNECT.value,
                                  ErrorMsg.ERR_MSG_NAS_NFS_URL_NOT_CONNECT.value))

    datas = []
    vm_dirs = nfs_client.list_dirs(".")
    for vm_dir in vm_dirs:
        try:
            vm_upload_path = os.path.join(url, vm_dir)
            stat_info = nfs_client.stat_file(vm_dir)
            if stat_info.get("nlink") == 1:
                continue

            vm_file_list = nfs_client.list_dirs(vm_dir)
            vm_ovf_path = ''
            ovf_count = 0
            vmdk_count = 0

            for vm_file in vm_file_list:
                if vm_file.endswith('ovf'):
                    vm_ovf_path = os.path.join(vm_dir, vm_file)
                    ovf_count += 1
                elif vm_file.endswith("vmdk"):
                    vmdk_count += 1
            if not vm_ovf_path:
                logger.error("ovf file not found in vm image path: %s "
                             "" % vm_upload_path)
                continue

            if ovf_count > 1:
                logger.error("the count of ovf file is {ovf_count}, vm upload "
                             "path: {vm_upload_path}"
                             "".format(vm_upload_path=vm_upload_path,
                                       ovf_count=ovf_count))
                continue

            ovf_xml = nfs_client.read_file(vm_ovf_path)
            vm_img = vmware_img.VmWareImg(ovf_xml, vm_upload_path)
            vm = vm_img.parse_vm_dict()

            vmdk_img_ct = vm_img.get_vmdk_file_count()
            if vmdk_img_ct != vmdk_count:
                logger.error("vm vmdk file: %s not match image vmdk: %s" % (
                    vmdk_img_ct, vmdk_count))
                continue

            datas.append(vm)
        except Exception as e:
            logger.exception("get vm image path failed, url: %s, error: %s"
                             "" % (url, e))
            continue
    try:
        result_list, total = order_list_and_paginate(datas, sort_key, offset,
                                                     limit, reverse)
    except Exception as e:
        logger.exception("pageinate error, data: {data}, error: {e}"
                         "".format(data=datas, e=e))
        return return_error(req,
                            Error(ErrorCode.ERR_CODE_NAS_PARSE_VMS_FROM_NFS_PAGINATE_ERROR.value,
                                  ErrorMsg.ERR_MSG_NAS_PARSE_VMS_FROM_NFS_PAGINATE_ERROR.value))

    return return_success(req, None, datas={"src_vm_set": result_list,
                                            "total_count": total})

def handle_check_nas_connectivity(req):
    """检查NAS的联通性"""
    logger.debug('handle check nas connectivity start, {}'.format(req))
    url = req.get("image_addr")
    port = req.get("port", 2049)

    # 中文检查
    is_cn = is_contains_chinese(url)
    if is_cn:
        logger.error("url address of url contain chinese, url: %s" % url)
        return return_error(req,
                            Error(ErrorCode.ERR_CODE_NAS_INVAILD_NFS_URL_ADDRESS.value,
                                  ErrorMsg.ERR_MSG_NAS_INVAILD_NFS_URL_ADDRESS.value))

    # 统一nfs路径格式
    if url.startswith("nfs://"):
        nfs_arr = url.split("nfs://")
        if len(nfs_arr) == 2:
            url = nfs_arr[1]

    # 解析主机地址
    host = ""
    if len(url.split("/")) > 0:
        host = url.split("/")[0]

    # 主机检查
    ret = os.system("ping {host} -c 1".format(host=host))
    if ret != 0:
        logger.error("nfs connectivity check failed, url: %s, host, %s, port: "
                     "%s, reason: host is cot connect" % (url, host, port))
        return return_error(req,
                            Error(ErrorCode.ERR_CODE_NAS_NFS_URL_IP_NOT_CONNECT.value,
                                  ErrorMsg.ERR_MSG_NAS_NFS_URL_IP_NOT_CONNECT.value))

    # 端口检查
    try:
        tn = telnetlib.Telnet()
        tn.open(host, port, timeout=2)
    except Exception:
        logger.error("nfs connectivity check failed, url: %s, host, %s, port: "
                     "%s, reason: port is cot connect" % (url, host, port))
        return return_error(req,
                            Error(ErrorCode.ERR_CODE_NAS_NFS_URL_PORT_NOT_CONNECT.value,
                                  ErrorMsg.ERR_MSG_NAS_NFS_URL_PORT_NOT_CONNECT.value))

    # URL地址检查
    nfs_client = interface.NFSClient(url)
    if not nfs_client.mount():
        logger.error("nfs connectivity check failed, url: %s, port: %s"
                     "" % (url, port))
        return return_error(req,
                            Error(ErrorCode.ERR_CODE_NAS_NFS_URL_NOT_CONNECT.value,
                                  ErrorMsg.ERR_MSG_NAS_NFS_URL_NOT_CONNECT.value))

    return return_success(req, None, data=True)

HANDLER_MAP = {
    ACTION_V2V_NAS_PARSE_VMS_FROM_NAS: handle_parse_vms_from_nas,
    ACTION_V2V_NAS_CHECK_NAS_CONNECTIVITY: handle_check_nas_connectivity
}


# -*- coding: utf-8 -*-
"""
Created on Sat Dec 17 17:03:57 2016

@author: Chase
"""

from keras.layers.advanced_activations import LeakyReLU
from keras.layers import Input, merge, SpatialDropout2D
from keras.layers import Convolution2D, AveragePooling2D, UpSampling2D
from keras.models import Model
from keras.optimizers import Adam

def get_cnn_model(img_rows, img_cols):
    input = Input(shape=(1, img_rows, img_cols))

    conv1 = Convolution2D(32, 3, 3, border_mode='same', init='he_normal')(input)
    conv1 = LeakyReLU()(conv1)
    conv1 = SpatialDropout2D(0.2)(conv1)
    conv1 = Convolution2D(32, 3, 3, border_mode='same', init='he_normal')(conv1)
    conv1 = LeakyReLU()(conv1)
    conv1 = SpatialDropout2D(0.2)(conv1)
    pool1 = AveragePooling2D(pool_size=(2,2))(conv1)

    conv2 = Convolution2D(64, 3, 3, border_mode='same', init='he_normal')(pool1)
    conv2 = LeakyReLU()(conv2)
    conv2 = SpatialDropout2D(0.2)(conv2)
    conv2 = Convolution2D(64, 3, 3, border_mode='same', init='he_normal')(conv2)
    conv2 = LeakyReLU()(conv2)
    conv2 = SpatialDropout2D(0.2)(conv2)
    pool2 = AveragePooling2D(pool_size=(2,2))(conv2)

    conv3 = Convolution2D(128, 3, 3, border_mode='same', init='he_normal')(pool2)
    conv3 = LeakyReLU()(conv3)
    conv3 = SpatialDropout2D(0.2)(conv3)
    conv3 = Convolution2D(128, 3, 3, border_mode='same', init='he_normal')(conv3)
    conv3 = LeakyReLU()(conv3)
    conv3 = SpatialDropout2D(0.2)(conv3)

    comb1 = merge([conv2, UpSampling2D(size=(2,2))(conv3)], mode='concat', concat_axis=1)
    conv4 = Convolution2D(64, 3, 3, border_mode='same', init='he_normal')(comb1)
    conv4 = LeakyReLU()(conv4)
    conv4 = SpatialDropout2D(0.2)(conv4)
    conv4 = Convolution2D(64, 3, 3, border_mode='same', init='he_normal')(conv4)
    conv4 = LeakyReLU()(conv4)
    conv4 = SpatialDropout2D(0.2)(conv4)

    comb2 = merge([conv1, UpSampling2D(size=(2,2))(conv4)], mode='concat', concat_axis=1)
    conv5 = Convolution2D(32, 3, 3, border_mode='same', init='he_normal')(comb2)
    conv5 = LeakyReLU()(conv5)
    conv5 = SpatialDropout2D(0.2)(conv5)
    conv5 = Convolution2D(32, 3, 3, border_mode='same', init='he_normal')(conv5)
    conv5 = LeakyReLU()(conv5)
    conv5 = SpatialDropout2D(0.2)(conv5)

    output = Convolution2D(1, 1, 1, activation='sigmoid')(conv5)

    model = Model(input=input, output=output)
    model.compile(optimizer=Adam(lr=3e-4), loss='binary_crossentropy')
    return model

def get_iterative_model(img_rows, img_cols):
    # the heatmap from CNN
    input1 = Input(shape=(1, img_rows, img_cols))

    # the image heatmap was generated from (change 1 to 3 for color images)
    input2 = Input(shape=(1, img_rows, img_cols))

    # these are shared layer declarations
    convh1 = Convolution2D(32, 3, 3, activation='relu', border_mode='same')
    convh2 = Convolution2D(64, 3, 3, activation='relu', border_mode='same')

    convi1 = Convolution2D(32, 3, 3, activation='relu', border_mode='same')

    convmlp1 = Convolution2D(32, 1, 1, activation='relu')
    convmlp2 = Convolution2D(32, 1, 1, activation='relu')
    convmlp3 = Convolution2D(1, 1, 1)

    # now the actual model
    upperconv1 = convh1(input1)
    lowerconv1 = convi1(input2)

    concat1 = merge([upperconv1, lowerconv1], mode='concat', concat_axis=1)
    upperconv2 = convh2(concat1)

    mlp11 = convmlp1(upperconv2)
    mlp12 = convmlp2(mlp11)
    mlp13 = convmlp3(mlp12)

    add1 = merge([input1, mlp13], mode='sum')

    upperconv2 = convh1(add1)
    lowerconv2 = convi1(input2)

    concat2 = merge([upperconv2, lowerconv2], mode='concat', concat_axis=1)
    upperconv3 = convh2(concat2)

    mlp21 = convmlp1(upperconv3)
    mlp22 = convmlp2(mlp21)
    mlp23 = convmlp3(mlp22)

    add2 = merge([add1, mlp23], mode='sum')

    upperconv3 = convh1(add2)
    lowerconv3 = convi1(input2)

    concat3 = merge([upperconv3, lowerconv3], mode='concat', concat_axis=1)
    upperconv4 = convh2(concat3)

    mlp31 = convmlp1(upperconv4)
    mlp32 = convmlp2(mlp31)
    mlp33 = convmlp3(mlp32)

    add3 = merge([add2, mlp33], mode='sum')

    upperconv4 = convh1(add3)
    lowerconv4 = convi1(input2)

    concat4 = merge([upperconv4, lowerconv4], mode='concat', concat_axis=1)
    upperconv5 = convh2(concat4)

    mlp41 = convmlp1(upperconv5)
    mlp42 = convmlp2(mlp41)
    mlp43 = convmlp3(mlp42)

    add4 = merge([add3, mlp43], mode='sum')

    upperconv5 = convh1(add4)
    lowerconv5 = convi1(input2)

    concat5 = merge([upperconv5, lowerconv5], mode='concat', concat_axis=1)
    upperconv6 = convh2(concat5)

    mlp51 = convmlp1(upperconv6)
    mlp52 = convmlp2(mlp51)
    mlp53 = convmlp3(mlp52)

    add5 = merge([add4, mlp53], mode='sum')

    model = Model(input=[input1, input2], output=add5)
    model.compile(Adam(lr=3e-4), 'binary_crossentropy')

    return model


from __future__ import annotations
from typing import Union, Type, Any
from src.filter.Filter import Filter
from src.controller.Controller import Controller
from src.metric.CompoundMetricManager import CompoundMetricManager
from src.metric.SampleMetricManager import SampleMetricManager
from src.metric.SampleMetric import SampleMetric
from src.metric.CompoundMetric import CompoundMetric
from src.population.Population import Population
import numpy as np

COMPOUND_MODEL_FACTORY_PREFIX_NAME = "CMF"

class CompoundModelFactory:
    """
    This class represents a compound model factory. Each instance is responsible for producing
    compound models that are based on a fixed controller and generator.

    A compound model factory consists mainly of a controller, generator, and filters.
    Furthermore, a compound model is defined as the controller, generator and a filter, such that
    the number of filters determines how many compound models a factory represents.

    Each factory shares a population between all of its compound models, named after the controller and the generator.
    However, each compound model calculates its own metrics using their own metric manager on aforementioned
    population , by applying them to the separate filterings.
    """

    def __init__(
        self,
        controller: Controller,
        filters: list[Type[Filter]],
        sample_metrics: list[Type[SampleMetric]],
        compound_metrics: list[Type[CompoundMetric]],
    ):
        """
        Constructs a CompoundModelFactory with the specified arguments.

        Args:
            controller (Controller): Controller of the CompoundModelFactory. Generator and Dataset will be inferred from
                the controller.
            filters (list[Type[Filter]]): Filters of the CompoundModelFactory, each filter in combination with the
                controller is defined as a compound model.
            sample_metrics (list[Type[SampleMetric]]): All sample metrics used to evaluate the model.
            compound_metrics (list[Type[CompoundMetric]]): All compound metrics used to evaluate the model.
        """
        # Define key compound model components
        self._controller = controller
        self._generator = controller.get_generator()
        self._filters = filters

        # Define data objects
        self._dataset = self._generator.get_dataset()
        self._compound_model_name_prefix = "_".join(
            [
                controller.get_name(),
                self._generator.get_name(),
            ]
        )
        self._name = "_".join(
            [COMPOUND_MODEL_FACTORY_PREFIX_NAME, self._compound_model_name_prefix]
        )
        self._population = Population(self._name)

        # Define metric related objects
        self._smm = SampleMetricManager(sample_metrics, self._population, self._dataset)

        self._cmms = {
            filter.get_name(): CompoundMetricManager(
                compound_metrics,
                self._population,
                self._dataset,
                self._smm,
                controller,
                filter.get_bit(),
            )
            for filter in self._filters
        }

        # Define context
        self._context = None

    def apply_filters(
        self,
        filters: Union[Type[Filter], list[Type[Filter]]] = None,
    ) -> None:
        """
        Apply filters to the population.

        The result is stored in disk according to the population class.
        If any changes occur to the population, this function should be called again
        before evaluation.

        Note that this should be done before calling any evaluate functions.

        Args:
            filters (Type[Filter] | list[Type[Filter]], optional): Selection of filters to apply, must be subset
                of the filters that were specified on construction, see `get_filters()`.
                `filters` can either be a filter, a list of filters or `None` which will result in the use of all available filters.
                Defaults to None.
        """

        if isinstance(filters, type(Filter)):
            filters = [filters]

        for filter in self._filters:
            if filters is None or filter in filters:
                self._population.apply_filter(filter, self._smm)

    def give_context(self, context: CompoundModelFactoryContext) -> None:
        """
        Gives a context to the factory, used to keep track of all other
        factories.

        Note that this context will be given to compound metric managers as well,
        such that it can be reached from compound metrics.

        Args:
            context (CompoundModelFactoryContext): The context.
        """
        # Give context to the factory
        self._context = context

        # Propagate the context to CMMs
        for cmm in self._cmms.values():
            cmm.give_context(context)

    def generate_custom_population(
        self,
        latent_codes: np.ndarray,
        input: dict[str, np.ndarray],
        pop_name: str,
        append=True,
    ) -> None:
        """
        Generates samples based on specified latent codes and input (which specify
        attribute manipulations).

        Samples will be saved under the specified population name.

        An example of the input:

        Let input be defined as:
        `input_example = {"gender" : np.ndarray[-1, 1, -0.5], "age": np.ndarray[1,0,-1]}`
        then input for image 1 should then be parsed as an image with gender = -1
        and age = 1. Thus, `input_example` defines parameters for 3 images (length of the array).

        Note that a value of 0 means that no changes for that parameter will be done for that image.

        Note that the custom population is not meant to be evaluated. However, it can be done
        by constructing a metric manager for the population, this is not intended and may produce
        strange results.

        Note that this function clears local and internal compound metrics stored by the compound metric manager for this
        population.

        Args:
            latent_codes (np.ndarray): Latent codes that will be generated. Should be at least
                2-dimensional. First axis should represent the number of samples, the other ones should
                represent the dimensions of a latent code.
            input (dict[str, np.ndarray]): Standardized input representation.
                Where key = parameter name, and the value = array of values
                (in float range of [-1,1]) of that parameter. Each value in the
                array represents one generated image. For example, image 1 should
                be based on value indexed 1 in the array for each parameter in input.keys().
                Standardized representation refers to the float range of [-1,1].
            pop_name (str): Name of the population.
            append (bool, optional): True if generated samples should be appended to the population.
                False if the new samples should replace the population instead.
                Note that images to which URIs of replaced samples point are not removed from disk -
                only the references to them will be dropped. New samples are free to override the old images on disk.
                Defaults to True.

        Raises:
            AssertionError: When the number of latent codes are below 1.
            AssertionError: When shape of the `latent_codes` is not valid.
            AssertionError: When a attribute is not supported by the controller.
        """
        # Check input
        assert len(latent_codes.shape) > 1
        num_samples = latent_codes.shape[0]
        assert num_samples > 0
        for attr in input.keys():
            assert attr in self._controller.get_attributes()

        pop = Population(pop_name)

        # Parse input
        parsed_input = self._controller.parse_native_input(input)

        # Generate images
        uris, manipulated_latent_codes = self._controller.generate_native(
            latent_codes, parsed_input
        )

        # Add images to population
        pop.add_all(
            manipulated_latent_codes,
            latent_codes,
            uris,
            [1] * len(uris),
            append=append,
            **input,
        )

        # Clear compound metrics
        self._clear_compound_metrics(pop_name)

    def generate_population(
        self, num_samples: int, num_attrs=-1, replace=False
    ) -> None:
        """
        Generates a random population with specified number of samples, or appends them to a
        already existing population.

        The number of attributes manipulated for each sample is specified by number of attributes.
        (Input sampling to be used by the controller is defined by the controller function `sample_random_input()`.)

        An example how the generated population interacts with pre-existing population:

        Let P be a population with 5 number of samples with id 1,2,3,4,5.
        'generate_population(num_samples = 15, replace=False)' will result in a population with the prexisting samples
        1-5 and newly created samples 6-15. If replace would be set to true, id 1-5 would be replaced by new samples.

        Another example:

        Let P be a population with 5 number of samples with id 1,2,3,4,5.
        'generate_population(num_samples = 3, replace=True)' will result in id 1-3 being replaced by new samples and
        4-5 removed from population (note that underlying files are not removed).

        Note that this function clears local and internal compound metrics stored by the compound metric manager for this
        population.

        Args:
            num_samples (int): Aspiring number of samples to generate. The actual number of samples to generate
                depends on `replace`. When `replace` is True, `num_samples` samples will be generated, these samples
                will replace any pre-existing samples (with that ID) in the population associated this factory.
                When `replace` is False, generated samples will be appended to the existing population (even if its empty)
                such that the population grows to up to `num_samples` samples.
            num_attrs (int, optional): Number of attributes to manipulate for each sample. Should not exceed the number of
                attributes of a controller. If set to -1, each sample will get random number of manipulated attributes.
                Defaults to -1.
            replace (bool, optional): True if samples in the population should be replaced, if false samples are appended to
                the population instead. Defaults to False.

        Raises:
            AssertionError: When `num_samples` or `num_attrs` are not valid values.
        """

        # Check input
        assert num_samples > 0
        assert num_attrs >= -1
        assert num_attrs <= len(self._controller.get_attributes())

        pop_samples = self._population.num_samples()

        # Check early stopping criteria
        if (not replace) and pop_samples >= num_samples:
            print(
                f"The population already consists of {pop_samples} ({num_samples} was the goal) samples."
            )
            return

        # Set number of samples to generate
        samples_to_gen = num_samples if replace else num_samples - pop_samples

        # Set sample mode, see sample_random_input() docs for more info
        attr_sample_mode = "random" if num_attrs == -1 else num_attrs
        random_input = self._controller.sample_random_input(
            samples_to_gen, attr_sample_mode
        )

        # Parse input
        parsed_input = self._controller.parse_native_input(random_input)

        # Get latent codes
        latent_codes = self._generator.random_latent_code(samples_to_gen)

        # Generate images
        uris, manipulated_latent_codes = self._controller.generate_native(
            latent_codes, parsed_input
        )

        # Add images to population
        self._population.add_all(
            manipulated_latent_codes,
            latent_codes,
            uris,
            [1] * len(uris),
            append=(not replace),
            **random_input,
        )

        # Clear metrics
        self._clear_compound_metrics(self._population.get_name())
        self._clear_sample_metrics(self._population.get_name())

    def _clear_compound_metrics(self, population_name: str) -> None:
        # Clear compound metrics from manager
        for filter in self._filters:
            cmm = self._cmms[filter.get_name()]
            if population_name == cmm.get_population().get_name():
                cmm.clear_compound_metrics()

        # Clear locally stored metrics from population
        CompoundMetricManager.clear_local_compound_metrics(population_name)
        SampleMetricManager.clear_local_sample_metrics(
            Population.POPULATION_ROOT_DIR / population_name
        )

    def _clear_sample_metrics(self, population_name: str) -> None:
        # Clear compound metrics from manager
        self._smm.clear_sample_metrics()

        # Clear locally stored compound metrics from population
        self._smm.clear_local_sample_metrics(
            self._smm.get_population().POPULATION_ROOT_DIR / population_name
        )

    def evaluate_compound_models(
        self,
        filters: Union[Type[Filter], list[Type[Filter]]] = None,
        compound_metrics: Union[list[str], str] = None,
        recalculate_metrics: bool = False,
        **parameters: Any,
    ) -> None:
        """
        Evaluates a selection or all of the compound models on a selection of compound metrics.
        Result are saved to the managers storage.

        Note that filters should be applied with function call to `apply_filters()` before evaluation!

        Args:
            filters (Type[Filter] | list[Type[Filter]], optional): Selection of filters to use, must be subset
                of the filters which was specified in the construction, see `get_filters()`.
                `filters` can either be a filter, a list of filters or `None` which will result in the use of all available filters.
                Defaults to None.
            compound_metrics (list[str] | str, optional): List of compound metric names to calculate, must be subset
                of the compound metrics which was specified in the construction, see `get_compound_metric_names()`.
                `compound_metrics` can either be a compound metric name, a list of compound metric names or `None` which
                will result in the use of all available compound metrics. Defaults to None.
            recalculate_metrics (bool, optional): True if metrics should be recalculated even if they already exist.
                False means that metrics already calculated will be skipped. Defaults to False.
            **parameters(Any): Parameters passed down to the calculation functions `calc()` and `get()`
                of the compound metric manager. These parameters will then be passed down to the each
                underlying metric of that compound metric manager. See their documentation for more details.

        Raises:
            AssertionError: When arguments `filters` or `compound_metrics` are not valid values.
        """

        # Format input
        filters = self._format_filters_input(filters)
        compound_metrics = self._format_compound_metrics_input(compound_metrics)

        # Check input
        self._check_filters(filters)
        self._check_compound_metrics(compound_metrics)

        # Evaluate each compound model
        for filter in filters:
            self._evaluate_compound_model(
                filter, compound_metrics, recalculate_metrics, **parameters
            )

    def _evaluate_compound_model(
        self,
        filter: Type[Filter],
        compound_metrics: list[str],
        recalculate_metrics: bool = False,
        **parameters: Any,
    ) -> None:
        print(
            "# ~ Evaluating metrics for compound model: "
            + f"{self._compound_model_name_prefix}_{filter.get_name()}"
        )

        # Calc all
        if recalculate_metrics:
            self._cmms[filter.get_name()].calc(
                compound_metrics,
                **parameters,
            )

        # Calc missing
        else:
            print(f"# ~ Trying to fetch metrics")
            self._cmms[filter.get_name()].get(
                compound_metrics,
                calc_if_missing=True,
                **parameters,
            )

        print("# ~ Metric Evaluation complete")

    def get_filters(self) -> list[Type[Filter]]:
        """
        Returns the filters.

        Returns:
            list[Type[Filter]]: The filters.
        """
        return self._filters

    def get_compound_metric_names(self) -> list[str]:
        """
        Returns a list of all compound metric names.

        Returns:
            list[str]: Name of the compound metrics.
        """
        return self._cmms[self._filters[0].get_name()].get_metric_names()

    def get_name(self) -> str:
        """
        Returns the name of the compond metric factory.

        Returns:
            str: The name.
        """
        return self._name

    def get_population(self) -> Population:
        """
        Returns the population of the compound model factory.

        Returns:
            Population: The population.
        """
        return self._population

    def get_controller(self) -> Controller:
        """
        Returns the controller of this compound model factory.

        Returns:
            Controller: The controller of this compound model factory.
        """
        return self._controller

    def get_compound_metric_managers(self) -> list[CompoundMetricManager]:
        """
        Returns the compound metric managers of this compound model factory.

        There should be precisely one compound metric manager for each filter
        supplied on construction, i.e.,
        `len(get_filters()) == len(get_compound_metric_managers())`.

        Returns:
            list[CompoundMetricManager]: The compound metric managers.
        """
        return [cmm for cmm in self._cmms.values()]

    def get_compound_model_names(self) -> list[str]:
        """
        Returns a list of all compound model names.

        Returns:
            list[str]: Name of the compound models.
        """
        return [
            f"{self._compound_model_name_prefix}_{filter.get_name()}"
            for filter in self._filters
        ]

    def _check_filters(self, filters: list[Type[Filter]]) -> None:
        for filter in filters:
            assert filter in self._filters

    def _check_compound_metrics(self, compound_metrics: list[str]) -> None:
        for compound_metric in compound_metrics:
            assert compound_metric in self.get_compound_metric_names()

    def _format_filters_input(
        self, filters: Union[Type[Filter], list[Type[Filter]]] = None
    ) -> list[Type[Filter]]:
        if type(filters) is list:
            return filters
        elif filters is None:
            return self._filters
        else:
            return [filters]

    def _format_compound_metrics_input(
        self, compound_metrics: Union[str, list[str]] = None
    ) -> list[Type[Filter]]:
        if type(compound_metrics) is list:
            return compound_metrics
        elif compound_metrics is None:
            return self.get_compound_metric_names()
        else:
            return [compound_metrics]

class CompoundModelFactoryContext:
    """
    Used to create a larger context, such that
    a factory can be aware of compound models from other factories.
    """

    def __init__(self, factories: list[CompoundModelFactory]):
        """
        Constructs a new CompoundModelContext instance.

        Args:
            factories (list[CompoundModelFactory]): All compound factories in the context.

        """
        # Init properties
        self._populations = dict()
        self._filter_bits = dict()
        self._cmms = dict()
        self._factory_names = [factory.get_name() for factory in factories]

        # Extract properties
        for name, factory in zip(self._factory_names, factories):
            filters = factory.get_filters()
            self._populations[name] = factory.get_population()
            self._filter_bits[name] = [
                (filter.get_bit(), filter.get_name()) for filter in filters
            ]
            self._cmms[name] = factory.get_compound_metric_managers()

        # Give all factories the context
        for factory in factories:
            factory.give_context(self)

    def __eq__(self, other):
        return (
            self._factory_names == other.factory_names
            and self._populations == other.populations
            and self._filter_bits == other.filter_bits
            and self._cmms == other.cmms
        )

    @property
    def compound_metric_managers(self) -> list[CompoundMetricManager]:
        """
        Get the compound metric managers of each factory in the context.
        """
        return self._cmms

    @property
    def factory_names(self) -> list[str]:
        """
        Get the  factory names of the context.
        """
        return self._factory_names

    @property
    def populations(self) -> dict[str, Population]:
        """
        Get the  populations of the context.
        """
        return self._populations

    @property
    def filter_bits(self) -> dict[str, list[int]]:
        """
        Get the filter bits of the context.
        """
        return self._filter_bits


import matplotlib.pyplot as plt
import os
import numpy as np

def plot_histogram(hist):
    levels = [i for i in range(len(hist))]
    fig, ax = plt.subplots()

    ax.fill_between(levels, hist, step="pre", alpha=0.4)
    ax.plot(levels, hist, drawstyle="steps")

    return fig, ax

def plot_points(ax: plt.Axes, points: np.ndarray):
    x = points[:,0]
    y = points[:,1]

    ax.clear()
    ax.plot(x, y)

def getOutputFilePath(filename):
    if not os.path.exists("./out"):
        os.mkdir("./out")
    return f"./out/{filename}"


# !pip install PyMySQL
from sqlalchemy import create_engine

import yfinance as yf
import pandas as pd

def getDataM(listado, start='2000-01-01', interval='1d', end=None):
    data = yf.download(listado, start=start, end=end, interval=interval, auto_adjust=True)
    return data.swaplevel(i=1, j=0, axis=1)

#Seteo el USER : PASS @ HOST / BBDD_NAME
sql_engine = create_engine('mysql+pymysql://root:@35.193.28.44/screener_v1')
sql_conn = sql_engine.connect()

#sp500_wiki = pd.read_html('https://en.wikipedia.org/wiki/List_of_S%26P_500_companies')[0]
#sp500_tickers = list(sp500_wiki.Symbol)
sp500_tickers = ['AMZN' ,'BABA','TSLA' ,'KO' ]
sp500_tickers = [t for t in sp500_tickers if t not in ['BRK.B' ,'BF.B']]

pd.DataFrame(sp500_tickers).to_sql(con=sql_conn, name='sp500tickers', if_exists='replace')

df_sp500 = getDataM(sp500_tickers, start='2012-01-01')

def getDataByYfinance(sp500_tickers):
	tablas = []
	for ticker in sp500_tickers:
        	tabla = df_sp500[ticker].copy()
        	tabla['ticker'] = ticker
        	tabla['variacion'] = tabla.Close.pct_change() *100
        	tabla['volatilidad'] = tabla.variacion.rolling(250).std() * 250**0.5
        	tabla['vol_mln'] = tabla.Volume * tabla.Close / 1000000
        	tabla = tabla.dropna().round(2)
        	tablas.append(tabla)

	return  pd.concat(tablas)

tabla_full = getDataByYfinance(sp500_tickers)
print(tabla_full)

tabla_full.to_excel('df_sp500.xlsx')

tabla_full.to_sql(con=sql_conn, name='sp500', if_exists='replace',chunksize=990500)

print("bye")