import random
from collections import deque, namedtuple
import torch
from ._tree import SumSegmentTree, MinSegmentTree

class _ExpReplay:
    def __init__(self, capacity, batch_size, init_size):
        assert batch_size is not None and batch_size >= 1
        if type(init_size) != int or init_size < batch_size: init_size = batch_size
        self.capacity = capacity
        self.batch_size = batch_size
        self.init_size = init_size
        if self.capacity is None:
            print('MEMORY WARNING: '
                  'capacity of exp replay not specified (infinity)', flush=True)
        elif self.capacity <= 0:
            raise ValueError('invalid capacity of replay queue specified')

    def __len__(self):
        raise NotImplementedError

    def push(self, state):
        raise NotImplementedError

    def sample(self):
        raise NotImplementedError

    def clear(self):
        raise NotImplementedError

class VanillaReplay(_ExpReplay):
    def __init__(self, capacity=None, batch_size=32, init_size=None):
        super(VanillaReplay, self).__init__(capacity, batch_size, init_size)
        self.obj_type = namedtuple('Exp', ('curr_state', 'action', 'next_state', 'reward'))

        self.memory = deque(maxlen=self.capacity)

    def __len__(self):
        return len(self.memory)

    def clear(self):
        self.memory.clear()

    def push(self, *obj):
        processed_obj = self.form_obj(*obj)
        self.memory.append(processed_obj)

    def sample(self):
        return random.sample(self.memory, self.batch_size)

    def form_obj(self, *args):
        return self.obj_type(*args)

class LazyReplay(_ExpReplay):
    """
    this replay splits the stacked frames and makes sure each frame is only saved once, so it is
    meant to be memory-efficient. However, the sampling speed will be compromised since that
    process is kind of complicated
    """
    def __init__(self, capacity=None, batch_size=32, init_size=None, frames_stack=None):
        super(LazyReplay, self).__init__(capacity, batch_size, init_size)
        assert frames_stack is not None
        self.obj_type = namedtuple('Exp', ('curr_state', 'action', 'next_state', 'reward'))
        self.frames_stack = frames_stack
        self._valid_idx = deque([])
        self._valid_flag = [False] * self.capacity
        self._done = True
        self.memory = [None] * self.capacity
        self._idx = 0

    def clear(self):
        self.memory = [None] * self.capacity
        self._idx = 0
        self._valid_idx.clear()

    def __len__(self):
        return len(self._valid_idx)

    def push(self, *obj):
        """
        preprocess obj sequence and save it to the memory, note that preprocessing varies due to
        various obj format
        """
        curr_state, action, next_state, reward = obj
        if self._done:
            initial_frames = curr_state.squeeze().split(1)
            for f in initial_frames:
                if self.memory[self._idx] is not None and self._valid_flag[self._idx]:
                    self._valid_idx.popleft()
                    self._valid_flag[self._idx] = False
                self.memory[self._idx] = (f, None, None)
                self._idx = (self._idx + 1) % self.capacity
            self._done = False
        if next_state is None:
            self._done = True
            next_frame = None
        else:
            next_frame = next_state.squeeze().split(1)[-1]
        if self.memory[self._idx] is not None and self._valid_flag[self._idx]:
            self._valid_idx.popleft()
            self._valid_flag[self._idx] = False
        self.memory[self._idx] = (next_frame, action, reward)
        self._idx = (self._idx + 1) % self.capacity
        self._valid_flag[(self._idx - 5) % self.capacity] = True
        self._valid_idx.append((self._idx - 5) % self.capacity)

    def sample(self):
        ret_list = []
        frames_buffer = deque([], maxlen=self.frames_stack)
        indices = random.sample(self._valid_idx, self.batch_size)
        for idx in indices:
            for shift in range(self.frames_stack):
                frames_buffer.append(self.memory[(idx + shift) % self.capacity][0])
            curr_state = torch.cat(tuple(frames_buffer)).unsqueeze(0).clone()
            next_frame, action, reward = self.memory[(idx + self.frames_stack) % self.capacity]
            if next_frame is None:
                next_state = None
            else:
                frames_buffer.append(next_frame)
                next_state = torch.cat(tuple(frames_buffer)).unsqueeze(0).clone()
            ret_list.append((curr_state, action, next_state, reward))
        return tuple(ret_list)

    def form_obj(self, *args):
        return self.obj_type(*args)

class PrioritizedReplay(_ExpReplay):
    def __init__(
        self,
        capacity=None,
        batch_size=32,
        init_size=None,
        alpha=1,
        beta_func=lambda: 0,
        eps=1e-6
    ):
        super(PrioritizedReplay, self).__init__(capacity, batch_size, init_size)
        assert alpha >= 0 and capacity is not None
        self._alpha = alpha
        self.in_obj_type = namedtuple('Exp', ('curr_state', 'action', 'next_state', 'reward'))
        self.out_obj_type = namedtuple('PriorExp', (
            'curr_state', 'action', 'next_state', 'reward', 'weight', 'index'))
        self.memory = []
        self._idx = 0
        self._sum_prior = SumSegmentTree(self.capacity)
        self._min_prior = MinSegmentTree(self.capacity)
        self._max_prior = 1
        self.get_beta = beta_func
        self.eps = eps

    def clear(self):
        self.memory = []
        self._idx = 0
        self._sum_prior = SumSegmentTree(self.capacity)
        self._min_prior = MinSegmentTree(self.capacity)

    def __len__(self):
        return len(self.memory)

    def push(self, *obj):
        processed_obj = self._form_input_obj(*obj)
        if len(self) < self.capacity:
            self.memory.append(processed_obj)
        else:
            self.memory[self._idx] = processed_obj

        priority = self._max_prior ** self._alpha
        self._min_prior[self._idx] = self._sum_prior[self._idx] = priority
        self._idx = (self._idx + 1) % self.capacity

    def _sample_indices(self):
        indices = []
        for _ in range(self.batch_size):
            # sum(a, b): a is inclusive whereas b is exclusive
            mass = random.random() * self._sum_prior.sum(0, len(self))
            idx = self._sum_prior.find_prefixsum_idx(mass)
            indices.append(idx)
        return indices

    def sample(self):
        beta = self.get_beta()
        indices = self._sample_indices()
        weights = []
        ret_list = []
        p_min = self._min_prior.min() / self._sum_prior.sum()
        max_weight = (p_min * len(self)) ** (-beta)

        for idx in indices:
            p_sample = self._sum_prior[idx] / self._sum_prior.sum()
            weight = (p_sample * len(self)) ** (-beta) / max_weight
            ret_list.append(tuple(list(self.memory[idx]) + [weight, idx]))

        return ret_list

    def update_priorities(self, idxes, priorities):
        assert len(idxes) == len(priorities)
        for idx, priority in zip(idxes, priorities):
            assert priority > 0 and 0 <= idx < len(self.memory)
            self._sum_prior[idx] = priority ** self._alpha
            self._min_prior[idx] = priority ** self._alpha
            self._max_prior = max(self._max_prior, priority)

    def _form_input_obj(self, *args):
        return self.in_obj_type(*args)

    def form_obj(self, *args):
        return self.out_obj_type(*args)


# Copyright (c) 2020 Graphcore Ltd. All rights reserved.

import numpy as np
from tqdm import tqdm
from functools import partial
import tensorflow.compat.v1 as tf
from tensorflow.python.ipu import ipu_compiler, scopes, loops, ipu_infeed_queue, ipu_outfeed_queue
from tensorflow.python.ipu import dataset_benchmark
from tensorflow.python.ipu import rand_ops
from tensorflow.python.ipu.config import IPUConfig
import argparse
tf.disable_eager_execution()
tf.disable_v2_behavior()

def dense_layer(hiddenSize, input, scope_name):
    with tf.variable_scope(scope_name, reuse=tf.AUTO_REUSE, use_resource=True):
        w = tf.get_variable("weight", shape=[input.shape[-1], hiddenSize],
                            initializer=tf.glorot_uniform_initializer())
        b = tf.get_variable("bias", shape=[hiddenSize],
                            initializer=tf.zeros_initializer())
        return tf.nn.relu_layer(input, w, b)

def model(lr, outqueue, training: bool, inputs, labels):
    h1Size = 320
    h2Size = 10
    droprate = 0.2

    relu1 = dense_layer(h1Size, inputs, "d1")
    # Use the IPU optimised version of dropout:
    if training:
        drop1 = rand_ops.dropout(relu1, rate=droprate)
    else:
        drop1 = relu1
    relu2 = dense_layer(h2Size, drop1, "d2")

    with tf.variable_scope("metrics", reuse=tf.AUTO_REUSE, use_resource=True):
        acc, acc_op = tf.metrics.accuracy(labels=labels,
                                          predictions=tf.argmax(
                                              relu2, axis=1, output_type=tf.dtypes.int32),
                                          name="accuracy")
        loss = tf.nn.sparse_softmax_cross_entropy_with_logits(
            labels=labels, logits=relu2)

    if training:
        with tf.variable_scope("training", reuse=tf.AUTO_REUSE, use_resource=True):
            optimiser = tf.train.MomentumOptimizer(
                learning_rate=lr, momentum=0.0001, use_nesterov=True, name='optimise')
            train_op = optimiser.minimize(loss)
            with tf.control_dependencies([train_op, acc_op]):
                mean_loss = tf.reduce_mean(loss, name='train_loss')
    else:
        with tf.control_dependencies([acc_op]):
            mean_loss = tf.reduce_mean(loss, name='test_loss')

    return outqueue.enqueue({'mean_loss': mean_loss, 'acc': acc})

def scheduler(epoch):
    if epoch < 1:
        return 0.02
    if epoch < 3:
        return 0.01
    else:
        return 0.001

def loop_builder(iterations, builder_func, infeed):
    return loops.repeat(iterations, builder_func, [], infeed)

def run_model(opts):
    # Use Keras to get the dataset:
    mnist = tf.keras.datasets.mnist
    (x_train, y_train), (x_test, y_test) = mnist.load_data()
    x_train, x_test = x_train / 255.0, x_test / 255.0

    # Sizes/shapes for the dataset:
    image_shape = x_train.shape[1:]
    num_pixels = image_shape[0] * image_shape[1]
    batch_size = 16
    batch_shape = [batch_size, num_pixels]
    num_train = y_train.shape[0]
    num_test = y_test.shape[0]
    data_shape = [None, num_pixels]

    # Flatten the images and cast the labels:
    x_train_flat = x_train.astype(np.float32).reshape(-1, num_pixels)
    x_test_flat = x_test.astype(np.float32).reshape(-1, num_pixels)
    y_train = y_train.astype(np.int32)
    y_test = y_test.astype(np.int32)

    # Decide how to split epochs into loops up front:
    epochs = 5
    ipu_steps_per_epoch = 15
    batches_per_epoch = num_train // batch_size
    train_batches = (num_train * epochs) // batch_size
    test_batches = num_test // batch_size
    batches_per_step = batches_per_epoch // ipu_steps_per_epoch
    if not batches_per_epoch % ipu_steps_per_epoch == 0:
        raise ValueError(f"IPU steps per epoch {ipu_steps_per_epoch} must divide batches per epoch {batches_per_epoch}.")

    # Put placeholders on the CPU host:
    with tf.device("cpu"):
        place_x = tf.placeholder(dtype=tf.float32, shape=data_shape, name="input")
        place_y = tf.placeholder(dtype=tf.int32, shape=[None], name="label")
        lr_placeholder = tf.placeholder(tf.float32, shape=[])

    # Create dataset and IPU feeds:
    dataset = tf.data.Dataset.from_tensor_slices((place_x, place_y))
    dataset = dataset.cache().repeat().batch(batch_size, drop_remainder=True)
    infeed_train_queue = ipu_infeed_queue.IPUInfeedQueue(dataset)
    outfeed_train_queue = ipu_outfeed_queue.IPUOutfeedQueue()
    infeed_test_queue = ipu_infeed_queue.IPUInfeedQueue(dataset)
    outfeed_test_queue = ipu_outfeed_queue.IPUOutfeedQueue()

    # Use function binding to create all the builder functions that are neeeded:
    bound_train_model = partial(model, lr_placeholder, outfeed_train_queue, True)
    bound_train_loop = partial(
        loop_builder, batches_per_step, bound_train_model, infeed_train_queue)
    bound_test_model = partial(model, lr_placeholder, outfeed_test_queue, False)
    bound_test_loop = partial(loop_builder, test_batches,
                              bound_test_model, infeed_test_queue)

    # Use the bound builder functions to place the model on the IPU:
    with scopes.ipu_scope("/device:IPU:0"):
        train_loop = ipu_compiler.compile(bound_train_loop, inputs=[])
        test_loop = ipu_compiler.compile(bound_test_loop, inputs=[])

    # Initialisers should go on the CPU:
    with tf.device("cpu"):
        metrics_vars = tf.get_collection(
            tf.GraphKeys.LOCAL_VARIABLES, scope="metrics")
        metrics_initializer = tf.variables_initializer(var_list=metrics_vars)
        saver = tf.train.Saver()

    # Setup and acquire an IPU device:
    config = IPUConfig()
    config.auto_select_ipus = 1
    config.configure_ipu_system()

    # These allow us to retrieve the results of IPU feeds:
    dequeue_train_outfeed = outfeed_train_queue.dequeue()
    dequeue_test_outfeed = outfeed_test_queue.dequeue()

    # Create a benchmark program for the infeed to determine maximum achievable throughput:
    infeed_perf = dataset_benchmark.infeed_benchmark(
        infeed_train_queue, epochs, num_train, True)

    print(f"\nImage shape: {image_shape} Training examples: {num_train} Test examples: {num_test}")
    print(f"Epochs: {epochs} Batch-size: {batch_size} Steps-per-epoch: {ipu_steps_per_epoch} Batches-per-step: {batches_per_step}")

    # Run the model:
    with tf.Session() as sess:
        print(f"Benchmarking the infeed...")
        sess.run(infeed_perf, feed_dict={place_x: x_train_flat, place_y: y_train})

        sess.run(tf.global_variables_initializer())
        sess.run(infeed_train_queue.initializer, feed_dict={
                 place_x: x_train_flat, place_y: y_train})

        if opts.test_mode in ["all", "training"]:
            print(f"Training...")
            progress = tqdm(
                range(epochs), bar_format='{desc} Epoch: {n_fmt}/{total_fmt} {bar}')
            for e in progress:

                sess.run(metrics_initializer)
                for i in range(ipu_steps_per_epoch):
                    sess.run(train_loop, feed_dict={lr_placeholder: scheduler(e)})
                    result = sess.run(dequeue_train_outfeed)
                    if len(result['mean_loss'] != 0) and len(result['acc'] != 0):
                        progress.set_description(f"Loss {result['mean_loss'][0]:.5f} Accuracy {result['acc'][0]:.5f}")

            print(f"Saving...")
            saver.save(sess, "model")

        if opts.test_mode in ["all", "tests"]:
            print(f"Testing...")
            sess.run(metrics_initializer)
            sess.run(infeed_test_queue.initializer, feed_dict={
                     place_x: x_test_flat, place_y: y_test})
            sess.run(test_loop)
            result = sess.run(dequeue_test_outfeed)

            test_loss = np.mean(result['mean_loss'])
            test_acc = np.mean(result['acc'])
            print(f"Test loss: {test_loss:.8f} Test accuracy: {test_acc:.8f}")

if __name__ == '__main__':
    parser = argparse.ArgumentParser(
        description='Train and Test the simple Tensorflow model with MNIST dataset',
        formatter_class=argparse.ArgumentDefaultsHelpFormatter)

    parser.add_argument(
        "--test-mode",
        choices=['training', 'tests', 'all'],
        default="all",
        help="Use this flag to run the model in either 'training' or 'tests' mode or both ('all')")

    opts = parser.parse_args()
    run_model(opts)


from os import sched_getaffinity
import discord
import time
import asyncio
from discord import file
from discord import channel
from discord import message
from discord import embeds
from discord import utils
from discord.ext import commands
from discord.gateway import DiscordWebSocket
from User import User
from utils.utils import *
from youtubesearchpython import VideosSearch
from Games.tiny_games import *
from Games.Hangman import Hangman
from Games.connect4.Game import Game
from Bump import Bump
from shared import *

games = {}
colors = {}

intents = discord.Intents(messages=True, guilds=True, reactions=True, members=True, presences=True)
bot = commands.Bot(command_prefix=".", intents=intents)

# game = Game()
game = Hangman()
bump = Bump(bot, 841701695628247080)
board_message_history = []

@bot.event
async def on_ready():
    print(f"Logged in as {bot.user.name}")
    print("Ready!")
    print("------")

    while bump.active == True:
        command = await bump.bump_()
        # await self.clean(command)
@bot.event
async def on_member_join(member):
    channel = discord.utils.get(member.guild.text_channels, name="gelen-giden")
    User.add_new(member, 100, 3)
    print(User.coin(member))
    print(User.warn_ctr(member))
    await channel.send(f"Hos geldin {member.mention} c:")
    print(f"Hos geldin {member.mention} c:")

# @bot.event
# async def on_profanity(message, word):
#     warn_ctr = User.warn_ctr(message.author)
#     User.update_warn_ctr(message.author, warn_ctr + 1)
#     channel = message.channel
#     embed = discord.Embed(title="WARNING!",
#     description=f"{message.author.name} just said ||{word}||\n You have been warned **{warn_ctr}** times.", color=discord.Color.blurple())

#     if User.warn_ctr(message.author) >= 3:
#         duration = 10
#         unit = "s"
#         roleobject = discord.utils.get(message.guild.roles, id=863562272668385311)
#         await channel.send(f":white_check_mark: Muted {message.author} for {duration}{unit}")
#         await message.author.add_roles(roleobject)
#         # await channel.set_permissions(message.author, send_messages=False)
#         if unit == "s":
#             wait = 1 * duration
#             await asyncio.sleep(wait)
#         elif unit == "m":
#             wait = 60 * duration
#             await asyncio.sleep(wait)

#         await message.author.remove_roles(roleobject)
#         # await channel.set_permissions(message.author, send_messages=True)
#         await channel.send(f":white_check_mark: {message.author} was unmuted")

#     await channel.send(embed=embed)

# @bot.event
# async def on_member_update(before, after):
#     # text_channel_list = []
#     # for guild in bot.guilds:
#     #     for channel in guild.text_channels:
#     #         text_channel_list.append(channel)
#     # last_msg = None
#     # for channel in text_channel_list:
#     #     aux = await channel.history(limit=1).find(lambda m: m.author.id == after.user.id)
#     #     if aux.created_at > last_msg.created_at:
#     #         last_msg = aux

#     # channel = discord.utils.get(after.guild.text_channels, name="bot")
#     text_channel_list = []
#     for guild in bot.guilds:
#         for channel in guild.text_channels:
#             text_channel_list.append(channel)
#     for channel in text_channel_list:
#         for role in after.roles:
#             if role.name == "Muted":
#                 await channel.set_permissions(after, send_messages=False)
#                 return
#         await channel.set_permissions(after, send_messages=True)

@bot.event
async def on_message(message):
#     # last_message = await message.author.history(limit=1).flatten()[0]
#     # history = await message.author.history(limit=1)
#     # history_listesi = await history.flatten()
#     # last_message = history_listesi[0]
#     # print(last_message)
#     # print(dir(message.author.history))
#     # print(type(message.created_at))
#     # print(message.author.history)

#     for i in badwords:
#         if i in message.content:
#             await message.delete()
#             await message.channel.send(f"{message.author.mention} Don't use that word!")
#             bot.dispatch("profanity", message, i)
#             return # So that it does not try to delete the message again, which will cause an error.

#         await bot.process_commands(message)

    if message.member.hasPermission("ADMINISTRATOR"):
        if message.content == "!pause":
            await bump.pause_()
            await bump.channel.send(message, "Bot is paused :sleeping:")

        elif message.content == "!continue":
            await bump.continue_()
            await bump.channel.send(
                message,
                f"Bump is activated, next bump in {bump.diff} seconds :hourglass_flowing_sand:",
            )

# @bot.command(pass_context=True)
# async def ping(ctx):
#     await ctx.send(f'{round(bot.latency * 1000)} ms')
@bot.command(pass_context=True)
async def ping(ctx):
    time_1 = time.perf_counter()
    await ctx.trigger_typing()
    time_2 = time.perf_counter()
    ping = round((time_2-time_1) * 1000)
    await ctx.send(f"{ping} ms")

@bot.command()
@commands.has_permissions(kick_members=True)
async def mute(ctx, user : discord.Member, duration = 0,*, unit = None):
    roleobject = discord.utils.get(ctx.message.guild.roles, id=730016083871793163)
    await ctx.send(f":white_check_mark: Muted {user} for {duration}{unit}")
    await user.add_roles(roleobject)
    if unit == "s":
        wait = 1 * duration
        await asyncio.sleep(wait)
    elif unit == "m":
        wait = 60 * duration
        await asyncio.sleep(wait)
    await user.remove_roles(roleobject)
    await ctx.send(f":white_check_mark: {user} was unmuted")

@bot.command()
@commands.has_permissions(administrator=True)
# @commands.has_role(840869771356012555)
async def clear(ctx, amount:int=1):
    await ctx.channel.purge(limit = amount + 1)

@clear.error
async def clear_error(ctx, error):
    if isinstance(error, commands.MissingPermissions):
        await ctx.send("You do not have the permission to do that!")

@bot.command(aliases=["copy"])
@commands.has_permissions(administrator=True)
async def clone_channel(ctx, amount=1):
    print(ctx.channel)
    for i in range(amount):
        await ctx.channel.copy()

@bot.command(aliases=["oyun"])
async def deneme(ctx, *args):
    print(dir(ctx.channel))
    if "roll" in args:
        await ctx.send(game.roll_dice())
    else:
        await ctx.send("yee")

@bot.command()
async def av(ctx, *, avamember: discord.Member=None):
    user_avatar_url = avamember.avatar_url

    await ctx.send(user_avatar_url)

@bot.command(aliases=["yt"])
async def ytplay(ctx, *args:str):

    base = "https://www.youtube.com/results?search_query="
    query = args
    search_url = base + '+'.join(query)
    print(search_url)
    videos_search = VideosSearch(search_url, limit=5)
    print(videos_search.result())
    results = videos_search.result()["result"][0]
    url_of_music = results["link"]
    view_count = results["viewCount"]["short"]
    published_time = results["publishedTime"]
    # print(url_of_music)

    await ctx.send(f":globe_with_meridians: {view_count} :cyclone: {published_time}\n{url_of_music}")

@bot.command(aliases=["hm"])
async def hangman(ctx, *args:str):
    # elif message.content.startswith('!hangman'):
    game_message = ""
    author = ctx.author
    if args[0] == 'start':
        # User.add_new(author)
        print(author.id)
        User.coin(author)
        game.start_game(author)
        game_message = 'A word has been randomly selected (all lowercase). \nGuess letters by using `.hangman x` (x is the guessed letter). \n'
    else:
        # print(dir(ctx.message))
        User.coin(author)
        print(ctx.message.content)
        game.guess(ctx.message.content)
    print(game.remaining_guesses)
    await ctx.channel.send(game_message, embed=game.create_embed(hangman_body[game.remaining_guesses - 1]))

# Connect 4
@bot.command(aliases=["c4"])
async def connect4(ctx, *args:str):

    # print(dir(ctx.message.content))
    message = ctx.message
    print(message.content)
    if args[0] == "color":
        ind = message.content.find("#")
        print("ind: ", ind)
        if ind > -1:
            color = message.content[ind:]
            colors[message.author] = color
            await ctx.channel.send("Color has set.")
        else:
            await ctx.channel.send("Colors must be set as hex!")
        print(colors[ctx.author])

    elif args[0] == "resign":
        try:
            game_info = games[message.author.id]
        except KeyError:
            await ctx.channel.send("You are not currently in a game " + message.author.mention)
            return

        opp_id = game_info['opponent'].id

        del games[opp_id]
        del games[message.author.id]

    elif args[0] == "start":
        player = message.author
        opponent = message.mentions[0]

        if player in games or opponent in games:
            await ctx.channel.send("One of you is already in another game!")
            return

        game = Game()
        games[player.id] = {"game": game, "opponent": opponent, "team": 0}
        games[opponent.id] = {"game": game, "opponent": player, "team": 1}
        # await ctx.channel.send("`" + str(games[message.author]['game']) + "`")

        file_name = str(player.id) + ".png"

        game.generateImageBoard().save(file_name, "PNG")

        new_board = await ctx.send("Your turn " + player.mention, file=discord.File(file_name))
        for r in keycap_digits:
            await new_board.add_reaction(r)

@bot.event
async def on_reaction_add(reaction, user):
    if user.id == bot.user.id: return

    message = reaction.message
    channel = message.channel
    last_bot_msg = await channel.history(limit=1).find(lambda m: m.author.id == bot.user.id)
    if not last_bot_msg: return
    print(last_bot_msg.content)

    if message.id != last_bot_msg.id: return

    try:
        game_info = games[user.id]
    except KeyError:
        await channel.send("You are not currently in a game " + user.mention)
        await reaction.remove(user)
        return

    if game_info['game'].turn != game_info['team']:
        await reaction.remove(user)
        return
    else:
        if reaction.emoji in keycap_digits:
            column = str(keycap_digits.index(reaction.emoji))
        else:
            return

        if column.isdigit() and int(column) >= 0 and int(column) < 7:
            winner = game_info['game'].move(int(column))
            if winner == -1:
                await channel.send("You must give a valid column " + user.mention)
                return

            elif winner:
                file_name = str(user.id) + ".png"

                if game_info["team"] == 0:
                    pc = colors.get(user, "red")
                    oc = colors.get(game_info['opponent'], "black")
                    game_info['game'].generateImageBoard(pc, oc).save(file_name, "PNG")
                else:
                    pc = colors.get(user, "black")
                    oc = colors.get(game_info['opponent'], "red")
                    game_info['game'].generateImageBoard(oc, pc).save(file_name, "PNG")

                new_board = await channel.send(file=discord.File(file_name), content=user.mention + " won!")
                for r in keycap_digits:
                    await new_board.add_reaction(r)
                opp_id = game_info['opponent'].id

                del games[opp_id]
                del games[user.id]

            else:
                file_name = str(user.id) + ".png"

                if game_info["team"] == 0:
                    pc = colors.get(user, "red")
                    oc = colors.get(game_info['opponent'], "black")
                    game_info['game'].generateImageBoard(pc, oc).save(file_name, "PNG")
                else:
                    pc = colors.get(user, "black")
                    oc = colors.get(game_info['opponent'], "red")
                    game_info['game'].generateImageBoard(oc, pc).save(file_name, "PNG")

                new_board = await channel.send("Your turn " + game_info["opponent"].mention, file=discord.File(file_name))
                for r in keycap_digits:
                    await new_board.add_reaction(r)

        else:
            await channel.send("You must give a valid column " + game_info["opponent"].mention)

if __name__ == "__main__":
    bot.run(TOKEN)


# -*- coding=utf-8 -*-

import calendar
import logging
import time
import datetime

from functools import wraps

from utils.const_def import *
from utils.const_err import *
from utils.utils_except import BusinessException

logger = logging.getLogger(__name__)

def record_time(func):
    """
        装饰器： 记录函数运行时间
    """
    @wraps(func)
    def returned_wrapper(*args, **kwargs):
        start_time = datetime.datetime.now()
        return_value = func(*args, **kwargs)
        end_time = datetime.datetime.now()
        logger.debug('runtime %s : %s' % (func.__name__, str(end_time - start_time)))
        return return_value
    return returned_wrapper

def is_today(date_str):
    """
    检查传入的日期是否为当天
    :param date_str:
    :return:
    """
    today_str = today(is_str=True)
    if date_str == today_str:
        return TRUE_STR

    return FALSE_STR

def today(FORMAT_DATE='%Y-%m-%d', is_str=True):
    """
        返回今天的时间
    """
    if is_str:
        return time.strftime(FORMAT_DATE, time.localtime(time.time()))
    else:
        return datetime.datetime.strptime(
            time.strftime(FORMAT_DATE, time.localtime(time.time())),
            FORMAT_DATE)

def tomorrow(FORMAT_DATE='%Y-%m-%d', is_str=True):
    """
        返回明天的时间
    """
    if is_str:
        return time.strftime(FORMAT_DATE, time.localtime(time.time() + 24 * 60 * 60))
    else:
        return datetime.datetime.strptime(
            time.strftime(FORMAT_DATE, time.localtime(time.time() + 24 * 60 * 60)),
            FORMAT_DATE)

def yesterday(FORMAT_DATE='%Y-%m-%d', is_str=True):
    """
        返回昨天的时间
    """
    if is_str:
        return time.strftime(FORMAT_DATE, time.localtime(time.time() - 24 * 60 * 60))
    else:
        return datetime.datetime.strptime(
            time.strftime(FORMAT_DATE, time.localtime(time.time() - 24 * 60 * 60)),
            FORMAT_DATE)

def now(FORMAT_DATETIME='%Y-%m-%d %H:%M:%S'):
    """
        获取当前时间：2013-09-10 11:22:11这样的时间年月日时分秒字符串
    """
    return time.strftime(FORMAT_DATETIME, time.localtime(time.time()))

def now_datetime():
    return datetime.datetime.now()

def datetime2str(datetime_para, format=TIME.DATE_STD):
    """
        时间转字符串
    """
    if not datetime_para:
        return ''
    return datetime_para.strftime(format)

def str2datetime(datetime_str, format=TIME.DATE_STD):
    """
        字符串转时间
    """
    try:
        day = datetime.datetime.strptime(datetime_str, format)
    except:
        day = None
    return day

def str2dt_DATE_STD(time_str, **kwargs):
    return str2datetime(time_str, format=TIME.DATE_STD)

def str2dt_DAY_STD(time_str, **kwargs):
    return str2datetime(time_str, format=TIME.DAY_STD)

def datetime2timestamp(dt, without_point=False):
    timeStamp = int(time.mktime(dt.timetuple()))
    timeStamp = '%.3f'% (float(str(timeStamp) + str("%06d" % dt.microsecond)) / 1000000)
    if without_point:
        timeStamp = timeStamp.replace('.', '')
    return timeStamp

def timestamp2datetime(ts):
    return datetime.datetime.fromtimestamp(ts)

def get_day_cycle(time_in, offset=0):
    """
    获取传入日期当天的开始结束时间
    :param time_in: 时间
    :param offset: 偏移量
    :return: 传入日期当天的开始结束时间
    """
    result = dict()
    date = time_in.date() + datetime.timedelta(days=offset)  # 日期不带时分秒
    result['startdate'] = datetime.datetime.combine(date, datetime.time.min)
    result['enddate'] = datetime.datetime.combine(date, datetime.time.max)
    result['enddate_pro'] = datetime.datetime.combine(date + datetime.timedelta(days=1), datetime.time.min)
    return result

def get_week_cycle(time_in, offset=0):
    """
        获取本周的开始结束时间（周的概念为周一00:00:00到周日23:59:59）
        inparam: 时间，偏移量（周）  例如offset=0时，查询本周开始结束时间 ，为-1是查询上周开始结束时间
        outparam: {'startdate':startdate, 'enddate':enddate}
    """
    result = dict()
    week = time_in.weekday()  # 当前周数0-6 表示周一到周日
    date = time_in.date()  # 日期不带时分秒
    startdate = date + datetime.timedelta(days=(0-week+7*offset))
    enddate = date + datetime.timedelta(days=(6-week+7*offset))

    result['startdate'] = datetime.datetime.combine(startdate, datetime.time.min)
    result['enddate'] = datetime.datetime.combine(enddate, datetime.time.max)
    result['enddate_pro'] = datetime.datetime.combine(enddate + datetime.timedelta(days=1), datetime.time.min)
    return result

def get_month_cycle(time_in, offset=0):
    """
    获取传入日期当月的开始结束时间
    :param time_in: 时间
    :param offset: 偏移量
    :return: 传入日期当月的开始结束时间
    """
    result = dict()
    date = time_in.date()

    weekday, last_day_num = calendar.monthrange(date.year, date.month)
    date_month_firstday = datetime.date(date.year, date.month, 1)
    date_month_lastday = datetime.date(date.year, date.month, last_day_num)

    # 获取offset个月前的月份
    for i in range(abs(offset)):
        if offset > 0:
            date = date_month_lastday + datetime.timedelta(days=1)
        elif offset < 0:
            date = date_month_firstday - datetime.timedelta(days=1)
        else:
            break

        weekday, last_day_num = calendar.monthrange(date.year, date.month)
        date_month_firstday = datetime.date(date.year, date.month, 1)
        date_month_lastday = datetime.date(date.year, date.month, last_day_num)

    result['startdate'] = datetime.datetime.combine(date_month_firstday, datetime.time.min)
    result['enddate'] = datetime.datetime.combine(date_month_lastday, datetime.time.max)
    result['enddate_pro'] = datetime.datetime.combine(date_month_lastday + datetime.timedelta(days=1), datetime.time.min)

    return result

def get_season_cycle(time_in, offset=0):
    """
    获取传入日期当季度的开始结束时间(一季度为1到3月，二季度为4到6月，三季度为7到9月，二季度为10到12月，)
    :param time_in: 时间
    :param offset: 偏移量
    :return: 传入日期当季度的开始结束时间
    """
    result = dict()
    termlist = [
        {'termid': 1, 'startmonth': 1, 'endmonth': 3},
        {'termid': 2, 'startmonth': 4, 'endmonth': 6},
        {'termid': 3, 'startmonth': 7, 'endmonth': 9},
        {'termid': 4, 'startmonth': 10, 'endmonth': 12},
    ]
    curterm = (time_in.month - 1) / 3 + 1
    queryterm = (curterm + offset - 1) % 4 + 1
    queryyear = time_in.year + int((curterm + offset - 1)/4)

    querystartmonth = termlist[queryterm-1]['startmonth']
    queryendmonth = termlist[queryterm-1]['endmonth']

    weekday, last_day_num = calendar.monthrange(queryyear, queryendmonth)

    startdate = datetime.datetime.strptime("%d-%d-1" % (queryyear, querystartmonth), "%Y-%m-%d")
    enddate = datetime.datetime.strptime("%d-%d-%d" % (queryyear, queryendmonth, last_day_num), "%Y-%m-%d")

    result['startdate'] = datetime.datetime.combine(startdate, datetime.time.min)
    result['enddate'] = datetime.datetime.combine(enddate, datetime.time.max)
    result['enddate_pro'] = datetime.datetime.combine(enddate + datetime.timedelta(days=1), datetime.time.min)

    return result

def get_term_cycle(time_in, offset=0):
    """
    获取传入日期当学期的开始结束时间(上半学年为上次9月1日0点0分0秒到之后的第一个1月31日23点59分59秒，下半学年为2月1日到下次8月31日23点59分59秒)
    :param time_in: 时间
    :param offset: 偏移量
    :return: 传入日期当学期的开始结束时间
    """
    # 查询传入的时间是第几学期，起止于哪年
    result = dict()
    if time_in.month >= 9:
        startyear = time_in.year + (offset+1)/2
        firstudyyear = True
    elif time_in.month < 2:
        startyear = time_in.year - 1 + (offset+1)/2
        firstudyyear = True
    else:
        startyear = time_in.year + offset/2
        firstudyyear = False

    if offset % 2 != 0:
        firstudyyear = not firstudyyear

    # 根据学期和年份获取起止时间
    if firstudyyear:
        endyear = startyear + 1
        startdate = datetime.datetime.strptime("%s-9-1" % startyear, "%Y-%m-%d")
        enddate = datetime.datetime.strptime("%s-1-31" % endyear, "%Y-%m-%d")
    else:
        endyear = startyear
        startdate = datetime.datetime.strptime("%s-2-1" % startyear, "%Y-%m-%d")
        enddate = datetime.datetime.strptime("%s-8-31" % endyear, "%Y-%m-%d")

    result['startdate'] = datetime.datetime.combine(startdate, datetime.time.min)
    result['enddate'] = datetime.datetime.combine(enddate, datetime.time.max)
    result['enddate_pro'] = datetime.datetime.combine(enddate + datetime.timedelta(days=1), datetime.time.min)
    return result

def get_year_cycle(time_in, offset=0):
    """
    获取传入日期当年的开始结束时间（当年时间为上次1月1日0点0分0秒到下次12月31日23点59分59秒）
    :param time_in: 时间
    :param offset: 偏移量
    :return: 传入日期当年的开始结束时间
    """
    result = dict()
    year = time_in.year + offset
    startdate = datetime.datetime.strptime("%s-1-1" % year, "%Y-%m-%d")
    enddate = datetime.datetime.strptime("%s-12-31" % year, "%Y-%m-%d")

    result['startdate'] = datetime.datetime.combine(startdate, datetime.time.min)
    result['enddate'] = datetime.datetime.combine(enddate, datetime.time.max)
    result['enddate_pro'] = datetime.datetime.combine(enddate + datetime.timedelta(days=1), datetime.time.min)
    return result

def get_schoolyear_cycle(time_in, offset=0):
    """
    获取传入日期当学年的开始结束时间，（学年时间为上次9月1日0点0分0秒到下次8月30日23点59分59秒）
    :param time_in: 时间
    :param offset: 偏移量
    :return: 传入日期当学年的开始结束时间
    """
    result = dict()
    if time_in.month >= 9:
        year = time_in.year + offset
    else:
        year = time_in.year - 1 + offset
    nextyear = year + 1

    startdate = datetime.datetime.strptime("%s-9-1" % year, "%Y-%m-%d")
    enddate = datetime.datetime.strptime("%s-8-31" % nextyear, "%Y-%m-%d")

    result['startdate'] = datetime.datetime.combine(startdate, datetime.time.min)
    result['enddate'] = datetime.datetime.combine(enddate, datetime.time.max)
    result['enddate_pro'] = datetime.datetime.combine(enddate + datetime.timedelta(days=1), datetime.time.min)
    return result

def get_unify_time(timestr):
    """
    将时间格式统一转换为时间格式，
    支持时间2017-02-28或2017-02-28 08:58:18或python时间戳1488767197或java时间戳1488767197123
    :param timestr:时间字符串，格式如上
    :return:%Y-%m-%d %H:%M:%S格式的时间字符串
    """
    # 检查传入的是时间戳还是时间，如果是时间戳则自动转换为时间
    if timestr.find('-') < 0:
        if len(timestr) == 13:
            result = datetime.datetime.fromtimestamp(float(timestr) / 1000)
        else:
            result = datetime.datetime.fromtimestamp(float(timestr))
    else:
        if timestr.find(':') < 0:
            result = datetime.datetime.strptime(timestr, "%Y-%m-%d")
        else:
            result = datetime.datetime.strptime(timestr, "%Y-%m-%d %H:%M:%S")

    return result

def get_time_perform(timestr):
    """
    将timestr转换为前端可以展示的时间
    :param timestr: %Y-%m-%d %H:%M:%S
    :return: %Y-%m-%d %H:%M，当天显示为“今天 %H:%M”，昨天显示为“昨天 %H:%M”
    """
    if not timestr:
        return ''

    in_datetime = datetime.datetime.strptime(timestr, "%Y-%m-%d %H:%M:%S")
    in_date = datetime.datetime.combine(in_datetime, datetime.time.min)
    in_date_str = in_date.strftime("%Y-%m-%d")
    in_time_str = in_datetime.strftime("%H:%M")

    today_datetime = datetime.datetime.now()
    today_date = datetime.datetime.combine(today_datetime, datetime.time.min)
    yestody_date = today_date - datetime.timedelta(days=1)

    if in_date == today_date:
        out_date = '今天'
    elif in_date == yestody_date:
        out_date = '昨天'
    else:
        out_date = in_date_str

    result = out_date + ' ' + in_time_str

    return result

def get_datetypelist_daterange(startdate, enddate, datetype):
    """
    通过时间范围获取对应的日期类型列表
    例如：
    传入时间
    入参:日期:时间格式，非字符串格式
         时间类型：天MEDAL_DATE_TYPE_DAY、周MEDAL_DATE_TYPE_WEEK、月MEDAL_DATE_TYPE_MONTH
    出参：[{序号，时间类型，开始时间，结束时间}]
        时间列表，字符串格式%Y-%m-%d %H:%M:%S
    """
    result = list()
    ordersn = 1  # 序号
    datetemp = datetime.datetime.combine(startdate, datetime.time.min)

    if datetype == DATE_TYPE_DAY:
        while True:
            if enddate < datetemp:
                break

            result_row = dict()
            result_row['order'] = ordersn
            result_row['datetype'] = DATE_TYPE_DAY
            result_row['startdate'] = datetime.datetime.combine(datetemp, datetime.time.min)
            result_row['enddate'] = datetime.datetime.combine(datetemp, datetime.time.max)
            result_row['enddate_pro'] = datetime.datetime.combine(datetemp + datetime.timedelta(days=1), datetime.time.min)
            result.append(result_row)

            datetemp += datetime.timedelta(days=1)
            ordersn += 1
    elif datetype == DATE_TYPE_WEEK:  # 注意：此处查出来的周有可能跨月。
        while True:
            if enddate < datetemp:
                break
            result_row = dict()
            result_row['order'] = ordersn
            result_row['datetype'] = DATE_TYPE_MONTH
            weekcycle = get_week_cycle(datetemp)
            result_row['startdate'] = weekcycle['startdate']
            result_row['enddate'] = weekcycle['enddate']
            result_row['enddate_pro'] = weekcycle['enddate_pro']
            result.append(result_row)

            datetemp += datetime.timedelta(days=7)
            ordersn += 1
    elif datetype == DATE_TYPE_MONTH:
        while True:
            if enddate < datetemp:
                break
            result_row = dict()
            result_row['order'] = ordersn
            result_row['datetype'] = DATE_TYPE_MONTH
            monthcycle = get_month_cycle(datetemp)
            result_row['startdate'] = monthcycle['startdate']
            result_row['enddate'] = monthcycle['enddate']
            result_row['enddate_pro'] = monthcycle['enddate_pro']
            result.append(result_row)

            datetemp = get_nextmonth_firstday(datetemp)
            ordersn += 1

    else:
        raise BusinessException(WRONG_TIME_TYPE)

    return result

def get_nextmonth_firstday(time_in):
    """
    获取下月1日时间
    :param time_in: 时间，时间格式
    :return:时间，时间格式
    """
    # 先获取本月1号
    indate_firstday = datetime.datetime.strptime("%d-%d-1" % (time_in.year, time_in.month), "%Y-%m-%d")

    # 本月1号+32天一定是下月
    outdate = indate_firstday + datetime.timedelta(days=32)
    outdate_firstday = datetime.datetime.strptime("%d-%d-1" % (outdate.year, outdate.month), "%Y-%m-%d")

    return outdate_firstday

def get_day_list(startdate, enddate):
    """
    将时间范围转为按天的时间列表，时间均为日期格式
    :param startdate:
    :param enddate:
    :return: 日期格式列表
    """
    result = list()
    if startdate > enddate:
        raise BusinessException(PARAM_ERROR_LARGE_STARTDATE)

    # 转成只带日期的时间格式
    startdate = datetime.datetime.strptime(startdate.strftime("%Y-%m-%d"), "%Y-%m-%d")
    enddate = datetime.datetime.strptime(enddate.strftime("%Y-%m-%d"), "%Y-%m-%d")

    datetmp = startdate
    while datetmp <= enddate:
        result.append(datetmp)
        datetmp = datetmp + datetime.timedelta(days=1)

    return result

def get_cycletime_from_date_type(date_type, startdate=None, enddate=None, offset=0):
    """
        将时段类型转换为开始结束时间
    """
    result = dict()
    if date_type:
        date_type = int(date_type)
        if date_type == DATE_TYPE_STUDYYEAR:
            result = get_schoolyear_cycle(datetime.datetime.now(), offset)
        elif date_type == DATE_TYPE_TODAY:
            result = get_day_cycle(datetime.datetime.now(), offset)
        elif date_type == DATE_TYPE_THISWEEK:
            result = get_week_cycle(datetime.datetime.now(), offset)
        elif date_type == DATE_TYPE_THISMONTH:
            result = get_month_cycle(datetime.datetime.now(), offset)
        elif date_type == DATE_TYPE_LASTMONTH:
            result = get_month_cycle(datetime.datetime.now(), offset-1)
        elif date_type == DATE_TYPE_THISSEASON:
            result = get_season_cycle(datetime.datetime.now(), offset)
        elif date_type == DATE_TYPE_THISTERM:
            result = get_term_cycle(datetime.datetime.now(), offset)
        elif date_type == DATE_TYPE_THISSCHOOLYEAR:
            result = get_schoolyear_cycle(datetime.datetime.now(), offset)
        else:
            raise BusinessException(DATETYPE_ERR)
    else:
        if not startdate or not enddate:
            raise BusinessException(REQUEST_PARAM_ERROR)

        result['startdate'] = get_unify_time(startdate)
        result['enddate'] = get_unify_time(enddate)
        result['enddate_pro'] = result['enddate']
    return result

def is_future_time(time_dt, delta=0):
    return time_dt > (now_datetime() + datetime.timedelta(days=delta))


import logging

LOGGING_FORMAT = '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
DEFAULT_LOG_LEVEL = logging.DEBUG

def configure_logger(cls, log_level=None, stream_handling=True):
    """ Takes a python class and returns a logger for that class. Convenience
    method.

    """
    logger_name = cls.__name__ if cls else 'default'
    logger = logging.getLogger(logger_name) if cls else logging.getLogger()
    log_level = DEFAULT_LOG_LEVEL if not log_level else log_level
    logger.setLevel(log_level)

    formatter = logging.Formatter(LOGGING_FORMAT)

    if stream_handling:
        ch = logging.StreamHandler()
        ch.setLevel(log_level)
        ch.setFormatter(formatter)
        logger.addHandler(ch)

    return logger

def load_file(file_name, file_lines=None, logger=None):
    """ Utility method for loading any configuration files. """
    try:
        with open(file_name, 'r') as file_contents:
            file_lines = file_contents.readlines()
    except IOError:
        if logger:
            logger.error(
                "The file %s was not found. Help may not work.".format(
                file_name))
    return file_lines


class Solution:
    def spiralOrder(self, matrix):
        if not matrix:
            return []

        row_len, col_len = len(matrix), len(matrix[0])
        boundaries = (len(matrix), len(matrix[0]), 0, 0)
        updates = [(0, 0, 1, 0), (0, -1, 0, 0), (-1, 0, 0, 0), (0, 0, 0, 1)]
        directions = [(0, 1), (1, 0), (0, -1), (-1, 0)]
        cnt = 0
        res = []
        k = 0
        x, y = 0, -1
        i, j = directions[k]

        while cnt < row_len * col_len:
            n_x, n_y = x + i, y + j
            if boundaries[0] > n_x >= boundaries[2] and boundaries[1] > n_y >= boundaries[3]:
                res.append(matrix[n_x][n_y])
                cnt += 1
                x, y = n_x, n_y
            else:
                boundaries = tuple(b + u for b, u in zip(boundaries, updates[k]))
                k = (k + 1) % 4
                i, j = directions[k]

        return res


#ftLinuxQAOps.py

import getopt, os, sys
from ftLinuxQAOperations import *
#
#=====================================Variable Declarations==============================================================================
global Obj
global QAOperation

#========================================================================================================================================
def EvaluateArgs():
    try:
        opts, args = getopt.getopt(sys.argv[1:], "hf:s:o:", ["help", "file=", "server=", "operation="])
    except getopt.GetoptError as err:
        # print help information and exit:
        print(err)
        Usage()
        sys.exit(2)

    Cfg_given=False
    Opr_given=False
    Srv_given=False

    for o, a in opts:
        if o in ("-o", "--operation"):
            Opr_given=True
            print('QA Operation mandated is %s ' % a )
            Obj.Set_Operation_Name(a)
            QAOperation=a

        elif o in ("-s", "--server"):
            Srv_given=True
            print('Target server specified is %s ' % a )
            Obj.Set_Host_Name(a)

        elif o in ("-f", "--file"):
            Cfg_given=True
            print('Setting QA Operations Config file to : %s ' %  a )
            Obj.Set_Config_File(a)

        elif o in ("-h", "--help"):
            Usage()
            sys.exit()
        else:
            assert False, "Unhandled option."

    if False in (Cfg_given, Opr_given, Srv_given):
        print("ERROR: One of the mandatory parameters is missing. Must specify mandatory parameters.")
        Usage()
        sys.exit(2)

    # if not Cfg_given:
        # print("Config file was not given. Must specify a Config file.")
        # Usage()
        # sys.exit(2)

    # if not Opr_given:
        # print("QA Operation not specified. Must specify a QA Operation.")
        # Usage()
        # sys.exit(2)

    print('List of major QA operations : ')
    print(">>>>" , args)
    global Operations
    Operations=Obj.STD_Operations if len(args)==0 else args
    print(Operations)
    Illegal_Operations=[Operation for Operation in Operations if Operation not in Obj.All_Operations]
    if (Illegal_Operations):
        print("Illegal operation names supplied : ", Illegal_Operations)
        Usage()
        sys.exit()
#========================================================================================================================================
def Usage():
    print("Usage: To be constructed...")
    print("Config file , Target Server IP address and Operation name  are mandatory options. ")
    print("All supported QA operations:")
    print(Obj.STD_Operations)

    print("Additionally available QA operations:")
    print([Operation for Operation in Obj.All_Operations if Operation not in Obj.STD_Operations])
#========================================================================================================================================

if __name__ == "__main__":
    Obj=ftLinuxQAOperations()
    Operations=[]
    QAOperation=""
    #
    EvaluateArgs()

    print(Obj.All_Operations)

    print(sys.argv)

    print("----->",Operations)
    #Obj.Perform_IPL()
    Obj.Set_Config_Dict()
    Obj.Set_Contact_IpAddr(Obj.CONFIG['DHCP-IPA'])
    #Obj.Set_Contact_IpAddr(Obj.CONFIG['STATIC-IPA'])
    Obj.Set_SSHO_Object()
    # ##Obj.Get_SSHClient()
    # ##Obj.SSHOObj=Obj.Get_SSHO_Object()
    # #Obj.Unregister_OS()
    # #Obj.Register_OS()
    # Obj.Set_Linux_Release(Obj.CONFIG['LINUX_RELEASE'])
    # #Obj.Update_OS()
    # #Obj.Generate_NetCfg_Script()
    # #Obj.Schedule_NetCfg()
    # #Obj.Install_AUL()
    # #print("Waiting untill server is reachable over network... (40 minutes)")
    # #time.sleep(2400)
    #Obj.Set_Contact_IpAddr(Obj.CONFIG['STATIC-IPA'])
    #Obj.Check_Server_Availability(Obj.CONTACT_IPADDR)
    #Obj.Set_SSHO_Object()
    #Obj.Unregister_OS()
    #Obj.Register_OS()
    #Obj.Install_QATools()
    #Obj.Verify_FTServer_Version()
    #Obj.Verify_HW_Duplexing()
    QAOperation=Obj.QA_OPERATION
    print('+++' + QAOperation)
    getattr(Obj,QAOperation)()
    # #Obj.Install_QATools()
    # #[getattr(Obj, Operation)() for Operation in Operations]
        # #print("***>",action)
        # #getattr(Obj,action)()
        # #Remember to implement error check and save_error routine later.


import collections

class Node:
    def __init__(self, val, lv):
        self.val = val
        self.children = {}
        self.lv = lv
        self.tails = collections.defaultdict(set)

class Trie:
    def __init__(self):
        self.head = Node(None, 0)

    def insert(self, string, suffix_cnt, suffixes):
        curr_node = self.head
        for idx in range(len(string)):
            # print(curr_node.tails, curr_node.children)
            if (curr_node.lv, string[idx:]) in suffixes:
                suffix_cnt['?'*curr_node.lv+string[idx:]] += 1
            if len(string) - idx - 1 >= 1:
                # print(000, curr_node.val)
                curr_node.tails[len(string) - idx ].add(string[idx:])
            if string[idx] not in curr_node.children:
                curr_node.children[string[idx]] = Node(string[idx], idx+1)
            # print(988,curr_node.tails, curr_node.children, curr_node.lv)
            curr_node = curr_node.children[string[idx]]

    def starts_with(self, prefix, remained_len):
        curr_node = self.head
        subtrie = None
        for char in prefix:
            if char in curr_node.children:
                curr_node = curr_node.children[char]
                subtrie = curr_node
            else:
                return 0
        # print(5839, prefix, subtrie.tails)
        if remained_len in subtrie.tails:
            return len(subtrie.tails[remained_len])
        else:
            return 0

def split(string):
    for i in range(len(string)):
        if string[i] == '?':
            return i
    return -1

def split2(string):
    for i in range(len(string)):
        if string[i] != '?':
            return i
    return -1

def solution(words, queries):
    answer = []

    suffixes = set()
    for q in queries:
        if q[0] == '?' and q[-1] != '?':
            lv = split2(q)
            suffixes.add((lv, q[lv:]))
    # print(suffixes)

    t = Trie()
    suffix_cnt = collections.defaultdict(int)
    for word in words:
        t.insert(word, suffix_cnt, suffixes)
    # print(45345,suffix_cnt)

    # print(t.head.tails, t.head.children)
    for q in queries:
        if q[0] == '?' and q[-1] == '?':
            answer.append(t.head.tails[len(q)])
        elif q[-1] == '?':
            idx = split(q)
            prefix = q[:idx]
            answer.append(t.starts_with(prefix, len(q) - idx))
        elif q[0] == '?':
            answer.append(suffix_cnt[q])
    return answer

print(solution(["frodo", "front", "frost", "frozen", "frame", "kakao"], ["fro??", "????o", "fr???", "fro???", "pro?"]))


#https://towardsdatascience.com/handling-overfitting-in-deep-learning-models-c760ee047c6e

# Problem statement
# we need to see the viz when network picks up new samples afer every xth epoch

from __future__ import absolute_import, division, print_function, unicode_literals

import tensorflow as tf

from keras import layers
from keras import regularizers

print(tf.__version__)

import pandas as pd
import numpy as np
import re
import collections
import matplotlib.pyplot as plt

import nltk
nltk.download('stopwords')

from pathlib import Path
# Packages for data preparation
from sklearn.model_selection import train_test_split
from nltk.corpus import stopwords
from keras.callbacks import ModelCheckpoint, LambdaCallback,CSVLogger
from keras.preprocessing.text import Tokenizer
from keras.utils.np_utils import to_categorical
from sklearn.preprocessing import LabelEncoder
# Packages for modeling
from keras import models
from keras import layers
from keras import regularizers

import sys
import os
sys.path.append(os.getcwd())
from tools.multi_model_info import save_graph_plot, save_graph_json
from tools.project_tools import get_project_name, get_project_paths
#from tensorflow.keras.callbacks import ModelCheckpoint
NB_WORDS = 800  # Parameter indicating the number of words we'll put in the dictionary
NB_START_EPOCHS = 51  # Number of epochs we usually start to train with
BATCH_SIZE = 512  # Size of the batches used in the mini-batch gradient descent
MAX_LEN = 500  # Maximum number of words in a sequence
root = Path('../')
input_path = root / 'input/'
ouput_path = root / 'output/'
source_path = root / 'source/'
logs_path = "/home/pasricha/keras_log-weights/log/UC3/"

def deep_model(model, X_train, y_train, X_valid, y_valid):
    logs = logs_path + "uc2_model_history_log.csv"
    csv_logger = CSVLogger(logs, append=True)
    '''
    Function to train a multi-class model. The number of epochs and
    batch_size are set by the constants at the top of the
    notebook.

    Parameters:
        model : model with the chosen architecture
        X_train : training features
        y_train : training target
        X_valid : validation features
        Y_valid : validation target
    Output:
        model training history
    '''
    model.compile(optimizer='rmsprop'
                  , loss='categorical_crossentropy'
                  , metrics=['accuracy'])
    argv = "deep.py"
    project_paths = get_project_paths(argv[0], to_tmp=False)

    save_graph_plot(model, project_paths["plots"] + "/model.ps")
    save_graph_json(model, project_paths["graphs"] + "/model.json")
    #import pdb; pdb.set_trace()
    history = model.fit(X_train
                       , y_train
                       , epochs=NB_START_EPOCHS
                       , batch_size=BATCH_SIZE
                       , validation_data=(X_valid, y_valid)
                       , callbacks=[
                           ModelCheckpoint(
                           project_paths["checkpoints"] + "/weights_epoch-{epoch:02d}.hdf5",
                           save_weights_only=True,
                           period = 1),
                                    csv_logger]
#                        , callbacks = [ModelCheckpoint(project_paths["checkpoints"] + "/weights_epoch-{epoch:02d}.hdf5",
#                                                       monitor='val_acc', verbose=1,  save_weights_only=True,  period=1)]
                       , verbose=1)
    return history
def eval_metric(model, history, metric_name):
    '''
    Function to evaluate a trained model on a chosen metric.
    Training and validation metric are plotted in a
    line chart for each epoch.

    Parameters:
        history : model training history
        metric_name : loss or accuracy
    Output:
        line chart with epochs of x-axis and metric on
        y-axis
    '''
    #import pdb;
    #pdb.set_trace()
    plt.clf()
    metric = history.history[metric_name]
    val_metric = history.history['val_' + metric_name]
    e = range(1, NB_START_EPOCHS + 1)
    plt.plot(e, metric, 'bo', label='Train ' + metric_name)
    plt.plot(e, val_metric, 'b', label='Validation ' + metric_name)
    plt.xlabel('Epoch number')
    plt.ylabel(metric_name)
    plt.title('Comparing Loss training and validation for ' + model.name)
    plt.legend()
    plt.show()
    plt.savefig(logs_path  + model.name + "_" + metric_name)

def test_model(model, X_train, y_train, X_test, y_test, epoch_stop):
    '''
    Function to test the model on new data after training it
    on the full training data with the optimal number of epochs.

    Parameters:
        model : trained model
        X_train : training features
        y_train : training target
        X_test : test features
        y_test : test target
        epochs : optimal number of epochs
    Output:
        test accuracy and test loss
    '''
    model.fit(X_train
              , y_train
              , epochs=epoch_stop
              , batch_size=BATCH_SIZE
              , verbose=0)
    results = model.evaluate(X_test, y_test)
    print()
    print('Test accuracy for ' + model.name + ' : {0:.2f}%'.format(results[1]*100))
    return results

def remove_stopwords(input_text):
    '''
    Function to remove English stopwords from a Pandas Series.

    Parameters:
        input_text : text to clean
    Output:
        cleaned Pandas Series
    '''
    stopwords_list = stopwords.words('english')
    # Some words which might indicate a certain sentiment are kept via a whitelist
    whitelist = ["n't", "not", "no"]
    words = input_text.split()
    clean_words = [word for word in words if (word not in stopwords_list or word in whitelist) and len(word) > 1]
    return " ".join(clean_words)

def remove_mentions(input_text):
    '''
    Function to remove mentions, preceded by @, in a Pandas Series

    Parameters:
        input_text : text to clean
    Output:
        cleaned Pandas Series
    '''
    return re.sub(r'@\w+', '', input_text)
def compare_models_by_metric(model_1, model_2, model_hist_1, model_hist_2, metric):
    '''
    Function to compare a metric between two models

    Parameters:
        model_hist_1 : training history of model 1
        model_hist_2 : training history of model 2
        metrix : metric to compare, loss, acc, val_loss or val_acc

    Output:
        plot of metrics of both models
    '''
    plt.clf() # To Erase previous plots
    #import pdb;pdb.set_trace()
    metric_model_1 = model_hist_1.history[metric]
    metric_model_2 = model_hist_2.history[metric]
    e = range(1, NB_START_EPOCHS + 1)

    metrics_dict = {
        'acc' : 'Training Accuracy',
        'loss' : 'Training Loss',
        'val_acc' : 'Validation accuracy',
        'val_loss' : 'Validation loss',
        'val_accuracy' :  'Validation accuracy'
    }
    metric_label = metrics_dict[metric]
    plt.plot(e, metric_model_1, 'bo', label=model_1.name)
    plt.plot(e, metric_model_2, 'b', label=model_2.name)
    plt.xlabel('Epoch number')
    plt.ylabel(metric_label)
    plt.title('Comparing ' + metric_label + ' between models')
    plt.legend()
    plt.show()
    plt.savefig(logs_path  + model_1.name + model_2.name)
    plt.clf()

def optimal_epoch(model_hist):
    '''
    Function to return the epoch number where the validation loss is
    at its minimum

    Parameters:
        model_hist : training history of model
    Output:
        epoch number with minimum validation loss
    '''
    min_epoch = np.argmin(model_hist.history['val_loss']) + 1
    print("Minimum validation loss reached in epoch {}".format(min_epoch))
    return min_epoch

df = pd.read_csv('Tweets.csv')
df = df.reindex(np.random.permutation(df.index))
df = df[['text', 'airline_sentiment']]
df.text = df.text.apply(remove_stopwords).apply(remove_mentions)

X_train, X_test, y_train, y_test = train_test_split(df.text, df.airline_sentiment, test_size=0.1, random_state=37)

tk = Tokenizer(num_words=NB_WORDS,
               filters='!"#$%&()*+,-./:;<=>?@[\\]^_`{"}~\t\n',
               lower=True,
               char_level=False,
               split=' ')
tk.fit_on_texts(X_train)

X_train_oh = tk.texts_to_matrix(X_train, mode='binary')
X_test_oh = tk.texts_to_matrix(X_test, mode='binary')

le = LabelEncoder()
y_train_le = le.fit_transform(y_train)
y_test_le = le.transform(y_test)
y_train_oh = to_categorical(y_train_le)
y_test_oh = to_categorical(y_test_le)

print("************************Overfit MODEL test_size=0.999 *******************")
X_train_rest, X_valid, y_train_rest, y_valid = train_test_split(X_train_oh, y_train_oh, test_size=0.999, random_state=37)
#import pdb; pdb.set_trace()
overfit_model = models.Sequential()
overfit_model.add(layers.Dense(16, activation='relu', input_shape=(NB_WORDS,)))
overfit_model.add(layers.Dense(3, activation='softmax'))
overfit_model.name = 'UC2_OverFit_low_train_size'
overfit_history = deep_model(overfit_model, X_train_rest, y_train_rest, X_valid, y_valid)
overfit_min = optimal_epoch(overfit_history)
eval_metric(overfit_model, overfit_history, 'loss')
eval_metric(overfit_model, overfit_history, 'accuracy')

print("************************Perfect MODEL test_size=0.1 *******************")
X_train_rest, X_valid, y_train_rest, y_valid = train_test_split(X_train_oh, y_train_oh, test_size=0.1, random_state=37)
perfect_model = models.Sequential()
perfect_model.add(layers.Dense(16, activation='relu', input_shape=(NB_WORDS,)))
perfect_model.add(layers.Dense(3, activation='softmax'))
perfect_model.name = 'UC2_perfect'
perfect_history = deep_model(perfect_model, X_train_rest, y_train_rest, X_valid, y_valid)
perfect_min = optimal_epoch(perfect_history)
eval_metric(perfect_model, perfect_history, 'loss')
eval_metric(perfect_model, perfect_history, 'accuracy')

overfit_results = test_model(overfit_model, X_train_oh, y_train_oh, X_test_oh, y_test_oh, overfit_min)
perfect_results = test_model(perfect_model, X_train_oh, y_train_oh, X_test_oh, y_test_oh, perfect_min)

compare_models_by_metric(overfit_model, perfect_model, overfit_history, perfect_history, 'val_accuracy')
# compare_models_by_metric(base_model, reduced_model, base_history, reduced_history, 'val_accuracy')
# compare_models_by_metric(base_model,rereduced_model , base_history, rereduced_history, 'val_accuracy')
# compare_models_by_metric(reduced_model, rereduced_model, reduced_history, rereduced_history, 'val_accuracy')


import os
import tabix
import time
from .util import file_util
from .util import vcf_util
from .util import proc_util
from .util import struct_util
from .util import region_util
from .model import datasource
from .annot2.annotvcf import VCFAnnotator, VCFVariant
from .model.datastructure import DataSourceListStructure
from .model.datasource import DataSourceList
from . import external_functions
from .renderer import TSIRenderer

REFIDX = 3
ALTIDX = 4
entrezmap = {}
refseqmap = {}

class BlankVCFReader():
    def __init__(self, opt, dslist):
        self.dslist = dslist
        self.opt = opt
        self.region = region_util.Region(opt.region)
        self.tpos = self.region.spos
        self.variantkeylist = []
        self.eof = False
        self.colnames = vcf_util.VCF_COL

    def get_variant(self):
        if len(self.variantkeylist) == 0 and self.tpos < self.region.epos:
            self.load_variantkeys()

        if len(self.variantkeylist) > 0:
            vkey = self.variantkeylist.pop(0)
            vcfline = vkey.replace('_', '\t') + '\t\t\t\t'
            variant = VCFVariant(vcfline, self.colnames, self.opt)
            self.tpos = int(vkey.split('_')[1])
        else:
            variant = None
            self.eof = True
        return variant

    def load_variantkeys(self):
        for r1 in self.region.get_split_region(1000):
            variantkeys = self.dslist.get_variantkeys(r1.chrom, r1.spos, r1.epos)
            poslist = list(variantkeys.keys())
            for pos in sorted(poslist):
                for vkey in variantkeys[pos]:
                    self.variantkeylist.append(vkey)

class DataSourceGenerator(VCFAnnotator):
    def __init__(self, opt):
        self.opt = opt
        self.dslist = DataSourceList(self.opt.ds)
        self.region = region_util.Region(opt.region)
        self.out = None
        self.set_outfile_extension()
        self.vcfreader = BlankVCFReader(self.opt, self.dslist)
        self.tsirenderer = TSIRenderer()

    def set_outfile_extension(self):
        out2 = self.opt.out.replace('.tsi.gz', '.tsi')
        if out2[-4:] != ".tsi":
            out2 += ".tsi"
        self.out = out2

    def get_tsi_header(self):
        header = ["#CHROM", "POS", "ID", "REF", "ALT"]
        info_arr = []
        for s1 in self.dslist.available_source_list:
            info_header = []
            for f1 in s1.available_field_list:
                info_header.append(f1.name2)
            info_arr.append(s1.name + '=' + '|'.join(info_header))
        header.append(';'.join(info_arr))
        return '\t'.join(header)

    def print_i(self, ivar, variant, stime1, stime0):
        if ivar % 500 == 0:
            etime1 = time.time()
            elapsed1 = etime1 - stime1
            elapsed0 = etime1 - stime0
            log = 'processed... ' + str(ivar)
            log += " elapsed:" + str(round(elapsed1, 2)) + "s " + str(round(elapsed0, 2)) + "s , time for 9G variants:" + str(round(elapsed0 / ivar * 9000000000 / 3600, 3)) + "hr"
            # print(log, ivar, variant)

    def make_single_source_file(self):
        stime0 = time.time()
        self.open_outpointer()
        self.fp.write(self.get_tsi_header() + '\n')

        ivar = 0
        stime1 = time.time()
        while(not self.vcfreader.eof):
            ivar += 1
            variant = self.vcfreader.get_variant()
            if self.vcfreader.eof or variant == None:
                break
            variant.set_annot(self.dslist)
            self.fp.write(self.tsirenderer.render_vcfvariant(variant, False, True))
            self.print_i(ivar, variant, stime1, stime0)
            stime1 = time.time()

        self.close_outpointer()
        etime0 = time.time()
        elapsed0 = etime0 - stime0
        print("total:", elapsed0, ", time per variant:", elapsed0 / ivar, ", time for 9G variants:", str(round(elapsed0 / ivar * 9000000000 / 3600, 3)) + "hr")

        # time.sleep(10)
        # proc_util.run_cmd('gz ' + self.out)

    def make_single_source_file_with_block(self):
        fp = open(self.opt.out, 'w')
        blockreaders = {}
        for s1 in self.dslist.available_source_list:
            blockreaders[s1.name] = TSVBlockReader(s1)

        tbm = TSVBlockMerger(blockreaders, self.region, 1000)
        print_header = True
        while not tbm.eof:
            start = time.time()
            block = tbm.get_block_tsi()
            if print_header:
                fp.write(tbm.get_header())
                print_header = False
            fp.write(block)

            end = time.time()
            elapsed = end - start
            log = 'processing.. ' + str(round(elapsed, 3)) + ' sec elapsed. '
            log += str(len(tbm.block.keys())) + ' variants added. '
            log += str(tbm.block_spos) + '~' + str(tbm.block_epos)
            print(log)
        print('Saved', self.out)
        fp.close()

    def check_datasourcefile(self):
        mt_seq_reader = datasource.MutannoDataSequentialReader(self.out)

        dslist = datasource.DataSourceList()
        dslist.set_datastructure(self.datastruct)

        for mtdata in mt_seq_reader:
            annot = dslist.get_sourcedata_variant(mtdata['CHROM'], mtdata['POS'], mtdata['REF'], mtdata['ALT'])

            break
        pass

class TSVBlockReader():

    def __init__(self, datasource):
        self.source = datasource
        self.fileformat = self.source.format
        self.target_colidx = []
        self.header = []
        self.filter_start_with = {}
        self.filter_skip_equal_str = {}
        self.filter_keep_equal_str = {}
        self.delimiter = {}
        self.defaultvalue = {}
        self.field_function = {}
        self.field_function_param = {}
        self.field_names = []
        self.tp = None
        self.chrom = ''
        self.refidx = self.source.ref_column_index
        self.altidx = self.source.alt_column_index

    def add_block_bed(self, block, region, sid):
        if self.tp is None:
            sourcefile = self.source.sourcefile2.replace('#CHROM#', region.chrom)
            if file_util.is_exist(sourcefile) and file_util.is_exist(sourcefile + ".tbi"):
                self.tp = tabix.open(sourcefile)
                self.source.set_header(sourcefile)

        if self.tp is not None:
            self.set_target_column()
            for arr in self.tp.querys(region.region_str):
                d = {}
                spos = int(arr[1])
                epos = int(arr[2])
                ref = arr[self.refidx]
                if (spos+1) >= region.spos and (spos+1) <= region.epos:
                    cont = (spos,epos,ref, self.source.name + "=" + "|".join(self.select_field(arr, self.source.header)))
                    try:
                        block[spos].append(cont)
                    except KeyError:
                        block[spos] = [cont]
        return block

    def set_target_column(self):
        for idx, h1 in enumerate(self.source.header):
            if self.source.format == "bed":
                self.target_colidx = list(range(3, len(self.source.header)))
            elif self.fileformat == "tsi":
                self.target_colidx = [len(self.source.header)-1]
            else:
                for idx, h1 in enumerate(self.source.header):
                    if h1.upper() not in ['CHROM','POS','REF','ALT']:
                        if idx not in self.target_colidx:
                            self.target_colidx.append(idx)

    def select_field(self, arr, header):
        rst = []
        for idx in self.target_colidx:
            rst.append(arr[idx])
        return rst

    def add_block(self, block, region, sid):
        if self.tp is None:
            sourcefile = self.source.sourcefile2.replace('#CHROM#', region.chrom)
            if file_util.is_exist(sourcefile) and file_util.is_exist(sourcefile + ".tbi"):
                self.tp = tabix.open(sourcefile)
                self.source.set_header(sourcefile)
        if self.tp is not None:
            self.set_target_column()
            try:
                for arr in self.tp.querys(self.source.chrompre + region.region_str):
                    pos = int(arr[1])
                    if pos >= region.spos and pos <= region.epos:
                        refalt = arr[self.refidx] + '_' + arr[self.altidx]
                        try:
                            block[pos]
                        except KeyError:
                            block[pos] = {}
                        try:
                            block[pos][refalt]
                        except KeyError:
                            block[pos][refalt] = {}

                        if self.fileformat == "tsi":
                            cont = arr[-1]
                        else:
                            cont = '|'.join(self.select_field(arr, self.source.header))

                        try:
                            block[pos][refalt][sid].append(cont)
                        except KeyError:
                            block[pos][refalt][sid] = [cont]
            except tabix.TabixError:
                pass

        return block

    def get_header(self):
        rst = []
        for idx in self.target_colidx:
            rst.append(self.source.header[idx])
        return rst

class TSVBlockMerger():
    def __init__(self, block_readers, region, block_size):
        self.block_readers = block_readers
        self.block_size = block_size
        self.region = region  # region object
        self.block_spos = 0
        self.block_epos = 0
        self.block_lpos = 0
        self.block = {}
        self.eof = False

    def read_block(self):
        if self.block_spos == 0:
            self.block_spos = self.region.spos
        else:
            self.block_spos = self.block_epos + 1

        if (self.block_spos + self.block_size) <= self.region.epos:
            self.block_epos = self.block_spos + self.block_size - 1
        else:
            self.block_epos = self.region.epos
        this_range = region_util.Region(self.region.chrom + ':' + str(self.block_spos) + '-' + str(self.block_epos))

        self.block = {}
        for sid in self.block_readers.keys():
            if self.block_readers[sid].fileformat != "bed":
                self.block = self.block_readers[sid].add_block(self.block, this_range, sid)

        for sid in self.block_readers.keys():
            if self.block_readers[sid].fileformat == "bed":
                self.block = self.block_readers[sid].add_block_bed(self.block, this_range, sid)

    def get_block_tsi(self):
        self.read_block()
        if self.block_epos >= self.region.epos:
            self.eof = True

        cont = ''
        poslist = list(self.block.keys())
        for pos in sorted(poslist):
            if isinstance(self.block[pos], dict):
                for refalt in self.block[pos].keys():
                    infoarr = []
                    for sid in self.block_readers.keys():
                        if sid in self.block[pos][refalt].keys():
                            if self.block[pos][refalt][sid] != "":
                                infoarr.append(sid + '=' + ",".join(self.block[pos][refalt][sid]))

                    cont += self.region.chrom + '\t' + str(pos) + '\t\t' + refalt.replace('_', '\t') + '\t\t\t\t' + ';'.join(infoarr) + '\n'
            elif isinstance(self.block[pos], list): # bed file foramt
                for bedline in self.block[pos]:
                    spos = bedline[0]
                    epos = bedline[1]
                    ref = bedline[2]
                    infoarr = bedline[3]
                    cont += self.region.chrom + '\t' + str(spos+1) + '\t\t' + ref + '\t\t\t\t' + 'END=' + str(epos) +  '\t' + infoarr + '\n'
            self.block_lpos = pos
        return cont

    def get_header(self):
        cont = ""
        for sid in self.block_readers.keys():
            header = self.block_readers[sid].get_header()
            if cont != "":
                cont += ";"
            cont += sid + '=' + '|'.join(header)
        cont = "#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\t" + cont+ "\n"
        return cont


# -*- coding: utf-8 -*-
"""
Created on Mon Dec  2 17:03:06 2019

@author: Administrator
"""

import pdblp
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
#import seaborn as sns

plt.style.use('seaborn')

#con = pdblp.BCon(debug=True, port=8194, timeout=5000)
con = pdblp.BCon(debug=False, port=8194, timeout=5000)
con.start()
index_tickers = ['NYA Index', 'SPX Index', 'CCMP Index' ,'CDAX Index' ,
            'ASX Index', 'TPX Index', 'SHCOMP Index' ,
           'SZCOMP Index', 'XUTUM Index', 'MEXBOL Index',
           'IBOV Index', 'IMOEX Index' , 'JALSH Index']

from datetime import date

start = '2004-1-1'
today = date.today().strftime('%Y%m%d')
firstday = '19991230'

window_long = 52
window_short = 13

ohlc_tickers = ['OPEN','HIGH','LOW', 'LAST']

prices_open = con.bdh(index_tickers, 'PX OPEN',firstday , today)
prices_open.columns = [i[0] for i in prices_open.columns]
prices_open_int = prices_open.interpolate(method='linear')[index_tickers]
prices_open_w = prices_open_int.groupby(pd.Grouper(freq='W')).first()

prices_high = con.bdh(index_tickers, 'PX HIGH',firstday , today)
prices_high.columns = [i[0] for i in prices_high.columns]
prices_high_int = prices_high.interpolate(method='linear')[index_tickers]
prices_high_w = prices_high_int.groupby(pd.Grouper(freq='W')).max()

prices_low  = con.bdh(index_tickers, 'PX LOW',firstday , today)
prices_low.columns = [i[0] for i in prices_low.columns]
prices_low_int = prices_low.interpolate(method='linear')[index_tickers]
prices_low_w = prices_low_int.groupby(pd.Grouper(freq='W')).min()

prices_close  = con.bdh(index_tickers, 'PX LAST',firstday , today)
prices_close.columns = [i[0] for i in prices_close.columns]
prices_close_int = prices_close.interpolate(method='linear')[index_tickers]
prices_close_w = prices_close_int.groupby(pd.Grouper(freq='W')).last()

returns_open = prices_open_w / prices_close_w.shift(1) - 1
returns_high = prices_high_w / prices_close_w.shift(1) - 1
returns_low = prices_low_w / prices_close_w.shift(1) - 1
returns_close = prices_close_w / prices_close_w.shift(1) - 1

returns_fromClose_ohlc =  pd.concat([returns_open, returns_high, returns_low, returns_close],axis=1)
returns_fromClose_ohlc.columns =  [('_').join(i) for i in zip(returns_fromClose_ohlc.columns,np.repeat(ohlc_tickers,len(index_tickers)))]

std_fromClose_ohlc_long = returns_fromClose_ohlc.rolling(window_long).std()[returns_fromClose_ohlc.index>=start]
std_fromClose_ohlc_long.columns = ['23-2_'+i for i in std_fromClose_ohlc_long.columns]
#std_fromClose_ohlc_long.columns = ['23_1_US_NY','23_1_US_SPX','23_1_US_CCMP','23_1_DE','23_1_UK','23_1_JP','23_1_CH_SH','23_1_SZ','23_1_TR',
#                                   '23_1_MX','23_1_BR','23_1_RU','23_1_ZA']

mean_fromClose_ohlc_long = returns_fromClose_ohlc.rolling(window_long).mean()[returns_fromClose_ohlc.index>=start]
mean_fromClose_ohlc_long.columns = ['23-1_'+i for i in mean_fromClose_ohlc_long.columns]

std_fromClose_ohlc_short= returns_fromClose_ohlc.rolling(window_short).std()[returns_fromClose_ohlc.index>=start]
std_fromClose_ohlc_short.columns = ['23-4_'+i for i in std_fromClose_ohlc_short.columns]

mean_fromClose_ohlc_short= returns_fromClose_ohlc.rolling(window_short).mean()[returns_fromClose_ohlc.index>=start]
mean_fromClose_ohlc_short.columns = ['23-3_'+i for i in mean_fromClose_ohlc_short.columns]

#std_fromClose_ohlc_short.columns = ['23_2_US_NY','23_2_US_SPX','23_2_US_CCMP','23_2_DE','23_2_UK','23_2_JP','23_2_CH_SH','23_2_SZ','23_2_TR',
#                                   '23_2_MX','23_2_BR','23_2_RU','23_2_ZA']
##############################################################################

prices_ohlc = pd.concat([prices_open_w, prices_high_w, prices_low_w, prices_close_w],axis=1)
prices_ohlc.columns =  [('_').join(i) for i in zip(prices_ohlc.columns,np.repeat(ohlc_tickers,len(index_tickers)))]

std_delta_ohlc_long = prices_ohlc.pct_change().rolling(window_long).std()
mean_delta_ohlc_long = prices_ohlc.pct_change().rolling(window_long).mean()

std_ohlc_long = std_delta_ohlc_long[std_delta_ohlc_long.index>=start]
std_ohlc_long.columns = ['23-6_'+i for i in std_ohlc_long.columns]

mean_ohlc_long = mean_delta_ohlc_long[mean_delta_ohlc_long.index>=start]
mean_ohlc_long.columns = ['23-5_'+i for i in mean_ohlc_long.columns]

#std_ohlc_long.columns = ['23_3_US_NY','23_3_US_SPX','23_3_US_CCMP','23_3_DE','23_3_UK','23_3_JP','23_3_CH_SH','23_3_SZ','23_3_TR',
#                                   '23_3_MX','23_3_BR','23_3_RU','23_3_ZA']

std_delta_ohlc_short = prices_ohlc.pct_change().rolling(window_short).std()
mean_delta_ohlc_short = prices_ohlc.pct_change().rolling(window_short).mean()

std_ohlc_short = std_delta_ohlc_short[std_delta_ohlc_short.index>=start]
std_ohlc_short.columns = ['23-8_'+i for i in std_ohlc_short.columns]

mean_ohlc_short = mean_delta_ohlc_short[mean_delta_ohlc_short.index>=start]
mean_ohlc_short.columns = ['23-7_'+i for i in mean_ohlc_short.columns]

#std_ohlc_short.columns = ['23_4_US_NY','23_4_US_SPX','23_4_US_CCMP','23_4_DE','23_4_UK','23_4_JP','23_4_CH_SH','23_4_SZ','23_4_TR',
#                                   '23_4_MX','23_4_BR','23_4_RU','23_4_ZA']

mean_fromClose_ohlc_long.to_excel('C:/Users/sb0538/Desktop/15022020/excels/23-1_rvolmeanfromCloseohlclong.xlsx')

std_fromClose_ohlc_long.to_excel('C:/Users/sb0538/Desktop/15022020/excels/23-2_rvolstdfromcloseohlclong.xlsx')

mean_fromClose_ohlc_short.to_excel('C:/Users/sb0538/Desktop/15022020/excels/23-3_rvolmeanfromcloseohlcshort.xlsx')

std_fromClose_ohlc_short.to_excel('C:/Users/sb0538/Desktop/15022020/excels/23-4_rvolstdfromcloseohlcshort.xlsx')

mean_ohlc_long.to_excel('C:/Users/sb0538/Desktop/15022020/excels/23-5_rvolmeanohlcdeltalong.xlsx')

std_ohlc_long.to_excel('C:/Users/sb0538/Desktop/15022020/excels/23-6_rvolstdohlcdeltalong.xlsx')

mean_ohlc_short.to_excel('C:/Users/sb0538/Desktop/15022020/excels/23-7_rvolmeanohlcdeltashort.xlsx')

std_ohlc_short.to_excel('C:/Users/sb0538/Desktop/15022020/excels/23-8_rvolstdohlcdeltashort.xlsx')


import argparse
import sys
import torch
import os
from torchvision import transforms
from utils.read_data import Covid_19_Dataset
from torch.utils.data import DataLoader
from networks.unet import UNet
from collections import OrderedDict
from utils.util import weight_to_cpu
from matplotlib import pyplot as plt
import numpy as np
from PIL import Image
from sklearn.metrics import precision_recall_curve, auc
from tqdm import tqdm

def parse_args():
    parser = argparse.ArgumentParser(description='Segment Covid-19 lesion area')
    parser.add_argument('--test_set_dir', '-td', type=str, default='./data_1/covid_19_ct',
                        help='folder to load test set')
    parser.add_argument('--save_dir', '-sd', type=str, default='./result/ep_62_2', help='folder to save result')
    parser.add_argument('--batch_size', '-bs', type=int, default=4, help='batch size')
    parser.add_argument('--power', '-k', type=int, default=2, help='power of weight')
    parser.add_argument('--input_channel', type=int, default=1, help='input channel of model')
    parser.add_argument('--u_depth', type=int, default=5, help='unet dpeth')
    parser.add_argument('--pretrain_unet_path', type=str, default='./check_points/train_3_ep_62/g.pkl',
                        help='pretrained unet')
    parser.add_argument('--gt_dir', '-gtd', type=str, default='./data_1/covid_19_ct/mask',
                        help='folder of ground truth')
    return parser.parse_args()

class Predict:
    def __init__(self, args):
        self.mean, self.std = 0.5, 0.5
        self.args = args
        self.dataloader = self.get_dataloader()
        self.auto_encoder = self.get_unet()
        self.prediction = []
        self.label = []
        self.segmentation_threshold = 60

    def get_dataloader(self):
        transform = transforms.Compose([
            transforms.ToTensor(),
            transforms.Normalize(self.mean, self.std)
        ])
        dataset = Covid_19_Dataset(data_dir=self.args.test_set_dir,
                                   transform=transform)
        data_loader = DataLoader(dataset,
                                 batch_size=self.args.batch_size,
                                 shuffle=True,
                                 num_workers=2,
                                 drop_last=True,
                                 pin_memory=False)
        return data_loader

    def get_unet(self):
        unet = UNet(num_classes=1, depth=self.args.u_depth, in_channels=self.args.input_channel)
        print('loading the parameters of generator......')
        unet.load_state_dict(weight_to_cpu(self.args.pretrain_unet_path))
        print('success loading!')
        return unet

    def save_single_image(self, saved_dir, names, inputs):
        """
        save unet output as a form of image
        """
        if not os.path.exists(saved_dir):
            os.makedirs(saved_dir)
        with torch.no_grad():
            outputs, gdps = self.auto_encoder(inputs)
        # print(outputs.shape, gdps.shape)
        total_dice = 0
        for i in range(inputs.shape[0]):
            input = inputs[i]
            output = outputs[i]
            gdp = gdps[i]
            name = names[i]
            left = self.restore(input)
            right = self.restore(output)
            lesion = self.restore(gdp)

            shotname, extension = os.path.splitext(name)

            # gt.resize muust use Nearest neighbor interpolation
            gt = Image.open(os.path.join(self.args.gt_dir, shotname + '_mask' + extension)).resize((256, 256),
                                                                                Image.NEAREST)
            gt = np.array(gt)
            diff = np.where(left > right, left - right, right - left).clip(0, 255).astype(np.uint8)
            diff_1 = np.where(diff >= self.segmentation_threshold, 255, 0).astype(np.uint8)
            self.save_prediction_label(diff, gt)
            dice_score = self.dice_coeff(diff_1 / 255, gt / 255)
            # print(dice_score)
            total_dice += dice_score
            plt.figure(num='unet result', figsize=(8, 8))

            plt.subplot(2, 3, 1)
            plt.title('source image')
            plt.imshow(left, cmap='gray')
            plt.axis('off')

            plt.subplot(2, 3, 2)
            plt.title('unet output')
            plt.imshow(right, cmap='gray')
            plt.axis('off')

            plt.subplot(2, 3, 3)
            plt.title('GDP')
            plt.imshow(lesion, cmap='gray')
            plt.axis('off')

            plt.subplot(2, 3, 4)
            plt.imshow(diff, cmap='jet')
            plt.colorbar(orientation='horizontal')
            plt.title('difference in heatmap')
            plt.axis('off')

            plt.subplot(2, 3, 5)
            plt.imshow(gt, cmap='gray')
            plt.title('GT')
            plt.axis('off')

            plt.subplot(2, 3, 6)
            plt.imshow(diff_1, cmap='gray')
            plt.title('segmentation result ' + str(round(dice_score, 3)))
            plt.axis('off')

            plt.tight_layout()
            plt.savefig(os.path.join(saved_dir, name))
            plt.close()
        return total_dice

    def restore(self, x):
        x = torch.squeeze(x)
        x = x.data.cpu()
        for t, m, s in zip([x], [self.mean], [self.std]):
            t.mul_(s).add_(m)
        # transform Tensor to numpy
        x = x.numpy()
        # x = np.transpose(x, (1, 2, 0))
        x = np.clip(x * 255, 0, 255).astype(np.uint8)
        return x

    def dice_score(self, X, Y):
        assert X.shape == Y.shape
        return self.dice_coeff(X, Y)

    def dice_coeff(self, pred, target):
        smooth = 1.
        assert pred.shape == target.shape
        # num = pred.size(0)
        m1 = pred  # Flatten
        m2 = target  # Flatten
        intersection = (m1 * m2).sum()

        return (2. * intersection + smooth) / (m1.sum() + m2.sum() + smooth)

    def save_prediction_label(self, diff, gt):
        predict = (diff / 255.).reshape(-1)
        label = (gt / 255.).reshape(-1)
        self.prediction.extend(list(predict))
        self.label.extend(list(label))

    def show_PR_curve(self):
        print(len(self.label), len(self.prediction))
        precision, recall, thresholds = precision_recall_curve(self.label, self.prediction)
        pr_auc = auc(recall, precision)
        print('auc:', pr_auc)
        plt.plot(recall, precision, color='blue')
        plt.title('Precision/Recall Curve')
        plt.xlabel('Recall')
        plt.ylabel('Precision')
        plt.legend(loc="upper right", labels=['Ours(AUC={})'.format(round(pr_auc, 3))])
        plt.savefig(os.path.join(self.args.save_dir, 'PR_curve.png'))
        plt.show()

        np.savez(os.path.join(self.args.save_dir, 'PR_data.npz'), precision=precision, recall=recall)

    def predictor(self):
        self.auto_encoder.eval()

        save_dir = self.args.save_dir
        print(save_dir)
        total_num = 0
        total_dice = 0

        for idx, item in tqdm(enumerate(self.dataloader), desc='processing result...', total=len(self.dataloader),
                              ncols=100):
            inputs, names = item['image'], item['name']
            # print(inputs.shape, names)
            total_num += len(names)
            batch_dice = self.save_single_image(save_dir, names, inputs)
            total_dice += batch_dice
        print('average dice:', total_dice / total_num)

if __name__ == '__main__':
    args = parse_args()
    predict = Predict(args)
    predict.predictor()
    predict.show_PR_curve()