# coding=utf-8
# Androxus bot
# Botinfo.py

__author__ = 'Rafael'

from datetime import datetime
from os import getpid
from sys import version

import discord
import psutil
from discord.ext import commands
from stopwatch import Stopwatch

from Classes import Androxus
from database.Conexao import Conexao
from database.Repositories.ComandoDesativadoRepository import ComandoDesativadoRepository
from database.Repositories.ComandoPersonalizadoRepository import ComandoPersonalizadoRepository
from database.Repositories.InformacoesRepository import InformacoesRepository
from database.Repositories.ServidorRepository import ServidorRepository
from utils.Utils import get_last_commit, capitalize, pegar_o_prefixo
from utils.Utils import get_last_update, datetime_format
from utils.Utils import random_color

class Botinfo(commands.Cog, command_attrs=dict(category='bot_info')):
    def __init__(self, bot):
        self.bot = bot

    @commands.command(name='botinfo',
                      aliases=['info', 'detalhes'],
                      description='Mostra algumas informações sobre mim!',
                      examples=['``{prefix}botinfo``'],
                      cls=Androxus.Command)
    @commands.max_concurrency(1, commands.BucketType.user)
    async def _botinfo(self, ctx):
        stopwatch_banco = Stopwatch()
        conexao = Conexao()
        stopwatch_banco.stop()
        # se a pessoa usou o comando, mencionando o bot:
        if ctx.prefix.replace("!", "").replace(" ", "") == self.bot.user.mention:
            # vai pegar o prefixo que está no banco
            prefixo = pegar_o_prefixo(self.bot, ctx, False, conexao)
        else:
            # se a pessoa não marcou o bot:
            prefixo = ctx.prefix
        embed = discord.Embed(title=f'<:Info:756712227221930102> Detalhes sobre mim!',
                              colour=discord.Colour(random_color()),
                              description=f'Caso você queira saber meus outros comandos, use ``{prefixo}cmds``!',
                              timestamp=datetime.utcnow())
        embed.set_author(name='Androxus',
                         icon_url=self.bot.user.avatar_url)
        embed.set_footer(text=f'{ctx.author}',
                         icon_url=ctx.author.avatar_url)
        embed.add_field(name=':calendar: Data que fui criado:',
                        value=f'``{self.bot.user.created_at.strftime("%d/%m/%Y")}``',
                        inline=True)
        embed.add_field(name=':older_adult: Idade:',
                        value=f'``{datetime_format(self.bot.user.created_at)}``',
                        inline=True)
        embed.add_field(name=':robot: Meu perfil:',
                        value=f'``{str(self.bot.user)}``',
                        inline=True)
        embed.add_field(name=':id: Meu id:',
                        value=f'``{self.bot.user.id}``',
                        inline=True)
        embed.add_field(name='<:WumpusCrown:756712226978660392> Meu dono:',
                        value=f'``{self.bot.get_user(self.bot.owner_id)}``',
                        inline=True)
        embed.add_field(name='<a:pato:755774683348992060> Quantos servidores estão me usando:',
                        value=f'``{len(self.bot.guilds)}``',
                        inline=True)
        embed.add_field(name='<a:parrot_dancando:755774679670718575> Quantas pessoas têm acesso a mim:',
                        # observer, que aqui, estamos pegando a lista de membros e jogando para um set
                        # pois, o "set" não permite que haja itens duplicados, ou seja, fazendo desta forma
                        # cada item vai ser único
                        value=f'``{str(len(set(self.bot.get_all_members())))}``',
                        inline=True)
        embed.add_field(name=':ping_pong: Latência da API:',
                        value=f'``{int(self.bot.latency * 1000)}ms``',
                        inline=True)
        embed.add_field(name='<:DatabaseCheck:756712226303508530> Tempo para se conectar ao banco:',
                        value=f'``{stopwatch_banco}``',
                        inline=True)
        this_process = psutil.Process(getpid())
        embed.add_field(name='<a:loading:756715436149702806> Uso da CPU:',
                        value=f'``{this_process.cpu_percent():.2f}%``',
                        inline=True)
        embed.add_field(name=':frog: Memória RAM:',
                        value=f'``{(this_process.memory_info().rss / (1e+6)):.2f}Mb' +
                              '/100Mb``',
                        inline=True)
        embed.add_field(name='<:WumpusPizza:756712226710356122> Versão do discord.py:',
                        value=f'``{discord.__version__}``',
                        inline=True)
        embed.add_field(name='<:python:756712226210971660> Versão do python:',
                        value=f'``{version[0:5]}``',
                        inline=True)
        embed.add_field(name=':bank: Banco de dados que estou usando:',
                        value=f'``{InformacoesRepository().get_sql_version(conexao)[:15]}``',
                        inline=True)
        comandos = 0
        for cog in self.bot.cogs:  # adiciona os comandos padrões no embed
            for command in self.bot.get_cog(cog).get_commands():
                if not command.hidden:  # se o comando não estiver privado
                    comandos += 1
        embed.add_field(name=':desktop: Quantos comandos eu tenho:',
                        value=f'``{comandos}``')
        embed.add_field(name=':stopwatch: Quando eu liguei:',
                        value=f'``{datetime_format(self.bot.uptime)}``',
                        inline=True)
        embed.add_field(name=':watch: Última atualização que tive foi:',
                        value=f'``{datetime_format(get_last_update())}``',
                        inline=True)
        await ctx.send(embed=embed)
        conexao.fechar()

    @commands.command(name='source',
                      aliases=['github', 'programação'],
                      description='Mostra o meu código fonte!',
                      examples=['``{prefix}source``'],
                      cls=Androxus.Command)
    async def _source(self, ctx):
        embed = discord.Embed(title=f'Olá {ctx.author.name}, eu sou um bot feito em python, com ' +
                                    'a API do discord e um banco de dados!',
                              colour=discord.Colour(random_color()),
                              description='Caso você queira ver o meu código fonte, clique [aqui]' +
                                          '(https://github.com/devRMA/Androxus)\n' +
                                          'Caso você queira ver a documentação da API do discord ' +
                                          'para python, clique [aqui](https://discordpy.readthedo' +
                                          'cs.io/en/latest/index.html).',
                              timestamp=datetime.utcnow())
        embed.set_author(name='Androxus', icon_url=f'{self.bot.user.avatar_url}')
        embed.set_footer(text=f'{ctx.author}', icon_url=f'{ctx.author.avatar_url}')
        await ctx.send(embed=embed)

    @commands.command(name='ping',
                      aliases=['latency', 'latência'],
                      description='Mostra a minha latência atual.',
                      examples=['``{prefix}ping``'],
                      cls=Androxus.Command)
    async def _ping(self, ctx):
        from stopwatch import Stopwatch
        stopwatch_banco = Stopwatch()
        conexao = Conexao()
        stopwatch_banco.stop()
        conexao.fechar()
        stopwatch_message = Stopwatch()
        mensagem_do_bot = await ctx.send(f'Calculando ping...')
        stopwatch_message.stop()
        await mensagem_do_bot.edit(content=f'Latência da API do discord: {(self.bot.latency * 1000):.2f}ms!\n'
                                           f'Tempo para se conectar ao banco de dados: {str(stopwatch_banco)}!\n'
                                           f'Tempo para enviar uma mensagem: {str(stopwatch_message)}!\n'
                                           '<a:hello:755774680949850173>')

    @commands.command(name='invite',
                      aliases=['convidar', 'convite'],
                      description='Mostra o link que você usa para me adicionar em seu servidor',
                      examples=['``{prefix}invite``'],
                      cls=Androxus.Command)
    async def _invite(self, ctx):
        e = discord.Embed(title=f'Invite',
                          colour=discord.Colour(random_color()),
                          description=f'Clique [aqui](https://discord.com/oauth2/authorize?client_id={self.bot.user.id}'
                                      '&scope=bot&permissions=604892375) para me adicionar em outro servidor! <a:love:7'
                                      '63523322675068958>',
                          timestamp=datetime.utcnow())
        await ctx.send(embed=e)

    @commands.command(name='changelog',
                      aliases=['ultima_att', 'última_att', 'att_log'],
                      description='Mostra quando foi a minha última atualização e ainda mostra o que foi alterado.',
                      examples=['``{prefix}changelog``'],
                      cls=Androxus.Command)
    @commands.max_concurrency(1, commands.BucketType.user)
    async def _changelog(self, ctx):
        embed = discord.Embed(title=f'Ultima atualização que eu tive:',
                                colour=discord.Colour(random_color()),
                                description=f'```{get_last_commit()}```',
                                timestamp=datetime.utcnow())
        embed.set_footer(text=f'{ctx.author}',
                            icon_url=ctx.author.avatar_url)
        embed.add_field(name='Atualização feita em:',
                        value=f'{datetime_format(get_last_update())}',
                        inline=True)
        await ctx.send(embed=embed)

    @commands.command(name='uptime',
                      aliases=['tempo_on'],
                      description='Mostra a quanto tempo eu estou online!',
                      examples=['``{prefix}uptime``'],
                      cls=Androxus.Command)
    async def _uptime(self, ctx):
        embed = discord.Embed(title=f':timer: Quando eu liguei:',
                              description=f'``{datetime_format(self.bot.uptime)}``',
                              colour=discord.Colour(random_color()),
                              timestamp=datetime.utcnow())
        embed.set_author(name='Androxus', icon_url=f'{self.bot.user.avatar_url}')
        embed.set_footer(text=f'{ctx.author}', icon_url=f'{ctx.author.avatar_url}')
        await ctx.send(embed=embed)

    @commands.command(name='cmds',
                      aliases=['comandos', 'listar_comandos', 'list_cmds'],
                      description='A lista com todos os comandos que eu tenho!',
                      examples=['``{prefix}cmds``', '``{prefix}comandos``'],
                      cls=Androxus.Command)
    @commands.max_concurrency(1, commands.BucketType.user)
    async def _cmds(self, ctx):
        conexao = Conexao()
        e = discord.Embed(title='Todos os meus comandos',
                          colour=discord.Colour(random_color()),
                          description=f'Caso você queira saber mais informações sobre um comando, '
                                      'digite \'help comando\'',
                          timestamp=datetime.utcnow())
        e.set_author(name='Androxus', icon_url=self.bot.user.avatar_url)
        e.set_footer(text=f'{ctx.author}', icon_url=ctx.author.avatar_url)
        categories = self.bot.get_all_categories()
        servidor = None
        if ctx.guild:
            servidor = ServidorRepository().get_servidor(conexao, ctx.guild.id)
            comandos_desativados = [c.comando for c in ComandoDesativadoRepository().get_commands(conexao, servidor)]
        for category in categories:
            commands = self.bot.get_commands_from_category(category)
            if len(commands) != 0:
                for i in range(len(commands)):
                    commands[i] = f'``{commands[i]}``'
                if servidor:
                    # vai remover todos os comandos desativados, da lista que vai aparecer na mensagem
                    for cmds_off in comandos_desativados:
                        if cmds_off in commands:
                            try:
                                commands = commands.remove(cmds_off)
                            except:
                                pass
                e.add_field(
                    name=f'{self.bot.get_emoji_from_category(category)} {capitalize(category)} ({len(commands)})',
                    value=f'{", ".join(commands)}.',
                    inline=False)
        if servidor:
            cmds_personalizados = ComandoPersonalizadoRepository().get_commands(conexao, servidor)
            commands = []
            if len(cmds_personalizados) >= 1:
                for comando_personalizado in cmds_personalizados:
                    commands.append(f'``{comando_personalizado.comando}``')
                commands.sort()
                e.add_field(name=f'{self.bot.get_emoji_from_category("personalizado")} Comandos personalizados (são '
                                 'comandos exclusivos deste servidor, e não precisam do prefixo)'
                                 f'({len(commands)})',
                            value=f'{", ".join(commands)}.',
                            inline=False)
        conexao.fechar()
        await ctx.send(embed=e)

def setup(bot):
    bot.add_cog(Botinfo(bot))


import os
import time
from collections import deque
import pickle

import numpy as np
import tensorflow as tf
from mpi4py import MPI

from baselines.ddpg.ddpg import DDPG
import baselines.common.tf_util as tf_util
from baselines import logger

def train(env, nb_epochs, nb_epoch_cycles, render_eval, reward_scale, render, param_noise, actor, critic,
          normalize_returns, normalize_observations, critic_l2_reg, actor_lr, critic_lr, action_noise,
          popart, gamma, clip_norm, nb_train_steps, nb_rollout_steps, nb_eval_steps, batch_size, memory,
          tau=0.01, eval_env=None, param_noise_adaption_interval=50):
    """
    Runs the training of the Deep Deterministic Policy Gradien (DDPG) model

    DDPG: https://arxiv.org/pdf/1509.02971.pdf

    :param env: (Gym Environment) the environment
    :param nb_epochs: (int) the number of training epochs
    :param nb_epoch_cycles: (int) the number cycles within each epoch
    :param render_eval: (bool) enable rendering of the evalution environment
    :param reward_scale: (float) the value the reward should be scaled by
    :param render: (bool) enable rendering of the environment
    :param param_noise: (AdaptiveParamNoiseSpec) the parameter noise type (can be None)
    :param actor: (TensorFlow Tensor) the actor model
    :param critic: (TensorFlow Tensor) the critic model
    :param normalize_returns: (bool) should the critic output be normalized
    :param normalize_observations: (bool) should the observation be normalized
    :param critic_l2_reg: (float) l2 regularizer coefficient
    :param actor_lr: (float) the actor learning rate
    :param critic_lr: (float) the critic learning rate
    :param action_noise: (ActionNoise) the action noise type (can be None)
    :param popart: (bool) enable pop-art normalization of the critic output
        (https://arxiv.org/pdf/1602.07714.pdf)
    :param gamma: (float) the discount rate
    :param clip_norm: (float) clip the gradients (disabled if None)
    :param nb_train_steps: (int) the number of training steps
    :param nb_rollout_steps: (int) the number of rollout steps
    :param nb_eval_steps: (int) the number of evalutation steps
    :param batch_size: (int) the size of the batch for learning the policy
    :param memory: (Memory) the replay buffer
    :param tau: (float) the soft update coefficient (keep old values, between 0 and 1)
    :param eval_env: (Gym Environment) the evaluation environment (can be None)
    :param param_noise_adaption_interval: (int) apply param noise every N steps
    """
    rank = MPI.COMM_WORLD.Get_rank()

    assert (np.abs(env.action_space.low) == env.action_space.high).all()  # we assume symmetric actions.
    max_action = env.action_space.high
    logger.info('scaling actions by {} before executing in env'.format(max_action))
    agent = DDPG(actor, critic, memory, env.observation_space.shape, env.action_space.shape, param_noise=param_noise,
                 action_noise=action_noise, gamma=gamma, tau=tau, normalize_returns=normalize_returns,
                 enable_popart=popart, normalize_observations=normalize_observations, batch_size=batch_size,
                 critic_l2_reg=critic_l2_reg, actor_lr=actor_lr, critic_lr=critic_lr, clip_norm=clip_norm,
                 reward_scale=reward_scale)
    logger.info('Using agent with the following configuration:')
    logger.info(str(agent.__dict__.items()))

    # Set up logging stuff only for a single worker.
    if rank == 0:
        tf.train.Saver()

    eval_episode_rewards_history = deque(maxlen=100)
    episode_rewards_history = deque(maxlen=100)
    with tf_util.single_threaded_session() as sess:
        # Prepare everything.
        agent.initialize(sess)
        sess.graph.finalize()

        agent.reset()
        obs = env.reset()
        if eval_env is not None:
            eval_obs = eval_env.reset()
        episode_reward = 0.
        episode_step = 0
        episodes = 0
        step = 0

        start_time = time.time()

        epoch_episode_rewards = []
        epoch_episode_steps = []
        epoch_actions = []
        epoch_qs = []
        epoch_episodes = 0
        for epoch in range(nb_epochs):
            for _ in range(nb_epoch_cycles):
                # Perform rollouts.
                for _ in range(nb_rollout_steps):
                    # Predict next action.
                    action, q_value = agent.policy(obs, apply_noise=True, compute_q=True)
                    assert action.shape == env.action_space.shape

                    # Execute next action.
                    if rank == 0 and render:
                        env.render()
                    assert max_action.shape == action.shape
                    # scale for execution in env (as far as DDPG is concerned, every action is in [-1, 1])
                    new_obs, reward, done, _ = env.step(max_action * action)
                    step += 1
                    if rank == 0 and render:
                        env.render()
                    episode_reward += reward
                    episode_step += 1

                    # Book-keeping.
                    epoch_actions.append(action)
                    epoch_qs.append(q_value)
                    agent.store_transition(obs, action, reward, new_obs, done)
                    obs = new_obs

                    if done:
                        # Episode done.
                        epoch_episode_rewards.append(episode_reward)
                        episode_rewards_history.append(episode_reward)
                        epoch_episode_steps.append(episode_step)
                        episode_reward = 0.
                        episode_step = 0
                        epoch_episodes += 1
                        episodes += 1

                        agent.reset()
                        obs = env.reset()

                # Train.
                epoch_actor_losses = []
                epoch_critic_losses = []
                epoch_adaptive_distances = []
                for t_train in range(nb_train_steps):
                    # Adapt param noise, if necessary.
                    if memory.nb_entries >= batch_size and t_train % param_noise_adaption_interval == 0:
                        distance = agent.adapt_param_noise()
                        epoch_adaptive_distances.append(distance)

                    critic_loss, actor_loss = agent.train()
                    epoch_critic_losses.append(critic_loss)
                    epoch_actor_losses.append(actor_loss)
                    agent.update_target_net()

                # Evaluate.
                eval_episode_rewards = []
                eval_qs = []
                if eval_env is not None:
                    eval_episode_reward = 0.
                    for _ in range(nb_eval_steps):
                        eval_action, eval_q = agent.policy(eval_obs, apply_noise=False, compute_q=True)
                        # scale for execution in env (as far as DDPG is concerned, every action is in [-1, 1])
                        eval_obs, eval_r, eval_done, _ = eval_env.step(max_action * eval_action)
                        if render_eval:
                            eval_env.render()
                        eval_episode_reward += eval_r

                        eval_qs.append(eval_q)
                        if eval_done:
                            eval_obs = eval_env.reset()
                            eval_episode_rewards.append(eval_episode_reward)
                            eval_episode_rewards_history.append(eval_episode_reward)
                            eval_episode_reward = 0.

            mpi_size = MPI.COMM_WORLD.Get_size()
            # Log stats.
            # XXX shouldn't call np.mean on variable length lists
            duration = time.time() - start_time
            stats = agent.get_stats()
            combined_stats = stats.copy()
            combined_stats['rollout/return'] = np.mean(epoch_episode_rewards)
            combined_stats['rollout/return_history'] = np.mean(episode_rewards_history)
            combined_stats['rollout/episode_steps'] = np.mean(epoch_episode_steps)
            combined_stats['rollout/actions_mean'] = np.mean(epoch_actions)
            combined_stats['rollout/Q_mean'] = np.mean(epoch_qs)
            combined_stats['train/loss_actor'] = np.mean(epoch_actor_losses)
            combined_stats['train/loss_critic'] = np.mean(epoch_critic_losses)
            combined_stats['train/param_noise_distance'] = np.mean(epoch_adaptive_distances)
            combined_stats['total/duration'] = duration
            combined_stats['total/steps_per_second'] = float(step) / float(duration)
            combined_stats['total/episodes'] = episodes
            combined_stats['rollout/episodes'] = epoch_episodes
            combined_stats['rollout/actions_std'] = np.std(epoch_actions)
            # Evaluation statistics.
            if eval_env is not None:
                combined_stats['eval/return'] = eval_episode_rewards
                combined_stats['eval/return_history'] = np.mean(eval_episode_rewards_history)
                combined_stats['eval/Q'] = eval_qs
                combined_stats['eval/episodes'] = len(eval_episode_rewards)

            def as_scalar(scalar):
                """
                check and return the input if it is a scalar, otherwise raise ValueError

                :param scalar: (Any) the object to check
                :return: (Number) the scalar if x is a scalar
                """
                if isinstance(scalar, np.ndarray):
                    assert scalar.size == 1
                    return scalar[0]
                elif np.isscalar(scalar):
                    return scalar
                else:
                    raise ValueError('expected scalar, got %s' % scalar)
            combined_stats_sums = MPI.COMM_WORLD.allreduce(np.array([as_scalar(x) for x in combined_stats.values()]))
            combined_stats = {k: v / mpi_size for (k, v) in zip(combined_stats.keys(), combined_stats_sums)}

            # Total statistics.
            combined_stats['total/epochs'] = epoch + 1
            combined_stats['total/steps'] = step

            for key in sorted(combined_stats.keys()):
                logger.record_tabular(key, combined_stats[key])
            logger.dump_tabular()
            logger.info('')
            logdir = logger.get_dir()
            if rank == 0 and logdir:
                if hasattr(env, 'get_state'):
                    with open(os.path.join(logdir, 'env_state.pkl'), 'wb') as file_handler:
                        pickle.dump(env.get_state(), file_handler)
                if eval_env and hasattr(eval_env, 'get_state'):
                    with open(os.path.join(logdir, 'eval_env_state.pkl'), 'wb') as file_handler:
                        pickle.dump(eval_env.get_state(), file_handler)


from django.conf.urls import url
from . import views

urlpatterns = [
    # Examples:
    # url(r'^$', 'ecommerce.views.home', name='home'),
    # url(r'^blog/', include('blog.urls')),
    url(r'^$', views.show_menus, name='show_menus'),
    url(r'^spide_menus/$', views.spide_menus, name='spide_menus'),
    url(r'^menu_page/(?P<menu_folder>[a-z_0-9]+)/$', views.menu_page, name='menu_page'),
    url(r'^delete_menu/(?P<menu_folder>[a-z_0-9]+)/$', views.delete_menu, name='delete_menu'),
]


"""
This file may not be shared/redistributed without permission. Please read copyright notice in the git repo. If this file contains other copyright notices disregard this text.
"""
import numpy as np
import matplotlib.pyplot as plt
from tqdm import tqdm
from irlc import savepdf
from irlc.ex01.agent import train
from irlc.ex12.semi_grad_nstep_sarsa import LinearSemiGradSarsaN
import gym
from irlc import main_plot
from irlc.ex10.mc_evaluate_blackjack import plot_surface_2
from irlc.ex12.semi_grad_sarsa_lambda import LinearSemiGradSarsa

def plot_mountaincar_value_function(env, value_function, ax):
    """
    3d plot
    """
    grid_size = 40
    low = env.unwrapped.observation_space.low
    high = env.unwrapped.observation_space.high
    X,Y = np.meshgrid( np.linspace(low[0], high[0], grid_size), np.linspace(low[1], high[1], grid_size)  )
    Z = X*0
    for i, (x,y) in enumerate(zip(X.flat, Y.flat)):
        Z.flat[i] = value_function([x,y])

    plot_surface_2(X,Y,Z,ax=ax)
    ax.set_xlabel('Position')
    ax.set_ylabel('Velocity')
    ax.set_zlabel('Cost to go')

def figure_10_1():
    episodes = 9000
    plot_episodes = [1, 99, episodes - 1]
    scale = 8
    fig = plt.figure(figsize=(4*scale, scale))
    axes = [fig.add_subplot(1, len(plot_episodes), i+1, projection='3d') for i in range(len(plot_episodes))]
    num_of_tilings = 8
    alpha = 0.3

    env = gym.make("MountainCar-v0")
    agent = LinearSemiGradSarsa(env, gamma=1, alpha=alpha/num_of_tilings, epsilon=0)
    for ep in tqdm(range(episodes)):
        train(env, agent, num_episodes=1, max_steps=np.inf, verbose=False)
        if ep in plot_episodes:
            v = lambda s: -agent.v(s)
            ax = axes[plot_episodes.index(ep)]
            plot_mountaincar_value_function(env, v, ax=ax)
            ax.set_title('Episode %d' % (ep + 1))

    from irlc import savepdf
    savepdf("semigrad_sarsa_10-1")
    plt.show()

def figure_10_2():
    episodes = 500
    num_of_tilings = 8
    alphas = [0.1, 0.2, 0.5]
    env = gym.make("MountainCar500-v0")

    experiments = []
    for alpha in alphas:
        agent = LinearSemiGradSarsa(env, gamma=1, alpha=alpha / num_of_tilings, epsilon=0)
        experiment = f"experiments/mountaincar_10-2_{agent}_{episodes}"
        train(env, agent, experiment_name=experiment, num_episodes=episodes,max_runs=10)
        experiments.append(experiment)

    main_plot(experiments=experiments, y_key="Length")
    plt.xlabel('Episode')
    plt.ylabel('Steps per episode')
    plt.title(env.spec._env_name + " - Semigrad Sarsa - Figure 10.2")
    savepdf("mountaincar_10-2")
    plt.show()

def figure_10_3():
    from irlc.ex12.semi_grad_sarsa_lambda import LinearSemiGradSarsaLambda
    from irlc.ex11.semi_grad_q import LinearSemiGradQAgent

    max_runs = 10
    episodes = 500
    num_of_tilings = 8
    alphas = [0.5, 0.3]
    n_steps = [1, 8]

    env = gym.make("MountainCar500-v0")
    experiments = []

    """ Plot results of experiments here. """
    # TODO: 16 lines missing.
    raise NotImplementedError("")

    main_plot(experiments=experiments, y_key="Length")
    plt.xlabel('Episode')
    plt.ylabel('Steps per episode')
    plt.title(env.spec._env_name + " - Semigrad N-step Sarsa - Figure 10.3")
    savepdf("mountaincar_10-3")
    plt.show()

def figure_10_4():
    from irlc import log_time_series
    alphas = np.arange(0.25, 1.75, 0.25)
    n_steps = np.power(2, np.arange(0, 5))
    episodes = 50
    env = gym.make("MountainCar500-v0")
    experiments = []
    num_of_tilings = 8
    max_asteps = 500
    run = True
    for n_step_index, n_step in enumerate(n_steps):
        aexp = []
        did_run = False
        for alpha_index, alpha in enumerate(alphas):
            if not run:
                continue
            if (n_step == 8 and alpha > 1) or (n_step == 16 and alpha > 0.75):
                # In these cases it won't converge, so ignore them
                asteps = max_asteps #max_steps * episodes
            else:
                n = n_step
                agent = LinearSemiGradSarsaN(env, gamma=1, alpha=alpha / num_of_tilings, epsilon=0, n=n)
                _, stats, _ = train(env, agent, num_episodes=episodes)
                asteps = np.mean( [s['Length'] for s in stats] )
                did_run = did_run or stats is not None

            aexp.append({'alpha': alpha, 'average_steps': asteps})

        experiment = f"experiments/mc_10-4_lsgn_{n_step}"
        experiments.append(experiment)
        if did_run:
            log_time_series(experiment, aexp)

    main_plot(experiments, x_key="alpha", y_key="average_steps", ci=None)
    plt.xlabel('alpha')
    plt.ylabel('Steps per episode')
    plt.title("Figure 10.4: Semigrad n-step Sarsa on mountain car")
    plt.ylim([150, 300])
    savepdf("mountaincar_10-4")
    plt.show()

if __name__ == '__main__':
    # figure_10_1()
    # figure_10_2()
    figure_10_3()
    # figure_10_4()


import pickle
from sklearn.ensemble import RandomForestClassifier
import numpy as np
from sklearn.model_selection import KFold
from sklearn import metrics
import sys
import create_contact_features
import matplotlib.pyplot as plt
import math
from joblib import dump, load
from sklearn.metrics import confusion_matrix
from collections import defaultdict

#from xgboost import XGBClassifier
#import eli5
#from eli5.sklearn import PermutationImportance

alleles = []
allele_indices = []
running_index = 0
print("Reading Data")

data = pickle.load(open("data_ensemble.pkl",'rb'))
X_o = data['X']
y_o = data['y']
peptides = data['peptides']
alleles = data['alleles']

rows_finite = np.isfinite(X_o).all(axis=1)
X_o = X_o[rows_finite]
y_o = y_o[rows_finite]
peptides = peptides[rows_finite]
alleles = alleles[rows_finite]

allele_peptides = defaultdict(lambda: [])
labels = defaultdict(lambda: [])
for i in range(len(peptides)):
    if (i%10000) == 0: print(i)
    ap_name = alleles[i] + "-" + peptides[i]
    allele_peptides[ap_name].append(X_o[i])
    labels[ap_name].append(y_o[i])

X = []
y = []
peptides_n = []
alleles_n = []
for allele_peptide in allele_peptides.keys():
    allele, peptide = allele_peptide.split("-")
    X.append(np.mean(allele_peptides[allele_peptide], axis=0))
    y.append(labels[allele_peptide][0])
    peptides_n.append(peptide)
    alleles_n.append(allele)

X = np.array(X)
y = np.array(y)

print(X.shape, y.shape, len(peptides_n), len(alleles_n))

data = {'X':X, 'y':y, 'peptides':peptides_n, 'alleles':alleles_n}

f = open("data_ensemble_avg.pkl", 'wb')
pickle.dump(data, f)
f.close()


"""Plugwise Binary Sensor component for Home Assistant."""

import logging

import voluptuous as vol

from homeassistant.components.binary_sensor import BinarySensorEntity, DOMAIN as BINARY_SENSOR_DOMAIN
from homeassistant.const import ATTR_DEVICE_CLASS, ATTR_ICON, ATTR_NAME, ATTR_STATE

from homeassistant.core import callback
from homeassistant.helpers import config_validation as cv, entity_platform

from .const import (
    API,
    ATTR_SCAN_DAYLIGHT_MODE,
    ATTR_SCAN_SENSITIVITY_MODE,
    ATTR_SCAN_RESET_TIMER,
    ATTR_SED_STAY_ACTIVE,
    ATTR_SED_SLEEP_FOR,
    ATTR_SED_MAINTENANCE_INTERVAL,
    ATTR_SED_CLOCK_SYNC,
    ATTR_SED_CLOCK_INTERVAL,
    CB_NEW_NODE,
    COORDINATOR,
    DOMAIN,
    FLAME_ICON,
    FLOW_OFF_ICON,
    FLOW_ON_ICON,
    GW_BINARY_SENSORS,
    IDLE_ICON,
    NO_NOTIFICATION_ICON,
    NOTIFICATION_ICON,
    PW_CLASS,
    PW_MODEL,
    PW_TYPE,
    SCAN_SENSITIVITY_MODES,
    SERVICE_CONFIGURE_BATTERY,
    SERVICE_CONFIGURE_SCAN,
    SEVERITIES,
    STICK,
    STICK_API,
    USB,
    USB_AVAILABLE_ID,
    USB_MOTION_ID,
)

from .gateway import SmileGateway
from .usb import NodeEntity

PARALLEL_UPDATES = 0

_LOGGER = logging.getLogger(__name__)

async def async_setup_entry(hass, config_entry, async_add_entities):
    """Set up the Smile switches from a config entry."""
    if hass.data[DOMAIN][config_entry.entry_id][PW_TYPE] == USB:
        return await async_setup_entry_usb(hass, config_entry, async_add_entities)
    # Considered default and for earlier setups without usb/network config_flow
    return await async_setup_entry_gateway(hass, config_entry, async_add_entities)

async def async_setup_entry_usb(hass, config_entry, async_add_entities):
    """Set up Plugwise binary sensor based on config_entry."""
    api_stick = hass.data[DOMAIN][config_entry.entry_id][STICK]
    platform = entity_platform.current_platform.get()

    async def async_add_binary_sensor(mac):
        """Add plugwise binary sensor."""
        if USB_MOTION_ID in api_stick.devices[mac].features:
            _LOGGER.debug("Add binary_sensors for %s", mac)
            async_add_entities([USBBinarySensor(api_stick.devices[mac])])

            # Register services
            platform.async_register_entity_service(
                SERVICE_CONFIGURE_SCAN,
                {
                    vol.Required(ATTR_SCAN_SENSITIVITY_MODE): vol.In(
                        SCAN_SENSITIVITY_MODES
                    ),
                    vol.Required(ATTR_SCAN_RESET_TIMER): vol.All(
                        vol.Coerce(int), vol.Range(min=1, max=240)
                    ),
                    vol.Required(ATTR_SCAN_DAYLIGHT_MODE): cv.boolean,
                },
                "_service_configure_scan",
            )
            platform.async_register_entity_service(
                SERVICE_CONFIGURE_BATTERY,
                {
                    vol.Required(ATTR_SED_STAY_ACTIVE): vol.All(
                        vol.Coerce(int), vol.Range(min=1, max=120)
                    ),
                    vol.Required(ATTR_SED_SLEEP_FOR): vol.All(
                        vol.Coerce(int), vol.Range(min=10, max=60)
                    ),
                    vol.Required(ATTR_SED_MAINTENANCE_INTERVAL): vol.All(
                        vol.Coerce(int), vol.Range(min=5, max=1440)
                    ),
                    vol.Required(ATTR_SED_CLOCK_SYNC): cv.boolean,
                    vol.Required(ATTR_SED_CLOCK_INTERVAL): vol.All(
                        vol.Coerce(int), vol.Range(min=60, max=10080)
                    ),
                },
                "_service_configure_battery_savings",
            )

    for mac in hass.data[DOMAIN][config_entry.entry_id][BINARY_SENSOR_DOMAIN]:
        hass.async_create_task(async_add_binary_sensor(mac))

    def discoved_binary_sensor(mac):
        """Add newly discovered binary sensor."""
        hass.async_create_task(async_add_binary_sensor(mac))

    # Listen for discovered nodes
    api_stick.subscribe_stick_callback(discoved_binary_sensor, CB_NEW_NODE)

async def async_setup_entry_gateway(hass, config_entry, async_add_entities):
    """Set up the Smile binary_sensors from a config entry."""
    api = hass.data[DOMAIN][config_entry.entry_id][API]
    coordinator = hass.data[DOMAIN][config_entry.entry_id][COORDINATOR]

    entities = []
    is_thermostat = api.single_master_thermostat()

    devices = api.get_all_devices()
    for dev_id in devices:
        if devices[dev_id][PW_CLASS] == "heater_central":
            _LOGGER.debug("Plugwise device_class %s found", devices[dev_id][PW_CLASS])
            data = api.get_device_data(dev_id)
            for binary_sensor in GW_BINARY_SENSORS:
                _LOGGER.debug("Binary_sensor: %s", binary_sensor)
                if binary_sensor not in data:
                    continue

                _LOGGER.debug(
                    "Plugwise binary_sensor Dev %s", devices[dev_id][ATTR_NAME]
                )
                entities.append(
                    GwBinarySensor(
                        api,
                        coordinator,
                        devices[dev_id][ATTR_NAME],
                        dev_id,
                        True,
                        binary_sensor,
                        GW_BINARY_SENSORS[binary_sensor],
                        devices[dev_id][PW_MODEL],
                    )
                )
                _LOGGER.info(
                    "Added binary_sensor.%s",
                    f"{devices[dev_id][ATTR_NAME]}_{binary_sensor}",
                )
        if devices[dev_id][PW_CLASS] == "gateway" and is_thermostat is not None:
            _LOGGER.debug("Plugwise device_class %s found", devices[dev_id][PW_CLASS])
            entities.append(
                GwNotifySensor(
                    api,
                    coordinator,
                    devices[dev_id][ATTR_NAME],
                    dev_id,
                    "plugwise_notification",
                    devices[dev_id][PW_MODEL],
                )
            )
            _LOGGER.info(
                "Added binary_sensor.%s",
                f"{devices[dev_id][ATTR_NAME]}_{'plugwise_notification'}",
            )

    async_add_entities(entities, True)

class SmileBinarySensor(SmileGateway):
    """Represent Smile Binary Sensors."""

    def __init__(
        self, api, coordinator, name, dev_id, enabled_default, binary_sensor, key, model
    ):
        """Initialise the binary_sensor."""
        super().__init__(api, coordinator, name, dev_id)

        self._binary_sensor = binary_sensor
        self._enabled_default = enabled_default
        self._model = model

        self._icon = None
        self._is_on = False

        self._name = key[ATTR_NAME] if key else None

        if dev_id == self._api.heater_id:
            self._entity_name = "Auxiliary"

        if not self._name:
            sensorname = binary_sensor.replace("_", " ").title()
            self._name = f"{self._entity_name} {sensorname}"

        if dev_id == self._api.gateway_id:
            self._entity_name = f"Smile {self._entity_name}"

        self._unique_id = f"{dev_id}-{binary_sensor}"

    @property
    def entity_registry_enabled_default(self) -> bool:
        """Return if the entity should be enabled when first added to the entity registry."""
        return self._enabled_default

    @property
    def icon(self):
        """Return the icon of this entity."""
        return self._icon

    @property
    def is_on(self):
        """Return true if the binary sensor is on."""
        return self._is_on

    @callback
    def _async_process_data(self):
        """Update the entity."""
        raise NotImplementedError

class GwBinarySensor(SmileBinarySensor, BinarySensorEntity):
    """Representation of a Gateway binary_sensor."""

    @callback
    def _async_process_data(self):
        """Update the entity."""
        _LOGGER.debug("Update binary_sensor called")
        data = self._api.get_device_data(self._dev_id)

        if self._binary_sensor not in data:
            self.async_write_ha_state()
            return

        self._is_on = data[self._binary_sensor]

        if self._binary_sensor == "dhw_state":
            self._icon = FLOW_ON_ICON if self._is_on else FLOW_OFF_ICON
        if self._binary_sensor == "flame_state":
            self._icon = FLAME_ICON if self._is_on else IDLE_ICON
        if self._binary_sensor == "slave_boiler_state":
            self._icon = FLAME_ICON if self._is_on else IDLE_ICON

        self.async_write_ha_state()

class GwNotifySensor(SmileBinarySensor, BinarySensorEntity):
    """Representation of a Plugwise Notification binary_sensor."""

    def __init__(self, api, coordinator, name, dev_id, binary_sensor, model):
        """Initialise the notification binary_sensor."""
        super().__init__(
            api, coordinator, name, dev_id, False, binary_sensor, None, model
        )

        self._attributes = {}

    @property
    def device_state_attributes(self):
        """Return the state attributes."""
        return self._attributes

    @callback
    def _async_process_data(self):
        """Update the entity."""
        for severity in SEVERITIES:
            self._attributes[f"{severity.upper()}_msg"] = []

        notify = self._api.notifications

        self._is_on = False
        self._icon = NO_NOTIFICATION_ICON

        if notify != {}:
            self._is_on = True
            self._icon = NOTIFICATION_ICON

            for notify_id, details in notify.items():
                for msg_type, msg in details.items():
                    if msg_type not in SEVERITIES:
                        msg_type = "other"

                    self._attributes[f"{msg_type.upper()}_msg"].append(msg)
                    self.hass.components.persistent_notification.async_create(
                        f"{msg_type.title()}: {msg}",
                        "Plugwise Notification:",
                        f"{DOMAIN}.{notify_id}",
                    )

        self.async_write_ha_state()

class USBBinarySensor(NodeEntity, BinarySensorEntity):
    """Representation of a Stick Node Binary Sensor."""

    def __init__(self, node):
        """Initialize a Node entity."""
        super().__init__(node, USB_MOTION_ID)
        self.node_callbacks = (USB_AVAILABLE_ID, USB_MOTION_ID)

    @property
    def is_on(self):
        """Return true if the binary_sensor is on."""
        return getattr(self._node, STICK_API[USB_MOTION_ID][ATTR_STATE])

    @property
    def unique_id(self):
        """Get unique ID."""
        return f"{self._node.mac}-{USB_MOTION_ID}"

    def _service_configure_scan(self, **kwargs):
        """Service call to configure motion sensor of Scan device."""
        sensitivity_mode = kwargs.get(ATTR_SCAN_SENSITIVITY_MODE)
        reset_timer = kwargs.get(ATTR_SCAN_RESET_TIMER)
        daylight_mode = kwargs.get(ATTR_SCAN_DAYLIGHT_MODE)
        _LOGGER.debug(
            "Configure Scan device '%s': sensitivity='%s', reset timer='%s', daylight mode='%s'",
            self.name,
            sensitivity_mode,
            str(reset_timer),
            str(daylight_mode),
        )
        self._node.Configure_scan(reset_timer, sensitivity_mode, daylight_mode)

    def _service_configure_battery_savings(self, **kwargs):
        """Configure battery powered (sed) device service call."""
        stay_active = kwargs.get(ATTR_SED_STAY_ACTIVE)
        sleep_for = kwargs.get(ATTR_SED_SLEEP_FOR)
        maintenance_interval = kwargs.get(ATTR_SED_MAINTENANCE_INTERVAL)
        clock_sync = kwargs.get(ATTR_SED_CLOCK_SYNC)
        clock_interval = kwargs.get(ATTR_SED_CLOCK_INTERVAL)
        _LOGGER.debug(
            "Configure SED device '%s': stay active='%s', sleep for='%s', maintenance interval='%s', clock sync='%s', clock interval='%s'",
            self.name,
            str(stay_active),
            str(sleep_for),
            str(maintenance_interval),
            str(clock_sync),
            str(clock_interval),
        )
        self._node.Configure_SED(
            stay_active,
            maintenance_interval,
            sleep_for,
            clock_sync,
            clock_interval,
        )


import mailchimp
from django.conf import settings
from django.db import models
from django.db.models.signals import pre_delete, pre_save, post_save
from django.dispatch import receiver

import hq.models
from hq.models import Person

class Subscription(models.Model):
    name = models.CharField(max_length=50, unique=True)
    description = models.TextField()
    subscribers = models.ManyToManyField(hq.models.Person,
        related_name='subscriptions', through='UserSubscription')
    mailchimp_id = models.CharField(max_length=32, blank=True)

    def __unicode__(self):
        return self.name

class UserSubscription(models.Model):
    user = models.ForeignKey(hq.models.Person)
    subscription = models.ForeignKey(Subscription)

    class Meta:
        unique_together = ('user', 'subscription')

    def mailchimp_add(self):
        if hasattr(self, 'no_api'):
            return
        listid = self.subscription.mailchimp_id
        if listid:
            try:
                mc = mailchimp.Mailchimp(settings.MAILCHIMP_APIKEY)
                mc.lists.subscribe(
                    listid,
                    {'email': self.user.email},
                    {'FNAME': self.user.given_name,
                     'LNAME': self.user.family_name,
                     'UID': self.user.id},
                    double_optin=False,
                    send_welcome=False)
            except mailchimp.ListAlreadySubscribedError:
                pass

    def mailchimp_remove(self, old_email=None):
        if hasattr(self, 'no_api'):
            return
        listid = self.subscription.mailchimp_id
        if listid:
            try:
                mc = mailchimp.Mailchimp(settings.MAILCHIMP_APIKEY)
                mc.lists.unsubscribe(
                    listid,
                    {'email': old_email if old_email else self.user.email},
                    send_goodbye=False,
                    send_notify=False)
            except (mailchimp.ListNotSubscribedError,
                    mailchimp.EmailNotExistsError):
                pass

    def mailchimp_update(self, old_email=None):
        if hasattr(self, 'no_api'):
            return
        listid = self.subscription.mailchimp_id
        if listid:
            mc = mailchimp.Mailchimp(settings.MAILCHIMP_APIKEY)
            args = {
                'id': listid,
                'email': {'email': self.user.email},
                'merge_vars': {
                    'FNAME': self.user.given_name,
                    'LNAME': self.user.family_name,
                    'UID': self.user.id,
                },
                'update_existing': True,
            }

            if old_email:
                args['email']['email'] = old_email
                args['merge_vars']['new-email'] = self.user.email
            try:
                mc.lists.subscribe(**args)
            except mailchimp.ListAlreadySubscribedError:
                self.mailchimp_remove(old_email=old_email)

@receiver(pre_save, sender=UserSubscription)
def _add_to_mailchimp(sender, instance, **kwargs):
    instance.mailchimp_add()

@receiver(pre_delete, sender=UserSubscription)
def _remove_from_mailchimp(sender, instance, **kwargs):
    instance.mailchimp_remove()

@receiver(pre_save, sender=Person)
def _update_person_mailchimp(sender, instance, **kwargs):
    try:
        old = Person.objects.get(id=instance.id)
    except (Person.DoesNotExist, Person.MultipleObjectsReturned):
        return

    if not instance.email:
        for usub in old.usersubscription_set.all():
            usub.delete()
    elif old.email != instance.email:
        for usub in instance.usersubscription_set.all():
            usub.mailchimp_update(old_email=old.email)

@receiver(post_save, sender=Person)
def _sync_new_person_to_mailchimp(sender, instance, created, **kwargs):
    if created and instance.email:
        mc = mailchimp.Mailchimp(settings.MAILCHIMP_APIKEY)
        for sub in Subscription.objects.exclude(mailchimp_id=''):
            result = mc.lists.member_info(
                sub.mailchimp_id, [{'email': instance.email}])
            if result['data']:
                usub = UserSubscription(user=instance, subscription=sub)
                usub.no_api = True
                usub.save()
                del usub.no_api
                usub.mailchimp_update()


import chess
import random
import os
import time
import copy
from socket import *

BOLD = '\033[1m'
CEND = '\033[0m'
CRED = '\033[31m'
CGRAY = '\033[90m'

def send(sock, move):
    sendData = move
    sock.send(sendData.encode('utf-8'))

def receive(sock):
    recvData = sock.recv(1024)
    decodeMove = recvData.decode('utf-8')
    board.push(chess.Move.from_uci(decodeMove))

# board.move_stack을 사용, 마지막 움직임 출력
def moveStackList():
    moveList=[]
    for i in board.move_stack:
        moveList.append(str(i))
    if len(moveList) > 0 :
        print('\nmove: %s'%(moveList[-1]))

# display
def display():
    reboard = str(board)
    for i in range(len(reboard)):
        if reboard[i].islower() is True:
            print(CRED+BOLD+reboard[i]+CEND, end='')
        elif reboard[i].islower() is False:
            print(reboard[i], end='')
        else:
            print(CGRAY+BOLD+reboard[i]+CEND, end='')
    print()
    moveStackList()

    # p,r,n,b,q,k,P,R,N,B,Q,K = 0
    for i in range(12):
        if piece[i][1] >= 1:
            print('%s: %d'%(piece[i][0],piece[i][1]))

# 잡은 말 count
def capturedCount(before_board, after_board):
    before_list=[0]*12
    after_list=[0]*12

    for i in range(len(before_board)):
        for j in range(len(piece_list)):
            if piece_list[j] is before_board[i]:
                before_list[j] += 1

    for i in range(len(after_board)):
        for j in range(len(piece_list)):
            if piece_list[j] is after_board[i]:
                after_list[j] += 1

    # for i in range(len(before_list)):
    #     if before_list[i] > after_list[i]:
    #         piece[i][1]+=1

    for i in range(len(before_list)):
        if before_list[i] > after_list[i]:
            if (sum(before_list) > sum(after_list)):
                piece[i][1]+=1

port = 8080
serverSock = socket(AF_INET, SOCK_STREAM)
serverSock.bind(('', port))
serverSock.listen(1)
print('%d : LOADING...' % port)
connectionSock, addr = serverSock.accept()
print('CONNECTED in ', str(addr))

board = chess.Board()
piece_list = ['p','r','n','b','q','k','P','R','N','B','Q','K']
piece = []
for i in range(12):
    line = [0, 0]
    piece.append(line)
    piece[i][0]= piece_list[i]

beforeBoard = str(board)
os.system('clear')
display()
while True:
    legal_list = []
    for i in board.legal_moves:
        legal_list.append(str(i))
    rmove = chess.Move.from_uci(random.choice(legal_list))
    board.push(rmove)

    afterBoard = str(board)
    capturedCount(beforeBoard, afterBoard)

    os.system('clear')
    display()

    send(connectionSock, str(rmove))
    beforeBoard = copy.deepcopy(afterBoard)
    if board.is_game_over() is True:
        print()
        print(board.result())
        break

    receive(connectionSock)
    afterBoard = str(board)
    #capturedCount(beforeBoard, afterBoard)
    os.system('clear')
    display()
    if board.is_game_over() is True:
        print()
        print(board.result())
        break


from django.urls import path
from products import views

app_name = 'products'

urlpatterns = [
    path('', views.index, name='index'),
    path('shop/', views.all_products_view, name='all_products'),
    path('shop/<slug>/', views.single_product_view, name='single_product'),
]


#!/usr/bin/env python

from argparse import FileType, ArgumentParser
import csv
from operator import itemgetter
import os

parser = ArgumentParser(description='Calculate the total of a column.');
parser.add_argument("input_csv",
                    type=FileType('rU'),
                    help="CSV containing (sum,product) pairs");
parser.add_argument("output_csv",
                    type=FileType('w'),
                    help="CSV containing one (sum,product) pair");
args = parser.parse_args();

reader = csv.DictReader(args.input_csv);
writer = csv.DictWriter(args.output_csv,
                        ['sum', 'product'],
                        lineterminator=os.linesep)
writer.writeheader()

# Copy first row unchanged
for row in reader:
    writer.writerow(row)
    break

sum_total = sum(map(int, map(itemgetter('sum'), reader)))
product_total = 0
writer.writerow(dict(sum=sum_total, product=product_total))


import cv2
import cv2.aruco
import numpy as np
import sys
# import pygame
import time
import serial
from threading import Thread
ser1 = serial.Serial('/dev/ttyACM0', 9600)
ser2 = serial.Serial('/dev/ttyACM1', 9600)

# alpha-100  beta-150
def direction(img, top_left, top_right, centre, ids):
    x1 = top_left[0]
    y1 = top_left[1]
    x2 = top_right[0]
    y2 = top_right[1]
    s1 = (x1 + x2) / 2
    s2 = (y1 + y2) / 2
    if abs(x2 - x1) <= 15:
        if y1 > y2:
            orient = "west"
        elif y2 > y1:
            orient = "east"
    elif (x2 - x1) > 0:     # upright
        if abs(y2 - y1) <= 15:
            orient = "north"
        elif (y2 > y1):
            orient = "ne"
        elif (y1 > y2):
            orient = "nw"
    elif (x2 - x1) < 0:
        if abs(y2 - y1) <= 15:
            orient = "south"
        elif (y1 > y2):
            orient = "sw"
        elif (y2 > y1):
            orient = "se"
    return orient

def radio1(data, ids):
    if ids == 100:
        if data == 'for':
            print("forward")
            ser1.write('w'.encode())
        if data == 'lt':
            print("left")
            ser1.write('a'.encode())
        if data == 'rev':
            print("reverse")
            ser1.write('s'.encode())
        if data == 'rt':
            print("right")
            ser1.write('d'.encode())
        if data == 'z':
            sys.exit()
        if data == 'c':
            print("not pressed")
            ser1.write('q'.encode())
        time.sleep(0.1)

def radio2(data, ids):
    if ids == 150:
        if data == 'for':
            print("forward")
            ser2.write('w'.encode())
        if data == 'lt':
            print("left")
            ser2.write('a'.encode())
        if data == 'rev':
            print("reverse")
            ser2.write('s'.encode())
        if data == 'rt':
            print("right")
            ser2.write('d'.encode())
        if data == 'z':
            sys.exit()
        if data == 'c':
            print("not pressed")
            ser2.write('q'.encode())
        time.sleep(0.1)

def send_radio(data, ids):
    t1 = Thread(target=radio1, args=(data, ids))
    t2 = Thread(target=radio2, args=(data, ids))
    t1.start()
    t2.start()
    t1.join()
    t2.join()

def orientation(img, top_left, top_right, centre, ids):
    x1 = top_left[0]
    y1 = top_left[1]
    x2 = top_right[0]
    y2 = top_right[1]
    s1 = (x1 + x2) / 2
    s2 = (y1 + y2) / 2
    data = 'for'
    # centre1 = centre[0]
    # centre2 = centre[1]
    # destination coord
    d1 = 250.0
    d2 = 250.0
    orient = ""

    if abs(s1 - d1) <= 15 and abs(s2 - d2) <= 15:
        data = 'c'
    # north
    # orient = direction(img, top_left, top_right, centre, ids)
    if direction(img, top_left, top_right, centre, ids) == "north":
    	send_radio('c', ids)
    else:
        direc = direction(img, top_left, top_right, centre, ids)
        while direc != 'north':
            print("Stuck in loop")
            send_radio('rt', ids)
            send_radio('c', ids)
            direc = direction(img, top_left, top_right, centre, ids)
            break
        send_radio('c', ids)
    # send_radio(data, ids)

if __name__=='__main__':
    cap = cv2.VideoCapture(2)
    dictionary = cv2.aruco.getPredefinedDictionary(cv2.aruco.DICT_6X6_250)
    while True:
        ret, frame = cap.read()
        # getting all the values in different variables
        corners, ids, rejected = cv2.aruco.detectMarkers(frame, dictionary)
        cv2.aruco.drawDetectedMarkers(frame, corners, ids)
        # createGrid(frame)
        cv2.imshow('out', frame)
        if cv2.waitKey(1) & 0xFF == ord('q'):
            break
        if len(corners) > 0:
            # draw only if aruco detected
            i = 0

            # nested loop only for centre coordinates
            while i < len(corners):
                # corners contains 4 coordinates of the aruco (2d array)
                j = 0
                while j < len(corners[i]):
                    # first taking sum of all x coord, then divide by 4. Then y
                    pos = np.sum(corners[i], axis=1) / 4
                    j = j + 1
                    # call the function for position
                    # checkMarker(frame, ids[i], pos)
                    # orient north and move
                    orientation(frame, corners[0][0][0], corners[0][0][1], pos, ids[i])
                i = i + 1
            print("\n")
        # cv2.imshow('out', frame)
        # if cv2.waitKey(1) & 0xFF == ord('q'):
        #     break

    cap.release()
    cv2.destroyAllWindows()


# -*- coding: utf-8 -*-

from time import time
from common import JsonDict, b64decode, b64encode

class ParamError(BaseException):pass

class Base64(str):
    def __repr__(self) -> str:
        return f'[Base64] size:{len(self)}'

def Source(
        id,
        t=None
    ):
    params = JsonDict(type="Source")
    params.id = id
    params.time = t or int(time())
    return params

def Quote(
        sender:int,
        target:int,
        *message:dict,
        id:int=None
    ):
    r'''回复对象
sender  | Int    | 被引用回复的原消息的发送者的QQ号
target  | Int    | 被引用回复的原消息的接收者者的QQ号（或群号）
id      | Int    | 被引用回复的原消息的messageId
message | Object | 被引用回复的原消息的消息链对象'''
    params = JsonDict(type="Quote")
    if sender:params.senderId = sender
    if target:params.targetId = target
    if id:params.id = id
    if message:params.origin = list(message)
    return params

def At(
        id:int
    ) -> list:
    r'''At
target | Int | 群员QQ号'''
    return JsonDict(type="At",target=id)

def AtAll() -> dict:
    return JsonDict(type="AtAll")

def Face(
        id:int=None,
        name:str=None
    ):
    r'''表情
id   | Int | QQ表情编号，可选，优先高于name
name | Str | QQ表情拼音，可选'''
    params = JsonDict(type="Face")
    if id:params.faceId = id
    if name:params.name = name
    return params

def Plain(s:str) -> dict:
    r'''文本
text | Str | 文字消息'''
    return JsonDict(type="Plain",text=str(s))

def Image(
        url:str=None,
        path:str=None,
        base64:str|bytes|dict=None,
        id=None
    ) -> dict:
    r'''图片
url    | Str | 图片的URL，发送时可作网络图片的链接；接收时为腾讯图片服务器的链接，可用于图片下载
path   | Str | 图片的路径，发送本地图片，路径相对于 JVM 工作路径（默认是当前路径，可通过 `-Duser.dir=...`指定），也可传入绝对路径
base64 | Str | 图片的 str 或 bytes 类型的Base64编码或二进制
id     | Str | 图片的id，群图片与好友图片格式不同。不为空时将忽略url属性'''
    params = JsonDict(type="Image")
    if url:params.url = url
    if path:params.path = path
    if id:params.imageId = id
    if base64:
        try:b64decode(base64)
        except: base64 = b64encode(base64)
        params.base64 = Base64(base64)
    return params

def FlashImage(
        url:str=None,
        path:str=None,
        base64:str|bytes|dict=None,
        id=None
    ) -> dict:
    r'''闪图
url    | Str | 图片的URL，发送时可作网络图片的链接；接收时为腾讯图片服务器的链接，可用于图片下载
path   | Str | 图片的路径，发送本地图片，路径相对于 JVM 工作路径（默认是当前路径，可通过 `-Duser.dir=...`指定），也可传入绝对路径
base64 | Str | 图片的 str 或 bytes 类型的Base64编码或二进制
id     | Str | 图片的id，群图片与好友图片格式不同。不为空时将忽略url属性'''
    params = JsonDict(type="FlashImage")
    if url:params.url = url
    if path:params.path = path
    if id:params.imageId = id
    if base64:
        try:b64decode(base64)
        except: base64 = b64encode(base64)
        params.base64 = Base64(base64)
    return params

def Voice(
        url:str=None,
        path:str=None,
        base64:str|bytes|dict=None,
        id=None
    ) -> dict:
    r'''语音
url    | Str | 语音的URL，发送时可作网络语音的链接；接收时为腾讯语音服务器的链接，可用于语音下载
path   | Str | 语音的路径，发送本地语音，路径相对于 JVM 工作路径（默认是当前路径，可通过 `-Duser.dir=...`指定），也可传入绝对路径。
base64 | Str | 语音的 str 或 bytes 类型的Base64编码或二进制
id     | Str | 语音的id，不为空时将忽略url属性

#> 参数任选其一，出现多个参数时，按照id > url > path > base64的优先级'''
    params = JsonDict(type="Voice")
    if url:params.url = url
    if path:params.path = path
    if id:params.voiceId = id
    if base64:
        try:b64decode(base64)
        except: base64 = b64encode(base64)
        params.base64 = Base64(base64)
    return params

def Xml(xml:str) -> dict:
    r'''XML
xml | Str | XML文本'''
    return JsonDict(type="Xml",xml=xml)

def Json(json:str):
    r'''JSON
json | Str | Json文本'''
    return JsonDict(type="Json",json=json)

def App(content:str):
    r'''小程序
content | Str | 内容'''
    return JsonDict(type="App",content=content)

def Poke(name:str):
    r'''戳一戳
name | Str | 戳一戳名称
1. "Poke": 戳一戳
2. "ShowLove": 比心
3. "Like": 点赞
4. "Heartbroken": 心碎
5. "SixSixSix": 666
6. "FangDaZhao": 放大招'''
    return JsonDict(type="Poke",name=name)

def Dice(value:int):
    r'''骰子
value | Int | 点数'''
    return JsonDict(type="Dice",value=value)

def MusicShare(
        kind=None,
        title=None,
        summary=None,
        jumpUrl=None,
        pictureUrl=None,
        musicUrl=None,
        brief=None
    ):
    r'''音乐分享
kind       | Str | 类型
title      | Str | 标题
summary    | Str | 概括
jumpUrl    | Str | 跳转路径
pictureUrl | Str | 封面路径
musicUrl   | Str | 音源路径
brief      | Str | 简介'''
    params = JsonDict(type="MusicShare")
    if kind:params.kind = kind
    if title:params.title = title
    if summary:params.summary = summary
    if jumpUrl:params.jumpUrl = jumpUrl
    if pictureUrl:params.pictureUrl = pictureUrl
    if musicUrl:params.musicUrl = musicUrl
    if brief:params.brief = brief
    return params

def Forward(
    *node:dict,
    title:str=None,
    brief:str=None,
    summary:str=None,
    preview:list=None,
    source:str=None
) -> dict:
    r'''聊天记录
nodeList | object | 消息节点
title    | Str    | 标题（群聊的聊天记录）
brief    | Str    | 简介（[聊天记录]）
summary  | Str    | 总结（查看x条转发消息）
preview  | List   | 预览（["msg1", "msg2", "msg3", "msg4"]）
source   | Str    | 来源（聊天记录）'''
    params = JsonDict(type="Forward",nodeList=list(node))
    if title or brief or source or preview or summary:
        params.display = JsonDict()
        if title:params.display.title = title
        if brief:params.display.brief = brief
        if source:params.display.source = source
        if preview:params.display.preview = preview
        if summary:params.display.summary = summary
    return params

def Node(
    sender:int=None,
    name:str=None,
    *message:dict,
    t:int=None,
    target:int=None,
    id:int=None,
    ref:dict|list|tuple=None
) -> dict:
    r'''聊天记录节点
senderid   | Int             | 发送人QQ号
time       | Int             | 发送时间
senderName | Str             | 显示名称
message    | List            | 消息数组
id         | Int             | 可以只使用消息id，从缓存中读取一条消息作为节点
target     | Int             | 引用的上下文目标，群号、好友账号
ref        | Dict|List|Tuple | {'id' or 'messageId':Int,'target':Int} 或者 [target:Int,messageId:Int]'''
    params = JsonDict()
    if sender:params.senderId = sender
    if not name:name = str(sender)
    if name:params.senderName = name
    params.messageChain = list(message)
    params.time = t or (int(time())if not id else None)

    if ref:
        if type(ref) in [list,tuple]:
            params.messageRef = JsonDict(target=ref[0], messageId=ref[1])
        else:
            if 'messageId' in ref and 'target' in ref:
                params.messageRef = JsonDict(ref)
            elif 'id' in ref and 'target' in ref:
                params.messageRef = JsonDict(target=ref.target, messageId=ref.id)
            else:
                raise ParamError('Error:The key of ref is only (id or messageId) and target')
    elif target:
        if id:
            params.messageRef = {'messageId':id,'target':target}
        else:
            raise ParamError('Error:id is None')
    elif id:
        params.messageId = id
    return params

def File(
    id=None,
    name=None,
    size=None
):
    r'''文件类型
id   | Str | 文件识别id
name | Str | 文件名
size | Int   | 文件大小'''
    return JsonDict({
        "type": "File",
        "id": id,
        "name": name,
        "size": size
    })

def MiraiCode(MiraiCode):
    r'''Mirai命令行
code | Str | MiraiCode
MiraiCode的使用(https://github.com/mamoe/mirai/blob/dev/docs/Messages.md#%E6%B6%88%E6%81%AF%E5%85%83%E7%B4%A0)'''
    return JsonDict(type="MiraiCode",code=MiraiCode)


import direct.directbase.DirectStart
from direct.showbase.DirectObject import DirectObject

class Game(DirectObject):
    def __init__(self):
        self.accept('escape', __import__('sys').exit)
        self.parent = loader.loadModel('models/longslug.egg')
        self.parent.reparentTo(render)
        self.child = loader.loadModel('models/longslug.egg')
        self.child.reparentTo(self.parent)
        self.parent.setColor(1, 0, 0, 1)
        self.child.setColor(0, 0, 1, 1)
        self.parent.setPos(-0.5, 5, 0)
        self.child.setPos(1, 0, 0)

Game()
run()


# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
#
# See LICENSE for more details.
#
# Copyright: Red Hat Inc. 2020
# Author: Amador Pahim <apahim@redhat.com>

"""
The GitHub Mirror endpoints
"""

import hashlib
import logging
import requests

import flask

from ghmirror.app.response import MirrorResponse
from ghmirror.data_structures.monostate import RequestsCache
from ghmirror.data_structures.monostate import StatsCache

logging.basicConfig(level=logging.INFO,
                    format='%(asctime)-15s %(message)s')

LOG = logging.getLogger(__name__)
GH_API = 'https://api.github.com'

APP = flask.Flask(__name__)

@APP.route('/healthz', methods=["GET"])
def healthz():
    """
    Health check endpoint for Kubernetes.
    """
    return flask.Response('OK')

@APP.route('/stats', methods=["GET"])
def stats():
    """
    Cache statistics endpoint.
    """
    stats_cache = StatsCache()

    return flask.jsonify(
        {
            'cache_hit': stats_cache.hits,
            'cache_miss': stats_cache.misses,
        }
    )

@APP.route('/', defaults={'path': ''})
@APP.route('/<path:path>', methods=["GET", "POST", "PUT", "PATCH", "DELETE"])
def ghmirror(path):
    """
    Default endpoint, matching any url without a specific endpoint.
    """
    url = f'{GH_API}/{path}'

    if flask.request.args:
        url += '?'
        for key, value in flask.request.args.items():
            url += f'{key}={value}&'
        url = url.rstrip('&')

    headers = {}
    authorization = flask.request.headers.get('Authorization')
    auth_sha = None
    if authorization is not None:
        headers['Authorization'] = authorization
        # The authorization token will be used as cache key,
        # so let's hash it for additional security.
        auth_sha = hashlib.sha1(authorization.encode()).hexdigest()

    if flask.request.method != 'GET':
        LOG.info('[%s] BYPASS %s', flask.request.method, url)

        resp = requests.request(flask.request.method,
                                url=url,
                                headers=headers,
                                data=flask.request.data)

        return flask.Response(resp.content,
                              resp.status_code)

    stats_cache = StatsCache()
    cache = RequestsCache()

    cache_key = (url, auth_sha)
    if cache_key in cache:
        etag = cache[cache_key].headers.get('ETag')
        if etag is not None:
            headers['If-None-Match'] = etag
        last_mod = cache[cache_key].headers.get('Last-Modified')
        if last_mod is not None:
            headers['If-Modified-Since'] = last_mod

    resp = requests.request(flask.request.method,
                            url=url,
                            headers=headers)

    if resp.status_code != 304:
        LOG.info('[GET] CACHE_MISS %s', url)
        stats_cache.miss()
        # Caching only makes sense when at least one
        # of those headers is present
        if any(['ETag' in resp.headers,
                'Last-Modified' in resp.headers]):
            cache[cache_key] = resp
        mirror_response = MirrorResponse(original_response=resp,
                                         gh_api_url=GH_API,
                                         gh_mirror_url=flask.request.host_url)
    else:
        LOG.info('[GET] CACHE_HIT %s', url)
        stats_cache.hit()
        mirror_response = MirrorResponse(original_response=cache[cache_key],
                                         gh_api_url=GH_API,
                                         gh_mirror_url=flask.request.host_url)

    return flask.Response(mirror_response.content,
                          mirror_response.status_code,
                          mirror_response.headers)

if __name__ == '__main__':  # pragma: no cover
    APP.run(host='127.0.0.1', debug=True, port='8080')


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models, migrations
import tagging.fields

class Migration(migrations.Migration):

    dependencies = [
    ]

    operations = [
        migrations.CreateModel(
            name='Bookmark',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('title', models.CharField(max_length=100, null=True, blank=True)),
                ('slug', models.SlugField(unique=True, verbose_name='slug')),
                ('url', models.URLField(unique=True, verbose_name='url')),
                ('description', models.TextField(verbose_name='description')),
                ('extended', models.TextField(verbose_name='extended', blank=True)),
                ('created', models.DateTimeField(auto_now_add=True, verbose_name='created')),
                ('modified', models.DateTimeField(auto_now=True, verbose_name='modified')),
                ('tags', tagging.fields.TagField(max_length=255, blank=True)),
            ],
            options={
                'db_table': 'bookmarks',
                'verbose_name': 'bookmark',
                'verbose_name_plural': 'bookmarks',
            },
        ),
    ]


from nose.tools import eq_
import libs.aws as aws_lib

def test_get_object_keys():
    """should return all objects in all buckets"""

    # class MockObject(object):
    #     def __init__(self, key, value):
    #         self.key = key
    #         self.value = value

    class MockObjects(object):
        def __init__(self, values):
            self.values = values

        def all(self):
            return self.values

    class MockBucket(object):
        def __init__(self, values):
            self.objects = MockObjects(values)

    _buckets = [
        MockBucket([
            {'key': 'A', 'value': 1},
            {'key': 'B', 'value': 2},
        ]),
        MockBucket([
            {'key': 'C', 'value': 3},
        ]),
    ]

    def extract_objects(r, buckets):
        return buckets.objects.all()

    r = aws_lib.get_object_keys(extract_objects)(_buckets)

    eq_(r[0], 'A')
    eq_(r[1], 'B')
    eq_(r[2], 'C')


#!python3
#Download reddit links to file

import bs4,requests, os
from bs4 import BeautifulSoup

pagefile= requests.get('https://reddit.com')
pagefile.raise_for_status()
soup = BeautifulSoup(pagefile.text,'html.parser')
sall = soup.select('p  a')

file= open('redditLinks.txt','a')

for href in sall:
    file.write(str(href))

file.close()


'''
Manages the translation system.

Last update: 26/04/19
'''
# Dependancies
import gettext, asyncio, time

from data.queries.select.user_info import User_language

async def Translator(client, ctx):
    '''
    This function returns the right translated strings in function of the user's language

    Return: str
    '''
    caller = ctx.message.author
    caller_lang = await User_language(client, caller)
    if(type(caller_lang) == str):
        caller_lang = caller_lang.upper()
    else:
        error_time = time.strftime('%d/%m/%y - %H:%M', time.gmtime())
        print('{} - Error in cogs.utils.translation.translation.Translator : The \'caller_lang\' type is uncorrect'.format(error_time))
        return

    # Get the translations
    if(caller_lang == 'FR'):
        # French translation
        french_t = gettext.translation('fr', localedir= 'locale', languages= ['fr'])
        french_t.install()

        # The '_' will be the function used to translate different strings
        _ = french_t.gettext

        return(_)

    # If the language is the default language ('EN') we don't translate anything
    else:
        gettext.install('locale/pantheist_translation')

        _ = gettext.gettext

        return(_)


import logging

from tornado import web

from .core import RequestHandler, MyApp, Resources

logger = logging.getLogger(__name__)

class Info(RequestHandler):
    def get(self):
        resp = {'ncores': self.server.ncores,
                'nkeys': len(self.server.data),
                'status': self.server.status}
        self.write(resp)

def HTTPWorker(worker):
    application = MyApp(web.Application([
        (r'/info.json', Info, {'server': worker}),
        (r'/resources.json', Resources, {'server': worker}),
        ]))
    return application


import os
import os.path as op
import numpy as np
import re
import csv
import nltk
from collections import defaultdict
from nltk.corpus import stopwords
from nltk.tokenize import word_tokenize

#nqa_dir = os.environ["NARRATIVEQA_DIR"]
mimir_dir = os.environ["MIMIR_DIR"]
data_dir = op.join(mimir_dir, "data")

def map_words_to_named_entities(obj_dict, classes = ["ORG","LOC","PERSON"]):

	word2entity = {}

	for ne_class in classes:
		for idx, obj in obj_dict[ne_class].items():
			for name in obj.name_variants:
				word2entity[name] = idx

	return(word2entity)

def make_dataset_dict():
	data_dir = op.join(mimir_dir, "data")
	id_name_dict = make_id_name_dict()
	summary_csv = op.join(data_dir, "summaries.csv")
	summary_list = csv_to_list(summary_csv)
	name_dataset_dict = {}
	for row in summary_list[1:]:
		try:
			doc_id, corpus_set, _, _ = row
			book_name = id_name_dict[doc_id]
			name_dataset_dict[book_name] = corpus_set #Test, train, valid#
		except KeyError:
			pass
	return name_dataset_dict

def make_qa_dict_valid(qaps_line_list, id_name_dict):
	qa_dict_valid = {}
	for line in qaps_line_list:
		if line[0] in id_name_dict: #if it's a book
			if line[1] == "valid":
				qa_dict_valid[line[2]] = (id_name_dict[line[0]],[line[3],line[4]])
	return(qa_dict_valid)

def tokenize(sent:str):
	return word_tokenize(sent)

def remove_stopwords(sent:list):
	stop_words = set(stopwords.words("english"))
	return ([word for word in sent if word.lower() not in stop_words and word.isalnum()])

def csv_to_list(csv_file_path):

	line_list = []

	with open(csv_file_path, newline="") as csvfile:
		csvreader = csv.reader(csvfile)
		for row in csvreader:
			line_list.append(row)

	return(line_list)

def list_to_csv(csv_file_path, line_list):

	with open(csv_file_path, "w+") as csvfile:
		csvwriter = csv.writer(csvfile)
		for row in line_list:
			csvwriter.writerow(row)

def load_or_create_object(numpy_filename: str, obj: object):
	if not numpy_filename.endswith(".npy"):
		numpy_filename += ".npy"
	if not op.exists(numpy_filename):
		return obj
	else:
		return np.load(numpy_filename, allow_pickle=True)[0]

def make_id_name_dict():
	id_name_dict = {}
	docs_csv = op.join(data_dir, "documents.csv")
	docs_list = csv_to_list(docs_csv)
	for line in docs_list[1:]:
		try:
			if line[2] == "gutenberg":
				id_name_dict[line[0]] = line[6]
		except:
			pass
	return(id_name_dict)

def make_name_url_dict():
	id_name_dict = make_id_name_dict()
	name_url_dict = {}
	id_url_csv = op.join(data_dir, "id_url.csv")
	id_url_list = csv_to_list(id_url_csv)
	for line in id_url_list[1:]:
		try:
			if line[1] == "gutenberg":
				name_url_dict[id_name_dict[line[0]]] = line[2]
		except:
			pass
	return name_url_dict

def map_words_to_named_entities(obj_dict, classes = ["PERSON"]):

	word2entity = {}

	for ne_class in set(classes)&set(obj_dict.keys()):
		for idx, obj in obj_dict[ne_class].items():
			for name in obj.name_variants:
				word2entity[name] = idx

	return(word2entity)

def levenshtein(input_sentence, target_sentence):
	#Calculates the minimum edit distance (Levenshtein distance) between two strings (or lists)
	if type(input_sentence) != type(target_sentence):
		raise TypeError("Input and target should probably be the same type. \
Input (type {}): {} \
Target (type {}): {}".format(type(input_sentence),input_sentence, type(target_sentence),target_sentence))

	trellis = np.zeros((len(input_sentence) + 1, len(target_sentence) + 1))
	trellis[:,0] = np.arange(len(input_sentence)+1)
	trellis[0,:] = np.arange(len(target_sentence)+1)

	for i in range(1, len(input_sentence) + 1):
		w_in = input_sentence[i-1]
		for j in range(1, len(target_sentence) +1):
			w_t = target_sentence[j-1]
			if w_in == w_t:
				square_score = 0
			else:
				square_score = 1
			trellis[i,j] = min(trellis[i-1,j], trellis[i-1,j-1], trellis[i,j-1]) +square_score

	return(trellis[-1,-1])

def download_models():
	nltk.download('stopwords')
	nltk.download('punkt')
	nltk.download('averaged_perceptron_tagger')
	nltk.download('maxent_ne_chunker')
	nltk.download('words')

def get_line_list_from_file(file):
	with open(file) as f:
		line_list = f.readlines()

	return line_list

def file_path_to_text(file):
    line_list = get_line_list_from_file(file)
    raw_text = " ".join([line.strip("\n") for line in line_list])
    return(raw_text)

def get_tokens_from_text(line):
	tokens = nltk.word_tokenize(line)

	return tokens

def get_named_entities(tokens):
	entities = nltk.chunk.ne_chunk(nltk.pos_tag(tokens))
	return entities

def ne_labels_from_file(file_path):
	line_list = get_line_list_from_file(file_path)
	entity_list = []

	all_lines = " ".join([line.strip("\n") for line in line_list])
	entities = get_named_entities(tokenize(all_lines))
	return(entities)

def context_to_tokens(labelled_string: list, stemmer):
	tokens = []
	for item in labelled_string:
		if isinstance(item, nltk.tree.Tree):
			leaves = item.leaves()
			tokens += [stemmer.stem(leaf[0]) for leaf in leaves]
		else:
			tokens.append(stemmer.stem(item[0]))
	return(tokens)

def ne_BOWs_from_file(file_path, context_length=5, stemmer=None):
	ne_labels = ne_labels_from_file(file_path)

	BOWs_dict = defaultdict(list)

	for i, item in enumerate(ne_labels):
		if isinstance(item, nltk.tree.Tree):
			if item.label() == "PERSON":
				person_name = " ".join([leaf[0] for leaf in item.leaves()])
				before_context = ne_labels[i-context_length:i]
				after_context = ne_labels[i+1:i+1+context_length]
				BOWs_dict[person_name] += context_to_tokens(before_context + after_context, stemmer)

	return(BOWs_dict)

def merge_list(list_list):
	return [l for sub_l in list_list for l in sub_l]

def BOWs_to_TFIDF(BOWs_dict):
	types = list(set(merge_list(list(BOWs_dict.values()))))
	DFs = defaultdict(int)
	for word in types:
		for BOW in BOWs_dict.values():
			if word in BOW:
				DFs[word] += 1

	person_TFs_dict = {}
	for person in BOWs_dict.keys():
		TFs = defaultdict(int)
		BOW = BOWs_dict[person]
		for token in BOW:
			TFs[token] += 1
		token_sum = sum(TFs.values())
		for token in TFs:
			TFs[token] = TFs[token]/token_sum
		person_TFs_dict[person] = TFs

	tf_idf_dict = defaultdict(dict)
	for person, tf_dict in person_TFs_dict.items():
		for word, tf in tf_dict.items():
			tf_idf_dict[person][word] = tf * (1/DFs[word])

#	print(tf_idf_dict)
#	import pdb; pdb.set_trace()
	return(tf_idf_dict)

def ne_tokens_from_file(file_path, pause=False):
	line_list = get_line_list_from_file(file_path)
	entity_list = []

	for line in line_list: #for each sentence in the input file
		line.strip("\n")
		t = get_tokens_from_text(line) #get tokens
		entities = get_named_entities(t) # get entities tree
		if pause == True:
			print(entities)

			stop = input("stop? y/n")
			if stop == "y":
				import pdb; pdb.set_trace()

		for e in entities:
			if isinstance(e, nltk.tree.Tree): #it's an entity
				ent_string = " ".join([leaf[0] for leaf in e.leaves()])
				ent_tuple = (ent_string, e.label())

				entity_list.append(ent_tuple)
	return(entity_list)

def list_to_dict(l:list):
	out_dict = defaultdict(int)

	for thing in l:
		out_dict[thing] += 1
	return(out_dict)

def ne_list_from_file(file_path, pause=False):
	line_list = get_line_list_from_file(file_path)
	entity_list = []

	for line in line_list: #for each sentence in the input file
		line.strip("\n")
		t = get_tokens_from_text(line) #get tokens
		entities = get_named_entities(t) # get entities tree
		if pause == True:
			print(entities)

			stop = input("stop? y/n")
			if stop == "y":
				import pdb; pdb.set_trace()

		for e in entities:
			if isinstance(e, nltk.tree.Tree): #it's an entity
				ent_string = " ".join([leaf[0] for leaf in e.leaves()])
				ent_tuple = (ent_string, e.label())

				if ent_tuple not in entity_list:
					entity_list.append(ent_tuple)
	return(entity_list)

def extract_text_from_gutenberg(text):
	text = re.split(r"\*\*\*.*?START.*?PROJECT GUTENBERG EBOOK.*?\*\*\*", text)[1]
	text = re.split(r"\*\*\*.*?END.*?PROJECT GUTENBERG EBOOK.*?\*\*\*", text)[0]
	return(text)

def ne_sent_dict_from_file(file_path, pause=False):
	full_text = file_path_to_text(file_path)
	full_text = extract_text_from_gutenberg(full_text)
	sentences = full_text.split(".")

	entity_list = []

	entity_sentence_dict = defaultdict(list)

	for sentence in sentences: #for each sentence in the input file
		sentence.strip("\n")
		t = get_tokens_from_text(sentence) #get tokens
		entities = get_named_entities(t) # get entities tree
		if pause == True:
			print(entities)

			stop = input("stop? y/n")
			if stop == "y":
				import pdb; pdb.set_trace()

		for e in entities:
			if isinstance(e, nltk.tree.Tree): #it's an entity
				ent_string = " ".join([leaf[0] for leaf in e.leaves()])
				ent_tuple = (ent_string, e.label())

				if ent_tuple not in entity_list:
					entity_sentence_dict[ent_tuple].append(t)

	return(entity_sentence_dict)

if __name__ == "__main__":

	print(make_dataset_dict())
	exit(1)
	download_models()
	print(ne_list_from_file(op.join(mimir_dir,"data","nqa_summary_text_files","train", "Anna Karenina")))

	print(BOWs_to_TFIDF(ne_BOWs_from_file(op.join(mimir_dir,"data","dune_plot.txt"))))


import pandas as pd
import numpy as np
import re

import bokeh.plotting as bk

from bokeh.palettes import Spectral6
from bokeh.plotting import ColumnDataSource

from bokeh.models import (HoverTool,
                          BoxZoomTool,
                          ResetTool,
                          PanTool,
                          WheelZoomTool,
                          Slider,
                          CustomJS,
                          Range1d,
                          Circle,
                          Text,
                          NumeralTickFormatter)

from bokeh.io import vform

gdp_df = pd.read_csv('data/gapminder/'
                     'indicator_gapminder_gdp_per_capita_ppp.csv',
                     thousands=',')

gdp_df.rename(columns={'GDP per capita': 'Entity'},
              inplace=True)

min_gdp = gdp_df.iloc[:,1:].min().min()
max_gdp = gdp_df.iloc[:,1:].max().max()

geographical_regions_df = pd.read_csv('data/gapminder/'
                                      'geographical_regions.csv')

all_df = pd.merge(gdp_df,
                  geographical_regions_df[['Entity', 'Group']],
                  on='Entity')

lex_df = pd.read_csv('data/gapminder/'
                     'indicator_gapminder_life_expectancy_at_birth.csv')

lex_df.rename(columns={'Life expectancy with projections. Yellow is IHME':
                       'Entity'},
              inplace=True)

min_lex = lex_df.iloc[:,1:].min().min()
max_lex = lex_df.iloc[:,1:].max().max()

all_df = pd.merge(all_df,
                  lex_df,
                  on='Entity',
                  suffixes=("_gdp", "_lex"))

pop_df = pd.read_csv('data/gapminder/'
                     'indicator_gapminder_population.csv',
                     thousands=',')

pop_df.rename(columns={'Total population':
                       'Entity'},
             inplace=True)

all_df = pd.merge(all_df,
                  pop_df,
                  on='Entity')

for column in all_df.columns:
    col_match = re.match(r'(\d+)(_(.+))?', column)
    if col_match:
        if col_match.lastindex > 1:
            new_name = col_match.group(3) + '_' + col_match.group(1)
            all_df.rename(columns={column: new_name},
                          inplace=True)
        else:
            new_name = 'pop_' + col_match.group(1)
            all_df.rename(columns={column: new_name},
                          inplace=True)
            sizes = 0.003 * np.sqrt(all_df[new_name] / np.pi)
            all_df['size_' + col_match.group(1)] = sizes

groups = geographical_regions_df['Group'].drop_duplicates()
group_map = {v: k for k, v in enumerate(groups)}
all_df['Group Code'] = all_df['Group'].map(group_map)

all_df['x'] = all_df['gdp_2015']
all_df['y'] = all_df['lex_2015']
all_df['pop'] = all_df['pop_2015']

all_df['size'] = all_df['size_2015']

colors = all_df['Group Code'].map(lambda x: Spectral6[x])

source = ColumnDataSource(all_df)

hover = HoverTool(tooltips=[
        ("Country Name", "@Entity"),
        ("Population", "@pop")
        ]
    )

min_x = 100 * (min_gdp // 100)
max_x = 1000 * (max_gdp // 1000) + 1000

min_y = 10 * (min_lex // 10)
max_y = 100 * (max_lex // 100) + 100

tools = [
    hover,
    WheelZoomTool(),
    PanTool(),
    BoxZoomTool(),
    ResetTool(),
]

p = bk.Figure(tools=tools,
              x_axis_type="log",
              x_axis_label="Income per person "
              "(GDP/capita, PPP$ inflation-adjusted)",
              y_axis_label="Life expectancy (years)",
              x_range=Range1d(min_x, max_x),
              y_range=Range1d(min_y, max_y),
              plot_width=700)

p.xaxis[0].formatter = NumeralTickFormatter(format='0a')

text_x = 15000
text_y = 20

for i, group in enumerate(groups):
    p.add_glyph(Text(x=text_x,
                     y=text_y,
                     text=[group],
                     text_font_size='10pt',
                     text_color='#666666'))
    p.add_glyph(Circle(x=text_x - 2000,
                       y=text_y + 1,
                       fill_color=Spectral6[i],
                       size=10,
                       line_color=None, fill_alpha=0.8))
    text_y = text_y - 3

callback = CustomJS(args=dict(source=source), code="""
        var data = source.get('data');
        var v = cb_obj.get('value')
        x = data['gdp_' + v];
        y = data['lex_' + v];
        data['x'] = x;
        data['y'] = y;
        data['size'] = data['size_' + v];
        data['pop'] = data['pop_' + v];
        source.trigger('change');
    """)

p.scatter('x',
          'y',
          source=source,
          size='size',
          fill_color=colors,
          fill_alpha=0.8)

slider = Slider(start=1800, end=2015, value=2015, step=1, title="Year",
                callback=callback)

layout = vform(p, slider)

bk.output_file("gdp_lex_gapminder.html")

bk.show(layout)