import os
import numpy as np
import pandas as pd
from sklearn.preprocessing import QuantileTransformer, StandardScaler, MinMaxScaler
from sklearn.model_selection import train_test_split

import torch.utils.data as data

def get_scaler(n_quantiles):
    if n_quantiles > 0:
        return QuantileTransformer(n_quantiles=n_quantiles, output_distribution='normal', subsample=int(1e10))
    else:
        return StandardScaler()

class DataHandler:
    def __init__(self, config):
        self.config = config
        self.scalers = {
            'data': get_scaler(config.data.scaler.n_quantiles),
            'context': get_scaler(config.data.scaler.n_quantiles),
            'weight': get_scaler(config.experiment.weights.n_quantiles),
        }

        if self.config.experiment.weights.positive:
            self.scalers['weight'] = MinMaxScaler()
        if not self.config.experiment.weights.enable:
            self.scalers['weight'] = NoneProcessor()

        if config.data.download:
            if not os.path.exists(config.data.data_path):
                os.makedirs(config.data.data_path)
            log.info('config.data.download is True, starting dowload')
            target_path = os.path.join(config.data.data_path, 'data-calibsample')
            if os.path.exists(target_path):
                print("It seems that data is already downloaded. Are you sure?")
            os.system(f"wget https://cernbox.cern.ch/index.php/s/Fjf3UNgvlRVa4Td/download -O {target_path + '.tar.gz'}")
            log.info('files downloaded, starting unpacking')
            os.system(f"tar xvf {target_path + '.tar.gz'}")
            log.info('files unpacked')

        # todo rethink
        config.data.data_path = os.path.join(config.data.data_path, 'data-calibsample')

        table = np.array(get_particle_table(config.data.data_path, config.experiment.particle))
        train_table, val_table = train_test_split(table, test_size=self.config.data.val_size, random_state=42)
        self.scalers['data'].fit(train_table[:, :config.experiment.data.data_dim])
        self.scalers['context'].fit(
            train_table[:, config.experiment.data.data_dim:
                           config.experiment.data.data_dim + config.experiment.data.context_dim]
        )
        self.scalers['weight'].fit(train_table[:, -1].reshape(-1, 1))
        # todo assert weight on last col, mb add to config

        train_table = np.concatenate([
            self.scalers['data'].transform(train_table[:, :config.experiment.data.data_dim]),
            self.scalers['context'].transform(
                train_table[:, config.experiment.data.data_dim:
                               config.experiment.data.data_dim + config.experiment.data.context_dim]),
            self.scalers['weight'].transform(train_table[:, -1].reshape(-1, 1))
        ], axis=1)
        val_table = np.concatenate([
            self.scalers['data'].transform(val_table[:, :config.experiment.data.data_dim]),
            self.scalers['context'].transform(
                val_table[:, config.experiment.data.data_dim:
                             config.experiment.data.data_dim + config.experiment.data.context_dim]),
            self.scalers['weight'].transform(val_table[:, -1].reshape(-1, 1))
        ], axis=1)

        train_dataset = ParticleDataset(config, train_table)
        self.train_loader = data.DataLoader(
            dataset=train_dataset,
            batch_size=config.experiment.batch_size,
            sampler=data.DistributedSampler(train_dataset) if config.utils.use_ddp else None,
            shuffle=True if not config.utils.use_ddp else None,
            pin_memory=True,
            drop_last=True
        )
        val_dataset = ParticleDataset(config, val_table)
        self.val_loader = data.DataLoader(
            dataset=val_dataset,
            batch_size=config.experiment.batch_size,
            sampler=None,
            shuffle=False,
            drop_last=True
        )

class ParticleDataset(torch.utils.data.Dataset):
    def __init__(self, config, table):
        self.data = table[:, :config.experiment.data.data_dim]
        self.context = table[:, config.experiment.data.data_dim:
                                     config.experiment.data.data_dim + config.experiment.data.context_dim]
        self.weight = table[:, -1]
        assert config.experiment.data.data_dim + config.experiment.data.context_dim + 1 == table.shape[1]

    def __len__(self):
        return len(self.data)

    def __getitem__(self, idx):
        return self.data[idx], self.context[idx], self.weight[idx]

dll_columns = ['RichDLLe', 'RichDLLk', 'RichDLLmu', 'RichDLLp', 'RichDLLbt']
raw_feature_columns = ['Brunel_P', 'Brunel_ETA', 'nTracks_Brunel']
weight_col = 'probe_sWeight'
list_particles = ['kaon', 'pion', 'proton', 'muon', 'electron']

def load_and_cut(file_name):
    data = pd.read_csv(file_name, delimiter='\t')
    return data[dll_columns + raw_feature_columns + [weight_col]]

def load_and_merge_and_cut(filename_list):
    return pd.concat([load_and_cut(fname) for fname in filename_list], axis=0, ignore_index=True)

def get_particle_table(data_path, particle):
    particle_files = [os.path.join(data_path, name) for name in os.listdir(data_path) if particle in name]
    particle_csv = []
    for path in particle_files:
        table = pd.read_csv(path, delimiter='\t')
        table = table[dll_columns + raw_feature_columns + [weight_col]]
        particle_csv.append(table)
    particle_csv = pd.concat(particle_csv, axis=0, ignore_index=True)
    return particle_csv

class NoneProcessor(StandardScaler):
    def fit(self, X, y=None):
        pass

    def transform(self, X, copy=None):
        return X

    def inverse_transform(self, X, copy=None):
        return X

class DotDict(dict):
    __getattr__ = dict.__getitem__
    __setattr__ = dict.__setitem__
    __delattr__ = dict.__delitem__

    def __init__(self, dct):
        for key, value in dct.items():
            if hasattr(value, 'keys'):
                value = DotDict(value)
            self[key] = value

    def _getattr(self, key):
        target = self
        for dot in key.split('.'):
            target = target[dot]
        return target

    def _setattr(self, key, value):
        target = self
        for dot in key.split('.')[:-1]:
            target = target[dot]
        target[key.split('.')[-1]] = value


import os, shutil
import joblib
from rllab.algos.base import RLAlgorithm
from rllab.misc.overrides import overrides
from rllab.misc import special2 as special
from sandbox.rocky.tf.misc import tensor_utils
from rllab.sampler import parallel_sampler
from rllab.misc import ext
import rllab.misc.logger as logger
import numpy as np
import pyprind
import tensorflow as tf
from sandbox.rocky.tf.optimizers.first_order_optimizer import FirstOrderOptimizer
from sandbox.rocky.tf.optimizers.conjugate_gradient_optimizer import ConjugateGradientOptimizer
from rllab.core.serializable import Serializable
from rllab.pool.simple_pool import SimpleReplayPool
from sandbox.rocky.tf.misc.common_utils import memory_usage_resource
from sandbox.rocky.tf.misc.common_utils import pickle_load, pickle_dump
from rllab.exploration_strategies.base import ExplorationStrategy
import gc
from time import time
from sandbox.rocky.tf.algos.poleval import Poleval

class DDPG(RLAlgorithm, Poleval):

    def __init__(
            self,
            env,
            qf,
            es,
            policy=None,
            policy_batch_size=32,
            n_epochs=200,
            epoch_length=1000,
            discount=0.99,
            max_path_length=250,
            policy_weight_decay=0,
            policy_update_method='adam',
            policy_learning_rate=1e-3,
            policy_step_size=0.01,
            policy_optimizer_args=dict(),
            policy_updates_ratio=1.0,
            policy_use_target=True,
            policy_sample_last=False,
            eval_samples=10000,
            updates_ratio=1.0, # #updates/#samples
            scale_reward=1.0,
            include_horizon_terminal_transitions=False,
            save_freq=0,
            save_format='pickle',
            restore_auto=True,
            **kwargs):

        self.env = env
        self.policy = policy
        if self.policy is None: self.qf_dqn = True
        else: self.qf_dqn = False
        self.qf = qf
        self.es = es
        if self.es is None: self.es = ExplorationStrategy()
        self.n_epochs = n_epochs
        self.epoch_length = epoch_length
        self.discount = discount
        self.max_path_length = max_path_length

        self.init_critic(**kwargs)

        if not self.qf_dqn:
            self.policy_weight_decay = policy_weight_decay
            if policy_update_method == 'adam':
                self.policy_update_method = \
                    FirstOrderOptimizer(
                        update_method=policy_update_method,
                        learning_rate=policy_learning_rate,
                        **policy_optimizer_args,
                    )
                self.policy_learning_rate = policy_learning_rate
            elif policy_update_method == 'cg':
                self.policy_update_method = \
                    ConjugateGradientOptimizer(
                        **policy_optimizer_args,
                    )
                self.policy_step_size = policy_step_size
            self.policy_optimizer_args = policy_optimizer_args
            self.policy_updates_ratio = policy_updates_ratio
            self.policy_use_target = policy_use_target
            self.policy_batch_size = policy_batch_size
            self.policy_sample_last = policy_sample_last
            self.policy_surr_averages = []
            self.exec_policy = self.policy
        else:
            self.policy_batch_size = 0
            self.exec_policy = self.qf

        self.eval_samples = eval_samples
        self.updates_ratio = updates_ratio
        self.include_horizon_terminal_transitions = include_horizon_terminal_transitions

        self.paths = []
        self.es_path_returns = []
        self.paths_samples_cnt = 0

        self.scale_reward = scale_reward

        self.train_policy_itr = 0

        self.save_freq = save_freq
        self.save_format = save_format
        self.restore_auto = restore_auto

    def start_worker(self):
        parallel_sampler.populate_task(self.env, self.exec_policy)

    def save(self, checkpoint_dir=None):
        if checkpoint_dir is None: checkpoint_dir = logger.get_snapshot_dir()

        pool_file = os.path.join(checkpoint_dir, 'pool.chk')
        if self.save_format == 'pickle':
            pickle_dump(pool_file + '.tmp', self.pool)
        elif self.save_format == 'joblib':
            joblib.dump(self.pool, pool_file + '.tmp', compress=1, cache_size=1e9)
        else: raise NotImplementedError
        shutil.move(pool_file + '.tmp', pool_file)

        checkpoint_file = os.path.join(checkpoint_dir, 'params.chk')
        sess = tf.get_default_session()
        saver = tf.train.Saver()
        saver.save(sess, checkpoint_file)

        tabular_file = os.path.join(checkpoint_dir, 'progress.csv')
        if os.path.isfile(tabular_file):
            tabular_chk_file = os.path.join(checkpoint_dir, 'progress.csv.chk')
            shutil.copy(tabular_file, tabular_chk_file)

        logger.log('Saved to checkpoint %s'%checkpoint_file)

    def restore(self, checkpoint_dir=None):
        if checkpoint_dir is None: checkpoint_dir = logger.get_snapshot_dir()
        checkpoint_file = os.path.join(checkpoint_dir, 'params.chk')
        if os.path.isfile(checkpoint_file + '.meta'):
            sess = tf.get_default_session()
            saver = tf.train.Saver()
            saver.restore(sess, checkpoint_file)

            tabular_chk_file = os.path.join(checkpoint_dir, 'progress.csv.chk')
            if os.path.isfile(tabular_chk_file):
                tabular_file = os.path.join(checkpoint_dir, 'progress.csv')
                logger.remove_tabular_output(tabular_file)
                shutil.copy(tabular_chk_file, tabular_file)
                logger.add_tabular_output(tabular_file)

            pool_file = os.path.join(checkpoint_dir, 'pool.chk')
            if self.save_format == 'pickle':
                pickle_load(pool_file)
            elif self.save_format == 'joblib':
                self.pool = joblib.load(pool_file)
            else: raise NotImplementedError

            logger.log('Restored from checkpoint %s'%checkpoint_file)
        else:
            logger.log('No checkpoint %s'%checkpoint_file)

    @overrides
    def train(self):
        global_itr = tf.Variable(0, name='global_itr', trainable=False, dtype=tf.int32)
        increment_global_itr_op = tf.assign(global_itr, global_itr+1)
        global_epoch = tf.Variable(0, name='global_epoch', trainable=False, dtype=tf.int32)
        increment_global_epoch_op = tf.assign(global_epoch, global_epoch+1)
        with tf.Session() as sess:
            sess.run(tf.global_variables_initializer())
            # This seems like a rather sequential method
            self.pool = SimpleReplayPool(
                max_pool_size=self.replay_pool_size,
                observation_dim=self.env.observation_space.flat_dim,
                action_dim=self.env.action_space.flat_dim,
                replacement_prob=self.replacement_prob,
                env=self.env,
            )
            self.start_worker()

            self.init_opt()
            # This initializes the optimizer parameters
            sess.run(tf.global_variables_initializer())
            path_length = 0
            path_return = 0
            terminal = False
            initial = False
            n_updates = 0
            observation = self.env.reset()

            sample_policy = Serializable.clone(self.exec_policy, name="sample_policy")

            if self.restore_auto: self.restore()
            itr = sess.run(global_itr)
            epoch = sess.run(global_epoch)
            t0 = time()
            logger.log("Critic batch size=%d, Actor batch size=%d"%(self.qf_batch_size, self.policy_batch_size))
            while epoch < self.n_epochs:
                logger.push_prefix('epoch #%d | ' % epoch)
                logger.log("Mem: %f"%memory_usage_resource())
                logger.log("Training started")
                train_qf_itr, train_policy_itr = 0, 0
                for epoch_itr in pyprind.prog_bar(range(self.epoch_length)):
                    # Execute policy
                    if terminal:  # or path_length > self.max_path_length:
                        # Note that if the last time step ends an episode, the very
                        # last state and observation will be ignored and not added
                        # to the replay pool
                        observation = self.env.reset()
                        self.es.reset()
                        sample_policy.reset()
                        self.es_path_returns.append(path_return)
                        path_length = 0
                        path_return = 0
                        initial = True
                    else:
                        initial = False
                    action = self.es.get_action(itr, observation, policy=sample_policy)  # qf=qf)

                    next_observation, reward, terminal, _ = self.env.step(action)
                    path_length += 1
                    path_return += reward

                    if not terminal and path_length >= self.max_path_length:
                        terminal = True
                        # only include the terminal transition in this case if the flag was set
                        if self.include_horizon_terminal_transitions:
                            self.pool.add_sample(observation, action, reward * self.scale_reward, terminal, initial)
                    else:
                        self.pool.add_sample(observation, action, reward * self.scale_reward, terminal, initial)

                    observation = next_observation

                    if self.pool.size > max(self.min_pool_size, self.qf_batch_size):
                        n_updates += self.updates_ratio
                        while n_updates > 0:
                            # Train policy
                            itrs = self.do_training(itr)
                            train_qf_itr += itrs[0]
                            train_policy_itr += itrs[1]
                            n_updates -= 1
                        sample_policy.set_param_values(self.exec_policy.get_param_values())

                    itr = sess.run(increment_global_itr_op)
                    if time() - t0 > 100: gc.collect(); t0 = time()

                logger.log("Training finished")
                logger.log("Trained qf %d steps, policy %d steps"%(train_qf_itr, train_policy_itr))
                if self.pool.size >= self.min_pool_size:
                    self.evaluate(epoch, self.pool)
                    params = self.get_epoch_snapshot(epoch)
                    logger.save_itr_params(epoch, params)
                logger.dump_tabular(with_prefix=False)
                logger.pop_prefix()
                epoch = sess.run(increment_global_epoch_op)
                if self.save_freq > 0 and (epoch-1) % self.save_freq == 0: self.save()
            self.env.terminate()
            self.exec_policy.terminate()

    def init_opt(self):
        self.init_opt_critic()
        self.init_opt_policy()

    def init_opt_policy(self):
        if not self.qf_dqn:
            obs = self.policy.env_spec.observation_space.new_tensor_variable(
                'pol_obs',
                extra_dims=1,
            )

            if self.policy_use_target:
            	logger.log("[init_opt] using target policy.")
            	target_policy = Serializable.clone(self.policy, name="target_policy")
            else:
            	logger.log("[init_opt] no target policy.")
            	target_policy = self.policy

            policy_weight_decay_term = 0.5 * self.policy_weight_decay * \
                                   sum([tf.reduce_sum(tf.square(param))
                                        for param in self.policy.get_params(regularizable=True)])
            policy_qval = self.qf.get_e_qval_sym(
                obs, self.policy,
                deterministic=True
            )
            policy_surr = -tf.reduce_mean(policy_qval)

            policy_reg_surr = policy_surr + policy_weight_decay_term

            policy_input_list = [obs]

            if isinstance(self.policy_update_method, FirstOrderOptimizer):
                self.policy_update_method.update_opt(
                    loss=policy_reg_surr, target=self.policy, inputs=policy_input_list)

                f_train_policy = tensor_utils.compile_function(
                    inputs=policy_input_list,
                    outputs=[policy_surr, self.policy_update_method._train_op],
                )
            else:
                f_train_policy = self.policy_update_method.update_opt_trust_region(
                        loss=policy_reg_surr,
                        input_list=policy_input_list,
                        obs_var=obs,
                        target=self.policy,
                        policy=self.policy,
                        step_size=self.policy_step_size,
                )

            self.opt_info = dict(
                f_train_policy=f_train_policy,
                target_policy=target_policy,
            )

    def do_training(self, itr):
        batch = self.pool.random_batch(self.qf_batch_size)
        self.do_critic_training(itr, batch=batch)

        train_policy_itr = 0

        if not self.qf_dqn and self.pool.size > max(
                self.min_pool_size, self.policy_batch_size):
            self.train_policy_itr += self.policy_updates_ratio
            while self.train_policy_itr > 0:
                if self.policy_sample_last:
                    pol_batch = self.pool.last_batch(self.policy_batch_size)
                else:
                    pol_batch = self.pool.random_batch(self.policy_batch_size)
                self.do_policy_training(itr, batch=pol_batch)
                self.train_policy_itr -= 1
                train_policy_itr += 1

        return 1, train_policy_itr # number of itrs qf, policy are trained

    def do_policy_training(self, itr, batch):
        target_policy = self.opt_info["target_policy"]
        obs, = ext.extract(batch, "observations")
        f_train_policy = self.opt_info["f_train_policy"]
        if isinstance(self.policy_update_method, FirstOrderOptimizer):
            policy_surr, _ = f_train_policy(obs)
        else:
            agent_infos = self.policy.dist_info(obs)
            state_info_list = [agent_infos[k] for k in self.policy.state_info_keys]
            dist_info_list = [agent_infos[k] for k in self.policy.distribution.dist_info_keys]
            all_input_values = (obs, obs, ) + tuple(state_info_list) + tuple(dist_info_list)
            policy_results = f_train_policy(all_input_values)
            policy_surr = policy_results["loss_after"]
        if self.policy_use_target:
            target_policy.set_param_values(
                target_policy.get_param_values() * (1.0 - self.soft_target_tau) +
                self.policy.get_param_values() * self.soft_target_tau)
        self.policy_surr_averages.append(policy_surr)

    def evaluate(self, epoch, pool):
        logger.log("Collecting samples for evaluation")
        paths = parallel_sampler.sample_paths(
            policy_params=self.exec_policy.get_param_values(),
            max_samples=self.eval_samples,
            max_path_length=self.max_path_length,
        )

        average_discounted_return = np.mean(
            [special.discount_return(path["rewards"], self.discount) for path in paths]
        )

        returns = [sum(path["rewards"]) for path in paths]

        average_action = np.mean(np.square(np.concatenate(
            [path["actions"] for path in paths]
        )))

        qfun_reg_param_norm = np.linalg.norm(
            self.qf.get_param_values(regularizable=True)
        )

        logger.record_tabular('Epoch', epoch)
        logger.record_tabular('Iteration', epoch)
        logger.record_tabular('AverageReturn', np.mean(returns))
        logger.record_tabular('StdReturn',
                              np.std(returns))
        logger.record_tabular('MaxReturn',
                              np.max(returns))
        logger.record_tabular('MinReturn',
                              np.min(returns))
        if len(self.es_path_returns) > 0:
            logger.record_tabular('AverageEsReturn',
                                  np.mean(self.es_path_returns))
            logger.record_tabular('StdEsReturn',
                                  np.std(self.es_path_returns))
            logger.record_tabular('MaxEsReturn',
                                  np.max(self.es_path_returns))
            logger.record_tabular('MinEsReturn',
                                  np.min(self.es_path_returns))
        logger.record_tabular('AverageDiscountedReturn',
                              average_discounted_return)
        logger.record_tabular('AverageAction', average_action)

        logger.record_tabular('QFunRegParamNorm',
                              qfun_reg_param_norm)
        self.env.log_diagnostics(paths)
        self.log_critic_training()

        self.es_path_returns = []

        if not self.qf_dqn:
            average_policy_surr = np.mean(self.policy_surr_averages)
            policy_reg_param_norm = np.linalg.norm(
                self.policy.get_param_values(regularizable=True)
            )
            logger.record_tabular('AveragePolicySurr', average_policy_surr)
            logger.record_tabular('PolicyRegParamNorm',
                              policy_reg_param_norm)
            self.policy.log_diagnostics(paths)
            self.policy_surr_averages = []

    def get_epoch_snapshot(self, epoch):
        snapshot = dict(
            env=self.env,
            epoch=epoch,
            qf=self.qf,
            target_qf=self.opt_info_critic["target_qf"],
            es=self.es,
        )
        if not self.qf_dqn:
            snapshot.update(dict(
                policy=self.policy,
                target_policy=self.opt_info["target_policy"],
            ))
        return snapshot


import cloudinary
import cloudinary.uploader
from flask import Blueprint, jsonify, request
from models.imagen import Imagen
from models.noticia import Noticia

api = Blueprint('api_noticias', __name__)

#Ruta para agregar una noticia
@api.route('/administrar/agregarNoticia', methods=['POST'])
def upload_image():

    # Asignamos a una variable los valores rescatados del form,
    titulo = request.form['titulo']
    descripcion = request.form['descripcion']
    image = None

    if not titulo: return jsonify({"message": "Titulo is required!"}), 400
    if not descripcion: return jsonify({"message": "Descripcion is required!"}), 400
    if not 'image' in request.files:
        return jsonify({"message": "Image is required!"}), 400
    else:
        image = request.files['image']

    response = cloudinary.uploader.upload(image, folder="imagenesFidi")

    if response:
        nuevaImagen = Imagen()
        nuevaImagen.src_imagen = response['secure_url']
        nuevaImagen.id_publico = response['public_id']
        nuevaImagen.activo = True
        nuevaImagen.save()

        nuevaNoticia = Noticia()
        nuevaNoticia.titulo = titulo
        nuevaNoticia.descripcion = descripcion
        nuevaNoticia.src_imagen = response['secure_url']
        nuevaNoticia.save()

    return jsonify({ "image": nuevaImagen.serialize(), "message": "Noticia creada satisfactoriamente"}), 201

@api.route('/administrar/modificarNoticia')
def modify_notice():

    noticias = Noticia.query.all() # [<User 1>, <User 2>]
    noticias = list(map(lambda noticia: noticia.serialize(), noticias)) # [{"id": 1}, {"id": 2}]

    return jsonify(noticias), 200


from utils.db import connect

def get_active(chat_id):

    db = connect()
    cursor = db.cursor(prepared=True)
    query = "SELECT char_id FROM active WHERE chat_id = %s"

    try:
        cursor.execute(query, (chat_id,))
    except:
        print("can not retriever char_id from active")

    result = cursor.fetchall()

    try:
        char_id = int(result[0][0])
    except:
        return None

    if not result:
        return None

    query = "SELECT char_name FROM `character` WHERE char_id = %s"

    try:
        cursor.execute(query, (char_id,))
    except:
        print("can not retriever char name from character")

    if not result and result[0][0] is not None:
        return None
    else:
        return char_id

def get_hp(char_id):

    db = connect()
    cursor = db.cursor(prepared=True)
    query = "SELECT life FROM `character` WHERE char_id = %s"

    try:
        cursor.execute(query, (char_id,))
    except:
        print("can not retriever char life from character")

    result = cursor.fetchall()

    return int(result[0][0])

def get_name(char_id):

    db = connect()
    cursor = db.cursor(prepared=True)
    query = "SELECT char_name FROM `character` WHERE char_id = %s"

    try:
        cursor.execute(query, (char_id,))
    except:
        print("can not retriever char life from character")

    result = cursor.fetchall()
    return result[0][0]

def update_life(char_id, char_hp):

    db = connect()
    cursor = db.cursor(prepared=True)
    query = "UPDATE `character` SET life = %s WHERE char_id = %s "

    try:
        cursor.execute(query, (char_hp, char_id))
        db.commit()
    except:
        print("can not update char life from character")


from django.urls import path
from . import views

urlpatterns = [
    path('', views.index),
    path('addBook/',views.add_book),
    path('addAuthor/',views.add_author),
    path('book/<book_ID>',views.details),
    path('authors/<author_ID>',views.author_details),
    path('authors/', views.author_index),
    path('addNewAuthor/',views.add_new_author),
    path('addBookToAuthor/',views.add_book_to_author),

]


import json
from django.http import JsonResponse, FileResponse
import os

# Create your views here.
from gif_gen.src.service.gif_generator import GifGenerator
from gif_gen.src.utils import build_gif_name

def upload_video(request):
    if request.method == "POST":
        my_file = request.FILES.get("my_file", None)
        if not my_file:
            return JsonResponse({'message': 'no files to upload.'})
        file_path = os.path.join(os.getcwd(), "tmp/videos", my_file.name)
        destination = open(file_path, "wb+")
        for chunk in my_file.chunks():
            destination.write(chunk)
        destination.close()
        return JsonResponse({'file_path': file_path, 'file_name': my_file.name})

def video_to_gif_quick(request):
    GIFS_DIR = os.path.join(os.getcwd(), "tmp/gifs/")
    gif_gen = GifGenerator()
    # video_path = eval(request.body.decode()).get('video_path')
    body = json.loads(request.body.decode())
    video_path = body.get('video_path')
    video_name = body.get('video_name')
    gif_name = build_gif_name(video_name)
    gif_gen.video2gif_quick(video_path, GIFS_DIR + gif_name)
    return JsonResponse({'message': 'transform over.'})

def download(request):
    GIFS_DIR = os.path.join(os.getcwd(), "tmp/gifs/")
    video_name = request.GET.get('video_name')
    gif_name = build_gif_name(video_name)
    file = open(GIFS_DIR + gif_name, 'rb')
    response = FileResponse(file)
    response['Content-Type'] = 'application/form-data'
    # content_dis = 'attachment;filename="' + gif_name + '"'
    # response['Content-Disposition'] = content_dis
    return response

if __name__ == '__main__':
    PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.getcwd())))
    a = os.path.join(PROJECT_ROOT, "tmp", "my_file.name")
    print(PROJECT_ROOT)
    print(a)


import wx
import wx.xrc

###########################################################################
## Class SetDefenseSkillsDialog
###########################################################################

class SetDefenseSkillsDialog(wx.Dialog):

    def __init__(self, parent):
        wx.Dialog.__init__(self, parent, id=wx.ID_ANY, title=u"Set Defense Skills", pos=wx.DefaultPosition,
                           size=wx.Size(699, 377), style=wx.DEFAULT_DIALOG_STYLE)

        skills_dict = parent.user_character.calc_skill_total()
        skills = ['']
        for skill, value in skills_dict.items():
            skills.append(skill)

        self.SetSizeHints(wx.DefaultSize, wx.DefaultSize)

        bs_sdsd_main = wx.BoxSizer(wx.VERTICAL)

        self.st_ = wx.StaticText(self, wx.ID_ANY,
                                 u"For MELEE and RANGED DEFENSE, you may use acrobatics, dodging, "
                                 u"or foresight.\n\nFor MENTAL DEFENSE you may use concentration, meditation, "
                                 u"bravery, discipline, religion, conviction, leadership, psychology, "
                                 u"or rulership.\n\nFor VITAL DEFENSE you may use resistance.\n\nSkill with a melee "
                                 u"weapon or unarmed combat applies to MELEE DEFENSE, but not ranged attacks.",
                                 wx.DefaultPosition, wx.DefaultSize, 0)
        self.st_.Wrap(-1)
        self.st_.SetMinSize(wx.Size(-1, 175))

        bs_sdsd_main.Add(self.st_, 0, wx.ALL, 5)

        bs_sdsd_def_skills = wx.BoxSizer(wx.HORIZONTAL)

        sbs_sdsd_melee = wx.StaticBoxSizer(wx.StaticBox(self, wx.ID_ANY, u"Melee Defense"), wx.VERTICAL)

        c_sdsd_meleeChoices = skills
        self.c_sdsd_melee = wx.Choice(sbs_sdsd_melee.GetStaticBox(), wx.ID_ANY, wx.DefaultPosition, wx.DefaultSize,
                                      c_sdsd_meleeChoices, 0)
        self.c_sdsd_melee.SetSelection(0)
        sbs_sdsd_melee.Add(self.c_sdsd_melee, 0, wx.ALL | wx.EXPAND, 5)

        bs_sdsd_def_skills.Add(sbs_sdsd_melee, 1, wx.EXPAND, 5)

        sbs_sdsd_ranged = wx.StaticBoxSizer(wx.StaticBox(self, wx.ID_ANY, u"Ranged Defense"), wx.VERTICAL)

        c_sdsd_rangedChoices = skills
        self.c_sdsd_ranged = wx.Choice(sbs_sdsd_ranged.GetStaticBox(), wx.ID_ANY, wx.DefaultPosition, wx.DefaultSize,
                                       c_sdsd_rangedChoices, 0)
        self.c_sdsd_ranged.SetSelection(0)
        sbs_sdsd_ranged.Add(self.c_sdsd_ranged, 0, wx.ALL | wx.EXPAND, 5)

        bs_sdsd_def_skills.Add(sbs_sdsd_ranged, 1, wx.EXPAND, 5)

        sbs_sdsd_mental = wx.StaticBoxSizer(wx.StaticBox(self, wx.ID_ANY, u"Mental Defense"), wx.VERTICAL)

        c_sdsd_mentalChoices = skills
        self.c_sdsd_mental = wx.Choice(sbs_sdsd_mental.GetStaticBox(), wx.ID_ANY, wx.DefaultPosition, wx.DefaultSize,
                                       c_sdsd_mentalChoices, 0)
        self.c_sdsd_mental.SetSelection(0)
        sbs_sdsd_mental.Add(self.c_sdsd_mental, 0, wx.ALL | wx.EXPAND, 5)

        bs_sdsd_def_skills.Add(sbs_sdsd_mental, 1, wx.EXPAND, 5)

        sbs_sdsd_vital = wx.StaticBoxSizer(wx.StaticBox(self, wx.ID_ANY, u"Vital Defense"), wx.VERTICAL)

        c_sdsd_vitalChoices = skills
        self.c_sdsd_vital = wx.Choice(sbs_sdsd_vital.GetStaticBox(), wx.ID_ANY, wx.DefaultPosition, wx.DefaultSize,
                                      c_sdsd_vitalChoices, 0)
        self.c_sdsd_vital.SetSelection(0)
        sbs_sdsd_vital.Add(self.c_sdsd_vital, 0, wx.ALL | wx.EXPAND, 5)

        bs_sdsd_def_skills.Add(sbs_sdsd_vital, 1, wx.EXPAND, 5)

        bs_sdsd_main.Add(bs_sdsd_def_skills, 1, wx.EXPAND, 5)

        bs_sdsd_buttons = wx.BoxSizer(wx.HORIZONTAL)

        self.b_sdsd_ok = wx.Button(self, wx.ID_ANY, u"OK", wx.DefaultPosition, wx.DefaultSize, 0)
        bs_sdsd_buttons.Add(self.b_sdsd_ok, 0, wx.ALL, 5)

        self.b_sdsd_cancel = wx.Button(self, wx.ID_ANY, u"Cancel", wx.DefaultPosition, wx.DefaultSize, 0)
        bs_sdsd_buttons.Add(self.b_sdsd_cancel, 0, wx.ALL, 5)

        bs_sdsd_main.Add(bs_sdsd_buttons, 0, wx.ALIGN_BOTTOM | wx.ALIGN_RIGHT, 5)

        self.SetSizer(bs_sdsd_main)
        self.Layout()

        self.Centre(wx.BOTH)

        # Connect Events
        self.b_sdsd_ok.Bind(wx.EVT_BUTTON, self.on_ok)
        self.b_sdsd_cancel.Bind(wx.EVT_BUTTON, self.on_cancel)

    def __del__(self):
        pass

        ################################################
        # Additional SetDefenseSkillsDialog init stuff #
        ################################################
        # skills_dict = parent.user_character.calc_skill_total()
        # skills = ['']
        # for skill, value in skills_dict.items():
        #     skills.append(skill)

    def on_ok(self, event):
        self.EndModal(wx.ID_OK)

    def on_cancel(self, event):
        self.EndModal(wx.ID_CANCEL)


import numpy as np

from ..common import radial_average
from ..HeightContainer import UniformTopographyInterface

def autocorrelation_1D(topography):
    r"""
    Compute the one-dimensional height-difference autocorrelation function (ACF).

    For non-periodic surfaces the ACF at distance d is given by:

       .. math::
         :nowrap:

         \begin{equation}
         \begin{split}

          \text{ACF}(d) =& \sum_{i=0}^{n-d-1} \frac{1}{n-d} \frac{1}{2} \left( h_i - h_{i+d} \right)^2 \\

                        =& \frac{1}{2(n-d)} \sum_{i=0}^{n-d-1} \left( h_i^2 + h_{i+d}^2 \right)
                         - \frac{1}{n-d} \sum_{i=0}^{n-d-1} h_i h_{i+d}
          \end{split}
          \end{equation}

    Parameters
    ----------
    topography : :obj:`Topography` or :obj:`UniformLineScan`
        Container storing the uniform topography map

    Returns
    -------
    r : array
        Distances. (Units: length)
    A : array
        Autocorrelation function. (Units: length**2)
    """
    if topography.dim == 2:
        nx, dummy_ny = topography.nb_grid_pts
        sx, dummy_sy = topography.physical_sizes
    else:
        nx, = topography.nb_grid_pts
        sx, = topography.physical_sizes

    if topography.is_periodic:
        # Compute height-height autocorrelation function from a convolution
        # using FFT. This is periodic by nature.
        surface_qy = np.fft.fft(topography.heights(), axis=0)
        C_qy = abs(surface_qy) ** 2  # pylint: disable=invalid-name
        A_xy = np.fft.ifft(C_qy, axis=0).real / nx

        # Convert height-height autocorrelation to height-difference
        # autocorrelation:
        #     <(h(x) - h(x+d))^2>/2 = <h^2(x)> - <h(x)h(x+d)>
        A_xy = A_xy[0] - A_xy

        A = A_xy[:nx // 2]
        A[1:nx // 2] += A_xy[nx - 1:(nx + 1) // 2:-1]
        A /= 2

        r = sx * np.arange(nx // 2) / nx
    else:
        p = topography.heights()

        # Compute height-height autocorrelation function. We need to zero
        # pad the FFT for the nonperiodic case in order to separate images.
        surface_qy = np.fft.fft(p, n=2 * nx - 1, axis=0)
        C_qy = abs(surface_qy) ** 2  # pylint: disable=invalid-name
        A_xy = np.fft.ifft(C_qy, axis=0).real

        # Correction to turn height-height into height-difference
        # autocorrelation:
        #     <(h(x) - h(x+d))^2>_d/2 = <h^2(x)>_d - <h(x)h(x+d)>_d
        # but we need to take care about h_rms^2=<h^2(x)>, which in the
        # nonperiodic case needs to be computed only over a subsection of
        # the surface. This is because the average < >_d now depends on d,
        # which determines the number of data points that are actually
        # included into the computation of <h(x)h(x+d)>_d. h_rms^2 needs to
        # be computed over the same data points.
        p_sq = p**2
        A0_xy = (p_sq.cumsum(axis=0)[::-1] + p_sq[::-1].cumsum(axis=0)[::-1])/2

        # Convert height-height autocorrelation to height-difference
        # autocorrelation
        A = ((A0_xy - A_xy[:nx]).T / (nx - np.arange(nx))).T

        r = sx * np.arange(nx) / nx

    if topography.dim == 2:
        return r, A.mean(axis=1)
    else:
        return r, A

def autocorrelation_2D(topography, nbins=100, return_map=False):
    """
    Compute height-difference autocorrelation function and radial average.

    Parameters
    ----------
    topography : :obj:`Topography`
        Container storing the (two-dimensional) topography map.
    nbins : int
        Number of bins for radial average. Note: Returned array can be smaller
        than this because bins without data point are discarded.
    return_map : bool, optional
        Return full 2D autocorrelation map. (Default: False)

    Returns
    -------
    r : array
        Distances. (Units: length)
    A : array
        Autocorrelation function. (Units: length**2)
    A_xy : array
        2D autocorrelation function. Only returned if return_map=True.
        (Units: length**2)
    """
    nx, ny = topography.nb_grid_pts
    sx, sy = topography.physical_sizes

    # Pixel physical_sizes
    area0 = (sx / nx) * (sy / ny)

    # Compute FFT and normalize
    if topography.is_periodic:
        surface_qk = np.fft.fft2(topography[...])
        C_qk = abs(surface_qk) ** 2  # pylint: disable=invalid-name
        A_xy = np.fft.ifft2(C_qk).real / (nx * ny)

        # Convert height-height autocorrelation to height-difference
        # autocorrelation
        A_xy = A_xy[0, 0] - A_xy

        if nbins is None:
            return A_xy

        # Radial average
        r_edges, n, r_val, A_val = radial_average(  # pylint: disable=invalid-name
            A_xy, (sx + sy) / 4, nbins, physical_sizes=(sx, sy))
    else:
        p = topography.heights()

        # Compute height-height autocorrelation function
        surface_qk = np.fft.fft2(p, s=(2 * nx - 1, 2 * ny - 1))
        C_qk = abs(surface_qk) ** 2  # pylint: disable=invalid-name
        A_xy = np.fft.ifft2(C_qk).real

        # Correction to turn height-height into height-difference autocorrelation
        p_sq = p**2
        A0_xy = (p_sq.cumsum(axis=0).cumsum(axis=1)[::-1, ::-1] + \
                 p_sq[::-1, ::-1].cumsum(axis=0).cumsum(axis=1)[::-1, ::-1])/2

        # Convert height-height autocorrelation to height-difference
        # autocorrelation
        A_xy = (A0_xy - A_xy[:nx, :ny]) / ((nx - np.arange(nx)).reshape(-1, 1) * (ny - np.arange(ny)).reshape(1, -1))

        if nbins is None:
            return A_xy

        # Radial average
        r_edges, n, r_val, A_val = radial_average(  # pylint: disable=invalid-name
            A_xy, (sx + sy) / 2, nbins, physical_sizes=(sx, sy), full=False)

    if return_map:
        return r_val[n > 0], A_val[n > 0], A_xy
    else:
        return r_val[n > 0], A_val[n > 0]

### Register analysis functions from this module

UniformTopographyInterface.register_function('autocorrelation_1D', autocorrelation_1D)
UniformTopographyInterface.register_function('autocorrelation_2D', autocorrelation_2D)


from dotenv import load_dotenv
import os
from langchain.embeddings import OpenAIEmbeddings
from langchain.text_splitter import CharacterTextSplitter
from langchain.document_loaders import TextLoader
from langchain.vectorstores.redis import Redis

load_dotenv()

twitter_bearer_token = os.getenv('TW_BEARER_TOKEN')
openai_api_key = os.getenv('T_OPENAI_API_KEY')
os.environ['OPENAI_API_KEY'] = openai_api_key

loader = TextLoader("denemegpt.txt")
documents = loader.load()
text_splitter = CharacterTextSplitter(chunk_size=500, chunk_overlap=0)
docs = text_splitter.split_documents(documents)

embeddings = OpenAIEmbeddings()

rds = Redis.from_documents(docs, embeddings, redis_url="redis://localhost:6379",  index_name='link04')
print(rds.index_name)

query = "What is the gptverse and their utilities?"
results = rds.similarity_search(query)
print(results[0].page_content.replace("\n", " "))


### pyspark_logr_prediction
from pyspark.ml import PipelineModel

test_df = read("FreddieMac_Spark.PySpark_TestingData")
# load logr pipeline model
lr_pipeline_model = PipelineModel.load("/incorta/IncortaAnalytics/Tenants/ebs_cloud/incorta.ml/models"+ "/logr-pipeline-model2")

# predict
predictions = lr_pipeline_model.transform(test_df) # can work
df_output = predictions.select("seq_num", "label", "predictedLabel", "probability")
df_output.show(5)
save(df_output)

### check performance
import matplotlib.pyplot as plt
from sklearn.metrics import roc_curve, auc, classification_report, confusion_matrix
from pyspark.ml.evaluation import MulticlassClassificationEvaluator, BinaryClassificationEvaluator
from pyspark.mllib.evaluation import BinaryClassificationMetrics
# Accuracy
evaluator = MulticlassClassificationEvaluator(labelCol="indexedLabel", predictionCol="prediction", metricName="accuracy")
accuracy = evaluator.evaluate(predictions)
print("Accuracy = %f" % accuracy)

# AUC
lr_eval = BinaryClassificationEvaluator(rawPredictionCol="probability", labelCol="indexedLabel")
lr_AUC  = lr_eval.evaluate(predictions)
print("AUC = %.2f" % lr_AUC)

# confusion matrix
# method 1
y_true = predictions.select(['label']).collect()
y_pred = predictions.select(['predictedLabel']).collect()
print("Confusion matrix:")
print(classification_report(y_true, y_pred))

# method 2
y_true = predictions.select("label")
y_true = y_true.toPandas()
y_pred = predictions.select("predictedLabel")
y_pred = y_pred.toPandas()
cnf_matrix = confusion_matrix(y_true, y_pred,labels=['delinquent', 'not_delinquent'])
cnf_matrix

# method 3
cm_lr_result = predictions.crosstab("predictedLabel", "label")
cm_lr_result = cm_lr_result.toPandas()
cm_lr_result

TP = float(cm_lr_result.iat[0,1])
FP = float(cm_lr_result.iat[0,2])
TN = float(cm_lr_result.iat[1,2])
FN = float(cm_lr_result.iat[1,1])
Accuracy = (TP+TN)/(TP+FP+TN+FN)
Recall = TP/(TP+FN)
Specificity = TN/(TN+FP)
Precision = TP/(TP+FP)
F1 = (2*Precision*Recall) / (Precision+Recall)

print ("Accuracy = %0.2f" %Accuracy )
print ("Recall = %0.2f" %Recall )
print ("Specificity = %0.2f" %Specificity )
print ("Precision = %0.2f" %Precision )
print("F1 score = %0.2f" %F1)

# ROC Grafik
# Create ROC grafik from lr_result
PredAndLabels           = predictions.select("probability", "indexedLabel")
PredAndLabels_collect   = PredAndLabels.collect()
PredAndLabels_list      = [(float(i[0][0]), 1.0-float(i[1])) for i in PredAndLabels_collect]
PredAndLabels           = sc.parallelize(PredAndLabels_list)
metrics = BinaryClassificationMetrics(PredAndLabels)
# Visualization ROC curve
FPR = dict()                     # FPR: False Positive Rate
tpr = dict()                     # TPR: True Positive Rate
roc_auc = dict()

y_test = [i[1] for i in PredAndLabels_list]
y_score = [i[0] for i in PredAndLabels_list]

fpr, tpr, _ = roc_curve(y_test, y_score)
roc_auc = auc(fpr, tpr)

plt.figure(figsize=(5,4))
plt.plot(fpr, tpr, label='ROC curve (area = %0.2f)' % roc_auc)
plt.plot([0, 1], [0, 1], 'k--')
plt.xlim([0.0, 1.0])
plt.ylim([0.0, 1.05])
plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate')
plt.title('ROC Curve - Logistic Regression')
plt.legend(loc="lower right")
plt.show()


import json
import unittest

from embedded_jubatus import Anomaly
from jubatus.anomaly.types import IdWithScore
from jubatus.common import Datum
import numpy as np

CONFIG = {
    'method': 'lof',
    'parameter': {
        'nearest_neighbor_num': 3,
        'reverse_nearest_neighbor_num': 5,
        'method': 'lsh',
        'parameter': {
            'hash_num': 512,
        }
    },
    'converter': {
        'num_filter_types': {},
        'num_filter_rules': [],
        'string_filter_types': {},
        'string_filter_rules': [],
        'num_types': {},
        'num_rules': [
            {'key': '*', 'type': 'num'}
        ],
        'string_types': {},
        'string_rules': [
            {'key': '*', 'type': 'space',
             'sample_weight': 'bin', 'global_weight': 'bin'}
        ]
    },
}

class TestAnomaly(unittest.TestCase):
    def test_invalid_configs(self):
        self.assertRaises(TypeError, Anomaly)
        self.assertRaises(ValueError, Anomaly, {})
        self.assertRaises(TypeError, Anomaly, {'method': 'hoge'})
        self.assertRaises(RuntimeError, Anomaly,
                          {'method': 'hoge', 'converter': {}})
        invalid_config = dict(CONFIG)
        invalid_config['method'] = 'hoge'
        self.assertRaises(RuntimeError, Anomaly, invalid_config)

    def test(self):
        x = Anomaly(CONFIG)
        self.assertEqual(float('inf'), x.add(Datum({'x': 0.1})).score)
        self.assertTrue(isinstance(x.add(Datum({'x': 0.101})), IdWithScore))
        x.add(Datum({'x': 0.1001}))
        x.calc_score(Datum({'x': 0.1001}))
        self.assertEqual(set(['0', '1', '2']), set(x.get_all_rows()))

        model = x.save_bytes()
        x.clear()
        self.assertEqual([], x.get_all_rows())

        x = Anomaly(CONFIG)
        x.load_bytes(model)
        self.assertEqual(set(['0', '1', '2']), set(x.get_all_rows()))
        self.assertEqual(CONFIG, json.loads(x.get_config()))

        p = x.add(Datum({'x': 0.2}))
        self.assertEqual(set(['0', '1', '2', '3']), set(x.get_all_rows()))
        self.assertEqual(p.score, x.update(p.id, Datum({'x': 0.2})))
        self.assertEqual(p.score, x.overwrite(p.id, Datum({'x': 0.2})))

        st = x.get_status()
        self.assertTrue(isinstance(st, dict))
        self.assertEqual(len(st), 1)
        self.assertEqual(list(st.keys())[0], 'embedded')
        self.assertTrue(isinstance(st['embedded'], dict))

    def test_add_bulk(self):
        x = Anomaly(CONFIG)
        data = [
            Datum({'x': 0.0999}),
            Datum({'x': 0.1}),
            Datum({'x': -0.1009}),
            Datum({'x': -0.101}),
            Datum({'x': 0.1011}),
        ]
        ret = x.add_bulk(data)
        self.assertEqual(['0', '1', '2', '3', '4'], ret)
        self.assertEqual(set(ret), set(x.get_all_rows()))

        x = Anomaly(CONFIG)
        x.fit(np.array([[d.num_values[0][1]] for d in data]))
        self.assertEqual(['0', '1', '2', '3', '4'], ret)
        self.assertEqual(set(ret), set(x.get_all_rows()))


from aiohttp.web import middleware

from apps.aiohttp.extensions import db

@middleware
async def sqla_middleware(request, handler):
    request.app._current_request = id(request)
    resp = await handler(request)
    db.session.remove()
    return resp


import cv2
assert cv2.__version__[0] == '3', 'The fisheye module requires opencv version >= 3.0.0'
import numpy as np
import os
import glob
import io
import tkinter
import PIL.Image, PIL.ImageTk
import shutil

# Defines
calib_image_folder = '../config/imgsCalib/'
calib_param_file = "../config/calib_param.cnf"
NUM_CAMERA = 0

def chess_corners_draw(frame):
    # termination criteria
    criteria = (cv2.TERM_CRITERIA_EPS + cv2.TERM_CRITERIA_MAX_ITER, 30, 0.001)

    # prepare object points, like (0,0,0), (1,0,0), (2,0,0) ....,(6,5,0)
    objp = np.zeros((6*7,3), np.float32)
    objp[:,:2] = np.mgrid[0:7,0:6].T.reshape(-1,2)

    # Arrays to store object points and image points from all the images.
    objpoints = [] # 3d point in real world space
    imgpoints = [] # 2d points in image plane.

    img = frame.copy()
    # Encontrar bordas xadrez na imagem
    gray = cv2.cvtColor(img,cv2.COLOR_BGR2GRAY)
    ret, corners = cv2.findChessboardCorners(gray, (7,6),None)

    # Se encontrou mostrar
    if ret == True:
        print('teste')
        objpoints.append(objp)

        corners2 = cv2.cornerSubPix(gray,corners,(11,11),(-1,-1),criteria)
        imgpoints.append(corners2)

        # Draw and display the corners
        img = cv2.drawChessboardCorners(img, (7,6), corners2,ret)

    return img

def param_clean():
    open(calib_param_file, 'w').close()

def param_add(string):
    file = open(calib_param_file, "a+", encoding="utf-8")
    file.write(string)
    file.close()

def calibrar():
    CHECKERBOARD = (6,9)
    subpix_criteria = (cv2.TERM_CRITERIA_EPS+cv2.TERM_CRITERIA_MAX_ITER, 30, 0.1)
    calibration_flags = cv2.fisheye.CALIB_RECOMPUTE_EXTRINSIC+cv2.fisheye.CALIB_CHECK_COND+cv2.fisheye.CALIB_FIX_SKEW
    objp = np.zeros((1, CHECKERBOARD[0]*CHECKERBOARD[1], 3), np.float32)
    objp[0,:,:2] = np.mgrid[0:CHECKERBOARD[0], 0:CHECKERBOARD[1]].T.reshape(-1, 2)
    _img_shape = None
    objpoints = [] # 3d point in real world space
    imgpoints = [] # 2d points in image plane.

    images = glob.glob(calib_image_folder+'*.jpg')
    #images = glob.glob('../img/samples/*.jpg')

    for fname in images:
        img = cv2.imread(fname)
        if _img_shape == None:
            _img_shape = img.shape[:2]
        else:
            assert _img_shape == img.shape[:2], "All images must share the same size."
        gray = cv2.cvtColor(img,cv2.COLOR_BGR2GRAY)
        # Find the chess board corners
        ret, corners = cv2.findChessboardCorners(gray, CHECKERBOARD, cv2.CALIB_CB_ADAPTIVE_THRESH+cv2.CALIB_CB_FAST_CHECK+cv2.CALIB_CB_NORMALIZE_IMAGE)
        # If found, add object points, image points (after refining them)
        if ret == True:
            objpoints.append(objp)
            cv2.cornerSubPix(gray,corners,(3,3),(-1,-1),subpix_criteria)
            imgpoints.append(corners)

    N_OK = len(objpoints)
    K = np.zeros((3, 3))
    D = np.zeros((4, 1))
    rvecs = [np.zeros((1, 1, 3), dtype=np.float64) for i in range(N_OK)]
    tvecs = [np.zeros((1, 1, 3), dtype=np.float64) for i in range(N_OK)]
    rms, _, _, _, _ = \
        cv2.fisheye.calibrate(
            objpoints,
            imgpoints,
            gray.shape[::-1],
            K,
            D,
            rvecs,
            tvecs,
            calibration_flags,
            (cv2.TERM_CRITERIA_EPS+cv2.TERM_CRITERIA_MAX_ITER, 30, 1e-6)
        )
    param_clean()
    param_add("Found " + str(N_OK) + " valid images for calibration\n")
    param_add("DIM=" + str(_img_shape[::-1])+"\n")
    param_add("K=np.array(" + str(K.tolist()) + ")\n")
    param_add("D=np.array(" + str(D.tolist()) + ")")

class FinishWindow:
    def __init__(self,window,parent):
        self.window = window
        self.window.title("Calibração terminada")

        # Definir quem é o objeto Pai
        self.parent = parent

        # Realiza o cálculo dos parâmetros
        try:
            calibrar()
            mensagem = "Calibração completa com sucesso!"
        except:
            mensagem = "Ocorreu um erro durante a calibração, por favor, refaça a calibração!"

        # Texto da janela
        self.texto = tkinter.Label(window, text=mensagem)
        self.texto.pack()

        # Botão da janela
        self.btn_OK = tkinter.Button(window, text="OK PORRA!",command=self.quit)
        self.btn_OK.pack()

        self.window.mainloop()

    def quit(self):
        self.parent.window.destroy()
        self.window.destroy()

class App:
    def __init__(self,window,window_title,video_source=NUM_CAMERA):
        self.window = window
        self.window.title(window_title)
        self.video_source = video_source

        # Cria objeto de captura
        self.vid = VideoCapture(self.video_source)

        # Cria área na janela para imprimir a imagem
        self.canvas = tkinter.Canvas(window,width=960,height=540)
        self.canvas.pack()

        # Cria variável de contagem do número de fotos
        self.contagem = 1

        # Cria botão para tirar fotos
        self.btn_snapshot=tkinter.Button(window, text="Gravar!", width=50, command=self.snapshot)
        self.btn_snapshot.pack(anchor=tkinter.CENTER, expand=True)

        # Seta variável e método para atualização da imagem
        self.delay = 15
        self.update()

        # Limpa o diretório de imagens
        if os.path.exists(calib_image_folder):
            shutil.rmtree(calib_image_folder)
        os.makedirs(calib_image_folder)

        self.window.mainloop()

    def snapshot(self):
        ret,frame = self.vid.get_frame()
        if ret:
            file_name = calib_image_folder+"img"+str(self.contagem)+".jpg"
            cv2.imwrite(file_name, cv2.cvtColor(frame, cv2.COLOR_RGB2BGR))
            self.contagem = self.contagem+1
            if self.contagem == 6:
                FinishWindow(tkinter.Tk(),self)

    def update(self):
        ret, frame = self.vid.get_frame()
        font                   = cv2.FONT_HERSHEY_SIMPLEX
        bottomLeftCornerOfText = (10,50)
        fontScale              = 1
        fontColor              = (255,0,0)
        lineType               = 2

        cv2.putText(frame, "IMAGEM "+str(self.contagem),
        bottomLeftCornerOfText,
        font,
        fontScale,
        fontColor,
        lineType)

        if ret:
            self.photo = PIL.ImageTk.PhotoImage(image = PIL.Image.fromarray(cv2.resize(frame,(960,540))))
            self.canvas.create_image(0, 0, image = self.photo, anchor = tkinter.NW)

        self.window.after(self.delay, self.update)

class VideoCapture:
    def __init__(self,video_source=NUM_CAMERA):
        # Abre a fonte de vídeo
        self.vid = cv2.VideoCapture(video_source)
        self.vid.set(cv2.CAP_PROP_FRAME_WIDTH,1920)
        self.vid.set(cv2.CAP_PROP_FRAME_HEIGHT,1080)

        if not self.vid.isOpened():
            raise ValueError("Não foi possível abrir a câmera", video_source)

        self.width = self.vid.get(cv2.CAP_PROP_FRAME_WIDTH)
        self.height = self.vid.get(cv2.CAP_PROP_FRAME_HEIGHT)

    def get_frame(self):
        if self.vid.isOpened():
            ret,frame = self.vid.read()
            if ret:
                #frame = chess_corners_draw(frame)
                return (ret, cv2.cvtColor(frame, cv2.COLOR_BGR2RGB))
            else:
                return (ret, None)
        else:
            return (ret, None)

    def __del__(self):
        if self.vid.isOpened():
            self.vid.release()

App(tkinter.Tk(),"Fotos para calibração")


import codecs
import io
import json
import os
from pathlib import Path
import sys
import threading
from typing import Callable, Dict, Union
import unittest

import filelock

# Test for availability of matplotlib.
# (And remove matplotlib as an iris.tests dependency.)
try:
    import matplotlib

    # Override any user settings e.g. from matplotlibrc file.
    matplotlib.rcdefaults()
    # Set backend *after* rcdefaults, as we don't want that overridden (#3846).
    matplotlib.use("agg")
    # Standardise the figure size across matplotlib versions.
    # This permits matplotlib png image comparison.
    matplotlib.rcParams["figure.figsize"] = [8.0, 6.0]
    import matplotlib.pyplot as plt
except ImportError:
    MPL_AVAILABLE = False
else:
    MPL_AVAILABLE = True

# Whether to display matplotlib output to the screen.
_DISPLAY_FIGURES = False

if MPL_AVAILABLE and "-d" in sys.argv:
    sys.argv.remove("-d")
    plt.switch_backend("tkagg")
    _DISPLAY_FIGURES = True

# Threading non re-entrant blocking lock to ensure thread-safe plotting in the
# GraphicsTestMixin.
_lock = threading.Lock()

#: Default perceptual hash size.
HASH_SIZE = 16
#: Default maximum perceptual hash hamming distance.
HAMMING_DISTANCE = 2
# Prefix for image test results (that aren't yet verified as good to add to
# reference images)
RESULT_PREFIX = "result-"
# Name of the imagerepo json and associated file lock
IMAGE_REPO_DIR = Path(__file__).parents[1] / "results"
IMAGE_REPO_PATH = IMAGE_REPO_DIR / "imagerepo.json"
IMAGE_REPO_LOCK_PATH = IMAGE_REPO_DIR / "imagerepo.lock"

__all__ = [
    "GraphicsTestMixin",
    "MPL_AVAILABLE",
    "RESULT_PREFIX",
    "check_graphic",
    "fully_qualify",
    "generate_repo_from_baselines",
    "get_phash",
    "read_repo_json",
    "repos_equal",
    "skip_plot",
    "write_repo_json",
]

def _output_dir() -> Path:
    test_output_dir = Path(__file__).parents[1] / Path(
        "result_image_comparison"
    )

    if not os.access(test_output_dir, os.W_OK):
        if not os.access(Path("."), os.W_OK):
            raise IOError(
                "Write access to a local disk is required "
                "to run image tests. Run the tests from a "
                "current working directory you have write "
                "access to to avoid this issue."
            )
        else:
            test_output_dir = Path(".") / "iris_image_test_output"

    return test_output_dir

def read_repo_json() -> Dict[str, str]:
    with open(IMAGE_REPO_PATH, "rb") as fi:
        repo: Dict[str, str] = json.load(codecs.getreader("utf-8")(fi))
    return repo

def write_repo_json(data: Dict[str, str]) -> None:
    string_data = {}
    for key, val in data.items():
        string_data[key] = str(val)
    with open(IMAGE_REPO_PATH, "wb") as fout:
        json.dump(
            string_data,
            codecs.getwriter("utf-8")(fout),
            indent=4,
            sort_keys=True,
        )

def repos_equal(repo1: Dict[str, str], repo2: Dict[str, str]) -> bool:
    if sorted(repo1.keys()) != sorted(repo2.keys()):
        return False
    for key, val in repo1.items():
        if str(val) != str(repo2[key]):
            return False
    return True

def get_phash(input: Path) -> str:
    from PIL import Image
    import imagehash

    return imagehash.phash(Image.open(input), hash_size=HASH_SIZE)

def generate_repo_from_baselines(baseline_image_dir: Path) -> Dict[str, str]:
    repo = {}
    for path in baseline_image_dir.iterdir():
        phash = get_phash(path)
        repo[path.stem] = phash
    return repo

def fully_qualify(test_id: str, repo: str) -> Dict[str, str]:
    # If the test_id isn't in the repo as it stands, look for it
    if test_id not in repo:
        test_id_candidates = [x for x in repo.keys() if x.endswith(test_id)]
        if len(test_id_candidates) == 1:
            (test_id,) = test_id_candidates
    return test_id

def check_graphic(test_id: str, results_dir: Union[str, Path]) -> None:
    """
    Check the hash of the current matplotlib figure matches the expected
    image hash for the current graphic test.

    To create missing image test results, set the IRIS_TEST_CREATE_MISSING
    environment variable before running the tests. This will result in new
    and appropriately "<hash>.png" image files being generated in the image
    output directory, and the imagerepo.json file being updated.

    """
    from imagehash import hex_to_hash

    dev_mode = os.environ.get("IRIS_TEST_CREATE_MISSING")

    #: The path where the images generated by the tests should go.
    test_output_dir = _output_dir()
    test_output_dir.mkdir(exist_ok=True)

    # The path where the image matching this test should be saved if necessary
    result_path = test_output_dir / f"{RESULT_PREFIX}{test_id}.png"

    results_dir = Path(results_dir)
    repo = read_repo_json()

    # Check if test_id is fully qualified, if it's not then try to work
    # out what it should be
    test_id = fully_qualify(test_id, repo)

    try:

        def _create_missing(phash: str) -> None:
            output_path = test_output_dir / (test_id + ".png")

            print(f"Creating image file: {output_path}")
            figure.savefig(output_path)

            msg = "Creating imagerepo entry: {} -> {}"
            print(msg.format(test_id, phash))
            # The imagerepo.json file is a critical resource, so ensure
            # thread safe read/write behaviour via platform independent
            # file locking.
            lock = filelock.FileLock(IMAGE_REPO_LOCK_PATH)
            with lock.acquire(timeout=600):
                # Read the file again in case it changed, then edit before
                # releasing lock
                repo = read_repo_json()
                repo[test_id] = phash
                write_repo_json(repo)

        # Calculate the test result perceptual image hash.
        buffer = io.BytesIO()
        figure = plt.gcf()
        figure.savefig(buffer, format="png")
        buffer.seek(0)
        phash = get_phash(buffer)

        if test_id in repo:
            expected = hex_to_hash(repo[test_id])

            # Calculate hamming distance vector for the result hash.
            distance = expected - phash

            if distance > HAMMING_DISTANCE:
                if dev_mode:
                    _create_missing(phash)
                else:
                    figure.savefig(result_path)
                    msg = (
                        "Bad phash {} with hamming distance {} " "for test {}."
                    )
                    msg = msg.format(phash, distance, test_id)
                    if _DISPLAY_FIGURES:
                        emsg = "Image comparison would have failed: {}"
                        print(emsg.format(msg))
                    else:
                        emsg = "Image comparison failed: {}"
                        raise AssertionError(emsg.format(msg))
        else:
            if dev_mode:
                _create_missing(phash)
            else:
                figure.savefig(result_path)
                emsg = "Missing image test result: {}."
                raise AssertionError(emsg.format(test_id))

        if _DISPLAY_FIGURES:
            plt.show()

    finally:
        plt.close()

class GraphicsTestMixin:
    def setUp(self) -> None:
        # Acquire threading non re-entrant blocking lock to ensure
        # thread-safe plotting.
        _lock.acquire()
        # Make sure we have no unclosed plots from previous tests before
        # generating this one.
        if MPL_AVAILABLE:
            plt.close("all")

    def tearDown(self) -> None:
        # If a plotting test bombs out it can leave the current figure
        # in an odd state, so we make sure it's been disposed of.
        if MPL_AVAILABLE:
            plt.close("all")
        # Release the non re-entrant blocking lock.
        _lock.release()

def skip_plot(fn: Callable) -> Callable:
    """
    Decorator to choose whether to run tests, based on the availability of the
    matplotlib library.

    Example usage:
        @skip_plot
        class MyPlotTests(test.GraphicsTest):
            ...

    """
    skip = unittest.skipIf(
        condition=not MPL_AVAILABLE,
        reason="Graphics tests require the matplotlib library.",
    )

    return skip(fn)


import numpy as np
import matplotlib.pyplot as plt
x=np.arange(0,3*np.pi,0.1)
#sin
y=np.sin(x)
plt.plot(x,y)
plt.show()
#cos
y=np.cos(x)
plt.plot(x,y)
plt.show()
#tan
y=np.tan(x)
plt.plot(x,y)
plt.show()
ar=np.array([1,2,3])
print(np.exp(ar))
print(np.log(ar))
print(np.log10(ar))


import os
import sys

import matplotlib.pyplot as plt
import pandas as pd
from scipy.sparse import csr_matrix
from sklearn.cluster import KMeans
from sklearn.feature_extraction.text import TfidfVectorizer

def calculate_inertia(tfidf_matrix: csr_matrix, cluster_counts: range) -> list:
    """
    Calculate inertia for various number of cluster centers.
    """

    inertias = []
    for count in cluster_counts:
        kmeans = KMeans(n_clusters=count, random_state=0, n_init='auto')
        kmeans.fit(tfidf_matrix)
        inertias.append(kmeans.inertia_)

    return inertias

def plot_elbow_curve(cluster_counts: range, inertias: list, output_png: str) -> None:
    """
    Plot the elbow curve indicating the optimal number of clusters.
    """

    plt.figure(figsize=(6, 6))
    plt.plot(cluster_counts, inertias, 'bo-')
    plt.xlabel('Number of Clusters')
    plt.ylabel('Inertia')
    plt.title('The Elbow Method showing the optimal number of clusters')
    plt.savefig(output_png, dpi=300, bbox_inches='tight')

def main(input_csv: str, cluster_counts: range, output_png: str, overwrite: bool = False) -> None:
    """
    Main function to load preprocessed data, vectorize the 'Review_Text' column,
    perform elbow analysis and plot the resulting curve.
    """

    if not os.path.exists(input_csv):
        raise FileNotFoundError(f"The input file '{input_csv}' does not exist.")

    if os.path.exists(output_png) and not overwrite:
        should_proceed = input(f"The output file '{output_png}' already exists. Do you want to overwrite it? (y/n): ")
        if should_proceed.lower() != 'y':
            sys.exit(f"Execution stopped. The output file '{output_png}' already exists.")

    df = pd.read_csv(input_csv)

    vectorizer = TfidfVectorizer()

    tfidf_matrix = vectorizer.fit_transform(df['Preprocessed_Review_Text'].values.astype('U'))

    inertias = calculate_inertia(tfidf_matrix, cluster_counts)
    plot_elbow_curve(cluster_counts, inertias, output_png)

if __name__ == '__main__':
    input_csv = './output/word_count_analysis/preprocessed_reviews.csv'
    cluster_counts = range(1, 30)
    output_png = './output/word_count_analysis/png/elbow_analysis.png'
    overwrite = False

    main(input_csv, cluster_counts, output_png, overwrite)


from janome.tokenizer import Tokenizer
from janome.analyzer import Analyzer
from janome.charfilter import UnicodeNormalizeCharFilter, RegexReplaceCharFilter
from janome.tokenfilter import LowerCaseFilter, CompoundNounFilter, POSKeepFilter, TokenCountFilter

import logging
logging.basicConfig(level='INFO')

print('Analyzer example:')
text = '蛇の目はPure Ｐｙｔｈｏｎな形態素解析器です。'
char_filters = [UnicodeNormalizeCharFilter(), RegexReplaceCharFilter('蛇の目', 'janome')]
tokenizer = Tokenizer()
token_filters = [CompoundNounFilter(), LowerCaseFilter()]
a = Analyzer(char_filters=char_filters, tokenizer=tokenizer, token_filters=token_filters)
for token in a.analyze(text):
    print(token)

print('')
print('Analyzer example: Count nouns with POSKeepFilter and TokenCountFilter')
text = 'すもももももももものうち'
token_filters = [POSKeepFilter(['名詞']), TokenCountFilter()]
a = Analyzer(token_filters=token_filters)
for k, v in a.analyze(text):
    print('%s: %d' % (k, v))


# -*- coding: utf-8 -*-
"""
Created on Mon Jun 29 15:15:06 2020

@author: teeja
"""

import setuptools

with open("README.md", "r") as fh:
    long_description = fh.read()

setuptools.setup(
    name="faceMongo",
    version="0.0.5",
    author="Tajudeen Abdulazeez",
    author_email="toabdula@syr.edu",
    description="Scrape facebook public page to mongo database",
    long_description=long_description,
    long_description_content_type="text/markdown",
    url="https://github.com/toraaglobal/faceMongo",
    packages=setuptools.find_packages(),
    install_requires=[
          'facebook_scraper','pymongo','tqdm',
      ],
    classifiers=[
        "Programming Language :: Python :: 3",
        "License :: OSI Approved :: MIT License",
        "Operating System :: OS Independent",
    ],
    python_requires='>=3.6',
)


import asyncio
import json
import socket
import sys
import gui
import argparse
import aiofiles
from datetime import datetime
from functools import partial
import tools
import logging
from anyio import create_task_group, ExceptionGroup
from tkinter import messagebox
from async_timeout import timeout

WATCHDOG_TIMEOUT = 60
MAX_RECONNECT_ATTEMPTS = 5
CHECK_CONN_TIMEOUT = 30

watchdog_logger = logging.getLogger('watchdog_logger')

async def save_message(text):

    async with aiofiles.open('history.txt', 'a', encoding='utf-8') as file:
        await file.write(f'[{datetime.now().strftime("%d.%m %H:%M")}] {text}')

async def write_down_account_info(account_info):
    async with aiofiles.open('account.txt', 'w', encoding='utf-8') as file:
        await file.write(account_info)

async def register(reader, writer, sending_queue, messages_queue, status_updates_queue):

    messages_queue.put_nowait('Введите ник, который бы хотели использовать')
    username = await sending_queue.get()
    writer.write(f'{username}\n'.encode())
    await writer.drain()

    response = await reader.readline()
    response_decoded = response.decode()
    logging.debug(response_decoded)
    await write_down_account_info(response_decoded)

    tools.set_username(username, status_updates_queue)
    messages_queue.put_nowait(f'Добро пожаловать, {username}!')
    messages_queue.put_nowait('Ваша информация об аккаунте была записана в account.txt!')

    writer.write('\n'.encode())
    await writer.drain()
    await reader.readline()

async def login(reader, writer, hash, sending_queue, messages_queue, status_updates_queue):

    writer.write(f'{hash}\n'.encode())
    await writer.drain()
    response = await reader.readline()
    response_decoded = response.decode()

    logging.debug(response_decoded)

    if 'null' in response_decoded:
        messagebox.showinfo('Неверный хеш!', 'Неверный хеш, дружище, отправляй ник!')
        await register(reader, writer, sending_queue, messages_queue, status_updates_queue)
        return

    writer.write('\n'.encode())
    await writer.drain()
    await reader.readline()

    username = json.loads(response_decoded)['nickname']
    tools.set_username(username, status_updates_queue)
    messages_queue.put_nowait(f'Добро пожаловать, {username}!')

async def authorize(reader, writer, sending_queue, messages_queue, status_updates_queue, watchdog_queue):

    response = await reader.readline()
    response_decoded = response.decode()
    logging.debug(response_decoded)

    messages_queue.put_nowait('Добро пожаловать в чат майнкрафтера!')
    messages_queue.put_nowait('Есть ли у вас хеш? Если да, то отправьте его, если нет - напишите "нет"')
    watchdog_queue.put_nowait('Prompt before auth')

    while True:
        message = await sending_queue.get()

        if 'нет' == message.lower():
            writer.write('\n'.encode())
            await writer.drain()
            await reader.readline()
            await register(reader, writer, sending_queue, messages_queue, status_updates_queue)
            break
        else:
            await login(reader, writer, message, sending_queue, messages_queue, status_updates_queue)
            break
    watchdog_queue.put_nowait('Authorization done')

async def read_old_messages(messages_queue):

    try:
        async with aiofiles.open('history.txt', 'r', encoding='utf-8') as file:
            data = await file.readlines()

        for line in data:
            messages_queue.put_nowait(line)

    except FileNotFoundError:
        pass

async def write_in_chat(open_writer_socket_function,
                        authorize_function,
                        sending_queue,
                        watchdog_queue,
                        status_updates_queue):

    async with open_writer_socket_function() as streamers:
        reader, writer = streamers

        while True:

            if not tools.AUTHORIZED:
                await authorize_function(reader, writer)
                tools.AUTHORIZED = True
                status_updates_queue.put_nowait(gui.SendingConnectionStateChanged.ESTABLISHED)

            else:

                message = await sending_queue.get()
                message = message.replace('\n', '')
                writer.write(f'{message}\n\n'.encode())
                watchdog_queue.put_nowait('Message sent')
                await writer.drain()
                response = await reader.readline()
                logging.debug(response.decode())

async def read_chat(open_reader_socket_function, messages_queue, status_updates_queue, watchdog_queue):

    status_updates_queue.put_nowait(gui.ReadConnectionStateChanged.ESTABLISHED)

    async with open_reader_socket_function() as streamers:
        reader = streamers[0]

        while True:

            if tools.AUTHORIZED:
                data = await reader.readline()
                data_decoded = data.decode()
                watchdog_queue.put_nowait('New message in chat')
                messages_queue.put_nowait(f'[{datetime.now().strftime("%d.%m %H:%M")}] {data_decoded}')
                await save_message(f'{data_decoded}')

            else:
                await asyncio.sleep(1)

async def watch_for_connection(watchdog_queue):

    while True:
        async with timeout(WATCHDOG_TIMEOUT) as time_out:
            try:
                msg = await watchdog_queue.get()
                watchdog_logger.debug(f'[{datetime.now().strftime("%d.%m %H:%M")}] Connection is alive. {msg}')
            finally:
                if time_out.expired:
                    watchdog_logger.debug(f'[{datetime.now().strftime("%d.%m %H:%M")}] Timeout is elapsed.')
                    raise ConnectionError

async def check_the_connection(open_writer_socket_function, messages_queue, status_updates_queue):

    attempts = 0
    while True:
        async with timeout(CHECK_CONN_TIMEOUT) as time_out:
            try:
                async with open_writer_socket_function() as streamers:

                    status_updates_queue.put_nowait(gui.SendingConnectionStateChanged.ESTABLISHED)
                    status_updates_queue.put_nowait(gui.ReadConnectionStateChanged.ESTABLISHED)

                    reader, writer = streamers
                    writer.write('\n\n'.encode())
                    await writer.drain()
                    await reader.readline()
                    await asyncio.sleep(1)

            except (socket.gaierror, OSError, ExceptionGroup):
                status_updates_queue.put_nowait(gui.SendingConnectionStateChanged.INITIATED)
                status_updates_queue.put_nowait(gui.ReadConnectionStateChanged.INITIATED)
                messages_queue.put_nowait('Соединение нарушено!')
                if attempts < MAX_RECONNECT_ATTEMPTS:
                    messages_queue.put_nowait('Попытка восстановить соединение...')
                    await asyncio.sleep(5)
                    attempts += 1
                else:
                    messages_queue.put_nowait('Невозможно установить соединение!')
                    messagebox.showerror('Ошибка!', 'Соединение с сервером нарушено!')
                    raise RuntimeError('Соединение с сервером нарушено!')
            finally:
                if time_out.expired:
                    raise ConnectionError

async def handle_connection(check_the_connection_function,
                            watch_for_connection_function,
                            read_chat_function,
                            write_in_chat_function,
                            authorize_function,
                            open_reader_socket_function,
                            open_writer_socket_function):

    await save_message('Установлено соединение!\n')

    async with create_task_group() as task_group:
        task_group.start_soon(check_the_connection_function, open_writer_socket_function)
        task_group.start_soon(read_chat_function, open_reader_socket_function)
        task_group.start_soon(write_in_chat_function, open_writer_socket_function, authorize_function)
        task_group.start_soon(watch_for_connection_function)

async def main(args):

    messages_queue = asyncio.Queue()
    sending_queue = asyncio.Queue()
    status_updates_queue = asyncio.Queue()
    watchdog_queue = asyncio.Queue()

    read_chat_function = partial(
        read_chat,
        messages_queue=messages_queue,
        watchdog_queue=watchdog_queue,
        status_updates_queue=status_updates_queue
    )
    write_in_chat_function = partial(
        write_in_chat,
        sending_queue=sending_queue,
        watchdog_queue=watchdog_queue,
        status_updates_queue=status_updates_queue
    )
    authorize_function = partial(
        authorize,
        sending_queue=sending_queue,
        messages_queue=messages_queue,
        status_updates_queue=status_updates_queue,
        watchdog_queue=watchdog_queue
    )
    open_reader_socket_function = partial(
        tools.open_socket,
        host=args.host,
        port=args.reading_port
    )
    open_writer_socket_function = partial(
        tools.open_socket,
        host=args.host,
        port=args.writing_port
    )
    watch_for_connection_function = partial(
        watch_for_connection,
        watchdog_queue=watchdog_queue
    )
    check_the_connection_function = partial(
        check_the_connection,
        messages_queue=messages_queue,
        status_updates_queue=status_updates_queue
    )

    if args.debug:
        logging.basicConfig(level=logging.DEBUG)
    watchdog_logger.setLevel(level=logging.DEBUG)

    await read_old_messages(messages_queue)

    async with create_task_group() as task_group:

        task_group.start_soon(
            gui.draw,
            messages_queue,
            sending_queue,
            status_updates_queue),

        task_group.start_soon(
            handle_connection,
            check_the_connection_function,
            watch_for_connection_function,
            read_chat_function,
            write_in_chat_function,
            authorize_function,
            open_reader_socket_function,
            open_writer_socket_function),

if __name__ == '__main__':

    parser = argparse.ArgumentParser()
    parser.add_argument('--host', help='указать хост', default='minechat.dvmn.org')
    parser.add_argument('--writing_port', type=int, help='указать порт для чтения сообщений из чата', default=5050)
    parser.add_argument('--reading_port', type=int, help='указать порт для отправки сообщений в чат', default=5000)
    parser.add_argument('--debug', action='store_true', help='указать включен/выключен дебаг сообщений в терминал')
    args = parser.parse_args()

    try:
        asyncio.run(main(args))
    except KeyboardInterrupt:
        sys.stderr.write("Чат завершен\n")
    except gui.TkAppClosed:
        sys.stderr.write("Вы вышли из чата\n")


"""
File name: add_features.py
Author: HOJA
Date created: 03/12/2021
Date last modified: 17/12/2021
Python Version: 3.8
"""

import json
import pywikibot
import collections
import numpy as np
from src.CONSTS import COUNTRY_IDENTIFIER, SPEAKER_ATTRIBUTES, URL_END_LIB, URL_COUNTRY
import requests
import time

"""FUnctions to get country of speaker"""

def get_country_from_wikidata(q_country) -> str:
    """This function finds the country of a q-identifier on wikidata.
    param:  q_country: str
    return: : str
    """
    country = None
    site = pywikibot.Site("wikidata", "wikidata")
    repo = site.data_repository()
    try:
        item = pywikibot.ItemPage(repo, q_country)
    except ConnectionError:
        return country

    item_dict = item.get()
    try:
        country = item_dict['labels']['en']
    except KeyError:
        pass

    return country

def get_country_identifier(q_person) -> list:
    """
    This functions returns the country identifier of a given person identifier
    param: q_person: list
    return: q_country: list
    """
    q_country = list()
    for identfier in q_person:
        if identfier in SPEAKER_ATTRIBUTES:
            q_identifiers = SPEAKER_ATTRIBUTES[identfier].tolist()

            if q_identifiers:
              for q in q_identifiers:
                  if isinstance(q, (list, np.ndarray)):
                      q_country.append(q[0])
                  elif q:
                      q_country.append(q)

    return q_country

def get_most_common_values_list(input_list) ->list:
    """
    This function returns a list of most common values in input list.
    param: input_list: list
    return: output_list: list
    """
    output_list =list()
    freq_map = collections.Counter(input_list)
    max_freq = freq_map.most_common(1)[0][1]
    for key in freq_map.keys():
        if freq_map[key] == max_freq:
            output_list.append(key)
    return output_list

def assign_country_to_speaker(q_person) ->list:
    """ This function finds the country(ies) of a list of qids of a person.
    param: q_person: list
    return: speaker_country: list
    """
    q_country = get_country_identifier(q_person)
    # Get most common country if multiple found
    if q_country:
        q_country = get_most_common_values_list(q_country)

    # Multiple countries found for speaker
    if len(q_country) > 1:
        speaker_country = list()
        for id_country in q_country:
            if id_country in COUNTRY_IDENTIFIER:
                speaker_country.append(COUNTRY_IDENTIFIER[id_country])
            else:
                country = get_country_from_wikidata(id_country)
                speaker_country.append(country)
                COUNTRY_IDENTIFIER[id_country] = country

    # One country found
    elif len(q_country) == 1:
        id_country = q_country[0]
        if id_country in COUNTRY_IDENTIFIER:
            speaker_country = COUNTRY_IDENTIFIER[id_country]
        else:
            country = get_country_from_wikidata(id_country)
            speaker_country = country
            COUNTRY_IDENTIFIER[id_country] = country

    # No country found
    else:
      speaker_country = None
    return speaker_country

"""Functions to get country of url"""

def get_identifier(item) -> str:
    """This function finds the wikidata identifier for a given string input (item)
    input:
        -item: str, item of which you want the wikidata identfier
    output:
        : str, wikidata identfier"""

    params = dict (
            action='wbsearchentities',
            format='json',
            language='en',
            uselang='en',
            type='item',
            search=item
            )

    response = None
    try:
        response = requests.get('https://www.wikidata.org/w/api.php?', params).json()
    except ValueError or ConnectionError:
        pass

    if response:
        if response.get('search'):
            return response.get('search')[0]['id']

def get_country_from_identifier(q_website):
    site = pywikibot.Site("wikidata", "wikidata")
    repo = site.data_repository()
    try:
      item = pywikibot.ItemPage(repo, q_website)
    except ConnectionError:
      return None

    country = None
    if not item.isRedirectPage():
        item_dict = item.get()
        if "P17" in item_dict["claims"]:
            clm_list = item_dict["claims"]["P17"]
            for clm in clm_list:
                clm_trgt = clm.getTarget()
                if clm_trgt:
                    try:
                        country = clm_trgt.text["labels"]["en"]
                    except KeyError:
                        pass
                    return country

def get_country_website(url) -> str:
    """This function finds the country in which the company of the url is based,
    e.g. for www.guardian.co.uk it will return Great-Brittain
    input:
        url: str, url of which the country needs to be found
        url_end_dic: dict, dictionary of countries for which url was already found
    output:
        country: str, found country"""

    country =  None

    url_ending = url.split('.')[-1]
    if url_ending in URL_END_LIB:
        country = URL_END_LIB[url_ending]

    """ Change this back before Final push
    else:
        q_website = get_identifier(url)
        if q_website:
            country = get_country_from_identifier(q_website)

        if q_website is None or country is None:
            url_try_list = url.split('.')
            url_try = max(url_try_list, key=len)
            q_website = get_identifier(url_try)
            if q_website:
                country = get_country_from_identifier(q_website)
    """
    return country

def assign_country_to_url(urls) -> list:
    """This function finds all the countries of the url list
    param: urls: list
    return: countries: list"""

    countries = list()
    for url in urls:
        if url in URL_COUNTRY:
            countries.append(URL_COUNTRY[url])
        else:
            country = get_country_website(url)
            URL_COUNTRY[url] = country
            countries.append(country)

    return countries

def expand_line(line):
    """Adds the country of the speaker, urls and unique urls to the line
    param: line: json
    return: expanded_line: json"""
    # load from json
    parsed = json.loads(line)

    # adapt for country speaker
    q_person = parsed['qids']
    if q_person:
        speaker_country = assign_country_to_speaker(q_person)
    else:
        speaker_country = []
    parsed['country_speaker'] = speaker_country

    # adapt for country url
    urls = parsed['urls']
    unique_urls = list()
    for url in urls:
        new_url = url.split('/')[2]
        new_url = new_url.split('?')[0]
        new_url = new_url.split(':')[0]
        unique_urls.append(new_url)

    unique_urls = list(set(unique_urls))
    parsed['n_appearances'] = len(unique_urls)
    url_countries = assign_country_to_url(unique_urls)
    parsed['country_urls'] = url_countries

    # back to json
    expanded_line = json.dumps(parsed).encode('utf-8')
    return expanded_line


import vlc
import pafy
from time import sleep

rickroll: str = "https://www.youtube.com/watch?v=dQw4w9WgXcQ"
short: str = "https://www.youtube.com/watch?v=C0DPdy98e4c"

class TV():
    "Simple wrapper for a TV "

    def __init__(self, media: str = "https://www.youtube.com/watch?v=C0DPdy98e4c"):
        # Get the correct media URL
        video = pafy.new(media)
        best = video.getbest()
        playurl: str = best.url
        self.url = playurl
        # This disables the video and audio output, and loops the video pretty much forever
        self.instance = vlc.Instance("--vout=dummy", "--aout=dummy", "--input-repeat=99999")
        self.player: vlc.MediaPlayer = self.instance.media_player_new()
        self.media: vlc.Media = self.instance.media_new(self.url)
        self.media.get_mrl()
        self.player.set_media(self.media)

        self._state: bool = False

    @property
    def state(self) -> bool:
        return bool(self.player.is_playing())

    def on(self) -> None:
        err = self.player.play()
        sleep(3)
        if err != 0:
            print('Error while starting the player')
        else:
            print("TV started")

    def off(self) -> None:
        self.player.stop()

    def toggle(self) -> None:
        if self.state is True:
            self.off()
        else:
            self.on()

if __name__ == "__main__":

    tv = TV()
    tv.on()
    while True:
        print(tv.state)
        sleep(1)


import logging
import pathlib
from pathlib import Path

import tensorflow as tf
from keras import models
from keras.models import Model

from src.data.makedataset_recog import process_directory
from src.models.processhelpers import (
    HOLDOUT_DIR,
    encoder,
    load_single_image,
    load_test_data,
    load_unseen_data,
    zip_prediction_labels,
)

# ! to avoid some common hardware/environment errors
gpus = tf.config.experimental.list_physical_devices('GPU')
if gpus:
    try:
        # Currently, memory growth needs to be the same across GPUs
        for gpu in gpus:
            tf.config.experimental.set_memory_growth(gpu, True)
        logical_gpus = tf.config.experimental.list_logical_devices('GPU')
        logging.info(len(gpus), 'Physical GPUs,', len(logical_gpus), 'Logical GPUs')
    except RuntimeError:
        logging.exception('Memory growth must be set before GPUs have been initialized')

def load_model_with_weights(model_name: str = 'model_full', directory: str = './models/') -> Model:
    model = models.load_model(directory + model_name)
    assert isinstance(model, Model)
    return model

def load_data_to_validate(source, sample_frac: float = 1.0):
    images_array, labels_array, encoder = load_test_data(
        HOLDOUT_DIR,
        sample_frac=0.01,
        validate=True,
    )
    logging.info(f'Images Shape: {images_array.shape}')
    logging.info(f'Labels Shape: {labels_array.shape}')
    return images_array, labels_array, encoder

def validate_model(model_name: str = 'model_full', holdout_data: str = HOLDOUT_DIR):
    logging.info('Validating Model')
    model: Model = load_model_with_weights(model_name=model_name)
    logging.info(model.summary())
    x, y, encoder = load_data_to_validate(HOLDOUT_DIR)
    score = model.evaluate(x, y, verbose='0')
    logging.info(f'Test score: {score[0]}')
    logging.info(f'Test accuracy: {score[1]}')
    predictions_array = model.predict(x)
    predictions_labeled = zip_prediction_labels(predictions_array, encoder)
    logging.info(f'Sample predictions: {predictions_labeled[:5]}')
    logging.info('Completed Validation', '\n')
    return predictions_labeled

def load_data_to_predict(source, sample_frac: float = 1.0):
    return load_unseen_data(source, sample_frac)

def predict_new_images(source, destination):
    model = load_model_with_weights('model_full')
    process_directory(UNPROCESSED_IMAGES_DIR, PREDICTION_IMAGES_DIR, size=-1)
    x = load_data_to_predict(PREDICTION_IMAGES_DIR)
    predictions_array = model.predict(x)
    predictions_labeled = zip_prediction_labels(predictions_array, encoder)
    return predictions_labeled, x, model

def predict_single_image(processed_image, model):
    if not model:
        model = load_model_with_weights('model_full')
    x = load_single_image(processed_image)
    x = x.reshape(x.shape[0], x.shape[1], x.shape[2], 1)
    x = x.astype('float32')
    x /= 255
    logging.info(type(x), x.shape)
    prediction = model.predict(x)
    return zip_prediction_labels(prediction, encoder)

if __name__ == '__main__':
    validate_model()

    # ! Directory with unprocessed images to be read
    UNPROCESSED_IMAGES_DIR: Path = pathlib.Path().cwd() / 'data/raw/'
    PREDICTION_IMAGES_DIR: Path = pathlib.Path().cwd() / 'data/processed/3_prediction/'
    predictions, X, model = predict_new_images(
        UNPROCESSED_IMAGES_DIR,
        PREDICTION_IMAGES_DIR,
    )
    logging.info(predictions)
    # ! uncomment to save predictions as a csv
    # np.savetxt("models/predictions.csv", predictions,
    #            delimiter=",", newline='\n', fmt='%s')


from django.urls import path
from .views import (
    Dashboard,
    LocalidadCreate,
    LocalidadListView,
    LocalidadCreateView,
    LocalidadUpdateView,
    LocalidadDetailView,
    LocalidadDeleteView,
)
from users.views import choice_homepage

app_name = "core"
urlpatterns = [
    path("", choice_homepage, name="choice-homepage"),
    path("inicio/", Dashboard.as_view(), name="home"),
    # Usada con AJAX en el modalform de creación del cliente
    path("localidades/", LocalidadListView.as_view(), name="localidades-list"),
    path("localidades/new/", LocalidadCreateView.as_view(), name="localidades-create"),
    path(
        "localidades/<int:pk>/", LocalidadDetailView.as_view(), name="localidades-detail"
    ),
    path(
        "localidades/<int:pk>/update/",
        LocalidadUpdateView.as_view(),
        name="localidades-update",
    ),
    path(
        "localidades/<int:pk>/delete/",
        LocalidadDeleteView.as_view(),
        name="localidades-delete",
    ),
    # AJAX
    path("localidades/ajax-new/", LocalidadCreate, name="crearLocalidad"),
]


import justext
import os
import statistics

""" Fichier qui calcule avec just text la moyenne et l'écart type du nombre de caractères
et fait la comparaison avec le fichier clean """

directory = r'/home/romaric/PycharmProjects/scrapping/Corpus_detourage/html'

save_path = "./JT"
count = 0
ecartypeLigne = 0
moyenneLigne = 0
nbChar = 0
ecartypeChar = 0
moyenneChar = 0

jtStats = []
for filename in os.listdir(directory):
    count += 1
    nbline = 0
    completeName = os.path.join(save_path, filename)

    # Création des fichiers en écriture
    cf = open(completeName, "w+")

    # Parcours des fichiers en lecture
    f = open(directory+"/"+filename, "r",encoding="utf-8",errors='ignore')

    paragraphs = justext.justext(f.read(),justext.get_stoplist("English"))

    # Ecriture de chaques paragraphes dans le fichier d'écriture
    for paragraph in paragraphs:
        if not paragraph.is_boilerplate:
            cf.write("<p>"+paragraph.text+"</p>"+"\n")

f.close()
cf.close()

data = []

#calcul des statistiques justtext
filelist = os.listdir(save_path+"/")
somme = 0
for file in filelist:
    chars = 0
    lines = 0
    fchar =open(save_path+"/" + file,'r')
    for line in fchar.readlines():
        lines +=1
        for char in line:
            chars +=1
    data.append(chars)

def cleanStat():

    path = "./Corpus_detourage/clean/"
    data = []
    filelist = os.listdir(path)
    for file in filelist:
        data.append(len(open(path + file).readlines()))

    return data

def compare(cleanStats,jtStats):
    comparaison = []
    for i in range(len(cleanStats)):
        comparaison.append(abs(cleanStats[i]-jtStats[i]))
    return comparaison

print(compare(cleanStat(),data))


import os

from PIL import Image
import cv2

def get_file_paths(folder):
    image_file_paths = []
    for root, dirs, filenames in os.walk(folder):
        filenames = sorted(filenames)
        for filename in filenames:
            input_path = os.path.abspath(root)
            file_path = os.path.join(input_path, filename)
            if filename.endswith('.png') or filename.endswith('.jpg') or filename.endswith('.ppm') or filename.endswith('.pgm'):
                image_file_paths.append(file_path)

        break  # prevent descending into subfolders
    return image_file_paths

def open_image(image_path):
    if image_path[-3:] == 'ppm':
        return Image.fromarray(cv2.cvtColor(cv2.imread(image_path, cv2.IMREAD_COLOR), cv2.COLOR_RGB2BGR))
    elif image_path[-3:] == 'pgm':
        return Image.fromarray(cv2.imread(image_path, cv2.IMREAD_GRAYSCALE))
    else:
        return Image.open(image_path)

def load_meta(dataset_folder):
    path = os.path.join(dataset_folder, 'meta.txt')
    meta = {}
    with open(path, 'r') as f:
        for line in f:
            dat = line.split(',')
            meta[dat[0]] = dat[1:]
    return meta

def align_images(meta, meta_target, a_file_paths, b_file_paths, target_path):
    if not os.path.exists(target_path):
        os.makedirs(target_path)
    with open(meta_target, 'w') as f:
        for i in range(len(a_file_paths)):
            img_a = open_image(a_file_paths[i])
            img_b = open_image(b_file_paths[i])
            assert(img_a.size == img_b.size)

            identifier = os.path.basename(a_file_paths[i])[:-4]
            m = [str(i)] + [identifier] + meta[identifier]
            f.write(','.join(m))

            aligned_image = Image.new("RGB", (img_a.size[0] * 2, img_a.size[1]))
            aligned_image.paste(img_a, (0, 0))
            aligned_image.paste(img_b, (img_a.size[0], 0))
            aligned_image.save(os.path.join(target_path, '{:04d}.jpg'.format(i)))

if __name__ == '__main__':
    import argparse
    parser = argparse.ArgumentParser()
    parser.add_argument(
        '--dataset-path',
        dest='dataset_path',
        help='Which folder to process (it should have subfolders testA, testB, trainA and trainB'
    )
    args = parser.parse_args()

    dataset_folder = args.dataset_path
    print(dataset_folder)

    test_a_path = os.path.join(dataset_folder, 'testA')
    test_b_path = os.path.join(dataset_folder, 'testB')
    val_a_path = os.path.join(dataset_folder, 'valA')
    val_b_path = os.path.join(dataset_folder, 'valB')
    test_a_file_paths = get_file_paths(test_a_path)
    test_b_file_paths = get_file_paths(test_b_path)
    if os.path.exists(val_a_path) and os.path.exists(val_b_path):
        test_a_file_paths.extend(get_file_paths(val_a_path))
        test_b_file_paths.extend(get_file_paths(val_b_path))
    assert(len(test_a_file_paths) == len(test_b_file_paths))
    test_path = os.path.join(dataset_folder, 'test')

    train_a_path = os.path.join(dataset_folder, 'trainA')
    train_b_path = os.path.join(dataset_folder, 'trainB')
    train_a_file_paths = get_file_paths(train_a_path)
    train_b_file_paths = get_file_paths(train_b_path)
    assert(len(train_a_file_paths) == len(train_b_file_paths))
    train_path = os.path.join(dataset_folder, 'train')

    meta = load_meta(dataset_folder)

    align_images(meta, os.path.join(dataset_folder, 'meta_test.txt'), test_a_file_paths, test_b_file_paths, test_path)
    align_images(meta, os.path.join(dataset_folder, 'meta_train.txt'), train_a_file_paths, train_b_file_paths, train_path)