'''
neural net output as follows:
21x9 2d tensor = a
    a[0-9,] encode for playing card in hand in position
    a[10-16,] encode for attacking with minion at position
    a[17,] encode for hero power
    a[18,] encode for hero attack
    a[19,] encode for end turn
    a[20,] encode for card index when given choice
    a[,0-8] encode for attacking enemy at position, 8 is for playable without target
'''
import sys
sys.path.append('..')
from utils import *
import argparse
import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
from torchvision import datasets, transforms
from torch.autograd import Variable

# def get_emb(ni,nf):
#     e = nn.Embedding(ni, nf)
#     e.weight.data.uniform_(-0.01,0.01)
#     return e

# class EmbeddingDotBias(nn.Module):
#     def __init__(self, n_users, n_movies):
#         super().__init__()
#         (self.u, self.m, self.ub, self.mb) = [get_emb(*o) for o in [
#             (n_users, n_factors), (n_movies, n_factors), (n_users,1), (n_movies,1)
#         ]]

#     def forward(self, cats, conts):
#         users,movies = cats[:,0],cats[:,1]
#         um = (self.u(users)* self.m(movies)).sum(1)
#         res = um + self.ub(users).squeeze() + self.mb(movies).squeeze()
#         res = F.sigmoid(res) * (max_rating-min_rating) + min_rating
#         return res

# class EmbeddingNet(nn.Module):
#     def __init__(self, n_users, n_movies, nh=10, p1=0.05, p2=0.5):
#         super().__init__()
#         (self.u, self.m) = [get_emb(*o) for o in [
#             (n_users, n_factors), (n_movies, n_factors)]]
#         self.lin1 = nn.Linear(n_factors*2, nh)
#         self.lin2 = nn.Linear(nh, 1)
#         self.drop1 = nn.Dropout(p1)
#         self.drop2 = nn.Dropout(p2)

#     def forward(self, cats, conts):
#         users,movies = cats[:,0],cats[:,1]
#         x = self.drop1(torch.cat([self.u(users),self.m(movies)], dim=1))
#         x = self.drop2(F.relu(self.lin1(x)))
#         return F.sigmoid(self.lin2(x)) * (max_rating-min_rating+1) + min_rating-0.5

class ResBlock(nn.Module):
    def __init__(self, ni, nf, kernel_size=3):
        super(ResLayer, self).__init__()

        self.conv1 = nn.Conv2d(ni, nf,
                kernel_size=kernel_size, padding=1)
        self.conv2 = nn.Conv2d(nf, nf,
                kernel_size=kernel_size, padding=1)

        self.bn1 = nn.BatchNorm2d(nf)
        self.bn2 = nn.BatchNorm2d(nf)

    def forward(self, x):
        residual = x
        x = F.relu(self.bn1(self.conv1(x)))
        x = F.relu(residual + self.bn2(self.conv2(x)))
        return x

class DQN(nn.Module):
    def __init__(self, game, args):
        """
        layers = [16, 32, 64, 128, 256]
        """
        self.state_size = game.getBoardSize()
        self.args = args

        super(DQN, self).__init__()

        #first conv layer (input as state, feed into res layers)
        self.conv1 = nn.Conv2d(4, 16, #append last three turns onto input as 3rd dim
            kernel_size=3, padding=1)
        self.bn1 = nn.BatchNorm2d(16)

        #residual layers (10)
        self.layers1 = nn.ModuleList([ResBlock(layers[i], layers[i+1])
            for i in range(len(layers) - 1)])
        self.layers2 = nn.ModuleList([ResBlock(layers[i], layers[i+1])
            for i in range(len(layers) - 1)])

        #policy head (output as action probabilities (size of actions))
        self.pi_conv1 = nn.Conv2d(256, 8, kernel_size=1)
        self.pi_bn1 = nn.BatchNorm2d(8)
        self.pi_fc1 = nn.Linear(21, 9) #!!!!output must be resized to 19x8

        #value head (output as state value [-1,1])
        self.v_conv1 = nn.Conv2d(256, 4, kernel_size=1)
        self.v_bn1 = nn.BatchNorm2d(2)
        self.v_fc1 = nn.Linear(3*self.state_size, 64)
        self.v_fc2 = nn.Linear(64, 1)

    def forward(self, state_input):
        #train conv layer --> feed into res layers,
        #pass into action and value seperately and return

        x = F.relu(self.bn1(self.conv1(state_input))) #convlayer of state into bn into relu

        #residual layers
        for l,l2 in zip(self.layers1, self.layers2):
            x = l2(l(x)) #conv into bn into relu(orig + conv into bn) into each layer batch

        #action policy head (action probabilities)
        pi = F.relu(self.pi_bn1(self.pi_conv1(x))) #feed resnet into policy head
        pi = pi.view(-1, 21)
        pi = F.log_softmax((self.pi_fc1(pi)))

        #value head (score of board state)
        v = F.relu(self.v_bn1(self.v_conv1(x))) #feed resnet into value head
        v = v.view(-1, 3*self.state_size)
        v = F.relu(self.v_fc1(v))
        v = F.tanh(self.v_fc2(v))

        return pi, v


import numpy as np
import cv2

#Load the image
filename = 'planIRL4.jpg'
img = cv2.imread(filename)

#Convert image to grayscale and denoise
imgray = cv2.cvtColor(img,cv2.COLOR_BGR2GRAY)
imgray = cv2.fastNlMeansDenoising(imgray, None, 10, 7, 21)

#Adaptive threshold attempt

imgray = cv2.adaptiveThreshold(imgray, 255, cv2.ADAPTIVE_THRESH_GAUSSIAN_C, cv2.THRESH_BINARY, 11, 2)
imgray = cv2.fastNlMeansDenoising(imgray, None, 10, 7, 21)
cv2.imwrite('planBW.jpg', imgray)

#Binary threshold conversion (0 or 1)
#ret, imgray = cv2.threshold(imgray, 128, 255, cv2.THRESH_BINARY);

ret,thresh = cv2.threshold(imgray,127,255,0)
im2, contours, hierarchy = cv2.findContours(thresh,cv2.RETR_EXTERNAL,cv2.CHAIN_APPROX_SIMPLE)

#cnt = contours[4]
#cv2.drawContours(img, [cnt], 0, (0,255,0), 3)

cv2.drawContours(img, contours, -1, (0,255,0), 3)

cv2.imwrite('planOUT.jpg', img)


from typing import Set
from pyvalidity.GroupTerm import GroupTerm

import itertools

# The name of this class should probably be "Truncated
# Multiplicatively Closed Set".
class MultiplicativelyClosedSet:
    elements: Set[GroupTerm]
    max_length: int

    # The parameter is_already_closed bypasses the closure procedure, which
    # can make initialisation more efficient in case the input set is already
    # known to be multiplicatively closed.
    def __init__(self,
                 some_set: Set[GroupTerm],
                 max_length: int,
                 is_already_closed=False):
        self.max_length = max_length
        self.elements = some_set.copy()

        if is_already_closed:
            self.unchecked_pairs = []
        else:
            self.unchecked_pairs = itertools.product(self.elements, repeat=2)
            self.close()
        assert self.unchecked_pairs == []

    # seems slow ...
    def close(self):
        while self.unchecked_pairs:
            # sets to avoid duplicate entries
            # but this takes long for > 100 000 unchecked pairs:
            new_elements = {s.times(t) for s, t in self.unchecked_pairs
                            if len(s.times(t)) <= self.max_length
                            and s.times(t) not in self.elements}

            self.elements |= new_elements
            self.unchecked_pairs = []

            # this also takes a while.
            for s in new_elements:
                for t in self.elements:
                    self.unchecked_pairs.append((s, t))
                    if s != t:
                        self.unchecked_pairs.append((t, s))

    def add(self, element: GroupTerm):
        assert self.unchecked_pairs == []
        assert element.__len__() <= self.max_length
        self.unchecked_pairs = [(element, t) for t in self.elements] \
            + [(t, element) for t in self.elements] \
            + [(element, element)]

        self.unchecked_pairs.append((element, element))
        self.elements.add(element)
        self.close()

    def __len__(self):
        return self.elements.__len__()

    def __str__(self):
        return '{' + ', '.join([str(t) for t in self.elements]) + '}'


import nltk
from nltk.corpus import stopwords
from nltk.tokenize import word_tokenize
from nltk.stem import WordNetLemmatizer
from sklearn.feature_extraction.text import TfidfVectorizer
import pandas as pd

Kaguya_sama_max_ep = 13
Boogiepop_max_ep = 19
Slime_anime_max_ep = 2

def penntag(pen):
    morphy_tag = {'NN': 'n', 'JJ': 'a',
                  'VB': 'v', 'RB': 'r'}
    try:
        return morphy_tag[pen[:2]]
    except:
        return 'n'

for ep in range(1, Slime_anime_max_ep):

    kaguya_file = 'D:\Github Projects\Heriot-Watt-Msc-Project-Sentiment-Analysis\Manually determines\Kaguya-sama cleaned\Kaguya-sama Episode ' + str(3) + ' .csv'

    # print('File '+str(ep)+ ' read')
    df = pd.read_csv(kaguya_file, index_col=0, encoding='cp1252')
    pos_df = df[df['Actual Polarity'] == 0]
    pos_df = pos_df.reset_index(drop=True)
    print(pos_df['Comment'])

    # Convert dataframe values into string
    pos_df['Comment'] = pos_df['Comment'].astype(str)
    # Remove punctuation marks and tokenize each and every word
    pos_df['Comment'] = pos_df['Comment'].str.replace('[^\w\s]', ' ')
    comment_words = pos_df['Comment'].apply(word_tokenize)

    lemma = WordNetLemmatizer()
    stopword = set(stopwords.words('english'))

    # Put words into lowercase and remove stopwords
    lowercase_words = comment_words.apply(lambda x: [word.lower() for word in x])
    words_without_stopwords = lowercase_words.apply(lambda x: [word for word in x if word not in stopword])

    # remove further stopwords
    stopwords_json = {
        "en": ["a", "a's", "able", "about", "above", "according", "accordingly", "across", "actually", "after",
               "afterwards", "again", "against", "ain't", "all", "allow", "allows", "almost", "alone", "along",
               "already", "also", "although", "always", "am", "among", "amongst", "an", "and", "another", "any",
               "anybody", "anyhow", "anyone", "anything", "anyway", "anyways", "anywhere", "apart", "appear",
               "appreciate", "appropriate", "are", "aren't", "around", "as", "aside", "ask", "asking", "associated",
               "at", "available", "away", "awfully", "b", "be", "became", "because", "become", "becomes", "becoming",
               "been", "before", "beforehand", "behind", "being", "believe", "below", "beside", "besides", "best",
               "better", "between", "beyond", "both", "brief", "but", "by", "c", "c'mon", "c's", "came", "can", "can't",
               "cannot", "cant", "cause", "causes", "certain", "certainly", "changes", "clearly", "co", "com", "come",
               "comes", "concerning", "consequently", "consider", "considering", "contain", "containing", "contains",
               "corresponding", "could", "couldn't", "course", "currently", "d", "definitely", "described", "despite",
               "did", "didn't", "different", "do", "does", "doesn't", "doing", "don't", "done", "down", "downwards",
               "during", "e", "each", "edu", "eg", "eight", "either", "else", "elsewhere", "enough", "entirely",
               "especially", "et", "etc", "even", "ever", "every", "everybody", "everyone", "everything", "everywhere",
               "ex", "exactly", "example", "except", "f", "far", "few", "fifth", "first", "five", "followed",
               "following", "follows", "for", "former", "formerly", "forth", "four", "from", "further", "furthermore",
               "g", "get", "gets", "getting", "given", "gives", "go", "goes", "going", "gone", "got", "gotten",
               "greetings", "h", "had", "hadn't", "happens", "hardly", "has", "hasn't", "have", "haven't", "having",
               "he", "he's", "hello", "help", "hence", "her", "here", "here's", "hereafter", "hereby", "herein",
               "hereupon", "hers", "herself", "hi", "him", "himself", "his", "hither", "hopefully", "how", "howbeit",
               "however", "i", "i'd", "i'll", "i'm", "i've", "ie", "if", "ignored", "immediate", "in", "inasmuch",
               "inc", "indeed", "indicate", "indicated", "indicates", "inner", "insofar", "instead", "into", "inward",
               "is", "isn't", "it", "it'd", "it'll", "it's", "its", "itself", "j", "just", "k", "keep", "keeps", "kept",
               "know", "known", "knows", "l", "last", "lately", "later", "latter", "latterly", "least", "less", "lest",
               "let", "let's", "like", "liked", "likely", "little", "look", "looking", "looks", "ltd", "m", "mainly",
               "many", "may", "maybe", "me", "mean", "meanwhile", "merely", "might", "more", "moreover", "most",
               "mostly", "much", "must", "my", "myself", "n", "name", "namely", "nd", "near", "nearly", "necessary",
               "need", "needs", "neither", "never", "nevertheless", "new", "next", "nine", "no", "nobody", "non",
               "none", "noone", "nor", "normally", "not", "nothing", "novel", "now", "nowhere", "o", "obviously", "of",
               "off", "often", "oh", "ok", "okay", "old", "on", "once", "one", "ones", "only", "onto", "or", "other",
               "others", "otherwise", "ought", "our", "ours", "ourselves", "out", "outside", "over", "overall", "own",
               "p", "particular", "particularly", "per", "perhaps", "placed", "please", "plus", "possible",
               "presumably", "probably", "provides", "q", "que", "quite", "qv", "r", "rather", "rd", "re", "really",
               "reasonably", "regarding", "regardless", "regards", "relatively", "respectively", "right", "s", "said",
               "same", "saw", "say", "saying", "says", "second", "secondly", "see", "seeing", "seem", "seemed",
               "seeming", "seems", "seen", "self", "selves", "sensible", "sent", "serious", "seriously", "seven",
               "several", "shall", "she", "should", "shouldn't", "since", "six", "so", "some", "somebody", "somehow",
               "someone", "something", "sometime", "sometimes", "somewhat", "somewhere", "soon", "sorry", "specified",
               "specify", "specifying", "still", "sub", "such", "sup", "sure", "t", "t's", "take", "taken", "tell",
               "tends", "th", "than", "thank", "thanks", "thanx", "that", "that's", "thats", "the", "their", "theirs",
               "them", "themselves", "then", "thence", "there", "there's", "thereafter", "thereby", "therefore",
               "therein", "theres", "thereupon", "these", "they", "they'd", "they'll", "they're", "they've", "think",
               "third", "this", "thorough", "thoroughly", "those", "though", "three", "through", "throughout", "thru",
               "thus", "to", "together", "too", "took", "toward", "towards", "tried", "tries", "truly", "try", "trying",
               "twice", "two", "u", "un", "under", "unfortunately", "unless", "unlikely", "until", "unto", "up", "upon",
               "us", "use", "used", "useful", "uses", "using", "usually", "uucp", "v", "value", "various", "very",
               "via", "viz", "vs", "w", "want", "wants", "was", "wasn't", "way", "we", "we'd", "we'll", "we're",
               "we've", "welcome", "well", "went", "were", "weren't", "what", "what's", "whatever", "when", "whence",
               "whenever", "where", "where's", "whereafter", "whereas", "whereby", "wherein", "whereupon", "wherever",
               "whether", "which", "while", "whither", "who", "who's", "whoever", "whole", "whom", "whose", "why",
               "will", "willing", "wish", "with", "within", "without", "won't", "wonder", "would", "wouldn't", "x", "y",
               "yes", "yet", "you", "you'd", "you'll", "you're", "you've", "your", "yours", "yourself", "yourselves",
               "z", "zero"]}
    stopwords_jsa = set(stopwords_json['en'])
    stopword_removed = words_without_stopwords.apply(lambda x: [word for word in x if word not in stopwords_jsa])

    # Apply part-of-speech to tokenized words
    POS_words = stopword_removed.apply(nltk.pos_tag)
    samples = POS_words[:POS_words.shape[0]]

    list_of_lemmad_words = []

    for i in range(0, samples.shape[0]):
        # lemmatize the tokens using Penn Treebank POS tokens in the penntag() function
        lemma_words = [lemma.lemmatize(words, pos=penntag(tags)) for words, tags in samples[i]]
        list_of_lemmad_words.append([lemma_words])

        tokeniz = pd.DataFrame(list_of_lemmad_words, columns=['Values'])

    token_lemma_values = tokeniz['Values']
    def return_back_df(doc):
        return doc
    # perfom TF-IDF on the tokens

    tfidf = TfidfVectorizer(analyzer='word',preprocessor=return_back_df,tokenizer=return_back_df,ngram_range=(2, 2))
    response = tfidf.fit_transform(token_lemma_values)

    feature_names = tfidf.get_feature_names()

    dense = response.todense()
    wordlist = dense.tolist()

    # Put TF-IDF values onto the dataframe and sum up the values
    tfif_val = pd.DataFrame(wordlist, columns=feature_names)
    sum_of_tfidf = tfif_val.sum(numeric_only=True)

    # Normalize and sort the TF-IDF values
    # normalized_values = (sum_of_tfidf - sum_of_tfidf.mean()) / (sum_of_tfidf.max() - sum_of_tfidf.min())
    #sorted_norms = normalized_values.sort_values(ascending=False)

    print(sum_of_tfidf.sort_values(ascending=False))
    # csv_file_name_kaguya = 'D:\Github Projects\Heriot-Watt-Msc-Project-Sentiment-Analysis\Tokenizer\Kaguya-sama TF-IDF\TF-IDF words of Kaguya-sama Episode ' +str(ep)+' .csv'
    # csv_file_name_boogiepop = 'D:\Github Projects\Heriot-Watt-Msc-Project-Sentiment-Analysis\Tokenizer\Boogiepop Wa Waranai TF-IDF\TF-IDF words of Boogiepop Wa Waranai Episode ' + str(ep) + ' .csv'
    # csv_file_name_slime = 'D:\Github Projects\Heriot-Watt-Msc-Project-Sentiment-Analysis\Tokenizer\Tensei Slime TF-IDF\TF-IDF words of Tensei Slime Episode ' +str(ep)+' .csv'

    # Write the TF-IDF values to the file.
    # print('File ' + str(ep) + ' Written\n')
    # sorted_norms.to_csv(csv_file_name_slime,encoding='utf-8-sig')


import scrapy

class FunSpider(scrapy.Spider):
    name = "blog_scrape"

    start_urls = [
        'https://blog.scrapinghub.com/'
        ]

    def parse(self,response):
        for url in response.css('.read-more .more-link::attr(href)').getall():
            yield scrapy.Request(url, callback=self.parse_blog_data)

        next_link = response.css('.blog-pagination .next-posts-link::attr(href)').get()
        if next_link is not None:
            yield response.follow(next_link, self.parse)

    def parse_blog_data(self,response):
        yield {
                'tite':response.css(".post-header h1 span::text").get(),
                'date':response.css('.post-header .date a::text').get(),
                'author':response.css('.post-header .author a::text').get(),
                'blog_data':response.css('.post-body p::text').getall()

                }


# -*- coding: utf-8 -*-

#############################################
#              dependencies                 #
#############################################

import gym # the env
import sys
import os.path
import matplotlib.pyplot as plt
import numpy as np
from datetime import datetime as dt  # for the logs ans saves
# networks
# --------
from neuralNetworkMaster import mlp
from neuralNetworkMaster import elman
from neuralNetworkMaster import jordan
from neuralNetworkMaster import operators
# others
import global_vars as g
from graph_init import *
from save_and_load import *
from log_writer import *

#############################################
#                  code                     #
#############################################

#===========================================#
# function create_gammas:
# in: rl_gamma
# out: float array gammas: gammas[i] = rl_gamma ** i
#===========================================#

def create_gammas(rl_gamma, rl_epsilon):
    if (rl_gamma < 0 or rl_gamma >= 1 or rl_epsilon <= 0):
        raise Exception('Uncorrect gamma or epsilon')
    gammas = [1, rl_gamma]
    power = 1
    while gammas[power] > rl_epsilon:
        gammas.append(gammas[power]*rl_gamma)
        power += 1
    return gammas

#===========================================#
# function breadth_propagate:
# in: float array observation, the vector given by the OpenAI Gym env at every movements
# out: (id, result) the id of the networks which gave the answer to "out" node, and the result
#-------------------------------------------#
# two points:
# - fifo.pop(0) isn't enough when the operators take several parameters,
#   so we capture all the results to propagate in the current networks,
#   store their index (they will be removed from the fifo stack) into a
#   lifo stack to be sure one pop won't prevent the others (as we pop
#   in the decreasing order of index)
# - we don't know, out of this function, which network's should throw
#   the result into "out", so we need to return the result (and we are
#   sure that the execution will reach "out" thanks to breadth_init())
#===========================================#

def breadth_propagate(observation):
    fifo = [ (i, observation) for i in g.global_links_array[0] ] # Doesn't visit the network of index 0 as his type is "in", it is a faked network.
    while fifo: # The list fifo isn't empty
        # propagation
        # -----------
        (id_network, result_to_propagate) = fifo.pop(0)
        result_array_to_propagate = [result_to_propagate]
        pop_indexes = []
        for i in range(len(fifo)):
            (id_prime, result_prime) = fifo[i]
            if id_prime == id_network: # Gathers several parameters, for operators
                pop_indexes.insert(0,i)
                result_array_to_propagate.append(result_prime)
        for i in pop_indexes:
            fifo.pop(i)

        result = g.networks[id_network].propagate_forward(*result_array_to_propagate)
        g.networks_answers_sequences[id_network].insert(0,np.copy(result)) # Saving the sequences for the reinforcement learning.
        # fifo completion for breadth propag
        # ----------------------------------
        # we differ case with the operator of decomposition since it needs to communique different results to the networks after it.
        j = 0
        for i in g.global_links_array[id_network]:
            if (g.global_types_array[i] == "out"):
                return id_network, result
            if g.global_types_array[id_network] == "op_decomp":
                fifo += [(i, result[j])]
                j += 1
            else :
                fifo += [(i, result)]

#===========================================#
# function reinforcement_learning:
# in:
# out:
#===========================================#

# def reinforcement_learning(done, current_score, max_score, gammas, rl_epsilon, id_result_giver, errors): # no need to alter anything since a new game wiil begin just after this function, which is called at defeat or at victory

#     power = 0
#     # building errors that will be backwarded into all the networks
#     # last_answer = networks_answers_sequences[id_result_giver][current_score]
#     for gamma in gammas:
#         # answer_to_bw = networks_answers_sequences[id_result_giver][current_score - power]
#         # real_target = target + last_answer - answer_to_bw
#         sign = np.sign(g.networks_answers_sequences[id_result_giver][power])
#         if (done) :
#             target = -sign * g.global_punishs_array[id_result_giver]
#         elif (current_score == max_score-1) :
#             target =  sign * g.global_rewards_array[id_result_giver]
#         errors.insert( 0, g.networks[id_result_giver].propagate_backward(target, gammas[power]) )
#         power += 1

    # backwarding the errors
    # for id_net in range(1, g.nb_networks-1): # all networks except "in", "out" and
    #     if id_net != id_result_giver :     # <-- the result giver (already done)
    #         for id_err in range(len(errors)):
    #             target = g.networks_answers_sequences[id_net][current_score - id_err] + errors[id_err]
    #             if (done) :
    #                 real_target = presumed_target * g.global_punishs_array[id_result_giver]
    #             elif (current_score == max_score-1) :
    #                 real_target = presumed_target * g.global_rewards_array[id_result_giver]
    #             g.networks[id_net].propagate_backward(target, gammas[id_err])

# Trying to debug
# -------------
def reinforcement_learning(done, current_score, max_score, gammas, rl_epsilon, id_result_giver, errors): # no need to alter anything since a new game wiil begin just after this function, which is called at defeat or at victory
    if done: # Defeat
        power = 0
        for gamma in gammas:
            sign = np.sign(g.networks_answers_sequences[id_result_giver][power][0])
            g.networks[id_result_giver].propagate_backward(-sign*g.global_punishs_array[id_result_giver], gammas[power])
            power += 1

    else: # victory
        power = 0
        for gamma in gammas:
            sign = np.sign(g.networks_answers_sequences[id_result_giver][power])
            g.networks[id_result_giver].propagate_backward(sign*g.global_rewards_array[id_result_giver]/2, gammas[power])
            power += 1

#===========================================#
# function" multi_networks:
# in: string array p_types_array:       networks' types (among "in" and "out" for the first and last elements which are the limits of the graph,
#                                       "mlp", "elman" and  "jordan" for the networks, or "op_plus", "op_minus", etc for the operators)
#     float array p_weights_array:      weight_multiplicator used for each network's init
#     float array p_rewards_array:      victory_reward for each network
#     float array p_punishs_array:      defeat_punishment for each network
#                                       ## weights, rewards and punishs = 0 for the types "in", "out" and the operators
#
#     positive int list array p_inner_sizes_array:      Each element i of global_inner_sizes_array corresponds to the network of index i.
#                                                       so global_inner_sizes_array[i] is the list containing the sizes of all layers,
#                                                       except the first one which depends of the last layer size of the network before.
#
#     positive int list array p_links_array:            Each element i of global_links_array corresponds to the network of index i.
#                                                       So global_links_array[i] is the list containing the index of all the reachable
#                                                       neighbors of the network of index i.
#
#     string user_name                        used for saving the logs and the results
#     string game_name,                       ex 'CartPole-v0'
#     positive int game_observation_size,     number of dimensions for the observation vector
#     positive int game_decision_size,        positive int: number of dimensions for the decision vector
#     positive int max_score = 200,           positive int: score which makes the game end (200 is the basic OpenAI gym value)
#     positive int nb_epochs = 1000,          positive int: lenght of a simulation (in term of number of games victories or deafeats)
#     positive int nb_simulations = 10,        positive int: number of simulations
#     positive int save_frequency = 100,      positive int: the networks are saved at frequency of save_frequency*epochs
#     flaot in [0,1[ rl_gamma = 0.6,          used for the reinforced learning (if gamma = 0 there won't have reinforced learning)
#     rl_epsilon = 0.05,
#     was_loaded = False
#
# out: nothing
#===========================================#

def multi_networks (p_types_array,
                    p_weights_array,
                    p_rewards_array,
                    p_punishs_array,
                    p_inner_sizes_array,
                    p_links_array,
                    user_name,
                    game_name,
                    game_observation_size,
                    game_decision_size,
                    max_score = 200,
                    nb_epochs = 1000,
                    nb_simulations = 10,
                    save_frequency = 100,
                    rl_gamma = 0.6,
                    rl_epsilon = 0.05,
                    was_loaded = False
                    ):

    # assigning the global vars valide for all the simulations
    # --------------------------------------------------------
    g.global_types_array = p_types_array
    g.global_init_weights_array = p_weights_array
    g.global_rewards_array = p_rewards_array
    g.global_punishs_array = p_punishs_array
    g.global_inner_sizes_array = p_inner_sizes_array
    g.global_links_array = p_links_array
    g.nb_networks = len(g.global_types_array)

    # finding cycles --> infinite calcul # NO MORE NEEDED thanks to the grid
    # if(cycle_exists()):
    #     raise Exception("/!\ There is a forbidden cycle")

    # log creation
    log = initiate_log(user_name, game_name, nb_simulations, nb_epochs, max_score, rl_gamma, rl_epsilon, was_loaded)
    # running the game
    env = gym.make(game_name)
    # preparing reinforcement learning
    gammas = create_gammas(rl_gamma, rl_epsilon)  # float array: gammas[i] = rl_gamma ** i, putted here for optimisation because of their are a lot of repetitions
    # prepare the plot
    score_evolution = np.zeros([nb_simulations, nb_epochs])
    # prepare the save
    save_header = [p_types_array, p_weights_array,
                   p_rewards_array, p_punishs_array,
                   p_inner_sizes_array, p_links_array,
                   [user_name, game_name,
                    game_observation_size, game_decision_size,
                    max_score, nb_epochs, nb_simulations,
                    save_frequency, rl_gamma, rl_epsilon]
                   ] # don't put was_loaded into the header, it just serve on the loading
    g.networks_weights = [ [-1 for i in range(g.nb_networks)] for i in range(nb_simulations)]
    save_path = None
    if user_name is not None: # The player is connected
        today = dt.now()
        save_path = "saves/save_" + user_name + "_" + game_name + "_" + str(today.day) + "-" + str(today.month) + "-" + str(today.year) + "_" + str(today.hour) + "-" + str(today.minute) +"-" + str(today.second) + ".npz"
    has_been_saved = False

    ########################
    # BEGINING OF THE GAME #
    ########################
    for i_simul in range (nb_simulations):
        g.networks = [-1 for i in range(g.nb_networks)]
        breadth_init(game_observation_size, game_decision_size)
        # loading
        if was_loaded :  # It remains to update the weights for this simulation.
            for i_net in range(g.nb_networks):   # After the "for", global_load[0] = networks[0] for the next simulation.
                g.networks[net].weights = g.global_load.pop(0).tolist()

        for i_epoch in range(nb_epochs): # One epoch is like one game : it lasts until the victory or the defeat.

            if user_name is not None: # The player is connected
                if(save(save_path, i_epoch, nb_epochs, i_simul, save_frequency, save_header)):   # saving
                    has_been_saved = True

            observation = env.reset()   # new game
            errors = []    # float array: contains the errors compute by backwarding the targets into the decision giver network.
            g.networks_answers_sequences = [ [] for i in range(g.nb_networks) ]   # The answer sequence of each network is saved in order to opperate the reinforced learning.

            for current_score in range(max_score): # For each decision until we win or lost.
                g.networks_answers_sequences[0].append(observation) # if we want to animate the game
                id_result_giver, result = breadth_propagate(observation)
                if result[0] > 0 : # Transforming the networks graph result (so the result of the last network branched onto the "out")
                                # into an action usable by the Open AI environnement
                    action = 1 # often means "right"
                else:
                    action = 0 # often means "left"
                #print "Action is:" + action + "with decision = " + networks_answers_sequences[0][current_score]   # Printing result in order to know if the network makes a "strong" (close to -1 or 1) decision, for debbug.
                observation, reward, done, info = env.step(action)

                # Reinforcement learning
                # ----------------------
                if done or (current_score == max_score-1):  # the game is ended by defeat or victory
                    reinforcement_learning(done, current_score, max_score, gammas, rl_epsilon, id_result_giver, errors)
                    score_evolution[i_simul][i_epoch] = current_score + 1
                    write_log(log, i_simul, i_epoch, current_score+1)
                    break   # in the case of defeat, finish with the current score

    # Made by the frontend now
    # for i in range(nb_simulations) :
    #     plt.plot(range(nb_epochs), score_evolution[i],label="simulation n " + str(i))
    # plt.legend()
    # plt.xlabel("Training epochs")
    # plt.ylabel("Score")
    # plt.title("Each graph result")

    # if (nb_simulations > 1):
    #     plt.figure()
    #     # Return the mean along the row.
    #     plt.plot(range(nb_epochs),np.mean(score_evolution, axis=0)) # (axis=0 is for row) Axis or axes along which the means are computed.
    #     plt.xlabel("Training epochs")
    #     plt.ylabel("Total score")
    #     plt.title("Mean result")

    # plt.show()

    log.close()

    return save_path, has_been_saved, score_evolution


import pandas as pd
import numpy as np
import seaborn as sns
import matplotlib.pyplot as plt

df=pd.read_csv('df1',index_col=0)
df1=pd.read_csv('df2')
print (df.head())
print (df1)
df.plot.hist() #calling histogram using pandas inbuilt data viz,import seaborn so that image will be gud
plt.show()
df1.plot.area(alpha=0.5) #calling area plot which describes the area of the values.alpha for transparency
plt.show()
df1.plot.bar(stacked=True)  #plots based on the index value,stacks the value one over the other
plt.show()
df['A'].plot.hist(bins=30)
df.plot.line(x=df.index,y='B',figsize=(5,5))  #lineplot accepts x and y value and u can use all the matplotlib func like figsize,lw,ls
df.plot.scatter(x='A',y='B',c='C',cmap='coolwarm')  #two arg as x vs y and passing c as the color value
df.plot.scatter(x='A',y='B',s=df['C']*100)  #passing c values as the size here
df.plot.box()
df2=pd.DataFrame(np.random.randn(1000,2),columns=['A','B'])
print (df2)
df2.plot.hexbin(x='A',y='B',gridsize=20,cmap='coolwarm') #accepts two args
df2.plot.kde()  #kde plot

plt.style.use('ggplot')
df3 = pd.read_csv('df3')
#df3.plot.scatter(x='a',y='b',s=50,c='red')  #setting size and color for the plot
#df3['a'].plot.hist(bins=40,alpha=0.5)
print (df3.iloc[0:30])
#df3.iloc[0:30].plot.area()
#plt.legend(loc=9,bbox_to_anchor=(0.5, -0.1))
plt.show()


"""
2020.05.06
Andy Revell and Alex Silva
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Purpose:
    Calculate functional correlations between given time series data and channels.
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Logic of code:
    1. Calculate correlations within a given time window: Window data in 1 second
    2. Calculate broadband functional connectivity with echobase broadband_conn
    3. Calculate other band functional connectivity with echobase multiband_conn
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Input:
    inputfile: a pickled list. See get_iEEG_data.py and https://docs.python.org/3/library/pickle.html for more information
        index 0: time series data N x M : row x column : time x channels
        index 1: fs, sampling frequency of time series data
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Output:
    Saves file outputfile as a pickel. For more info on pickeling, see https://docs.python.org/3/library/pickle.html
    Briefly: it is a way to save + compress data. it is useful for saving lists, as in a list of time series data and sampling frequency together along with channel names

    List index 0: ndarray. broadband. C x C x T. C: channels. T: times (1 second intervals). To change, see line 70: endInd = int(((t+1)*fs) - 1)
    List index 1: ndarray. alphatheta. C x C x T
    List index 2: ndarray. beta. C x C x T
    List index 3: ndarray. lowgamma. C x C x T
    List index 4: ndarray. highgamma. C x C x T
    List index 5: ndarray. C x _  Electrode row and column names. Stored here are the corresponding row and column names in the matrices above.
    List index 6: pd.DataFrame. N x 1. order of matrices in pickle file. The order of stored matrices are stored here to aid in transparency.
        Typically, the order in broadband, alphatheta, beta, lowgamma, highgamma

    To open the pickle file, use command:
    with open(outputfile, 'rb') as f: broadband, alphatheta, beta, lowgamma, highgamma, electrode_row_and_column_names, order_of_matrices_in_pickle_file = pickle.load(f)

~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Example:

inputfile = '/Users/andyrevell/mount/DATA/Human_Data/BIDS_processed/sub-RID0278/eeg/sub-RID0278_HUP138_phaseII_248432340000_248525740000_EEG.pickle'
outputfile = '/Users/andyrevell/mount/DATA/Human_Data/BIDS_processed/sub-RID0278/connectivity_matrices/functional/sub-RID0278_HUP138_phaseII_248432340000_248525740000_functionalConnectivity.pickle'
get_Functional_connectivity(inputfile,outputfile)
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Please use this naming convention if data is from iEEG.org
sub-RIDXXXX_iEEGFILENAME_STARTTIME_STOPTIME_functionalConnectivity.pickle
example: 'sub-RID0278_HUP138_phaseII_248432340000_248525740000_functionalConnectivity.pickle'

"""

import numpy as np
import pickle
import pandas as pd
from scipy.io import loadmat #fromn scipy to read mat files (structural connectivity)
from scipy.stats import spearmanr, pearsonr
import matplotlib.pyplot as plt

def SFC(structure_file_path,function_file_path,electrode_localization_by_atlas_file_path, outputfile):
    """
    :param structure_file_path:
    :param function_file_path:
    :param electrode_localization_by_atlas_file_path:
    :return:
    """

    """

    #Example:
    structure_file_path= '/Users/andyrevell/mount/DATA/Human_Data/BIDS_processed/sub-RID0278/connectivity_matrices/structural/RA_N1000/sub-RID0278_ses-preop3T_dwi-eddyMotionB0Corrected.nii.gz.trk.gz.RA_N1000_Perm0001.count.pass.connectivity.mat'
    electrode_localization_by_atlas_file_path = '/Users/andyrevell/mount/DATA/Human_Data/BIDS_processed/sub-RID0278/electrode_localization/electrode_localization_by_atlas/sub-RID0278_electrode_coordinates_mni_RA_N1000_Perm0001.csv'

    #seizure 3
    #pre-ictal
    function_file_path = '/Users/andyrevell/mount/DATA/Human_Data/BIDS_processed/sub-RID0278/connectivity_matrices/functional/sub-RID0278_HUP138_phaseII_415933490000_416023190000_functionalConnectivity_window0.20.pickle'
    #ictal
    function_file_path = '/Users/andyrevell/mount/DATA/Human_Data/BIDS_processed/sub-RID0278/connectivity_matrices/functional/sub-RID0278_HUP138_phaseII_416023190000_416112890000_functionalConnectivity_window0.20.pickle'
    #postictal
    function_file_path = '/Users/andyrevell/mount/DATA/Human_Data/BIDS_processed/sub-RID0278/connectivity_matrices/functional/sub-RID0278_HUP138_phaseII_416112890000_416292890000_functionalConnectivity_window0.20.pickle'

    #seizure 1
    #pre-ictal
    function_file_path = '/Users/andyrevell/mount/DATA/Human_Data/BIDS_processed/sub-RID0278/connectivity_matrices/functional/sub-RID0278_HUP138_phaseII_248432340000_248525740000_functionalConnectivity.pickle'
    #ictal
    function_file_path = '/Users/andyrevell/mount/DATA/Human_Data/BIDS_processed/sub-RID0278/connectivity_matrices/functional/sub-RID0278_HUP138_phaseII_248525740000_248619140000_functionalConnectivity.pickle'
    #postictal
    function_file_path = '/Users/andyrevell/mount/DATA/Human_Data/BIDS_processed/sub-RID0278/connectivity_matrices/functional/sub-RID0278_HUP138_phaseII_248619140000_248799140000_functionalConnectivity.pickle'

    #Output Files:
    #seizure 3
    #pre-ictal
    outputfile = '/Users/andyrevell/mount/DATA/Human_Data/BIDS_processed/sub-RID0278/connectivity_matrices/structure_function_correlation/RA_N1000/sub-RID0278_HUP138_phaseII_415933490000_416023190000_RA_N1000_Perm0001_correlation_window0.20.pickle'
    #ictal
    outputfile = '/Users/andyrevell/mount/DATA/Human_Data/BIDS_processed/sub-RID0278/connectivity_matrices/structure_function_correlation/RA_N1000/sub-RID0278_HUP138_phaseII_416023190000_416112890000_RA_N1000_Perm0001_correlation_window0.20.pickle'
    #postictal
    outputfile = '/Users/andyrevell/mount/DATA/Human_Data/BIDS_processed/sub-RID0278/connectivity_matrices/structure_function_correlation/RA_N1000/sub-RID0278_HUP138_phaseII_416112890000_416292890000_RA_N1000_Perm0001_correlation_window0.20.pickle'

    """
    #Get functional connecitivty data in pickle file format
    with open(function_file_path, 'rb') as f: broadband, alphatheta, beta, lowgamma, highgamma, electrode_row_and_column_names, order_of_matrices_in_pickle_file = pickle.load(f)
    FC_list = [broadband, alphatheta, beta, lowgamma,highgamma ]

    #Get Structural Connectivity data in mat file format. Output from DSI studio
    structural_connectivity_array = np.array(pd.DataFrame(loadmat(structure_file_path)['connectivity']))

    #Get electrode localization by atlas csv file data. From get_electrode_localization.py
    electrode_localization_by_atlas = pd.read_csv(electrode_localization_by_atlas_file_path)

    # normalizing and log-scaling the structural matrices
    structural_connectivity_array[structural_connectivity_array == 0] = 1;
    structural_connectivity_array = np.log10(structural_connectivity_array)  # log-scaling. Converting 0s to 1 to avoid taking log of zeros
    structural_connectivity_array = structural_connectivity_array / np.max(structural_connectivity_array)  # normalization

    #Remove electrodes in "electrode localization" not found in Functional Connectivity matrices
    electrode_localization_names = np.array(electrode_localization_by_atlas['electrode_name'])
    electrode_localization_by_atlas = electrode_localization_by_atlas[np.in1d(electrode_localization_names,electrode_row_and_column_names )]
    # Remove electrodes in the Functional Connectivity matrices not found in "electrode localization"
    not_in_functional_connectivity = np.in1d(electrode_row_and_column_names, electrode_localization_names)
    for i in range(len(FC_list)):
        FC_list[i] = FC_list[i][not_in_functional_connectivity,:,:]
        FC_list[i] = FC_list[i][:,not_in_functional_connectivity, :]

    #Fisher z-transform of functional connectivity data. This is to take means of correlations and do correlations to the structural connectivity
    #Fisher z transform is just arctanh
    for i in range(len(FC_list)):
        FC_list[i] = np.arctanh(FC_list[i])

    # Remove structural ROIs not in electrode_localization ROIs
    electrode_ROIs = np.unique(np.array( electrode_localization_by_atlas.iloc[:,4]))
    electrode_ROIs = electrode_ROIs[~(electrode_ROIs == 0)] #remove region 0
    structural_index = electrode_ROIs - 1 #subtract 1 because of python's zero indexing
    structural_connectivity_array = structural_connectivity_array[structural_index,:]
    structural_connectivity_array = structural_connectivity_array[:,structural_index]

    #taking average functional connectivity for those electrodes in same atlas regions
    for i in range(len(FC_list)):
        ROIs = np.array( electrode_localization_by_atlas.iloc[:,4])
        for r in range(len(electrode_ROIs)):
            index_logical = (ROIs == electrode_ROIs[r])
            index_first = np.where(index_logical)[0][0]
            index_second_to_end = np.where(index_logical)[0][1:]
            mean = np.mean(FC_list[i][index_logical,:,:], axis=0)
            # Fill in with mean.
            FC_list[i][index_first,:,:] = mean
            FC_list[i][:, index_first, :] = mean
            #delete the other rows and oclumns belonging to same region.
            FC_list[i] = np.delete(FC_list[i], index_second_to_end, axis=0)
            FC_list[i] = np.delete(FC_list[i], index_second_to_end, axis=1)
            #keeping track of which electrode labels correspond to which rows and columns
            ROIs = np.delete(ROIs, index_second_to_end, axis=0)
        #remove electrodes in the ROI labeld as zero
        index_logical = (ROIs == 0)
        index = np.where(index_logical)[0]
        FC_list[i] = np.delete(FC_list[i], index, axis=0)
        FC_list[i] = np.delete(FC_list[i], index, axis=1)
        ROIs = np.delete(ROIs, index, axis=0)

    #order FC matrices by ROIs
    order = np.argsort(ROIs)
    for i in range(len(FC_list)):
        FC_list[i] = FC_list[i][order,:,:]
        FC_list[i] = FC_list[i][:, order, :]

    #un-fisher ztranform
    for i in range(len(FC_list)):
        FC_list[i] = np.tanh(FC_list[i])

    #initialize correlation arrays
    Corrrelation_list = [None] * len(FC_list)
    for i in range(len(FC_list)):
        Corrrelation_list[i] = np.zeros(  [FC_list[0].shape[2]], dtype=float)

    correlation_type = 'spearman'
    #calculate Structure-Function Correlation.
    for i in range(len(FC_list)):
        for t in range(FC_list[i].shape[2]-1):
            #Spearman Rank Correlation: functional connectivity and structural connectivity are non-normally distributed. So we should use spearman
            if correlation_type == 'spearman':
                Corrrelation_list[i][t] = spearmanr(  np.ndarray.flatten(FC_list[i][:,:,t]) , np.ndarray.flatten(structural_connectivity_array)   ).correlation
                #print("spearman")
            # Pearson Correlation: This is calculated bc past studies use Pearson Correlation and we want to see if these results are comparable.
            if correlation_type == 'pearson':
                Corrrelation_list[i][t] = pearsonr(np.ndarray.flatten(FC_list[i][:, :, t]), np.ndarray.flatten(structural_connectivity_array))[0]

    order_of_matrices_in_pickle_file = pd.DataFrame(["broadband", "alphatheta", "beta", "lowgamma" , "highgamma" ], columns=["Order of matrices in pickle file"])
    with open(outputfile, 'wb') as f: pickle.dump([Corrrelation_list[0], Corrrelation_list[1], Corrrelation_list[2], Corrrelation_list[3], Corrrelation_list[4], order_of_matrices_in_pickle_file], f)

"""
#Vizualizing distributions of raw data:
fig, axs = plt.subplots(2,1)
axs[0].hist( np.ndarray.flatten( np.array(structural_connectivity_array)), bins = 40 )
axs[0].title.set_text('Structural Streamline Counts Distribution')
axs[0].set_xlabel('Log normalized Stremline Counts (only ones in electrode regions)')
axs[0].set_ylabel('Frequency')
axs[1].hist(np.ndarray.flatten(np.array(  np.arctanh(broadband)  )), bins=40)
axs[1].title.set_text('BroadBand Cross Correlation Distribution')
axs[1].set_xlabel('Cross Correlation Fisher Z transform')
axs[1].set_ylabel('Frequency')
plt.show()
#Notes: streamline counts seems to have a HEAVY right skew distribution: Okay for correlating Structure and Function distirbutions with just a pearson correlation? Should do spearman rank

#visualize structure-function correlation on a correlational line
x = np.ndarray.flatten(structural_connectivity_array)
i = 0
t=0
y = np.ndarray.flatten(FC_list[i][:,:,t])
fig, axs = plt.subplots(1,1)
axs.scatter(x, y )
axs.axes.set_xlim(-0.1,1)
axs.axes.set_ylim(0, 1)
axs.title.set_text('')
axs.set_xlabel('Structural Correlation')
axs.set_ylabel('Functional Correlation')
plt.show()

# visualize structure-function correlation by time
i = 0
x = range(len(Corrrelation_list[i]))
y = np.ndarray.flatten(Corrrelation_list[i])
fig, axs = plt.subplots(1, 1)
axs.scatter(x, y)
#axs.axes.set_xlim(-0.1, 1)
axs.axes.set_ylim(0.1, 0.6)
axs.title.set_text('')
axs.set_xlabel('')
axs.set_ylabel('')
plt.show()

#save FC_list for vizualization
# seizure 3
# pre-ictal
outputfile_FC_list = '/Users/andyrevell/mount/USERS/arevell/papers/paper001/data_processed/supplemental_data/sub-RID0278_HUP138_phaseII_415933490000_416023190000_RA_N1000_Perm0001_matrices_window0.20.pickle'
# ictal
outputfile_FC_list = '/Users/andyrevell/mount/USERS/arevell/papers/paper001/data_processed/supplemental_data/sub-RID0278_HUP138_phaseII_416023190000_416112890000_RA_N1000_Perm0001_matrices_window0.20.pickle'
# postictal
outputfile_FC_list = '/Users/andyrevell/mount/USERS/arevell/papers/paper001/data_processed/supplemental_data/sub-RID0278_HUP138_phaseII_416112890000_416292890000_RA_N1000_Perm0001_matrices_window0.20.pickle'

order_of_matrices_in_pickle_file = pd.DataFrame(["broadband", "alphatheta", "beta", "lowgamma" , "highgamma", "structural" ], columns=["Order of matrices in pickle file"])
with open(outputfile_FC_list, 'wb') as f: pickle.dump([FC_list[0], FC_list[1], FC_list[2], FC_list[3], FC_list[4],structural_connectivity_array, ROIs, order_of_matrices_in_pickle_file], f)

"""


from .baseCommand import BaseCommand
from .commandCache import CommandCache

from utils import *

from json_minify import json_minify
from mcstatus import MinecraftServer
from nbt import nbt

import json
import os

def first(iterable, condition = lambda x: True):
    for i in iterable:
        if condition(i):
            return i

class KnownLocation:
    """Contains information such as name and 2D bounding box for a known location on the Dugged server."""
    @staticmethod
    def parse(values):
        dimension, location_name, x1, z1, x2, z2 = values

        # Skip locations with incomplete coords
        if x1 != 0 and z1 != 0 and x2 != 0 and z2 != 0:
            min_x = x1 if x1 < x2 else x2
            min_z = z1 if z1 < z2 else z2
            max_x = x2 if x2 > x1 else x1
            max_z = z2 if z2 > z1 else z1

            return KnownLocation(dimension, location_name, min_x, min_z, max_x, max_z)
        return None

    def __init__(self, dimension, location_name, min_x, min_z, max_x, max_z):
        self.dimension = dimension
        self.location_name = location_name
        self.min_x = min_x
        self.min_z = min_z
        self.max_x = max_x
        self.max_z = max_z

    def is_contained(self, x, z):
        # Determine whether the point is contained in the locations 2D bounding box
        return self.min_x <= x <= self.max_x and self.min_z <= z <= self.max_z

class ListCommand(BaseCommand):
    command_text = "!!list"
    dimensions = { -1: 'N', 0: 'O', 1: 'E' }
    known_locations = { -1: [], 0: [], 1: [] }

    def __init__(self, bot, command_cache, minecraft_ip, minecraft_port, current_folder, playerdata_folder):
        super().__init__(bot, command_cache)

        self.minecraft_ip = minecraft_ip
        self.minecraft_port = minecraft_port
        self.current_folder = current_folder
        self.playerdata_folder = playerdata_folder

    def help(self):
        return '`' + self.command_text + '`  **-**  Displays currently online players and their dimension\n'

    async def load_files(self):
        file = os.path.join(self.current_folder,'known_locations.json')

        with open(file, 'r') as f:
            json_data = json.loads(json_minify(f.read(None)))

            for entry in json_data:
                known_location = KnownLocation.parse(entry)

                if known_location != None:
                    ListCommand.known_locations[known_location.dimension].append(known_location)

    async def process(self, message, args):
        try:
            server = MinecraftServer(self.minecraft_ip, self.minecraft_port)
            query = server.query()

            if query.players.online > 0:
                online_list = query.players.names
                text1 = []

                async with CommandCache.semaphore:
                    for item in online_list:
                        nbt_file = nbt.NBTFile(os.path.join(self.playerdata_folder, convert_uuid(self.cache.uuids[self.cache.names.index(item)]) + '.dat'))
                        #nbt_file = nbt.NBTFile(os.path.join(self.playerdata_folder, '33804744-b7a3-4816-8ddc-02ac2d8c80cf.dat'))

                        dimension = nbt_file['Dimension'].value
                        x, y, z = (int(i.value) for i in nbt_file['Pos'])
                        known_location = first(ListCommand.known_locations[dimension], lambda i: i.is_contained(x, z))

                        text1.append('**' + formatNameForEmbed(item) + '** (' + ListCommand.dimensions.get(dimension, '?') + ('' if known_location == None else '@*' + known_location.location_name + '*') + ')')

                if self.bot:
                    await self.bot.send_message(message.channel, 'Players: ' + ', '.join(text1))
                else:
                    print('Players: ' + ', '.join(text1))
            else:
                if self.bot:
                    await self.bot.send_message(message.channel, 'No Player is currently online')
                else:
                    print('No Player is currently online')
        except:
            if self.bot:
                await self.bot.send_message(message.channel, 'An error occured while trying to reach the server')
            else:
                print('An error occured while trying to reach the server')


import discord
from discord.ext import commands
import asyncio
from discord.utils import get

class Mod(commands.Cog):
    def __init__(self, client):
        self.client = client

    @commands.command(aliases=['k'])
    @commands.has_permissions(kick_members=True)
    async def kick(self, ctx, member: discord.Member=None, *,reason=None):
        if not member:
            embed = discord.Embed(
                title = 'Please specify a member to Kick.',
                color = discord.Color.green()
            )
            await ctx.send(embed=embed)
            return
        await member.kick(reason=reason)
        embed = discord.Embed(
        title = f'Kick',
        color = discord.Color.green()
        )
        embed.add_field(name = "User :", value = f'> {member.name}', inline = 'true')
        embed.add_field(name = "Reason :", value = f'> {reason}', inline = 'true')
        embed.set_thumbnail(url=member.avatar_url)
        await ctx.send(embed=embed)

    @commands.command(aliases=['b'])
    @commands.has_permissions(ban_members=True)
    async def ban(self, ctx, member: discord.Member=None, *,reason=None):
        if not member:
            embed = discord.Embed(
                title = 'Please specify a member to Ban.',
                color = discord.Color.green()
            )
            await ctx.send(embed=embed)
            return
        await member.ban(reason=reason)
        embed = discord.Embed(
        title = f'Ban',
        description = f'```Reason :``` >  **{reason}**',
        color = discord.Color.green()
        )
        embed.add_field(name = "User :", value = f'> {member.name}', inline = 'true')
        embed.add_field(name = "Reason :", value = f'> {reason}', inline = 'true')
        embed.set_thumbnail(url=member.avatar_url)
        await ctx.send(embed=embed)

    @commands.command(aliases=['ub'])
    @commands.has_permissions(ban_members=True)
    async def unban(self, ctx, member:discord.Member, *,reason=None):
        banned_users = await ctx.guild.bans()
        member_name, member_discriminator = member.split('#')

        for ban_entry in banned_users:
            user = ban_entry.user

            if (user.name, user.discriminator) == (member_name, member_discriminator):
                await ctx.guild.unban(user,reason=reason)
                embed = discord.Embed(
                title = f'UnBan',
                description = f'```Reason :``` >  **{reason}**',
                color = discord.Color.green()
                )
                embed.add_field(name = "User :", value = f'> {member.name}', inline = 'true')
                embed.add_field(name = "Reason :", value = f'> {reason}', inline = 'true')
                embed.set_thumbnail(url=member.avatar_url)
                await ctx.send(embed=embed)
                return

    @commands.command(aliases=['m'])
    @commands.has_permissions(kick_members=True)
    async def mute(self, ctx, member: discord.Member=None,mute_minutes=0, *,reason=None):
        if get(ctx.guild.roles, name="Muted"):
            role = discord.utils.get(ctx.guild.roles, name='Muted')
            if not member:
                embed = discord.Embed(
                    title = 'Please specify a member to Mute.',
                    color = discord.Color.green()
                )
                await ctx.send(embed=embed)
                return

            await member.add_roles(role,reason=reason)
            mute = discord.Embed(
            title = f'Muted',
            color = discord.Color.green()
            )
            mute.add_field(name = "User :", value = f'> {member.name}', inline = 'true')
            mute.add_field(name = "Reason :", value = f'> {reason}', inline = 'true')
            mute.add_field(name = "Time :", value = f'> {mute_minutes}m', inline = 'false')
            mute.set_thumbnail(url=member.avatar_url)
            await ctx.send(embed=mute)

            if mute_minutes > 0:
                await asyncio.sleep(mute_minutes * 60)
                await member.remove_roles(role,reason='Times up')
                unmute = discord.Embed(
                    title = f'Unmuted',
                    color = discord.Color.green()
                )
                unmute.add_field(name = "User :", value = f'> {member.name}', inline = 'true')
                unmute.add_field(name = "Reason :", value = f'> Automatic Unmute', inline = 'true')
                unmute.add_field(name = "Time :", value = f'> {mute_minutes}m', inline = 'false')
                unmute.set_thumbnail(url=member.avatar_url)
                await ctx.send(embed=unmute)
        else:
            await ctx.send('A ``Muted`` role was not found in this server. Try Creating one and decline ``Send Message`` permissions to the channels u wish.')
            author = ctx.message.author
            await self.client.create_role(author.guild, name="Muted")

    @commands.command(aliases=['um'])
    @commands.has_permissions(kick_members=True)
    async def unmute(self, ctx, member: discord.Member=None, *,reason=None):
        role = discord.utils.get(ctx.guild.roles, name='Muted')
        if not member:
            await ctx.send('Please specify a member')
            return

        await member.remove_roles(role,reason=reason)
        embed = discord.Embed(
        title = f'Unmuted',

        color = discord.Color.green()
        )
        embed.add_field(name = "User :", value = f'> {member.name}', inline = 'true')
        embed.add_field(name = "Reason :", value = f'> {reason}', inline = 'true')
        embed.set_thumbnail(url=member.avatar_url)
        await ctx.send(embed=embed)

    @commands.command(aliases=['c','clear'])
    @commands.has_permissions(manage_messages=True)
    async def purge(self, ctx, amount=10):
        await ctx.channel.purge(limit=amount)
        embed = discord.Embed(
        title = f'``Logs :`` {ctx.author} deleted {amount} messages',
        color = discord.Color.green()
        )
        await ctx.send(embed=embed,delete_after=3)

def setup(client):
    client.add_cog(Mod(client))


# -*- coding: utf-8 -*-
import os.path
import io
from math import *
from random import shuffle
import numpy as np
import sys
import time
from collections import defaultdict

reload(sys)
sys.setdefaultencoding('utf8')

idx_unk = 0
str_unk = "<unk>"

idx_pad = 1
str_pad = "<pad>"

class Embeddings():

    def __init__(self, file, voc):
        w2e = {}
        if file is not None:
            with io.open(file, 'r', encoding='utf-8', newline='\n', errors='ignore') as f:
                self.num, self.dim = map(int, f.readline().split())
                i = 0
                for line in f:
                    i += 1
                    if i%10000 == 0:
                        if i%100000 == 0: sys.stderr.write("{}".format(i))
                        else: sys.stderr.write(".")
                    tokens = line.rstrip().split(' ')
                    if voc.exists(tokens[0]): w2e[tokens[0]] = tokens[1:]
            sys.stderr.write('Read {} embeddings ({} missing in voc)\n'.format(len(w2e),len(voc)-len(w2e)))
        else:
            sys.stderr.write('Embeddings not used!\n')

        # i need an embedding for each word in voc
        # embedding matrix must have tokens in same order than voc 0:<unk>, 1:<pad>, 2:le, ...
        self.matrix = []
        for tok in voc:
            if tok == str_unk or tok == str_pad or not tok in w2e: ### random initialize these tokens
                self.matrix.append(np.random.normal(0, 1.0, self.dim))
            else:
                self.matrix.append(np.asarray(w2e[tok], dtype=np.float32))

        self.matrix = np.asarray(self.matrix, dtype=np.float32)
        self.matrix = self.matrix / np.sqrt((self.matrix ** 2).sum(1))[:, None]

class Vocab():

    def __init__(self, dict_file):
        curr_time = time.strftime("[%Y-%m-%d_%X]", time.localtime())
        self.tok_to_idx = {}
        self.idx_to_tok = []
        self.idx_to_tok.append(str_unk)
        self.tok_to_idx[str_unk] = len(self.tok_to_idx) #0
        self.idx_to_tok.append(str_pad)
        self.tok_to_idx[str_pad] = len(self.tok_to_idx) #1
        with io.open(dict_file, 'r', encoding='utf-8', newline='\n', errors='ignore') as f:
            nline = 0
            for line in f:
                nline += 1
                line = line.strip()
                self.idx_to_tok.append(line)
                self.tok_to_idx[line] = len(self.tok_to_idx)

        self.length = len(self.idx_to_tok)
        sys.stderr.write('Read vocab ({} entries)\n'.format(self.length))

    def __len__(self):
        return len(self.idx_to_tok)

    def exists(self, s):
        return s in self.tok_to_idx

    def __iter__(self):
        for tok in self.idx_to_tok:
            yield tok

    def get(self,s):
        if type(s) == int: ### I want the string
            if s < len(self.idx_to_tok): return self.idx_to_tok[s]
            else:
                sys.stderr.write('error: key \'{}\' not found in vocab\n'.format(s))
                sys.exit()
        ### I want the index
        if s not in self.tok_to_idx: return idx_unk
        return self.tok_to_idx[s]

class Replace():

    def __init__(self, file):
        curr_time = time.strftime("[%Y-%m-%d_%X]", time.localtime())
        self.max_length = -1
        self.min_length = -1
        self.pos_to_wrd = defaultdict(list)
        with io.open(file, 'r', encoding='utf-8', newline='\n', errors='ignore') as f:
            nline = 0
            for line in f:
                nline += 1
                if nline==1:
                    tok = line.split(",")
                    self.min_length = int(tok[0])
                    self.max_length = int(tok[1])
                    continue
                seqpos, seqwrd = line.strip().split('\t')
                self.pos_to_wrd[seqpos].append(seqwrd)

        sys.stderr.write('Read replace ({} pos entries)\n'.format(len(self.pos_to_wrd)))

    def get(self,seq,pos):
        seqpos = " ".join(pos) #string
        if "-" in seqpos: return []
        seqwrd = " ".join(seq) #string

#        print("\tseqpos: {}".format(seqpos))
        if not seqpos in self.pos_to_wrd: return []
        seqwrds = self.pos_to_wrd[seqpos] #is a list of strings

        indexs = [i for i in range(min(100,len(seqwrds)))]
        shuffle(indexs)
        for i in indexs:
            seqwrd2=seqwrds[i] #string
#            print("\t\ti={} {}".format(i,seqwrd2))
            seq2=seqwrd2.split(' ') #vector
            totally_different = True
            for k in range(len(seq2)):
                if seq2[k] == seq[k]:
#                    print("not totally different")
                    totally_different = False
                    break
            if totally_different: return seq2

        return []

class Dataset():

    def __init__(self, file, voc_src, voc_tgt, replace_src, seq_size, max_sents, do_shuffle, mode):
        if file is None:
            return
        self.voc_src = voc_src
        self.voc_tgt = voc_tgt
        self.file = file
        self.seq_size = seq_size
        self.max_sents = max_sents
        self.do_shuffle = do_shuffle
        self.mode = mode
        self.data = []
        self.replace_src = replace_src
        self.length = 0 ### length of the data set to be used (not necessarily the whole set)
        self.annotated = False
        with io.open(self.file, 'r', encoding='utf-8', newline='\n', errors='ignore') as f:
            first = True
            for line in f:
                if first:
                    tokens = line.strip().split('\t')
                    if len(tokens)==3 and (tokens[0]=="P" or tokens[0]=="U" or tokens[0]=="I" or tokens[0]=="R"):
                        self.annotated = True
                    first = False
                self.data.append(line)
                self.length += 1
        if self.max_sents > 0:
            self.length = min(self.length,self.max_sents)
        sys.stderr.write('({} contains {} examples)\n'.format(self.file,len(self.data)))

    def __iter__(self):
        nsent = 0
        self.nsrc = 0
        self.ntgt = 0
        self.nunk_src = 0
        self.nunk_tgt = 0
        self.ndiv_src = 0
        self.ndiv_tgt = 0
        ### every iteration i get shuffled data examples if do_shuffle
        do_delete = True
        do_source = True
        indexs = [i for i in range(len(self.data))]
        if self.do_shuffle: shuffle(indexs)
        for index in indexs:
            tokens = self.data[index].strip().split('\t')
            if len(tokens) < 2 or len(tokens) > 4 or len(tokens[0])==0 or len(tokens[1])==0:
                sys.stderr.write("error: bad data entry \'{}\' in line={} [skipped]\n".format(self.data[index],index+1))
                continue

            if len(tokens)==3 and (tokens[0]=="P" or tokens[0]=="U" or tokens[0]=="I" or tokens[0]=="R"):
                self.annotated = True ### should not be done at each line... :-(
                src = tokens[1].split(' ')
                tgt = tokens[2].split(' ')
                yield self.tagged_sentence(src,tgt)
                nsent += 1
                continue

            src = tokens[0].split(' ')
            tgt = tokens[1].split(' ')

            if self.seq_size > 0 and (len(src) > self.seq_size or len(tgt) > self.seq_size): continue # filter out examples with more than seq_size tokens
            if self.max_sents > 0 and nsent > self.max_sents: break # already generated max_sents examples

            if 'p' in self.mode:
                yield self.parallel_sentence(src, tgt)
                nsent += 1

            if len(self.data)>1:
                if 'u' in self.mode:
                    yield self.uneven_sentence(src, tgt, index)
                    nsent += 1

                if 'i' in self.mode:
                    yield self.insert_sentence(src, tgt, index)
                    nsent += 1

            if 'd' in self.mode and len(tokens)==3:
                yield self.delete_sentence(src, tgt, tokens[2])
                nsent += 1

            if 'r' in self.mode and self.replace_src is not None and len(tokens)==4:
                yield self.replace_sentence(src, tgt, tokens[2], tokens[3])
                nsent += 1

    def __len__(self):
        return self.length

    def debug_sentence(self,tag,orig_src,src,isrc,src_tag,orig_tgt,tgt,itgt,tgt_tag):
        print(tag)
        print("\torig_src: {}".format(" ".join([s for s in orig_src])))
        print("\tsrc     : {}".format(" ".join([s for s in src])))
        print("\tisrc    : {}".format(" ".join([str(s) for s in isrc])))
        print("\tsrc_tag : {}".format(" ".join([str(s) for s in src_tag])))
        print("\torig_tgt: {}".format(" ".join([t for t in orig_tgt])))
        print("\ttgt     : {}".format(" ".join([t for t in tgt])))
        print("\titgt    : {}".format(" ".join([str(t) for t in itgt])))
        print("\ttgt_tag : {}".format(" ".join([str(t) for t in tgt_tag])))
#        print("   {}\nKK:{}\t{}\t{}\t{}".format(" ".join([s for s in orig_src])," ".join([s for s in src])," ".join([t for t in orig_tgt])," ".join([str(s) for s in src_tag])," ".join([str(t) for t in tgt_tag])))

    def tagged_sentence(self,src,tgt):
        isrc = []
        src_tag =[]
        for s in range(len(src)):
            if src[s][0]=="@":
                src_tag.append(1.0)
                src[s]=src[s][1:]
            else:
                src_tag.append(-1.0)
            isrc.append(self.voc_src.get(src[s]))

        itgt = []
        tgt_tag =[]
        for t in range(len(tgt)):
            if tgt[t][0]=="@":
                tgt_tag.append(1.0)
                tgt[t]=tgt[t][1:]
            else:
                tgt_tag.append(-1.0)
            itgt.append(self.voc_tgt.get(tgt[t]))
        self.keep_records(src_tag, tgt_tag, isrc, itgt)
        #self.debug_sentence("TAGGED",src,src,isrc,src_tag,tgt,tgt,itgt,tgt_tag)
        return isrc, itgt, src, tgt, src_tag, tgt_tag

    def original_words(self,src,tgt):
        isrc = []
        src_tag = []
        for s in src:
            isrc.append(self.voc_src.get(s))
            src_tag.append(-1.0) #positive example (non divergent)
        itgt = []
        tgt_tag = []
        for t in tgt:
            itgt.append(self.voc_tgt.get(t))
            tgt_tag.append(-1.0) #positive example (non divergent)

        return isrc, itgt, src, tgt, src_tag, tgt_tag

    def keep_records(self, src_tag, tgt_tag, isrc, itgt):
        self.ndiv_src += sum(1 for s in src_tag if s==1.0)
        self.ndiv_tgt += sum(1 for t in tgt_tag if t==1.0)
        self.nunk_src += sum(1 for s in isrc if s==idx_unk)
        self.nunk_tgt += sum(1 for t in itgt if t==idx_unk)
        self.nsrc += len(src_tag)
        self.ntgt += len(tgt_tag)

    def parallel_sentence(self,src,tgt):
        orig_src = src
        orig_tgt = tgt
        isrc, itgt, src, tgt, src_tag, tgt_tag = self.original_words(src,tgt)
        self.keep_records(src_tag, tgt_tag, isrc, itgt)
        #self.debug_sentence("PARALLEL",orig_src,src,isrc,src_tag,orig_tgt,tgt,itgt,tgt_tag)
        return isrc, itgt, src, tgt, src_tag, tgt_tag

    def uneven_sentence(self,src,tgt,ind):
        orig_src = src
        orig_tgt = tgt
        if ind%2==0: #replace tgt sentence
            correct_ratio = False
            while not correct_ratio:
                index = np.random.randint(0,len(self.data))
                while ind==index:
                    index = np.random.randint(0,len(self.data))
                tgt = self.data[index].strip().split('\t')[1].split(' ')
                ratio = len(tgt)/float(len(src))
                correct_ratio = (ratio>=0.5 and ratio<=2.0)
        else: #replace src sentence
            correct_ratio = False
            while not correct_ratio:
                index = np.random.randint(0,len(self.data))
                while ind==index:
                    index = np.random.randint(0,len(self.data))
                src = self.data[index].strip().split('\t')[0].split(' ')
                ratio = len(tgt)/float(len(src))
                correct_ratio = (ratio>=0.5 and ratio<=2.0)
        isrc, itgt, src, tgt, src_tag, tgt_tag = self.original_words(src,tgt)
        for i in range(len(src_tag)): src_tag[i] = 1.0 ### converting positive (-1.0) into negative examples (+1.0)
        for i in range(len(tgt_tag)): tgt_tag[i] = 1.0 ### converting positive (-1.0) into negative examples (+1.0)
        self.keep_records(src_tag, tgt_tag, isrc, itgt)
        #self.debug_sentence("UNEVEN",orig_src,src,isrc,src_tag,orig_tgt,tgt,itgt,tgt_tag)
        return isrc, itgt, src, tgt, src_tag, tgt_tag

    def insert_sentence(self,src,tgt,ind):
        orig_src = src
        orig_tgt = tgt
        ### base sentence
        isrc, itgt, src, tgt, src_tag, tgt_tag = self.original_words(src,tgt)
        where_src = -1
        where_tgt = -1
        ###
        ### add words in source side (src,isrc), mark these source words as divergent (src_tag)
        ###
        if len(src)<=len(tgt):
            ### choose randomly the added src sentence in data
            i = np.random.randint(0,len(self.data))
            while ind==i:
                i = np.random.randint(0,len(self.data))
            add_source = self.data[i].strip().split('\t')[0].split(' ')
            #i will consider only the initial/final nsrc words of the new add_source sentence
            nsrc = np.random.randint(1,len(tgt)+1)
            if nsrc>len(add_source): nsrc=len(add_source)
            if ind%2==0: #add in the begining (take the last nsrc words)
                add_src = add_source[len(add_source)-nsrc:]
                j=0
            else: #add in the end (take the initial nsrc words)
                add_src = add_source[:nsrc]
                j=len(src)
            where_src=j
            add_src_tag = [1.0] * len(add_src) # divergent
            add_isrc = []
            for s in add_src: add_isrc.append(self.voc_src.get(s))
            prev = src[:j]
            post = src[j:]
            src = prev + add_src + post
            prev = isrc[:j]
            post = isrc[j:]
            isrc = prev + add_isrc + post
            prev = src_tag[:j]
            post = src_tag[j:]
            src_tag = prev + add_src_tag + post
        ###
        ### add words in target side (tgt,itgt), mark these target words as divergent (tgt_tag)
        ###
        else:
            j = np.random.randint(0,len(self.data))
            while ind==j:
                j = np.random.randint(0,len(self.data))
            add_target = self.data[j].strip().split('\t')[1].split(' ')
            #i will consider only the initial/final ntgt words of the new add_target sentence
            ntgt = np.random.randint(1,len(src)+1)
            if ntgt>len(add_target): ntgt=len(add_target)
            if ind%2==0: #add in the begining (take the last ntgt words)
                add_tgt = add_target[len(add_target)-ntgt:]
                i=0
            else: #add in the end (take the initial ntgt words)
                add_tgt = add_target[:ntgt]
                i=len(tgt)
            where_tgt=i
            add_tgt_tag = [1.0] * len(add_tgt) # divergent
            add_itgt = []
            for t in add_tgt: add_itgt.append(self.voc_tgt.get(t))
            ### choose where to include the add_tgt sequence
            prev = tgt[:i]
            post = tgt[i:]
            tgt = prev + add_tgt + post
            prev = itgt[:i]
            post = itgt[i:]
            itgt = prev + add_itgt + post
            prev = tgt_tag[:i]
            post = tgt_tag[i:]
            tgt_tag = prev + add_tgt_tag + post

        self.keep_records(src_tag, tgt_tag, isrc, itgt)
        #self.debug_sentence("INSERT where_src={} where_tgt={}".format(where_src,where_tgt),orig_src,src,isrc,src_tag,orig_tgt,tgt,itgt,tgt_tag)
        return isrc, itgt, src, tgt, src_tag, tgt_tag

    def replace_sentence(self,src,tgt,align,srcpos):
        orig_src = list(src)
        orig_tgt = list(tgt)
        if len(src)<3: return self.parallel_sentence(src,tgt)
        src_from, src_to, tgt_from, tgt_to, replace_by = self.find_replacement(src, tgt, align, srcpos)
        if len(replace_by)==0 or src_from<0 or src_to<src_from or src_to>=len(src) or (src_from==0 and src_to==len(src)-1): return self.parallel_sentence(src,tgt)

        isrc, itgt, src, tgt, src_tag, tgt_tag = self.original_words(src,tgt)
        ###
        ### replace words in source side
        ###
        for s in range(src_from,src_to+1):
            src[s] = replace_by[s-src_from]
            isrc[s] = self.voc_src.get(src[s])
            src_tag[s] = 1.0
        ###
        ### keep words in target side except those aligned with the replaced in the source side
        ###
        for t in range(tgt_from,tgt_to+1):
            tgt_tag[t] = 1.0

        self.keep_records(src_tag, tgt_tag, isrc, itgt)
        #self.debug_sentence("REPLACE[{},{}] ({}=>{}) [{},{}]".format(src_from,src_to," ".join(orig_src[src_from:src_to+1])," ".join(src[src_from:src_to+1]),tgt_from,tgt_to), orig_src,src,isrc,src_tag, orig_tgt,tgt,itgt,tgt_tag)
        return isrc, itgt, src, tgt, src_tag, tgt_tag

    def find_replacement(self,src,tgt,align,srcpos):
        if len(src) < self.replace_src.min_length*2:
            return -1, -1, -1, -1, []
        ali = align.split(" ")
        if len(ali)==0: ### there must be at leat one alignment
            return -1, -1, -1, -1, []
        pos = srcpos.split(" ")
        if len(pos)!=len(src):
            return -1, -1, -1, -1, []

        #build alignment structures for fast access
        s2t_minmax = [[None, None] for i in range(len(src))]
        t2s_minmax = [[None, None] for i in range(len(tgt))]
        for a in ali:
            s, t = map(int, a.split('-'))
            if s2t_minmax[s][0] is None:
                s2t_minmax[s][0] = t
                s2t_minmax[s][1] = t
            else:
                if t < s2t_minmax[s][0]: s2t_minmax[s][0] = t
                if t > s2t_minmax[s][1]: s2t_minmax[s][1] = t
            if t2s_minmax[t][0] is None:
                t2s_minmax[t][0] = s
                t2s_minmax[t][1] = s
            else:
                if s < t2s_minmax[t][0]: t2s_minmax[t][0] = s
                if s > t2s_minmax[t][1]: t2s_minmax[t][1] = s

        #find the largest replacement that does not exceeds len(src)/2
        replace_by = []
        src_from = -1
        src_to = -1
        tgt_from = -1
        tgt_to = -1
        indexs = [i for i in range(len(src))]
        shuffle(indexs)
        for src_from in indexs:
            #for src_to in range(src_from+self.replace_src.min_length-1,src_from+self.replace_src.max_length):
            for src_to in range(src_from+self.replace_src.max_length-1,src_from+self.replace_src.min_length-2,-1):
                if src_to>=len(src): continue
                if src_to-src_from+1 > len(src)//2: continue
                tgt_from, tgt_to, replace_by = self.replacement(src,tgt,src_from,src_to,pos,s2t_minmax,t2s_minmax)
                if len(replace_by)>0:
                    return src_from, src_to, tgt_from, tgt_to, replace_by
        return src_from, src_to, tgt_from, tgt_to, replace_by

    def replacement(self,src,tgt,src_from,src_to,pos,s2t_minmax,t2s_minmax):
        src_seq = src[src_from:src_to+1]
        pos_seq = pos[src_from:src_to+1]
        src_rep = self.replace_src.get(src_seq,pos_seq)
        if len(src_rep)!=len(src_seq): return -1, -1, []

        tgt_from = 9999
        tgt_to = -1
        ### first i collect the [tgt_from,tgt_to] words aligned to [src_from, src_to]
        for s in range(src_from, src_to+1):
            tgt_min = s2t_minmax[s][0]
            tgt_max = s2t_minmax[s][1]
            if tgt_min is not None and tgt_min < tgt_from: tgt_from = tgt_min
            if tgt_max is not None and tgt_max > tgt_to:   tgt_to   = tgt_max
        if tgt_from == 9999 or tgt_to == -1: return -1, -1, []

        ### second i make sure that any target word in [tgt_from, tgt_to] is not aligned outside [src_from,src_to]
        for t in range(tgt_from, tgt_to+1):
            src_min = t2s_minmax[t][0]
            src_max = t2s_minmax[t][1]
            if src_min is not None and src_min < src_from: return -1, -1, []
            if src_max is not None and src_max > src_to:   return -1, -1, []

        return tgt_from, tgt_to, src_rep

    def delete_sentence(self,src,tgt,ali):
        orig_src = src
        orig_tgt = tgt
        src_from, src_to, tgt_from, tgt_to = self.find_self_aligned_sequence(src, tgt, ali)
        if src_from<0 or src_to<src_from or src_to>=len(src) or (src_from==0 and src_to==len(src)-1): return self.parallel_sentence(src, tgt)
        ### delete on the side with more words
        ### remove words in source side, mark divergence in target side
        ###
        if len(orig_src)>=len(orig_tgt):
            new_src = []
            isrc = []
            src_tag = []
            for s in range(len(src)):
                if s < src_from or s > src_to: ### keep the word
                    new_src.append(src[s])
                    isrc.append(self.voc_src.get(src[s]))
                    src_tag.append(-1.0) # positive example
            src = new_src
            #tgt remains as it was
            itgt = []
            tgt_tag = []
            for t in range(len(tgt)):
                itgt.append(self.voc_tgt.get(tgt[t]))
                if t < tgt_from or t > tgt_to: tgt_tag.append(-1.0) ### positive example (link is aligned)
                else: tgt_tag.append(1.0) ### negative example (link is deleted)
        ###
        ### remove words in target side, mark divergence in source side
        ###
        else:
            new_tgt = []
            itgt = []
            tgt_tag = []
            for t in range(len(tgt)):
                if t < tgt_from or t > tgt_to: ### keep the word
                    new_tgt.append(tgt[t])
                    itgt.append(self.voc_tgt.get(tgt[t]))
                    tgt_tag.append(-1.0) # positive example
            tgt = new_tgt
            #src remains as it was
            isrc = []
            src_tag = []
            for s in range(len(src)):
                isrc.append(self.voc_src.get(src[s]))
                if s < src_from or s > src_to: src_tag.append(-1.0) ### positive example (link is aligned)
                else: src_tag.append(1.0) ### negative example (link is deleted)

        self.keep_records(src_tag, tgt_tag, isrc, itgt)
        #self.debug_sentence("DELETE is_source={} [{},{}] [{},{}]".format(len(orig_src)>=len(orig_tgt),src_from,src_to,tgt_from,tgt_to), orig_src,src,isrc,src_tag, orig_tgt,tgt,itgt,tgt_tag)
        return isrc, itgt, src, tgt, src_tag, tgt_tag

    def find_self_aligned_sequence(self, src, tgt, align):
        ali = align.split(" ")
        if len(ali)==0: ### there must be at leat one alignment
            return -1, -1, -1, -1
        #0 : build alignment structures for fast access
        s2t_minmax = [[None, None] for i in range(len(src))]
        t2s_minmax = [[None, None] for i in range(len(tgt))]
        for a in ali:
            s, t = map(int, a.split('-'))
            if s2t_minmax[s][0] is None:
                s2t_minmax[s][0] = t
                s2t_minmax[s][1] = t
            else:
                if t < s2t_minmax[s][0]: s2t_minmax[s][0] = t
                if t > s2t_minmax[s][1]: s2t_minmax[s][1] = t
            if t2s_minmax[t][0] is None:
                t2s_minmax[t][0] = s
                t2s_minmax[t][1] = s
            else:
                if s < t2s_minmax[t][0]: t2s_minmax[t][0] = s
                if s > t2s_minmax[t][1]: t2s_minmax[t][1] = s
        #1 : find a target sequence [imin, imax] with at least one alignment
        max_size = 9
        while True:
            imin = np.random.randint(0,len(tgt))
            imax = imin + np.random.randint(0,min(max_size,len(tgt)-imin))
            jmin, jmax = self.MinMax(t2s_minmax,imin,imax)
            if jmin is not None and jmax is not None: break
        length_sequences = (imax-imin+1) + (jmax-jmin+1)
        j_or_i = 1
        #2 : find the minimum sequence of source/target words that contains i and are not aligned outside the sequence
        while True:
            if j_or_i == 0: jmin, jmax = self.MinMax(t2s_minmax,imin,imax)
            else:           imin, imax = self.MinMax(s2t_minmax,jmin,jmax)
            if length_sequences == (imax-imin+1) + (jmax-jmin+1): break #no added words stop iterating
            length_sequences = (imax-imin+1) + (jmax-jmin+1)
            j_or_i = (j_or_i + 1)%2
        #extend with unaligned words ???
        #filter bad examples: all words, too many tokens, etc.
        max_ratio = 0.5 #maximum ratio of deleted words
        if jmin==0 and jmax==len(src)-1: return -1, -1, -1, -1
        if imin==0 and imax==len(tgt)-1: return -1, -1, -1, -1
        if ((imax-imin+1) + (jmax-jmin+1))/(len(src)+len(tgt)) > max_ratio: return -1, -1, -1, -1
        return jmin,jmax,imin,imax

    def MinMax(self, a2b_minmax, amin, amax):
        bmin=None
        bmax=None
        for a in range(amin, amax+1):
            if a2b_minmax[a][0] is not None:
                if bmin is None or a2b_minmax[a][0] < bmin:
                    bmin = a2b_minmax[a][0]
            if a2b_minmax[a][1] is not None:
                if bmax is None or a2b_minmax[a][1] > bmax:
                    bmax = a2b_minmax[a][1]
        return bmin, bmax

def minibatches(data, minibatch_size, npad_src, npad_tgt):
    SRC, TGT, RAW_SRC, RAW_TGT, SRC_TAG, TGT_TAG = [], [], [], [], [], []
    max_src, max_tgt = 0, 0
    for (src, tgt, raw_src, raw_tgt, src_tag, tgt_tag) in data:
        if len(SRC) == minibatch_size:
            yield build_batch(SRC, TGT, RAW_SRC, RAW_TGT, SRC_TAG, TGT_TAG, max_src, max_tgt, npad_src, npad_tgt)
            SRC, TGT, RAW_SRC, RAW_TGT, SRC_TAG, TGT_TAG = [], [], [], [], [], []
            max_src, max_tgt = 0, 0
        if len(src) > max_src: max_src = len(src)
        if len(tgt) > max_tgt: max_tgt = len(tgt)
        SRC.append(src)
        TGT.append(tgt)
        RAW_SRC.append(raw_src)
        RAW_TGT.append(raw_tgt)
        SRC_TAG.append(src_tag)
        TGT_TAG.append(tgt_tag)

    if len(SRC) != 0:
        yield build_batch(SRC, TGT, RAW_SRC, RAW_TGT, SRC_TAG, TGT_TAG, max_src, max_tgt, npad_src, npad_tgt)

def build_batch(SRC, TGT, RAW_SRC, RAW_TGT, SRC_TAG, TGT_TAG, max_src, max_tgt, npad_src, npad_tgt):
    src_batch, tgt_batch, raw_src_batch, raw_tgt_batch, sign_src_batch, sign_tgt_batch, sign_batch, len_src_batch, len_tgt_batch = [], [], [], [], [], [], [], [], []
    ### build: src_batch, pad_src_batch sized of max_src
    batch_size = len(SRC)
    for i in range(batch_size):
        # add padding to have max_src/max_tgt words in all examples of current batch
        sign_src = SRC_TAG[i]
        sign_tgt = TGT_TAG[i]
        sign = max(max(sign_src),max(sign_tgt)) ### sign is divergent (+1) if there is any divergent word
        while len(sign_src) < max_src: sign_src.append(-1.0) ### continue filling up to max_tgt cells
        while len(sign_tgt) < max_tgt: sign_tgt.append(-1.0) ### continue filling up to max_tgt cells
        src = list(SRC[i])
        tgt = list(TGT[i])
        while len(src) < max_src: src.append(idx_pad) #<pad>
        while len(tgt) < max_tgt: tgt.append(idx_pad) #<pad>
        raw_src = list(RAW_SRC[i])
        raw_tgt = list(RAW_TGT[i])
        len_src = len(SRC[i])
        len_tgt = len(TGT[i])
        #add padding at the begining/end of the sequences for convolutions (npad = k1+k2/2 + 1)
        for _ in range(npad_src):
            src.append(idx_pad)
            src.insert(0,idx_pad)
        for _ in range(npad_tgt):
            tgt.append(idx_pad)
            tgt.insert(0,idx_pad)
        ### add to batches
        sign_src_batch.append(sign_src)
        sign_tgt_batch.append(sign_tgt)
        sign_batch.append(sign)
        src_batch.append(src)
        tgt_batch.append(tgt)
        raw_src_batch.append(raw_src)
        raw_tgt_batch.append(raw_tgt)
        len_src_batch.append(len_src)
        len_tgt_batch.append(len_tgt)

    return src_batch, tgt_batch, raw_src_batch, raw_tgt_batch, sign_src_batch, sign_tgt_batch, sign_batch, len_src_batch, len_tgt_batch


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"Created on 24 Jun 21:30 2017"
"@author: ratnadeepb"

import pandas as pd
import numpy as np
from scipy.stats import mode
from itertools import product
from sklearn.preprocessing import Imputer
from sklearn.preprocessing import LabelEncoder
from sklearn.preprocessing import OneHotEncoder
from sklearn.preprocessing import StandardScaler
from sklearn.model_selection import train_test_split

#################### Import Data ####################
data = pd.read_csv("/home/ratnadeepb/app/machine_learning/ml_programming/Data/Data.csv")
# data = pd.read_csv("Data/Data.csv")
columns = data.columns  # Reference to feature names

#################### Separate into features and target ####################
X = data[['Country', 'Age', 'Salary']].values
y = data['Purchased'].values

############### Separating Categorical and Numerical Data ############
cat_cls = [0]  # Row numbers in X with categorical data
cls = [cl for cl in range(X.shape[1]) if cl not in cat_cls]

C = np.array([list(elem) for elem in zip(*[X[:, i] for i in cat_cls])])
C = C.reshape(C.shape[0], -1)
N = np.array([list(elem) for elem in zip(*[X[:, i] for i in cls])])
N = N.reshape(N.shape[0], -1)

#################### Handling missing numerical data ####################
# Impute numerical variables
imputer_num = Imputer()
N = imputer_num.fit_transform(N)
c_cls = {}
for cl in cat_cls:
    c_cls[cl] = cat_cls.index(cl)
vals = {cl:mode(C[:,cl])[0][0] for cl in c_cls.values()}

missing_indices = [[rw,cl] for rw, cl in product(range(C.shape[0]), range(C.shape[1]))
                   if pd.isnull(C[rw, cl]) or  C[rw, cl] == 'nan']

for i in missing_indices:
    C[i[0]] = vals[i[1]]

#################### Encoding categorical data ####################
original_tags = []
for cl in list(c_cls.values()):
    original_tags.extend(np.unique(C[:, cl]))
    C_labels = LabelEncoder()
    C = C_labels.fit_transform(C[:, cl])

C = C.reshape(C.shape[0], -1)

categorical = []

for cl in list(c_cls.values()):
    onehotencoder = OneHotEncoder(n_values='auto', categorical_features=[cl])
    temp = onehotencoder.fit_transform(C[:, cl].reshape(-1, 1)).toarray()
    temp = np.delete(temp, -1, 1)  # Dropping the last array
    categorical.extend(temp)

C = np.array(categorical)

#################### Recreate features ####################
X = np.concatenate((C, N), axis=1)

#################### Handling missing response data ####################
response = mode(y)[0][0]

missing_y = [ind for ind in range(y.shape[0])
                   if pd.isnull(y[ind]) or  y[ind] == 'nan']

for i in missing_y:
    y[i] = response

y_labels = LabelEncoder()
y = y_labels.fit_transform(y)

#################### Train -Test Set ####################
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3)

#################### Feature Scaling ####################
# Here we are scaling the categorical variables
# The scaling can also be done to the numerical variables only
sc_X = StandardScaler()
X_train = sc_X.fit_transform(X_train)
X_test = sc_X.transform(X_train)


import tweepy
from tweepy import OAuthHandler
import json
import datetime as dt
import time
import os
import sys
import couchdb
from tweepy.utils import import_simplejson
from CouchDBConnect import CouchDBConnect
import core

'''
Python 3 and tweepy version 3.5.0.
'''

def load_api():
    ''' Function that loads the twitter API after authorizing the user. '''
    args = core.config('twitter','OAuth')

    consumer_key = args['consumer_key']
    consumer_secret = args['consumer_secret']
    access_token = args['access_token']
    access_secret = args['access_secret']
    auth = OAuthHandler(consumer_key, consumer_secret)
    auth.set_access_token(access_token, access_secret)
    # load the twitter API via tweepy
    return tweepy.API(auth)

def tweet_search(api, max_tweets,geocode,since_id,max_id=-1):
    ''' Function that takes in a search string 'query', the maximum
        number of tweets 'max_tweets'.
        It returns a list of tweepy.models.Status objects. '''

    searchedtweets = []
    while len(searchedtweets) < max_tweets:
        remainingtweets = max_tweets - len(searched_tweets)
        try:
            if(max_id== -1) :
                td = dt.datetime.now() - dt.timedelta(days=2)
                tweet_date = '{0}-{1:0>2}-{2:0>2}'.format(td.year, td.month, td.day)
                # get list of up to 100 tweets
                new_tweets = api.search(count=remaining_tweets,geocode=geocode,until=tweet_date)

            else :
                # Search tweets according to since_id and max_id
                new_tweets = api.search(count=remaining_tweets,geocode=geocode,max_id=max_id,since_id=since_id)
            print('found', len(new_tweets), 'tweets')
            if not new_tweets:
                print('no tweets found')
                break
            searchedtweets.extend(new_tweets)
        except tweepy.TweepError:
            print('exception raised, waiting 15 minutes')
            print('(until:', dt.datetime.now() + dt.timedelta(minutes=15), ')')
            time.sleep(15 * 60)
            break  # stop the loop
    return searchedtweets

def write_tweets(tweets):
    ''' Function that appends tweets to couchdb. '''

    json = import_simplejson()
    _db_handle=CouchDBConnect.connect_CouchDB()
    db = _db_handle["tweets_all"]
    dtweet={}
    for tweet in tweets:

            dtweet= json.dumps(tweet._json)
            store_tweet(db,dtweet)

def store_tweet(db,dtweet) :
    """Tweet as Dict"""
    try:
        doc = json.loads(dtweet)
        doc.update({'_id': doc['id_str']})
    except KeyError:
        print(
            "Warning: store_tweet() called, but tweet input "
            + "parameter does not contain an id_str."
        )
        return None
    if not isinstance(doc, dict):
        print(
            "Warning: store_dict() called, but doc input parameter"
            + " is not a dict."
        )
        return None
    # Attempt to save document to CouchDB
    try:
        response = db.save(doc)
    # If the _id already exists
    except couchdb.http.ResourceConflict as e:
        print(
            "Warning: The doc (_id: "
            + doc['_id']
            + ") is already exists "
        )

        print(
            "Overwriting doc (_id: "
            + doc['_id']
            + ")"
        )
def get_tweet_id(api, days_ago,geocode="Australia"):
    ''' Function that gets the ID of a tweet. I trecives api , days_ago
        and region for search as parameters'''
    # return an ID from __ days ago
    td = dt.datetime.now() - dt.timedelta(days=days_ago)
    tweet_date = '{0}-{1:0>2}-{2:0>2}'.format(td.year, td.month, td.day)
    # get list of up to 10 tweets
    tweet = api.search(count=10,until=tweet_date,geocode=geocode)
    # return the id of the first tweet in the list
    return tweet[0].id

def get_replies(db):
    """Retrieves a list of reply-to's that need to be
    downloaded.
    """
    print("get_replies",db.view('replies/replies'))
    try:
    #if(True) :
        for row in db.view('replies/replies',
                                 wrapper=None,
                                 group_level=1):
            if (row.value == 1):
                queue.update({row.key[0]: {}})
        for row in db.view('replies/replies',
                                 wrapper=None,
                                 group='true'):
            if row.key[0] in queue:
                queue[row.key[0]]['reply'] = row.key[1]
                queue[row.key[0]]['reply_user'] = row.key[2]
        print("get_replies1")
    except:
        print("Error")
    return queue
def iterate_replies(db,db2,api):
    """This method downloads tweets that tweets in our database
    have replied to.
    """
    try:
        replies = get_replies(db2)
        for r in replies:
            try:
                tweet = api.get_status(r)
                reply = replies[r]['reply']
                reply_user = replies[r]['reply_user']
                dtweet= json.dumps(tweet._json)
                store_tweet(db,dtweet)
            except:
                print("Exception1")
                pass
    except:
        print("Exception2")
        pass
def main():
    ''' This is a script that continuously searches for tweets
        that were created over a given number of days. The search
        dates and search phrase can be changed below. '''

    _db_handle=CouchDBConnect.connect_CouchDB()
    db2=_db_handle["tweets"]
    db = _db_handle["tweets_all"]

    time_limit = 25  # runtime limit in hours
    max_tweets = 100  # number of tweets per search (will be
    # iterated over) - maximum is 100
    min_days_old, max_days_old = 0, 7  # search limits e.g., from 7 to 8
    # gives current weekday from last week,
    # min_days_old=0 will search from right now

    Australia = '-34.337655,135.697131,2700km'

    api = load_api()
    max_id=-1
    while(True):
        #for search_phrase in search_phrases:
         if(True):
            if(max_id==-1) :
                  since_id=get_tweet_id(api, days_ago=4,geocode=Australia)
            else :
                  since_id=max_id
            json_file="json_file.json"

            ''' tweet gathering loop  '''
            start = dt.datetime.now()
            end = start + dt.timedelta(hours=time_limit)
            count = 0
            exitcount = 0
            while dt.datetime.now() < end:
                count += 1
                print('count =', count)
                # collect tweets
                try:
                    with open(json_file, 'r') as f:
                        lines= f.readlines()
                        max_id = json.loads(lines[-1])['id']

                        print('Searching from the bottom ID in file ', max_id)
                except FileNotFoundError:
                    max_id = -1
                tweets= tweet_search(api, max_tweets,geocode=Australia,since_id=since_id,max_id=max_id)
                # write tweets to couchdb in JSON format

                if tweets:
                    write_tweets(tweets)
                    with open(json_file, 'w') as f:
               # write the tweets to a json file for getting the max_id after each iteration.
                       for tweet in tweets:
                            json.dump(tweet._json, f)
                            f.write('\n')
                    exitcount = 0
                else:
                    exitcount += 1
                    if exitcount == 3:
                           break

if __name__ == "__main__":
    main()