# -*- coding: utf-8 -*-
"""
Created on 10-04-2015

@author: Diogo Silva

Evidence accumulation clustering. This module aims to include all
features of the Matlab toolbox plus addressing NxK co-association
matrices.

TODO:
- clustering of non-square co-association matrix
- link everything
- add sanity checks on number of samples of partitions
- robust exception handling
- fix centroid-based prototype creation
- convert dissimilarity matrix to float32 (why?)
"""

import numpy as np
from random import sample
from numba import jit, njit
# from sklearn.neighbors import NearestNeighbors

from scipy.sparse import csr_matrix
from scipy.cluster.hierarchy import linkage,dendrogram
from scipy.sparse.csgraph import connected_components

from scipy_numba.sparse.csgraph import minimum_spanning_tree
from scipy_numba.spatial.distance import squareform

from MyML.cluster.linkage import scipy_numba_slink_wraper as slink
from MyML.cluster.linkage import labels_from_Z
from MyML.cluster.K_Means3 import K_Means

from MyML.EAC.sparse import EAC_CSR, _compute_max_assocs_from_ensemble as biggest_cluster_size
from MyML.EAC.full import EAC_FULL

def sqrt_rule(n):
    n_clusters = [np.sqrt(n)/2, np.sqrt(n)]
    n_clusters = map(np.ceil, n_clusters)
    n_clusters = map(int, n_clusters)
    return n_clusters

def sqrt2_rule(n):
    n_clusters = [np.sqrt(n), np.sqrt(n)*2]
    n_clusters = map(np.ceil, n_clusters)
    n_clusters = map(int, n_clusters)
    return n_clusters

class EAC():

    def __init__(self, n_samples, **kwargs):
        """
        mat_sparse         : stores co-associations in a sparse matrix
        mat_half         : stores co-associations in pdist format, in an (n*(n-1))/2 length array
        """

        self.n_samples = n_samples

        # check if all arguments were passed as a dictionary
        args = kwargs.get("args")
        if args is not None and type(args) == dict:
            kwargs == args

        ## generate ensemble parameters
        self.n_partitions = kwargs.get("n_partitions", 100)
        self.iters = kwargs.get("iters", 3)
        self.n_clusters = kwargs.get("n_clusters", "sqrt")
        self.toFiles = False
        self.toFiles_folder = None

        ## build matrix parameters
        self.condensed = kwargs.get("condensed", True)
        self.kNN = kwargs.get("kNN", False)
        self.assoc_dtype = kwargs.get("assoc_dtype", np.uint8)

        # sparse matrix parameters
        self.sparse = kwargs.get("sparse", False)
        self.sp_sort = kwargs.get("sparse_sort_mode", "surgical")
        self.sp_max_assocs = kwargs.get("sparse_max_assocs", None)
        self.sp_max_assocs_factor = kwargs.get("sparse_max_assocs_factor", 3)
        self.sp_max_assocs_mode = kwargs.get("sparse_max_assocs_mode", "linear")
        self.sp_keep_degree = kwargs.get("sparse_keep_degree", False)

        # if not sparse and not kNN then it is full matrix
        if not self.sparse  and not self.kNN:
            self.full = True
        else:
            self.full = False

        ## final clustering parameters
        self.linkage = kwargs.get("linkage", "SL")

    def _validate_params(self):
        pass

    def generateEnsemble(self):
        pass

    def buildMatrix(self, ensemble):

        if self.sparse:
            if self.sp_max_assocs is None:
                self.sp_max_assocs = biggest_cluster_size(ensemble)
                self.sp_max_assocs *= self.sp_max_assocs_factor

            coassoc = EAC_CSR(self.n_samples, max_assocs=self.sp_max_assocs,
                              condensed=self.condensed,
                              max_assocs_type=self.sp_max_assocs_mode,
                              sort_mode=self.sp_sort,
                              dtype=self.assoc_dtype)

            coassoc.update_ensemble(ensemble)
            coassoc._condense(keep_degree = self.sp_keep_degree)
        elif self.full:
            coassoc = EAC_FULL(self.n_samples, condensed=self.condensed,
                               dtype=self.assoc_dtype)
            coassoc.update_ensemble(ensemble)
        elif self.kNN:
            raise NotImplementedError("kNN matrix building has not been included in this version.")
        else:
            raise ValueError("Build matrix: No sparse, no full, no kNN. No combination possible.")

        self.coassoc = coassoc

        # received names of partition files
        # if files:
        #     for partition_file in ensemble:
        #         partition = self._readPartition(partition_file) # read partition from file
        #         self._update_coassoc_matrix(partition) # update co-association matrix
        # # received partitions
        # else:
        #     for partition in ensemble:
        #         self._update_coassoc_matrix(partition) # update co-association matrix

    def finalClustering(self, n_clusters=0):
        if self.sparse:
            n_fclusts, labels = sp_sl_lifetime(self.coassoc.csr,
                                               max_val=self.n_partitions,
                                               n_clusters=n_clusters)
        elif self.full:
            n_fclusts, labels = full_sl_lifetime(self.coassoc.coassoc,
                                                 self.n_samples,
                                                 max_val=self.n_partitions,
                                                 n_clusters=n_clusters)
        elif self.kNN:
            raise NotImplementedError("kNN not included in this version yet.")
        else:
            raise ValueError("Final clustering: No sparse, no full, no kNN. No combination possible.")

        self.n_fclusts = n_fclusts
        self.labels = labels
        return labels

    def fit(self, ensemble,files=False, assoc_mode="full", prot_mode="none",
            nprot=None, link='single', build_only=False):
        """
        ensemble    : list of partitions; each partition is a list of
                      arrays (clusterings); each array contains the indices
                      of the cluster's data;  if files=True, partitions is
                      a list of file names, each corresponding to a partition
        assoc_mode  : type of association matrix; "full" - NxN, "prot" - NxK prototypes
        prot_mode   : how to build the prototypes; "random" - random selection
                      of K data points, "knn" for K-nearest neighbours, "other"
                      for K centroids/medoids
        nprot       : num. of prototypes to use; default = sqrt(num. of samples)
        """
        # how to build association matrix
        if self._assoc_mode is None:
            self._assoc_mode = assoc_mode
        # how to build prototypes
        if self._prot_mode is None:
            self._prot_mode = prot_mode

        # create co-association matrix
        self._coassoc = self._create_coassoc(assoc_mode, self.n_samples, nprot=nprot)

        if prot_mode is not "none":
            # changing assoc_mode for the matrix updates
            if prot_mode == "knn":
                self._assoc_mode="knn"

            elif assoc_mode == "full" and prot_mode == "random":
                self._assoc_mode = "full_random"

            elif prot_mode == "random":
                self._assoc_mode = "random"

            else:
                self._assoc_mode="other"

            self._build_prototypes(nprot=nprot, mode=prot_mode, data=self.data)

        self.n_partitions = 0

        # delete diagonal
        #self._coassoc[xrange(self.n_samples),xrange(self.n_samples)] = np.zeros(self.n_samples)

        # convert sparse matrix to convenient format, if it is sparse
        if self.mat_sparse:
            self._coassoc = self._coassoc.tocsr()
        # else:
        # 	self._coassoc[np.diag_indices_from(self._coassoc)] = 0

    def _create_coassoc(self, mode, nsamples, nprot=None):
        if self.condensed:
            n = sum(xrange(1, nsamples))
            coassoc = np.zeros(n, dtype=self.assoc_type)
        elif mode == "full":
            if self.mat_sparse:
                coassoc = sparse_type((nsamples, nsamples), dtype=self.assoc_type)
            else:
                coassoc = np.zeros((nsamples, nsamples), dtype=self.assoc_type)
        elif mode =="prot":
            if nprot == None:
                nprot = np.sqrt(nsamples)
            coassoc = np.zeros((nsamples,nprot), dtype=self.assoc_type)
        else:
            validValues=("full", "prot")
            raise ValueError("mode value should be from the list:\t" + str(validValues))

        return coassoc

    def _readPartition(self, filename):
        # list to hold the cluster arrays
        partition = list()

        with open(filename, "r") as pfile:
            # read cluster lines
            for cluster_line in pfile:
                if cluster_line == '\n':
                    continue
                cluster = np.fromstring(cluster_line, sep=',', dtype=np.int32)
                partition.append(cluster)

        return partition

    # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
    # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
    # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
    # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
    # # # #                                                             # # # #
    # # # #                                                             # # # #
    # # # #                                                             # # # #
    # # # #                      BUILD PROTOTYPES                       # # # #
    # # # #                                                             # # # #
    # # # #                                                             # # # #
    # # # #                                                             # # # #
    # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
    # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
    # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
    # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #

    def _build_prototypes(self, nprot=None, mode="random", data=None):
        if nprot == None:
            nprot = np.sqrt(self.n_samples)

        if mode == "random":
            self.k_labels = self._build_random_prototypes(nprot, self.n_samples)
            self.k_labels.sort()

        elif mode == "knn":
            if data is None:
                raise Exception("Data needs to be set for this method of choosing prototypes.")
            self.k_neighbours = self._build_knn_prototypes(nprot, data)

        elif mode == "other":
            if data is None:
                raise Exception("Data needs to be set for this method of choosing prototypes.")
            self.k_labels = self._build_k_prototypes(nprot, data)

        else:
            validValues=("random","knn","other")
            raise ValueError("Mode value should be from the list:\t" + str(validValues))

    def _build_random_prototypes(self, nprot, nsamples):

        # select nprot unique random samples from the dataset
        return np.array(sample(xrange(nsamples), nprot), dtype=np.int32)

    def _build_knn_prototypes(self, nprot, data):
        """
        K-Nearest Neighbours algorithm
        should return an NxK array with the labels
        """
        #first neighbour is the point itself, it gets discarded afterwards
        nneigh = nprot + 1

        # Minkowski distance is a generalization of Euclidean distance and
        # is equivelent to it for p=2
        neigh = NearestNeighbors(n_neighbors=nneigh, radius=1.0,
                                 algorithm='auto', leaf_size=30,
                                 metric='minkowski', p=2)
        neigh.fit(data)

        k_indices = neigh.kneighbors(X=data, return_distance=False)
        k_indices = k_indices[:,1:] # discard first neighbour

        return k_indices

    def _build_k_prototypes(self, nprot, data):
        # K-Means / K-Medoids algorithm
        # should return a N-length array with he indices of the chosen data
        grouper = K_Means()
        grouper._centroid_mode = "index"
        grouper.fit(data, nprot, iters=300, mode="cuda", cuda_mem='manual',
                    tol=1e-4, max_iters=300)
        centroids = grouper.centroids

        nclusters = centroids.shape[0]

        # TODO - very inefficient
        k_labels = np.zeros(nclusters, dtype=np.int32)

        for k in xrange(nclusters):
            dist = data - centroids[k]
            dist = dist ** 2
            dist = dist.sum(axis=1)

            k_labels[k] = dist.argmin()
        return k_labels

    # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
    # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
    # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
    # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
    # # # #                                                             # # # #
    # # # #                                                             # # # #
    # # # #                                                             # # # #
    # # # #              UPDATE CO-ASSOCIATION MATRIX                   # # # #
    # # # #                                                             # # # #
    # # # #                                                             # # # #
    # # # #                                                             # # # #
    # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
    # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
    # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
    # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #

    def _update_coassoc_k(self, assoc_mat, clusters, k_labels):
        """
        Updates an NxK co-association matrix.
        k_labels is an array (List, not np.ndarray) of length K where the k-th
        element is the index of a data point that corresponds to the
        k-th prototype.
        """

        nclusters = len(clusters)
        for i in xrange(nclusters): # for each cluster in ensemble
            # if cluster has more than 1 sample (i.e. not outlier)
            if clusters[i].size > 1:

                # all data points in cluster - rows to select
                n_in_cluster = clusters[i]

                ## select prototypes present in cluster - columns to select
                # in1d checks common values between two 1-D arrays (a,b) and
                # returns boolean array with the shape of a with value True on
                # the indices of common values
                k_in_cluster = np.where(np.in1d(k_labels, n_in_cluster))[0]

                if k_in_cluster.size == 0:
                    continue

                # this indexing selects the rows and columns specified by
                # n_in_cluster and k_in_cluster; np.newaxis is alias for None
                assoc_mat[n_in_cluster[:, np.newaxis], k_in_cluster] += 1

    def _update_coassoc_full_k(self, assoc_mat, clusters, k_labels):
        """
        Updates an NxN co-association matrix with only K prototypes specified
        by k_labels. k_labels is an array (List, not np.ndarray) of length K
        where the k-th element is the index of a data point that corresponds
        to the k-th prototype.
        """

        nclusters = len(clusters)
        for i in xrange(nclusters): # for each cluster in ensemble
            # if cluster has more than 1 sample (i.e. not outlier)
            if clusters[i].size > 1:

                # all data points in cluster - rows to select
                n_in_cluster = clusters[i]

                ## select prototypes present in cluster - columns to select
                # in1d checks common values between two 1-D arrays (a,b) and
                # returns boolean array with the shape of a with value True on
                # the indices of common values
                prots_in_cluster = np.intersect1d(k_labels, n_in_cluster)
                points_in_cluster = np.setdiff1d(n_in_cluster, prots_in_cluster)

                if prots_in_cluster.size == 0:
                    continue

                # this indexing selects the rows and columns specified by
                # n_in_cluster and k_in_cluster; np.newaxis is alias for None
                # select all rows that are not prots and all columns that are
                # prots and increment them
                assoc_mat[points_in_cluster[:, np.newaxis], prots_in_cluster] += 1
                assoc_mat[prots_in_cluster[:, np.newaxis], points_in_cluster] += 1

                # select all rows and columns that are prots and increment them
                assoc_mat[prots_in_cluster[:, np.newaxis], prots_in_cluster] += 1

    def _update_coassoc_knn(self, assoc_mat, clusters, k_neighbours):
        """
        Updates an NxK co-association matrix.
        k_neighbours is an NxK array where the k-th element of the i-th row is
        the index of a data point that corresponds to the k-th nearest neighbour
        of the i-th data point. That neighbour is the k-th prototype of the
        i-th data point.
        """
        nclusters = len(clusters)
        for i in xrange(nclusters):

            if clusters[i].size > 1:

                # all data points in cluster - rows to select
                n_in_cluster = clusters[i]

                # update row j of matrix
                for j in n_in_cluster:
                    # all prototypes in cluster - columns to select
                    k_in_cluster = np.in1d(k_neighbours[j], n_in_cluster)

                    if k_in_cluster.size == 0:
                        continue

                    # this indexing selects the rows and columns specified by
                    # n_in_cluster and k_in_cluster
                    assoc_mat[j, k_in_cluster] += 1 # newaxis is alias for None
        pass
    # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
    # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
    # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
    # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
    # # # #                                                             # # # #
    # # # #                                                             # # # #
    # # # #                                                             # # # #
    # # # #                      OPERATIONS                             # # # #
    # # # #                                                             # # # #
    # # # #                                                             # # # #
    # # # #                                                             # # # #
    # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
    # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
    # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
    # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
    def apply_threshold(self, threshold):
        """
        threshold   : all co-associations whose value is below
                      threshold * max_val are zeroed
        max_val     : usually number of partitions
        assoc_mat   : co-association matrix
        """
        assoc_mat = self._coassoc
        max_val = self.n_partitions
        apply_threshold_to_coassoc(threshold, max_val, assoc_mat)

    def getMaxAssocs(self):
        """\
        Returns the maximum number of co-associations a sample has and the
        index of that sample.\
        """
        # if not hasattr(self, 'degree'):
        #     self._getAssocsDegree()
        # return self.degree.max()

        if not self.mat_sparse:
            max_assocs, max_idx = get_max_assocs_in_sample(self._coassoc)
        else:
            max_assocs, max_idx = get_max_assocs_in_sample_csr(self._coassoc)
        return max_assocs, max_idx

    def _getAssocsDegree(self):
        self.degree = np.zeros(self.n_samples, dtype=np.int32)
        if self.condensed:
            error_str = "Getting degree from condensed matrix. Alternative: " +\
                         "convert to 2d, get degree, multiply by 2."
            raise NotImplementedError(error_str)
        elif not self.mat_sparse:
            full_get_assoc_degree(self._coassoc, self.degree)
        else:
            self.degree = self._coassoc.indptr[1:] - self._coassoc.indptr[:-1]
        self.nnz = self.degree.sum()

    def getNNZAssocs(self):
        """Get total number of associations in co-association matrix."""
        if not self.mat_sparse:
            #return np.count_nonzero(self.._coassoc)
            return numba_array2d_nnz(self._coassoc, self._coassoc.shape[0],
                                     self._coassoc.shape[1])
        else:
            return self._coassoc.getnnz()

# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
# # # #                                                             # # # #
# # # #                                                             # # # #
# # # #                                                             # # # #
# # # #                      FINAL CLUSTERING                       # # # #
# # # #                                                             # # # #
# # # #                                                             # # # #
# # # #                                                             # # # #
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #

def sp_sl_lifetime(mat, max_val=False, n_clusters=0):
    """
    Converts graph weights to dissimilarities if input graph is in
    similarities. Computes MST (Kruskal) of dissimilarity graph.
    Compute number of disconnected clusters (components).
    Sort MST in increasing order to get equivalent of SL clustering.
    Compute lifetimes if number of clusters is not provided.
    Make necessary cuts to have the desired number of clusters.
    Compute connected components (clusters) after the cuts.

    Inputs:
        graph           : dis/similarity matrix in CS form.
        max_val         : maximum value from which dissimilarity will be
                          computed. If False (default) assumes input graph
                          already encodes dissimilarities.
        n_clusters      : number of clusters to compute. If 0 (default),
                          use lifetime criteria.
    Outputs:
        n_fclusts       : final number of clusters
        labels          : final clustering labels
    """

    dtype = mat.dtype

    # converting to diassociations
    if max_val != False:
        mat.data = max_val + 1 - mat.data

    # get minimum spanning tree
    mst = minimum_spanning_tree(mat)

    # compute number of disconnected components
    n_disconnect_clusters = mst.shape[0] - mst.nnz

    # sort associations by weights
    asort = mst.data.argsort()
    sorted_weights = mst.data[asort]

    if n_clusters == 0:
        cont, max_lifetime = lifetime_n_clusters(sorted_weights)

        if n_disconnect_clusters > 1:
            # add 1 to max_val as the maximum weight because I also added
            # 1 when converting to diassoc to avoid having zero weights
            disconnect_lifetime = max_val + 1 - sorted_weights[-1]

            # add disconnected clusters to number of clusters if disconnected
            # lifetime is smaller
            if max_lifetime > disconnect_lifetime:
                cont += n_disconnect_clusters - 1
            else:
                cont = n_disconnect_clusters

        nc_stable = cont
    else:
        nc_stable = n_clusters

    # cut associations if necessary
    if nc_stable > n_disconnect_clusters:
        n_cuts = nc_stable - n_disconnect_clusters

        mst.data[asort[-n_cuts:]] = 0
        mst.eliminate_zeros()

    if nc_stable > 1:
        n_comps, labels = connected_components(mst)
    else:
        labels = np.empty(0, dtype=np.int32)
        n_comps = 1

    return n_comps, labels

def full_sl_lifetime(mat, n_samples, max_val=False, n_clusters=0):

    dtype = mat.dtype

    # convert to diassoc
    if mat.ndim == 2:
        mat = squareform(mat)

    # converting to diassociations
    if max_val != False:
        make_diassoc_1d(mat, max_val + 1)

    #Z = linkage(mat, method="single")
    Z = slink(mat, n_samples)

    if n_clusters == 0:

        cont, max_lifetime = lifetime_n_clusters(Z[:,2])

        nc_stable = cont
    else:
        nc_stable = n_clusters

    if nc_stable > 1:
        labels = labels_from_Z(Z, n_clusters=nc_stable)
        # rename labels
        i=0
        for l in np.unique(labels):
            labels[labels == l] = i
            i += 1
    else:
        labels = np.empty(0, dtype=np.int32)

    return nc_stable, labels

def lifetime_n_clusters(weights):
    # compute lifetimes
    lifetimes = weights[1:] - weights[:-1]

    # maximum lifetime
    m_index = np.argmax(lifetimes)
    th = lifetimes[m_index]

    # get number of clusters from lifetimes
    indices = np.where(weights >weights[m_index])[0]
    if indices.size == 0:
        cont = 1
    else:
        cont = indices.size + 1

    #testing the situation when only 1 cluster is present
    # if maximum lifetime is smaller than 2 times the minimum
    # don't make any cuts (= 1 cluster)
    # max>2*min_interval -> nc=1
    close_to_zero_indices = np.where(np.isclose(lifetimes, 0))
    minimum = np.min(lifetimes[close_to_zero_indices])

    if th < 2 * minimum:
        cont = 1

    return cont, th

# 2d
@njit
def make_diassoc_2d(ary, val):
    for row in range(ary.shape[0]):
        for col in range(ary.shape[1]):
            tmp = ary[row,col]
            ary[row,col] = val - tmp

#1d
@njit
def make_diassoc_1d(ary, val):
    for i in range(ary.size):
        tmp = ary[i]
        ary[i] = val - tmp


import pandas as pd
from keras.models import Sequential
from keras.layers import Dense, Dropout
from keras.wrappers.scikit_learn import KerasClassifier
from keras.utils import np_utils

# return the best three results
def top_n(matrix_prob, label_map):
	ans = []
	for line in matrix_prob:
		rank = [label_map[item[0]] for item in sorted(enumerate(line), key=lambda v:v[1], reverse=True)]
		ans.append(rank[:3])
	return ans
# basic neural network model
def basic_model():
	model = Sequential()
	model.add(Dense(output_dim=500, input_dim=100, activation='relu'))
	model.add(Dropout(0.2))
	model.add(Dense(output_dim=42, input_dim=500, activation='softmax'))
	model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
	return model

if __name__ == '__main__':
	X = pd.read_csv('./data/triple_train_x_mean.txt', header=None, encoding='utf-8')
	Y = pd.read_csv('./data/triple_train_y.txt', header=None, encoding='utf-8')
	X_test = pd.read_csv('./data/triple_test_x_mean.txt', header=None, encoding='utf-8')
	matrix_y = np_utils.to_categorical(Y,42)
	# KerasClassifier analysis
	classifier = KerasClassifier(build_fn=basic_model, nb_epoch=10, batch_size=500)
	classifier.fit(X, Y)

	pred_prob = classifier.predict_proba(X_test)

	with open('./model/task2_label_space.txt', encoding='utf-8') as flabel:
		label_map = flabel.read().split()
	pd.DataFrame(top_n(pred_prob, label_map)).to_csv('./data/task2_ans_int_index.txt', index=None, header=None, encoding='utf-8')


#! /usr/bin/python3
""" hamming.py

Error correction and detection using Hamming Codes

Usage:
	$ ./hamming.py [options] [input_string]

Options:
	-e ..., --encode=...		string to encode to hamming code
	-d ..., --decode=...		hamming code to decode into message after error correction

Examples:
	$ ./hamming.py -e secretmessgage

	$ ./hamming.py -d recievedmessage

Author: Aman Hussain
Last Modified: 28th March, 2017
E-mail: aman@amandavinci.me
"""

import sys
import math
import copy
import getopt
import binascii
from itertools import zip_longest

def main(argv):
    try:
        opts, args = getopt.getopt(argv, "he:d:", ["help", "encode=", "decode="])
    except getopt.GetoptError:
        usage()
        sys.exit(2)

    if not opts:
    	usage()
    	sys.exit()

    for opt, arg in opts:
        if opt in ("-h", "--help"):
            usage()
            sys.exit()

        elif opt in ("-e", "--encode"):
            data = arg
            print(hamming_encode(data))
            sys.exit()

        elif opt in ("-d", "--decode"):
            data = arg
            print(hamming_decode(data))
            sys.exit()

def usage():
	usage = "Usage: ./hamming.py [options] [input_string]"
	print(usage)

def hamming_encode(data):
	"""Encodes data into hamming code

	Args:
		Data to be encoded

	Returns:
		Hamming coded data
	"""
	# Convert string into binary bits
	databits = list(text_to_bits(data))

	# Set parity bits and return the hamming code
	code = set_parity(databits)

	return ''.join(code)

def hamming_decode(code):
	"""Decodes hamming code after error correction and detection

	Args:
		Hamming code to be decoded

	Returns:
		Original Data if error can be corrected, null otherwise
	"""

	# Converting string to list
	data=list(code)
	data.reverse()
	c,ch,j,r,error,h,parity_list,h_copy=0,0,0,0,0,[],[],[]

	for k in range(0,len(data)):
		p=(2**c)
		h.append(int(data[k]))
		h_copy.append(data[k])
		if(p==(k+1)):
			c=c+1

	for parity in range(0,(len(h))):
		ph=(2**ch)
		if(ph==(parity+1)):

			startIndex=ph-1
			i=startIndex
			toXor=[]

			while(i<len(h)):
				block=h[i:i+ph]
				toXor.extend(block)
				i+=2*ph

			for z in range(1,len(toXor)):
				h[startIndex]=h[startIndex]^toXor[z]
			parity_list.append(h[parity])
			ch+=1
	parity_list.reverse()
	error=sum(int(parity_list) * (2 ** i) for i, parity_list in enumerate(parity_list[::-1]))

	if((error)==0):
		c,data_bits=0,[]
		for k in range(0,len(data)):
			p=2**c
			if(p==(k+1)):
				c=c+1
			else:
				data_bits.append(h[k])
		data_bits.reverse()
		print(int(''.join(map(str, data_bits))))

	elif((error)>=len(h_copy)):
		return -1

	else:
		print('Error is in',error,'bit')

		if(h_copy[error-1]=='0'):
			h_copy[error-1]='1'

		elif(h_copy[error-1]=='1'):
			h_copy[error-1]='0'
		c,data_bits=0,[]
		for k in range(0,len(data)):
			p=2**c
			if(p==(k+1)):
				c=c+1
			else:
				data_bits.append(h[k])
		data_bits.reverse()
		print(int(''.join(map(str, data_bits))))

def check_parity(code):

	error_sum = 0

	# Checking with the correct parity bits
	for i in range(len(code)):

		# Parity locations
		if isPowerOf2(i+1):
			parity = compute_parity(code, i+1)

			# Computed parity does not match
			if parity != code[i]:
				error_sum = error_sum + i+1

	return error_sum

def set_parity(bits):
	"""Sets parity bits into the original data bits"""

	# Inserting the parity positions
	for i in range(len(bits)):
		if isPowerOf2(i+1):
			bits.insert(i,'_')

	# Filling up the parity bits
	for i in range(len(bits)):
		if isPowerOf2(i+1):
			bits[i] = compute_parity(bits, i+1)

	return bits

def compute_parity(bits, pos):
	"""Computes the parity bit of a given position"""

	tempbits = copy.deepcopy(bits)

	# Adding a temp bit in the beginning
	# for the hamming code algorithm to work
	tempbits.insert(0, '*')

	# Grouping the bits which are needed to
	# compute parity of position pos
	chunks = list(grouper(tempbits, pos))

	# Chunks to check parity in, skipping others
	parity_chunks = chunks[1::2]

	# Finding the number of ones in the groups
	# To perform XOR over the bits
	ones = 0
	for li in parity_chunks:
		ones+=li.count('1')

	# Parity is one for odd number of ones
	if ones%2 != 0:
		return '1'
	# Parity is zero otherwise
	elif ones%2 == 0:
		return '0'

def grouper(iterable, n, fillvalue=None):
    "Collect data into fixed-length chunks or blocks"
    # grouper('ABCDEFG', 3, 'x') --> ABC DEF Gxx"
    args = [iter(iterable)] * n
    return zip_longest(*args, fillvalue=fillvalue)

def text_to_bits(text, encoding='utf-8', errors='surrogatepass'):
    bits = bin(int.from_bytes(text.encode(encoding, errors), 'big'))[2:]
    return bits.zfill(8 * ((len(bits) + 7) // 8))

def text_from_bits(bits, encoding='utf-8', errors='surrogatepass'):
    n = int(bits, 2)
    return n.to_bytes((n.bit_length() + 7) // 8, 'big').decode(encoding, errors) or '\0'

def isPowerOf2(n):
    m = math.log(n, 2)
    return m == int(m)

if __name__ == "__main__":
	main(sys.argv[1:])


import collections
import csv
import glob
import os.path
import pickle

import cv2
import numpy as np

def prepare_im(path):
    # loads the image from a path and converts it to greyscale
    img = cv2.imread(path)
    grey = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
    grey = cv2.rotate(grey, cv2.ROTATE_90_CLOCKWISE)
    grey = grey.astype(np.float64)
    return grey

def does_have_xandy(pixel_sample):
    has_front = False
    has_side = False

    for angle_name in pixel_sample:
        if "front" in angle_name or "back" in angle_name:
            has_front = True
        elif "left" in angle_name or "right" in angle_name:
            has_side = True

    return has_front and has_side

def get_brightest_front_side(brights):
    front, f_val = None, 0
    side, s_val = None, 0
    for angle_name in brights:
        if "front" in angle_name or "back" in angle_name:
            if brights[angle_name] > f_val:
                f_val = brights[angle_name]
                front = angle_name
        elif "left" in angle_name or "right" in angle_name:
            if brights[angle_name] > s_val:
                s_val = brights[angle_name]
                side = angle_name
    return [front, side]

if __name__ == "__main__":
    # so long as the brightness is this many units greater than the background, we detected a light
    threshold = 50

    pixels_samples = collections.defaultdict(dict)
    pixels_samples_brightness = collections.defaultdict(dict)

    sides = ["front1", "left1", "back1", "right1"]
    for side in sides:
        # load the background image
        background = prepare_im(f"./data/{side}/background.jpg")

        # for each of the images we have in data
        for img_path in glob.glob(f"./data/{side}/*.jpg"):
            light_im = prepare_im(img_path)

            # remove the background so that we are looking at the change in lighting conditions
            diff = np.clip(light_im - background, 0, None)

            # eroding to remove reflections
            kernel = np.ones((6, 6), np.uint8)
            diff = cv2.erode(diff, kernel, cv2.BORDER_REFLECT)

            #  threshold out any change below our defined threshold
            threshed = np.zeros_like(diff, dtype=np.uint8)
            threshed[diff > threshold] = diff[diff > threshold]

            # get the location of the maximum brightness in the image
            (minVal, maxVal, minLoc, maxLoc) = cv2.minMaxLoc(threshed)

            # if we detected a change then
            if maxVal > 0:
                filename = os.path.basename((img_path))
                pixel_id = int(filename.replace(".jpg", ""))

                pixels_samples[pixel_id][side]= [maxLoc[0], maxLoc[1]]
                pixels_samples_brightness[pixel_id][side]= maxVal

            # show the images and the detections
            image = light_im.copy().astype(np.uint8)
            image = np.dstack([image, image, image])
            cv2.circle(image, maxLoc, 10, (255, 0, 0), 2)
            cv2.imshow("im", image)
            cv2.imshow("diff", diff.astype(np.uint8))
            cv2.waitKey(1)

    # for each angle we imaged from, normalize the range of the locations
    for side in sides:
        locations = []
        for pixel_id in pixels_samples:
            xy_loc = pixels_samples[pixel_id].get(side, None)
            if xy_loc:
                locations.append(xy_loc)

        min_pos = np.min(locations, axis=0)
        max_pos = np.max(locations, axis=0)

        for pixel_id in pixels_samples:
            xy_loc = pixels_samples[pixel_id].get(side, None)
            if xy_loc:
                # normalize the coordinate
                xy_loc = (xy_loc - min_pos) / (max_pos - min_pos)

                # flip the coordinates if we are in the back or the right hand side
                if "back" in side or "right" in side:
                    xy_loc[0] = 1 - xy_loc[0]

                #flip vertically because the top of an image is 0, and we want the bottom of the tree to be zero
                xy_loc[1] = 1 - xy_loc[1]

                # rescale the coordinate to the dimensions of the tree
                xy_loc[0] *= 100
                xy_loc[1] *= 300

                pixels_samples[pixel_id][side] = xy_loc

    pixel_coords = []
    for pixel_id in pixels_samples:
        if does_have_xandy(pixels_samples[pixel_id]):
            best_samples = get_brightest_front_side(pixels_samples_brightness[pixel_id])

            # get all X samples
            X_samples = [pixels_samples[pixel_id][s][0] for s in best_samples if "front" in s or "back" in s]
            x = np.array(X_samples).mean()

            Y_samples = [pixels_samples[pixel_id][s][0] for s in best_samples if "left" in s or "right" in s]
            y = np.array(Y_samples).mean()

            Z_samples = [pixels_samples[pixel_id][s][1] for s in [best_samples[0],]]
            z = np.array(Z_samples).mean()

            pixel_coords.append((pixel_id, x, y, z))

    with open("./data/coords.csv", "w", newline="\n") as f:
        writer = csv.writer(f)
        for row in pixel_coords:
            writer.writerow(row)


"""Main training file for citation graph datasets."""

import argparse
import numpy as np
import time
from model_archs.gat_utils import build_model
from model_archs.gat_utils import collect_attention_weights
from model_archs.gat_utils import compute_sparsity_stats
from model_archs.gat_utils import compare_tree_sparsification
from model_archs.gat_utils import compute_att_on_same_class
from analysis.attention_vis import plot_attentions
from analysis.attention_vis import plot_att_trees
from analysis.attention_analysis import plot_attention_entropy
from analysis.attention_analysis import plot_unlabeled_samples
from analysis.attention_analysis import analyze_noise_detection
from analysis.attention_analysis import compute_att_on_shortest_paths
from data_reader.citation_graph_dgl import load_data
from sklearn.metrics import f1_score
from sklearn import metrics
import copy
import torch

def accuracy(logits, labels):
  """Compute an evaluation score based on logits and labels.

  Args:
      logits (tensor): Output logits.
      labels (tensor): Labels of all nodes.

  Returns:
      score: evaluation score.
  """
  _, indices = torch.max(logits, dim=1)
  correct = torch.sum(indices == labels)
  return correct.item() * 1.0 / len(labels)

def compute_f1_score(logits, labels, average_mode='micro'):
  """Compute an evaluation score based on logits and labels.

  Args:
      logits (tensor): Output logits.
      labels (tensor): Labels of all nodes.
      average_mode (str, optional): Description

  Returns:
      score: evaluation score.
  """
  _, indices = torch.max(logits, dim=1)

  score = f1_score(
      labels.data.cpu().numpy(), indices.cpu().numpy(), average=average_mode)
  return score

def compute_f1_score_binary(logits, labels):
  """Compute an evaluation score based on logits and labels.

  Args:
      logits (tensor): Output logits.
      labels (tensor): Labels of all nodes.

  Returns:
      score: evaluation score.
  """
  probs = torch.softmax(logits, dim=1)[:, 1]
  probs[probs >= 0.5] = 1
  probs[probs < 0.5] = 0

  score = f1_score(
      labels.data.cpu().numpy(), probs.cpu().numpy(), average=None)[1]
  return score

def evaluate(model, features, labels, mask, args):
  """Evaluate a trained model.

  Args:
      model (model): graph model.
      features (tensor): input features of all nodes.
      labels (tensor): labels of all nodes.
      mask (tensor): mask to choose samples
      args (namespace): all hyperparameters.

  Returns:
      score: evaluation score.
      secondary_score: a second score.
  """
  model.eval()
  with torch.no_grad():
    logits = model(features)
    if args.data.startswith('yelp'):
      logits = logits[mask]
      labels = labels[mask]
      average_mode = 'binary'
      probs = torch.softmax(logits, dim=1)[:, 1]
      fpr, tpr, _ = metrics.roc_curve(labels.data.cpu().numpy(),
                                      probs.cpu().numpy())
      auc_scores = metrics.auc(fpr, tpr)
      score = auc_scores
      f1_binary = compute_f1_score_binary(logits, labels)

      secondary_score = f1_binary
      score = auc_scores
    else:
      logits = logits[mask]
      labels = labels[mask]
      average_mode = 'micro'
      secondary_score = compute_f1_score(logits, labels, average_mode)
      score = accuracy(logits, labels)

    return score, secondary_score

def main(args):
  """Training and computation of some stats.

  This function handles the entire training process and analyses the learned
  results.

  Args:
      args (namespace): All hyperparameters

  Returns:
      resutls: Collected results after traning.
  """
  if args.model == 'sparselossgat' and args.attention_weight == 0:
    args.model = 'unsupsparsegat'
  assert args.labeling_rate is None
  assert (args.samples_per_class is None or
          args.samples_per_class > -1) or args.use_provided_data_split
  torch.backends.cudnn.deterministic = True
  torch.backends.cudnn.benchmark = False

  data = load_data(args)
  features = torch.FloatTensor(data.features)
  labels = torch.LongTensor(data.labels)

  if hasattr(torch, 'BoolTensor'):
    train_mask = torch.BoolTensor(data.train_mask)
    val_mask = torch.BoolTensor(data.val_mask)
    test_mask = torch.BoolTensor(data.test_mask)
    trainable_masks = torch.ones(len(train_mask), dtype=torch.bool)
    trainable_masks[val_mask] = False
    trainable_masks[test_mask] = False
  else:
    train_mask = torch.ByteTensor(data.train_mask)
    val_mask = torch.ByteTensor(data.val_mask)
    test_mask = torch.ByteTensor(data.test_mask)

  num_feats = features.shape[1]
  n_classes = data.num_labels
  n_edges = data.graph.number_of_edges()
  print("""----Data statistics------'
      #Edges %d
      #Classes %d
      #Train samples %d
      #Val samples %d
      #Test samples %d""" %
        (n_edges, n_classes, train_mask.int().sum().item(),
         val_mask.int().sum().item(), test_mask.int().sum().item()))

  if args.gpu < 0:
    cuda = False
  else:
    cuda = True
    torch.cuda.set_device(args.gpu)
    features = features.cuda()
    labels = labels.cuda()
    train_mask = train_mask.cuda()
    val_mask = val_mask.cuda()
    test_mask = test_mask.cuda()
    trainable_masks = trainable_masks.cuda()
    if 'w' in data.g.edata:
      data.g.edata['w'] = data.g.edata['w'].cuda()

  g = data.g
  if cuda:
    g = g.to(args.gpu)
  n_edges = g.number_of_edges()
  # create model
  heads = ([args.num_heads] * args.num_layers) + [args.num_out_heads]
  model = build_model(g, args, num_feats, n_classes, heads)
  print(model)

  if cuda:
    model.cuda()

  loss_fcn = torch.nn.CrossEntropyLoss()
  # use optimizers
  optimizer = torch.optim.Adam(
      model.parameters(), lr=args.lr, weight_decay=args.weight_decay)

  # initialize graph
  dur = []
  best_valid_acc = -1
  best_valid_f1 = -1
  best_epoch = -1
  for epoch in range(args.epochs):
    model.train()
    if epoch >= 3:
      t0 = time.time()
    # forward
    logits = model(features)
    loss = loss_fcn(logits[train_mask], labels[train_mask])

    optimizer.zero_grad()
    loss.backward()

    optimizer.step()

    if epoch >= 3:
      dur.append(time.time() - t0)

    train_acc = accuracy(logits[train_mask], labels[train_mask])
    train_f1 = compute_f1_score(logits[train_mask], labels[train_mask])

    val_acc, val_f1 = evaluate(model, features, labels, val_mask, args)
    attention_list = collect_attention_weights(
        model,
        mean=True,
        keep_list=True,
    )
    compute_sparsity_stats(attention_list)

    if val_acc >= best_valid_acc:
      best_valid_acc = val_acc
      best_valid_f1 = val_f1
      best_test_acc, best_test_f1 = evaluate(model, features, labels, test_mask,
                                             args)
      best_epoch = epoch
      best_model = copy.deepcopy(model.state_dict())
    if epoch - best_epoch >= args.patience:
      break

    print('Epoch {:05d} | Time(s) {:.4f} | Loss {:.4f} | TrainAcc {:.4f} |'
          ' ValAcc {:.4f} | Testacc {:.4f} |'
          ' TrainF1 {:.4f} | ValF1 {:.4f} |'
          ' TestF1 {:.4f}| ETputs(KTEPS) {:.2f}'.format(
              epoch, np.mean(dur), loss.item(), train_acc, val_acc,
              best_test_acc, train_f1, val_f1, best_test_f1,
              n_edges / np.mean(dur) / 1000))

  print()
  print('Test Accuracy {:.4f} Test F1 (pos-class) {:.4f}'.format(
      best_test_acc, best_test_f1))
  more_info = (best_epoch, best_test_acc, best_test_f1, best_valid_acc,
               best_valid_f1)
  model.load_state_dict(best_model)
  best_test_acc, best_test_f1 = evaluate(model, features, labels, test_mask,
                                         args)
  attention_weights = collect_attention_weights(model, mean=True)
  more_info = [best_epoch, best_test_acc, best_valid_acc, None]

  attention_list = collect_attention_weights(
      model,
      mean=True,
      keep_list=True,
  )
  if args.compute_tree_sparsification:
    saving, average_saving, most_sparse_node = compare_tree_sparsification(
        g, attention_list)
    print('Saving %f Best Node: %s sparse_rate: %f' % (
        average_saving,
        most_sparse_node,
        saving[most_sparse_node],
    ))
    plot_att_trees(
        g,
        attention_list,
        labels,
        model_name=args.model,
        data=args.data,
        samples_per_class=args.samples_per_class,
        best_epoch=best_epoch,
        nodes=[most_sparse_node],
    )
  if args.plot_tree:
    attention_list = attention_list[:3]
    compute_sparsity_stats(attention_list)
    plot_att_trees(
        g,
        attention_list,
        labels,
        model_name=args.model,
        data=args.data,
        samples_per_class=args.samples_per_class,
        best_epoch=best_epoch,
    )
  if args.noise_injected_ratio > 0:
    noisy_edges_ids = data.noisy_edges_ids
    n_nodes = len(g.nodes())
    att_on_noisy_edge = analyze_noise_detection(attention_weights,
                                                noisy_edges_ids, n_nodes)
    more_info[-1] = att_on_noisy_edge
  if args.noise_robustness:
    att_on_same_class = compute_att_on_same_class(g, model, features, labels)
    more_info[-1] = att_on_same_class

  if args.vis_att:
    plot_attentions(g, model, labels, args)

  if args.att_on_shortest_paths:
    att_mean = compute_att_on_shortest_paths(g, attention_weights, labels)
    more_info[-1] = att_mean
  if args.att_analysis:
    attention_heads = collect_attention_weights(model)
    plot_attention_entropy(
        graph=g,
        attention_heads=attention_heads,
        dataset=args.data,
        samples_per_class=args.samples_per_class,
        model_name=args.model,
        labels=labels,
    )
    plot_unlabeled_samples(
        graph=g,
        attention_weights=attention_heads[:, -1],
        train_mask=train_mask,
        dataset=args.data,
        samples_per_class=args.samples_per_class,
        model_name=args.model,
        labels=labels,
    )

  assert len(more_info) == 4, 'Too many elements in the list %d' \
                              % len(more_info)

  return best_test_acc, more_info

def get_args():
  """Provides a parser with standard arguments.

  Returns:
      parser: parser with standard arguments
  """
  parser = argparse.ArgumentParser(description='GAT')
  parser.add_argument(
      '--gpu', type=int, default=1, help='which GPU to use. Set -1 to use CPU.')
  parser.add_argument(
      '--epochs', type=int, default=400, help='number of training epochs')
  parser.add_argument(
      '--num-heads',
      type=int,
      default=8,
      help='number of hidden attention heads')
  parser.add_argument(
      '--num-out-heads',
      type=int,
      default=1,
      help='number of output attention heads')
  parser.add_argument(
      '--num-layers', type=int, default=2, help='number of hidden layers')
  parser.add_argument(
      '--num-hidden', type=int, default=8, help='number of hidden units')
  parser.add_argument(
      '--residual',
      action='store_true',
      default=False,
      help='use residual connection')
  parser.add_argument(
      '--in-drop', type=float, default=.6, help='input feature dropout')
  parser.add_argument(
      '--attn-drop', type=float, default=.6, help='attention dropout')
  parser.add_argument('--lr', type=float, default=0.005, help='learning rate')
  parser.add_argument(
      '--weight-decay', type=float, default=5e-4, help='weight decay')
  parser.add_argument(
      '--alpha',
      type=float,
      default=0.2,
      help='the negative slop of leaky relu')
  parser.add_argument(
      '--batch-size',
      type=int,
      default=2,
      help='batch size used for training, validation and test')
  parser.add_argument(
      '--patience', type=int, default=20, help='used for early stop')
  parser.add_argument(
      '--dropout', type=float, default=0, help='Dropout for non-gat models ')
  parser.add_argument(
      '--graphsage_aggregator_type',
      type=str,
      default='gcn',
      help='Aggregator type: mean/gcn/pool/lstm')

  parser.add_argument(
      '--data', type=str, default=10, help='cora | citeseer | pubmed')

  parser.add_argument('--model', type=str, help='which model to be used')
  parser.add_argument(
      '--labeling_rate',
      type=float,
      default=None,
      help='how much labels should be used?')
  parser.add_argument(
      '--samples_per_class',
      type=float,
      default=-1,
      help='Alternative: how mnay samples per classes should be used. -1 means use labeling rate instead'
  )
  parser.add_argument('--attention_weight', type=float, default=1)
  parser.add_argument('--lambda_sparsemax', type=float, default=0)
  parser.add_argument(
      '--alpha_sparsemax',
      type=float,
      default=0,
      help='1 = only sparse_max, 0 = only softmax')

  parser.add_argument('--layer_aggregation', type=bool)
  parser.add_argument('--self_loop', type=bool)

  parser.add_argument('--correct_data_split', type=bool, default=True)
  parser.add_argument('--label_prop_steps', type=float, default=None)
  parser.add_argument('--pseudo_label_mode', type=str, default='final_only')
  parser.add_argument('--use_adj_matrix', type=bool, default=None)
  parser.add_argument(
      '--vis_att',
      type=bool,
      default=None,
      help='visualize attention after training ')
  parser.add_argument(
      '--n_nodes_to_plot', type=int, default=20, help='How many nodes to plot.')
  parser.add_argument(
      '--noise_robustness',
      type=bool,
      default=None,
      help='Performs noise robustness experiments or not')
  parser.add_argument(
      '--noise_injected_ratio',
      type=float,
      default=0,
      help='Percentual noise to be added.')
  parser.add_argument(
      '--noise_type', type=str, default='random', help='random | heterophily')
  parser.add_argument(
      '--att_on_shortest_paths',
      type=bool,
      default=False,
      help='Compute the att-mean on shortest path')
  parser.add_argument(
      '--plot_tree',
      type=bool,
      default=False,
      help='Plot explanation trees for some nodes')
  parser.add_argument(
      '--compute_tree_sparsification',
      type=bool,
      default=False,
      help='How much of the interpretability tree is saved')
  parser.add_argument(
      '--att_analysis',
      type=bool,
      default=False,
      help='Plot explanation trees for some nodes')

  return parser

if __name__ == '__main__':
  parser = get_args()
  args = parser.parse_args()

  config = args.__dict__
  config['data'] = 'cora'
  config['gpu'] = -1
  config['attn_drop'] = 0
  config['in_drop'] = 0
  config['SLGAT_apply_on_up_to_layers'] = 3
  config['test_level'] = 'ALL'
  config['patience'] = 100
  config['num_epochs'] = 1000
  config['SLGAT_loss'] = 'L2'
  config['SLGAT_apply_on_all_samples'] = True
  config['SLGAT_use_same_projection'] = False
  config['SLGAT_projection_detach'] = False
  config['model'] = 'sparselossgat'
  config['labeling_rate'] = -2

  print(args)
  main(args)


'''
Helper script to check the status of the redshift cluster.

'''

import boto3
import json
import configparser
from botocore.exceptions import ClientError

config = configparser.ConfigParser()
config.read('dwh.cfg')

'''
    Builds a connection to the AWS services.

'''
redshift = boto3.client('redshift',
            aws_access_key_id=config.get('AWS','KEY'),
            aws_secret_access_key=config.get('AWS','SECRET'),
            region_name=config.get('AWS','AWS_REGION')
            )

try:
    cluster_status = redshift.describe_clusters(
    ClusterIdentifier=config.get('REDSHIFT_CLUSTER', 'CLUSTER_IDENTIFIER')
        )
except Exception as e:
    print('could not get cluster status', e)

redshift_cluster = cluster_status['Clusters'][0]
print(redshift_cluster)

print("Redshift endpoint :: ", redshift_cluster['Endpoint']['Address'])
print("IAM role ARN :: ", redshift_cluster['IamRoles'][0]['IamRoleArn'])


import argschema
import pandas as pd
import numpy as np
from functools import partial
from PIL import Image
from pathlib import Path
from ophys_etl.schemas.fields import H5InputFile
from ophys_etl.utils.array_utils import normalize_array
from ophys_etl.utils.video_utils import downsample_h5_video, encode_video

import matplotlib as mpl
mpl.use('Agg')
from matplotlib import pyplot as plt  # noqa: E402

class RegistrationQCException(Exception):
    pass

class RegistrationQCInputSchema(argschema.ArgSchema):
    movie_frame_rate_hz = argschema.fields.Float(
        required=True,
        description="frame rate of movie, usually 31Hz or 11Hz")
    preview_frame_bin_seconds = argschema.fields.Float(
        required=False,
        default=2.0,
        description=("before creating the webm, the movies will be "
                     "aveaged into bins of this many seconds."))
    preview_playback_factor = argschema.fields.Float(
        required=False,
        default=10.0,
        description=("the preview movie will playback at this factor "
                     "times real-time."))
    uncorrected_path = H5InputFile(
        required=True,
        description=("path to uncorrected original movie."))
    motion_corrected_output = H5InputFile(
        required=True,
        description=("path to motion corrected movie."))
    motion_diagnostics_output = argschema.fields.InputFile(
        required=True,
        description=("Saved path for *.csv file containing motion "
                     "correction offset data")
    )
    max_projection_output = argschema.fields.InputFile(
        required=True,
        description=("Saved path for *.png of the max projection of the "
                     "motion corrected video."))
    avg_projection_output = argschema.fields.InputFile(
        required=True,
        description=("Saved path for *.png of the avg projection of the "
                     "motion corrected video."))
    registration_summary_output = argschema.fields.OutputFile(
        required=True,
        description=("Desired path for *.png summary plot."))
    motion_correction_preview_output = argschema.fields.OutputFile(
        required=True,
        description="Desired path for *.webm motion preview")
    movie_lower_quantile = argschema.fields.Float(
        required=False,
        default=0.1,
        description=("lower quantile threshold for avg projection "
                     "histogram adjustment of movie"))
    movie_upper_quantile = argschema.fields.Float(
        required=False,
        default=0.999,
        description=("upper quantile threshold for avg projection "
                     "histogram adjustment of movie"))

def make_png(max_proj_path: Path, avg_proj_path: Path,
             summary_df: pd.DataFrame, dst_path: Path):
    """
    """
    xo = np.abs(summary_df['x']).max()
    yo = np.abs(summary_df['y']).max()

    fig = plt.figure(figsize=(16, 10))
    gs = fig.add_gridspec(4, 4)
    mx_ax = fig.add_subplot(gs[0:2, 0:2])
    av_ax = fig.add_subplot(gs[0:2, 2:4])
    xyax = fig.add_subplot(gs[2, :])
    corrax = fig.add_subplot(gs[3, :])

    for ax, im_path in zip([mx_ax, av_ax], [max_proj_path, avg_proj_path]):
        with Image.open(im_path) as im:
            ax.imshow(im, cmap='gray')
            sz = im.size
        ax.axvline(xo, color='r', linestyle='--')
        ax.axvline(sz[0] - xo, color='r', linestyle='--')
        ax.axhline(yo, color='g', linestyle='--')
        ax.axhline(sz[1] - yo, color='g', linestyle='--')
        ax.set_title(f"{im_path.parent}\n{im_path.name}", fontsize=8)

    xyax.plot(summary_df['x'], linewidth=0.5, color='r', label='xoff')
    xyax.axhline(xo, color='r', linestyle='--')
    xyax.axhline(-xo, color='r', linestyle='--')
    xyax.plot(summary_df['y'], color='g', linewidth=0.5, alpha=0.5,
              label='yoff')
    xyax.axhline(yo, color='g', linestyle='--')
    xyax.axhline(-yo, color='g', linestyle='--')
    xyax.legend(loc=0)
    xyax.set_ylabel("correction offset [pixels]")

    corrax.plot(summary_df['correlation'], color='k', linewidth=0.5,
                label='corrXY')
    corrax.set_xlabel("frame index")
    corrax.set_ylabel("correlation peak value")
    corrax.legend(loc=0)
    fig.tight_layout()
    fig.savefig(dst_path)

    return dst_path

def downsample_normalize(movie_path: Path, frame_rate: float,
                         bin_size: float, lower_quantile: float,
                         upper_quantile: float) -> np.ndarray:
    """reads in a movie (nframes x nrows x ncols), downsamples,
    creates an average projection, and normalizes according to
    quantiles in that projection.

    Parameters
    ----------
    movie_path: Path
        path to an h5 file, containing an (nframes x nrows x ncol) dataset
        named 'data'
    frame_rate: float
        frame rate of the movie specified by 'movie_path'
    bin_size: float
        desired duration in seconds of a downsampled bin, i.e. the reciprocal
        of the desired downsampled frame rate.
    lower_quantile: float
        arg supplied to `np.quantile()` to determine lower cutoff value from
        avg projection for normalization.
    upper_quantile: float
        arg supplied to `np.quantile()` to determine upper cutoff value from
        avg projection for normalization.

    Returns
    -------
    ds: np.ndarray
        a downsampled and normalized array

    Notes
    -----
    This strategy was satisfactory in the labeling app for maintaining
    consistent visibility.

    """
    ds = downsample_h5_video(
            movie_path,
            input_fps=frame_rate,
            output_fps=1.0 / bin_size)
    avg_projection = ds.mean(axis=0)
    lower_cutoff, upper_cutoff = np.quantile(
                avg_projection.flatten(), (lower_quantile, upper_quantile))
    ds = normalize_array(ds,
                         lower_cutoff=lower_cutoff,
                         upper_cutoff=upper_cutoff)
    return ds

class RegistrationQC(argschema.ArgSchemaParser):
    default_schema = RegistrationQCInputSchema

    def run(self):
        self.logger.name = type(self).__name__
        self.logger.setLevel(self.args['log_level'])

        # create and write the summary png
        motion_offset_df = pd.read_csv(self.args['motion_diagnostics_output'])
        png_out_path = make_png(Path(self.args['max_projection_output']),
                                Path(self.args['avg_projection_output']),
                                motion_offset_df,
                                Path(self.args['registration_summary_output']))
        self.logger.info(f"wrote {png_out_path}")

        # downsample and normalize the input movies
        ds_partial = partial(downsample_normalize,
                             frame_rate=self.args['movie_frame_rate_hz'],
                             bin_size=self.args['preview_frame_bin_seconds'],
                             lower_quantile=self.args['movie_lower_quantile'],
                             upper_quantile=self.args['movie_upper_quantile'])
        processed_vids = [ds_partial(i)
                          for i in [
                              Path(self.args['uncorrected_path']),
                              Path(self.args['motion_corrected_output'])]]
        self.logger.info("finished downsampling motion corrected "
                         "and non-motion corrected movies")

        # tile into 1 movie, raw on left, motion corrected on right
        tiled_vids = np.block(processed_vids)

        # make into a viewable artifact
        playback_fps = (self.args['preview_playback_factor'] /
                        self.args['preview_frame_bin_seconds'])
        encode_video(tiled_vids,
                     self.args['motion_correction_preview_output'],
                     playback_fps)
        self.logger.info("wrote "
                         f"{self.args['motion_correction_preview_output']}")

        return

if __name__ == "__main__":
    rqc = RegistrationQC()
    rqc.run()


from enum import Enum,auto
import sys
from items import change,shuffle
from module import standard_numer0n as nm
class DefenceItem(Enum):
    CHANGE = "c"
    SHUFFLE = "sh"
    NONE = "n"

    def use_defence(self,info,turn,digit=3):
        fin_flag = False
        if self == DefenceItem.CHANGE:
            info[not turn]["ans"] = change.use(info[not turn]["ans"],digit)
        elif self == DefenceItem.SHUFFLE:
            info[not turn]["ans"] = shuffle.use(info[not turn]["ans"],digit)
        elif self == DefenceItem.NONE:
            pass
        else:
            print("コマンドが見つかりません")
        return (info,turn,fin_flag)

    @classmethod
    def return_item(cls,item_command):
        try:
            item_object = cls(item_command)
            return (True,item_object)
        except:
            return (False,None)

    @classmethod
    def print_list(cls):
        print(list(cls))


from _warnings import warn

from comet_ml import API
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
import pandas as pd
sns.set(style="darkgrid")

api = API()
reported_metrics = ["Reward Mean", "Episodic Success Rate", "Episode Length Mean "]

def build_plot_dict(orig_env_name, raw_tuples, final_data, log_ts):
    for i in range(len(raw_tuples)):

        env_name = orig_env_name + ' - ' + reported_metrics[i]
        temp_data = np.array([list(x) for x in raw_tuples[i]]).transpose()

        # Preprocess for equal log intervals
        data = np.zeros((temp_data.shape[0], len(log_ts)))
        data[0, :] = log_ts
        for j in range(len(log_ts)):
            index = np.where(temp_data[0] == log_ts[j])[0]
            if index.size == 0:
                continue
            data[:, j] = temp_data[:, index[0]]

        # If same experiment with different seed
        if env_name in final_data:
            final_data[env_name]['n'] += 1
            data_concat = np.vstack((final_data[env_name]['data0'], data[1, :]))
            final_data[env_name]['data0'] = data_concat

        # If first experiment with these hyperparams.
        else:
            final_data[env_name] = {'metric': reported_metrics[i], 'data0': data, 'n': 1}

    return final_data

def running_mean(x, N):
    cumsum = np.cumsum(np.insert(x, 0, 0))
    return (cumsum[N:] - cumsum[:-N]) / float(N)

gps_exp_ids = {
    "SEVN-Test-NoisyGPS-1-v1-s0-10p": "mweiss17/navi-corl-2019/2d3670a8af1e4c9d83f06a889e02862e",
    "SEVN-Test-NoisyGPS-1-v1-s1-10p": "mweiss17/navi-corl-2019/604adedd9fdd4f0bb18fff5d76a4400f",
    "SEVN-Test-NoisyGPS-1-v1-s2-10p": "mweiss17/navi-corl-2019/76078c37a6934a24bb644bda65fcff0e",
    "SEVN-Test-NoisyGPS-5-v1-s0-10p": "mweiss17/navi-corl-2019/60fb0b41644c4ba89949eb03ada53948",
    "SEVN-Test-NoisyGPS-5-v1-s1-10p": "mweiss17/navi-corl-2019/a887d1de91c0463e94985441bbf8b126",
    "SEVN-Test-NoisyGPS-5-v1-s2-10p": "mweiss17/navi-corl-2019/76078c37a6934a24bb644bda65fcff0e",
    "SEVN-Test-NoisyGPS-25-v1-s0-10p": "mweiss17/navi-corl-2019/281db6264d9d46fb8b0ada97ecd19903",
    "SEVN-Test-NoisyGPS-25-v1-s1-10p": "mweiss17/navi-corl-2019/f4f8901bd4944340bd8beb2c203ccca9",
    "SEVN-Test-NoisyGPS-25-v1-s2-10p": "mweiss17/navi-corl-2019/69a88388d483456c9174453956414f97",
    "SEVN-Test-NoisyGPS-100-v1-s0-10p": "mweiss17/navi-corl-2019/5824c0711976465cb1c2363434246961",
    "SEVN-Test-NoisyGPS-100-v1-s1-10p": "mweiss17/navi-corl-2019/79e37ee59d0e41b094eafb5a70f8df23",
    "SEVN-Test-NoisyGPS-100-v1-s2-10p": "mweiss17/navi-corl-2019/cb839ab2d3444f3aaadfb8497c7604ad",
}

plot_info = {
    "SEVN-Test-NoisyGPS-1": {'color': '#404387', 'plot_name': 'NoisyGPS-1'},
    "SEVN-Test-NoisyGPS-5": {'color': '#22a784', 'plot_name': 'NoisyGPS-5'},
    "SEVN-Test-NoisyGPS-25": {'color': '#fde724', 'plot_name': 'NoisyGPS-25'},
    "SEVN-Test-NoisyGPS-100": {'color': '#29788e', 'plot_name': 'NoisyGPS-100'},
}

gps_exp_data = {}
min_numframes = 0000000
max_numframes = 100000000
for name, exp_id in gps_exp_ids.items():
    gps_exp_data[name] = {}
    experiment = api.get(exp_id)

    reward_mean = experiment.metrics_raw[reported_metrics[0]]
    reward_arr = np.array(reward_mean).transpose()
    reward_arr = reward_arr[:, np.where(np.logical_and(reward_arr[0] >= min_numframes,
                                                       reward_arr[0] < max_numframes))[0]]
    gps_exp_data[name][reported_metrics[0]] = {
        'mean': np.mean(reward_arr[1]),
        'std': np.std(reward_arr[1]),
        'raw_data': reward_arr
    }

    episodic_success_rate = experiment.metrics_raw[reported_metrics[1]]
    ep_succes_rate_arr = np.array(episodic_success_rate).transpose()
    ep_succes_rate_arr = ep_succes_rate_arr[:, np.where(np.logical_and(ep_succes_rate_arr[0] >= min_numframes,
                                                                       ep_succes_rate_arr[0] < max_numframes))[0]]
    gps_exp_data[name][reported_metrics[1]] = {
        'mean': np.mean(ep_succes_rate_arr[1]),
        'std': np.std(ep_succes_rate_arr[1]),
        'raw_data': ep_succes_rate_arr
    }

    episode_length_mean = experiment.metrics_raw[reported_metrics[2]]
    ep_length_mean_arr = np.array(episode_length_mean).transpose()
    ep_length_mean_arr = ep_length_mean_arr[:, np.where(np.logical_and(ep_length_mean_arr[0] >= min_numframes,
                                                                       ep_length_mean_arr[0] < max_numframes))[0]]
    gps_exp_data[name][reported_metrics[2]] = {
        'mean': np.mean(ep_length_mean_arr[1]),
        'std': np.std(ep_length_mean_arr[1]),
        'raw_data': ep_length_mean_arr
    }

from collections import defaultdict
exps = defaultdict(list)
for name, exp_id in gps_exp_ids.items():
    exp = "-".join(name.split("-")[:4])
    if exp in name:
        exps[exp].append(gps_exp_data[name])

compiled_data = []
for name, data in exps.items():
    c_data = {'name': name}
    for metric in reported_metrics:
        try:
            trials = np.concatenate([np.expand_dims(data[0][metric]['raw_data'][1], axis=0), np.expand_dims(data[1][metric]['raw_data'][1], axis=0), np.expand_dims(data[2][metric]['raw_data'][1], axis=0)], axis=0)
        except Exception as e:
            continue
        mean = trials.mean(axis=0)
        std = trials.std(axis=0)
        c_data[metric] = np.concatenate([np.expand_dims(data[0][metric]['raw_data'][0], axis=0), np.expand_dims(mean, axis=0), np.expand_dims(std, axis=0)], axis=0)
    compiled_data.append(c_data)

for metric in reported_metrics:
    # fig = plt.figure()
    # plt.title(metric, fontsize=18)
    # plt.xlabel('Timesteps', fontsize=14)
    # plt.ylabel(metric, fontsize=14)
    for data in compiled_data:
        color = plot_info[data['name']]['color']
        label = plot_info[data['name']]['plot_name']
        try:
            import pdb; pdb.set_trace()
            to_plot = pd.DataFrame(data[metric].T, columns=['timestep', 'mean', 'std'])
            sns.lineplot(x="Timesteps", y=metric, data=to_plot)
            # plt.errorbar(data0[metric][0], data0[metric][1], yerr=data0[metric][2], color=color, label=label, alpha=0.5)
        except Exception:
            pass
        # plt.plot(running_mean(data0[0], 100), running_mean(data0[1], 100), color, label=label)

    # plt.ticklabel_format(style='sci', axis='x', scilimits=(0, 0))
    # plt.legend(fontsize=14)
    # plt.savefig('plots/gps_exp'+metric + ".png")
    #


import serial
import logging
import logging.handlers
import sys
import os
import traceback

# To enable the serial port on the GPIO connector, use raspi-config or:
# sudo nano /boot/config.txt
# enable_uart=1
# reboot

DEBUG = False

port = "/dev/ttyAMA0"
serialPort = serial.Serial(port, baudrate=9600, timeout=10.0)

# data path is on a USB stick to protect the SD card
data_path = "/mnt/usb/"

# -- Logger definitions
LOG_FILENAME = data_path+"ocxo.log"
LOG_LEVEL = logging.INFO  # Could be e.g. "INFO", "DEBUG", "ERROR" or "WARNING"

class MyLogger(object):
    '''
    Replace stdout and stderr with logging to a file so we can run this script
    even as a daemon and still capture all the stdout and stderr messages in the log.

    '''
    def __init__(self, logger, level):
            """Needs a logger and a logger level."""
            self.logger = logger
            self.level = level

    def write(self, message):
            # Only log if there is a message (not just a new line)
            # typical for serial data with a cr/lf ending
            if message.rstrip() != "":
                self.logger.log(self.level, message.rstrip())

def init():
    global logger, handler

    if DEBUG:print ("Setting up the logger functionality")
    logger = logging.getLogger(__name__)
    logger.setLevel(LOG_LEVEL)
    handler = logging.handlers.TimedRotatingFileHandler(LOG_FILENAME, when="midnight", backupCount=31)
    formatter = logging.Formatter('%(asctime)s %(levelname)-8s %(message)s')
    handler.setFormatter(formatter)
    logger.addHandler(handler)

    # pipe the stdout and stderr messages to the logger
    sys.stdout = MyLogger(logger, logging.INFO)
    sys.stderr = MyLogger(logger, logging.ERROR)

def main():
    if DEBUG:print("Serial logger")

    init()

    if DEBUG:print("Opened port", port, "for serial tracing")

    try:
        while True:
            while (serialPort.inWaiting()>0):
                try:
                    ser_input = serialPort.readline().decode('utf-8')
                    print(ser_input)
                except (OSError, serial.serialutil.SerialException):
                    pass
                    if DEBUG : print("No data available")
                except UnicodeDecodeError:
                    pass
                    if DEBUG: print("decode error")

    except KeyboardInterrupt: # Ctrl-C
        print("\nCtrl-C - Terminated")
        os._exit(1)

    except Exception as e:
        sys.stderr.write("Got exception: %s" % (e))
        print(traceback.format_exc())
        os._exit(1)

if __name__ == '__main__':
    main()


"""Test the create context helper function."""

from werkzeug.datastructures import Headers

from .create_context import create_context
from tdf3_kas_core.models import Context

def test_create_context():
    """Test the create context header function."""
    # create a werkzeug.datastructures.Header object, as Flask produces
    headers = Headers()
    # Load it up with stuff.  Note the implicit string conversions.
    headers.add("foo", "AAA")
    headers.add("foo", "BBB")  # creates second kv pair with same key
    headers.add("foo", "CCC")  # creates third kv pair with same key
    headers.add("bar", 34)  # implicit string conversion
    headers.add("gee", True)  # implict string conversion
    headers.add("hah", ["one", "two", "three"])  # implict string conversion
    # Create the context from the headers
    ctx = create_context(headers)
    assert isinstance(ctx, Context)
    assert ctx.size == 4
    # Check the data property.  Note aggregation of duplicate header keys
    actual = ctx.data
    assert actual["foo"] == ["AAA", "BBB", "CCC"]  # aggregated keys
    assert actual["bar"] == "34"
    assert actual["gee"] == "True"
    assert actual["hah"] == "['one', 'two', 'three']"