import numpy as np
import time
from joblib import Parallel, delayed
import multiprocessing as ml
import itertools
from collections import Counter
import multiprocessing as ml
from joblib import parallel_backend
import math
import pickle
from functools import partial
from .surrogate import *
from scipy.spatial.distance import cdist

from sklearn.dummy import DummyClassifier
from sklearn.metrics import accuracy_score

from .rule import Rule, compact_premises, get_counterfactual_rules_supert, get_rule_supert

from .explanation import Explanation, MultilabelExplanation
from .neighgen import RandomGenerator, GeneticGenerator, RandomGeneticGenerator, ClosestInstancesGenerator, CFSGenerator, CounterGenerator
from .neighgen import GeneticProbaGenerator, RandomGeneticProbaGenerator
from .rule import get_rule, get_counterfactual_rules
from .util import calculate_feature_values, neuclidean, multilabel2str, multi_dt_predict, record2str
from .discretizer import *
from .encdec import *

def default_kernel(d, kernel_width):
    return np.sqrt(np.exp(-(d ** 2) / kernel_width ** 2))

# LOcal Rule-based Explanation Method
class LOREM(object):

    def __init__(self, K, bb_predict, predict_proba, feature_names, class_name, class_values, numeric_columns, features_map,
                 neigh_type='genetic', K_transformed=None, categorical_use_prob=True, continuous_fun_estimation=False,
                 size=1000, ocr=0.1, multi_label=False, one_vs_rest=False, filter_crules=True, init_ngb_fn=True,
                 kernel_width=None, kernel=None, random_state=None, encdec = None, dataset = None, binary=False, discretize=True, verbose=False,
                 extreme_fidelity = False, constraints = None, **kwargs):

        self.random_state = random_state
        self.bb_predict = bb_predict
        self.bb_predict_proba = predict_proba
        self.class_name = class_name
        self.unadmittible_features = None
        self.feature_names = feature_names
        self.class_values = class_values
        self.numeric_columns = numeric_columns
        self.features_map = features_map
        self.neigh_type = neigh_type
        self.multi_label = multi_label
        self.one_vs_rest = one_vs_rest
        self.filter_crules = self.bb_predict if filter_crules else None
        self.binary = binary
        self.verbose = verbose
        self.discretize = discretize
        self.extreme_fidelity = extreme_fidelity
        self.predict_proba = predict_proba
        if encdec is not None:
            print('che dataset passo qui', dataset)
            self.dataset = dataset
            if encdec == 'target':
                print('preparo targetencoding')
                self.encdec = MyTargetEnc(self.dataset, self.class_name)
                self.encdec.enc_fit_transform()
            elif encdec == 'onehot':
                print('preparo onehotencoding')
                self.encdec = OneHotEnc(self.dataset, self.class_name)
                self.encdec.enc_fit_transform()
            Y = self.bb_predict(K)
            print('la y calcolata ', Y)
            self.K = self.encdec.enc(K, Y)
        else:
            self.encdec = None
            self.K = K
        self.K_original = K_transformed
        self.features_map_inv = None
        if self.features_map:
            self.features_map_inv = dict()
            for idx, idx_dict in self.features_map.items():
                for k, v in idx_dict.items():
                    self.features_map_inv[v] = idx
        self.constraints = constraints

        kernel_width = np.sqrt(len(self.feature_names)) * .75 if kernel_width is None else kernel_width
        self.kernel_width = float(kernel_width)

        kernel = default_kernel if kernel is None else kernel
        self.kernel = partial(kernel, kernel_width=kernel_width)

        np.random.seed(self.random_state)

        if init_ngb_fn:
            self.__init_neighbor_fn(ocr, categorical_use_prob, continuous_fun_estimation, size, kwargs)

    def explain_instance(self, x, samples=1000, use_weights=True, metric=neuclidean):

        if isinstance(samples, int):
            if self.verbose:
                print('generating neighborhood - %s' % self.neigh_type)
            Z = self.neighgen_fn(x, samples)
        else:
            Z = samples
        Yb = self.bb_predict(Z)
        if self.multi_label:
            Z = np.array([z for z, y in zip(Z, Yb) if np.sum(y) > 0])
            Yb = self.bb_predict(Z)

        if self.verbose:
            if not self.multi_label:
                neigh_class, neigh_counts = np.unique(Yb, return_counts=True)
                neigh_class_counts = {self.class_values[k]: v for k, v in zip(neigh_class, neigh_counts)}
            else:
                neigh_counts = np.sum(Yb, axis=0)
                neigh_class_counts = {self.class_values[k]: v for k, v in enumerate(neigh_counts)}

            print('synthetic neighborhood class counts %s' % neigh_class_counts)

        weights = None if not use_weights else self.__calculate_weights__(Z, metric)

        if self.one_vs_rest and self.multi_label:
            exp = self.__explain_tabular_instance_multiple_tree(x, Z, Yb, weights)
        else:  # binary, multiclass, multilabel all together
            exp = self.__explain_tabular_instance_single_tree(x, Z, Yb, weights)

        return exp

    def __calculate_weights__(self, Z, metric):
        if np.max(Z) != 1 and np.min(Z) != 0:
            Zn = (Z - np.min(Z)) / (np.max(Z) - np.min(Z))
            distances = cdist(Zn, Zn[0].reshape(1, -1), metric=metric).ravel()
        else:
            distances = cdist(Z, Z[0].reshape(1, -1), metric=metric).ravel()
        weights = self.kernel(distances)
        return weights

    def multi_neighgen_fn_parallel(self, x, runs, samples, n_jobs = 2 ):
        Z_list = [list() for i in range(runs)]
        if self.neigh_type == 'cfs' or self.neigh_type == 'random':
            Z_list = Parallel(n_jobs=n_jobs, verbose=self.verbose, prefer='threads')(
                delayed(self.neighgen_fn)(x, samples)
                for i in range(runs))
        else:
            with parallel_backend('multiprocessing'):
                Z_list = Parallel(n_jobs=n_jobs, verbose=self.verbose, prefer='threads')(
                    delayed(self.neighgen_fn)(x, samples)
                    for i in range(runs))
        return Z_list

    def multi_neighgen_fn(self, x, runs, samples, kwargs=None):
        Z_list = list()
        for i in range(runs):
            if self.verbose:
                print('generating neighborhood [%s/%s] - %s' % (i, runs, self.neigh_type))
                #print(samples, x)
            Z = self.neighgen_fn(x, samples)
            Z_list.append(Z)
        return Z_list

    def get_feature_importance_supert(self, dt, x, tot_samples):
        dt.set_impurity(dt)
        dt.calculate_features_importance(tot_samples)
        all_features = dt.calculate_all_importances()
        single_features = dt.calculate_fi_path(x)
        return single_features, all_features

    def get_feature_importance_binary(self, dt, x):
        att_list = []
        # dt.apply: Return the index of the leaf that each sample is predicted as.
        leave_id_dt = dt.apply(x.reshape(1, -1))
        node_index_dt = dt.decision_path(x.reshape(1, -1)).indices
        feature_dt = dt.tree_.feature
        for node_id in node_index_dt:
            if leave_id_dt[0] == node_id:
                break
            else:
                att = self.feature_names[feature_dt[node_id]]
                att_list.append(att)
        # Feature importance
        feature_importance_all = dt.feature_importances_
        dict_feature_importance = dict(zip(self.feature_names, feature_importance_all))
        feature_importance_rule = {k: v for k, v in dict_feature_importance.items() if k in att_list}
        return feature_importance_rule, dict_feature_importance

    '''Explain Instance Stable
            x the instance to explain
            samples the number of samples to generate during the neighbourhood generation
            use weights True or False
            metric default is neuclidean, it is the metric employed to measure the distance between records
            runs number of times the neighbourhood generation is done
            exemplar_num number of examplars to retrieve
            kwargs a dictionary in which add the parameters needed for cfs generation'''

    def explain_instance_stable_neigh(self, x, Z_list, use_weights=True, metric=neuclidean, runs=3, exemplar_num=5,
                                n_jobs=-1, prune_tree=False, kwargs=None):

        if self.encdec is not None:
            y = self.bb_predict(x.reshape(1, -1))
            x = self.encdec.enc(x, y)

        Yb_list = list()
        # print('la Z creata ', len(Z_list), Z_list[0])
        if self.encdec is not None:
            for Z in Z_list:
                Z = self.encdec.dec(Z)
                # print('Z decodificata ', Z)
                Z = np.nan_to_num(Z)
                Yb = self.bb_predict(Z)
                print('la yb ', Counter(Yb))
                Yb_list.append(Yb)
        else:
            for Z in Z_list:
                Yb = self.bb_predict(Z)
                Yb_list.append(Yb)

        if self.verbose:
            neigh_class_counts_list = list()
            for Yb in Yb_list:
                neigh_class, neigh_counts = np.unique(Yb, return_counts=True)
                neigh_class_counts = {self.class_values[k]: v for k, v in zip(neigh_class, neigh_counts)}
                neigh_class_counts_list.append(neigh_class_counts)

            for neigh_class_counts in neigh_class_counts_list:
                print('Synthetic neighborhood class counts %s' % neigh_class_counts)

        weights_list = list()
        for Z in Z_list:
            weights = None if not use_weights else self.__calculate_weights__(Z, metric)
            weights_list.append(weights)

        if self.verbose:
            print('Learning local decision trees')

        # discretize the data employed for learning decision tree
        if self.discretize:
            Z = np.concatenate(Z_list)
            Yb = np.concatenate(Yb_list)

            discr = RMEPDiscretizer()
            discr.fit(Z, Yb)
            temp = list()
            for Zl in Z_list:
                temp.append(discr.transform(Zl))
            Z_list = temp

            # caso binario da Z e Y da bb
        if self.binary == 'binary_from_bb':
            surr = DecTree()
            weights = None if not use_weights else self.__calculate_weights__(Z, metric)
            superT = surr.learn_local_decision_tree(Z, Yb, weights, self.class_values)
            fidelity = superT.score(Z, Yb, sample_weight=weights)

            # caso binario da Z e Yb
            # caso n ario
            # caso binario da albero n ario
        else:
                # qui prima creo tutti i dt, che servono sia per unirli con metodo classico o altri
            dt_list = [DecTree() for i in range(runs)]
            dt_list = Parallel(n_jobs=n_jobs, verbose=self.verbose, prefer='threads')(
                delayed(t.learn_local_decision_tree)(Zl, Yb, weights, self.class_values, prune_tree=prune_tree)
                for Zl, Yb, weights, t in zip(Z_list, Yb_list, weights_list, dt_list))

            Z = np.concatenate(Z_list)
            Z = np.nan_to_num(Z)
            Yb = np.concatenate(Yb_list)

                # caso binario da Z e Yb dei vari dt
            if self.binary == 'binary_from_dts':
                weights = None if not use_weights else self.__calculate_weights__(Z, metric)
                surr = DecTree()
                superT = surr.learn_local_decision_tree(Z, Yb, weights, self.class_values)
                fidelity = superT.score(Z, Yb, sample_weight=weights)

                # caso n ario
                # caso binario da albero n ario
            else:
                if self.verbose:
                    print('Pruning decision trees')
                surr = SuperTree()
                for t in dt_list:
                    surr.prune_duplicate_leaves(t)
                if self.verbose:
                    print('Merging decision trees')

                weights_list = list()
                for Zl in Z_list:
                    weights = None if not use_weights else self.__calculate_weights__(Zl, metric)
                    weights_list.append(weights)
                weights = np.concatenate(weights_list)
                n_features = list()
                for d in dt_list:
                    n_features.append(list(range(0, len(self.feature_names))))
                roots = np.array([surr.rec_buildTree(t, FI_used) for t, FI_used in zip(dt_list, n_features)])

                superT = surr.mergeDecisionTrees(roots, num_classes=np.unique(Yb).shape[0], verbose=False)

                if self.binary == 'binary_from_nari':
                    superT = surr.supert2b(superT, Z)
                    Yb = superT.predict(Z)
                    fidelity = superT.score(Z, Yb, sample_weight=weights)
                else:
                    Yz = superT.predict(Z)
                    fidelity = accuracy_score(Yb, Yz)

                if self.extreme_fidelity:
                    res = superT.predict(x)
                    if res != y:
                        raise Exception('The prediction of the surrogate model is different wrt the black box')

                if self.verbose:
                    print('Retrieving explanation')
        x = x.flatten()
        Yc = superT.predict(X=Z)
        if self.binary == 'binary_from_nari' or self.binary == 'binary_from_dts' or self.binary == 'binary_from_bb':
            rule = get_rule(x, self.bb_predict(x.reshape(1, -1)), superT, self.feature_names, self.class_name,
                            self.class_values,
                            self.numeric_columns, encdec=self.encdec,
                            multi_label=self.multi_label)
        else:
            rule = get_rule_supert(x, superT, self.feature_names, self.class_name, self.class_values,
                                   self.numeric_columns,
                                   self.multi_label, encdec=self.encdec)
        if self.binary == 'binary_from_nari' or self.binary == 'binary_from_dts' or self.binary == 'binary_from_bb':
            crules, deltas = get_counterfactual_rules(x, Yc[0], superT, Z, Yc, self.feature_names,
                                                      self.class_name, self.class_values, self.numeric_columns,
                                                      self.features_map, self.features_map_inv, encdec=self.encdec,
                                                      filter_crules=self.filter_crules,
                                                      constraints=self.constraints, unadmittible_features=self.unadmittible_features)
        else:
            crules, deltas = get_counterfactual_rules_supert(x, Yc[0], superT, Z, Yc, self.feature_names,
                                                                 self.class_name, self.class_values,
                                                                 self.numeric_columns,
                                                                 self.features_map, self.features_map_inv,
                                                                 filter_crules=self.filter_crules, unadmittible_features=self.unadmittible_features)

        exp = Explanation()
        exp.bb_pred = Yb[0]
        exp.dt_pred = Yc[0]
        exp.rule = rule
        exp.crules = crules
        exp.deltas = deltas
        exp.dt = superT
        exp.fidelity = fidelity
            # Feature Importance
        if self.binary:
            feature_importance, feature_importance_all = self.get_feature_importance_binary(superT, x)

        else:
            feature_importance, feature_importance_all = self.get_feature_importance_supert(superT, x, len(Yb))
            # Exemplar and Counter-exemplar
        '''exemplars_rec, cexemplars_rec = self.get_exemplars_cexemplars_binary(superT, x, exemplar_num)
        if exemplars_rec is not None:
            print('entro con exemplars ', exemplars_rec, self.feature_names)
            exemplars = self.get_exemplars_str(exemplars_rec)
        else:
            exemplars = 'None'
        if cexemplars_rec is not None:
            cexemplars = self.get_exemplars_str(cexemplars_rec)
        else:
            cexemplars = 'None'
            '''
        exp.feature_importance = feature_importance
        exp.feature_importance_all = feature_importance_all
        #exp.exemplars = exemplars
        #exp.cexemplars = cexemplars
        return exp

    # qui l'istanza arriva originale
    def explain_instance_stable(self, x, samples=100, use_weights=True, metric=neuclidean, runs=3, exemplar_num=5,
                                n_jobs=-1, prune_tree=False, single=False, kwargs=None):

        if self.multi_label:
            print('Not yet implemented')
            raise Exception

        if self.encdec is not None:
            y = self.bb_predict(x.reshape(1, -1))
            x = self.encdec.enc(x, y)

        if isinstance(samples, int):
            if self.neigh_type == 'cfs':
                Z_list = self.multi_neighgen_fn_parallel(x, runs, samples, n_jobs)
            else:
                Z_list = self.multi_neighgen_fn(x, runs, samples, kwargs)
        else:
            Z_list = list()
            for z in samples:
                Z_list.append(np.array(z))

        Yb_list = list()
        #print('la Z creata ', len(Z_list), Z_list[0])
        if self.encdec is not None:
            for Z in Z_list:
                Z = self.encdec.dec(Z)
                #print('Z decodificata ', Z)
                Z = np.nan_to_num(Z)
                Yb = self.bb_predict(Z)
                #print('la yb ', Counter(Yb))
                Yb_list.append(Yb)
        else:
            if single:
                Yb = self.bb_predict(Z_list)
                Yb_list.append(Yb)
            else:
                for Z in Z_list:
                    Yb = self.bb_predict(Z)
                    Yb_list.append(Yb)

        if self.verbose:
            neigh_class_counts_list = list()
            for Yb in Yb_list:
                neigh_class, neigh_counts = np.unique(Yb, return_counts=True)
                neigh_class_counts = {self.class_values[k]: v for k, v in zip(neigh_class, neigh_counts)}
                neigh_class_counts_list.append(neigh_class_counts)

            for neigh_class_counts in neigh_class_counts_list:
                print('Synthetic neighborhood class counts %s' % neigh_class_counts)

        weights_list = list()
        if single:
            weights = None if not use_weights else self.__calculate_weights__(Z_list, metric)
            weights_list.append(weights)
        #print('la shape di z e come e fatta ', len(Z_list), Z[0].dtype)
        else:
            for Z in Z_list:
                #print('nel calcolo del peso', Z.dtype, Z.shape)
                weights = None if not use_weights else self.__calculate_weights__(Z, metric)
                weights_list.append(weights)

        if self.verbose:
            print('Learning local decision trees')

        # discretize the data employed for learning decision tree
        if self.discretize:
            if single:
                discr = RMEPDiscretizer()
                discr.fit(Z, Yb)
                Z_list = discr.transform(Z_list)
            else:
                Z = np.concatenate(Z_list)
                Yb = np.concatenate(Yb_list)

                discr = RMEPDiscretizer()
                discr.fit(Z, Yb)
                temp = list()
                for Zl in Z_list:
                    temp.append(discr.transform(Zl))
                Z_list = temp
        # caso binario da Z e Y da bb
        if self.binary == 'binary_from_bb':
            surr = DecTree()
            weights = None if not use_weights else self.__calculate_weights__(Z, metric)
            superT = surr.learn_local_decision_tree(Z, Yb, weights, self.class_values)
            fidelity = superT.score(Z, Yb, sample_weight=weights)

        # caso binario da Z e Yb
        # caso n ario
        # caso binario da albero n ario
        else:
            #qui prima creo tutti i dt, che servono sia per unirli con metodo classico o altri
            dt_list = [DecTree() for i in range(runs)]
            dt_list = Parallel(n_jobs=n_jobs, verbose=self.verbose,prefer='threads')(
                delayed(t.learn_local_decision_tree)(Zl, Yb, weights, self.class_values, prune_tree=prune_tree)
                for Zl, Yb, weights, t in zip(Z_list, Yb_list, weights_list, dt_list))

            Z = np.concatenate(Z_list)
            Z = np.nan_to_num(Z)
            Yb = np.concatenate(Yb_list)

            # caso binario da Z e Yb dei vari dt
            if self.binary == 'binary_from_dts':
                weights = None if not use_weights else self.__calculate_weights__(Z, metric)
                surr = DecTree()
                superT = surr.learn_local_decision_tree(Z, Yb, weights, self.class_values)
                fidelity = superT.score(Z, Yb, sample_weight=weights)

            # caso n ario
            # caso binario da albero n ario
            else:
                if self.verbose:
                    print('Pruning decision trees')
                surr = SuperTree()
                for t in dt_list:
                    surr.prune_duplicate_leaves(t)
                if self.verbose:
                    print('Merging decision trees')

                weights_list = list()
                for Zl in Z_list:
                    weights = None if not use_weights else self.__calculate_weights__(Zl, metric)
                    weights_list.append(weights)
                weights = np.concatenate(weights_list)
                n_features = list()
                for d in dt_list:
                    n_features.append(list(range(0, len(self.feature_names))))
                roots = np.array([surr.rec_buildTree(t, FI_used) for t, FI_used in zip(dt_list, n_features)])

                superT = surr.mergeDecisionTrees(roots, num_classes=np.unique(Yb).shape[0], verbose=False)

                if self.binary == 'binary_from_nari':
                    superT = surr.supert2b(superT, Z)
                    Yb = superT.predict(Z)
                    fidelity = superT.score(Z, Yb, sample_weight=weights)
                else:
                    Yz = superT.predict(Z)
                    fidelity = accuracy_score(Yb, Yz)

                if self.extreme_fidelity:
                    res = superT.predict(x)
                    if res != y:
                        raise Exception('The prediction of the surrogate model is different wrt the black box')

                if self.verbose:
                    print('Retrieving explanation')
        x = x.flatten()
        Yc = superT.predict(X=Z)
        if self.binary == 'binary_from_nari' or self.binary == 'binary_from_dts' or self.binary == 'binary_from_bb':
            rule = get_rule(x, self.bb_predict(x.reshape(1, -1)), superT, self.feature_names, self.class_name, self.class_values,
                                self.numeric_columns, encdec=self.encdec,
                                multi_label=self.multi_label)
        else:
            rule = get_rule_supert(x, superT, self.feature_names, self.class_name, self.class_values,
                                       self.numeric_columns,
                                       self.multi_label, encdec=self.encdec)
        if self.binary == 'binary_from_nari' or self.binary == 'binary_from_dts' or self.binary == 'binary_from_bb':
            #print('la shape di x che arriva fino alla get counter ', x, x.shape)
            crules, deltas = get_counterfactual_rules(x, Yc[0], superT, Z, Yc, self.feature_names,
                                                          self.class_name, self.class_values, self.numeric_columns,
                                                          self.features_map, self.features_map_inv, encdec=self.encdec,
                                                          filter_crules = self.filter_crules, constraints= self.constraints)
        else:
            #print('la shaoe di x che arriva a get counter con super t', x, x.shape)
            crules, deltas = get_counterfactual_rules_supert(x, Yc[0], superT, Z, Yc, self.feature_names,
                                                                 self.class_name, self.class_values, self.numeric_columns,
                                                                 self.features_map, self.features_map_inv,
                                                                 filter_crules = self.filter_crules)

        exp = Explanation()
        exp.bb_pred = Yb[0]
        exp.dt_pred = Yc[0]
        exp.rule = rule
        exp.crules = crules
        exp.deltas = deltas
        exp.dt = superT
        exp.fidelity = fidelity
        # Feature Importance
        if self.binary:
            feature_importance, feature_importance_all = self.get_feature_importance_binary(superT, x)
            exemplars_rec, cexemplars_rec = self.get_exemplars_cexemplars_binary(superT, x, exemplar_num)
        else:
            feature_importance, feature_importance_all = self.get_feature_importance_supert(superT, x, len(Yb))
            exemplars_rec, cexemplars_rec = self.get_exemplars_cexemplars_supert(superT, x, exemplar_num)
        # Exemplar and Counter-exemplar

        if exemplars_rec is not None:
            print('entro con exemplars ', exemplars_rec, self.feature_names)
            exemplars = self.get_exemplars_str(exemplars_rec)
        else:
            exemplars = 'None'
        if cexemplars_rec is not None:
            cexemplars = self.get_exemplars_str(cexemplars_rec)
        else:
            cexemplars = 'None'
        exp.feature_importance = feature_importance
        exp.feature_importance_all = feature_importance_all
        exp.exemplars = exemplars
        exp.cexemplars = cexemplars
        return exp

    def __init_neighbor_fn(self, ocr, categorical_use_prob, continuous_fun_estimation, size, kwargs):

        neighgen = None
        numeric_columns_index = list()
        self.feature_names = list(self.feature_names)
        for f in self.feature_names:
            if f in self.numeric_columns:
                numeric_columns_index.append(self.feature_names.index(f))
        #numeric_columns_index = [i for i, c in enumerate(self.feature_names) if c in self.numeric_columns]

        self.feature_values = None
        if self.neigh_type in ['random', 'genetic', 'rndgen', 'geneticp', 'rndgenp', 'counter']:
            if self.verbose:
                print('calculating feature values', self.K_original.shape)
            self.feature_values = calculate_feature_values(self.K_original, numeric_columns_index,
                                                           categorical_use_prob=categorical_use_prob,
                                                           continuous_fun_estimation=continuous_fun_estimation,
                                                           size=size)

        nbr_features = len(self.feature_names)
        nbr_real_features = self.K_original.shape[1]

        if self.neigh_type in ['genetic', 'rndgen', 'geneticp', 'rndgenp']:
            alpha1 = kwargs.get('alpha1', 0.5)
            alpha2 = kwargs.get('alpha2', 0.5)
            metric = kwargs.get('metric', neuclidean)
            ngen = kwargs.get('ngen', 10)
            mutpb = kwargs.get('mutpb', 0.5)
            cxpb = kwargs.get('cxpb', 0.7)
            tournsize = kwargs.get('tournsize', 3)
            halloffame_ratio = kwargs.get('halloffame_ratio', 0.1)
            random_seed = self.random_state

            if self.neigh_type == 'genetic':
                neighgen = GeneticGenerator(self.bb_predict, self.feature_values, self.features_map, nbr_features,
                                            nbr_real_features, numeric_columns_index, ocr=ocr, alpha1=alpha1,
                                            alpha2=alpha2, metric=metric, ngen=ngen,
                                            mutpb=mutpb, cxpb=cxpb, tournsize=tournsize,
                                            halloffame_ratio=halloffame_ratio, random_seed=random_seed, encdec=self.encdec,
                                            verbose=self.verbose)
            elif self.neigh_type == 'rndgen':
                neighgen = RandomGeneticGenerator(self.bb_predict, self.feature_values, self.features_map,
                                                  nbr_features, nbr_real_features, numeric_columns_index,
                                                  ocr=ocr, alpha1=alpha1, alpha2=alpha2,
                                                  metric=metric, ngen=ngen, mutpb=mutpb, cxpb=cxpb,
                                                  tournsize=tournsize, halloffame_ratio=halloffame_ratio,
                                                  random_seed=random_seed, encdec=self.encdec, verbose=self.verbose)
            elif self.neigh_type == 'geneticp':
                neighgen = GeneticProbaGenerator(self.bb_predict, self.feature_values, self.features_map, nbr_features,
                                                 nbr_real_features, numeric_columns_index, ocr=ocr, alpha1=alpha1,
                                                 alpha2=alpha2, metric=metric, ngen=ngen,
                                                 mutpb=mutpb, cxpb=cxpb, tournsize=tournsize,
                                                 halloffame_ratio=halloffame_ratio,
                                                 bb_predict_proba=self.bb_predict_proba,
                                                 random_seed=random_seed, encdec=self.encdec,
                                                 verbose=self.verbose)

            elif self.neigh_type == 'rndgenp':
                neighgen = RandomGeneticProbaGenerator(self.bb_predict, self.feature_values, self.features_map,
                                                       nbr_features, nbr_real_features, numeric_columns_index,
                                                       ocr=ocr, alpha1=alpha1, alpha2=alpha2,
                                                       metric=metric, ngen=ngen, mutpb=mutpb, cxpb=cxpb,
                                                       tournsize=tournsize, halloffame_ratio=halloffame_ratio,
                                                       bb_predict_proba=self.bb_predict_proba,
                                                       random_seed=random_seed, encdec=self.encdec, verbose=self.verbose)

        elif self.neigh_type == 'random':
            neighgen = RandomGenerator(self.bb_predict, self.feature_values, self.features_map, nbr_features,
                                       nbr_real_features, numeric_columns_index, ocr=ocr, encdec=self.encdec)
        elif self.neigh_type == 'closest':
            Kc = kwargs.get('Kc', None)
            k = kwargs.get('k', None)
            type = kwargs.get('core_neigh_type', 'simple')
            alphaf = kwargs.get('alphaf', 0.5)
            alphal = kwargs.get('alphal', 0.5)
            metric_features = kwargs.get('metric_features', neuclidean)
            metric_labels = kwargs.get('metric_labels', neuclidean)
            neighgen = ClosestInstancesGenerator(self.bb_predict, self.feature_values, self.features_map, nbr_features,
                                                 nbr_real_features, numeric_columns_index, ocr=ocr,
                                                 K=Kc, rK=self.K, k=k, core_neigh_type=type, alphaf=alphaf,
                                                 alphal=alphal, metric_features=metric_features,
                                                 metric_labels=metric_labels, categorical_use_prob=categorical_use_prob,
                                                 continuous_fun_estimation=continuous_fun_estimation, size=size, encdec=self.encdec,
                                                 verbose=self.verbose)
        elif self.neigh_type == 'cfs':
            if self.verbose:
                print('Neigh kind ', self.neigh_type)
                print('sampling kind ', kwargs.get('kind', None))
            neighgen = CFSGenerator(self.bb_predict, self.feature_values, self.features_map, nbr_features,
                                                 nbr_real_features, numeric_columns_index,
                                    ocr=ocr,
                                    kind=kwargs.get('kind', None),
                                    sampling_kind=kwargs.get('sampling_kind', None),
                                    #vicinity_sampler_kwargs=kwargs.get('vicinity_sampler_kwargs', None),
                                    stopping_ratio=kwargs.get('stopping_ratio', 0.01),
                                    n_batch=kwargs.get('n_batch', 560),
                                    check_upper_threshold=kwargs.get('check_upper_threshold', True),
                                    final_counterfactual_search=kwargs.get('final_counterfactual_search',True),
                                    verbose=kwargs.get('verbose', False),
                                    custom_sampling_threshold=kwargs.get('custom_sampling_threshold', None),
                                    custom_closest_counterfactual=kwargs.get('custom_closest_counterfactual',None),
                                    n=kwargs.get('n', 10000), balance=kwargs.get('balance', None),
                                    forced_balance_ratio = kwargs.get('forced_balance_ratio', 0.5),
                                    cut_radius=kwargs.get('cut_radius', False),
                                    downward_only=kwargs.get('downward_only', None),
                                    encdec=self.encdec
                                    )
        elif self.neigh_type == 'counter':
            if self.verbose:
                print('Neigh kind ', self.neigh_type)
            print('ecco self. k ' , self.K)
            neighgen = CounterGenerator(self.bb_predict, self.bb_predict_proba, self.feature_values, self.features_map, nbr_features,
                                        nbr_real_features, numeric_columns_index, encdec=self.encdec,
                                        original_data=self.K, verbose=self.verbose)
        else:
            print('unknown neighborhood generator')
            raise Exception

        self.neighgen_fn = neighgen.generate

    def get_exemplars_str(self, exemplars_rec):
        exemplars = '\n'.join([record2str(s, self.feature_names, self.numeric_columns, encdec=self.encdec) for s in exemplars_rec])
        return exemplars

    def get_exemplars_cexemplars_binary(self, dt, x, n):
        if self.encdec is not None:
            dataset = self.dataset.copy(deep=True)
            labels = dataset.pop(self.class_name)
            dataset = self.encdec.enc(dataset.values, labels.values)
            leave_id_K = dt.apply(dataset)
        else:
            print('la self k ', self.K)
            leave_id_K = dt.apply(self.K)

        leave_id_x = dt.apply(x.reshape(1, -1))
        exemplar_idx = np.where(leave_id_K == leave_id_x)
        if self.encdec is not None:
            exemplar_vals = dataset[exemplar_idx]
        else:
            exemplar_vals = self.K[exemplar_idx]

        cexemplar_idx = np.where(leave_id_K != leave_id_x)
        if self.encdec is not None:
            cexemplar_vals = dataset[cexemplar_idx]
        else:
            cexemplar_vals = self.K[cexemplar_idx]

        # find instance x in obtained list and remove it
        idx_to_remove = None
        if x in exemplar_vals:
            idx_to_remove = np.where((exemplar_vals == x).all(axis=1))[0]
        if idx_to_remove is not None:
            exemplar_vals = np.delete(exemplar_vals, idx_to_remove, axis=0)
        print('exemplars ', exemplar_vals, cexemplar_vals)
        if len(exemplar_vals)==0 and len(cexemplar_vals)==0:
            print('IN CASO NONE NONE vals', exemplar_vals, cexemplar_vals)
            return None, None
        elif len(exemplar_vals)==0:
            print('CASO DI C EX E NONE')
            distance_x_cexemplar = cdist(x.reshape(1, -1), cexemplar_vals, metric='euclidean').ravel()
            n = len(cexemplar_vals)
            first_n_dist_id_c = distance_x_cexemplar.argsort()[:n]
            first_n_cexemplar = cexemplar_vals[first_n_dist_id_c]
            return None, first_n_cexemplar
        else:
            print('CASO DI EX E NONE')
            distance_x_exemplar = cdist(x.reshape(1, -1), exemplar_vals, metric='euclidean').ravel()
            n = len(exemplar_vals)
            first_n_dist_id = distance_x_exemplar.argsort()[:n]
            first_n_exemplar = exemplar_vals[first_n_dist_id]
            return first_n_exemplar, None

        distance_x_exemplar = cdist(x.reshape(1, -1), exemplar_vals, metric='euclidean').ravel()
        distance_x_cexemplar = cdist(x.reshape(1, -1), cexemplar_vals, metric='euclidean').ravel()

        if len(exemplar_vals) < n or len(cexemplar_vals) < n:
            if self.verbose:
                print('maximum number of exemplars and counter-exemplars founded is : %s, %s', len(exemplar_vals),
                  len(cexemplar_vals))
            n = min(len(cexemplar_vals),len(exemplar_vals))
        first_n_dist_id = distance_x_exemplar.argsort()[:n]
        first_n_exemplar = exemplar_vals[first_n_dist_id]

        first_n_dist_id_c = distance_x_cexemplar.argsort()[:n]
        first_n_cexemplar = cexemplar_vals[first_n_dist_id_c]

        return first_n_exemplar, first_n_cexemplar

    def get_exemplars_cexemplars_supert(self, dt, x, n):
        if self.encdec is not None:
            dataset = self.dataset.copy(deep=True)
            labels = dataset.pop(self.class_name)
            dataset = self.encdec.enc(dataset.values, labels.values)
            leave_id_K = dt.apply(dataset)
        else:
            leave_id_K = dt.apply(self.K)

        print('leave id applied ', leave_id_K)

        leave_id_x = dt.apply(x.reshape(1, -1))

        exemplar_idx = np.where(leave_id_K == leave_id_x)
        print('exemplar idx ', len(exemplar_idx))

        if self.encdec is not None:
            exemplar_vals = dataset[exemplar_idx]
        else:
            exemplar_vals = self.K[exemplar_idx]
        cexemplar_idx = np.where(leave_id_K != leave_id_x)
        print('cexemplar idx ', len(cexemplar_idx))
        if self.encdec is not None:
            cexemplar_vals = dataset[cexemplar_idx]
        else:
            cexemplar_vals = self.K[cexemplar_idx]
        print('exemplar and counter exemplars ', exemplar_vals, cexemplar_vals)
        # find instance x in obtained list and remove it
        idx_to_remove = None
        if x in exemplar_vals:
            print('cerco la x')
            idx_to_remove = np.where((exemplar_vals == x).all(axis=1))[0]
        if idx_to_remove is not None:
            print('la tolgo')
            exemplar_vals = np.delete(exemplar_vals, idx_to_remove, axis=0)

        distance_x_exemplar = cdist(x.reshape(1, -1), exemplar_vals, metric='euclidean').ravel()
        distance_x_cexemplar = cdist(x.reshape(1, -1), cexemplar_vals, metric='euclidean').ravel()

        if len(exemplar_vals) < n or len(cexemplar_vals) < n:
            if self.verbose:
                print('maximum number of exemplars and counter-exemplars founded is : %s, %s', len(exemplar_vals),
                  len(cexemplar_vals))
            n = min(len(cexemplar_vals),len(exemplar_vals))
        first_n_dist_id = distance_x_exemplar.argsort()[:n]
        first_n_exemplar = exemplar_vals[first_n_dist_id]

        first_n_dist_id_c = distance_x_cexemplar.argsort()[:n]
        first_n_cexemplar = cexemplar_vals[first_n_dist_id_c]

        return first_n_exemplar, first_n_cexemplar

    def explain_set_instances_stable(self, X, n_workers, title, runs=3, n_jobs =4, n_samples=1000, exemplar_num=5, use_weights=True, metric=neuclidean, kwargs=None):
        # for parallelization
        items_for_worker = math.ceil( len(X)/ float(n_workers))
        start = 0
        print(items_for_worker)
        end = int(items_for_worker)
        processes = list()
        # create workers
        print("Dispatching jobs to workers...\n")
        for i in range(0, n_workers):
            print('start, end ', start, end)
            dataset = X[start:end]
            process = ml.Process(target=self.explain_workers_stable, args=(i, dataset, title, n_samples, use_weights, metric, runs, n_jobs, exemplar_num, kwargs))
            processes.append(process)
            process.start()

            if end > (len(X)-1):
                workers = n_workers - 1
                break
            start = end
            end += int(items_for_worker)

        # join workers
        for i in range(0, workers):
            processes[i].join()
        print("All workers joint.\n")

    def explain_workers_stable(self, i, dataset, title, n_samples, use_wieghts, metric, runs=3, n_jobs =4, exemplar_num=5, kwargs=None):
        count = 0
        results = list()
        title = 'explanations_lore' + title + '_' + str(i) + '.p'
        for d in dataset:
            print(i, count)
            count += 1
            d = np.array(d)
            exp = self.explain_instance_stable(d, samples=n_samples, use_weights=use_wieghts, metric=metric, runs=runs, exemplar_num=exemplar_num, n_jobs=n_jobs, kwargs=kwargs)
            results.append((d,exp))

        with open(title, "ab") as pickle_file:
            pickle.dump(results, pickle_file)

        return

    def set_unfeasibible_features(self, unadmittible_features):
        self.check_feasibility = True
        self.unadmittible_features = unadmittible_features


import colorama
import os
import random
def cls():
    os.system('cls')
from colorama import Fore,Back,Style
colorama.init(autoreset=1)
import time
def delay(tim):
    time.sleep(tim)
def load0anim(times):
    for i in range(times):
        print(Fore.MAGENTA + "\\")
        cls()
        print(Fore.CYAN + "-")
        cls()
        print(Fore.GREEN + Style.BRIGHT + "/")
        cls()
        print('|'+ Style.BRIGHT)
        cls()
def dice():
    ct = int(input(Fore.YELLOW + Style.BRIGHT + "Enter the No of Times to Roll the Dice: "))
    for i in range (ct):
        X = random.randrange(1,7)
        print(Fore.YELLOW + Style.BRIGHT + str(X))
        delay(0.5)
    tim = float(input("Shall I Wait For: "))
    delay(tim)
def tosscoin():
    ct = int(input(Fore.YELLOW + Style.BRIGHT + "Enter the No of Times to Toss the Coin: "))
    for i in range (ct):
        x = random.randrange(1,7)
        if x in [1,3,5]:
            print(Fore.RED + Style.BRIGHT + "HEADS")
            delay(0.5)
        else:
            print(Fore.CYAN + Style.BRIGHT + "TAILS")
            delay(0.5)
    tim = float(input("Shall I Wait For: "))
    delay(tim)
def xmas():
    tree = """
                *
               ***
              *****
            *********
        *****************
    *************************
*********************************
              *****
            *********
        *****************
    *************************
*********************************
              *****
            *********
        *****************
    *************************
*********************************
               ***
               ***
               ***
               ***
               ***
               ***
               ***
              *****
            *********
    """
    ucont = int(input("MERRY CHRISTMAS - ENTER THE NO OF TIMES TO RUN: "))
    for i in range(ucont):
        cls()
        print(Fore.RED + "               " + tree)
        time.sleep(0.5)
        cls()
        print(Fore.CYAN + Style.BRIGHT + "               " + tree)
        time.sleep(0.5)
        cls()
        print(Fore.GREEN + Style.BRIGHT + "               " + tree)
        time.sleep(0.5)
        cls()
        print(Fore.YELLOW + Style.BRIGHT + "               " + tree)
        time.sleep(0.5)
        cls()
        print(Fore.LIGHTMAGENTA_EX + "               " + tree)
        time.sleep(0.5)
        cls()
def gamesApp():
    while 1:
        cls()
        print(Fore.YELLOW + Style.DIM + """GAMES
            (1) Toss a Coin
            (2) Roll a Dice
            (X) Exit the Application and Go Back """)
        choice = input()
        if choice == '1':
            print("Lets Toss")
            time.sleep(1)
            cls()
            tosscoin()
        elif choice == '2':
            print("Preparing dice.")
            delay(1)
            cls()
            dice()
        elif choice == 'X':
            break
        elif choice == 'XMAS':
            xmas()
        else:
            continue


import pygame
from player import Player

class Overlay:
    def __init__(self, player: Player, display_surface: pygame.Surface):
        self.gameCanvas = display_surface
        self.player = player
        self.font = pygame.font.Font("../Assets/Fonts/monogram.ttf", 40)
        # Overlay
        self.overlayUI = pygame.image.load("../Assets/Overlay/overlay.png").convert_alpha()
        self.overlayUIRect = self.overlayUI.get_rect(topleft=(10, 10))

        self.playerIcon = pygame.image.load("../Assets/Overlay/player_icon.png").convert_alpha()
        self.playerIconRect = self.overlayUI.get_rect(topleft=(14, 9))

    def display(self):
        money = self.player.itemInventory["money"]
        money_surf = self.font.render(str(money), False, "#E2E2E2")
        self.gameCanvas.blit(self.overlayUI, self.overlayUIRect)
        self.gameCanvas.blit(self.playerIcon, self.playerIconRect)
        self.gameCanvas.blit(money_surf, (180, 63))


from collections import Counter

from aocd import data, submit

lines = data.split('\n')

cards = Counter({k: 1 for k in range(1, len(lines) + 1)})
for line in lines:
    card, numbers = line.split(': ')
    card = int(card.replace('Card ', ''))
    winning_, mine_ = numbers.split(' | ')
    winning = set(map(int, winning_.split()))
    mine = set(map(int, mine_.split()))
    overlap = len(winning & mine)
    for i in range(card + 1, card + overlap + 1):
        if i > len(lines):
            break
        cards[i] += cards[card]

answer = sum(cards.values())
print(answer)
submit(answer)


import argparse

import sgutils

def main():
    parser = argparse.ArgumentParser()

    parser.add_argument("device", action="store", help="Path to device to inquiry.")

    args = parser.parse_args()

    with open(args.device, mode="w+b") as device:
        print(sgutils.simple_inquiry(device))

if __name__ == "__main__":
    main()


from die import Die
import pygal

die_1 = Die()  # create a die using the Die Class
die_2 = Die() # create a second die using the Die Class

# try to make it take an imputted number of rolls
#number_rolls = int(input("How many rolls would you like (must be a number)? "))

# ------------------ Roll die and update the result list for each roll ------------------
results = []
for roll_num in range(1000000):
    result = die_1.roll() + die_2.roll()
    results.append(result)

# ------------------ Result Analysis ------------------
frequencies = []
max_result = die_1.num_sides + die_2.num_sides # adding the 2 dice maximum result
# recall that the list will start at 0 so we have to push it up one
for value in range(2, max_result+1):
    frequency = results.count(value)
    frequencies.append(frequency)

print(frequencies)

# ------------------ Visualize the results ------------------
hist = pygal.Bar() # Bar graph created with pygal

hist.title = "Results of rolling 6 sided die one million times"
hist.x_labels = list(range(2, max_result+1))
hist.x_title = "Results"
hist.y_title = "Frequency of Result"

hist.add('D6', frequencies)
hist.render_to_file('two_D6_visual.svg')


import pandas as pd
from sklearn.model_selection import train_test_split

# Read in the Iris data
iris_df = pd.read_csv("./../data/iris.txt", sep = ",")

# Create additional features that report whether a sample belongs to each species
iris_df['is_setosa'] = (iris_df['class'] == "Iris-setosa").astype(float)
iris_df['is_versicolor'] = (iris_df['class'] == "Iris-versicolor").astype(float)
iris_df['is_virginica'] = (iris_df['class'] == "Iris-virginica").astype(float)

# Create a feature for our future predictions
iris_df["predicted_class"] = "No prediction"

# Split data into training and test sets
train_df, test_df = train_test_split(iris_df, test_size = 0.2)

# Write the training and test sets to csv files
train_df.to_csv("./../data/iris_train.txt", index = False)
test_df.to_csv("./../data/iris_test.txt", index = False)


import torch.nn as nn
import torch
import data
from torch.autograd import Variable
class RNN(nn.Module):
   def __init__(self, input_size, h_size, output_size):
      super(RNN, self).__init__()

      self.h_size = h_size
      self.i2h = nn.Linear(input_size + h_size, h_size)
      self.i2o = nn.Linear(input_size + h_size, output_size)
      self.softmax = nn.LogSoftmax(dim = 1)

   def forward(self, input, hidden):
      combined = torch.cat((input, hidden), 1)
      hidden = self.i2h(combined)
      output = self.i2o(combined)
      output = self.softmax(output)
      return output, hidden

   def initHidden(self):
      return Variable(torch.zeros(1, self.h_size))

def get_output(output):
   _,top_i = output.topk(3)
   cat_index =[ top_i[0][i] for i in range(top_i[0].size()[0])]
   top_cats = []
   for index in cat_index:
     top_cats.append(data.all_categories[index])
   return top_cats
n_hidden = 128
rnn = RNN(data.n_letters, n_hidden, data.n_categories)


from django.conf.urls import url
from django.utils.translation import ugettext_lazy as _

from . import views

class MisagoAdminExtension(object):
    def register_urlpatterns(self, urlpatterns):
        urlpatterns.namespace(r'^settings/', 'settings', 'system')

        urlpatterns.patterns('system:settings',
            url(r'^$', views.index, name='index'),
            url(r'^(?P<key>(\w|-)+)/$', views.group, name='group'),
        )

    def register_navigation_nodes(self, site):
        site.add_node(
            name=_("Settings"),
            icon='fa fa-sliders',
            parent='misago:admin:system',
            link='misago:admin:system:settings:index',
        )


import multiprocessing
import subprocess

# Function to run the Dashboard
def run_Dash_Board():
    subprocess.run(['python', 'DB.py'])

# Function to run the received data
def run_rec_data():
    subprocess.run(['python', 'received_data.py'])

# Function to run the server script
def run_server():
    subprocess.run(['python', 'server.py'])

if __name__ == '__main__':
    # Create two processes to run the scripts concurrently
    Dashboard_process = multiprocessing.Process(target=run_Dash_Board)
    received_process = multiprocessing.Process(target=run_rec_data)
    server_process = multiprocessing.Process(target=run_server)

    # Start the processes
    Dashboard_process.start()
    received_process.start()
    server_process.start()

    # Wait for the processes to finish
    Dashboard_process.join()
    received_process.join()
    server_process.join()


import numpy as np

from collections import deque
from keras.models import Model
from keras.layers import Input, Dense
from keras.optimizers import adam_v2

import random
import utility_functions as util

REPLAY_MEMORY_SIZE = 42000  # How many of last   to keep for model training, 42000 means remember last ~200 games
MIN_REPLAY_MEMORY_SIZE = 4200  # Minimum number of tricks in memory to start training, 10500 means at least ~20 games
MINIBATCH_SIZE = 32  # How many steps (samples) to use for training

# Agent class
class PlayingNetwork:
    def __init__(self, input_size, name):

        self.input_size = input_size
        self.name = name

        # Main model
        self.model = self.create_model()

        # An array with last n steps for training
        self.replay_memory = deque(maxlen=REPLAY_MEMORY_SIZE)

    @staticmethod
    def dense_layer(num_units):
        return Dense(num_units, activation="relu")

    def create_model(self):

        input1 = Input(self.input_size)  # input size for observation state is 3732
        # input2 = Input(1)  # scalar input for action taken
        #
        # combined = concatenate([input1, input2])
        if self.input_size > 3000:
            if self.name == "small":
                x = self.dense_layer(1024)(input1)
            else:
                x = self.dense_layer(2048)(input1)
            if self.name == "large":
                x = self.dense_layer(1024)(x)
            x = self.dense_layer(512)(x)
            x = self.dense_layer(256)(x)
            x = self.dense_layer(128)(x)
        else:
            x = self.dense_layer(128)(input1)
        x = self.dense_layer(64)(x)
        x = self.dense_layer(32)(x)
        x = Dense(1, activation="linear")(x)  # evaluation

        model = Model(inputs=input1, outputs=x)

        model.compile(
            loss="mse",
            optimizer=adam_v2.Adam(learning_rate=0.0001),
            metrics=["accuracy"],
        )
        # print(model.summary())
        return model

    # Adds data to a memory replay array
    # (state, reward)
    def update_replay_memory(self, transition):
        # util.write_state(util.key_to_state(192, transition[0]), "backprop", 192)
        self.replay_memory.append(transition)

    # Trains main network once every backpropagation/round
    def train(self):
        # print("Calling train, Replay size: ", len(self.replay_memory))
        # Start training only if certain number of samples is already saved
        if len(self.replay_memory) < MIN_REPLAY_MEMORY_SIZE:
            return 0

        # Get a minibatch of random samples from memory replay table
        minibatch = random.sample(self.replay_memory, MINIBATCH_SIZE)

        # Get states (x) and rewards (y) from minibatch
        states = np.array(
            [
                util.key_to_state(self.input_size, transition[0])
                for transition in minibatch
            ]
        )
        rewards = np.array([transition[1] for transition in minibatch])

        # Fit on all samples as one batch, log only on terminal state
        history = self.model.fit(
            states,
            rewards,
            batch_size=MINIBATCH_SIZE,
            verbose=0,
            shuffle=False,
        )
        loss = history.history["loss"][0]
        return loss

    def predict(self, state):
        return self.model.predict(state.reshape(-1, *state.shape))[0]


#####################################
# Author: Davi Sidarta-Oliveira
# School of Medical Sciences,University of Campinas,Brazil
# contact: davisidarta[at]fcm[dot]unicamp[dot]com
# License: MIT
######################################
# Being clear about laplacian-type operators and spectral decompositions

from warnings import warn
import numpy as np
import pandas as pd
from scipy import sparse
from sklearn.utils import check_random_state, as_float_array

def _dense_degree(W):
    return np.diag(W.sum(axis=1))

def _dense_unnormalized_laplacian(W, return_D=False):
    D = _dense_degree(W)
    L = D - W
    # Return ndarray instead of np.matrix
    if return_D:
        return L.A, D
    else:
        return L.A

def _dense_symmetric_normalized_laplacian(W, return_D=False):
    D = _dense_degree(W)
    L = D - W
    Dinvs = np.diag(1/np.sqrt(D.diagonal()))
    # Lsym = D^-1/2 @ L @ D^-1/2 = I - D^-1/2 @ W @ D^-1/2
    Lsym_norm = Dinvs @ (L @ Dinvs)
    if return_D:
        return Lsym_norm, Dinvs
    else:
        return Lsym_norm

def _dense_normalized_random_walk_laplacian(W, return_D=False):
    D = _dense_degree(W)
    # Lr = D^-1@Lr = I - D^-1@W = I - P
    Lr = np.eye(*W.shape) - (np.diag(1/D.diagonal())@W)
    if return_D:
        return Lr, D
    else:
        return Lr

def _dense_diffusion(W, alpha=0, semi_aniso=False):
    D = _dense_degree(W)
    if alpha > 0:
        Da = np.diag(1/(D.diagonal()**alpha))
        Wa = Da @ (W @ Da)
        Dd = _dense_degree(Wa)
        if semi_aniso:
            Pa = (np.diag(1/Dd.diagonal())@W)
        else:
            Pa = (np.diag(1/Dd.diagonal())@Wa)
    else:
        Pa = (np.diag(1/D.diagonal())@W)
    return Pa

def _dense_diffusion_symmetric(W, alpha=0, semi_aniso=False, return_D_inv_sqrt=False):
    if alpha < 0:
        alpha = 0
    D = _dense_degree(W)
    if alpha > 0:
        # Dinva is D^-alpha
        Dinva = np.diag(1/(D.diagonal()**alpha))
        Wa = Dinva @ (W @ Dinva)
        Da = _dense_degree(Wa)
        Dalpha_inv = np.diag(1/(Da.diagonal()))
        if semi_aniso:
            Pa = Dalpha_inv @ W
            D_right = _dense_degree(W)
        else:
            Pa = Dalpha_inv @ Wa
            D_right = _dense_degree(Wa)
    else:
        Pa = (np.diag(1/D.diagonal())@W)
        D_right = _dense_degree(W)
    # Now let's build the symmetrized version Pasym:
    D_left = D_right.copy()
    D_right = np.sqrt(D_right.diagonal())
    D_left = np.diag(1/np.sqrt(D_left.diagonal()))
    Psym = D_right @ (Pa @ D_left)
    if return_D_inv_sqrt:
        return Psym, D_left
    else:
        return Psym

def _sparse_degree(W):
    N = np.shape(W)[0]
    D = np.ravel(W.sum(axis=1))
    return sparse.csr_matrix((D, (range(N), range(N))), shape=[N, N])

def degree(W):
    if sparse.issparse(W):
        return _sparse_degree(W)
    else:
        return _dense_degree(W)

def _sparse_unnormalized_laplacian(W, return_D=False):
    D = _sparse_degree(W)
    L = D - W
    if return_D:
        return L, D
    else:
        return L

def _sparse_symmetrized_normalized_laplacian(W, return_D=False):
    D = _sparse_degree(W)
    L = D - W
    N = np.shape(W)[0]
    D_tilde = np.ravel(W.sum(axis=1))
    # D ^-1/2:
    D_tilde[D_tilde != 0] = 1 / np.sqrt(D_tilde[D_tilde != 0])
    Dinvs = sparse.csr_matrix((D_tilde, (range(N), range(N))), shape=[N, N])
    # Lsym = D^-1/2 @ L @ D^-1/2 = I - D^-1/2 @ W @ D^-1/2
    Lsym_norm = Dinvs.dot(L).dot(Dinvs)
    if return_D:
        return Lsym_norm, Dinvs
    else:
        return Lsym_norm

def _sparse_normalized_random_walk_laplacian(W, return_D=False):
    N = np.shape(W)[0]
    D = np.ravel(W.sum(axis=1))
    # D ^-1:
    D[D != 0] = 1 / D[D != 0]
    Dinv = sparse.csr_matrix((D, (range(N), range(N))), shape=[N, N])
    I = sparse.identity(W.shape[0], dtype=np.float32)
    Lr = I - Dinv.dot(W)
    if return_D:
        return Lr, Dinv
    else:
        return Lr

def _sparse_diffusion(W, alpha=0, semi_aniso=False):
    # Note the resulting operator is not symmetric!
    N = np.shape(W)[0]
    D = np.ravel(W.sum(axis=1))
    if alpha > 0:
        D[D != 0] = D[D != 0] ** (-alpha)
        # Dinva is D^-alpha
        Dinva = sparse.csr_matrix((D, (range(N), range(N))), shape=[N, N])
        Wa = Dinva.dot(W).dot(Dinva)
        Da = np.ravel(Wa.sum(axis=1))
        Da[Da != 0] = 1 / Da[Da != 0]
        # Da is now D(alpha)^-1
        if semi_aniso:
            # Weights the original kernel with the reweighted degree (non-canonical idea of mine, but works quite well)
            P = sparse.csr_matrix((Da, (range(N), range(N))), shape=[N, N]).dot(W)
        else:
            P = sparse.csr_matrix((Da, (range(N), range(N))), shape=[N, N]).dot(Wa)
    else:
        D[D != 0] = 1 / D[D != 0]
        Dd = sparse.csr_matrix((D, (range(N), range(N))), shape=[N, N])
        P = Dd.dot(W)
    return P

def _sparse_diffusion_symmetric(W, alpha=0, semi_aniso=False, return_D_inv_sqrt=False):
    if alpha < 0:
        alpha = 0
    N = np.shape(W)[0]
    D = np.ravel(W.sum(axis=1))
    if alpha > 0:
        D[D != 0] = D[D != 0] ** (-alpha)
        # Dinva is D^-alpha
        Dinva = sparse.csr_matrix((D, (range(N), range(N))), shape=[N, N])
        Wa = Dinva.dot(W).dot(Dinva)
        Da = np.ravel(Wa.sum(axis=1))
        Da[Da != 0] = 1 / Da[Da != 0]
        Dalpha_inv = sparse.csr_matrix((Da, (range(N), range(N))), shape=[N, N])
        if semi_aniso:
            # Weights the original kernel with the reweighted degree (non canonical idea of mine, but works quite well)
            Pa = Dalpha_inv.dot(W)
            D_right = np.ravel(W.sum(axis=1))
        else:
            Pa = Dalpha_inv.dot(Wa)
            D_right = np.ravel(Wa.sum(axis=1))
    else:
        D[D != 0] = 1 / D[D != 0]
        Pa = sparse.csr_matrix((D, (range(N), range(N))), shape=[N, N]).dot(W)
        D_right = np.ravel(W.sum(axis=1))
    D_left = D_right.copy()
    D_right[D_right != 0] = np.sqrt(D_right[D_right != 0])
    D_left[D_left != 0] = 1 / np.sqrt(D_left[D_left != 0])
    D_right = sparse.csr_matrix((D_right, (range(N), range(N))), shape=[N, N])
    D_left = sparse.csr_matrix((D_left, (range(N), range(N))), shape=[N, N])
    # Note the resulting operator is symmetric!
    Psym = D_right.dot(Pa).dot(D_left)
    if return_D_inv_sqrt:
        return Psym, D_left
    else:
        return Psym

def graph_laplacian(W, laplacian_type='normalized', return_D=False):
    """
    Compute the graph Laplacian, given a adjacency or affinity graph W. For a friendly reference,
    see this material from James Melville: https://jlmelville.github.io/smallvis/spectral.html

    Parameters
    ----------
    W : scipy.sparse.csr_matrix or np.ndarray
        The graph adjacency or affinity matrix. Assumed to be symmetric and with zero diagonal.
        No further symmetrization is performed, so make sure to symmetrize W if necessary (usually done additively with W = (W + W.T)/2 ).

    laplacian : str (optional, default 'random_walk').
        The type of laplacian to use. Can be 'unnormalized', 'normalized' or 'random_walk'.

    return_D : bool (optional, default False).
        Whether to also return a degree matrix with the Laplacian in a tuple

    Returns
    -------
    L : scipy.sparse.csr_matrix
        The graph Laplacian.
    """
    if sparse.issparse(W):
        if laplacian_type == 'unnormalized':
            lap_fun = _sparse_unnormalized_laplacian
        elif laplacian_type == 'normalized':
            lap_fun = _sparse_symmetrized_normalized_laplacian
        elif laplacian_type == 'random_walk':
            lap_fun = _sparse_normalized_random_walk_laplacian
        else:
            raise ValueError('Unknown laplacian type: {}'.format(laplacian_type) + '. Should \
            be one of \"unnormalized\", \"normalized\", or \"random_walk\".')
    else:
        if laplacian_type == 'unnormalized':
            lap_fun = _dense_unnormalized_laplacian
        elif laplacian_type == 'normalized':
            lap_fun = _dense_symmetric_normalized_laplacian
        elif laplacian_type == 'random_walk':
            lap_fun = _dense_normalized_random_walk_laplacian
        else:
            raise ValueError('Unknown laplacian type: {}'.format(laplacian_type) + '. Should \
            be one of \"unnormalized\", \"normalized\", or \"random_walk\".')
    return lap_fun(W, return_D)

def LE(W, n_eigs=10, laplacian_type='random_walk', drop_first=True, return_evals=False, eigen_tol=0, random_state=None):
    """
    Performs [Laplacian Eigenmaps](https://www2.imm.dtu.dk/projects/manifold/Papers/Laplacian.pdf), given a adjacency or affinity graph W.
    The graph W can be a sparse matrix or a dense matrix. It is assumed to be symmetric (no further symmetrization is performed, be sure it is),
    and with zero diagonal (all diagonal elements are 0). The eigenvectors associated with the smallest eigenvalues
    form a new orthonormal basis which represents the graph in the feature space and are useful for denoising and clustering.

    Parameters
    ----------
    W : scipy.sparse.csr_matrix or np.ndarray
        The graph adjacency or affinity matrix. Assumed to be symmetric and with zero diagonal.

    n_eigs : int (optional, default 10).
        The number of eigenvectors to compute.

    laplacian : str (optional, default 'random_walk').
        The type of laplacian to use. Can be 'unnormalized', 'symmetric', or 'random_walk'.

    drop_first : bool (optional, default True).
        Whether to drop the first eigenvector.

    return_evals : bool (optional, default False).
        Whether to return the eigenvalues. If True, returns a tuple of (eigenvectors, eigenvalues).

    eigen_tol : float (optional, default 0).
        The tolerance for the eigendecomposition.

    random_state : int (optional, default None).
        The random state for the eigendecomposition in scipy.sparse.linalg.lobpcg() if the data has more than
        a million samples.

    Returns
    -----------
    evecs : np.ndarray of shape (W.shape[0], n_eigs)
        The eigenvectors of the graph Laplacian, sorted by ascending eigenvalues.

    If return_evals:
        evecs, evals : tuple of ndarrays
        The eigenvectors and associated eigenvalues, sorted by ascending eigenvalues.

    """
    random_state = check_random_state(random_state)
    if n_eigs > np.shape(W)[0]:
        raise ValueError(
            'n_eigs must be less than or equal to the number of nodes.')
    # Compute graph Laplacian
    L = graph_laplacian(W, laplacian_type)
    if not sparse.issparse(L):
        L = sparse.csr_matrix(L)  # for ARPACK efficiency
    # Add one more eig if drop_first is True
    if drop_first:
        n_eigs = n_eigs + 1
    # Compute eigenvalues and eigenvectors
    try:
        if L.shape[0] < 1000000:
            evals, evecs = sparse.linalg.eigsh(
                L, k=n_eigs, which='SM', tol=eigen_tol, maxiter=L.shape[0] * 5)
        else:
            evals, evecs = sparse.linalg.lobpcg(
                L, random_state.normal(size=(L.shape[0], n_eigs)), largest=False, tol=1e-8
            )
    except sparse.linalg.ArpackError:
            warn(
                "WARNING: spectral decomposition FAILED! This is likely due to too small an eigengap. Consider\n"
                "adding some noise or jitter to your data."
            )
            return None
    evals = np.real(evals)
    evecs = np.real(evecs)
    # Sort eigenvalues and eigenvectors in ascending order
    idx = evals.argsort()
    evals = evals[idx]
    evecs = evecs[:, idx]
    # Normalize
    for i in range(evecs.shape[1]):
        evecs[:, i] = evecs[:, i] / np.linalg.norm(evecs[:, i])
    # Return embedding and evals
    if drop_first:
        evecs = evecs[:, 1:]
        evals = evals[1:]
    if return_evals:
        return evecs, evals
    else:
        return evecs

def diffusion_operator(W, alpha=1.0, symmetric=False, semi_aniso=False, return_D_inv_sqrt=False):
    """
    Computes the [diffusion operator](https://doi.org/10.1016/j.acha.2006.04.006).

    Parameters
    ----------
    W : scipy.sparse.csr_matrix or np.ndarray
        The graph adjacency or affinity matrix. Assumed to be symmetric and with zero diagonal.
        No further symmetrization is performed, so make sure to symmetrize W if necessary (usually done additively with W = (W + W.T)/2 ).

    alpha : float (optional, default 1.0).
        Anisotropy to apply. 'Alpha' in the diffusion maps literature.

    symmetric : bool (optional, default True).
        Whether to use a symmetric version of the diffusion operator. This is particularly useful to yield a symmetric operator
        when using anisotropy (alpha > 0), as the diffusion operator P would be assymetric otherwise, which can be problematic
        during matrix decomposition. Eigenvalues are the same of the assymetric version, and the eigenvectors of the original assymetric
        operator can be obtained by left multiplying by D_inv_sqrt (returned if `return_D_inv_sqrt` set to True).

    semi_aniso : bool (optional, default False).
        Whether to use semi-anisotropic diffusion. This reweights the original kernel  (not the renormalized kernel) by the renormalized degree.

    return_D_inv_sqrt : bool (optional, default False).
        Whether to return a tuple of diffusion operator P and inverse square root of the degree matrix.

    Returns
    -------
    P : scipy.sparse.csr_matrix
        The graph diffusion operator.

    """
    # Compute diffusion operator
    if sparse.issparse(W):
        if symmetric:
            P, D_left = _sparse_diffusion_symmetric(
                W, alpha, semi_aniso=semi_aniso, return_D_inv_sqrt=return_D_inv_sqrt)
        else:
            P = _sparse_diffusion(W, alpha, semi_aniso)
    else:
        if symmetric:
            P, D_left = _dense_diffusion_symmetric(
                W, alpha, semi_aniso=semi_aniso, return_D_inv_sqrt=return_D_inv_sqrt)
        else:
            P = _dense_diffusion(W, alpha, semi_aniso)
    if symmetric:
        if return_D_inv_sqrt:
            return P, D_left
        else:
            return P
    else:
        return P

def DM(W, n_eigs=10, alpha=0, return_evals=False, symmetric=False, eigen_tol=10e-4, t=None):
    """
    Performs [Diffusion Maps](https://doi.org/10.1016/j.acha.2006.04.006), given a adjacency or affinity graph W.
    The graph W can be a sparse matrix or a dense matrix. It is assumed to be symmetric (no further symmetrization is performed, be sure it is),
    and with zero diagonal (all diagonal elements are 0). The eigenvectors associated with the largest eigenvalues
    form a new orthonormal basis which represents the graph in the feature space and are useful for denoising and clustering.

    Parameters
    ----------

    W : scipy.sparse.csr_matrix or np.ndarray
        The graph adjacency or affinity matrix. Assumed to be symmetric and with zero diagonal.

    n_eigs : int (optional, default 10).
        The number of eigenvectors to return.

    alpha : float (optional, default 0).
        Anisotropy to be applied to the diffusion map. Refered to as alpha in the diffusion maps literature.

    return_evals : bool (optional, default False).
        Whether to return the eigenvalues. If True, returns a tuple of (eigenvectors, eigenvalues).

    symmetric : bool (optional, default False).
        Whether to use a symmetric version of the diffusion operator. This is particularly useful to yield a symmetric operator, but
        that can also be obtained by a simplistic mean with its the transpose with near identical results.

    eigen_tol : float (optional, default 0).
        The tolerance for the eigendecomposition in scipy.sparse.linalg.eigsh().

    t : int (optional, default 1).
        The number of steps to take in the diffusion map.

    Returns
    ----------
        * If return_evals is True :
            A tuple of scaled eigenvectors (the diffusion maps) and eigenvalues.
        * If return_evals is False :
            An array of scaled eigenvectors (the diffusion maps).

    """
    if t is not None:
        if not isinstance(t, int):
            raise ValueError('t must be `None` or an integer.')
    if n_eigs > np.shape(W)[0]:
        raise ValueError(
            'n_eigs must be less than or equal to the number of nodes.')
    # Compute diffusion operator
    if symmetric:
        P, D_left = diffusion_operator(
            W, alpha, symmetric, return_D_inv_sqrt=True)
    else:
        P = diffusion_operator(W, alpha, symmetric, return_D_inv_sqrt=False)
    # Compute eigenvalues and eigenvectors
    evals, evecs = sparse.linalg.eigsh(
        P, k=n_eigs, which='LM', tol=eigen_tol, maxiter=P.shape[0] * 5)
    evecs = np.real(evecs)
    evals = np.real(evals)
    if symmetric:
        evecs = D_left.dot(evecs)
    idx = evals.argsort()[::-1]
    evals = evals[idx]
    evecs = evecs[:, idx]
    # Normalize
    for i in range(evecs.shape[1]):
        evecs[:, i] = evecs[:, i] / np.linalg.norm(evecs[:, i])
    if t is not None:
        if t > 1:
            evals = np.power(evals, t)
        diffmap = evecs * evals
    else:
        use_eigs = int(np.sum(evals > 0, axis=0))
        eigs_idx = list(range(1, int(use_eigs)))
        eig_vals = np.ravel(evals[eigs_idx])
        diffmap = evecs[:, eigs_idx] * (eig_vals / (1 - eig_vals))
    if return_evals:
        return diffmap, evals
    else:
        return diffmap

def _set_diag(laplacian, value, norm_laplacian):
    n_nodes = laplacian.shape[0]
    # We need all entries in the diagonal to values
    if not sparse.isspmatrix(laplacian):
        if norm_laplacian:
            laplacian.flat[:: n_nodes + 1] = value
    else:
        laplacian = laplacian.tocoo()
        if norm_laplacian:
            diag_idx = laplacian.row == laplacian.col
            laplacian.data[diag_idx] = value
        # If the matrix has a small number of diagonals (as in the
        # case of structured matrices coming from images), the
        # dia format might be best suited for matvec products:
        n_diags = np.unique(laplacian.row - laplacian.col).size
        if n_diags <= 7:
            # 3 or less outer diagonals on each side
            laplacian = laplacian.todia()
        else:
            # csr has the fastest matvec and is thus best suited to
            # arpack
            laplacian = laplacian.tocsr()
    return laplacian

def spectral_clustering(init, max_svd_restarts=50, n_iter_max=50, random_state=None, copy=True):
    """
    Search for a partition matrix (clustering) which is closest to the eigenvector embedding.

    Parameters
    ----------
    init : array-like of shape (n_samples, n_clusters)
        The embedding space of the samples.
    max_svd_restarts : int, default=30
        Maximum number of attempts to restart SVD if convergence fails
    n_iter_max : int, default=30
        Maximum number of iterations to attempt in rotation and partition
        matrix search if machine precision convergence is not reached
    random_state : int, RandomState instance, default=None
        Determines random number generation for rotation matrix initialization.
        Use an int to make the randomness deterministic.
        See :term:`Glossary <random_state>`.
    copy : bool, default=True
        Whether to copy vectors, or perform in-place normalization.

    Returns
    -------
    labels : array of integers, shape: n_samples
        The labels of the clusters.

    References
    ----------
    - Multiclass spectral clustering, 2003
      Stella X. Yu, Jianbo Shi
      https://www1.icsi.berkeley.edu/~stellayu/publication/doc/2003kwayICCV.pdf

    Notes
    -----
    The eigenvector embedding is used to iteratively search for the
    closest discrete partition.  First, the eigenvector embedding is
    normalized to the space of partition matrices. An optimal discrete
    partition matrix closest to this normalized embedding multiplied by
    an initial rotation is calculated.  Fixing this discrete partition
    matrix, an optimal rotation matrix is calculated.  These two
    calculations are performed until convergence.  The discrete partition
    matrix is returned as the clustering solution.  Used in spectral
    clustering, this method tends to be faster and more robust to random
    initialization than k-means.
    """

    from scipy.sparse import csc_matrix
    from scipy.linalg import LinAlgError

    random_state = check_random_state(random_state)
    vectors = as_float_array(init, copy=copy)

    eps = np.finfo(float).eps
    n_samples, n_components = vectors.shape

    # Normalize the eigenvectors to an equal length of a vector of ones.
    # Reorient the eigenvectors to point in the negative direction with respect
    # to the first element.  This may have to do with constraining the
    # eigenvectors to lie in a specific quadrant to make the discretization
    # search easier.
    norm_ones = np.sqrt(n_samples)
    for i in range(vectors.shape[1]):
        vectors[:, i] = (vectors[:, i] / np.linalg.norm(vectors[:, i])) \
            * norm_ones
    if vectors[0, i] != 0:
        vectors[:, i] = -1 * vectors[:, i] * np.sign(vectors[0, i])

    # Normalize the rows of the eigenvectors.  Samples should lie on the unit
    # hypersphere centered at the origin.  This transforms the samples in the
    # embedding space to the space of partition matrices.
    vectors = vectors / np.sqrt((vectors ** 2).sum(axis=1))[:, np.newaxis]

    svd_restarts = 0
    has_converged = False

    # If there is an exception we try to randomize and rerun SVD again
    # do this max_svd_restarts times.

    while (svd_restarts < max_svd_restarts) and not has_converged:

        # Initialize first column of rotation matrix with a row of the
        # eigenvectors
        rotation = np.zeros((n_components, n_components))
        rotation[:, 0] = vectors[random_state.randint(n_samples), :].T

        # To initialize the rest of the rotation matrix, find the rows
        # of the eigenvectors that are as orthogonal to each other as
        # possible
        c = np.zeros(n_samples)
        for j in range(1, n_components):
            # Accumulate c to ensure row is as orthogonal as possible to
            # previous picks as well as current one
            c += np.abs(np.dot(vectors, rotation[:, j - 1]))
            rotation[:, j] = vectors[c.argmin(), :].T

        last_objective_value = 0.0
        n_iter = 0

        while not has_converged:
            n_iter += 1

            t_discrete = np.dot(vectors, rotation)

            labels = t_discrete.argmax(axis=1)
            vectors_discrete = csc_matrix(
                (np.ones(len(labels)), (np.arange(0, n_samples), labels)),
                shape=(n_samples, n_components))

            t_svd = vectors_discrete.T * vectors

            try:
                U, S, Vh = np.linalg.svd(t_svd)
                svd_restarts += 1
            except LinAlgError:
                print("SVD did not converge, randomizing and trying again")
                break

            ncut_value = 2.0 * (n_samples - S.sum())
            if ((abs(ncut_value - last_objective_value) < eps) or
                    (n_iter > n_iter_max)):
                has_converged = True
            else:
                # otherwise calculate rotation and continue
                last_objective_value = ncut_value
                rotation = np.dot(Vh.T, U.T)

    if not has_converged:
        raise LinAlgError('SVD did not converge')
    return labels


from rtree import index
from numpy import percentile

def colorize(column):
    data = []
    p33 = percentile(column.values, 33)
    p66 = percentile(column.values, 66)
    for x in column.values:
        if x > p66:
            data.append(3)
            continue
        elif x > p33:
            data.append(2)
            continue
        else:
            data.append(1)
    return data

def set_spatial_index(coordinates):
    p = index.Property()
    p.dimension = 2
    ind= index.Index(properties=p)
    for x,y in zip(coordinates.keys(),coordinates.values()):
        ind.add(x,y)
    return ind


from pydantic.main import BaseModel
from worker import (
    Worker,
    WorkerMeta
)
from domain import Domain
import csv
import trio
from loguru import logger as l
from typing import Dict

class ReporterMeta(BaseModel):
    outcome: Dict[str, int] = {}

    def colorful_outcome(self, oc: str) -> str:
        colors = {"OK": "green", "NXDOMAIN": "bold magenta",
                  "ValueError": "bold red", "Timeout": "bold red"}
        if oc in colors:
            return f"[{colors[oc]}]{oc}"
        return f"[dim]{oc}"

class Reporter(Worker):
    def __init__(self, data_source: trio.MemoryReceiveChannel, dest_file: str):
        self.__data_source = data_source
        self.__dest_file = dest_file
        self.__csv_dest = open(self.__dest_file, 'w')
        self.writer = csv.writer(self.__csv_dest)
        self.__meta__ = WorkerMeta(
            w_id=0, name="CSVWriter", entity=self.__dest_file)
        self.__stats__ = ReporterMeta()

    def item_filter(self, item: Domain) -> bool:
        """Filters domains to be reported"""
        return item.outcome == "NXDOMAIN"

    def submit_stats(self, item: Domain):
        """Update the domain resolve outcome statistics"""
        if item.outcome in self.__stats__.outcome:
            self.__stats__.outcome[item.outcome] += 1
        else:
            self.__stats__.outcome[item.outcome] = 1

    async def run(self):
        """Writes the CSV report file and updates the statistcs"""
        async with self.__data_source:
            self.change_status("running")
            async for item in self.__data_source:
                self.submit_stats(item)
                if self.item_filter(item):
                    self.writer.writerow([item.name, item.popularity])
                    self.__meta__.items_processed += 1
        self.change_status("done")


from zipfile import ZipFile
from urllib.request import urlopen
from io import BytesIO
from bs4 import BeautifulSoup

wordFile = urlopen('http://pythonscraping.com/pages/AWordDocument.docx').read()
wordFile = BytesIO(wordFile)
document = ZipFile(wordFile)
xml_content = document.read('word/document.xml')

wordObj = BeautifulSoup(xml_content.decode('utf-8'), 'xml')
textStrings = wordObj.find_all('w:t')

for textElem in textStrings:
    style = textElem.parent.parent.find('w:pStyle')
    if style is not None and style['w:val'] == 'Title':
        print('Title is: {}'.format(textElem.text))
    else:
        print(textElem.text)


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models, migrations

class Migration(migrations.Migration):

    dependencies = [
        ('writing', '0003_auto_20171016_1003'),
    ]

    operations = [
        migrations.RemoveField(
            model_name='writing',
            name='uid',
        ),
        migrations.AlterField(
            model_name='writing',
            name='id',
            field=models.CharField(max_length=30, serialize=False, verbose_name=b'\xe7\x94\xa8\xe6\x88\xb7id', primary_key=True),
        ),
    ]