# -*- coding: utf-8 -*-
# occiput
# Harvard University, Martinos Center for Biomedical Imaging
# Aalto University, Department of Computer Science

# Interface to list-mode data. In order to cope with various data formats of the list-mode data and
# to enable fast decoding, occiput uses a simple plug-in system. The routines
# to decode the list-mode data are defined in a C function that is loaded as a plugin.
# The simple plugin system handles the compilation of the code.

from .PET_projection import PET_Projection
import numpy as np

__all__ = ["Listmode_Loader"]

class ListmodeWrapper:
    def open_stream(self, stream):
        PET_listmode_open_stream(stream)

    def close_stream(self):
        PET_listmode_close_stream()

    def load_scanner_plugin(self, plugin):
        scanner_library = plugin.get_library_path()
        PET_listmode_load_scanner_plugin(scanner_library)

    def load_stream_plugin(self, plugin):
        stream_library = plugin.get_library_path()
        PET_listmode_load_stream_plugin(stream_library)

    def bin_events(self, n_events, Naxial, Nazim, Nu, Nv, time_bins):
        PET_listmode_bin(n_events, Naxial, Nazim, Nu, Nv, time_bins)

    def bin_events_intrinsic(self, n_events):
        PET_listmode_bin_intrinsic(n_events)

    def get_static_measurement(self):
        R = PET_listmode_get_static()
        return R["counts"], R["offsets"], R["locations"], R["time_start"], R["time_end"]

    def get_dynamic_measurement(self):
        R = PET_listmode_get_dynamic()
        return R["counts"], R["offsets"], R["locations"], R["time_start"], R["time_end"]

class ListmodeLoader:
    def __init__(self):
        self._LM = ListmodeWrapper()

    def set_streamer(self, plugin):
        # call C function that loads dynamically the open(), close() and read() functions from the streamer dynamic library
        self._LM.load_stream_plugin(plugin)

    def set_scanner(self, plugin):
        # call C function that loads dynamically the index_to_lor() function of the scanner plugin
        self._LM.load_scanner(plugin)

    def open_stream(self, stream):
        # call the C function that opens a list-mode stream
        self._LM.open_stream(stream)

    def close_stream(self):
        # call the C function that closes the list-mode stream
        self._LM.close_stream()

    def make_projection_given_binning(self, binning, n_events=-1, time_bins=None):
        # call the C function that parses the list-mode data and bins it into (dynamic) projection data
        self._LM.bin_events(
            n_events,
            binning.N_axial,
            binning.N_azimuthal,
            binning.N_u,
            binning.N_v,
            time_bins,
        )
        counts, offsets, locations, time_start, time_end = (
            self._LM.get_static_measurement()
        )
        time_bins = np.int32(np.linspace(time_start, time_end, 2))
        return PET_Projection(binning, counts, offsets, locations, time_bins)

    def make_projection_intrinsic_geometry(
        self, n_events=-1, time_bins=None, N_azimuthal=-1
    ):
        # call the C function that parses the list-mode data and bins it into (dynamic) projection data
        self._LM.bin_events_intrinsic(
            n_events,
            binning.N_axial,
            binning.N_azimuthal,
            binning.N_u,
            binning.N_v,
            time_bins,
        )
        counts, offsets, locations, time_start, time_end = (
            self._LM.get_static_measurement()
        )
        time_bins = np.int32(np.linspace(time_start, time_end, 2))
        return PET_Projection(binning, counts, offsets, locations, time_bins)


"""
Source.py

Author: Matthew Yu, Array Lead (2020).
Contact: matthewjkyu@gmail.com
Created: 5/24/20
Last Modified: 8/22/20

Description: This file describes the Source class, which generates input values
for the Display and MPPT. The Source does not do any modelling, but it
aggregates the results of the cells.

Functionality: I should be able to do the following:
    - (setup) load from a configuration file or pass in an impulse or regime
        (the latter two are for single cell source modelling).
    - (iterate) able to step forward a cycle for all of the modules in the
        source and return PV voltage and current.
    - (get_source_IV) able to return a full IV curve across all of the modules
        in the source based on the current environmental conditions.
    - (get_source_gmpp) able to get the GMPP characteristics of the PV at the
        current cycle.
    - (get_env_conditions) able to get the current environmental conditions
        for all of the modules in the PV.
    - (get_model_type) able to get the current cell model name (i.e. nonideal,
        ideal).
    - (set_current_cycle, increment_cycle) able to jump forward and rewind
        back in time.

"""
from math import exp, pow, e
from numpy import log as ln
from src.cell import Cell
import json

SINGLE  = 1
DOUBLE  = 2
QUAD    = 4
OCT     = 8

MOD_NUM = 0
MOD_TYPE= 1
CELL    = 2

SOURCE_MODEL_PATH = "./src/source_models/"

class Source:
    MAX_CURRENT = 100

    cycle = 0
    modules = []
    model_type = "Default"

    use_file = False

    def __init__(self, model_type="Default", use_file=True):
        """
        init
        Sets up the model type of the cell.

        Params:
            - model_type (String): type of model to represent the solar cell
                - [Nonideal]    (Single diode nonideal cell), default
                - Ideal         (Single diode ideal cell)
            - use_file (bool): use source_file if true.

        Returns:
            - None
        """
        if model_type == "Ideal":
            self.model_type = "Ideal"
        else:
            self.model_type = "Nonideal"

        if use_file:
            self.use_file = True

    def setup(self, setup_type="", file_name="", regime=[], impulse=()):
        """
        setup Sets up an internal environmental regime either using an array of
        values, a file, or with initial conditions (an impulse).

        Args:
            - type      (String): Either "File", "Array", or "Impulse".
            - file_name (String): name of file in the `source_models` folder.
            - regime    ([[(int) Cycle, (float) Irradiance, (float) Temperature], ...]): Environmental regime.
            - impulse   (((float) Irradiance, (float) Temperature)): Impulse

        Return:
            - Whether the setup was successful (bool)

        Notes:
        - irradiance (float): light intensity on the cell (W/m^2)
        - temperature (float): surface temperature of the cell (C)
        - load (float): load on the system (W/s) NOTE: UNUSED

        # TODO: no support for load at the moment.
        """
        # clear out modules first
        self.modules = []

        if setup_type == "":
            print("[CELL] WARN: Empty setup type -", setup_type)
            return False

        elif setup_type == "Array":
            if not regime:
                print("[CELL] WARN: Array setup type was selected, but regime is an empty list.")
                return False

            # create a single cell
            cell = Cell(self.model_type, self.use_file)
            self.modules.append((len(self.modules), SINGLE, cell))
            return cell.setup(setup_type, regime=regime)

        elif setup_type == "Impulse":
            if not impulse:
                print("[CELL] WARN: Impulse setup type was selected, but impulse is an empty tuple.")
                return False

            # create a single cell
            cell = Cell(self.model_type, self.use_file)
            self.modules.append((len(self.modules), SINGLE, cell))
            return cell.setup(setup_type, impulse=impulse)

        elif setup_type == "File":
            with open(SOURCE_MODEL_PATH + file_name) as f:
                data = json.load(f)

                # get number of modules
                num_modules = int(data['num_modules'])

                # for each module, add a source object
                for module_index in range(0, num_modules):
                    module = data['pv_model'][module_index]

                    # get module type
                    module_type = 1
                    if module['module_type'] == '1x1':
                        module_type = SINGLE
                    elif module['module_type'] == '1x2':
                        module_type = DOUBLE
                    elif module['module_type'] == '2x2' or module['module_type'] == '1x4':
                        module_type = QUAD
                    elif module['module_type'] == '2x4':
                        module_type = OCT
                    else:
                        module_type = SINGLE

                    cell = Cell(self.model_type, self.use_file)
                    self.modules.append((module_index, module_type, cell))
                    if module['env_type'] == "Array":
                        # early exit if one of the modules fail to build.
                        if not cell.setup(module['env_type'], regime=module['env_regime']):
                            return False
                    elif module['env_type'] == "Impulse":
                        # TODO: using eval is bad security practice, possibly replace
                        if not cell.setup(module['env_type'], impulse=eval(module['env_regime'])):
                            return False
                    else:
                        print("[SOURCE] WARN: Invalid module type in file for module", module['module_num'])
                        return False

        else:
            print("[CELL] WARN: Invalid setup type -", setup_type)
            return False

    def iterate(self, v_in):
        """
        iterate Calculates the PV voltage and current the system should expect
        across a set of modules. If done in impulse mode, we take the current
        conditions, but if done in array mode, we take the passed in cycle and
        interpolate from the regime.

        Args:
            - v_in    (float): input voltage
            - cycle   (int): cycle to grab source data from

        Returns:
            - (v_out, i_out, irrad, temp, load) (tuple)

        # NOTE: This model is for Sunpower 1 cell at STD conditions
        """
        v_out_tot = 0.0
        i_out_tot = self.MAX_CURRENT

        # grab the last module's env. conditions; if we're testing a single cell
        # we can also use this.
        irrad = 0.0
        temp = 0.0
        load = 0.0
        for module in self.modules:
            # TODO: issue here to pick up on: I pass in v_ref, which should be the voltage across the ENTIRE PV, not across each cell. I need to be able to get the current across here, but it would be hard to implicitly calculate the voltage across each cell without ...
            (v_out, i_out, irrad, temp, load) = module[CELL].iterate(v_in)

            # simplistically, we just grab the total voltage and the lowest current.
            # multiply v_out by amount of cells in that module
            if module[MOD_TYPE] > 1:
                v_out *= module[MOD_TYPE]

            # TODO: add the effect of bypass diodes (essentially remove x volts for
            # TODO: every module based on y current)
            v_out_tot += v_out
            if i_out_tot > i_out:
                i_out_tot = i_out

        return (v_out_tot, i_out_tot, irrad, temp, load)

    def get_source_IV(self, step_size=.01):
        """
        get_source_IV
        Returns an array of voltage and current values to display current
        solar panel IV pre-altered by the mppt.

        Args:
            - step_size (float): step size. Defaults to .01 V.

        Returns:
            - ([[voltage, current], ...], gmpp)
                - gmpp: global maximum power point characteristics, (vmpp, impp, pmpp)
        """
        # for each module, get their IV curve, and search through the IV curve and grab the total voltage and min current for each cell voltage.
        # build up individual
        module_IVs = []
        for module in self.modules:
            output = module[CELL].get_cell_IV(step_size)[0]
            # multiply by amount of cells in that module
            if module[MOD_TYPE] > 1:
                output = [[entry[0] * module[MOD_TYPE], entry[1]] for entry in output]
            # TODO: deal with bypass diodes here as well
            module_IVs.append(output)

        characteristics = []
        for voltage_entry in range(0, len(module_IVs[0])):
            # get the total voltage and min current for the cell v_ref
            v_out_tot = 0
            i_out_tot = self.MAX_CURRENT
            for iv in module_IVs:
                v_out_tot += iv[voltage_entry][0]
                if i_out_tot > iv[voltage_entry][1]:
                    i_out_tot = iv[voltage_entry][1]

            characteristics.append([v_out_tot, i_out_tot])

        # seek through the characteristics to find the max power and extract that
        v_mpp = 0
        i_mpp = 0
        p_mpp = 0
        for [voltage, current] in characteristics:
            if p_mpp < voltage * current:
                p_mpp = voltage * current
                v_mpp = voltage
                i_mpp = current

        return (characteristics, (v_mpp, i_mpp, p_mpp))

    def get_source_gmpp(self):
        """
        get_source_gmpp
        Returns the global maximum power point parameters to be used by the
        simulation display.

        Returns:
            - (vmpp, impp, pmpp) tuple of global maximum power point characteristics
        """
        (characteristics, (v_mpp, i_mpp, p_mpp)) = self.get_source_IV()
        return (v_mpp, i_mpp, p_mpp)

    def get_num_cells(self):
        """
        get_num_cells
        Returns the total number of cells in the PV. Used for adjusting the
        bounds on the simulator.

        Returns:
            - number of cells (int)
        """
        num_cells = 0
        for module in self.modules:
            num_cells += module[MOD_TYPE]
        return num_cells

    def get_env_conditions(self):
        """
        get_env_conditions
        Returns source conditions for data analysis.

        Args:
            - None

        Returns:
            - [(irradiance, temperature, load), ...] (list of tuples)
        """
        env_conditions = [module[CELL].get_env_conditions() for module in self.modules]
        return env_conditions

    def get_model_type(self):
        """
        get_model_name
        Returns the name of the model.

        Args:
            - None
        Returns:
            - String name
        """
        return self.model_type

    def get_modules(self):
        """
        get_modules
        Returns the list of Modules.

        Args:
            - None
        Returns:
            - List (Cell)
        """
        return self.modules

    def set_current_cycle(self, cycle):
        """
        set_current_cycle
        Sets the current cycle for all modules in the PV.

        Args:
            - cycle (int): current cycle for the PV modules.
        """
        for module in self.modules:
            module[CELL].set_current_cycle(cycle)

    def increment_cycle(self):
        """
        increment_cycle
        Increments the current cycle for all modules in the PV

        Returns:
            current cycle.
        """
        for module in self.modules:
            module[CELL].increment_cycle()

        self.cycle += 1
        return self.cycle


import numpy as np
import glob
from scipy.special import softmax
from sklearn.metrics import pairwise_distances, pairwise, mutual_info_score, log_loss
import json
from model.metric import one_hot
from scipy.stats import pearsonr
from activation_base_class import BaseActvMap
from activation import *
from culprit import *

class Uncertainty(BaseActvMap):
    '''
    End-to-end pipeline:
    Input:
        resume: trained model checkpoint
        config_querydata: model training and metrics details, indicate the csv file for query dataset
        config_valdata: model training and metrics details, indicate the csv file for validation dataset (to generate culprit matrix)
    Output:
        - At application time:
            Given a trained model, a new query dataset (test set),
            for each new datapoint, give an uncertain_score for a given class, showing how severe the culprits active.
            visualize the uncertain_score w.r.t the given class.
            Steps:
            1. generate culprit matrix for the model (from a previous val set, for example)
            2. get activation map for query dataset
            3. A visualization user interface:
                - for each datapoint, generate a class-specific graph w/ user-selected class to inspect

        - At experiment validation:

    '''

    def __init__(self, experiment_saved_path, flatten_mode, model_path = 'skinmodel/checkpoint.pth', config_querydata = 'config_skin_alexnet_query.json', config_valdata = 'config_skin_alexnet_val.json'):
        '''
        load model activation for query and val dataset
        '''
        # get query dataset actv, gt, pred
        saved_query_path = experiment_saved_path + '/query_actv/'
        if len(glob.glob(saved_query_path+'*.pkl')) != 4:
            # if there is no saved query actv
            config_query = json.load(open(config_querydata))
            query_actv = ExtractActivation(config_query, model_path)
            query_actv.extract()
            query_actv.save_data(saved_query_path)
            self.query_gt = query_actv.get_gt().numpy()
            self.query_pred = query_actv.get_predict().numpy()
            self.query_actv_map = query_actv.get_activation()
            self.query_shape = query_actv.get_map_shape()
        else:
            self.query_actv_map, self.query_gt, self.query_pred, self.query_shape = super().load_pkl(saved_query_path)

        # prepare the query data for further processing
#         self.query_gt = (self.query_gt.cpu()).numpy()
#         self.query_pred = self.query_pred.numpy() # conver torch tensor to numpy
        self.query_actv, self.turnout = super().flatten_actv_map(self.query_actv_map, flatten_mode) # flatten the activation map NOTE: that flatten returns 2 objects
#         print(type(self.query_actv))
        assert np.isnan(self.query_actv).sum() == 0, '[uncertainty.init] !!! query flatten activation contains Nan !!!'
        # get genralization error as groung-truth for experiment
#         self.error = self.get_generalize_error(self.query_gt, self.query_pred)

        # get culprit matrix
        saved_val_path = experiment_saved_path + '/val_actv/'
        if len(glob.glob(saved_val_path+'*.pkl')) != 4:
            # if saved 4 pkl file do not exist, generate one
            config_val = json.load(open(config_valdata))
            val_actv = ExtractActivation(config_val, model_path)
            val_actv.extract()
            val_actv.save_data(saved_val_path)
        # instantiate culprit instance
        self.clpt = CulpritNeuronScore(saved_val_path, flatten_mode)
        # culprit methods dictionary
#         self.culprit_methods = \
#         {'freq': self.clpt.culprit_freq,
#          'select': self.clpt.culprit_selectivity,
#          'stat': self.clpt.culprit_stat}

#     def load_pkl(self, path):
#         '''
#         load pkl files in the folder with the filename: gt, pred, activationMap, map_shape
#         '''
#         with open(path + 'activationMap.pkl', 'rb') as f:
#             actv_map = pickle.load(f)
#         with open(path + 'gt.pkl', 'rb') as f:
#             gt = pickle.load(f)
#         with open(path + 'pred.pkl', 'rb') as f:
#             pred_prob = pickle.load(f)
#         with open(path + 'map_shape.pkl', 'rb') as f:
#             map_shape = pickle.load(f)
#         # sanity check for data shape
#         assert gt.shape[0] == pred_prob.shape[0], 'pred and gt do not have the same datapoints, pred {}, gt {}'.format(pred_prob.shape, gt.shape)
#         for i in range(len(map_shape)):
#             assert actv_map[i].size()[1:] == map_shape[i][1:], 'activation map {} and map shape are not at the same length, activateion map {}, map_shape {}.'.format(i, actv_map[i].size(), map_shape[i])
#         print('*** actv shape (ignore dim 0 - batch size) is: {} .'.format(map_shape))
#         print('*** data loaded ***')
#         return actv_map, gt, pred_prob, map_shape

#     def flatten_actv_map(self, actv_map, mode = 'mean'):
#         '''
#         Input:
#             - actv_map, a dict of {layer idx: activation map for that layer of shape (datapoints, activations) - FC layer, or (datapoints, 3D activation maps) - conv}
#         Output:
#             - actv_mtx, of shape (datapoints, neurons)
#         Method:
#             1. flatten the 2D HxW activation map of one channel/unit/neuron to be a 1D scalar.
#                 mode: average, max, median
#             2. aggregate the neurons/channels at each layer to be single activation vector.

#         '''
#         # flatten activation map
#         mode_dict = {'mean': torch.mean, 'max': torch.max, 'median':torch.median}
#         activation = []
#         for i in range(len(actv_map)):
#             if len(actv_map[i].size()) > 2:
#                 actv_map_flattened =  actv_map[i].reshape(actv_map[i].shape[0], actv_map[i].shape[1], -1)
#                 if mode == 'max':
#                     convert_map_to_scalar, _ = mode_dict[mode](actv_map_flattened, dim = 2)
#                 else:
#                     convert_map_to_scalar = mode_dict[mode](actv_map_flattened, dim = 2)
#                 activation.append(convert_map_to_scalar)
#             else:
#                 activation.append(actv_map[i])
#         actv_mtx = torch.cat(activation, dim=1)
#         print('*** Flattened actv vector shape is {}.'.format(actv_mtx.shape))
#         return actv_mtx

    def get_actv_shape(self):
        return self.query_shape
    def get_query_actv(self):
        return self.query_actv

    def get_culprit_matrix(self, method):
        '''
        generate culprit matrix from the saved actv, gt, pred
        '''
        return self.clpt.get_culprit_matrix(method)

    def get_query_gt_pred(self):
        return self.query_gt, self.query_pred

    def get_generalize_error(self, gt, pred):
        '''
        for experiment results. get the difference between output prob as the generalization error.
        self.query_pred is a 2D array of (# of data, # of class)
        compute softmax over dim 1
        '''
        nb_class = np.max(gt) +1
        one_hot_gt = one_hot(gt, nb_class)
        prob = softmax(pred, axis = 1)
        assert one_hot_gt.shape == prob.shape, '!!! one_hot_gt and prob are not the same shape !!!'
        return np.absolute(one_hot_gt - prob)

    def get_generalize_error_ce(self, gt, pred):
        '''
        Input:
            - gt: 1d array indicating the gt class label
            - pred: 2d array of (# of data, # of class) raw output without softmax function
        Method:
        compute the distance between pred and gt using cross-entropy loss
        for multi-class,
        Adapt from: https://github.com/scikit-learn/scikit-learn/blob/7b136e9/sklearn/metrics/classification.py#L1699
        since original is the sum of all class, here we want indiviudual class error
        '''
        y_pred = softmax(pred, axis = 1)
        y_pred = check_array(y_pred, ensure_2d=False)
        check_consistent_length(y_pred, y_true, sample_weight)

        lb = LabelBinarizer()

        if labels is not None:
            lb.fit(labels)
        else:
            lb.fit(y_true)

        if len(lb.classes_) == 1:
            if labels is None:
                raise ValueError('y_true contains only one label ({0}). Please '
                                 'provide the true labels explicitly through the '
                                 'labels argument.'.format(lb.classes_[0]))
            else:
                raise ValueError('The labels array needs to contain at least two '
                                 'labels for log_loss, '
                                 'got {0}.'.format(lb.classes_))

        transformed_labels = lb.transform(y_true)

        if transformed_labels.shape[1] == 1:
            transformed_labels = np.append(1 - transformed_labels,
                                           transformed_labels, axis=1)

        # Clipping
        y_pred = np.clip(y_pred, eps, 1 - eps)

        # If y_pred is of single dimension, assume y_true to be binary
        # and then check.
        if y_pred.ndim == 1:
            y_pred = y_pred[:, np.newaxis]
        if y_pred.shape[1] == 1:
            y_pred = np.append(1 - y_pred, y_pred, axis=1)

        # Check if dimensions are consistent.
        transformed_labels = check_array(transformed_labels)
        if len(lb.classes_) != y_pred.shape[1]:
            if labels is None:
                raise ValueError("y_true and y_pred contain different number of "
                                 "classes {0}, {1}. Please provide the true "
                                 "labels explicitly through the labels argument. "
                                 "Classes found in "
                                 "y_true: {2}".format(transformed_labels.shape[1],
                                                      y_pred.shape[1],
                                                      lb.classes_))
            else:
                raise ValueError('The number of classes in labels is different '
                                 'from that in y_pred. Classes found in '
                                 'labels: {0}'.format(lb.classes_))

        # Renormalize
        y_pred /= y_pred.sum(axis=1)[:, np.newaxis]
        loss = -(transformed_labels * np.log(y_pred)).sum(axis=1)

        return loss

#     def sim_cosine(self, vec1, vec2):
#         '''
#         cosine similairty for two given vector with the same length.
#         '''
#         sim = np.dot(vec1, vec2)
#         return sim

#     def sim_pearson(self, vec1, vec2):
#         '''
#         pearson r correlation for two given vector with the same length.
#         '''
#         # todo
#         return

    def sim_mutual_info(self, vec1, vec2):
        '''
        mutual information similarity of two vectors of the same lenght, but on different scale
        '''
        return mutual_info_score(vec1, vec2)

#     def get_uncertain_score(self, culprit_vec, query_actv_vec, method = 'pearson'):
#         '''
#         Input:
#             - culprit_vec: culprit vector for one class of the model
#             - query_actv_vec: the flatten activation of the query data passing over the model
#             - method: a str indicate which similarity method to use
#         Output:
#             - uncertain_score: a score summarizing how similar the two vectors are.
#         '''
#         sim_method = {'cos': self.sim_cosine, 'pearson': self.sim_pearson, \
#                 'mi': self.sim_mutual_info}
#         uncertain_score = sim_method[method](culprit_vec, query_actv_vec)
#         return uncertain_score

    def get_uncertain_matrix(self, culprit_mtx, actv_mtx, sim_method):
        '''
        Input:
            - culprit_matrix (from a trained model and val set), shape (# of class, # of neurons)
            - activation matrix (from query datasets) , shape (# of data, # of neurons)
        Output:
            - a uncertain_matrix of shape (# of datapoints, # of class)
            (each row: the uncertain_vector for each class, given one data. rows are a stack of multiple datapoints)
        Method:
            - given the activation vector (from query image) and culprit matrix (from trained model),
        compute a similarity score between the activation vector, and each row of the culprit matrix.
        aggregate the uncertainty score for each class, to be a uncertainty vector for the data point.
        the uncertain_matrix is simply the stack of uncertain vector for multiple datapoints.
        '''

        sim_methods = {'mi': self.sim_mutual_info}
        if sim_method in sim_methods:
            sim_mthd = sim_methods[sim_method]
        else:
            sim_mthd = sim_method
        uncertain_matrix = pairwise_distances(actv_mtx, culprit_mtx, metric = sim_mthd)
#        for query_actv_vec in actv_mtx:
#            # process datapoints row-wise in the query data actv_mtx
#            uncertain_vector = []
#            for culprit_vec in culprit_mtx:
#                uncertain_score = self.get_uncertain_score(culprit_vec, query_actv_vec, method)
#                uncertain_vector.append(uncertain_score)
#            uncertain_matrix.append(uncertain_vector)
#        uncertain_matrix = np.array(uncertain_matrix)
        return uncertain_matrix

    def compare_gt_error(self, uncertain_matrix, gt):
        '''
        Input:
            - self.error: a 2D array of (#of data, # of class)
            - uncertain_matrix: 2D array of shape (# of data, # of class)
        compare the uncertain matrix results with self.error
        do it row wise for two matrix.
        Output:
            - a vector, with each scalar showing the similarity between the correlation between each class's prediction error and the uncertain_vec.
                the vector is the aggregete of the datapoints in the query dataset.
        '''
        assert uncertain_matrix.shape == self.error.shape, '!!! uncertain_matrix and self.error are not in the same shape !!!'
        corr = pairwise.paired_distances(uncertain_matrix, gt, metric = pearsonr)
        return corr

    def get_baseline(self, culprit_mtx):
        '''
        generate random culprit score with the same distribution of the input culprit_mtx
        '''
        mean, sigma = culprit_mtx.mean(), culprit_mtx.std()
        rand_mtx = np.random.normal(mean, sigma, culprit_mtx.shape)
        return rand_mtx

#     def layer_specific_uncertainty(self):
#         '''
#         pick up some layer which has the strong indication of uncertainty
#         '''
#         # todo self.map_shape
#         return

#     def run_experiment(self, method = 'select', sim_method = 'cosine'):
#         '''
#         Experiment running pipeline for one single culprit method
#         '''
#         clpt_mtx = self.get_culprit_matrix(method)
#         uncty_mtx = self.get_uncertain_matrix(clpt_mtx, self.query_actv, sim_method)


#  Copyright (c) 2020 the original author or authors
#
#  Licensed under the Apache License, Version 2.0 (the "License");
#  you may not use this file except in compliance with the License.
#  You may obtain a copy of the License at
#
#       https://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
#  or implied. See the License for the specific language governing
#  permissions and limitations under the License.

import logging
import functools
from typing import List, Tuple
import attr
import numpy as np
import mxnet as mx
from cached_property import cached_property
from insightface.app import FaceAnalysis
from insightface.model_zoo import (model_store, face_detection,
                                   face_recognition, face_genderage)
from insightface.utils import face_align

from src.constants import ENV
from src.services.dto.bounding_box import BoundingBoxDTO
from src.services.dto.json_encodable import JSONEncodable
from src.services.facescan.imgscaler.imgscaler import ImgScaler
from src.services.facescan.plugins import base, mixins, exceptions
from src.services.facescan.plugins.insightface import helpers as insight_helpers
from src.services.dto import plugin_result
from src.services.imgtools.types import Array3D

logger = logging.getLogger(__name__)

class InsightFaceMixin:
    _CTX_ID = ENV.GPU_IDX
    _NMS = 0.4

    def get_model_file(self, ml_model: base.MLModel):
        if not ml_model.exists():
            raise exceptions.ModelImportException(
                f'Model {ml_model.name} does not exists')
        return model_store.find_params_file(ml_model.path)

class DetectionOnlyFaceAnalysis(FaceAnalysis):
    rec_model = None
    ga_model = None

    def __init__(self, file):
        self.det_model = face_detection.FaceDetector(file, 'net3')

class FaceDetector(InsightFaceMixin, mixins.FaceDetectorMixin, base.BasePlugin):
    ml_models = (
        ('retinaface_mnet025_v1', '1ggNFFqpe0abWz6V1A82rnxD6fyxB8W2c'),
        ('retinaface_mnet025_v2', '1EYTMxgcNdlvoL1fSC8N1zkaWrX75ZoNL'),
        ('retinaface_r50_v1', '1LZ5h9f_YC5EdbIZAqVba9TKHipi90JBj'),
    )

    IMG_LENGTH_LIMIT = ENV.IMG_LENGTH_LIMIT
    IMAGE_SIZE = 112
    det_prob_threshold = 0.8

    @cached_property
    def _detection_model(self):
        model_file = self.get_model_file(self.ml_model)
        model = DetectionOnlyFaceAnalysis(model_file)
        model.prepare(ctx_id=self._CTX_ID, nms=self._NMS)
        return model

    def find_faces(self, img: Array3D, det_prob_threshold: float = None) -> List[BoundingBoxDTO]:
        if det_prob_threshold is None:
            det_prob_threshold = self.det_prob_threshold
        assert 0 <= det_prob_threshold <= 1
        scaler = ImgScaler(self.IMG_LENGTH_LIMIT)
        img = scaler.downscale_img(img)
        results = self._detection_model.get(img, det_thresh=det_prob_threshold)
        boxes = []
        for result in results:
            downscaled_box_array = result.bbox.astype(np.int).flatten()
            downscaled_box = BoundingBoxDTO(x_min=downscaled_box_array[0],
                                            y_min=downscaled_box_array[1],
                                            x_max=downscaled_box_array[2],
                                            y_max=downscaled_box_array[3],
                                            probability=result.det_score,
                                            np_landmarks=result.landmark)
            box = downscaled_box.scaled(scaler.upscale_coefficient)
            if box.probability <= det_prob_threshold:
                logger.debug(f'Box Filtered out because below threshold ({det_prob_threshold}: {box})')
                continue
            logger.debug(f"Found: {box}")
            boxes.append(box)
        return boxes

    def crop_face(self, img: Array3D, box: BoundingBoxDTO) -> Array3D:
        return face_align.norm_crop(img, landmark=box._np_landmarks,
                                    image_size=self.IMAGE_SIZE)

class Calculator(InsightFaceMixin, mixins.CalculatorMixin, base.BasePlugin):
    ml_models = (
        ('arcface_mobilefacenet', '17TpxpyHuUc1ZTm3RIbfvhnBcZqhyKszV', (1.26538905, 5.552089201), 200),
        ('arcface_r100_v1', '11xFaEHIQLNze3-2RUV1cQfT-q6PKKfYp', (1.23132175, 6.602259425), 400),
        ('arcface_resnet34', '1ECp5XrLgfEAnwyTYFEhJgIsOAw6KaHa7', (1.2462842, 5.981636853), 400),
        ('arcface_resnet50', '1a9nib4I9OIVORwsqLB0gz0WuLC32E8gf', (1.2375747, 5.973354538), 400),
        ('arcface-r50-msfdrop75', '1gNuvRNHCNgvFtz7SjhW82v2-znlAYaRO', (1.2350148, 7.071431642), 400),
        ('arcface-r100-msfdrop75', '1lAnFcBXoMKqE-SkZKTmi6MsYAmzG0tFw', (1.224676, 6.322647217), 400),
    )

    def calc_embedding(self, face_img: Array3D) -> Array3D:
        return self._calculation_model.get_embedding(face_img).flatten()

    @cached_property
    def _calculation_model(self):
        model_file = self.get_model_file(self.ml_model)
        model = face_recognition.FaceRecognition(
            self.ml_model.name, True, model_file)
        model.prepare(ctx_id=self._CTX_ID)
        return model

@attr.s(auto_attribs=True, frozen=True)
class GenderAgeDTO(JSONEncodable):
    gender: str
    age: Tuple[int, int]

class BaseGenderAge(InsightFaceMixin, base.BasePlugin):
    ml_models = (
        ('genderage_v1', '1J9hqSWqZz6YvMMNrDrmrzEW9anhvdKuC'),
    )
    CACHE_FIELD = '_genderage_cached_result'

    def _evaluate_model(self, face: plugin_result.FaceDTO):
        cached_result = getattr(face, self.CACHE_FIELD, None)
        if not cached_result:
            cached_result = self._genderage_model.get(face._face_img)
            setattr(face, self.CACHE_FIELD, cached_result)
        return cached_result

    @cached_property
    def _genderage_model(self):
        model_file = self.get_model_file(self.ml_model)
        model = face_genderage.FaceGenderage(
            self.ml_model.name, True, model_file)
        model.prepare(ctx_id=self._CTX_ID)
        return model

class GenderDetector(BaseGenderAge):
    slug = "gender"
    GENDERS = ('female', 'male')

    def __call__(self, face: plugin_result.FaceDTO):
        gender, age = self._evaluate_model(face)
        return plugin_result.GenderDTO(gender=self.GENDERS[int(gender)])

class AgeDetector(BaseGenderAge):
    slug = "age"

    def __call__(self, face: plugin_result.FaceDTO):
        gender, age = self._evaluate_model(face)
        return plugin_result.AgeDTO(age=(age, age))

class LandmarksDetector(mixins.LandmarksDetectorMixin, base.BasePlugin):
    """ Extract landmarks from FaceDetector results."""

class Landmarks2d106DTO(plugin_result.LandmarksDTO):
    """
    106-points facial landmarks

    Points mark-up - https://github.com/deepinsight/insightface/tree/master/alignment/coordinateReg#visualization
    """
    NOSE_POSITION = 86

class Landmarks2d106Detector(InsightFaceMixin, mixins.LandmarksDetectorMixin,
                              base.BasePlugin):
    slug = 'landmarks2d106'
    ml_models = (
        ('2d106det', '1MBWbTEYRhZFzj_O2f2Dc6fWGXFWtbMFw'),
    )
    CROP_SIZE = (192, 192) # model requirements

    def __call__(self, face: plugin_result.FaceDTO):
        landmarks = insight_helpers.predict_landmark2d106(
            self._landmark_model, face._img, self.CROP_SIZE,
            face.box.center, (face.box.width, face.box.height),
        )
        return Landmarks2d106DTO(landmarks=landmarks.astype(int).tolist())

    @cached_property
    def _landmark_model(self):
        model_prefix = f'{self.ml_model.path}/{self.ml_model.name}'
        sym, arg_params, aux_params = mx.model.load_checkpoint(model_prefix, 0)
        ctx = mx.gpu(self._CTX_ID) if self._CTX_ID >= 0 else mx.cpu()
        all_layers = sym.get_internals()
        sym = all_layers['fc1_output']
        model = mx.mod.Module(symbol=sym, context=ctx, label_names=None)
        model.bind(for_training=False,
                   data_shapes=[('data', (1, 3, *self.CROP_SIZE))])
        model.set_params(arg_params, aux_params)
        return model


#####DONORSCHOOSE FUNCTIONS
import pandas as pd
import numpy as np

import datetime
from datetime import timedelta, date #for time duration calculations
from dateutil.parser import parse #for fuzzy finding year

def fixer(scaler,grd_boost4_model,
    dateDF2,total_price_excluding_optional_support,
    primary_focus_category,state_you_live,students_reached,
    resource_type,grade_level,school_metro,school_poverty):

    #DEFAULTS
    primary_focus_subject_AppliedSciences = 0
    primary_focus_subject_CharacterEducation = 0
    primary_focus_subject_Civics_Government = 0
    primary_focus_subject_College_CareerPrep = 0
    primary_focus_subject_CommunityService = 0
    primary_focus_subject_ESL = 0
    primary_focus_subject_EarlyDevelopment = 0
    primary_focus_subject_Economics = 0
    primary_focus_subject_EnvironmentalScience = 0
    primary_focus_subject_Extracurricular = 0
    primary_focus_subject_FinancialLiteracy = 0
    primary_focus_subject_ForeignLanguages = 0
    primary_focus_subject_Gym_Fitness = 0
    primary_focus_subject_Health_LifeScience = 0
    primary_focus_subject_Health_Wellness = 0
    primary_focus_subject_History_Geography = 0
    primary_focus_subject_Literacy = 0
    primary_focus_subject_Literature_Writing = 0
    primary_focus_subject_Mathematics = 0
    primary_focus_subject_Music = 0
    primary_focus_subject_Nutrition = 0
    primary_focus_subject_Other = 0
    primary_focus_subject_ParentInvolvement = 0
    primary_focus_subject_PerformingArts = 0
    primary_focus_subject_SocialSciences = 0
    primary_focus_subject_SpecialNeeds = 0
    primary_focus_subject_Sports = 0
    primary_focus_subject_TeamSports = 0
    primary_focus_subject_VisualArts = 0

    resource_type_Books = 0
    resource_type_Other = 0
    resource_type_Supplies = 0
    resource_type_Technology = 0
    resource_type_Trips = 0
    resource_type_Visitors = 0

    grade_level_Grades3_5 = 0
    grade_level_Grades6_8 = 0
    grade_level_Grades9_12 = 0
    grade_level_GradesPreK_2 = 0

    school_metro_none = 0
    school_metro_rural = 0
    school_metro_suburban = 0
    school_metro_urban = 0

    poverty_level_high = 0
    poverty_level_highest = 0
    poverty_level_low = 0
    poverty_level_minimal = 0
    poverty_level_moderate = 0
    poverty_level_unknown = 0

    #######################
    #Then use user input decides which one is 'active'
    if primary_focus_category == "AppliedSciences":
        primary_focus_subject_AppliedSciences = 1
    if primary_focus_category == "CharacterEducation":
        primary_focus_subject_CharacterEducation = 1
    if primary_focus_category == "Civics_Government":
        primary_focus_subject_Civics_Government = 1
    if primary_focus_category == "College_CareerPrep":
        primary_focus_subject_College_CareerPrep = 1
    if primary_focus_category == "CommunityService":
        primary_focus_subject_CommunityService = 1
    if primary_focus_category == "ESL":
        primary_focus_subject_ESL = 1
    if primary_focus_category == "EarlyDevelopment":
        primary_focus_subject_EarlyDevelopment = 1
    if primary_focus_category == "Economics":
        primary_focus_subject_Economics = 1
    if primary_focus_category == "EnvironmentalScience":
        primary_focus_subject_EnvironmentalScience = 1
    if primary_focus_category == "Extracurricular":
        primary_focus_subject_Extracurricular = 1
    if primary_focus_category == "FinancialLiteracy":
        primary_focus_subject_FinancialLiteracy = 1
    if primary_focus_category == "ForeignLanguages":
        primary_focus_subject_ForeignLanguages = 1
    if primary_focus_category == "Gym_Fitness":
        primary_focus_subject_Gym_Fitness = 1
    if primary_focus_category == "Health_LifeScience":
        primary_focus_subject_Health_LifeScience = 1
    if primary_focus_category == "Health_Wellness":
        primary_focus_subject_Health_Wellness = 1
    if primary_focus_category == "History_Geography":
        primary_focus_subject_History_Geography = 1
    if primary_focus_category == "Literacy":
        primary_focus_subject_Literacy = 1
    if primary_focus_category == "Literature_Writing":
        primary_focus_subject_Literature_Writing = 1
    if primary_focus_category == "Mathematics":
        primary_focus_subject_Mathematics = 1
    if primary_focus_category == "Music":
        primary_focus_subject_Music = 1
    if primary_focus_category == "Nutrition":
        primary_focus_subject_Nutrition = 1
    if primary_focus_category == "Other":
        primary_focus_subject_Other = 1
    if primary_focus_category == "ParentInvolvement":
        primary_focus_subject_ParentInvolvement = 1
    if primary_focus_category == "PerformingArts":
        primary_focus_subject_PerformingArts = 1
    if primary_focus_category == "SocialSciences":
        primary_focus_subject_SocialSciences = 1
    if primary_focus_category == "SpecialNeeds":
        primary_focus_subject_SpecialNeeds = 1
    if primary_focus_category == "Sports":
        primary_focus_subject_Sports = 1
    if primary_focus_category == "TeamSports":
        primary_focus_subject_TeamSports = 1
    if primary_focus_category == "VisualArts":
        primary_focus_subject_VisualArts = 1

    #######################
    #What resources being requested
    if resource_type == "Books":
        resource_type_Books = 1
    if resource_type == "Other":
        resource_type_Other = 1
    if resource_type == "Supplies":
        resource_type_Supplies = 1
    if resource_type == "Technology":
        resource_type_Technology = 1
    if resource_type == "Trips":
        resource_type_Trips = 1
    if resource_type == "Visitors":
        resource_type_Visitors = 1
    #######################
    #What grade level?
    if grade_level == 'pk2':
        grade_level_GradesPreK_2 = 1
    if grade_level == 'g35':
        grade_level_Grades3_5 = 1
    if grade_level == 'g68':
        grade_level_Grades6_8 = 1
    if grade_level == 'g912':
        grade_level_Grades9_12 = 1
    #######################
    #Is the school in a metro area?
    if school_metro == 'metro':
        school_metro_urban = 1
    if school_metro == 'suburban':
        school_metro_suburban = 1
    if school_metro == 'rural':
        school_metro_rural = 1
    if school_metro == 'other':
        school_metro_none = 1
    #######################
    #What is the school's poverty level?
    if school_poverty == 'highest':
        poverty_level_highest = 1
    if school_poverty == 'high':
        poverty_level_high = 1
    if school_poverty == 'moderate':
        poverty_level_moderate = 1
    if school_poverty == 'low':
        poverty_level_low = 1
    if school_poverty == 'minimal':
        poverty_level_minimal = 1
    if school_poverty == 'unknown':
        poverty_level_unknown = 1
    #######
    circlx=dateDF2.iloc[0,0]
    circly=dateDF2.iloc[0,1]
    ####################################
    scst=['school_state_AK', 'school_state_AL','school_state_AR',
                 'school_state_AZ',
                 'school_state_CA',
                 'school_state_CO',
                 'school_state_CT',
                 'school_state_DC',
                 'school_state_DE',
                 'school_state_FL',
                 'school_state_GA',
                 'school_state_HI',
                 'school_state_IA',
                 'school_state_ID',
                 'school_state_IL',
                 'school_state_IN',
                 'school_state_KS',
                 'school_state_KY',
                 'school_state_LA',
                 'school_state_MA',
                 'school_state_MD',
                 'school_state_ME',
                 'school_state_MI',
                 'school_state_MN',
                 'school_state_MO',
                 'school_state_MS',
                 'school_state_MT',
                 'school_state_NC',
                 'school_state_ND',
                 'school_state_NE',
                 'school_state_NH',
                 'school_state_NJ',
                 'school_state_NM',
                 'school_state_NV',
                 'school_state_NY',
                 'school_state_OH',
                 'school_state_OK',
                 'school_state_OR',
                 'school_state_PA',
                 'school_state_RI',
                 'school_state_SC',
                 'school_state_SD',
                 'school_state_TN',
                 'school_state_TX',
                 'school_state_UT',
                 'school_state_VA',
                 'school_state_VT',
                 'school_state_WA',
                 'school_state_WI',
                 'school_state_WV',
                 'school_state_WY']

    school_state = dict()
    for s in scst:
        if state_you_live in s:
            school_state[s] = 1
        else:
            school_state[s] = 0
    ####################################
    startingvals = [students_reached,total_price_excluding_optional_support,
                    circlx,circly]
    for key, value in school_state.items():
        startingvals.append(value)

    insertlist=startingvals+[school_metro_none,
    school_metro_rural,
    school_metro_suburban,
    school_metro_urban,
    primary_focus_subject_AppliedSciences,
    primary_focus_subject_CharacterEducation,
    primary_focus_subject_Civics_Government,
    primary_focus_subject_College_CareerPrep,
    primary_focus_subject_CommunityService,
    primary_focus_subject_ESL,
    primary_focus_subject_EarlyDevelopment,
    primary_focus_subject_Economics,
    primary_focus_subject_EnvironmentalScience,
    primary_focus_subject_Extracurricular,
    primary_focus_subject_FinancialLiteracy,
    primary_focus_subject_ForeignLanguages,
    primary_focus_subject_Gym_Fitness,
    primary_focus_subject_Health_LifeScience,
    primary_focus_subject_Health_Wellness,
    primary_focus_subject_History_Geography,
    primary_focus_subject_Literacy,
    primary_focus_subject_Literature_Writing,
    primary_focus_subject_Mathematics,
    primary_focus_subject_Music,
    primary_focus_subject_Nutrition,
    primary_focus_subject_Other,
    primary_focus_subject_ParentInvolvement,
    primary_focus_subject_PerformingArts,
    primary_focus_subject_SocialSciences,
    primary_focus_subject_SpecialNeeds,
    primary_focus_subject_Sports,
    primary_focus_subject_TeamSports,
    primary_focus_subject_VisualArts,
    resource_type_Books,
    resource_type_Other,
    resource_type_Supplies,
    resource_type_Technology,
    resource_type_Trips,
    resource_type_Visitors,
    poverty_level_high,
    poverty_level_highest,
    poverty_level_low,
    poverty_level_minimal,
    poverty_level_moderate,
    poverty_level_unknown,
    grade_level_Grades3_5,
    grade_level_Grades6_8,
    grade_level_Grades9_12,
    grade_level_GradesPreK_2]
    #################################
    valuearray=np.array([insertlist])

    valuearray=scaler.transform(valuearray)

    FundedFast = grd_boost4_model.predict_proba((valuearray.reshape(1, -1)))

    fundingstring = str(round(FundedFast[0][1],3))
    funding100 = round(100*round(FundedFast[0][1],3),2) #times funded out of 100

    ###############################
    ###############################
    ###############################
    ###############################

    predictlist=[]
    listcopy=insertlist

    for index, row in dateDF2.iterrows():
        listcopy[2]=row['circlx']
        listcopy[3]=row['circly']
        valuearray=np.array([listcopy])
        valuearray=scaler.transform(valuearray)
        FundedFastFuture = grd_boost4_model.predict_proba((valuearray.reshape(1, -1)))
        predictlist.append(round(FundedFastFuture[0][1],4))

    dateDF2['probsuccess']=predictlist

    predictlistdrop15=[]
    for index, row in dateDF2.iterrows():
        listcopy[1]=total_price_excluding_optional_support*.75
        listcopy[2]=row['circlx']
        listcopy[3]=row['circly']
        valuearray=np.array([listcopy])
        valuearray=scaler.transform(valuearray)
        FundedFastFuture = grd_boost4_model.predict_proba((valuearray.reshape(1, -1)))
        predictlistdrop15.append(round(FundedFastFuture[0][1],4))

    dateDF2['drop25prob']=predictlistdrop15

    predictlistplus15=[]
    for index, row in dateDF2.iterrows():
        listcopy[1]=total_price_excluding_optional_support*1.25
        listcopy[2]=row['circlx']
        listcopy[3]=row['circly']
        valuearray=np.array([listcopy])
        valuearray=scaler.transform(valuearray)
        FundedFastFuture = grd_boost4_model.predict_proba((valuearray.reshape(1, -1)))
        predictlistplus15.append(round(FundedFastFuture[0][1],4))

    dateDF2['plus25prob']=predictlistplus15

    resources=['Books','Other','Supplies','Technology','Trips','Visitors']
    defaultresource = resources.index(resource_type)
    startresource = resources.index(resource_type) + 88 #find default resource type
    listcopy[startresource]=0 #and set it to zero
    rind=0

    toptwo=[]

    for rt in range(88,94): #loop through different resources, skip if de
        predictlistnewresource1=[]
        nameofresource=resources[rind]

        if rind!=defaultresource:
            for index, row in dateDF2.iterrows():
                listcopy[1]=total_price_excluding_optional_support
                listcopy[2]=row['circlx']
                listcopy[3]=row['circly']
                listcopy[rt]=1

                valuearray=np.array([listcopy])
                valuearray=scaler.transform(valuearray)
                FundedFastFuture = grd_boost4_model.predict_proba((valuearray.reshape(1, -1)))
                predictlistnewresource1.append(round(FundedFastFuture[0][1],4))
                listcopy[rt]=0#reset resource

            dateDF2[nameofresource]=predictlistnewresource1

        rind+=1

    return(fundingstring,funding100,dateDF2)

def elapseddays(posted, completed):
    formatuse = '%Y-%m-%d %H:%M:%S' # The format: see down this page:https://docs.python.org/3/library/datetime.html
    otherformat = '%Y-%m-%d'

    try:
        elapsed_days=completed-posted
    except:
        try:
            elapsed_days = datetime.datetime.strptime(completed,formatuse)-datetime.datetime.strptime(posted,formatuse)
        except:
            try:
                elapsed_days = datetime.datetime.strptime(completed,otherformat)-datetime.datetime.strptime(posted,otherformat)
            except:
                elapsed_days = 'error'

    return(elapsed_days)

def elapsedseconds(posted, completed):

    formatuse = '%Y-%m-%d %H:%M:%S' # The format: see down this page:https://docs.python.org/3/library/datetime.html
    otherformat = '%Y-%m-%d'

    if isinstance(posted, datetime.datetime) or (type(posted) is pd.Timestamp):
        clock = completed
    else:
        try:
            clock = datetime.datetime.strptime(completed,formatuse)
        except:
            clock = datetime.datetime.strptime(completed,otherformat)

    if isinstance(completed, datetime.datetime) or (type(completed) is pd.Timestamp):
        startclock = completed
    else:
        try:
            startclock = datetime.datetime.strptime(posted,formatuse)
        except:
            startclock = datetime.datetime.strptime(posted,otherformat)

    elapsed = (clock-startclock).total_seconds()

    return(elapsed)

intervals = (
    ('weeks', 604800),  # 60 * 60 * 24 * 7
    ('days', 86400),    # 60 * 60 * 24
    ('hours', 3600),    # 60 * 60
    ('minutes', 60),
    ('seconds', 1),
    )

def display_time(seconds, granularity=2):
    result = []

    for name, count in intervals:
        value = seconds // count
        if value:
            seconds -= value * count
            if value == 1:
                name = name.rstrip('s')
            result.append("{} {}".format(value, name))
    return ', '.join(result[:granularity])

# Function convert seconds into day.decimal
def ConvertSectoDay(n):
    day = n // (24 * 3600)
    #print(day) #keep day
    n = n % (24 * 3600)
    daydec=(n/86400) # add this to day
    addem=day+daydec
    #https://stackoverflow.com/a/48812729/1602288
    holder='{:g}'.format(float('{:.{p}g}'.format(addem, p=5)))
    return(float(holder))

def projectover(posted, completed,expiration):
    formatuse = '%Y-%m-%d %H:%M:%S' # The format: see down this page:https://docs.python.org/3/library/datetime.html
    otherformat = '%Y-%m-%d'

    #failed projects were never completed, so in those cases, use the expiration date
    # if variable is None:
    if completed is None:
        try:
            clock = datetime.datetime.strptime(expiration,formatuse)
        except:
            try:
                clock = datetime.datetime.strptime(expiration,otherformat)
            except:
                clock = datetime.datetime.strptime('1900-01-01',otherformat)
    else:
        try:
            clock = datetime.datetime.strptime(completed,formatuse)
        except:
            try:
                clock = datetime.datetime.strptime(completed,otherformat)
            except:
                clock = datetime.datetime.strptime('1900-01-01',otherformat)

    return(clock)

def makedate(posted):
    formatuse = '%Y-%m-%d %H:%M:%S' # The format: see down this page:https://docs.python.org/3/library/datetime.html
    otherformat = '%Y-%m-%d'

    try:
        clock = datetime.datetime.strptime(posted,formatuse)
    except:
        try:
            clock = datetime.datetime.strptime(posted,otherformat)
        except:
            clock = datetime.datetime.strptime('1900-01-01',otherformat)

    return(clock)

def Convert_to_clock_x(m):
    m=int(m)
    if m == 1:
        a = 1
    if m == 2:
        a = 2
    if m == 3:
        a = 3
    if m == 4:
        a = 2
    if m == 5:
        a = 1
    if m == 6:
        a = 0
    if m == 7:
        a = -1
    if m == 8:
        a = -2
    if m == 9:
        a = -3
    if m == 10:
        a = -2
    if m == 11:
        a = -1
    if m == 12:
        a = 0
    return(a)

def Convert_to_clock_y(m):
    m=int(m)
    if m == 1:
        a = 2
    if m == 2:
        a = 1
    if m == 3:
        a = 0
    if m == 4:
        a = -1
    if m == 5:
        a = -2
    if m == 6:
        a = -3
    if m == 7:
        a = -2
    if m == 8:
        a = -1
    if m == 9:
        a = 0
    if m == 10:
        a = 1
    if m == 11:
        a = 2
    if m == 12:
        a = 3
    return(a)

import matplotlib.pyplot as plt
import seaborn as sns

#function for producing nice, smoothed line plots sorted by categorical variable, of a continues (var_dist) variable
def comp_dist(df_to_use, cat_to_subset, var_dist, figw,figh,linew):
    plt.figure(figsize=(figw,figh))
    sns.set_context( rc={"lines.linewidth": linew})

    for grp in sorted(df_to_use[cat_to_subset].unique()):
        grp_df = df_to_use.loc[df_to_use[cat_to_subset] == grp]

        sns.distplot(grp_df[var_dist], hist=False, label=grp)
        plt.xlim(0, 90)
    plt.show()

import math

def getxy(day):
    x = math.sin((180 - day * 0.9849521203830369)/180 * 3.141)
    y = math.cos((180 - day * 0.9849521203830369)/180 * 3.141)
    return x, y


# Copyright (c) 2012-2015 The GPy authors (see AUTHORS.txt)
# Licensed under the BSD 3-clause license (see LICENSE.txt)

import numpy as np
from scipy import stats,special
import scipy as sp
from . import link_functions
from ..util.misc import chain_1, chain_2, chain_3, blockify_dhess_dtheta, blockify_third, blockify_hessian, safe_exp
from scipy.integrate import quad
import warnings
from ..core.parameterization import Parameterized

class Likelihood(Parameterized):
    """
    Likelihood base class, used to defing p(y|f).

    All instances use _inverse_ link functions, which can be swapped out. It is
    expected that inheriting classes define a default inverse link function

    To use this class, inherit and define missing functionality.

    Inheriting classes *must* implement:
       pdf_link : a bound method which turns the output of the link function into the pdf
       logpdf_link : the logarithm of the above

    To enable use with EP, inheriting classes *must* define:
       TODO: a suitable derivative function for any parameters of the class
    It is also desirable to define:
       moments_match_ep : a function to compute the EP moments If this isn't defined, the moments will be computed using 1D quadrature.

    To enable use with Laplace approximation, inheriting classes *must* define:
       Some derivative functions *AS TODO*

    For exact Gaussian inference, define *JH TODO*

    """
    def __init__(self, gp_link, name):
        super(Likelihood, self).__init__(name)
        assert isinstance(gp_link,link_functions.GPTransformation), "gp_link is not a valid GPTransformation."
        self.gp_link = gp_link
        self.log_concave = False
        self.not_block_really = False

    def request_num_latent_functions(self, Y):
        """
        The likelihood should infer how many latent functions are needed for the likelihood

        Default is the number of outputs
        """
        return Y.shape[1]

    def exact_inference_gradients(self, dL_dKdiag,Y_metadata=None):
        return np.zeros(self.size)

    def update_gradients(self, partial):
        if self.size > 0:
            raise NotImplementedError('Must be implemented for likelihoods with parameters to be optimized')

    def _preprocess_values(self,Y):
        """
        In case it is needed, this function assess the output values or makes any pertinent transformation on them.

        :param Y: observed output
        :type Y: Nx1 numpy.darray

        """
        return Y

    def conditional_mean(self, gp):
        """
        The mean of the random variable conditioned on one value of the GP
        """
        raise NotImplementedError

    def conditional_variance(self, gp):
        """
        The variance of the random variable conditioned on one value of the GP
        """
        raise NotImplementedError

    def log_predictive_density(self, y_test, mu_star, var_star, Y_metadata=None):
        """
        Calculation of the log predictive density

        .. math:
            p(y_{*}|D) = p(y_{*}|f_{*})p(f_{*}|\mu_{*}\\sigma^{2}_{*})

        :param y_test: test observations (y_{*})
        :type y_test: (Nx1) array
        :param mu_star: predictive mean of gaussian p(f_{*}|mu_{*}, var_{*})
        :type mu_star: (Nx1) array
        :param var_star: predictive variance of gaussian p(f_{*}|mu_{*}, var_{*})
        :type var_star: (Nx1) array
        """
        assert y_test.shape==mu_star.shape
        assert y_test.shape==var_star.shape
        assert y_test.shape[1] == 1

        flat_y_test = y_test.flatten()
        flat_mu_star = mu_star.flatten()
        flat_var_star = var_star.flatten()

        if Y_metadata is not None:
            #Need to zip individual elements of Y_metadata aswell
            Y_metadata_flat = {}
            if Y_metadata is not None:
                for key, val in Y_metadata.items():
                    Y_metadata_flat[key] = np.atleast_1d(val).reshape(-1,1)

            zipped_values = []

            for i in range(y_test.shape[0]):
                y_m = {}
                for key, val in Y_metadata_flat.items():
                    if np.isscalar(val) or val.shape[0] == 1:
                        y_m[key] = val
                    else:
                        #Won't broadcast yet
                        y_m[key] = val[i]
                zipped_values.append((flat_y_test[i], flat_mu_star[i], flat_var_star[i], y_m))
        else:
            #Otherwise just pass along None's
            zipped_values = zip(flat_y_test, flat_mu_star, flat_var_star, [None]*y_test.shape[0])

        def integral_generator(yi, mi, vi, yi_m):
            """Generate a function which can be integrated
            to give p(Y*|Y) = int p(Y*|f*)p(f*|Y) df*"""
            def f(fi_star):
                #exponent = np.exp(-(1./(2*vi))*np.square(mi-fi_star))
                #from GPy.util.misc import safe_exp
                #exponent = safe_exp(exponent)
                #res = safe_exp(self.logpdf(fi_star, yi, yi_m))*exponent

                #More stable in the log space
                res = np.exp(self.logpdf(fi_star, yi, yi_m)
                              - 0.5*np.log(2*np.pi*vi)
                              - 0.5*np.square(fi_star-mi)/vi)
                if not np.isfinite(res):
                    import ipdb; ipdb.set_trace()  # XXX BREAKPOINT
                return res

            return f

        p_ystar, _ = zip(*[quad(integral_generator(yi, mi, vi, yi_m), -np.inf, np.inf)
                           for yi, mi, vi, yi_m in zipped_values])
        p_ystar = np.array(p_ystar).reshape(*y_test.shape)
        return np.log(p_ystar)

    def log_predictive_density_sampling(self, y_test, mu_star, var_star, Y_metadata=None, num_samples=1000):
        """
        Calculation of the log predictive density via sampling

        .. math:
            log p(y_{*}|D) = log 1/num_samples prod^{S}_{s=1} p(y_{*}|f_{*s})
            f_{*s} ~ p(f_{*}|\mu_{*}\\sigma^{2}_{*})

        :param y_test: test observations (y_{*})
        :type y_test: (Nx1) array
        :param mu_star: predictive mean of gaussian p(f_{*}|mu_{*}, var_{*})
        :type mu_star: (Nx1) array
        :param var_star: predictive variance of gaussian p(f_{*}|mu_{*}, var_{*})
        :type var_star: (Nx1) array
        :param num_samples: num samples of p(f_{*}|mu_{*}, var_{*}) to take
        :type num_samples: int
        """
        assert y_test.shape==mu_star.shape
        assert y_test.shape==var_star.shape
        assert y_test.shape[1] == 1

        #Take samples of p(f*|y)
        #fi_samples = np.random.randn(num_samples)*np.sqrt(var_star) + mu_star
        fi_samples = np.random.normal(mu_star, np.sqrt(var_star), size=(mu_star.shape[0], num_samples))

        from scipy.misc import logsumexp
        log_p_ystar = -np.log(num_samples) + logsumexp(self.logpdf(fi_samples, y_test, Y_metadata=Y_metadata), axis=1)
        log_p_ystar = np.array(log_p_ystar).reshape(*y_test.shape)
        return log_p_ystar

    def moments_match_ep(self,obs,tau,v,Y_metadata_i=None):
        """
        Calculation of moments using quadrature

        :param obs: observed output
        :param tau: cavity distribution 1st natural parameter (precision)
        :param v: cavity distribution 2nd natural paramenter (mu*precision)
        """
        #Compute first integral for zeroth moment.
        #NOTE constant np.sqrt(2*pi/tau) added at the end of the function
        mu = v/tau
        sigma2 = 1./tau
        #Lets do these for now based on the same idea as Gaussian quadrature
        # i.e. multiply anything by close to zero, and its zero.
        f_min = mu - 20*np.sqrt(sigma2)
        f_max = mu + 20*np.sqrt(sigma2)

        def int_1(f):
            return self.pdf(f, obs, Y_metadata=Y_metadata_i)*np.exp(-0.5*tau*np.square(mu-f))
        z_scaled, accuracy = quad(int_1, f_min, f_max)

        #Compute second integral for first moment
        def int_2(f):
            return f*self.pdf(f, obs, Y_metadata=Y_metadata_i)*np.exp(-0.5*tau*np.square(mu-f))
        mean, accuracy = quad(int_2, f_min, f_max)
        mean /= z_scaled

        #Compute integral for variance
        def int_3(f):
            return (f**2)*self.pdf(f, obs, Y_metadata=Y_metadata_i)*np.exp(-0.5*tau*np.square(mu-f))
        Ef2, accuracy = quad(int_3, f_min, f_max)
        Ef2 /= z_scaled
        variance = Ef2 - mean**2

        #Add constant to the zeroth moment
        #NOTE: this constant is not needed in the other moments because it cancells out.
        z = z_scaled/np.sqrt(2*np.pi/tau)

        return z, mean, variance

    #only compute gh points if required
    __gh_points = None
    def _gh_points(self, T=20):
        if self.__gh_points is None:
            self.__gh_points = np.polynomial.hermite.hermgauss(T)
        return self.__gh_points

    def variational_expectations(self, Y, m, v, gh_points=None, Y_metadata=None):
        """
        Use Gauss-Hermite Quadrature to compute

           E_p(f) [ log p(y|f) ]
           d/dm E_p(f) [ log p(y|f) ]
           d/dv E_p(f) [ log p(y|f) ]

        where p(f) is a Gaussian with mean m and variance v. The shapes of Y, m and v should match.

        if no gh_points are passed, we construct them using defualt options
        """

        if gh_points is None:
            gh_x, gh_w = self._gh_points()
        else:
            gh_x, gh_w = gh_points

        shape = m.shape
        m,v,Y = m.flatten(), v.flatten(), Y.flatten()

        #make a grid of points
        X = gh_x[None,:]*np.sqrt(2.*v[:,None]) + m[:,None]

        #evaluate the likelhood for the grid. First ax indexes the data (and mu, var) and the second indexes the grid.
        # broadcast needs to be handled carefully.
        logp = self.logpdf(X,Y[:,None], Y_metadata=Y_metadata)
        dlogp_dx = self.dlogpdf_df(X, Y[:,None], Y_metadata=Y_metadata)
        d2logp_dx2 = self.d2logpdf_df2(X, Y[:,None], Y_metadata=Y_metadata)

        #clipping for numerical stability
        #logp = np.clip(logp,-1e9,1e9)
        #dlogp_dx = np.clip(dlogp_dx,-1e9,1e9)
        #d2logp_dx2 = np.clip(d2logp_dx2,-1e9,1e9)

        #average over the gird to get derivatives of the Gaussian's parameters
        #division by pi comes from fact that for each quadrature we need to scale by 1/sqrt(pi)
        F = np.dot(logp, gh_w)/np.sqrt(np.pi)
        dF_dm = np.dot(dlogp_dx, gh_w)/np.sqrt(np.pi)
        dF_dv = np.dot(d2logp_dx2, gh_w)/np.sqrt(np.pi)
        dF_dv /= 2.

        if np.any(np.isnan(dF_dv)) or np.any(np.isinf(dF_dv)):
            stop
        if np.any(np.isnan(dF_dm)) or np.any(np.isinf(dF_dm)):
            stop

        if self.size:
            dF_dtheta = self.dlogpdf_dtheta(X, Y[:,None], Y_metadata=Y_metadata) # Ntheta x (orig size) x N_{quad_points}
            dF_dtheta = np.dot(dF_dtheta, gh_w)/np.sqrt(np.pi)
            dF_dtheta = dF_dtheta.reshape(self.size, shape[0], shape[1])
        else:
            dF_dtheta = None # Not yet implemented
        return F.reshape(*shape), dF_dm.reshape(*shape), dF_dv.reshape(*shape), dF_dtheta

    def predictive_mean(self, mu, variance, Y_metadata=None):
        """
        Quadrature calculation of the predictive mean: E(Y_star|Y) = E( E(Y_star|f_star, Y) )

        :param mu: mean of posterior
        :param sigma: standard deviation of posterior

        """
        #conditional_mean: the edpected value of y given some f, under this likelihood
        fmin = -np.inf
        fmax = np.inf
        def int_mean(f,m,v):
            exponent = -(0.5/v)*np.square(f - m)
            #If exponent is under -30 then exp(exponent) will be very small, so don't exp it!)
            #If p is zero then conditional_mean will overflow
            assert v.all() > 0
            p = safe_exp(exponent)

            #If p is zero then conditional_variance will overflow
            if p < 1e-10:
                return 0.
            else:
                return self.conditional_mean(f)*p
        scaled_mean = [quad(int_mean, fmin, fmax,args=(mj,s2j))[0] for mj,s2j in zip(mu,variance)]
        mean = np.array(scaled_mean)[:,None] / np.sqrt(2*np.pi*(variance))
        return mean

    def predictive_variance(self, mu,variance, predictive_mean=None, Y_metadata=None):
        """
        Approximation to the predictive variance: V(Y_star)

        The following variance decomposition is used:
        V(Y_star) = E( V(Y_star|f_star)**2 ) + V( E(Y_star|f_star) )**2

        :param mu: mean of posterior
        :param sigma: standard deviation of posterior
        :predictive_mean: output's predictive mean, if None _predictive_mean function will be called.

        """
        #sigma2 = sigma**2
        normalizer = np.sqrt(2*np.pi*variance)

        fmin_v = -np.inf
        fmin_m = np.inf
        fmin = -np.inf
        fmax = np.inf

        from ..util.misc import safe_exp
        # E( V(Y_star|f_star) )
        def int_var(f,m,v):
            exponent = -(0.5/v)*np.square(f - m)
            p = safe_exp(exponent)
            #If p is zero then conditional_variance will overflow
            if p < 1e-10:
                return 0.
            else:
                return self.conditional_variance(f)*p
        scaled_exp_variance = [quad(int_var, fmin_v, fmax,args=(mj,s2j))[0] for mj,s2j in zip(mu,variance)]
        exp_var = np.array(scaled_exp_variance)[:,None] / normalizer

        #V( E(Y_star|f_star) ) =  E( E(Y_star|f_star)**2 ) - E( E(Y_star|f_star) )**2

        #E( E(Y_star|f_star) )**2
        if predictive_mean is None:
            predictive_mean = self.predictive_mean(mu,variance)
        predictive_mean_sq = predictive_mean**2

        #E( E(Y_star|f_star)**2 )
        def int_pred_mean_sq(f,m,v,predictive_mean_sq):
            exponent = -(0.5/v)*np.square(f - m)
            p = np.exp(exponent)
            #If p is zero then conditional_mean**2 will overflow
            if p < 1e-10:
                return 0.
            else:
                return self.conditional_mean(f)**2*p

        scaled_exp_exp2 = [quad(int_pred_mean_sq, fmin_m, fmax,args=(mj,s2j,pm2j))[0] for mj,s2j,pm2j in zip(mu,variance,predictive_mean_sq)]
        exp_exp2 = np.array(scaled_exp_exp2)[:,None] / normalizer

        var_exp = exp_exp2 - predictive_mean_sq

        # V(Y_star) = E[ V(Y_star|f_star) ] + V[ E(Y_star|f_star) ]
        # V(Y_star) = E[ V(Y_star|f_star) ] + E(Y_star**2|f_star) - E[Y_star|f_star]**2
        return exp_var + var_exp

    def pdf_link(self, inv_link_f, y, Y_metadata=None):
        raise NotImplementedError

    def logpdf_link(self, inv_link_f, y, Y_metadata=None):
        raise NotImplementedError

    def dlogpdf_dlink(self, inv_link_f, y, Y_metadata=None):
        raise NotImplementedError

    def d2logpdf_dlink2(self, inv_link_f, y, Y_metadata=None):
        raise NotImplementedError

    def d3logpdf_dlink3(self, inv_link_f, y, Y_metadata=None):
        raise NotImplementedError

    def dlogpdf_link_dtheta(self, inv_link_f, y, Y_metadata=None):
        raise NotImplementedError

    def dlogpdf_dlink_dtheta(self, inv_link_f, y, Y_metadata=None):
        raise NotImplementedError

    def d2logpdf_dlink2_dtheta(self, inv_link_f, y, Y_metadata=None):
        raise NotImplementedError

    def pdf(self, f, y, Y_metadata=None):
        """
        Evaluates the link function link(f) then computes the likelihood (pdf) using it

        .. math:
            p(y|\\lambda(f))

        :param f: latent variables f
        :type f: Nx1 array
        :param y: data
        :type y: Nx1 array
        :param Y_metadata: Y_metadata which is not used in student t distribution - not used
        :returns: likelihood evaluated for this point
        :rtype: float
        """
        if isinstance(self.gp_link, link_functions.Identity):
            return self.pdf_link(f, y, Y_metadata=Y_metadata)
        else:
            inv_link_f = self.gp_link.transf(f)
            return self.pdf_link(inv_link_f, y, Y_metadata=Y_metadata)

    def logpdf_sum(self, f, y, Y_metadata=None):
        """
        Convenience function that can overridden for functions where this could
        be computed more efficiently
        """
        return np.sum(self.logpdf(f, y, Y_metadata=Y_metadata))

    def logpdf(self, f, y, Y_metadata=None):
        """
        Evaluates the link function link(f) then computes the log likelihood (log pdf) using it

        .. math:
            \\log p(y|\\lambda(f))

        :param f: latent variables f
        :type f: Nx1 array
        :param y: data
        :type y: Nx1 array
        :param Y_metadata: Y_metadata which is not used in student t distribution - not used
        :returns: log likelihood evaluated for this point
        :rtype: float
        """
        if isinstance(self.gp_link, link_functions.Identity):
            return self.logpdf_link(f, y, Y_metadata=Y_metadata)
        else:
            inv_link_f = self.gp_link.transf(f)
            return self.logpdf_link(inv_link_f, y, Y_metadata=Y_metadata)

    def dlogpdf_df(self, f, y, Y_metadata=None):
        """
        Evaluates the link function link(f) then computes the derivative of log likelihood using it
        Uses the Faa di Bruno's formula for the chain rule

        .. math::
            \\frac{d\\log p(y|\\lambda(f))}{df} = \\frac{d\\log p(y|\\lambda(f))}{d\\lambda(f)}\\frac{d\\lambda(f)}{df}

        :param f: latent variables f
        :type f: Nx1 array
        :param y: data
        :type y: Nx1 array
        :param Y_metadata: Y_metadata which is not used in student t distribution - not used
        :returns: derivative of log likelihood evaluated for this point
        :rtype: 1xN array
        """
        if isinstance(self.gp_link, link_functions.Identity):
            return self.dlogpdf_dlink(f, y, Y_metadata=Y_metadata)
        else:
            inv_link_f = self.gp_link.transf(f)
            dlogpdf_dlink = self.dlogpdf_dlink(inv_link_f, y, Y_metadata=Y_metadata)
            dlink_df = self.gp_link.dtransf_df(f)
            return chain_1(dlogpdf_dlink, dlink_df)

    @blockify_hessian
    def d2logpdf_df2(self, f, y, Y_metadata=None):
        """
        Evaluates the link function link(f) then computes the second derivative of log likelihood using it
        Uses the Faa di Bruno's formula for the chain rule

        .. math::
            \\frac{d^{2}\\log p(y|\\lambda(f))}{df^{2}} = \\frac{d^{2}\\log p(y|\\lambda(f))}{d^{2}\\lambda(f)}\\left(\\frac{d\\lambda(f)}{df}\\right)^{2} + \\frac{d\\log p(y|\\lambda(f))}{d\\lambda(f)}\\frac{d^{2}\\lambda(f)}{df^{2}}

        :param f: latent variables f
        :type f: Nx1 array
        :param y: data
        :type y: Nx1 array
        :param Y_metadata: Y_metadata which is not used in student t distribution - not used
        :returns: second derivative of log likelihood evaluated for this point (diagonal only)
        :rtype: 1xN array
        """
        if isinstance(self.gp_link, link_functions.Identity):
            d2logpdf_df2 = self.d2logpdf_dlink2(f, y, Y_metadata=Y_metadata)
        else:
            inv_link_f = self.gp_link.transf(f)
            d2logpdf_dlink2 = self.d2logpdf_dlink2(inv_link_f, y, Y_metadata=Y_metadata)
            dlink_df = self.gp_link.dtransf_df(f)
            dlogpdf_dlink = self.dlogpdf_dlink(inv_link_f, y, Y_metadata=Y_metadata)
            d2link_df2 = self.gp_link.d2transf_df2(f)
            d2logpdf_df2 = chain_2(d2logpdf_dlink2, dlink_df, dlogpdf_dlink, d2link_df2)
        return d2logpdf_df2

    @blockify_third
    def d3logpdf_df3(self, f, y, Y_metadata=None):
        """
        Evaluates the link function link(f) then computes the third derivative of log likelihood using it
        Uses the Faa di Bruno's formula for the chain rule

        .. math::
            \\frac{d^{3}\\log p(y|\\lambda(f))}{df^{3}} = \\frac{d^{3}\\log p(y|\\lambda(f)}{d\\lambda(f)^{3}}\\left(\\frac{d\\lambda(f)}{df}\\right)^{3} + 3\\frac{d^{2}\\log p(y|\\lambda(f)}{d\\lambda(f)^{2}}\\frac{d\\lambda(f)}{df}\\frac{d^{2}\\lambda(f)}{df^{2}} + \\frac{d\\log p(y|\\lambda(f)}{d\\lambda(f)}\\frac{d^{3}\\lambda(f)}{df^{3}}

        :param f: latent variables f
        :type f: Nx1 array
        :param y: data
        :type y: Nx1 array
        :param Y_metadata: Y_metadata which is not used in student t distribution - not used
        :returns: third derivative of log likelihood evaluated for this point
        :rtype: float
        """
        if isinstance(self.gp_link, link_functions.Identity):
            d3logpdf_df3 = self.d3logpdf_dlink3(f, y, Y_metadata=Y_metadata)
        else:
            inv_link_f = self.gp_link.transf(f)
            d3logpdf_dlink3 = self.d3logpdf_dlink3(inv_link_f, y, Y_metadata=Y_metadata)
            dlink_df = self.gp_link.dtransf_df(f)
            d2logpdf_dlink2 = self.d2logpdf_dlink2(inv_link_f, y, Y_metadata=Y_metadata)
            d2link_df2 = self.gp_link.d2transf_df2(f)
            dlogpdf_dlink = self.dlogpdf_dlink(inv_link_f, y, Y_metadata=Y_metadata)
            d3link_df3 = self.gp_link.d3transf_df3(f)
            d3logpdf_df3 = chain_3(d3logpdf_dlink3, dlink_df, d2logpdf_dlink2, d2link_df2, dlogpdf_dlink, d3link_df3)
        return d3logpdf_df3

    def dlogpdf_dtheta(self, f, y, Y_metadata=None):
        """
        TODO: Doc strings
        """
        if self.size > 0:
            if self.not_block_really:
                raise NotImplementedError("Need to make a decorator for this!")
            if isinstance(self.gp_link, link_functions.Identity):
                return self.dlogpdf_link_dtheta(f, y, Y_metadata=Y_metadata)
            else:
                inv_link_f = self.gp_link.transf(f)
                return self.dlogpdf_link_dtheta(inv_link_f, y, Y_metadata=Y_metadata)
        else:
            # There are no parameters so return an empty array for derivatives
            return np.zeros((0, f.shape[0], f.shape[1]))

    def dlogpdf_df_dtheta(self, f, y, Y_metadata=None):
        """
        TODO: Doc strings
        """
        if self.size > 0:
            if self.not_block_really:
                raise NotImplementedError("Need to make a decorator for this!")
            if isinstance(self.gp_link, link_functions.Identity):
                return self.dlogpdf_dlink_dtheta(f, y, Y_metadata=Y_metadata)
            else:
                inv_link_f = self.gp_link.transf(f)
                dlink_df = self.gp_link.dtransf_df(f)
                dlogpdf_dlink_dtheta = self.dlogpdf_dlink_dtheta(inv_link_f, y, Y_metadata=Y_metadata)

                dlogpdf_df_dtheta = np.zeros((self.size, f.shape[0], f.shape[1]))
                #Chain each parameter of hte likelihood seperately
                for p in range(self.size):
                    dlogpdf_df_dtheta[p, :, :] = chain_1(dlogpdf_dlink_dtheta[p,:,:], dlink_df)
                return dlogpdf_df_dtheta
                #return chain_1(dlogpdf_dlink_dtheta, dlink_df)
        else:
            # There are no parameters so return an empty array for derivatives
            return np.zeros((0, f.shape[0], f.shape[1]))

    def d2logpdf_df2_dtheta(self, f, y, Y_metadata=None):
        """
        TODO: Doc strings
        """
        if self.size > 0:
            if self.not_block_really:
                raise NotImplementedError("Need to make a decorator for this!")
            if isinstance(self.gp_link, link_functions.Identity):
                return self.d2logpdf_dlink2_dtheta(f, y, Y_metadata=Y_metadata)
            else:
                inv_link_f = self.gp_link.transf(f)
                dlink_df = self.gp_link.dtransf_df(f)
                d2link_df2 = self.gp_link.d2transf_df2(f)
                d2logpdf_dlink2_dtheta = self.d2logpdf_dlink2_dtheta(inv_link_f, y, Y_metadata=Y_metadata)
                dlogpdf_dlink_dtheta = self.dlogpdf_dlink_dtheta(inv_link_f, y, Y_metadata=Y_metadata)

                d2logpdf_df2_dtheta = np.zeros((self.size, f.shape[0], f.shape[1]))
                #Chain each parameter of hte likelihood seperately
                for p in range(self.size):
                    d2logpdf_df2_dtheta[p, :, :] = chain_2(d2logpdf_dlink2_dtheta[p,:,:], dlink_df, dlogpdf_dlink_dtheta[p,:,:], d2link_df2)
                return d2logpdf_df2_dtheta
                #return chain_2(d2logpdf_dlink2_dtheta, dlink_df, dlogpdf_dlink_dtheta, d2link_df2)
        else:
            # There are no parameters so return an empty array for derivatives
            return np.zeros((0, f.shape[0], f.shape[1]))

    def _laplace_gradients(self, f, y, Y_metadata=None):
        dlogpdf_dtheta = self.dlogpdf_dtheta(f, y, Y_metadata=Y_metadata)
        dlogpdf_df_dtheta = self.dlogpdf_df_dtheta(f, y, Y_metadata=Y_metadata)
        d2logpdf_df2_dtheta = self.d2logpdf_df2_dtheta(f, y, Y_metadata=Y_metadata)

        #Parameters are stacked vertically. Must be listed in same order as 'get_param_names'
        # ensure we have gradients for every parameter we want to optimize
        assert dlogpdf_dtheta.shape[0] == self.size #num_param array x f, d
        assert dlogpdf_df_dtheta.shape[0] == self.size #num_param x f x d x matrix or just num_param x f
        assert d2logpdf_df2_dtheta.shape[0] == self.size #num_param x f matrix or num_param x f x d x matrix, num_param x f x f or num_param x f x f x d

        return dlogpdf_dtheta, dlogpdf_df_dtheta, d2logpdf_df2_dtheta

    def predictive_values(self, mu, var, full_cov=False, Y_metadata=None):
        """
        Compute  mean, variance of the  predictive distibution.

        :param mu: mean of the latent variable, f, of posterior
        :param var: variance of the latent variable, f, of posterior
        :param full_cov: whether to use the full covariance or just the diagonal
        :type full_cov: Boolean
        """
        try:
            pred_mean = self.predictive_mean(mu, var, Y_metadata=Y_metadata)
            pred_var = self.predictive_variance(mu, var, pred_mean, Y_metadata=Y_metadata)
        except NotImplementedError:
            print("Finding predictive mean and variance via sampling rather than quadrature")
            Nf_samp = 300
            Ny_samp = 1
            s = np.random.randn(mu.shape[0], Nf_samp)*np.sqrt(var) + mu
            ss_y = self.samples(s, Y_metadata, samples=Ny_samp)
            pred_mean = np.mean(ss_y, axis=1)[:, None]
            pred_var = np.var(ss_y, axis=1)[:, None]

        return pred_mean, pred_var

    def predictive_quantiles(self, mu, var, quantiles, Y_metadata=None):
        #compute the quantiles by sampling!!!
        Nf_samp = 300
        Ny_samp = 1
        s = np.random.randn(mu.shape[0], Nf_samp)*np.sqrt(var) + mu
        ss_y = self.samples(s, Y_metadata)#, samples=Ny_samp)
        #ss_y = ss_y.reshape(mu.shape[0], mu.shape[1], Nf_samp*Ny_samp)

        pred_quantiles = [np.percentile(ss_y, q, axis=1)[:,None] for q in quantiles]
        return pred_quantiles

    def samples(self, gp, Y_metadata=None, samples=1):
        """
        Returns a set of samples of observations based on a given value of the latent variable.

        :param gp: latent variable
        :param samples: number of samples to take for each f location
        """
        raise NotImplementedError("""May be possible to use MCMC with user-tuning, see
                                  MCMC_pdf_samples in likelihood.py and write samples function
                                  using this, beware this is a simple implementation
                                  of Metropolis and will not work well for all likelihoods""")

    def MCMC_pdf_samples(self, fNew, num_samples=1000, starting_loc=None, stepsize=0.1, burn_in=1000, Y_metadata=None):
        """
        Simple implementation of Metropolis sampling algorithm

        Will run a parallel chain for each input dimension (treats each f independently)
        Thus assumes f*_1 independant of f*_2 etc.

        :param num_samples: Number of samples to take
        :param fNew: f at which to sample around
        :param starting_loc: Starting locations of the independant chains (usually will be conditional_mean of likelihood), often link_f
        :param stepsize: Stepsize for the normal proposal distribution (will need modifying)
        :param burnin: number of samples to use for burnin (will need modifying)
        :param Y_metadata: Y_metadata for pdf
        """
        print("Warning, using MCMC for sampling y*, needs to be tuned!")
        if starting_loc is None:
            starting_loc = fNew
        from functools import partial
        logpdf = partial(self.logpdf, f=fNew, Y_metadata=Y_metadata)
        pdf = lambda y_star: np.exp(logpdf(y=y_star[:, None]))
        #Should be the link function of f is a good starting point
        #(i.e. the point before you corrupt it with the likelihood)
        par_chains = starting_loc.shape[0]
        chain_values = np.zeros((par_chains, num_samples))
        chain_values[:, 0][:,None] = starting_loc
        #Use same stepsize for all par_chains
        stepsize = np.ones(par_chains)*stepsize
        accepted = np.zeros((par_chains, num_samples+burn_in))
        accept_ratio = np.zeros(num_samples+burn_in)
        #Whilst burning in, only need to keep the previous lot
        burnin_cache = np.zeros(par_chains)
        burnin_cache[:] = starting_loc.flatten()
        burning_in = True
        for i in xrange(burn_in+num_samples):
            next_ind = i-burn_in
            if burning_in:
                old_y = burnin_cache
            else:
                old_y = chain_values[:,next_ind-1]

            old_lik = pdf(old_y)
            #Propose new y from Gaussian proposal
            new_y = np.random.normal(loc=old_y, scale=stepsize)
            new_lik = pdf(new_y)
            #Accept using Metropolis (not hastings) acceptance
            #Always accepts if new_lik > old_lik
            accept_probability = np.minimum(1, new_lik/old_lik)
            u = np.random.uniform(0,1,par_chains)
            #print "Accept prob: ", accept_probability
            accepts = u < accept_probability
            if burning_in:
                burnin_cache[accepts] = new_y[accepts]
                burnin_cache[~accepts] = old_y[~accepts]
                if i == burn_in:
                    burning_in = False
                    chain_values[:,0] = burnin_cache
            else:
                #If it was accepted then new_y becomes the latest sample
                chain_values[accepts, next_ind] = new_y[accepts]
                #Otherwise use old y as the sample
                chain_values[~accepts, next_ind] = old_y[~accepts]

            accepted[~accepts, i] = 0
            accepted[accepts, i] = 1
            accept_ratio[i] = np.sum(accepted[:,i])/float(par_chains)

            #Show progress
            if i % int((burn_in+num_samples)*0.1) == 0:
                print("{}% of samples taken ({})".format((i/int((burn_in+num_samples)*0.1)*10), i))
                print("Last run accept ratio: ", accept_ratio[i])

        print("Average accept ratio: ", np.mean(accept_ratio))
        return chain_values


#!/usr/bin/env python3

# Example command:
# ./bin/predict_ailab.py

import os
import glob

import cv2
import PIL.Image as Image
import numpy as np
import torch
import tqdm

from saicinpainting.prediction.default import LaMa

def load_image(fname, mode='RGB'):
    img = np.array(Image.open(fname).convert(mode))
    if img.ndim == 3:
        img = np.transpose(img, (2, 0, 1))
    return img.astype('float32') / 255

def pad_img_to_modulo(img, mod):
    _, height, width = img.shape
    out_height = height if height % mod == 0 else (height // mod + 1) * mod
    out_width = width if width % mod == 0 else (width // mod + 1) * mod
    return np.pad(img, ((0, 0), (0, out_height - height), (0, out_width - width)), mode='symmetric')

def main():
    device = torch.device('cuda')

    checkpoint_path = os.path.join('big-lama', 'models', 'best.ckpt')
    state = torch.load(checkpoint_path)

    model = LaMa()
    model.load_state_dict(state['state_dict'], strict=False)
    model.eval()
    model.to(device)

    mask_filenames = sorted(list(glob.glob(os.path.join('LaMa_test_images', '**', '*mask*.png'), recursive=True)))
    img_filenames = [fname.rsplit('_mask', 1)[0] + '.png' for fname in mask_filenames]
    os.makedirs('output', exist_ok=True)
    with torch.no_grad():
        for img_i in tqdm.trange(len(mask_filenames)):
            mask_fname = mask_filenames[img_i]
            cur_out_fname = os.path.join(
                'output',
                os.path.splitext(mask_fname[len('LaMa_test_images/'):])[0] + '.png'
            )

            image = load_image(img_filenames[img_i], mode='RGB')
            image = pad_img_to_modulo(image, 8)
            image = torch.tensor(image).unsqueeze(0).to(device)

            mask = load_image(mask_filenames[img_i], mode='L')[None, ...]  # create channel dim by unsqueeze
            mask = pad_img_to_modulo(mask, 8)
            mask = torch.tensor(mask).unsqueeze(0).to(device)
            mask = (mask > 0) * 1

            batch = model(dict(image=image, mask=mask))

            cur_res = batch['inpainted'][0].permute(1, 2, 0).detach().cpu().numpy()
            cur_res = np.clip(cur_res * 255, 0, 255).astype('uint8')
            cur_res = cv2.cvtColor(cur_res, cv2.COLOR_RGB2BGR)
            cv2.imwrite(cur_out_fname, cur_res)

if __name__ == '__main__':
    main()


import numpy as np
import json
import pickle
from flask import Flask, render_template, request
from flask_cors import CORS
from sklearn.naive_bayes import MultinomialNB
import itertools
from tatabahasa import *
from unidecode import unidecode
import re

app = Flask(__name__)
CORS(app)

stopword_tatabahasa = list(set(tanya_list+perintah_list+pangkal_list+bantu_list+penguat_list+\
                penegas_list+nafi_list+pemeri_list+sendi_list+pembenar_list+nombor_list+\
                suku_bilangan_list+pisahan_list+keterangan_list+arah_list+hubung_list+gantinama_list))

with open('stop-word-kerulnet','r') as fopen:
    stopwords = list(filter(None, fopen.read().split('\n')))

with open('tfidf.p','rb') as fopen:
    tfidf = pickle.load(fopen)
with open('tfidf_bm.p','rb') as fopen:
    tfidf_bm = pickle.load(fopen)

with open('bayes.p','rb') as fopen:
    bayes = pickle.load(fopen)
with open('bayes_bm.p','rb') as fopen:
    bayes_bm = pickle.load(fopen)

VOWELS = "aeiou"
PHONES = ['sh', 'ch', 'ph', 'sz', 'cz', 'sch', 'rz', 'dz']

def isWord(word):
    if word:
        consecutiveVowels = 0
        consecutiveConsonents = 0
        for idx, letter in enumerate(word.lower()):
            vowel = True if letter in VOWELS else False
            if idx:
                prev = word[idx-1]
                prevVowel = True if prev in VOWELS else False
                if not vowel and letter == 'y' and not prevVowel:
                    vowel = True
                if prevVowel != vowel:
                    consecutiveVowels = 0
                    consecutiveConsonents = 0
            if vowel:
                consecutiveVowels += 1
            else:
                consecutiveConsonents +=1
            if consecutiveVowels >= 3 or consecutiveConsonents > 3:
                return False
            if consecutiveConsonents == 3:
                subStr = word[idx-2:idx+1]
                if any(phone in subStr for phone in PHONES):
                    consecutiveConsonents -= 1
                    continue
                return False
    return True

def stemming(word):
    try:
        word = re.findall(r'^(.*?)(%s)$'%('|'.join(hujung)), word)[0][0]
        mula = re.findall(r'^(.*?)(%s)'%('|'.join(permulaan[::-1])), word)[0][1]
        return word.replace(mula,'')
    except:
        return word

label = ['negative','positive']

list_stops = list(set(stopwords+stopword_tatabahasa))
list_laughing = ['huhu','haha','gaga','hihi','wkawka','wkwk','kiki','keke','rt']
def textcleaning(string):
    string = re.sub('http\S+|www.\S+', '',' '.join([i for i in string.split() if i.find('#')<0 and i.find('@')<0]))
    string = unidecode(string).replace('.', '. ')
    string = string.replace(',', ', ')
    string = re.sub('[^\'\"A-Za-z\- ]+', '', unidecode(string))
    string = [y.strip() for y in string.lower().split() if isWord(y.strip())]
    string = [stemming(y) for y in string if y not in list_stops]
    #string = [st.stem(y) for y in string]
    string = [y for y in string if all([y.find(k) < 0 for k in list_laughing]) and y[:len(y)//2] != y[len(y)//2:]]
    string = ' '.join(string).lower()
    string = (''.join(''.join(s)[:2] for _, s in itertools.groupby(string))).split()
    return ' '.join([y for y in string if y not in list_stops])

@app.route('/sentiment', methods = ['GET'])
def get_sentiment():
    string = textcleaning(request.args.get('text'))
    vectors = tfidf.transform([string])
    results = bayes.predict_proba(vectors)
    if results[0,0] == 0.5:
        vectors = tfidf_bm.transform([string])
        results = bayes_bm.predict_proba(vectors)
    u=np.where(results >= 0.6)
    res = []
    for i in range(results.shape[0]):
        if i in u[0]:
            res.append(label[np.argmax(results[i,:])])
        else:
            res.append('neutral')
    return json.dumps({'label':res[0],'string':string})

if __name__ == '__main__':
	app.run(host = '0.0.0.0', threaded = True, port = 8095)