# -*- coding: utf-8 -*-

# Author: Daniel Yang <daniel.yj.yang@gmail.com>
#
# License: BSD 3 clause

from sklearn.tree import DecisionTreeClassifier
from sklearn.tree import DecisionTreeRegressor

import numpy as np
import pandas as pd

from ..base import classifier

# Reduction in uncertainty = gain in information
#def Information_Gain(y, X):
#   IG(y, X) = Entropy(y) - Entropy(y | X)

# “purity” means how homogenized a group is.

#from numpy import ma # masked array

# A good read: https://towardsdatascience.com/entropy-how-decision-trees-make-decisions-2946b9c18c8#:~:text=Entropy%20is%20a%20measure%20of,general%20is%20to%20reduce%20uncertainty.&text=This%20is%20called%20Information%20Gain,gained%20about%20Y%20from%20X.

def Entropy(splitted_sample=[]):
    """
    For example: [194, 106]
    Entropy is a measure of disorder/uncertainty (= low purity, a lack of dominant class)
    Entropy vs. Gini impurity: both of them involve p_j * p_j, but Entropy takes the form of S = k_b*ln(Ω)
    """

    import math

    denominator = sum(splitted_sample)
    if denominator == 0:
        return 0
    Entropy_index = 0
    for numerator_i in range(len(splitted_sample)):
        p_i = splitted_sample[numerator_i]/denominator
        if p_i != 0:
            Entropy_index -= p_i * math.log2(p_i)
        #Entropy_index -= p_i * ma.array(ma.log2(p_i)).filled(0) # handle log2(0) warning
    return Entropy_index

def Gini_impurity(splitted_sample=[]):
    """
    For example: [194, 106]
    note: summation( p_j * (1 - p_j) ) = 1 - summation( p_j^2 )
    """
    denominator = sum(splitted_sample)
    if denominator == 0:
        return 0
    Gini_index = 0
    for numerator_i in range(len(splitted_sample)):
        p_i = splitted_sample[numerator_i]/denominator
        # p_j * (1 - p_j) is the likelihood of misclassifying a new instance
        Gini_index += p_i * (1 - p_i)
    return Gini_index

def impurity_measure_with_sample_weight(y_class0_value=0, y_class1_value=1, y=[], sample_weight=None, impurity_func=Entropy):
    """
    For example: y=[0,1,1,0,0,1], sample_weight=[0.1, 0.1, 0.2, 0.1, 0.1, 0.1]
    """

    y_class0_sample_weighted_sum = 0
    y_class1_sample_weighted_sum = 0

    length_y = len(y)
    if length_y == 0:
        return 0

    if sample_weight is None:
        sample_weight = np.ones(shape=(length_y,))

    #print(f"{sample_weight}")

    if length_y != len(sample_weight):
        raise ValueError("unequal inputs of y[] and sample_weight[]")

    y_class0_sample_weighted_sum = np.sum(sample_weight[y == y_class0_value])
    y_class1_sample_weighted_sum = np.sum(sample_weight[y == y_class1_value])

    for i in range(length_y):
        if y[i] not in [y_class0_value, y_class1_value]:
            raise ValueError(f"y[{i}]=[{y[i]}] must be either y_class0_value=[{y_class0_value}] or y_class1_value=[{y_class1_value}]")

    return impurity_func(splitted_sample=[y_class0_sample_weighted_sum, y_class1_sample_weighted_sum])

def Impurity_plot():

    import matplotlib.pyplot as plt

    n = 200
    X = np.arange(0,n/2+1,1)
    Y = X[::-1]

    def plot(impurity_index, X, index_name="", title="", y_ref=None):
        plt.figure(figsize=(5, 5))
        plt.plot(X, impurity_index)
        plt.axhline(y=y_ref, color='r', linestyle='--')
        plt.title(f"{title}")
        plt.xlabel(f'n splitted as y=0 (total #sample = {n})')
        plt.ylabel(f"{index_name}")
        plt.ylim([0, 1.1])
        plt.show()

    measure = np.empty_like(X)
    for i in range(len(X)):
        measure[i] = Gini_impurity([X[i], Y[i]])
    plot(measure, X, index_name="Gini Impurity", title="Gini Impurity Plot (highest = 0.5 for two classes)", y_ref=0.5)

    measure = np.empty_like(X)
    for i in range(len(X)):
        measure[i] = Entropy([X[i], Y[i]])
    plot(measure, X, index_name="Entropy",       title="Entropy Plot (highest = 1.0 for two classes)",       y_ref=1.0)

def demo_metrics():
    """
    These generally measure the homogeneity of the target variable within the subsets.
    """
    Impurity_plot()

class decision_tree_classifier_node(object):
    def __init__(self, curr_depth=None, curr_impurity=None, curr_sample_size=None, curr_y_distribution={}, best_split_feature_i=None, best_x_cutoff_value=None, y_class0_value=0, y_class1_value=1):
        self.curr_depth = curr_depth
        self.curr_impurity = curr_impurity
        self.curr_sample_size = curr_sample_size
        self.curr_y_distribution = curr_y_distribution

        y_class0_n = curr_y_distribution.get(y_class0_value)
        y_class1_n = curr_y_distribution.get(y_class1_value)
        y_classes_count = len(curr_y_distribution)

        if y_classes_count == 0:
            self.y_dominant_class = None
            self.y_class1_prob = None

        if y_classes_count == 1:
            self.y_dominant_class = list(curr_y_distribution.keys())[0]
            if self.y_dominant_class == y_class0_value:
                self.y_class1_prob = 0
            elif self.y_dominant_class == y_class1_value:
                self.y_class1_prob = 1
            else:
                raise ValueError(f"unexpected y_dominant_class (should be either {y_class0_value} or {y_class1_value})")

        if y_classes_count == 2:
            # must be {y_class0_value: sample_size_associated_with_y=y_class0_value, y_class1_value: sample_size_associated_with_y=y_class1_value}
            self.y_class1_prob = y_class1_n / (y_class1_n + y_class0_n)
            if self.y_class1_prob >= 0.50:
                self.y_dominant_class = y_class1_value
            else:
                self.y_dominant_class = y_class0_value

        if y_classes_count > 2:
            raise ValueError('more than 2 classes in y detected')

        self.best_split_feature_i = best_split_feature_i
        self.best_x_cutoff_value = best_x_cutoff_value
        self.left = None
        self.right = None

    def to_dict(self):
        return {'curr_depth': self.curr_depth, 'curr_impurity': f"{self.curr_impurity:.3f}" if self.curr_impurity is not None else None, 'curr_sample_size': self.curr_sample_size, 'curr_y_distribution': self.curr_y_distribution, 'curr_dominant_y_class': self.y_dominant_class, 'curr_y_class1_prob': f"{self.y_class1_prob:.3f}" if self.y_class1_prob is not None else None, 'best_split_feature_i': self.best_split_feature_i if self.best_x_cutoff_value is not None else None, 'best_x_cutoff_value': f"{self.best_x_cutoff_value:.3f}" if self.best_x_cutoff_value is not None else None}

class decision_tree_classifier_from_scratch(classifier):
    """
    Shortcoming of DT:
    (1) biggest information gain locally, but globally; that is, there is always a better tree in the overall picture
    (2) we may be asking too trivial questions at the greater depths (e.g., > 5 or 6-depth)
    """

    def __init__(self, max_depth = 10, impurity_measure='entropy', features_indices_actually_used='all', annotation=None, verbose=False):
        """
        "features_indices_actually_used": limits the analysis on only these feature indices if not 'all'
            for example, if there are 30 features, then "features_indices_actually_used" = [2, 15] means that only the 3th and 16th features will be used for analysis
        """
        super().__init__()
        self.features_indices_actually_used = features_indices_actually_used  # limits the analysis on only these feature indices
        self.max_depth = max_depth
        self.verbose=verbose
        self.root_node = decision_tree_classifier_node()
        if impurity_measure not in ['entropy', 'gini_impurity']:
            raise ValueError('invalid impurity_measure value')
        self.impurity_measure = impurity_measure
        if self.impurity_measure == 'entropy':
            self.impurity_func = Entropy
        elif self.impurity_measure == 'gini_impurity':
            self.impurity_func = Gini_impurity
        self.annotation = annotation
        # default values
        self.y_class0_value = 0
        self.y_class1_value = 1

    def find_best_split_in_one_specific_feature(self, x, y_true, sample_weight=None):

        if type(x) in [pd.DataFrame, pd.Series]:
            x = x.to_numpy()

        if type(y_true) in [pd.DataFrame, pd.Series]:
            y_true = y_true.to_numpy()

        if sample_weight is None:
            sample_weight = np.ones(shape=(len(y_true),))

        best_impurity = float('Inf')

        from sortedcontainers import SortedSet

        y_classes_values = list(SortedSet(y_true))
        if len(y_classes_values) != 2:
            raise ValueError("y must be binary")

        from collections import Counter

        #y_counts = Counter(y_true)
        #before_split_impurity = self.impurity_func([ y_counts[y_classes_values[0]], y_counts[y_classes_values[1]] ])
        before_split_impurity = impurity_measure_with_sample_weight(y_class0_value=self.y_class0_value, y_class1_value=self.y_class1_value, y=y_true, sample_weight=sample_weight, impurity_func=self.impurity_func)

        x_values_array = list(SortedSet(x))
        x_values_array_length = len(x_values_array)

        for value_i, this_x_value in enumerate(x_values_array):  # iterating through each value in this feature

            if value_i < (x_values_array_length-1):
                this_x_cutoff_value = (this_x_value + x_values_array[value_i+1]) / 2
            else:
                this_x_cutoff_value = this_x_value

            # let's say we split y on this value of x
            y_pred_left_node_list = y_true[x <= this_x_cutoff_value]
            y_pred_left_node_counts = Counter(y_pred_left_node_list)
            left_node_y_true_array  = [ y_pred_left_node_counts[y_classes_values[0]],  y_pred_left_node_counts[ y_classes_values[1]] ]
            #left_node_impurity  = self.impurity_func(left_node_y_true_array)
            left_node_impurity = impurity_measure_with_sample_weight(y_class0_value=self.y_class0_value, y_class1_value=self.y_class1_value, y=y_pred_left_node_list, sample_weight=sample_weight[x <= this_x_cutoff_value], impurity_func=self.impurity_func)
            left_node_n = sum(y_pred_left_node_counts.values())

            y_pred_right_node_list = y_true[x > this_x_cutoff_value]
            y_pred_right_node_counts = Counter(y_pred_right_node_list)
            right_node_y_true_array = [ y_pred_right_node_counts[y_classes_values[0]], y_pred_right_node_counts[y_classes_values[1]] ]
            #right_node_impurity = self.impurity_func(right_node_y_true_array)
            right_node_impurity = impurity_measure_with_sample_weight(y_class0_value=y_classes_values[0], y_class1_value=y_classes_values[1], y=y_pred_right_node_list, sample_weight=sample_weight[x > this_x_cutoff_value], impurity_func=self.impurity_func)
            right_node_n = sum(y_pred_right_node_counts.values())

            this_y_true_split_array = [left_node_y_true_array, right_node_y_true_array]

            total_n = left_node_n + right_node_n
            if total_n != len(y_true):
                raise ValueError("internal inconsistency")

            this_weighted_impurity = (left_node_impurity * left_node_n / total_n) + (right_node_impurity * right_node_n / total_n)
            this_information_gain = before_split_impurity - this_weighted_impurity

            if self.verbose:
                print(f"#{value_i:3d}: x_cutoff_value = {this_x_cutoff_value: .3f}, impurity = {this_weighted_impurity:.3f}, information_gain = {this_information_gain:.3f}, split_y_true_array = {this_y_true_split_array}")

            if this_weighted_impurity < best_impurity:
                best_impurity = this_weighted_impurity
                best_information_gain = this_information_gain
                best_x_cutoff_value = this_x_cutoff_value
                best_y_true_split_array = this_y_true_split_array

            if best_impurity == 0: # a perfect split was found
                break

        return best_x_cutoff_value, best_impurity, best_information_gain, best_y_true_split_array

    def find_best_split_across_all_features(self, X, y_true, sample_weight=None):

        if type(X) in [pd.DataFrame, pd.Series]:
            X = X.to_numpy()

        if type(y_true) in [pd.DataFrame, pd.Series]:
            y_true = y_true.to_numpy()

        if sample_weight is None:
            sample_weight = np.ones(shape=(len(y_true),))

        best_impurity = float('Inf')

        n_samples = X.shape[0]
        n_features = X.shape[1]

        if self.features_indices_actually_used == 'all':
            features_indices_actually_used = range(n_features)
        else:
            features_indices_actually_used = self.features_indices_actually_used

        for this_feature_i in features_indices_actually_used:
            x_cutoff_value, impurity, information_gain, y_true_split_array = self.find_best_split_in_one_specific_feature(x=X[:,this_feature_i], y_true=y_true, sample_weight=sample_weight)
            if self.verbose:
                print(f"feature # {this_feature_i: 2d}, x_cutoff_value = {x_cutoff_value: .3f}, impurity = {impurity:.3f}, information_gain = {information_gain:.3f}, y_true_split_array = {y_true_split_array}")
            if impurity == 0:  # a perfect split was found
                best_split_feature_i = this_feature_i
                best_x_cutoff_value, best_impurity, best_information_gain, best_y_true_split_array = x_cutoff_value, impurity, information_gain, y_true_split_array
                break
            elif impurity < best_impurity:
                best_split_feature_i = this_feature_i
                best_x_cutoff_value, best_impurity, best_information_gain, best_y_true_split_array = x_cutoff_value, impurity, information_gain, y_true_split_array

        return best_split_feature_i, best_x_cutoff_value, best_impurity, best_information_gain, best_y_true_split_array

    def fit(self, X, y, depth=0, sample_weight=None):

        y_true = y

        # init
        if depth == 0:
            from sortedcontainers import SortedSet
            y_classes_values = list(SortedSet(y_true))
            if len(y_classes_values) != 2:
                raise ValueError("y must be binary")
            else:
                self.y_class0_value = y_classes_values[0]
                self.y_class1_value = y_classes_values[1]

        if sample_weight is None:
            sample_weight = np.ones(shape=(len(y_true),))

        from collections import Counter

        if self.verbose:
            print(f"depth={depth}")

        if type(X) in [pd.DataFrame, pd.Series]:
            X = X.to_numpy()

        if type(y_true) in [pd.DataFrame, pd.Series]:
            y_true = y_true.to_numpy()

        curr_sample_size = len(y_true)
        if curr_sample_size == 0:  # no data left to be split
            return None

        from sortedcontainers import SortedDict
        curr_y_distribution = SortedDict(Counter(y_true))

        #curr_impurity = self.impurity_func(list(Counter(y_true).values()))
        curr_impurity = impurity_measure_with_sample_weight(y_class0_value=self.y_class0_value, y_class1_value=self.y_class1_value, y=y_true, sample_weight=sample_weight, impurity_func=self.impurity_func)

        if curr_impurity == 0 or depth >= self.max_depth: # curr_impurity = 0 means already perfect, no need to split
            curr_node = decision_tree_classifier_node(curr_depth = depth, curr_impurity = curr_impurity, curr_sample_size = curr_sample_size, curr_y_distribution = curr_y_distribution, best_split_feature_i = None, best_x_cutoff_value = None, y_class0_value=self.y_class0_value, y_class1_value=self.y_class1_value)
            return curr_node

        best_split_feature_i, best_x_cutoff_value, best_impurity, best_information_gain, best_y_true_split_array = self.find_best_split_across_all_features(X=X, y_true=y_true, sample_weight=sample_weight)
        left_rows = X[:, best_split_feature_i] <= best_x_cutoff_value
        right_rows = X[:, best_split_feature_i] > best_x_cutoff_value
        X_left, X_right = X[left_rows], X[right_rows]
        y_true_left, y_true_right = y_true[left_rows], y_true[right_rows]
        sample_weight_left, sample_weight_right = sample_weight[left_rows], sample_weight[right_rows]
        curr_node = decision_tree_classifier_node(curr_depth = depth, curr_impurity = curr_impurity, curr_sample_size = curr_sample_size, curr_y_distribution = curr_y_distribution, best_split_feature_i = best_split_feature_i, best_x_cutoff_value = best_x_cutoff_value, y_class0_value=self.y_class0_value, y_class1_value=self.y_class1_value)
        # adding left and right children nodes into the node dict
        curr_node.left  = self.fit( X=X_left,  y=y_true_left,  sample_weight=sample_weight_left,  depth=depth+1)
        curr_node.right = self.fit( X=X_right, y=y_true_right, sample_weight=sample_weight_right, depth=depth+1)
        #print(parent_node)

        if depth == 0:
            self.root_node = curr_node
            return self # return the fitted estimator
        else:
            return curr_node

    def _order(self, curr_node, type="Inorder"):
        # Recursive travesal
        if curr_node:
            if type == "Inorder":
                return_dict = {}
                return_dict['left'] = self._order(curr_node.left, type=type)
                return_dict['curr'] = curr_node.to_dict()
                return_dict['right'] = self._order(curr_node.right, type=type)
                return return_dict
            if type == "Preorder":
                return_dict = {}
                return_dict['curr'] = curr_node.to_dict()
                return_dict['left'] = self._order(curr_node.left, type=type)
                return_dict['right'] = self._order(curr_node.right, type=type)
                return return_dict
            if type == "Postorder":
                return_dict = {}
                return_dict['left'] = self._order(curr_node.left, type=type)
                return_dict['right'] = self._order(curr_node.right, type=type)
                return_dict['curr'] = curr_node.to_dict()
                return return_dict

    def order(self, type="Inorder"):
        return self._order(curr_node=self.root_node, type=type)

    def _predict(self, one_X_row, proba=False):
        curr_node = self.root_node
        while curr_node.best_x_cutoff_value: # not a leaf node yet
            if one_X_row[ curr_node.best_split_feature_i ] <= curr_node.best_x_cutoff_value:
                curr_node = curr_node.left  # but what if curr_node.left = None?
            else:
                curr_node = curr_node.right # but what if curr_node.right = None?
        # arriving at a leaf node now
        if proba:
            return np.array([1-curr_node.y_class1_prob, curr_node.y_class1_prob])
        else:
            return np.array([curr_node.y_dominant_class])

    def predict(self, X, proba=False):
        if type(X) in [pd.DataFrame, pd.Series]:
            X = X.to_numpy()

        n_rows = X.shape[0]
        if proba:
            prediction = np.zeros(shape=(n_rows, 2))
        else:
            prediction = np.zeros(shape=(n_rows,))

        for this_row_i in range(n_rows):
            prediction[this_row_i] = self._predict(X[this_row_i,:], proba=proba)
        return prediction

    def predict_proba(self, X):
        return self.predict(X, proba=True)

    def score(self, X_test, y_test):
        if type(X_test) in [pd.DataFrame, pd.Series]:
            X_test = X_test.to_numpy()
        if type(y_test) in [pd.DataFrame, pd.Series]:
            y_test = y_test.to_numpy()
        from sklearn.metrics import confusion_matrix
        cm = confusion_matrix(y_test, self.predict(X_test))
        accuracy = np.trace(cm) / np.sum(cm)
        return accuracy

def decision_tree_classifier(*args, **kwargs):
    """
    """
    return DecisionTreeClassifier(*args, **kwargs)

#######################################################################################################################################

class decision_tree_regressor_node(object):
    def __init__(self, X, y, subset_sample_indices, min_samples_leaf=5, max_depth=None, curr_depth=0):
        """
        min_samples_leaf: The minimum number of samples required to be at a leaf node.
        """
        self.X = X
        self.y = y
        self.subset_sample_indices = subset_sample_indices
        self.min_samples_leaf = min_samples_leaf
        self.max_depth = max_depth
        self.curr_depth = curr_depth
        self.n_samples = len(subset_sample_indices)
        self.n_features = X.shape[1]
        self.predicted_value = np.mean(y[subset_sample_indices]) # the decision (prediction) is based on the value the node holds.
        # best_after_split_purity_score, the lower the better, meaning more homogeneous elements within left and right nodes
        self.best_after_split_purity_score = float('inf') # the score indicates how effect the split was. It will be the weighted sum of the variance in the left and right nodes. leaf nodes do not have a score, thus set to infinity.
        if self.max_depth is None:
            self.find_best_feature_to_split()
        elif self.curr_depth < self.max_depth:
            self.find_best_feature_to_split()

    def find_best_feature_to_split(self):
        for feature_index in range(self.n_features):
            self.check_if_better_split_feature_and_its_value(feature_index)
        if self.is_leaf_node:
            return
        best_split_feature_x_series = self.best_split_feature_x_series
        best_split_left_node_series_indices  = np.nonzero(best_split_feature_x_series <= self.best_split_feature_value)[0]
        best_split_right_node_series_indices = np.nonzero(best_split_feature_x_series >  self.best_split_feature_value)[0]
        # recursively find all the left and right children nodes
        self.left  = decision_tree_regressor_node(X=self.X, y=self.y, subset_sample_indices=self.subset_sample_indices[best_split_left_node_series_indices],  min_samples_leaf=self.min_samples_leaf, max_depth=self.max_depth, curr_depth=self.curr_depth+1)
        self.right = decision_tree_regressor_node(X=self.X, y=self.y, subset_sample_indices=self.subset_sample_indices[best_split_right_node_series_indices], min_samples_leaf=self.min_samples_leaf, max_depth=self.max_depth, curr_depth=self.curr_depth+1)

    def check_if_better_split_feature_and_its_value(self, feature_index):
        """
        try to see if this feature and a certain value of the feature is a better split;
        criterion: a lower "weighted averages of the standard deviations", which is equivalent to minimizing RMSE.
        """
        specific_x_series = self.X[self.subset_sample_indices, feature_index]

        for this_row in range(self.n_samples):
            specific_x_value_in_this_row = specific_x_series[this_row]
            left_node_series_indices  = specific_x_series <= specific_x_value_in_this_row # left_node  holds those in specific_x_series smaller than (or equal to) specific_x_value_in_this_row
            right_node_series_indices = specific_x_series >  specific_x_value_in_this_row # right_node holds those in specific_x_series larger than                specific_x_value_in_this_row
            n_samples_in_left_node  = left_node_series_indices.sum() # a shorthand way to count
            n_samples_in_right_node = right_node_series_indices.sum()
            # the leaf nodes must have at least n = "min_samples_left" samples
            if (n_samples_in_right_node < self.min_samples_leaf) or (n_samples_in_left_node < self.min_samples_leaf):
                continue

            # this_row is now a candidate as it satisfies the condition that the leaf nodes must have at least n = "min_samples_left" samples
            curr_after_split_purity_score = self.find_after_split_purity_score(left_node_series_indices, right_node_series_indices)
            if curr_after_split_purity_score < self.best_after_split_purity_score:
                # a better split has been found
                self.best_split_feature_index = feature_index
                self.best_after_split_purity_score = curr_after_split_purity_score
                self.best_split_feature_value = specific_x_value_in_this_row

    def find_after_split_purity_score(self, left_node_indices, right_node_indices):
        """
        the weighted sum of the variance of y within left and right nodes
        """
        y_subset = self.y[self.subset_sample_indices]
        left_node_variance  = y_subset[left_node_indices].var()
        right_node_variance = y_subset[right_node_indices].var()
        n_samples_in_left_node  = left_node_indices.sum()
        n_samples_in_right_node = right_node_indices.sum()
        return (left_node_variance * n_samples_in_left_node) + (right_node_variance * n_samples_in_right_node)

    @property
    def best_split_feature_x_series(self):
        return self.X[self.subset_sample_indices, self.best_split_feature_index]

    @property
    def is_leaf_node(self):
        return self.best_after_split_purity_score == float('inf')

    def predict(self, X_test):
        return np.array([self.predict_row(X_test_specific_row) for X_test_specific_row in X_test])

    def predict_row(self, X_test_specific_row):
        if self.is_leaf_node:
            return self.predicted_value
        child_node = self.left if X_test_specific_row[self.best_split_feature_index] <= self.best_split_feature_value else self.right
        return child_node.predict_row(X_test_specific_row)

class decision_tree_regressor_from_scratch(object):

    def __init__(self, min_samples_leaf=5, max_depth=None):
        """
        min_samples_leaf: The minimum number of samples required to be at a leaf node.
        """
        self.X_train = None
        self.y_train = None
        self.DT_root_node = None
        self.min_samples_leaf = min_samples_leaf
        self.max_depth = max_depth

    def fit(self, X, y):
        self.X_train = X
        self.y_train = y
        if type(self.X_train) in [pd.DataFrame, pd.Series]:
            self.X_train = self.X_train.to_numpy()
        if type(self.y_train) in [pd.DataFrame, pd.Series]:
            self.y_train = self.y_train.to_numpy()
        n_samples = len(self.y_train)
        self.DT_root_node = decision_tree_regressor_node(X=self.X_train, y=self.y_train, subset_sample_indices=np.arange(n_samples), min_samples_leaf=self.min_samples_leaf, max_depth=self.max_depth)
        return self

    def predict(self, X):
        X_test = X
        if type(X_test) in [pd.DataFrame, pd.Series]:
            X_test = X_test.to_numpy()
        return self.DT_root_node.predict(X_test)

def decision_tree_regressor(*args, **kwargs):
    """
    """
    return DecisionTreeRegressor(*args, **kwargs)

#######################################################################################################################################

def _demo(dataset="Social_Network_Ads", classifier_func="decision_tree"): # DT: decision_tree
    """
    classifier_func: "decision_tree" or "DT", "GBM", "AdaBoost", "bagging"
    """
    from ..datasets import public_dataset
    from ..ensemble import bagging_classifier, random_forest_classifier, adaptive_boosting_classifier, gradient_boosting_classifier

    if dataset == "iris":
        data = public_dataset(name="iris")
        y_classes = ['setosa', 'versicolor', 'virginica']
        X = data[['sepal length (cm)', 'sepal width (cm)', 'petal length (cm)', 'petal width (cm)']]
        y = data['target']

    if dataset == "Social_Network_Ads":
        data = public_dataset(name="Social_Network_Ads")
        print(f"{data.head()}\n")
        del data['User ID']
        # Recode the data: Gender as Male
        mapper = {'Male': 1, 'Female': 0}
        data['Male'] = data['Gender'].map(mapper)
        # pairplot
        import seaborn as sns
        sns.pairplot(data, hue="Purchased", markers=["o", "s"])
        import matplotlib.pyplot as plt
        plt.show()
        # X and y
        X = data[['Age', 'EstimatedSalary']] # dropping Male to simplify the analysis
        y = data['Purchased']
        y_classes = ['not_purchased (y=0)', 'purchased (y=1)']

    if dataset == "bank_note_authentication":
        data = public_dataset(name="bank_note_authentication")
        y_classes = ['genuine (y=0)', 'forged (y=1)']
        X = data[['variance', 'skewness', 'curtosis', 'entropy']]
        y = data['class']
        import seaborn as sns
        sns.pairplot(data, hue="class", markers=["o", "s"])

    from sklearn.model_selection import train_test_split, GridSearchCV
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.25, random_state=123)

    # Feature scaling
    from sklearn.preprocessing import StandardScaler
    # scaler = StandardScaler()
    # X_train = scaler.fit_transform(X_train) # fit to the data and then transform it
    # X_test = scaler.transform(X_test) # 	uses a previously computed mean and std to scale the data

    # create pipeline
    from sklearn.pipeline import Pipeline

    ########################################################################################################################
    if classifier_func in ["decision_tree", "DT"]:
        pipeline = Pipeline(steps=[('scaler', StandardScaler(with_mean=True, with_std=True)),
                                   ('classifier', decision_tree_classifier(max_depth=1, random_state=123)),  # default criterion = 'gini'
                                   ])

        # pipeline parameters to tune
        hyperparameters={
            'scaler__with_mean': [True],
            'scaler__with_std': [True],
            'classifier__criterion': ("gini", "entropy",),
            'classifier__max_depth': range(1, 15),
        }

        model_name = "Decision Tree"

    ########################################################################################################################
    if classifier_func == "bagging":
        pipeline = Pipeline(steps=[('scaler', StandardScaler(with_mean=True, with_std=True)),
                                   ('classifier', bagging_classifier(random_state=123)),
                                   ])

        # pipeline parameters to tune
        hyperparameters = {
            'scaler__with_mean': [True],
            'scaler__with_std': [True],
        }

        model_name = "Bagging"

    ########################################################################################################################
    if classifier_func in ["random_forest"]:
        pipeline = Pipeline(steps=[('scaler', StandardScaler(with_mean=True, with_std=True)),
                                   ('classifier', random_forest_classifier(max_depth=1, random_state=123)),  # default criterion = 'gini'
                                   ])

        # pipeline parameters to tune
        hyperparameters={
            'scaler__with_mean': [True],
            'scaler__with_std': [True],
            'classifier__criterion': ("gini", "entropy",),
            'classifier__max_depth': range(1, 10),
        }

        model_name = "Random Forest"

    ########################################################################################################################
    if classifier_func == "AdaBoost":
        pipeline = Pipeline(steps=[('scaler', StandardScaler(with_mean=True, with_std=True)),
                                   ('classifier', adaptive_boosting_classifier(random_state=123)),
                                   ])

        # pipeline parameters to tune
        hyperparameters = {
            'scaler__with_mean': [True],
            'scaler__with_std': [True],
        }

        model_name = "Adaptive Boosting"

    ########################################################################################################################
    if classifier_func == "GBM":
        pipeline = Pipeline(steps=[('scaler', StandardScaler(with_mean=True, with_std=True)),
                                   ('classifier', gradient_boosting_classifier(max_depth=1, random_state=123)),
                                   ])

        # pipeline parameters to tune
        hyperparameters = {
            'scaler__with_mean': [True],
            'scaler__with_std': [True],
            'classifier__criterion': ("friedman_mse", "mse", "mae"),
            'classifier__max_depth': range(1, 10),
        }

        model_name = "Gradient Boosting"

    ########################################################################################################################

    grid=GridSearchCV(
        pipeline,
        hyperparameters,  # parameters to tune via cross validation
        refit=True,       # fit using all data, on the best detected classifier
        n_jobs=-1,
        scoring='accuracy',
        cv=5,
    )

    # Training and predicting
    # classifier = DecisionTreeClassifier(max_depth=3) # default criterion = 'gini'
    # classifier = classifier.fit(X_train,y_train)
    classifier_grid=grid.fit(X_train, y_train)

    ########################################################################################################################
    if classifier_func in ["decision_tree", "DT", "random_forest", "GBM"]:
        criterion=classifier_grid.best_params_['classifier__criterion']
        max_depth=classifier_grid.best_params_['classifier__max_depth']
        print(
            f"Using a grid search and a {model_name} classifier, the best hyperparameters were found as following:\n"
            f"Step1: scaler: StandardScaler(with_mean={repr(classifier_grid.best_params_['scaler__with_mean'])}, with_std={repr(classifier_grid.best_params_['scaler__with_std'])});\n"
            f"Step2: classifier: {classifier_func}(criterion={repr(criterion)}, max_depth={repr(max_depth)}).\n")
        model_desc = f"{model_name} (criterion={repr(criterion)}, max_depth={repr(max_depth)})"

    if classifier_func in ["AdaBoost", "bagging"]:
        print(
            f"Using a grid search and a {model_name} classifier, the best hyperparameters were found as following:\n"
            f"Step1: scaler: StandardScaler(with_mean={repr(classifier_grid.best_params_['scaler__with_mean'])}, with_std={repr(classifier_grid.best_params_['scaler__with_std'])});\n"
            f"Step2: classifier: {classifier_func}().\n")
        model_desc = f"{model_name}"

    ########################################################################################################################

    y_pred=classifier_grid.predict(X_test)
    y_pred_score=classifier_grid.predict_proba(X_test)

    # Model evaluation
    from ..model_evaluation import plot_confusion_matrix, plot_ROC_and_PR_curves, visualize_classifier_decision_boundary_with_two_features
    plot_confusion_matrix(y_true=y_test, y_pred=y_pred, y_classes=y_classes)

    if dataset in ['Social_Network_Ads', 'bank_note_authentication']:
        plot_ROC_and_PR_curves(fitted_model=classifier_grid, X=X_test,
                            y_true=y_test, y_pred_score=y_pred_score[:, 1], y_pos_label=1, model_name=f"{model_name}")

    if dataset in ['Social_Network_Ads',]:
        visualize_classifier_decision_boundary_with_two_features(
            classifier_grid, X_train.to_numpy(), y_train.to_numpy(), y_classes, title=f"{model_desc} / training set", X1_lab='Age', X2_lab='Estimated Salary')
        visualize_classifier_decision_boundary_with_two_features(
            classifier_grid, X_test.to_numpy(),  y_test.to_numpy(),  y_classes, title=f"{model_desc} / testing set",  X1_lab='Age', X2_lab='Estimated Salary')

    # Plotting the tree
    if classifier_func in ["decision_tree", "DT"]:

        if dataset == 'iris':
            feature_cols = list(X.columns)
            #class_names = ['0', '1', '2']
            class_names = ['0=setosa', '1=versicolor', '2=virginica']

        if dataset == 'bank_note_authentication':
            feature_cols = list(X.columns)
            #class_names = ['0', '1']
            class_names = ['0=genuine', '1=forged']

        if dataset == 'Social_Network_Ads':
            feature_cols=['Age', 'EstimatedSalary']
            #class_names = ['0', '1']
            class_names = ['0=not purchased', '1=purchased']

        # Approach 1
        from dtreeviz.trees import dtreeviz
        viz = dtreeviz(classifier_grid.best_estimator_.steps[1][1], StandardScaler().fit_transform(X_train), y_train, target_name="target", feature_names=feature_cols, class_names=class_names)
        # print(type(viz)) # <class 'dtreeviz.trees.DTreeViz'>
        viz.view()

        # Approach 2
        from sklearn.tree import export_graphviz
        import io
        import pydotplus
        # from IPython.display import Image
        dot_data=io.StringIO()

        export_graphviz(classifier_grid.best_estimator_.steps[1][1], out_file=dot_data,
                        filled=True, rounded=True,
                        special_characters=True, feature_names=feature_cols, class_names=class_names)

        graph=pydotplus.graph_from_dot_data(dot_data.getvalue())
        image=graph.create_png()
        from PIL import Image
        Image.open(io.BytesIO(image)).show()

def demo(dataset="Social_Network_Ads", classifier_func="decision_tree"):
    """
    This function provides a demo of selected functions in this module.

    Required arguments:
        - dataset:         A string. Possible values: "Social_Network_Ads", "iris", "bank_note_authentication"
        - classifier_func: A string. Possible values: "decision_tree" or "DT", "random_forest", "bagging", "AdaBoost", "GBM"
    """

    available_datasets = ("Social_Network_Ads","iris","bank_note_authentication")
    available_classifier_functions = ("decision_tree", "DT", "random_forest", "bagging", "AdaBoost", "GBM",)

    if dataset in available_datasets and classifier_func in available_classifier_functions:
        return _demo(dataset = dataset, classifier_func = classifier_func)
    else:
        raise TypeError(f"either dataset [{dataset}] or classifier function [{classifier_func}] is not defined")

def demo_from_scratch(question_type="classification"):
    """
    "classification" or "regression"
    """
    if question_type not in ['classification', 'regression',]:
        raise ValueError('question_type should be either classification or regression')

    if question_type == "regression":
        from ..datasets import public_dataset
        df = public_dataset(name="boston")
        print(f"{df.head()}\n")
        feature_names = df.columns.drop('MEDV')
        X, y = df[feature_names], df['MEDV']

        for model in [decision_tree_regressor_from_scratch(min_samples_leaf=5), decision_tree_regressor(min_samples_leaf=5)]:
            print(f"------ model: {repr(model)} ------")
            model.fit(X, y)
            y_pred = model.predict(X)
            from ..model_evaluation import evaluate_continuous_prediction
            RMSE, R_squared = evaluate_continuous_prediction(y, y_pred)
            print(f"R_squared = {R_squared:.3f}, RMSE = {RMSE:.3f}")

    if question_type == "classification":
        impurity_measure = 'entropy'  # impurity_measure: 'entropy' or 'gini_impurity'
        max_depth = 2

        from ..datasets import public_dataset
        data = public_dataset('Social_Network_Ads')
        X = data[['Age', 'EstimatedSalary']]
        y = data['Purchased']
        y_classes = ['not_purchased (y=0)', 'purchased (y=1)']
        from sklearn.model_selection import train_test_split
        X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.25, random_state=123)

        DT_model = decision_tree_classifier_from_scratch(impurity_measure=impurity_measure, max_depth = max_depth)
        from sklearn.preprocessing import scale
        print(DT_model.find_best_split_in_one_specific_feature(x=scale(X_train['Age']), y_true=y_train))
        print(DT_model.find_best_split_across_all_features(X=scale(X_train), y_true=y_train))

        DT_model.fit(X_train, y_train)
        print(DT_model.order(type="Preorder"))
        print(f"\nAccuracy in predicting the target in the testing set: {DT_model.score(X_test, y_test)}")

        #print(DT_model.predict(X_train))
        #print(DT_model.predict_proba(X_train))
        from ..model_evaluation import plot_confusion_matrix, plot_ROC_and_PR_curves
        y_pred_score = DT_model.predict_proba(X_test)
        plot_confusion_matrix(y_true=y_test, y_pred=DT_model.predict(X_test), y_classes=y_classes)
        plot_ROC_and_PR_curves(fitted_model=DT_model, X=X_test, y_true=y_test, y_pred_score=y_pred_score[:,1], model_name = 'DT from scratch')

#
# References
#
# decision tree:
#
# 1. Gini impurity:
# https://towardsdatascience.com/gini-impurity-measure-dbd3878ead33
# http://www.learnbymarketing.com/481/decision-tree-flavors-gini-info-gain/
# https://towardsdatascience.com/gini-index-vs-information-entropy-7a7e4fed3fcb
# https://medium.com/analytics-steps/understanding-the-gini-index-and-information-gain-in-decision-trees-ab4720518ba8
#
# 2. Entropy:
# https://www.geeksforgeeks.org/gini-impurity-and-entropy-in-decision-tree-ml/
#
# 3. Gini vs. Entropy
# https://datascience.stackexchange.com/questions/10228/when-should-i-use-gini-impurity-as-opposed-to-information-gain-entropy
#
# 4. DT from scratch in Python:
# https://medium.com/@penggongting/implementing-decision-tree-from-scratch-in-python-c732e7c69aea
# https://towardsdatascience.com/decision-tree-from-scratch-in-python-46e99dfea775
# https://machinelearningmastery.com/implement-decision-tree-algorithm-scratch-python/
#
# 5. Decision Tree Regressor in Python
# https://levelup.gitconnected.com/building-a-decision-tree-from-scratch-in-python-machine-learning-from-scratch-part-ii-6e2e56265b19
#

# References
#
# Ensemble methods:
#
# 1. Methods evolution and comparison
# https://scikit-learn.org/stable/modules/ensemble.html#bagging-meta-estimator
# https://towardsdatascience.com/decision-tree-ensembles-bagging-and-boosting-266a8ba60fd9


import numpy as np
import matplotlib.pyplot as plt
import matplotlib.animation as animation
from pylab import rcParams
rcParams['figure.figsize'] = 16, 9

fig, ax = plt.subplots(1,1)
plt.axis('equal')

ax.set_xlim([0, 2*np.pi])
ax.set_ylim([-2.5, 2.5])

# sine function plot
x = np.linspace(0, 2*np.pi, 100)
y = np.sin(x)

# x axis path
ax.plot(3*x - 3, 0*y, linewidth=1, color='#dddddd')
# y axis path
ax.plot(0*x, 2.5*y, linewidth=1, color='#dddddd')

# unit circle path
ax.plot(np.cos(x), np.sin(x), linewidth=1)
# sine path
ax.plot(x + 1, np.sin(x), linewidth=1)

# ------ anim -------
sineLine, = ax.plot([], [], linewidth=4)
sineDot, = ax.plot([], [], 'o', color='#ff0000')

circleLine, = ax.plot([], [],linewidth=4)
circleDot, = ax.plot([], [], 'o', color='black')

def sineAnim(i):
    # sine anim
    sineLine.set_data(x[:i] + 1,y[:i])
    sineDot.set_data(x[i] + 1, y[i])
    # circle anim
    circleLine.set_data(np.cos(x[:i]), np.sin(x[:i]))
    circleDot.set_data(np.cos(x[i]), np.sin(x[i]))

anim = animation.FuncAnimation(fig, sineAnim, frames=len(x), interval=50)
# -------------

anim.save('sine-py-effect.mp4', writer='ffmpeg')

# plt.show()
# HTML(anim.to_html5_video())


import numpy as np
from scipy.stats.distributions import norm

"""
    Simulate from a Multivariate Normal Distribution

    Produces one or more samples from the specified multivariate normal distribution.

    B. D. Ripley (1987) Stochastic Simulation. Wiley. Page 98.

    Borrowed from R: mvrnorm {MASS}

    TODO: numba
"""
def mvrnorm(n, mu, Sigma, tol = 1e-6):
    p = len(mu)
    # TODO: add exception: (p, p) == Sigma.shape
    eigValues, eigVectors = np.linalg.eig(Sigma)
    # TODO: check positive definite: eigen values > tol * abs(eigV[0])
    X = np.array(norm.rvs(size = n * p)).reshape(n, p)
    eigValues[eigValues < 0] = 0
    eigValues = np.sqrt(eigValues)
    X = mu + np.dot(np.dot(eigVectors, np.diag(eigValues)), X.T).T
    return X


import numpy as np

class Loss(object):
    """Loss 计算损失
    """
    def loss(self, predicted, actual):
        raise NotImplementedError

    def grad(self, predicted, actual):
        raise NotImplementedError

class MAE(Loss):
    """平均绝对误差
    """
    def loss(self, predicted, actual):
        # m = predicted.shape[0]
        # return np.sum(np.abs(predicted - actual)) / m
        return np.mean(np.abs(predicted - actual))

    def grad(self, predicted, actual):
        m = predicted.shape[0]
        return np.sign(predicted - actual) / m

class MSE(Loss):
    """均方误差 (Mean squared error)
    """
    def loss(self, predicted, actual):
        m = predicted.shape[0]
        return np.sum((predicted - actual) ** 2) / (2 * m)

    def grad(self, predicted, actual):
        m = predicted.shape[0]
        return (predicted - actual) / m

class SoftmaxCrossEntropy(Loss):
    def loss(self, predicted, actual):
        m = predicted.shape[0]

        # softmax
        exps = np.exp(predicted - np.max(predicted))
        p = exps / np.sum(exps)

        # cross entropy loss
        nll = -np.log(np.sum(p * actual, axis=1))

        return np.sum(nll) / m

    def grad(self, predicted, actual):
        m = predicted.shape[0]
        grads = predicted - actual
        return grads / m


import numpy as np
import scipy as sp
import numpy.linalg
import scipy.linalg
from math import sqrt

from mpl_toolkits.mplot3d import Axes3D
from mpl_toolkits.mplot3d.art3d import Poly3DCollection
import matplotlib.pyplot as plt
import matplotlib.colors as colors
import matplotlib.cm as cm

###-----------TRIANGLE--INTERSECTION--ALGORITHM--------------------------
def get_plane_equation(V):
    """
    Given a 3x3 ndarray with each row a vertex in R^3,
    compute the equation for the plane the triangle lies in.
    Plane: N \dot X + d = 0
    """
    N = np.cross(V[1,:] - V[0,:], V[2,:] - V[0,:])
    d = -np.dot(N, V[0,:])
    return N, d

def check_same_side(V, N, d):
    """
    Given a 3x3 ndarray V with each row a vertex in R^3,
    and the equation for a plane in R^3.
    Determine if all three verticies are on the same side of the plane.
    """
    ### AS IS ALLOWS FOR POINTS TO LIE IN PLANE
    c = np.dot(V, N) + d
    if max(c) <= 0.0:
        return True, c
    elif min(c) >= 0.0:
        return True, c
    else:
        return False, c

def get_vertex_projections(D, V):
    """
    Given planar intersection direction D and 3x3 triablge vertices V,
    return the projection of each projection onto the line.
    """
    max_ind = np.argmax(abs(D))
    return V[:,max_ind]

def get_isolated_vertex(c):
    """
    Return a permutation of the indices index of c with the first entry
    corresponding to the value with a different signed value from the other two and a
    """
    if c[0]*c[1] >= 0.0:
        return [2, 1, 0]
    elif c[0]*c[2] >= 0.0:
        return [1, 2, 0]
    else:
        return [0, 1, 2]

def get_line_location(p0, p1, c0, c1):
    return p0 + (p1 - p0)*c0/(c0 - c1)

def check_interval_intersection(t1a, t1b, t2a, t2b):
    """
    Determine if the intervals (t1a, t1b) and (t2a, t12b) intersect.
    """
    if min(t1a, t1b) < t2a and max(t1a, t1b) > t2a:
        return True
    elif min(t1a, t1b) < t2b and max(t1a, t1b) > t2b:
        return True
    elif min(t2a, t2b) < t1b and max(t2a, t2b) > t1b:
        return True
    else:
        return False

def rescale_triangle(V, scale):
    """
    Take triangle V and rescale each point by a factor of scale
    toward the triangle's center of mass.
    """
    C = np.tile(V.mean(axis=0), (3,1))
    return C + scale*(V - C)

def triangle_intersection(V1, V2, scaling=1.0):
    """
    Check for intersection between two triangles sitting in R^3.
    V1 and V2 are 3x3 ndarrays with each row a vertex.
    """

    # Rescale triangles to avoid edge or corner touching.
    V1 = rescale_triangle(V1, scaling)
    V2 = rescale_triangle(V2, scaling)

    ### From Moller 1997
    # Compute plane equation of triangle 2.
    N2, d2 = get_plane_equation(V2)

    # Reject if all points of triangle 1 are on same side.
    same_side_1, c1 = check_same_side(V1, N2, d2)
    if same_side_1 == True:
        return False

    # Compute plane equation of triangle 1.
    N1, d1 = get_plane_equation(V1)

    # Reject if all points of triangle 2 are on same side.
    same_side_2, c2 = check_same_side(V2, N1, d1)
    if same_side_2 == True:
        return False

    # Compute intersection line and project onto largest axis.
    D = np.cross(N1, N2)
    P1 = get_vertex_projections(D, V1)
    P2 = get_vertex_projections(D, V2)

    # Compute the intervals for each triangle.
    c_ind_1 = get_isolated_vertex(c1)
    c_ind_2 = get_isolated_vertex(c2)
    t1a = get_line_location(P1[c_ind_1[0]],
                            P1[c_ind_1[1]],
                            c1[c_ind_1[0]],
                            c1[c_ind_1[1]])
    t1b = get_line_location(P1[c_ind_1[0]],
                            P1[c_ind_1[2]],
                            c1[c_ind_1[0]],
                            c1[c_ind_1[2]])
    t2a = get_line_location(P2[c_ind_2[0]],
                            P2[c_ind_2[1]],
                            c2[c_ind_2[0]],
                            c2[c_ind_2[1]])
    t2b = get_line_location(P2[c_ind_2[0]],
                            P2[c_ind_2[2]],
                            c2[c_ind_2[0]],
                            c2[c_ind_2[2]])

    # Intersect the intervals
    return check_interval_intersection(t1a, t1b, t2a, t2b)

def get_face(x, f):
    """
    Given a flattened configuaration vector x, and indices for a triangle's corners f,
    return a 3x3 ndarray with the triangles coordinates.
    """
    t = np.zeros((3,3))
    for k in range(3):
        t[k,:] = x[3*f[k]:3*f[k]+3]
    return t

###-----------OTHER--GEOMETRIC--ALGORITHMS------------------------------

def Kabsch(P, Q, err_tol=10**-12):
    """
    Given two sets of N 3D points with centroid zero, find the rotation matrix
    that optimally rotates P onto Q.
    http://en.wikipedia.org/wiki/Kabsch_algorithm
    """
    assert abs(max(P.mean(axis=0))) < err_tol, abs(max(P.mean(axis=0)))
    assert abs(max(Q.mean(axis=0))) < err_tol, abs(max(Q.mean(axis=0)))

    A = np.dot(P.T, Q)
    V, s, WT = numpy.linalg.svd(A)
    #V, s, WT = scipy.linalg.svd(A)
    #d = np.sign(numpy.linalg.det(np.dot(WT, V)))
    #d = np.sign(numpy.linalg.det(np.dot(WT.T, V.T)))
    I = np.eye(3)
    U = numpy.dot(numpy.dot(V,I),WT)
    if numpy.linalg.det(U) < 0:
        I[2,2] = -1.0
        U = numpy.dot(numpy.dot(V,I),WT)
    return U

###-----------TESTING--CODE---------------------------------------------

def random_triangle():
    return np.random.uniform(size=(3,3))

def plot_triangles(V1, V2):
    ax = Axes3D(plt.figure())
    scale = 1.0
    ax.set_xlim(0.0,scale)
    ax.set_ylim(0.0,scale)
    ax.set_zlim(0.0,scale)

    T1 = Poly3DCollection([V1])
    color = colors.rgb2hex(sp.rand(3))
    T1.set_facecolor(color)
    T1.set_edgecolor('k')
    ax.add_collection(T1)
    T2 = Poly3DCollection([V2])
    color = colors.rgb2hex(sp.rand(3))
    T2.set_facecolor(color)
    T2.set_edgecolor('k')
    ax.add_collection(T2)

    plt.show()

def test_code(animate=False):
    V1 = random_triangle()
    V2 = random_triangle()
    print triangle_intersection(V1, V2)
    plot_triangles(V1, V2)

    if animate == True:
        for i in range(len(face_inds)):
            side = []
            for j in range(len(face_inds[i])):
                side.append([verts[face_inds[i][j],0],verts[face_inds[i][j],1],verts[face_inds[i][j],2]])

            tri = Poly3DCollection([side])
            color = colors.rgb2hex(sp.rand(3))
            tri.set_facecolor(color)
            tri.set_edgecolor('k')

            ax.add_collection3d(tri)

        plt.show()

def sample_so3():
    """
    Return a uniformly sampled matrix.
    """
    z = sp.randn(3,3)
    q,r = numpy.linalg.qr(z)
    d = sp.diagonal(r)
    ph = d/sp.absolute(d)
    return sp.multiply(q,ph,q)

def sample_SO3():
    while True:
        R = sample_so3()
        if numpy.linalg.det(R) > 0:
            return R


import numpy as np
import pandas as pd
from scipy.sparse.linalg import svds
from matplotlib import pyplot as plt
from sklearn.metrics import mean_squared_error as mse

from scipy import io
file = "facesOlivetti.mat"
data = io.loadmat(file)

X =np.matrix(data["X"])

h = 112
w = 92

mean_face = np.mean(X, axis = 0)
X_sub = X - mean_face

k_value = 400
u, s, vt= np.linalg.svd(X_sub)

Xproj = X_sub @ vt.T

selected_face = 60
recon_err = []

fig, ax = plt.subplots(2,2)
ax[0,0].imshow(np.reshape(X_sub[selected_face, : ], ( w, h)).T, extent=[0,1,0,1],cmap = "gray")
ax[0,0].set_axis_off()
ax[0,0].set_title("Original Image No." + str(selected_face))

for i in range(1, k_value+1):

    X_reconst = Xproj[selected_face, : i ] * vt[ : i ,  : ]+ mean_face
    X_err = X_reconst -  X[selected_face , : ]
    X_mse = np.mean(np.square(X_err))
    recon_err.append(X_mse)

    if i == 10 :
        ax[0,1].imshow(np.reshape(X_reconst, ( w, h)).T, extent=[0,1,0,1],cmap = "gray")
        ax[0,1].set_axis_off()
        ax[0,1].set_title("Reconstructed Image with 10 bases")

    elif i == 100 :
        ax[1,0].imshow(np.reshape(X_reconst, ( w, h)).T, extent=[0,1,0,1],cmap = "gray")
        ax[1,0].set_axis_off()
        ax[1,0].set_title("Reconstructed Image with 100 bases")

ax[1,1].plot( recon_err)
ax[1,1].set_title("Reconstruction Error")

plt.show()


from math import sqrt
from numpy import array
from numpy import mean
from numpy import std
from sklearn.metrics import mean_squared_error
from forecasttime.utils import train_test_split, series_to_supervised
from forecasttime.model_dev import model_fit, model_predict

from matplotlib import pyplot

# root mean squared error or rmse
def measure_rmse(actual, predicted):
    return sqrt(mean_squared_error(actual, predicted))

# walk-forward validation for univariate data
def walk_forward_validation(name, data, n_test, cfg):
    predictions = list()
    # split dataset
    train, test = train_test_split(data, n_test)
    # fit model
    model = model_fit(name, train, cfg)
    # seed history with training dataset
    history = [x for x in train]
    # step over each time step in the test set
    for i in range(len(test)):
        # fit model and make forecast for history
        yhat = model_predict(name, model, history, cfg)
        # store forecast in list of predictions
        predictions.append(yhat)
        # add actual observation to history for the next loop
        history.append(test[i])
    # estimate prediction error
    error = measure_rmse(test, predictions)
    print(' > %.4f' % error)
    return error

# repeat evaluation of a config
def repeat_evaluate(name, data, config, n_test, n_repeats=30):
    # fit and evaluate the model n times
    scores = [walk_forward_validation(name, data, n_test, config) for _ in range(n_repeats)]
    return scores

# summarize model performance
def summarize_scores(name, scores):
    # print a summary
    scores_m, score_std = mean(scores), std(scores)
    print('%s: %.3f RMSE (+/- %.3f)' % (name, scores_m, score_std))
    # box and whisker plot
    pyplot.boxplot(scores)
    pyplot.show()


from sympy.integrals.quadrature import gauss_legendre

PRECISION = 20

def get_gauss_legendre_pairs(order):
    qps, weights = gauss_legendre(order, PRECISION)
    apprx_qps = [float(qp) for qp in qps]
    apprx_weights = [float(weight) for weight in weights]
    return apprx_qps, apprx_weights

class Quadrature:
    def compute(self, func):
        total = 0.0
        for qp, weight in zip(self.qps, self.weights):
            total += weight * func(*qp)
        return total

class GaussLegendre1D(Quadrature):
    def __init__(self, n_qps):
        qps, weights = get_gauss_legendre_pairs(n_qps)
        self.qps = [(qp,) for qp in qps]
        self.weights = weights
        self.max_poly_order = (2 * len(self.qps)) - 1

if __name__ == '__main__':
    gl2 = GaussLegendre1D(2)
    def test_function_int(func):
        print("Around Origin  :", gl2.compute(func))

    print("Line")
    test_function_int(lambda x : x)
    print()

    print("Quad")
    test_function_int(lambda x : x * x)
    print()

    print("Cube")
    test_function_int(lambda x : x * x * x)
    print()

    print("Quart") # 2nd order GLQ is not exact
    test_function_int(lambda x : x * x * x * x)


import numpy as np
import matplotlib.pyplot as plt

def get_pi(N):
    x = np.random.random(N)
    y = np.random.random(N)
    r = np.sqrt(x*x + y*y)
    N_in = len(r[np.where(r <= 1)])
    return (N_in/N)*4.0

N = 10000
N_trials = 10000
pis = np.zeros(N_trials)
for i in range(0,N_trials):
    pis[i] = get_pi(N)

print(pis.mean(), "+/-", pis.std())
plt.hist(pis, bins='fd', density=True)
plt.show()

#use guassian mu (mean) and sigma (std dev) to get evenly spaced


import numpy as np

def model_simple_gaussian(x, coefficients):
    return - 1 / (coefficients[0] * np.sqrt(2 * np.pi)) * np.exp(
        - 0.5 * ((np.array(x) - coefficients[1]) / coefficients[0]) ** 2) + coefficients[2]

def model_double_gaussian(x, coefficients):
    return - 1 / (coefficients[0] * np.sqrt(2 * np.pi)) * np.exp(
        - 1 / 2 * ((np.array(x) - coefficients[1]) / coefficients[0]) ** 2) \
           - 1 / (coefficients[2] * np.sqrt(2 * np.pi)) * np.exp(
        - 1 / 2 * ((np.array(x) - coefficients[3]) / coefficients[2]) ** 2) + coefficients[4]

def model_simple_expo(x, coefficients):
    return coefficients[0] * np.exp(coefficients[1] * np.array(x)) + coefficients[2]

def model_double_expo(x, coefficients):
    return coefficients[0] * np.exp(coefficients[1] * np.array(x)) + coefficients[2] * np.exp(
        coefficients[3] * np.array(x)) + coefficients[4]

def model_lorentz(x, coefficients):
    return - (2 / (np.pi * coefficients[1])) / (1 + ((np.array(x) - coefficients[0]) / (0.5 * coefficients[1])) ** 2) +\
           coefficients[2]

def model_double_lorentz(x, coefficients):
    return - (2 / (np.pi * coefficients[1])) / (1 + ((np.array(x) - coefficients[0]) / (0.5 * coefficients[1])) ** 2) \
           - (2 / (np.pi * coefficients[3])) / (1 + ((np.array(x) - coefficients[2]) / (0.5 * coefficients[3])) ** 2) \
           + coefficients[4]

def model_linear(x, coefficients):
    return coefficients[0] * np.array(x) + coefficients[1]

def model_simple_gaussian_emission(x, coefficients):
    return 1 / (coefficients[0] * np.sqrt(2 * np.pi)) * np.exp(- 1 / 2 * ((x - coefficients[1]) / coefficients[0]) ** 2)\
           + coefficients[2]

def model_double_gaussian_emission(x, coefficients):
    return 1 / (coefficients[0] * np.sqrt(2 * np.pi)) * np.exp(- 1 / 2 * ((x - coefficients[1]) / coefficients[0]) ** 2)\
           + 1 / (coefficients[2] * np.sqrt(2 * np.pi)) * np.exp(- 1 / 2 * ((x - coefficients[3]) / coefficients[2]) ** 2)\
           + coefficients[4]

def model_lorentz_emission(x, coefficients):
    return (2 / (np.pi * coefficients[1])) / (1 + ((x - coefficients[0]) / (0.5 * coefficients[1])) ** 2) + \
           coefficients[2]

def model_double_lorentz_emission(x, coefficients):
    return (2 / (np.pi * coefficients[1])) / (1 + ((x - coefficients[0]) / (0.5 * coefficients[1])) ** 2) + (
            2 / (np.pi * coefficients[3])) / (1 + ((x - coefficients[2]) / (0.5 * coefficients[3])) ** 2) + \
           coefficients[4]


# basic method
from math import pi, cos, sin, sqrt, atan
import numpy as np

def build_complex_value(VM, VA):
    '''
    Build complex number  by amplitude and phase angle, and the phase angle is rad radian.
    Args:
        (1) VM, amplitude.
        (2) VA, phase angle in rad.
    Rets:
        value, complex number with the format a+jb.
    '''
    value = VM * (cos(VA) + 1j * sin(VA))
    return value

def convert_deg_to_rad(angle):
    '''
    Convert angle from deg to rad.
    Args:
        angle, angle in deg.
    Rets:
        angle, angle in rad.
    '''
    angle = angle * pi / 180.0
    return angle

def convert_rad_to_deg(angle):
    '''
    Convert angle from rad to deg.
    Args:
        angle, angle in rad.
    Rets:
        angle, angle in deg.
    '''
    angle = angle * 180.0 / pi
    return angle

def get_complex_phase_angle(value):
    '''
    Get complex phase angle in rad.
    Args:
        value, complex.
    Rets:
        angle, phase angle in rad.
    '''
    angle = atan(value.imag / value.real)
    return angle

def convert_xy_to_dq(derta, Ax, Ay):
    '''
    Convert the value of the xy frame to the value of the dq frame.
    Args:
        (1) derta, angle in rad.
        (2)(3) Ax, Ay,  x-component and y-component.
    Rets:
        (1)(2) Ad, Aq, d-component and q-component.
    '''
    convert_mat = np.array([[sin(derta), -cos(derta)], [cos(derta), sin(derta)]])
    Axy_mat = np.array([[Ax], [Ay]])
    Adq_mat = np.dot(convert_mat, Axy_mat)
    Ad, Aq = Adq_mat[0, 0], Adq_mat[1, 0]
    return Ad, Aq

def composite_three_phase_vector(Fa):
    '''
    Calculate three phase asymmetry vector from sequence component.
    Args:
        Fa, tuple, sequence component, (Fa1, Fa2, Fa0), (positive, negative, zero).
    Rets:
        F, A. B, C three phase components, (Fa, Fb, Fc).
    '''
    (Fa1, Fa2, Fa0) = Fa
    a = -0.5 + sqrt(3) * 0.5j

    Fa = Fa1 + Fa2 + Fa0
    Fb = a ** 2 * Fa1 + a * Fa2 + Fa0
    Fc = a * Fa1 + a ** 2 * Fa2 + Fa0

    return (Fa, Fb, Fc)


import numpy as np
import scipy.linalg as linalg
import matplotlib.pyplot as plt

def estimateGaussian(X):
    #   [mu sigma2] = estimateGaussian(X),
    #   The input X is the dataset with each n-dimensional data point in one row
    #   The output is an n-dimensional vector mu, the mean of the data set
    #   and the variances sigma^2, an n x 1 vector

    m, n = X.shape
    # You should return these values correctly
    mu = np.zeros((n, 1))
    sigma2 = np.zeros((n, 1))
    # estimating mu - at this point it is an n-column vector
    # mu = (1/m)*sum(X,1);
    mu = np.mean(X, axis=0)
    # turn into n-rows vector
    mu = mu.T

    # estimating sigma^2 = std.dev.
    # normalizes with 1/N, instead of with 1/(N-1) in formula std (x) = 1/(N-1) SUM_i (x(i) - mean(x))^2
    # i.e. degrees of freedom = 0 (by default)
    sigma2 = np.var(X, axis=0)

    # turn into n-rows vector
    sigma2 = sigma2.T

    return mu, sigma2

def multivariateGaussian(X, mu, sigma2):
    #    p = MULTIVARIATEGAUSSIAN(X, mu, sigma2) Computes the probability
    #    density function of the examples X under the multivariate gaussian
    #    distribution with parameters mu and sigma2. If sigma2 is a matrix, it is
    #    treated as the covariance matrix. If sigma2 is a vector, it is treated
    #    as the \sigma^2 values of the variances in each dimension (a diagonal
    #    covariance matrix)
    k = len(mu)

    # turns 1D array into 2D array
    if sigma2.ndim == 1:
        sigma2 = np.reshape(sigma2, (-1,sigma2.shape[0]))

    if sigma2.shape[1] == 1 or sigma2.shape[0] == 1:
        sigma2 = linalg.diagsvd(sigma2.flatten(), len(sigma2.flatten()), len(sigma2.flatten()))

    # mu is unrolled (and transposed) here
    X = X - mu.reshape(mu.size, order='F').T

    p = np.dot(np.power(2 * np.pi, - k / 2.0), np.power(np.linalg.det(sigma2), -0.5) ) * \
        np.exp(-0.5 * np.sum(np.dot(X, np.linalg.pinv(sigma2)) * X, axis=1))

    return p

def computeNumericalGradient(J, theta):
    #   numgrad = COMPUTENUMERICALGRADIENT(J, theta) computes the numerical
    #   gradient of the function J around theta. Calling y = J(theta) should
    #   return the function value at theta.

    # Notes: The following code implements numerical gradient checking, and
    #        returns the numerical gradient.It sets numgrad(i) to (a numerical
    #        approximation of) the partial derivative of J with respect to the
    #        i-th input argument, evaluated at theta. (i.e., numgrad(i) should
    #        be the (approximately) the partial derivative of J with respect
    #        to theta(i).)
    numgrad = np.zeros(theta.shape)
    perturb = np.zeros(theta.shape)
    e = 1e-4

    for p in range(theta.size):
        # Set perturbation vector
        perturb.reshape(perturb.size, order="F")[p] = e
        loss1, _ = J(theta - perturb)
        loss2, _ = J(theta + perturb)
        # Compute Numerical Gradient
        numgrad.reshape(numgrad.size, order="F")[p] = (loss2 - loss1) / (2*e)
        perturb.reshape(perturb.size, order="F")[p] = 0

    return numgrad

def cofiCostFunc(params, Y, R, num_users, num_movies, num_features, lambda_var):
    #   [J, grad] = COFICOSTFUNC(params, Y, R, num_users, num_movies, ...
    #   num_features, lambda) returns the cost and gradient for the
    #   collaborative filtering problem.

    # Unfold the U and W matrices from params
    X = np.reshape(params[:num_movies*num_features], (num_movies, num_features), order='F')
    Theta = np.reshape(params[num_movies*num_features:], (num_users, num_features), order='F')

    # You need to return the following values correctly
    J = 0
    X_grad = np.zeros(X.shape)
    Theta_grad = np.zeros(Theta.shape)

    # X * Theta performed according to low rank matrix vectorization
    squared_error = np.power(np.dot(X,Theta.T) - Y,2)

    # for cost function, sum only i,j for which R(i,j)=1
    J = (1/2.) * np.sum(squared_error * R)

    # NOTE where filtering through R is applied:
    # 	at ( (X * Theta') - Y ) - NOT after multiplying by Theta
    # 	that means that for purposes of the gradient, we're only interested
    # 	in the errors/differences in i, j for which R(i,j)=1
    # NOTE also that even though we do a sum, we only do it over users,
    # 	so we still get a matrix
    X_grad = np.dot((np.dot(X, Theta.T) - Y) * R, Theta)
    # NOTE also that even though we do a sum, we only do it over movies,
    # 	so we still get a matrix
    Theta_grad = np.dot(((np.dot(X, Theta.T) - Y) * R).T, X)

    ### COST FUNCTION WITH REGULARIZATION
    # only add regularized cost to J now
    J = J + (lambda_var / 2.) * (np.sum(np.power(Theta, 2)) + np.sum(np.power(X, 2)))

    ### GRADIENTS WITH REGULARIZATION
    # only add regularization terms
    X_grad = X_grad + lambda_var*X
    Theta_grad = Theta_grad + lambda_var*Theta

    grad = np.concatenate((X_grad.reshape(X_grad.size, order='F'), Theta_grad.reshape(Theta_grad.size, order='F')))

    return J, grad

def checkCostFunction(lambda_var=0):
    #   CHECKCOSTFUNCTION(lambda_var) Creates a collaborative filering problem
    #   to check your cost function and gradients, it will output the
    #   analytical gradients produced by your code and the numerical gradients
    #   (computed using computeNumericalGradient). These two gradient
    #   computations should result in very similar values.

    # Set lambda_var
    # if not lambda_var or not 'lambda_var' in locals():
    #     lambda_var = 0

    ## Create small problem
    X_t = np.random.rand(4, 3)
    Theta_t = np.random.rand(5, 3)

    # Zap out most entries
    Y = np.dot(X_t, Theta_t.T)
    Y[np.random.rand(Y.shape[0], Y.shape[1]) > 0.5] = 0
    R = np.zeros(Y.shape)
    R[Y != 0] = 1

    ## Run Gradient Checking
    X = np.random.randn(X_t.shape[0], X_t.shape[1])
    Theta = np.random.randn(Theta_t.shape[0], Theta_t.shape[1])
    num_users = Y.shape[1]
    num_movies = Y.shape[0]
    num_features = Theta_t.shape[1]

    params = np.concatenate((X.reshape(X.size, order='F'), Theta.reshape(Theta.size, order='F')))

    # Short hand for cost function
    def costFunc(p):
        return cofiCostFunc(p, Y, R, num_users, num_movies, num_features, lambda_var)

    numgrad = computeNumericalGradient(costFunc, params)

    cost, grad = cofiCostFunc(params, Y, R, num_users, num_movies, num_features, lambda_var)

    print(np.column_stack((numgrad, grad)))
    print('The above two columns you get should be very similar.\n' \
             '(Left-Your Numerical Gradient, Right-Analytical Gradient)')

    diff = np.linalg.norm(numgrad-grad)/np.linalg.norm(numgrad+grad)
    print('If your backpropagation implementation is correct, then \n' \
             'the relative difference will be small (less than 1e-9). ' \
             '\nRelative Difference: {:e}'.format(diff))

def selectThreshold(yval, pval):
    #   [bestEpsilon bestF1] = SELECTTHRESHOLD(yval, pval) finds the best
    #   threshold to use for selecting outliers based on the results from a
    #   validation set (pval) and the ground truth (yval).

    bestEpsilon = 0
    bestF1 = 0
    F1 = 0

    stepsize = (max(pval) - min(pval)) / 1000
    for epsilon in np.arange(min(pval), max(pval), stepsize):
        # Note: You can use predictions = (pval < epsilon) to get a binary vector
        #       of 0's and 1's of the outlier predictions

        # make anomaly predictions based on which values of pval are less than epsilon
        #   i.e. which examples from the cross-validation set have a very low p(x)
        cvPredictions = pval < epsilon

        # calculate F1 score, starting by calculating true positives(tp)
        # false positives (fp) and false negatives (fn)

        # true positives is the intersection between
        #   our positive predictions (cvPredictions==1) and positive ground truth values (yval==1)
        tp = np.sum(np.logical_and((cvPredictions == 1), (yval == 1)).astype(float))

        # false positives are the ones we predicted to be true (cvPredictions==1) but weren't (yval==0)
        fp = np.sum(np.logical_and((cvPredictions == 1), (yval == 0)).astype(float))

        # false negatives are the ones we said were false (cvPredictions==0) but which were true (yval==1)
        fn = np.sum(np.logical_and((cvPredictions == 0), (yval == 1)).astype(float))

        # compute precision, recall and F1
        precision = tp / (tp + fp)
        recall = tp / (tp + fn)
        F1 = (2 * precision * recall) / (precision + recall)

        if F1 > bestF1:
            bestF1 = F1
            bestEpsilon = epsilon

    return bestEpsilon, bestF1

def visualizeFit(X, mu, sigma2):
    #   VISUALIZEFIT(X, p, mu, sigma2) This visualization shows you the
    #   probability density function of the Gaussian distribution. Each example
    #   has a location (x1, x2) that depends on its feature values.

    X1,X2 = np.meshgrid(np.arange(0, 35.1, 0.5), np.arange(0, 35.1, 0.5))
    Z = multivariateGaussian(np.column_stack((X1.reshape(X1.size, order='F'), X2.reshape(X2.size, order='F'))), mu, sigma2)
    Z = Z.reshape(X1.shape, order='F')

    plt.plot(X[:, 0], X[:, 1],'bx', markersize=13, markeredgewidth=1)

    plt.hold(True)
    # Do not plot if there are infinities
    if (np.sum(np.isinf(Z)) == 0):
        plt.contour(X1, X2, Z, np.power(10,(np.arange(-20, 0.1, 3)).T))

    plt.hold(False)

def loadMovieList():
    #   movieList = GETMOVIELIST() reads the fixed movie list in movie.txt
    #   and returns a cell array of the words in movieList.

    ## Read the fixed movieulary list
    with open("ml/ex8/data/movie_ids.txt") as movie_ids_file:

        # Store all movies in movie list
        n = 1682  # Total number of movies

        movieList = [None]*n
        for i, line in enumerate(movie_ids_file.readlines()):
            movieName = line.split()[1:]
            movieList[i] = " ".join(movieName)

    return movieList

def normalizeRatings(Y, R):
    #   [Ynorm, Ymean] = NORMALIZERATINGS(Y, R) normalized Y so that each movie
    #   has a rating of 0 on average, and returns the mean rating in Ymean.

    m, n = Y.shape
    Ymean = np.zeros((m, 1))
    Ynorm = np.zeros(Y.shape)
    for i in range(m):
        idx = R[i, :] == 1
        Ymean[i] = np.mean(Y[i, idx])
        Ynorm[i, idx] = Y[i, idx] - Ymean[i]

    return Ynorm, Ymean


import numpy as np
import matplotlib.pyplot as plt

tab: str = "\t"

def plot_hist(data, title='histogram', x_label='bin center', y_label='frequency', c='blue', tc='red',
              is_z_score=False, bins=20):
    plt.xlabel(x_label)
    plt.ylabel(y_label)

    mean_plt = np.mean(data)
    std_plt = np.std(data)

    if is_z_score:
        plt.title(title)

        # plot histogram and return counts containing frequency and bin center data
        plt.hist(data, bins, color=c)
    else:
        plt.title(title)
        # plot histogram and return counts containing frequency and bin center data
        counts = plt.hist(data, bins, color=c)
        # text to be displayed on fig
        str_m = f'mean: {round(mean_plt, 3)}, std: {round(std_plt, 3)}'
        # location of text aligned with second bin center and half max frequency
        plt.annotate(str_m, [counts[1][1], np.max(counts[0]) / 2], color=tc)

def plot_norm_prob(data, y_label='Data', title='Normal Probability Plot'):
    a = np.random.randn(len(data))
    plt.scatter(np.sort(a), np.sort(data))
    plt.xlabel('Standard Normal')
    plt.ylabel(y_label)
    plt.title(title)

def plot_cdf(data, title="CDF", x_label='x', y_label='prob(value $\leq$ x)'):
    x_cdf = np.sort(data)
    size = len(data)
    y_cdf = (1 + np.arange(size)) / size
    plt.plot(x_cdf, y_cdf)
    plt.xlabel(x_label)
    plt.ylabel(y_label)  # between $$ is latex math mode
    plt.title(title)
    return x_cdf, y_cdf

def my_hist_data(data, bins=20):
    counts = plt.hist(data, bins)
    plt.close()
    bin_center_hist = (counts[1][1:] + counts[1][:-1]) / 2
    freq = counts[0] / len(data)
    return bin_center_hist, freq

def z_score(data):
    m = np.mean(data)
    s = np.std(data)

    return (data - m) / s

def corr_compute(x, y, x_label='x', y_label='y', modified=""):
    # corr x / y
    # compute pearson correlation coefficient
    pcc = x.corr(y, method='pearson')
    # compute spearman correlation coefficient
    scc = x.corr(y, method='spearman')
    # compute kendal correlation coefficient
    kcc = x.corr(y, method='kendall')

    corr_data = f"{tab * 4}Corelation between {x_label} and {y_label} {modified}\n" + \
                f"{tab * 5}Pearson CC = {round(pcc, 2)}\n" + \
                f"{tab * 5}Spearman CC = {round(scc, 2)}\n" + \
                f"{tab * 5}Kendall CC = {round(kcc, 2)}\n"
    return corr_data

def basic_stats_string(x_stats, x_label='x'):
    # output string
    stats_string = f"{tab * 4}{x_label} Statistics:\n" \
                   f"{tab * 5}Standard Deviation: {round(x_stats.loc['std'], 2)}\n" + \
                   f"{tab * 5}Mean: {round(x_stats.loc['mean'], 2)}\n" + \
                   f"{tab * 5}Min: {round(x_stats.loc['min'], 2)}\n" + \
                   f"{tab * 5}Max: {round(x_stats.loc['max'], 2)}\n"
    return stats_string

def data_set_stats(pokemon_data_set, name="Pokemon", total_attr=20, study_attr=9, cols=[], z_cols=[]):
    pokemon_set_data = f"{tab * 4}{name} Characteristics\n" + \
                       f"{tab * 5}Number of {name}: {len(pokemon_data_set)}\n" \
                       f"{tab * 5}Number of Attributes Total: {total_attr}\n" + \
                       f"{tab * 5}Number of Attributes in the Study: {study_attr}\n\n\n" + \
                       f"{tab * 10}- - - Attribute in study Information - - -\n\n"
    for col in cols:
        if col in z_cols:
            pokemon_set_data += f"{tab * 8}* {col}\n"

    return pokemon_set_data

def trim_stats(initial, trimmed, name='Pokemon', height_low=0, height_high=0, weight_low=0, weight_high=0, trimmed_height=False, trimmed_weight=False):
    trim_data = f"{tab * 4}Trimmed extreme Height(m) and Weight(kg) from {name} data!\n"

    if trimmed_height:
        trim_data += f'{tab * 5}min height = {height_low}(m), max height = {height_high}(m)\n'
    if trimmed_weight:
        trim_data += f'{tab * 5}min weight = {weight_low}(kg), max weight = {weight_high}(kg)\n'

    trim_data += f'{tab * 5}Removed {initial - trimmed} / {initial} = ' + \
                 f'{round(((initial - trimmed) / initial) * 100, 2)}%' \
                 f' of cases\n' + \
                 f'{tab * 5}{trimmed} Pokemon Remain\n'

    return trim_data


"""
    AUTO-CORRELATION

	This code is used to analyze different properties of auto-correltion

    Code by:- GSS

"""

# Let us first import all the required libraries
import numpy as np
import math
from scipy import signal
print("**************************************************************************")

#let us consider a discrete signal as shown below
x=[1,4,2,3]

#auto-coreltion function using inbuilt function in scipy
rxx=signal.correlate(x,x)
print("correlated signal:",rxx)

rxxcenter=math.floor(len(rxx)/2)

print('\nCenter position:',rxxcenter)
#deriving 'rxxleft' and 'rxxright' arrays

rxxleft=[]
rxxright=[]

for i in range(rxxcenter+1):
    rxxleft.append(rxx[rxxcenter-i])
print('\nrxx[rxxcenter]=',rxx[rxxcenter])

i=0

for i in range(rxxcenter+1):
    rxxright.append(rxx[rxxcenter+i])
print('\nrxxleft=',rxxleft)
print('\nrxxright=',rxxright)

# properties to be proved

if(rxxleft == rxxright):
    print("******************************************************")
    print("symmetry property proved")
    print("******************************************************")

def check(rxx, val):
    for z in rxx:
        if val< z:
            return False
    return True

if(check(rxx,rxx[rxxcenter])==True):
    print("******************************************************")
    print("Upper bound property Rxx(0)>=Rxx(k) is verified")
    print("******************************************************")
##########################################################################
#or 																	##
#																		##
#i=0																	##
# cntr=0																##
# for i in range(len(rxx)):												##
#     if(rxx[rxxcenter-1]<rxx[i]):										##
#         cntr=cntr+1													##
#																		##
# if (cntr==0):															##
#     print("******************************************************")	##
#     print("Upper bound property Rxx(0)>=Rxx(k) is verified")			##
#     print("******************************************************")	##
##########################################################################
y=np.square(x)
mean=sum(y)

if(rxx[rxxcenter]==mean):
    print("******************************************************")
    print("Mean square value=correlation value at zero shift")
    print("******************************************************")


#t83_1.py
#numpy Tutorial
#15-4-2016

import numpy as np

time = np.linspace(20, 145, 5)                 # time scale  #+ generates series from 20 to 145 in 5 steps, double
data = np.sin(np.arange(20)).reshape(5,4)      # 4 time-dependent series

#20 elements, 5 rows x 4 columns ... sinus of 0, 1, 2, 3, ... 19  -- sin(0), sin(1) ...

print(time)
print(data)

ind = data.argmax(axis=0)                   # index of the maxima for each series
# ahaaa ... vertical ...
print("axis=0")
print(ind)
#print(data[ind])

ind2 = data.argmax(axis=1)                   # index of the maxima for each series
# ahaaa ... vertical ...
print("axis=1")
print(ind2)

#Брои индексите ВЕРТИКАЛНО на това което показва с печатането!

#ind
#array([2, 0, 3, 1])
print("time_max... ind")
time_max = time[ ind]                       # times corresponding to the maxima
print(time_max)
data_max = data[ind, xrange(data.shape[1])] # => data[ind[0],0], data[ind[1],1]...
print(data_max)

print(type(data_max))
print(type(data))
# no -- errors
#print("time_max... ind2")
#time_max2 = time[ ind2]                       # times corresponding to the maxima
#print(time_max2)
#data_max2 = data[ind2, xrange(data.shape[1])] # => data[ind[0],0], data[ind[1],1]...
#print(data_max2)

#time_max
#array([  82.5 ,   20.  ,  113.75,   51.25])
# data_max
#array([ 0.98935825,  0.84147098,  0.99060736,  0.6569866 ])
#
# np.all(data_max == data.max(axis=0))
#True

import numpy as np
import matplotlib.pyplot as plt
# Build a vector of 10000 normal deviates with variance 0.5^2 and mean 2
mu, sigma = 2, 0.5
v = np.random.normal(mu,sigma,100)
# Plot a normalized histogram with 50 bins
plt.hist(v, bins=10, normed=1)       # matplotlib version (plot)
plt.show()

#v2 = np.linspace(20, 145, 0.35)
#v3 = np.linspace(10, 100, 1)
#v4 = np.ndarray(v2, 3)
mu, sigma = 4, 0.2
#v4 = np.random.
v4 = np.random.poisson(size=1000)
plt.hist(data, bins=2, normed=1.0)       # matplotlib version (plot)
plt.show()


import numpy as np
import pandas as pd

from chapter.four import LinearDiscriminantAnalysis, QuadraticDiscriminantAnalysis

def question_17():
    """ Misclassification error rate of QDA and LDA.

    Looks like they're both the same for the Height and Weight data: 11.9%.

    Returns:
        None.
    """
    # Get the height weight data
    hw_data = pd.read_csv(
        "cached_data/height_weight.csv",
        header=None,
        names=["gender", "height", "weight"],
        sep=",",
    )
    train = hw_data.drop(columns=["gender"]).values
    lda = LinearDiscriminantAnalysis()
    lda.fit(train, hw_data.gender.values)
    preds = lda.predict(train)
    miss_lda = np.not_equal(preds, hw_data.gender.values).sum()/hw_data.gender.shape[0]
    # Get the missclassification rate on each of LDA and QDA.
    print(f"LDA missclassification rate: {miss_lda*100:0.2f}")
    qda = QuadraticDiscriminantAnalysis()
    qda.fit(train, hw_data.gender.values)
    preds = qda.predict(train)
    miss_qda = np.not_equal(preds, hw_data.gender.values).sum()/hw_data.gender.shape[0]
    print(f"QDA missclassification rate: {miss_qda*100:0.2f}")
    return None

def question_19():
    r""" Stretched LDA.

    This is LDA with the additional parameter that we can multiply the grouped covariance matrix
    by a class-specific value to stretch the Gaussians.

    This change will cause the resulting model to still be quadratic in :math:`x` because we can't
    eliminate all of the squared terms as we did with LDA.

    .. math::
        p(y=c|x,\theta) &= \frac{\pi_c|2\pi\Sigma|^{-\frac{1}{2}}\exp(-\frac{1}{2}(x-\mu_c)^T\Sigma^{-1}(x-\mu_c))}{\Sigma_{c'}\pi_{c'}|2\pi\Sigma|^{-\frac{1}{2}}\exp(-\frac{1}{2}(x-\mu_{c'})^T\Sigma^{-1}(x-\mu_{c'}))}\\
            &= \frac{\pi_ck_c^n\exp(-\frac{1}{2}(x-\mu_c)^T\Sigma^{-1}(x-\mu_c))}{\Sigma_{c'}\pi_{c'}k_{c'}^n\exp(-\frac{1}{2}(x-\mu_{c'})^T\Sigma^{-1}(x-\mu_{c'}))}\\
            &= \frac{\pi_ck_c^n\exp(-\frac{1}{2}k_cx^T\Sigma^{-1}x)\exp(-\frac{1}{2}k_c\mu_c^T\Sigma^{-1}\mu_c+k_c\mu_c^T\Sigma^{-1}x}{\Sigma_{c'}\pi_{c'}k_{c'}^n\exp(-\frac{1}{2}k_{c'}x^T\Sigma^{-1}x)\exp(-\frac{1}{2}k_{c'}\mu_{c'}^T\Sigma^{-1}\mu_{c'}+k_{c'}\mu_{c'}^T\Sigma^{-1}x}\\

    Returns:
        None.
    """
    return None

def question_22():
    r""" QDA with 3 classes.

    Side effect: prints the answer that we're looking for!

    Point [-0.5, 0.5] is classified as 1, while [0.5, 0.5] is classified as 2.

    Interestingly, they're both very close to the decision boundary, with the first point
    almost in the 3rd class, and the second point almost in the 1st class. The results may
    be an artifact due to numerical instability in the code, but visual inspection marks them
    as the class labels given above.

    Returns:
        None.
    """
    qda = QuadraticDiscriminantAnalysis()
    qda.pi = [1 / 3, 1 / 3, 1 / 3]
    qda.mu = [np.array([0, 0]), np.array([1, 1]), np.array([-1, 1])]
    qda.sigma = [
        np.matrix([[0.7, 0], [0, 0.7]]),
        np.matrix([[0.8, 0.2], [0.2, 0.8]]),
        np.matrix([[0.8, 0.2], [0.2, 0.8]]),
    ]
    # Use this instead of the above 4 lines for LDA.
    # qda.sigma = [
    #    np.matrix([[0.766, 1.33], [1.33, 0.766]])
    # ]
    qda.pos_labels = [1, 2, 3]
    print(f"Point (-0.5, 0.5) has label: {qda.predict(np.array([-0.5, 0.5]))}")
    print(f"Point (0.5, 0.5) has label: {qda.predict(np.array([0.5, 0.5]))}")
    return None

def question_23():
    r""" Scalar QDA.

    Side-effect:
        We found the QDA values for each class as desired, and got an 83% probability for a 72 inch
        student to be a male at this university.

        The meatier question is about how to extend this method to the situation if we had multiple
        attributes per person. The clear answer would be to simply run QDA as implemented earlier, but I think
        that's not what the question is really getting at. I think you could run an ensemble of 1D QDA's and
        then take the average of the predicted values for each class across the set of 1D classifiers. This would
        ignore correlations between variables, but would take into account the multiple features.

    Returns:
        None.
    """
    data = np.matrix([
        [67, 79, 71, 68, 67, 60],
        [1, 1, 1, 2, 2, 2],
    ]).T
    qda = QuadraticDiscriminantAnalysis()
    print(list(data[:, 1].flat))
    qda.fit(data[:, 0], list(data[:, 1].flat))
    print(f"Priors: {qda.pi}")
    print(f"Means: {qda.mu}")
    print(f"Sigmas: {qda.sigma}")
    qda.predict(np.array([72]))
    print(f"P(y=m|x=72,\\theta): f{np.exp(qda.probs[0])}")
    return None


#!/usr/bin/env python

"""
generate a random walk to prove the equation.
"""
import numpy as np
from matplotlib import pyplot as plt

def gen_rand(prob):
    """ Decide where to go """
    if np.random.uniform() < prob:
        return 1        # move right
    else:
        return -1       # move left

def rw_next(cur_pos, probability):
    """evolve"""
    return cur_pos + gen_rand(probability)

def rw_simul(prob):
    """simulate RW for prob as probability of going left"""
    x_pos = np.zeros((10000, 10))

    for i in range(10000):
        pos = 0

        for j in range(10):
            for _ in range(100):
                pos = rw_next(pos, prob)

            x_pos[i, j] = pos

    x_mean = np.mean(x_pos, axis=0)
    sigma = np.mean(x_pos ** 2, axis=0) - x_mean

    return x_mean, sigma

def main():
    """main body"""
    sigmas = np.zeros((3, 10), dtype=float)
    x_means = np.zeros((3, 10), dtype=float)

    probs = [0.2, 0.35, 0.5]
    time = np.linspace(10, 1000, 10)

    for i in range(3):
        print("==> simulating for", probs[i])
        x_means[i], sigmas[i] = rw_simul(probs[i])

    print("==> plotting graphs")
    plt.plot(time, x_means[0], ls='', marker='o', color='b', label='p = 0.2')
    plt.plot(time, x_means[1], ls='', marker='o', color='r', label='p = 0.35')
    plt.plot(time, x_means[2], ls='', marker='o', color='g', label='p = 0.5')
    plt.xlabel('time')
    plt.ylabel('<x>')
    plt.title('mean of x over time for different p')
    plt.legend()
    plt.savefig("x_means.jpg", bbox_inches='tight')
    plt.close()

    plt.plot(time, sigmas[0], ls='', marker='^', color='b', label='p = 0.2')
    plt.plot(time, sigmas[1], ls='', marker='^', color='r', label='p = 0.35')
    plt.plot(time, sigmas[2], ls='', marker='^', color='g', label='p = 0.5')
    plt.xlabel('time')
    plt.ylabel('sigmas')
    plt.title('mean of x over time for different p')
    plt.legend()
    plt.savefig("sigmas.jpg", bbox_inches='tight')
    plt.close()

if __name__ == "__main__":
    main()


from math import factorial
from sympy import diff, symbols, lambdify, solveset
import numpy as np

########################################################################
def determinant(A):
    """
    Recursive calculation of the determinant of matrix A
    """
    if(A.shape == (2, 2)):
        return A[0,0]*A[1,1] - A[0,1]*A[1,0]
    else:
        det = 0.
        # iterate over row of matrix A
        for i in range(len(A)):
            sign = (-1.)**i

            # identify submatrix
            sublen = int(len(A)-1)
            As = np.zeros((sublen,sublen),dtype=type(A[0,0]))
            # count the number of row
            row = 0
            # iterate over row of matrix A
            for j in range(len(A)):
                # if row = i, skip
                if(i==j):
                    continue
                As[:,row] = A[1:,j]
                row += 1

            # determinant
            det += sign*A[0,i]*determinant(As)
    return det

###############################################################################
def init_Tmunu():
    """
    calculate the eigenvalues of T^{\mu \nu}
    Algebraic expressions for solve_Tmunu
    """
    T00 = symbols('T00')
    T01 = symbols('T01')
    T02 = symbols('T02')
    T03 = symbols('T03')
    T11 = symbols('T11')
    T12 = symbols('T12')
    T13 = symbols('T13')
    T22 = symbols('T22')
    T23 = symbols('T23')
    T33 = symbols('T33')

    Tmunu = np.array([[T00,T01,T02,T03],[T01,T11,T12,T13],[T02,T12,T22,T23],[T03,T13,T23,T33]])
    gmunu = np.array([[1,0,0,0],[0,-1,0,0],[0,0,-1,0],[0,0,0,-1]])

    # system to solve (x is the vector containing the eigenvalues)
    # ( T^{\mu \nu} - x g^{\mu \nu} ) u_\mu = 0
    x = symbols('x')
    tosolve = Tmunu - x*gmunu
    # calculate determinant of ( T^{\mu \nu} - x g^{\mu \nu} ) = characteristic polynomial
    det = determinant(tosolve)

    print(det)

    # we want det(x) = c0 + c1*x +c2*x**2 + c3*x**3 + c4*x**4
    # find coefficients of the characteristic polynomial
    coeff = 5*[0.]
    coeff[0] = det.subs(x,0)
    for order in range(1,5):
        det = diff(det, x, 1)
        coeff[order] = det.subs(x,0)/factorial(order)

    print(coeff[::-1])

init_Tmunu()


import numpy as np

### linefit
# From linefit.f from Stan Love
# Converted to python by Bryan Miller
# Fit a line to points with uncertainties in both axes

def linefit(x,sigx,y,sigy):
    m = 0.0
    b = 0.0
    csum = 0.0
    mold = -99999.
    sigsqb = 9999.0
    sigsqm = 9999.
    n = x.size
    while abs(m-mold) > 0.1*np.sqrt(sigsqm):
        mold = m
        sigsquen = sigy**2 + (mold*sigx)**2
        sumx = np.sum(x/sigsquen)
        sumy = np.sum(y/sigsquen)
        sumxx = np.sum((x**2)/sigsquen)
        sumxy = np.sum((x*y)/sigsquen)
        sums = np.sum(1.0/sigsquen)
        csum = np.sum((y-b-m*x)**2/sigsquen)
        det = sums*sumxx - sumx**2
        b = (sumxx*sumy - sumx*sumxy)/det
        m = (sums*sumxy - sumx*sumy)/det
        sigsqb = sumxx/det
        sigsqm = sums/det
    sigb = np.sqrt(sigsqb)
    sigm = np.sqrt(sigsqm)
    chi = csum/float(n-2)

    return m,sigm,b,sigb,chi


import math
import numpy as np
from matplotlib import pyplot as plt

lbdn2 = np.array([7.313, 44.61, 113.9, 215.2])
gn = np.array([0.749, 0.544, 0.463, 0.415])
lbdn4 = np.array([math.pow(e, 2) for e in lbdn2])

Pr = 0.71
r0 = 5
Re = 500
k = 0.024
a = 0.25e3

def xplus(x):
    return x / (r0 * Pr * Re)

xi1 = xplus(100)

def q(x):
    s = 0
    cxp = xplus(x)
    if cxp < xi1:
        for i in range(4):
            s += gn[i] / lbdn2[i] * (1 - math.exp(-lbdn2[i] * cxp))
    else:
        for i in range(4):
            s += gn[i] / lbdn2[i] * math.exp(-lbdn2[i] * cxp) * (math.exp(lbdn2[i] * xi1) - 1)

    return 2 * k * a / r0 * s

def dt(x):
    s = 0
    cxp = xplus(x)
    if cxp < xi1:
        for i in range(4):
            s += gn[i] / lbdn4[i] * (1 - math.exp(-lbdn2[i] * cxp))
    else:
        for i in range(4):
            s += gn[i] / lbdn4[i] * math.exp(-lbdn2[i] * cxp) * (math.exp(lbdn2[i] * xi1) - 1)

    return 8 * a * s

def nu(x):
    up = 0
    down = 0

    xp = xplus(x)
    if xp < xi1:
        for i in range(4):
            up += gn[i] / lbdn2[i] * (1 - math.exp(-lbdn2[i] * xp))
            down += gn[i] / lbdn4[i] * (1 - math.exp(-lbdn2[i] * xp))
    else:
        for i in range(4):
            up += gn[i] / lbdn2[i] * math.exp(-lbdn2[i] * xp) * (math.exp(lbdn2[i] * xi1) - 1)
            down += gn[i] / lbdn4[i] * math.exp(-lbdn2[i] * xp) * (math.exp(lbdn2[i] * xi1) - 1)

    return up / down / 2

if __name__ == '__main__':
    x = np.arange(25, 501, 5)
    qx = np.array([q(t) for t in x])
    dtx = np.array([dt(t) for t in x])
    nux = np.array([nu(t) for t in x])

    fig, (ax1, ax2, ax3) = plt.subplots(1, 3)
    ax1.plot(x, qx)
    ax1.set_title('q')
    ax2.plot(x, dtx)
    ax2.set_title('T0 - Tm')
    ax3.plot(x, nux)
    ax3.set_title('Nu')
    plt.show()


import numpy as np
from scipy import optimize

# Adapted from the SciPy Cookbook

def calc_R(x,y, xc, yc):
    """ calculate the distance of each 2D points from the center (xc, yc) """
    return np.sqrt((x-xc)**2 + (y-yc)**2)

def f(c, x, y):
    """ calculate the algebraic distance between the data points and the mean circle centered at c=(xc, yc) """
    Ri = calc_R(x, y, *c)
    return Ri - Ri.mean()

def leastsq_circle(x,y, centre_estimate):
    centre, ier = optimize.leastsq(f, centre_estimate, args=(x,y))
    xc, yc = centre
    Ri       = calc_R(x, y, *centre)
    R        = Ri.mean()
    residu   = np.sum((Ri - R)**2)
    return xc, yc, R, residu


import numpy as np
from PIL import Image
from matplotlib import pyplot as plt
from matplotlib import image as mpimg
from scipy import signal
from scipy import fftpack
import scipy.io

class Blurr_Fixer:
    def __init__(self, blurred_images, power=1,ifft_scale=1000, original_size=256, margin=0):
        self.blurred_images = blurred_images
        self.power = power
        self.ifft_scale = ifft_scale
        self.fixed = []
        self.margin = margin
        self.original_size = original_size
        self.F_images = [fftpack.fftshift(fftpack.fftn(img)) for img in blurred_images]

    def __get_weights_denom(self):
        weights_denom = np.zeros(self.F_images[0].shape)
        for mat in self.F_images:
            weights_denom = weights_denom + np.power(np.abs(mat), self.power)
        return weights_denom

    def unblur_images(self):
        denom = self.__get_weights_denom()
        accumulator = np.zeros(self.F_images[0].shape)
        for F in self.F_images:
            curr_weight = np.divide(np.power(np.abs(F), self.power), denom)
            accumulator = accumulator + np.multiply(F, curr_weight)
        fixed = fftpack.ifft2(fftpack.ifftshift(accumulator)).real
        fixed = np.divide(fixed,self.ifft_scale)
        # Crop
        size = self.original_size
        margin = self.margin
        self.fixed = fixed[margin:margin + size, margin:margin + size]
        return self.fixed

    def show_unblur_image(self,original_image):
        p1 = plt.subplot(1, 2, 1)
        plt.imshow(original_image.get_image(), cmap='gray')
        p1.set_title("Original")
        p2 = plt.subplot(1, 2, 2)
        plt.imshow(self.fixed, cmap='gray')
        p2.set_title("Fourier Burst Accumulation")
        plt.show()


import math
import numpy as np
from matplotlib import pyplot as plt

L_v = 2.5e6  # J/kg
R_v = 461  # J/K/kg
T_0 = 273.15  # K
e_0 = 6.113  # hPa
c_p = 1004 #J/K/kg
epsilon = 0.622 #kg/kg

def get_psychrometric_constant(pressure):
    # We could use a variable c_p depending on moisture content of air, but I don't think we have that info
    return c_p/L_v * pressure / epsilon

def clausius_clapeyron_e_s(T, is_C=True):
    if is_C:
        T = as_kelvin(T)
    return e_0 * math.exp(L_v/R_v*(1/T_0 - 1/T)) #hPa * e^(J/kg/(J/kg/K) * 1/K) = hPa * unitless^unitless = hPa

def slope_e_s(T, is_C=True):
    if is_C:
        T = as_kelvin(T)
    return L_v/R_v * 1/(T*T) * clausius_clapeyron_e_s(T, is_C=False)

def as_kelvin(temp_in_C):
    return temp_in_C + 273.15

def plot_sat_vapor_pressure_and_slope():
    """
        Test to see if the slope equation is correct by comparing to finite difference.
    """
    temps = np.linspace(0, 40, num=100)
    delta_T = 40/100
    e_s = [clausius_clapeyron_e_s(temp) for temp in temps]
    finite_difference_delta = finite_difference(e_s, delta_T)
    delta = [slope_e_s(temp) for temp in temps]

    fig, ax = plt.subplots()

    ax.plot(temps, e_s, 'y', label="e_S")
    ax.plot(temps, delta, 'r', label="d(e_s)/dT")
    ax.plot(temps[0:-1], finite_difference_delta, 'g', label="Finite difference")
    plt.legend()

    fig.savefig("sat_vapor_pressure_and_slope.png")

def finite_difference(series, delta):
    brackets = zip(series[0:-1], series[1:])
    return [(top - bottom)/delta for (bottom, top) in brackets]

plot_sat_vapor_pressure_and_slope()


import numpy as np
import scipy.stats as stat

def pOri(theta=0):
    return 1 / (2 * np.pi)

def pOriLog(theta=0):
    return np.log(pOri())

def pCon(con, conZero):
    return np.exp(-con / conZero) / conZero

def pConLog(con, conZero=1):
    return np.log(pCon(con, conZero))

def meanResp(con, ori, oriPreferred, tuningWidth=1, sensitivity=1):
    return sensitivity * con * np.exp(2 * np.cos(tuningWidth * (oriPreferred - ori)))

def pRespGivenOriCon(resp, ori, con, oriPreferred, tuningWidth=1, sensitivity=1):
    mr = meanResp(con, ori, oriPreferred, tuningWidth, sensitivity)
    return np.pi * resp / (2 * mr * mr) * np.exp(-np.pi * resp * resp / (4 * mr * mr))

def pRespGivenOriConLog(resp, ori, con, oriPreferred, tuningWidth=1, sensitivity=1):
    return np.log(pRespGivenOriCon(resp, ori, con, oriPreferred, tuningWidth, sensitivity))

def pJointConOri(ori, con, conZero):
    return pOri(ori) * pCon(con, conZero)

def pJointConOriLog(ori, con, conZero):
    return pOriLog(ori) + pConLog(con, conZero)

def pJointAll(resp, ori, con, conZero, oriPreferred=0, tuningWidth=1, sensitivity=1):
    if con == 0 or resp == 0:
        return 0
    cond = pRespGivenOriCon(resp, ori, con, oriPreferred, tuningWidth, sensitivity)
    return cond * pJointConOri(ori, con, conZero)

def pJointAllLog(resp, ori, con, conZero, oriPreferred=0, tuningWidth=1, sensitivity=1):
    cond = pRespGivenOriConLog(resp, ori, con, oriPreferred, tuningWidth, sensitivity)
    return cond + pJointConOriLog(ori, con, conZero)

def inverseExponCDF(x, lmb=1):
    return - np.log(1 - x) / lmb