import math
import random
import csv
import cProfile
import numpy as np
import hashlib

memoization = {}

class Similarity:
    """
    This class contains instances of similarity / distance metrics. These are used in centroid based clustering
    algorithms to identify similar patterns and put them into the same homogeneous sub sets
    :param minimum: the minimum distance between two patterns (so you don't divide by 0)
    """

    def __init__(self, minimum=1.0e-16):
        self.e = minimum
        self.vector_operators = VectorOperations()

    def manhattan_distance(self, p_vec, q_vec):
        """
        This method implements the manhattan distance metric
        :param p_vec: vector one
        :param q_vec: vector two
        :return: the manhattan distance between vector one and two
        """
        return max(np.sum(np.fabs(p_vec - q_vec)), self.e)

    def square_euclidean_distance(self, p_vec, q_vec):
        """
        This method implements the squared euclidean distance metric
        :param p_vec: vector one
        :param q_vec: vector two
        :return: the squared euclidean distance between vector one and two
        """
        diff = p_vec - q_vec
        return max(np.sum(diff**2), self.e)

    def euclidean_distance(self, p_vec, q_vec):
        """
        This method implements the euclidean distance metric
        :param p_vec: vector one
        :param q_vec: vector two
        :return: the euclidean distance between vector one and two
        """
        return max(math.sqrt(self.square_euclidean_distance(p_vec, q_vec)), self.e)

    def half_square_euclidean_distance(self, p_vec, q_vec):
        """
        This method implements the half squared euclidean distance metric
        :param p_vec: vector one
        :param q_vec: vector two
        :return: the half squared euclidean distance between vector one and two
        """
        return max(0.5 * self.square_euclidean_distance(p_vec, q_vec), self.e)

    def cosine_similarity(self, p_vec, q_vec):
        """
        This method implements the cosine similarity metric
        :param p_vec: vector one
        :param q_vec: vector two
        :return: the cosine similarity between vector one and two
        """
        pq = self.vector_operators.product(p_vec, q_vec)
        p_norm = self.vector_operators.norm(p_vec)
        q_norm = self.vector_operators.norm(q_vec)
        return max(pq / (p_norm * q_norm), self.e)

    def tanimoto_coefficient(self, p_vec, q_vec):
        """
        This method implements the cosine tanimoto coefficient metric
        :param p_vec: vector one
        :param q_vec: vector two
        :return: the tanimoto coefficient between vector one and two
        """
        pq = self.vector_operators.product(p_vec, q_vec)
        p_square = self.vector_operators.square(p_vec)
        q_square = self.vector_operators.square(q_vec)
        return max(pq / (p_square + q_square - pq), self.e)

    def fractional_distance(self, p_vec, q_vec, fraction=0.5):
        """
        This method implements the fractional distance metric. I have implemented memoization for this method to reduce
        the number of function calls required. The net effect is that the algorithm runs 400% faster. A similar approach
        can be used with any of the above distance metrics as well.
        :param p_vec: vector one
        :param q_vec: vector two
        :param fraction: the fractional distance value (power)
        :return: the fractional distance between vector one and two
        """
        # memoization is used to reduce unnecessary calculations ... makes a BIG difference
        memoize = True
        if memoize:
            key = self.get_key(p_vec, q_vec)
            x = memoization.get(key)
            if x is None:
                diff = p_vec - q_vec
                #diff_fraction = diff**fraction
                diff_fraction = np.power(np.abs(diff), fraction)
                return max(np.power(abs(np.sum(diff_fraction)), float(1.0/fraction)), self.e)
            else:
                return x
        else:
            diff = p_vec - q_vec
            diff_fraction = diff**fraction
            return max(math.pow(np.sum(diff_fraction), 1/fraction), self.e)

    @staticmethod
    def get_key(p_vec, q_vec):
        """
        This method returns a unique hash value for two vectors. The hash value is equal to the concatenated string of
        the hash value for vector one and vector two. E.g. is hash(p_vec) = 1234 and hash(q_vec) = 5678 then get_key(
        p_vec, q_vec) = 12345678. Memoization improved the speed of this algorithm 400%.
        :param p_vec: vector one
        :param q_vec: vector two
        :return: a unique hash
        """
        # return str(hash(tuple(p_vec))) + str(hash(tuple(q_vec)))
        return str(hashlib.sha1(p_vec)) + str(hashlib.sha1(q_vec))

class VectorOperations():
    """
    This class contains useful implementations of methods which can be performed on vectors
    """

    @staticmethod
    def product(p_vec, q_vec):
        """
        This method returns the product of two lists / vectors
        :param p_vec: vector one
        :param q_vec: vector two
        :return: the product of p_vec and q_vec
        """
        return p_vec * q_vec

    @staticmethod
    def square(p_vec):
        """
        This method returns the square of a vector
        :param p_vec: the vector to be squared
        :return: the squared value of the vector
        """
        return p_vec**2

    @staticmethod
    def norm(p_vec):
        """
        This method returns the norm value of a vector
        :param p_vec: the vector to be normed
        :return: the norm value of the vector
        """
        return np.sqrt(p_vec)


import numpy as np

dvar = np.float(0.0000000001)

#def deriv(func, val):
#    return (func(val+dvar)-func(val-dvar))/(dvar*2)

def deriv_func(func):
# description:  finding the derivate of the input function
# param:        func: the target function
# return:       func_out: the derivation function
    def func_out(val):
        result = (func(val+dvar)-func(val-dvar))/(dvar*2)
        return result
    return func_out

def solve_newton_method(func, initval):
# description:  find the zero point of input func, using Newton's method
# param:        func: the target function
#               initval: initial value of Newton's method
# return:       temp: the zero point
    func_deriv = deriv_func(func)
    temp = initval
    while abs(func(temp))>0.00001:
        if(func_deriv(temp)!=0):
            temp = temp - func(temp)/func_deriv(temp)
    return temp

#def func(x):
#    return x**2+2*x+1

def get_roc(func, val):
    deriv_1 = deriv(func, val)
    deriv_2 = (deriv(func, val+dvar)-deriv(func, val-dvar))/(dvar*2)
    return (1+deriv_1**2)**1.5/deriv_2

def get_min_roc(func, upper_lim, lower_lim):
    itera = np.linspace(upper_lim, lower_lim, 100)
    result = np.zeros(100)
    i = 0
    while i<100:
#        a = get_roc(func, itera[i])
        result[i]=(get_roc(func, itera[i]))
        i = i+1
#    print(np.min(result))
    for i in range(len(result)-1,-1,-1):
      if(result[i]<=0):
        result[i] = np.max(result)#negative replacer

    return np.min(result)


import numpy as np

# f = w*x

# f = 2*x
X = np.array([1,2,3,4], dtype=np.float32)
Y = np.array([2,4,6,8], dtype=np.float32)

w = 0.0

# model prediction
def forward(x): return w * x

# loss = MSE
def loss(y, y_predicted):
    return ((y - y_predicted)**2).mean()

# gradient
# MSE = 1/N * (w*x - y)**2
# dJ/dw = 1/N 2x (w*x - y)
def gradient(x, y, y_predicted):
    return np.dot(2*x, y_predicted-y).mean()

print(f'Prediction before training: f(5) = {forward(5):.3f}')

# Training
learning_rate = 0.01
n_iters = 10

for epoch in range(n_iters):
    # prediction = forward pass
    y_pred = forward(X)

    # loss
    l = loss(Y, y_pred)

    # gradients
    dw = gradient(X, Y, y_pred)

    # update weights
    w -= learning_rate * dw

    if epoch % 1 == 0:
        print(f'epoch {epoch+1}: w = {w:.3f}, loss = {l:.8f}')

print(f'Prediction after training: f(5) = {forward(5):.3f}')


import numpy as np
import pandas as pd
#
from pyta.overlays.simple_moving_average import simple_moving_average as sma
from pyta.helper import mean_dev

def commodity_channel_index(h: pd.Series, l: pd.Series, c: pd.Series, n: int = 20, const: float = 0.015) -> pd.Series:
    tp = (h + l + c) / 3
    tp_sma = sma(tp, n)
    tp_dev = mean_dev(tp, n)
    cci = (tp - tp_sma) / (tp_dev * const)
    return cci

commodity_channel_index.__doc__ = """Commodity Channel Index (CCI)

CCI essentially measures the current price level relative to an average over a given time.

Source:
    https://school.stockcharts.com/doku.php?id=technical_indicators:commodity_channel_index_cci

Formula:
    TP = (High + Low + Close) / 3
    CCI = (TP - SMA(n) of TP) / (const * MeanDev of TP)

Arguments:
    h (pdSeries)  : High series.
    ; (pdSeries)  : Low series.
    c (pdSeries)  : Close series.
    n (int)       : Period for MA [Default: 20]
    const (float) : Multiplier for MeanDev [Default: 0.015]

Returns:
    pdSeries
"""


import pandas as pd
import matplotlib.pyplot as plt
import statsmodels.api as sm

df_beetles=pd.read_csv('data/beetle.csv' , index_col=0)
df_beetles

# setting up the model
y=df_beetles[["ndied","nalive"]]
#since we want our random variable to be represented by a binomial we need two parameters
#to define a binomial.

X = df_beetles["dose"]
X = sm.add_constant(X)
model = sm.GLM(y, X,family=sm.families.Binomial())
results = model.fit()

print( results.summary() )

# adding a couple of diagnostic plots
fig, ax = plt.subplots(ncols=2,figsize=(14,6))
ax[0].plot(results.mu , results.resid_pearson , 'bo')
ax[0].set_xlabel("predicted values")
ax[0].set_ylabel("pearson residuals")

# results.mu corresponds to the fitted proportion of dead beetle
# equivalent to : 1./(1+np.exp(-(-14.8230+0.2494*df_beetles["dose"])))
ax[1].plot(results.mu , df_beetles["prop"] , 'bo')
ax[1].plot( [0,max(y)], [0,max(y)] )
ax[1].set_xlabel("predicted values")
ax[1].set_ylabel("observed values")

plt.tight_layout()

# plotting the prediction
predictedProportionOfDead =  results.mu
# equivalent to : 1./(1+np.exp(-(-14.8230+0.2494*df_beetles["dose"])))

fig, ax = plt.subplots(ncols=1,figsize=(6,6))
ax.plot(df_beetles["dose"] , df_beetles["prop"] ,'bo',label='data')
ax.plot(df_beetles["dose"], predictedProportionOfDead,'r--',label='prediction GLM')

ax.set_xlabel('dose')
ax.set_ylabel('proportion of dead')
ax.legend(loc='best',fontsize=12)


from numpy import sign

from .general_method import Method

class Bisection(Method,):
    """Bisection method class for bisection algorithm (https://en.wikipedia.org/wiki/Bisection_method).

    Attributes:
        f (function): function to search root against
        a (float): first interval boundary
        b (float): second interval boundary
        tol (float): tolerance
        max_iter (int): maximum number of iteration
    """

    def __init__(self, f, a, b, tol=1e-2, max_iter=1000):
        Method.__init__(self, f=f, tol=tol, max_iter=max_iter)
        self.a = a
        self.b = b

    def solve(self,):

        """run the Bisection root-finding method starting between points a and b.
        Stops if tolerance is matched or raise error if max iteration reached.

        Returns:
            float: computed root within interval.
        """
        a = self.a
        b = self.b
        self.epochs = []

        if 0 == self.f(a):
            return a
        elif 0 == self.f(b):
            return b

        if sign(self.f(a)) == sign(self.f(b)):
            raise ValueError(
                f"Same sign. f(a): {sign(self.f(a))},\tf(b): {sign(self.f(b))}"
            )

        for _ in range(self.max_iter):
            c = (a + b) / 2
            self.epochs.append(c)
            if (0 == self.f(c)) or ((b - a) / 2 < self.tol):  # solution found
                return c

            if sign(self.f(c)) == sign(self.f(a)):  # new interval
                a = c
            else:
                b = c
        raise Exception(
            "Convergence error. Reached maximum number of iterations without finding a stable enough solution"
        )

    def __repr__(self):
        return super().__repr__()

if __name__ == "__main__":
    print(Bisection(lambda x: x * x, 1e-1, 100))


"""
demo_Nd_unconstrained_optimization.py.py

Minimize a (quadratic) function of two variables:
Minimize f(X), where X = [x0, x1].T

Gradient algorithm with variable step size (steepest descent).
    Only for quadratic f(X) function. Requires gradient and hessian.
    Guaranteed to converge in n steps for n parameters, i.e. for X = [x1, ..., xn].T.
    X0 converges to X* for any X0.
    In parameter space, moving along orthogonal directions with variable step size.

Gradient algorithm with fixed step size.
    Only for quadratic f(X) function. Requires gradient and hessian.
    X0 converges to X* for any X0.
    In parameter space, moving along orthogonal directions with fixed step size.

Gradient algorithm with variable step size based on line search
    General algorithm for any f(X) function. Requires only gradient.
    X0 converges to X* for any X0.
    In parameter space, moving along orthogonal directions with variable step size determined by line search.
    Step size is determined by line search to ensure the descent property f(Xn+1) < f(Xn)

Conjugate gradient algorithm with variable step size (steepest descent).
    Only for quadratic f(X) function. Requires gradient and hessian.
    Guaranteed to converge in n steps for n parameters, i.e. for X = [x1, ..., xn].T.
    X0 converges to X* for any X0.
    In parameter space, moving along mutually conjugate directions with variable step size.

Conjugate gradient algorithm with variable step size based on line search
    General algorithm for any f(X) function. Requires only gradient.
    X0 converges to X* for any X0.
    In parameter space, moving along mutually conjugate directions with variable step size.
    Step size is determined by line search to ensure the descent property f(Xn+1) < f(Xn).
    For non-quadratic f(X) conjugate direction vectors will tend to deteriorate and need to be
        reset to the negative gradient every n steps.

Newton's algorithm.
    General algorithm for any f(X) function. Requires gradient and hessian.
    Can converge in as few as 1 step (for quadratic f(X)).
    X0 should be close to X*.
    In parameter space, moving to the MIN of parabola in 1 step.

Newton's algorithm with variable step size based on line search
    General algorithm for any f(X) function. Requires gradient and hessian.
    Can converge in as few as 1 step (for quadratic f(X)).
    X0 converges to X* for any X0.
    In parameter space, moving to the MIN of parabola in 1 step. Might be affected by linesearch.
    Step size is determined by line search to ensure the descent property f(Xn+1) < f(Xn).

Quasi-Newton algorithm with variable step size (steepest descent).
    Only for quadratic f(X) function. Requires gradient and hessian.
    Guaranteed to converge in n steps for n parameters, i.e. for X = [x1, ..., xn].T.
    X0 converges to X* for any X0.
    In parameter space, moving along mutually conjugate directions with variable step size.
    For non-quadratic f(X) conjugate direction vectors will tend to deteriorate and need to be
        reset to the negative gradient every n steps.

Quasi-Newton algorithm with variable step size based on line search
    General algorithm for any f(X) function. Requires only gradient.
    X0 converges to X* for any X0.
    In parameter space, moving along mutually conjugate directions with variable step size.
    Step size is determined by line search to ensure the descent property f(Xn+1) < f(Xn).
    For non-quadratic f(X) conjugate direction vectors will tend to deteriorate and need to be
        reset to the negative gradient every n steps.

Naive random walk algorithm
    General algorithm for any f(X) function.
    X0 converges to X* for any X0.

Simulated annealing algorithm
    General algorithm for any f(X) function.
    X0 converges to X* for any X0.

Particle swarm optimization algorithm
    General algorithm for any f(X) function.
    X0 converges to X* for any X0.
"""

import numpy as np
import matplotlib.pyplot as plt
from matplotlib import cm
from matplotlib.ticker import LinearLocator, FormatStrFormatter
from mpl_toolkits.mplot3d import Axes3D
from gradient_algorithm import gradient_algorithm_var_alpha, gradient_algorithm_fixed_alpha, gradient_algorithm_linesearch
from conjugate_gradient_algorithm import conjugate_gradient_algorithm, conjugate_gradient_algorithm_linesearch
from newton_algorithm import newton_algorithm, newton_algorithm_linesearch
from quasi_newton_algorithm import quasi_newton_algorithm, quasi_newton_algorithm_linesearch
from naive_random_search_algorithm import naive_random_search_algorithm
from simulated_annealing_algorithm import simulated_annealing_algorithm
from particle_swam_optimization_algorithm import particle_swam_optimization_algorithm
from print_report import print_report
from plot_progress_y import plot_progress_y
from plot_progress_x import plot_progress_x
import time

reg_coeff = 0.01; # regularization coefficient
# Minimize the following objective function:
# X = [x0, x1].T;
# f(X) = 1 / 2 * X.T @ Q @ X + X.T @ B + C + reg_coeff * X.T @ X;
Q = np.array([[8, 2 * np.sqrt(2)], [2 * np.sqrt(2), 10]]);
B = np.array([[3], [6]]);
C = 24;
func = lambda X : 1 / 2 * X.T @ Q @ X + X.T @ B + C + reg_coeff * X.T @ X;
func_grad = lambda X : Q @ X + B + 2 * reg_coeff * X;
func_hessian = lambda X : Q + 2 * reg_coeff;
# Objective function for particle swarm optimization algorithm (particles along row dimension, axis = 0)
func_ps = lambda X: 3 * X[:, [0]] + 6 * X[:, [1]] + X[:, [0]] * (4 * X[:, [0]] + np.sqrt(2) * X[:, [1]]) + X[:, [1]] \
    * (5 * X[:, [1]] + np.sqrt(2) * X[:, [0]]) + 24 + reg_coeff * (X[:, [0]] * X[:, [0]] + X[:, [1]] * X[:, [1]]);

fig = plt.figure();
X_data = np.arange(-100, 100, 5);
Y_data = np.arange(-100, 100, 5);
Z_data = np.zeros(shape = (X_data.size,Y_data.size));
for iX in range (X_data.size):
    for iY in range (Y_data.size):
        Z_data[iX][iY] = func(np.array([X_data[iX], Y_data[iY]]));
X_data, Y_data = np.meshgrid(X_data, Y_data);

# Plot the surface
ax = fig.add_subplot(1, 2, 1, projection ='3d')
surf = ax.plot_surface(X_data, Y_data, Z_data, cmap = cm.coolwarm,
                       linewidth = 0, antialiased = False)
ax.set_xlabel('X')
ax.set_xlim(-100, 100)
ax.set_ylabel('Y')
ax.set_ylim(-100, 100)
plt.title('Objective function: surface plot')

# Plot the contour
ax = fig.add_subplot(1, 2, 2)
cset = ax.contour(X_data, Y_data, Z_data, 50, cmap=cm.coolwarm)
ax.set_xlabel('X')
ax.set_xlim(-100, 100)
ax.set_ylabel('Y')
ax.set_ylim(-100, 100)
plt.title('Objective function: contour plot')
plt.show()

# Gradient algorithm with variable step size (steepest descent)
print('***********************************************************************');
print('Gradient algorithm with variable step size (steepest descent)');
N_iter_max = 1000;
tolerance_x = 10e-6;
tolerance_y = 10e-8;
options = {'tolerance_x' : tolerance_x, 'tolerance_y' : tolerance_y, 'N_iter_max' : N_iter_max};
X0 = np.array([[-100], [-100]]);
start = time.time();
X, report = gradient_algorithm_var_alpha(X0, func, func_grad, func_hessian, options);
end = time.time();
print_report(func, report);
# Plot path to X* for Y
algorithm_name = 'Gradient algorithm with variable alpha';
plot_progress_y(algorithm_name, report);
# Plot path to X* for X
plot_progress_x(X_data, Y_data, Z_data, algorithm_name, report);
print('Elapsed time [s]: %0.5f' % (end - start));
print('***********************************************************************\n');

# Gradient algorithm with fixed step size
print('***********************************************************************');
print('Gradient algorithm with fixed step size');
N_iter_max = 1000;
tolerance_x = 10e-6;
tolerance_y = 10e-8;
options = {'tolerance_x' : tolerance_x, 'tolerance_y' : tolerance_y, 'N_iter_max' : N_iter_max};
X0 = np.array([[-100], [-100]]);
start = time.time();
X, report = gradient_algorithm_fixed_alpha(X0, func, func_grad, func_hessian, options);
end = time.time();
print_report(func, report);
# Plot path to X* for Y
algorithm_name = 'Gradient algorithm with fixed alpha';
plot_progress_y(algorithm_name, report);
# Plot path to X* for X
plot_progress_x(X_data, Y_data, Z_data, algorithm_name, report);
print('Elapsed time [s]: %0.5f' % (end - start));
print('***********************************************************************\n');

# Gradient algorithm with variable step size based on line search
print('***********************************************************************');
print('Gradient algorithm with variable step size based on line search');
N_iter_max = 1000;
tolerance_x = 10e-6;
tolerance_y = 10e-8;
options = {'tolerance_x' : tolerance_x, 'tolerance_y' : tolerance_y, 'N_iter_max' : N_iter_max};
X0 = np.array([[-100], [-100]]);
start = time.time();
X, report = gradient_algorithm_linesearch(X0, func, func_grad, options);
end = time.time();
print_report(func, report);
# Plot path to X* for Y
algorithm_name = 'Gradient algorithm with line search';
plot_progress_y(algorithm_name, report);
# Plot path to X* for X
plot_progress_x(X_data, Y_data, Z_data, algorithm_name, report);
print('Elapsed time [s]: %0.5f' % (end - start));
print('***********************************************************************\n');

# Conjugate gradient algorithm with variable step size (steepest descent)
print('***********************************************************************');
print('Conjugate gradient algorithm with variable step size (steepest descent)');
N_iter_max = 1000;
tolerance_x = 10e-6;
tolerance_y = 10e-8;
options = {'tolerance_x' : tolerance_x, 'tolerance_y' : tolerance_y, 'N_iter_max' : N_iter_max};
X0 = np.array([[-100], [-100]]);
start = time.time();
X, report = conjugate_gradient_algorithm(X0, func, func_grad, func_hessian, options);
end = time.time();
print_report(func, report);
# Plot path to X* for Y
algorithm_name = 'Conjugate gradient algorithm with variable alpha';
plot_progress_y(algorithm_name, report);
# Plot path to X* for X
plot_progress_x(X_data, Y_data, Z_data, algorithm_name, report);
print('Elapsed time [s]: %0.5f' % (end - start));
print('***********************************************************************\n');

# Conjugate gradient algorithm with variable step size based on line search
print('***********************************************************************');
print('Conjugate gradient algorithm with variable step size based on line search');
N_iter_max = 1000;
tolerance_x = 10e-6;
tolerance_y = 10e-8;
options = {'tolerance_x' : tolerance_x, 'tolerance_y' : tolerance_y, 'N_iter_max' : N_iter_max};
X0 = np.array([[-100], [-100]]);
start = time.time();
X, report = conjugate_gradient_algorithm_linesearch(X0, func, func_grad, options);
end = time.time();
print_report(func, report);
# Plot path to X* for Y
algorithm_name = 'Conjugate gradient algorithm with line search';
plot_progress_y(algorithm_name, report);
# Plot path to X* for X
plot_progress_x(X_data, Y_data, Z_data, algorithm_name, report);
print('Elapsed time [s]: %0.5f' % (end - start));
print('***********************************************************************\n');

# Newton's algorithm
print('***********************************************************************');
print('Newton\'s algorithm');
N_iter_max = 1000;
tolerance_x = 10e-6;
tolerance_y = 10e-8;
options = {'tolerance_x' : tolerance_x, 'tolerance_y' : tolerance_y, 'N_iter_max' : N_iter_max};
X0 = np.array([[-100], [-100]]);
start = time.time();
X, report = newton_algorithm(X0, func, func_grad, func_hessian, options);
end = time.time();
print_report(func, report);
# Plot path to X* for Y
algorithm_name = 'Newton\'s algorithm';
plot_progress_y(algorithm_name, report);
# Plot path to X* for X
plot_progress_x(X_data, Y_data, Z_data, algorithm_name, report);
print('Elapsed time [s]: %0.5f' % (end - start));
print('***********************************************************************\n');

# Newton's algorithm with variable step size based on line search
print('***********************************************************************');
print('Newton\'s algorithm with variable step size based on line search');
N_iter_max = 1000;
tolerance_x = 10e-6;
tolerance_y = 10e-8;
options = {'tolerance_x' : tolerance_x, 'tolerance_y' : tolerance_y, 'N_iter_max' : N_iter_max};
X0 = np.array([[-100], [-100]]);
start = time.time();
X, report = newton_algorithm_linesearch(X0, func, func_grad, func_hessian, options);
end = time.time();
print_report(func, report);
# Plot path to X* for Y
algorithm_name = 'Newton\'s algorithm with line search';
plot_progress_y(algorithm_name, report);
# Plot path to X* for X
plot_progress_x(X_data, Y_data, Z_data, algorithm_name, report);
print('Elapsed time [s]: %0.5f' % (end - start));
print('***********************************************************************\n');

# Quasi-Newton algorithm with variable step size (steepest descent)
print('***********************************************************************');
print('Quasi-Newton algorithm with variable step size (steepest descent)');
N_iter_max = 1000;
tolerance_x = 10e-6;
tolerance_y = 10e-8;
options = {'tolerance_x' : tolerance_x, 'tolerance_y' : tolerance_y, 'N_iter_max' : N_iter_max};
X0 = np.array([[-100], [-100]]);
start = time.time();
X, report = quasi_newton_algorithm(X0, func, func_grad, func_hessian, options);
end = time.time();
print_report(func, report);
# Plot path to X* for Y
algorithm_name = 'Quasi-Newton algorithm with variable alpha';
plot_progress_y(algorithm_name, report);
# Plot path to X* for X
plot_progress_x(X_data, Y_data, Z_data, algorithm_name, report);
print('Elapsed time [s]: %0.5f' % (end - start));
print('***********************************************************************\n');

# Quasi-Newton algorithm with variable step size based on line search
print('***********************************************************************');
print('Quasi-Newton algorithm with variable step size based on line search');
N_iter_max = 1000;
tolerance_x = 10e-6;
tolerance_y = 10e-8;
options = {'tolerance_x' : tolerance_x, 'tolerance_y' : tolerance_y, 'N_iter_max' : N_iter_max};
X0 = np.array([[-100], [-100]]);
start = time.time();
X, report = quasi_newton_algorithm_linesearch(X0, func, func_grad, options);
end = time.time();
print_report(func, report);
# Plot path to X* for Y
algorithm_name = 'Quasi-Newton algorithm with line search';
plot_progress_y(algorithm_name, report);
# Plot path to X* for X
plot_progress_x(X_data, Y_data, Z_data, algorithm_name, report);
print('Elapsed time [s]: %0.5f' % (end - start));
print('***********************************************************************\n');

# Naive random walk algorithm
print('***********************************************************************');
print('Naive random walk algorithm');
N_iter_max = 1000;
tolerance_x = 10e-8;
tolerance_y = 10e-8;
X_lower = np.array([[-150], [-150]]); # X lower bound
X_upper = np.array([[150], [150]]); # X upper bound
alpha = 5; # step size
options = {'tolerance_x' : tolerance_x, 'tolerance_y' : tolerance_y, 'N_iter_max' : N_iter_max, 'x_lower' : X_lower,
           'x_upper' : X_upper, 'alpha' : alpha};
X0 = np.array([[-20], [-20]]); # X0 = X_lower + (X_upper - X_lower) * np.random.rand(X_lower.size, 1);
start = time.time();
X, report = naive_random_search_algorithm(X0, func, options);
end = time.time();
print_report(func, report);
# Plot path to X* for Y
algorithm_name = 'Naive random walk algorithm';
plot_progress_y(algorithm_name, report);
# Plot path to X* for X
plot_progress_x(X_data, Y_data, Z_data, algorithm_name, report);
print('Elapsed time [s]: %0.5f' % (end - start));
print('***********************************************************************\n');

# Simulated annealing algorithm
print('***********************************************************************');
print('Simulated annealing algorithm');
N_iter_max = 1000;
tolerance_x = 10e-8;
tolerance_y = 10e-8;
X_lower = np.array([[-150], [-150]]); # X lower bound
X_upper = np.array([[150], [150]]); # X upper bound
alpha = 5; # step size
gamma = 1.5; # controls temperature decay, gamma > 0
options = {'tolerance_x' : tolerance_x, 'tolerance_y' : tolerance_y, 'N_iter_max' : N_iter_max, 'x_lower' : X_lower,
           'x_upper' : X_upper, 'alpha' : alpha, 'gamma' : gamma};
X0 = np.array([[-20], [-20]]); # X0 = X_lower + (X_upper - X_lower) * np.random.rand(X_lower.size, 1);
start = time.time();
X, report = simulated_annealing_algorithm(X0, func, options);
end = time.time();
print_report(func, report);
# Plot path to X* for Y
algorithm_name = 'Simulated annealing algorithm';
plot_progress_y(algorithm_name, report);
# Plot path to X* for X
plot_progress_x(X_data, Y_data, Z_data, algorithm_name, report);
print('Elapsed time [s]: %0.5f' % (end - start));
print('***********************************************************************\n');

# Particle swarm optimization algorithm
print('***********************************************************************');
print('Particle swarm optimization algorithm');
N_iter_max = 1000;
tolerance_x = 10e-8;
tolerance_y = 10e-8;
X_lower = np.array([[-150], [-150]]); # X lower bound
X_upper = np.array([[150], [150]]); # X upper bound
d_lower = -1; # direction (aka velocity) lower bound
d_upper = 1; # direction (aka velocity) upper bound
N_ps = 10000; # number of particles
w = 0.9; # inertial constant, w < 1
c1 = 1.5; # cognitive/independent component, c1 ~ 2
c2 = 1.5; # social component, c2 ~ 2
alpha = 1; # step size
options = {'tolerance_x' : tolerance_x, 'tolerance_y' : tolerance_y, 'N_iter_max' : N_iter_max, 'x_lower' : X_lower,
           'x_upper' : X_upper, 'alpha' : alpha, 'd_lower' : d_lower, 'd_upper' : d_upper, 'N_ps' : N_ps, 'w' : w, 'c1' : c1, 'c2' : c2};
start = time.time();
X, report = particle_swam_optimization_algorithm(func_ps, options);
end = time.time();
print_report(func, report);
# Plot path to X* for Y
algorithm_name = 'Particle swarm optimization algorithm';
plot_progress_y(algorithm_name, report);
# Plot path to X* for X
plot_progress_x(X_data, Y_data, Z_data, algorithm_name, report);
print('Elapsed time [s]: %0.5f' % (end - start));
print('***********************************************************************\n');


"""  """

class Node(object):
    """  This class represents a node in MiniFlow architecture. """
    def __init__(self, inbound_nodes=[]):
        # Node(s) from which this Node receives values
        self.inbound_nodes = inbound_nodes
        # Node(s) to which this Node passes values
        self.outbound_nodes = []
        # For each inbound_node, add the current Node as an outbound_node.
        for n in self.inbound_nodes:
            n.outbound_nodes.append(self)
        # A calculated value
        self.value = None

    def forward(self):
        """ Forward propagation.

        Compute the output value based on `inbound_nodes` and store the result in self.value.

        :return:
        """
        raise NotImplemented

class Input(Node):
    """  This class represents a Inputs node in MiniFlow architecture. """
    def __init__(self):
        # An Input node has no inbound nodes,
        # so no need to pass anything to the Node instantiator.
        Node.__init__(self)

    # NOTE: Input node is the only node where the value
    # may be passed as an argument to forward().
    #
    # All other node implementations should get the value
    # of the previous node from self.inbound_nodes
    #
    # Example:
    # val0 = self.inbound_nodes[0].value
    def forward(self, value=None):
        """ Forward propagation.

        Compute the output value based on `inbound_nodes` or `passed arguments` and store the result in self.value.

        :return:
        """
        # Overwrite the value if one is passed in.
        if value is not None:
            self.value = value

class Add(Node):
    """  This class represents a add node in MiniFlow architecture. """
    # def __init__(self, x, y):
    # You could access `x` and `y` in forward with
    # self.inbound_nodes[0] (`x`) and self.inbound_nodes[1] (`y`)
    # Node.__init__(self, [x, y])
    # You may need to change this...
    def __init__(self, *inputs):
        Node.__init__(self, inputs)

    def forward(self):
        """
        Set the value of this node (`self.value`) to the sum of its inbound_nodes.
        Remember to grab the value of each inbound_node to sum!

        Your code here!
        """
        self.value = sum([n.value for n in self.inbound_nodes])

class Mul(Node):
    """  This class represents a multiplication node in MiniFlow architecture. """
    # def __init__(self, x, y):
    # You could access `x` and `y` in forward with
    # self.inbound_nodes[0] (`x`) and self.inbound_nodes[1] (`y`)
    # Node.__init__(self, [x, y])
    # You may need to change this...
    def __init__(self, *inputs):
        Node.__init__(self, inputs)

    def forward(self):
        """
        Set the value of this node (`self.value`) to the sum of its inbound_nodes.
        Remember to grab the value of each inbound_node to sum!

        Your code here!
        """
        from numpy import prod as npprod
        self.value = npprod([n.value for n in self.inbound_nodes])

class Linear(Node):
    """ This class represents a linear node in MiniFlow architecture. """
    def __init__(self, inputs, weights, bias):
        Node.__init__(self, [inputs, weights, bias])

        # NOTE: The weights and bias properties here are not
        # numbers, but rather references to other nodes.
        # The weight and bias values are stored within the
        # respective nodes.

    def forward(self):
        """
        Set self.value to the value of the linear function output.

        Your code goes here!
        """
        # from numpy import multiply as npmultiply
        # self.value = sum(npmultiply(self.inbound_nodes[0].value, self.inbound_nodes[1].value)) + self.inbound_nodes[2].value
        from numpy import dot as npdot
        self.value = npdot(self.inbound_nodes[0].value, self.inbound_nodes[1].value) + self.inbound_nodes[2].value

class Sigmoid(Node):
    """
    You need to fix the `_sigmoid` and `forward` methods.
    """
    def __init__(self, node):
        Node.__init__(self, [node])

    def _sigmoid(self, x):
        """
        This method is separate from `forward` because it
        will be used later with `backward` as well.

        `x`: A numpy array-like object.

        Return the result of the sigmoid function.

        Your code here!
        """
        from numpy import exp as npexp
        return 1/(1 + npexp(-x))

    def forward(self):
        """
        Set the value of this node to the result of the
        sigmoid function, `_sigmoid`.

        Your code here!
        """
        # This is a dummy value to prevent numpy errors
        # if you test without changing this method.
        self.value = self._sigmoid(self.inbound_nodes[0].value)

class MSE(Node):
    def __init__(self, y, a):
        """
        The mean squared error cost function.
        Should be used as the last node for a network.
        """
        # Call the base class' constructor.
        Node.__init__(self, [y, a])

    def forward(self):
        """
        Calculates the mean squared error.
        """
        # NOTE: We reshape these to avoid possible matrix/vector broadcast
        # errors.
        #
        # For example, if we subtract an array of shape (3,) from an array of shape
        # (3,1) we get an array of shape(3,3) as the result when we want
        # an array of shape (3,1) instead.
        #
        # Making both arrays (3,1) insures the result is (3,1) and does
        # an elementwise subtraction as expected.
        y = self.inbound_nodes[0].value.reshape(-1, 1)
        a = self.inbound_nodes[1].value.reshape(-1, 1)
        # TODO: your code here
        from numpy import square as npsquare
        from numpy.linalg import norm as npnorm
        self.value = npsquare(npnorm(y - a)) / len(y)

def topological_sort(feed_dict):
    """
    Sort generic nodes in topological order using Kahn's Algorithm.

    `feed_dict`: A dictionary where the key is a `Input` node and the value is the respective value feed to that node.

    Returns a list of sorted nodes.
    """

    input_nodes = [n for n in feed_dict.keys()]

    G = {}
    nodes = [n for n in input_nodes]
    while len(nodes) > 0:
        n = nodes.pop(0)
        if n not in G:
            G[n] = {'in': set(), 'out': set()}
        for m in n.outbound_nodes:
            if m not in G:
                G[m] = {'in': set(), 'out': set()}
            G[n]['out'].add(m)
            G[m]['in'].add(n)
            nodes.append(m)

    L = []
    S = set(input_nodes)
    while len(S) > 0:
        n = S.pop()

        if isinstance(n, Input):
            n.value = feed_dict[n]

        L.append(n)
        for m in n.outbound_nodes:
            G[n]['out'].remove(m)
            G[m]['in'].remove(n)
            # if no other incoming edges add to S
            if len(G[m]['in']) == 0:
                S.add(m)
    return L

def forward_pass(output_node, sorted_nodes=None):
    """
    Performs a forward pass through a list of sorted nodes.

    Arguments:

        `output_node`: A node in the graph, should be the output node (have no outgoing edges).
        `sorted_nodes`: A topologically sorted list of nodes.

    Returns the output Node's value
    """
    if sorted_nodes:
        for n in sorted_nodes:
            n.forward()
        return output_node.value
    else:
        # Forward pass
        for n in output_node:
            n.forward()

def main(which):
    """ This is main function. """
    if which == 1:
        x, y = Input(), Input()
        f = Add(x, y)
        feed_dict = {x: 10, y: 5}
        sorted_nodes = topological_sort(feed_dict)
        output = forward_pass(f, sorted_nodes)
        # NOTE: because topological_sort sets the values for the `Input` nodes we could also access
        # the value for x with x.value (same goes for y).
        print("{} + {} = {} (according to miniflow)".format(feed_dict[x], feed_dict[y], output))
    elif which == 2:
        x, y, z = Input(), Input(), Input()
        f = Add(x, y, z)
        feed_dict = {x: 4, y: 5, z: 10}
        graph = topological_sort(feed_dict)
        output = forward_pass(f, graph)
        print("{} + {} + {} = {} (according to miniflow)".format(feed_dict[x], feed_dict[y], feed_dict[z], output))
    elif which == 3:
        x, y, z = Input(), Input(), Input()
        f = Mul(x, y, z)
        feed_dict = {x: 4, y: 5, z: 10}
        graph = topological_sort(feed_dict)
        output = forward_pass(f, graph)
        print("{} * {} * {} = {} (according to miniflow)".format(feed_dict[x], feed_dict[y], feed_dict[z], output))
    elif which == 4:
        inputs, weights, bias = Input(), Input(), Input()
        f = Linear(inputs, weights, bias)
        feed_dict = {
            inputs: [6, 14, 3],
            weights: [0.5, 0.25, 1.4],
            bias: 2
        }
        graph = topological_sort(feed_dict)
        output = forward_pass(f, graph)
        print(output)  # should be 12.7 with this example
    elif which == 5:
        import numpy as np
        X, W, b = Input(), Input(), Input()
        f = Linear(X, W, b)
        g = Sigmoid(f)
        X_ = np.array([[-1., -2.], [-1, -2]])
        W_ = np.array([[2., -3], [2., -3]])
        b_ = np.array([-3., -5])
        feed_dict = {X: X_, W: W_, b: b_}
        graph = topological_sort(feed_dict)
        output = forward_pass(g, graph)
        """
        Output should be:
        [[  1.23394576e-04   9.82013790e-01]
         [  1.23394576e-04   9.82013790e-01]]
        """
        print(output)
    elif which == 6:
        import numpy as np
        y, a = Input(), Input()
        cost = MSE(y, a)
        y_ = np.array([1, 2, 3])
        a_ = np.array([4.5, 5, 10])
        feed_dict = {y: y_, a: a_}
        graph = topological_sort(feed_dict)
        # forward pass
        forward_pass(graph)
        """
        Expected output

        23.4166666667
        """
        print(cost.value)
    return None

if __name__ == "__main__":
    main(which=1)
    main(which=2)
    main(which=3)
    main(which=4)
    main(which=5)
    main(which=6)


# Copyright 2018 - Jonathan Alcantara e Osmar Fernandes

# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at

#     http://www.apache.org/licenses/LICENSE-2.0

# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import numpy as np

tolerance = 0.00001
A = [[1, 0.2, 0], [0.2, 1, 0.5], [0, 0.5, 1]]

def power_method(A):
    X = [1]*len(A[0])

    new_X = list(X)
    eigenvalue_candidate = 1
    solution_residue = eigenvalue_candidate

    while solution_residue > tolerance:
        print '\nLargest Eigenvalue Candidate: ', eigenvalue_candidate
        print 'Associated Eigenvector: ', X
        X = np.matmul(A, X)

        solution_residue = abs(X[0] - eigenvalue_candidate)/abs(X[0])
        eigenvalue_candidate = X[0]

        for X_element in range(len(X)):
            X[X_element] = X[X_element]/float(eigenvalue_candidate)

    print '\nLargest Eigenvalue: ', eigenvalue_candidate
    print 'Associated Eigenvector: ', X


"""
var = sd^2 = summ(Xi^2 - mu)/N = summ(Xi^2)/N - mu^2

Observation is:
Biased_Variance = (n-1)/n Population/Total_Variance
OR,
summ(Xi^2 - mu)/n = (n-1)/n Population/Total_Variance
=> Population_Variance = summ(Xi^2 - mu)/(n-1)
is a better Variance estimate than Sample Variance
"""

from math import sqrt, pi, e
from itertools import izip, imap
from numpy import var
import scipy.stats
from pandas import DataFrame, read_pickle
#zscore, t score, f score
from scipy.stats import norm, t, f, manwhitneyu, shapiro

def mean(x): return reduce(lambda a,b: (a+b), x)/float(len(x))

"""
access like this df[0.01][38][4] - df[critical percent][df][num of samples]
#Make sure the q_table csv is in same parent directory else calc_tukey_hsd won't work
"""
q_table = read_pickle('q_table')

def mean_arr(x):
    sumr = 0.
    prod_list = []
    for i in x:
        sumr += i
        prod_list.append(i*i)
    return sumr/len(x), prod_list

def variance_biased(x):
    """x is a list of numbers"""
    mean_x, prod_list = mean_arr(x)
    return sum(prod_list)/float(len(x)) - (mean_x)**2

def variance_unbiased(x):
    """x is a list of numbers"""
    xbar = mean(x)
    return sum([(i-xbar)**2 for i in x])/float(len(x)-1)

def std_deviation_biased(x):
    return sqrt(variance_biased(x))

def std_deviation_unbiased(x):
    return sqrt(variance_unbiased(x))

def normal_distro_probability(x, xbar, var):
    z2 = float(x-xbar)**2/(2*var)  #z squared
    return 1./sqrt((2*pi*var) * (e**z2))

def normal_sample_variance(original_var, sample_size):
    """
    given original variance but not all data points.
    sqrt(var/n) = Std. Error
    """
    return original_var/sample_size

############# ALL ABOUT CONFIDENCE INTERVAL & INFERENTIAL STATS ################

#standard error
se = lambda sd, sample_size: (sd/sqrt(sample_size))

def se_independent_sample(sd1, sd2, n1, n2):
    """
    1. WELCH's T TEST
    2. USE WHEN WE CMP 2 DIFF MEANS
    ---------------------------------
    Useful for finding out
    CI and comparing Z stats Given a population metric
    """
    return sqrt((sd1**2 + sd2**2)/float(n1)) if n1==n2 else sqrt((sd1**2/float(n1)) + (sd2**2/float(n2)))

def df_two_sampled_welch(var1, var2, n1, n2):
    #IMP dof FUNC!!
    #2 sample Deg of Freedom for 2 Sample Welch's T Test
    df1, df2 = n1-1, n2-1
    return float(var1/float(n1) + var2/float(n2))**2/\
    float((var1**2/float(n1**2 * df1)) + (var2**2/float(n2**2 * df2)))

pooled_variance = lambda x,y,n1,n2: ((var(x)*n1) + (var(y)*n2))/float(n1+n2-2)

def se_pooled_t(x, y):
    """
    Useful when calculating T stats reports
    since sample metrics are given.
    x, y: arrays
    n1, n2: sizes of x, y respectively.
    """
    n1,n2 = len(x), len(y)
    #sd^2 = \summ(x-xbar)^2 + \summ(y-ybar)^2/df1+df2
    pooled_var = pooled_variance(x,y,n1,n2)
    return round(sqrt(pooled_var * (1./n1 + 1./n2)), 2)

df_independent_sample = lambda n1,n2: (n1+n2-2)

#marginal error given z score range
marginal_z = lambda z_score, se: z_score * se

def calc_z(mu, xbar, sd, sample_size):
    """
    Calculate Z score given normal sample distribution mean
    and population mean, its std dev and the sample size for the
    sample distribution.
    """
    #std_error = se(sd, sample_size)
    return float(xbar - mu)/se(sd, sample_size)

#half if two tailed else not and /100
def critical_z(percentile, one_tailed):
    #calculate critical z from critical percentage
    return round(abs(scipy.stats.norm.ppf((100 - percentile)/100.)), 3) if one_tailed \
    else round(abs(scipy.stats.norm.ppf((100 - percentile)/200.)), 3)

def calc_probability_sample_mean(mu, xbar, sd, sample_size):
    #calculate percentage from z score
    return scipy.stats.norm.cdf(calc_z(mu, xbar, sd, sample_size))

def calc_t(mu, xbar, sd, sample_size):
    #calculating critical t
    return round(calc_z(mu, xbar, sd, sample_size), 2)

def calc_t_independent_sample(mu, xbar, se):
    #calculating critical t when se is given/independent samples
    return round(float(xbar-mu)/se, 2)

def t_percentile(t_val, df, one_tailed=0):
    """
    Find P-Value Given T score, DF and if its 1 Tailed or 2 Tailed
    """
    return t.sf(t_val, df) if one_tailed else t.sf(t_val, df) * 2

def critical_t(percentile, df, one_tailed):
    return round(t.isf((100-percentile)/100., df), 3) if one_tailed else round(t.isf((100-percentile)/200., df), 3)

#same as critical_t, but in proportion
def t_val_from_t_percentile(t_percentile, df, one_tailed = 0):
    """
    Find T score given T percentile, DF
    and if its 1 Tailed or 2 Tailed
    t_percentile: % in proportion
    """
    return round(t.isf(t_percentile, df), 3) if one_tailed else round(t.isf(t_percentile/2., df), 3)

def t_cmp(calculated_t, critical_t):
    """
    Given Critical T Value and T score,
    Return if its Significant or Not.
    """
    return abs(calculated_t) > abs(critical_t)

### Correlation Measures using T Stats ####
def t_r_squared(t_score, df):
    #calculated t score cousin of r squared for line fitting - correlation
    """
    R square generally tells us how much of the difference between 2 means
    in a particular hypothetisis is due to the given statistical factors
    OR R squared Error;
    Higher the Value better the Model of Assumption.
    Lower the value implies we are missing out on some stat factors and that
    its a weak hypothesis.
    """
    t= t_score**2
    return t/float(t+df)

def t_significant(x, y, percentile, one_sided):
    """
    by calculating from independent sampling from a bunch of samples
    when population params are not given
    """
    df = df_independent_sample(len(x), len(y))
    t_critical = critical_t(percentile, df, one_sided)
    xbar= mean(x)
    ybar = mean(y)
    std_dev = se_pooled_t(x, y)
    t_score = calc_t_independent_sample(ybar, xbar, std_dev)
    return t_score, t_cmp(t_score, t_critical)

def ci_t_margin_error(sd, sample_size, df, t_score = None, percentile = None, one_tailed = 0):
    critical_t_score = 0.
    if percentile:
        critical_t_score = critical_t(percentile, df, one_tailed)
    elif t_score:
        critical_t_score = t_score  #this is the Critical T value!
    #std_error = se(sd, sample_size)
    return marginal_z(critical_t_score, se(sd, sample_size))

def ci_t(xbar, sd, sample_size, df, t_score = None, percentile = None, one_tailed = 0):
    margin_error = ci_t_margin_error(sd, sample_size, df, t_score = t_score, percentile = percentile, one_tailed = one_tailed)
    return (xbar - margin_error, xbar + margin_error)

def ci_margin_error(sd, sample_size, z_score = None, percentile = None, one_tailed = 0):
    """
    z score * std_error is the margine of error i C.I. from the sample mean
    """
    critical_z_score = 0.
    if percentile: critical_z_score = critical_z(percentile, one_tailed)
    elif z_score: critical_z_score = z_score
    #std_error = se(sd, sample_size)
    return marginal_z(critical_z_score, se(sd, sample_size))

#confidence interval
def ci(xbar, sd, sample_size, z_score = None, percentile = None, one_tailed = 0):
    """
    xbar: sample mean
    percentile: in %
    """
    margin_error = ci_margin_error(sd, sample_size, z_score = z_score, percentile = percentile, one_tailed=one_tailed)
    return (xbar - margin_error, xbar + margin_error)

def ci_generic(xbar, margin_error):
    return (xbar-margin_error, xbar+margin_error)

#Decision process component
def z_cmp(calculated_z_score_proportion, criticalz_percentage_proportion):
    """
    calculated_z_score_proportion: the z score calculated from mu and xbar in proportion
    criticalz_percentage_proportion: Given Critical Value proportion for acceptance criteria
    if calculated_z_score_proportion > criticalz_percentage_proportion
    then the chance of that happening is p < criticalz_percentage_proportion
    """
    return abs(calculated_z_score_proportion) > scipy.stats.norm.cdf(scipy.stats.norm.ppf((100-criticalz_percentage_proportion)/100.))

######## ANALYSIS OF VARIANCE - ANOVA ###########
"""
F = Between Group Variability/Within Group Variability
"""
f_critical = lambda alpha, dfb, dfw: round(f.isf(alpha, dfb, dfw), 4)
#between group var
#xbar_groups is an array of x means
df_ssb = lambda xbar_groups: float(len(xbar_groups) - 1)

ssb = lambda xg, xbar_groups, each_grp_sample_size: \
sum([ni * (xbar - xg)**2 for ni, xbar in izip(each_grp_sample_size, xbar_groups)])/df_ssb(xbar_groups)

#within group var
#x_samples is an array of lists - this is like df for pooled variance
df_ssw = lambda x_samples: float(reduce(lambda x,y: x+y, imap(len, x_samples)) - len(x_samples))

#this is like pooled variance for more than 2 samples - generic
ssw = lambda x_samples: sum([var(samples) * len(samples) for samples in x_samples])/df_ssw(x_samples)

#F ratio stats
anova_ratio = lambda xg, xbar_groups, x_samples, each_grp_sample_size:ssb(xg, xbar_groups, each_grp_sample_size)/ssw(x_samples)

#if greater we reject null hypothesis else we fail to reject
f_cmp = lambda f_score, f_critical: abs(f_score) > abs(f_critical)

###### CALCULATING EFFECT MEASURES (TUKEYS HSD)- measures of significance not due to samplign error or chances #########

#this can be used when the num of samples per group are not the same N
var_num_samples = lambda x_samples: len(x_samples)/reduce(lambda n1,n2: (1/float(n1) + 1/float(n2)), imap(len, x_samples))

"""
#honesty significant differences - which samples are significantly different?
#q = df[critical_per][dfw][num_samples]
This is how you calculate Q:
Q = ml - ms / sqrt of mssw / n
k = num of groups
n = Total num of elements per group across all groups for an experiment where each group has same elements, else refer to var_num_samples(x_samples)
to calculate n;
"""

calc_tukey_hsd = lambda critical_per, dfw, ssw_score, k, n: round(q_table[critical_per][dfw][k] * sqrt(float(ssw_score)/n), 4)

def tukey_hsd_cmp(mean_diff, tukey_hsd):
    """
    Reject Null if mean diff > tukey hsd
    """
    return mean_diff > tukey_hsd

#cohens d multiple comparisons ANOVA
cohens_d_multiple = lambda x1,x2,ssw_score: float(x1-x2)/sqrt(ssw_score)

#eta squared - proportion of explained differences - sort of r sqrd
#Analogous to R squared Difference of T Test;
eta_sqrd = lambda ssb_score, ss: float(ssb_score)/ss
#ss = total or ssb+ssw = \summ((xi - xg)**2) summation of all xi minus grand xg

######## OTHER TESTS ON NON GAUSSIAN DATASETS ###############
"""
Man - Whitney U-Test
u,p = manwhitneyu(x_arr, y_arr)
Non-parametric test;
- non-normal data
- find descriptive variables.
"""
manwhitneyu_test = lambda x_arr, y_arr: manwhitneyu(x_arr, y_arr)

"""
Shapiro-Wilk Test
- Plot Histogram to check for Bell Curve
- Perform Shapiro-Wilk Test
s, p = shapiro(x_arr)
"""
shapiro_test = lambda x_arr: shapiro(x_arr)


# Applying PCA on two dimensional dummy data .............................

# PCA focuses only the data and not the output labels.
# Note: Reduction is always dependent on data. Which features to reduce, and how much information will be lost is dependent on the dataset.
import numpy as np
from sklearn.decomposition import PCA
from matplotlib import pyplot as plt

# Using PCA on 2-Dimensional Data.....................
# Plotting random/dummy 2-D data..........
x1 = np.array([1,2,3,4,5,6])
x2 = np.array([7.5, 11, 16, 18, 20, 26])
plt.scatter(x1, x2)
plt.show()

# Concatenating the two single feature arrays for 2-D data. It is done to create a two dimensional dataset(x1, x2 as features).
X = np.c_[x1, x2]           # Concatenate function,combines two lists.

# Creating the PCA object.....
# n_components: This parameter specifies how many features we need finally. If n_components is not passed then the no. of new dimensions is same as no of original features.
# Here we can try any number of feature values less than equal to the total number of features we originally had.
pca = PCA(n_components = 1)

# Here when we fit we try to find a line that finds a best line for the data.
# With transform function we transform the data according to x1' and x2' features. These new values are according to new directions.
X_reduced = pca.fit_transform(X)
print(X_reduced)
# The compnents_ parameter tells us about the unit vector in the new direction(basically i-cap and j-cap values (because originally its two dimensional data) for the x1' and x2' directions.)
# These are unit vectors along the new features which hold maximum variance. This is sorted and the first direction holds the information with maximum variance.
print(pca.components_)

# To bring back the data to original dimensions/directions we can use the inverse_transform function.
# If we reduce the feature values we will not be able to get back our data as some information is lost. The values will differ from original values.
X_approx = pca.inverse_transform(X_reduced)

#  Plotting the new x values after reducing in the original direction. After reducing components the point lie on a line as one direction is reduced.
plt.plot(X_approx[:, 0], X_approx[:, 1])
plt.scatter(x1, x2)
plt.show()


#21. Write a NumPy program to create a vector of length 10 with values evenly distributed between 5 and 50.
import numpy as np
v = np.linspace(10, 49, 5)
print("Length 10 with values evenly distributed between 5 and 50:")
print(v)
#22. Write a NumPy program to create a vector with values from 0 to 20 and change the sign of the numbers in the range from 9 to 15.
import numpy as np
x = np.arange(20)
print("Original vector:")
print(x)
print("After changing the sign of the numbers in the range from 9 to 15:")
x[(x >= 9) & (x <= 15)] *= -1
print(x)
#23. Write a NumPy program to create a vector of length 5 filled with arbitrary integers from 0 to 10.
import numpy as np
x = np.random.randint(0, 11, 5)
print("Vector of length 5 filled with arbitrary integers from 0 to 10:")
print(x)
#24. Write a NumPy program to multiply the values of two given vectors.
import numpy as np
x = np.array([1, 8, 3, 5])
print("Vector-1")
print(x)
y= np.random.randint(0, 11, 4)
print("Vector-2")
print(y)
result = x * y
print("Multiply the values of two said vectors:")
print(result)
#25. Write a NumPy program to create a 3x4 matrix filled with values from 10 to 21.
import numpy as np
m= np.arange(10,22).reshape((3, 4))
print(m)
#26. Write a NumPy program to find the number of rows and columns of a given matrix.
import numpy as np
m= np.arange(10,22).reshape((3, 4))
print("Original matrix:")
print(m)
print("Number of rows and columns of the said matrix:")
print(m.shape)
#27. Write a NumPy program to create a 3x3 identity matrix, i.e. diagonal elements are 1, the rest are 0.
import numpy as np
x = np.eye(3)
print(x)
#28. Write a NumPy program to create a 10x10 matrix, in which the elements on the borders will be equal to 1, and inside 0.
import numpy as np
x = np.ones((10, 10))
x[1:-1, 1:-1] = 0
print(x)
#29. Write a NumPy program to create a 5x5 zero matrix with elements on the main diagonal equal to 1, 2, 3, 4, 5.
import numpy as np
x = np.diag([1, 2, 3, 4, 5])
print(x)
#30. Write a NumPy program to create a 4x4 matrix in which 0 and 1 are staggered, with zeros on the main diagonal.
import numpy as np
x = np.zeros((4, 4))
x[::2, 1::2] = 1
x[1::2, ::2] = 1
print(x)
#31. Write a NumPy program to create a 3x3x3 array filled with arbitrary values.
import numpy as np
x = np.random.random((3, 3, 3))
print(x)
#32. Write a NumPy program to compute sum of all elements, sum of each column and sum of each row of a given array.
import numpy as np
x = np.array([[0,1],[2,3]])
print("Original array:")
print(x)
print("Sum of all elements:")
print(np.sum(x))
print("Sum of each column:")
print(np.sum(x, axis=0))
print("Sum of each row:")
print(np.sum(x, axis=1))
#33. Write a NumPy program to compute the inner product of two given vectors.
import numpy as np
x = np.array([4, 5])
y = np.array([7, 10])
print("Original vectors:")
print(x)
print(y)
print("Inner product of said vectors:")
print(np.dot(x, y))
#34. Write a NumPy program to add a vector to each row of a given matrix.
import numpy as np
m = np.array([[1,2,3], [4,5,6], [7,8,9], [10, 11, 12]])
v = np.array([1, 1, 0])
print("Original vector:")
print(v)
print("Original matrix:")
print(m)
result = np.empty_like(m)
for i in range(4):
  result[i, :] = m[i, :] + v
print("\nAfter adding the vector v to each row of the matrix m:")
print(result)
#35. Write a NumPy program to save a given array to a binary file .
import numpy as np
import os
a = np.arange(20)
np.save('temp_arra.npy', a)
print("Check if 'temp_arra.npy' exists or not?")
if os.path.exists('temp_arra.npy'):
    x2 = np.load('temp_arra.npy')
    print(np.array_equal(a, x2))
#36. Write a NumPy program to save two given arrays into a single file in compressed format (.npz format) and load it
import numpy as np
import os
x = np.arange(10)
y = np.arange(11, 20)
print("Original arrays:")
print(x)
print(y)
np.savez('temp_arra.npz', x=x, y=y)
print("Load arrays from the 'temp_arra.npz' file:")
with np.load('temp_arra.npz') as data:
    x2 = data['x']
    y2 = data['y']
    print(x2)
    print(y2)
#37. Write a NumPy program to save a given array to a text file and load it.
import numpy as np
import os
x = np.arange(12).reshape(4, 3)
print("Original array:")
print(x)
header = 'col1 col2 col3'
np.savetxt('temp.txt', x, fmt="%d", header=header)
print("After loading, content of the text file:")
result = np.loadtxt('temp.txt')
print(result)
#38. Write a NumPy program to convert a given array into bytes, and load it as array.
import numpy as np
import os
a = np.array([1, 2, 3, 4, 5, 6])
print("Original array:")
print(a)
a_bytes = a.tostring()
a2 = np.fromstring(a_bytes, dtype=a.dtype)
print("After loading, content of the text file:")
print(a2)
print(np.array_equal(a, a2))
#39. Write a NumPy program to convert a given array into a list and then convert it into a list again.
import numpy as np
a = [[1, 2], [3, 4]]
x = np.array(a)
a2 = x.tolist()
print(a == a2)
#40. Write a NumPy program to compute the x and y coordinates for points on a sine curve and plot the points using matplotlib.
import numpy as np
import matplotlib.pyplot as plt
# Compute the x and y coordinates for points on a sine curve
x = np.arange(0, 3 * np.pi, 0.2)
y = np.sin(x)
print("Plot the points using matplotlib:")
plt.plot(x, y)
plt.show()


# get the numerical gradient of multivariable function f(x1, x2, ...)

import sys
import os
sys.path.append(os.pardir)
import numpy as np
from ch04.numerical_diff import show

def numerical_gradient(f, x):
    """
    Returns the gradient of f at x.
    x is a NumPy array.
    """
    h = 1e-4  # rule of thumb
    gradient = np.zeros_like(x)

    for index in range(x.size):
        h_vector = get_unit_vector(x.size, index) * h
        # f(x_i + h)
        f_plus = f(x + h_vector)

        # f(x_i - h)
        f_minus = f(x - h_vector)

        # print('h_vector = {0}, x + h_vector = {1}, x - h_vector = {2}'.format(h_vector, x + h_vector, x - h_vector))
        # print('f_plus = {0}, f_minus = {1}'.format(f_plus, f_minus))
        gradient[index] = (f_plus - f_minus) / (2 * h)
    return gradient

def get_unit_vector(dimension, index):
    vector = np.zeros(dimension)
    vector[index] = 1
    return vector

if __name__ == '__main__':
    print('z = x ** 2 + y ** 2')

    def f1(x): return x[0] ** 2 + x[1] ** 2
    show(f1, np.array([3.0, 4.0]), numerical_gradient,
         'numerical_gradient', np.array([6.0, 8.0]))
    show(f1, np.array([5.0, 0.0]), numerical_gradient,
         'numerical_gradient', np.array([10.0, 0.0]))
    show(f1, np.array([0.0, 0.0]), numerical_gradient,
         'numerical_gradient', np.array([0.0, 0.0]))

    def f2(x): return x[0] + x[1] ** 2 + x[2] ** 3
    show(f2, np.array([1.0, 2.0, 3.0]), numerical_gradient,
         'numerical_gradient', np.array([1.0, 4.0, 27.0]))


#!/usr/bin/env python
# coding: utf-8

# # Random numbers in NumPy

# ## Random integers
#
# Here is the recommended way to make random integers in NumPy.  We first instantiate a "random number generator" that we call `rng`.

# In[1]:

import numpy as np
rng = np.random.default_rng()

# In[2]:

help(rng.integers)

# Making a 10x2 NumPy array of random integers between 1 (inclusive) and 5 (exclusive).

# In[3]:

rng.integers(1,5,size=(10,2))

# Here are two ways to include 5.

# In[4]:

rng.integers(1,6,size=(10,2))

# In[5]:

rng.integers(1,5,size=(10,2),endpoint=True)

# ## Random real numbers
#
# If making random real numbers, the range is always between 0 and 1; there is no way to specify the upper and lower bounds as inputs to the function.  So to increase the range of outputs, you multiply, and to shift the range of outputs, you add.

# In[6]:

rng.random(size=(10,2))

# Random real numbers between 0 and 30:

# In[7]:

30*rng.random(size=(10,2))

# Random real numbers between 5 and 35:

# In[8]:

5 + 30*rng.random(size=(10,2))


from notbook import show_plot
from bokeh.plotting import figure
import numpy as np
import scipy.optimize as opt

"""md
_This is an example of a better way to work on and display numerical python code._

[Here's](https://github.com/samuelcolvin/notbook/blob/master/demo-script.py) the input script used to generate
this document.

---

# Logistic curve fitting example

Example is taken mostly from
[here](https://ipython-books.github.io/93-fitting-a-function-to-data-with-nonlinear-least-squares/).

We define a logistic function with four parameters:

```maths
  \[f_{a,b,c,d}(x) = \frac{a}{1 + \exp\left(-c (x-d)\right)} + b\]
```
"""

# {
def f(x, a, b, c, d):
    return a / (1. + np.exp(-c * (x - d))) + b
# } The equation we'll try and fit

"""md
Define some random parameters:
"""
# {
a, c = np.random.exponential(size=2)
b, d = np.random.randn(2)
# }

# we actually cheat here and keep a "good" set of paramters
a = 0.8744
b = -2.0836
c = 1.3389
d = 0.4991

"""md
which are:
"""
print(f'a={a:0.4f} b={b:0.4f} c={c:0.4f} d={d:0.4f}')

"""md
Now, we generate random data points by using the sigmoid function and adding a bit of noise:
"""
# {
n = 100
x = np.linspace(-10, 10, n)
y_model = f(x, a, b, c, d)
y = y_model + a * 0.2 * np.random.randn(n)
# }

"""md
Plot that to see how it looks:
"""

p = figure(title='Model and random data')
p.line(x, y_model, color='black', line_dash='dashed', line_width=5, legend_label='model')
p.circle(x, y, size=10, legend_label='random data')
p.legend.location = 'top_left'
p.legend.click_policy = 'hide'
show_plot(p)

"""md
We now assume that we only have access to the data points and not the underlying generative function.
These points could have been obtained during an experiment.

By looking at the data, the points appear to approximately follow a sigmoid, so we may want to try to fit such a
curve to the points.
That's what curve fitting is about.

SciPy's [`curve_fit()`](https://docs.scipy.org/doc/scipy/reference/generated/scipy.optimize.curve_fit.html)
function allows us to fit a curve defined by an
arbitrary Python function to the data:
"""

# {
(a_, b_, c_, d_), _ = opt.curve_fit(f, x, y)

"""md
And use those parameters to estimate our underlying curve
"""
y_fit = f(x, a_, b_, c_, d_)
# }

p = figure(title='Model, random data and curve fit')
p.line(x, y_model, color='black', line_dash='dashed', line_width=5, legend_label='model')
p.circle(x, y, size=10, legend_label='random data')
p.line(x, y_fit, color='red', line_width=5, legend_label='curve fit')
p.legend.location = 'top_left'
p.legend.click_policy = 'hide'
show_plot(p)

"""md
we can also manually compare our initial parameters with with those from `curve_fit()`:
"""

print(f'initial parameters:    a={a:0.4f} b={b:0.4f} c={c:0.4f} d={d:0.4f}')
print(f'curve_fit parameters:  a={a_:0.4f} b={b_:0.4f} c={c_:0.4f} d={d_:0.4f}')


import numpy as np
from joblib import Parallel, delayed
import matplotlib.pyplot as plt
np.random.seed(3112163)
def random_walk_next(state):
    if np.random.uniform() < 0.5:
        return 0
    return state + 1
def pi(k): return k/2**(k+1)

NA = 10**4
Sim = []
Real = []
Error = []
for N in range(1,NA,10):
    running_state = 0
    sum_result = 0

    for i in range(N):
        running_state = random_walk_next(running_state)
        sum_result += running_state

    realMath = sum([pi(k) for k in range(N)])
    Sim.append(sum_result/N)
    Real.append(realMath)
    Error.append(abs( sum_result/N - realMath ) )

plt.figure(figsize=(12, 7))
plt.subplot()
plt.plot(Sim, color = 'b', label = 'Simulated')
plt.plot(Real, color = 'g', label = 'Actual')
plt.plot(Error, color = 'r', label = 'Error')
plt.xlabel(r'Number of simulations')
plt.ylabel(r'Probability')
plt.title(r'$\sum^\infty_{k=0}k\pi (k)$')
plt.legend()
plt.grid()
plt.show()
plt.cla()
plt.clf()

### Ergodic Limit Theorem
print("The simulated ergodic average [X1+X2+X3+...+XN]/N is: %.4f" % (sum_result / N))
print("The real ergodic average is: %.4f" % (realMath))