import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from scipy import stats
pd.set_option('display.width', 100)
pd.set_option('precision', 3)

import statsmodels.api as sm
from statsmodels.stats.stattools import durbin_watson
from statsmodels.sandbox.regression.predstd import wls_prediction_std
from sklearn.metrics import mean_squared_error, r2_score
from linearmodels import PanelOLS

class linear_regression:
    def __init__(self):
        self.reg = None

    def ols_fit(self, X, y, add_const=True, plot=None, assumption_test=False, if_print=True):
        """
        plot: column name of X that want to plot
        """
        if add_const:
            X_ = sm.add_constant(X)
        mod = sm.OLS(y.values, np.asarray(X_))
        res = mod.fit()
        #print(res.summary())
        res = res.get_robustcov_results()
        self.reg = res
        if if_print:
            print(res.summary()) # Robusted Results

        if plot is not None:
            prstd, iv_l, iv_u = wls_prediction_std(self.reg)
            plt.figure(figsize=(10,6))
            plt.plot(X[plot], y, 'o', label='Sample Data')
            fit_data = pd.DataFrame({'X': X[plot],
                                     'y': res.fittedvalues,
                                    'upper': iv_u,
                                    'lower': iv_l})
            fit_data.sort_values(by='X', inplace=True)
            plt.plot(fit_data.X, fit_data.y, 'r', label='OLS model')
            plt.plot(fit_data.X, fit_data.upper, color='darksalmon', ls='--')
            plt.plot(fit_data.X, fit_data.lower, color='darksalmon', ls='--')
            plt.legend(loc='best')
            plt.show()

        if assumption_test:
            error = res.fittedvalues - y
            print('Residual Tests:')
            # Durbin-Waston test:
            dw = durbin_watson(error)
            print('DW test statistic: ', dw)
            print('(2: no serial correlation; 0: positive serial correlation; 4: negative serial correlation)')

            sm.graphics.tsa.plot_acf(error, lags=20) # residual auto-correlation

            sm.qqplot(error, loc = error.mean(), scale = error.std(), line='s') # residual normality

    def gls_fit(self, X, y, add_const=True):
        if add_const:
            X_ = sm.add_constant(X)
        mod = sm.GLS(y.values, np.asarray(X_))
        res = mod.fit()
        #print(res.summary())
        res = res.get_robustcov_results()
        self.reg = res
        print(res.summary()) # Robusted Results

    def get_predict(self, X_test, add_const=True):
        if add_const:
            X_test = sm.add_constant(X_test)
        return self.reg.predict(X_test)

    def test_performance(self, X_test, y_test, add_const=True, plot=None):
        y_pred = self.get_predict(X_test, add_const=add_const)
        print("Mean squared error: %.2f"% mean_squared_error(y_test, y_pred))
        # Explained variance score: 1 is perfect prediction
        print('R-2 score: %.2f' % r2_score(y_test, y_pred))

        if plot is not None:
            plt.figure(figsize=(10,6))
            plt.scatter(X_test[plot], y_test, color='black', label='Test Sample Data')
            fit_data = pd.DataFrame({'X': X_test[plot],
                                     'y': y_pred})
            fit_data.sort_values(by='X', inplace=True)
            plt.plot(fit_data.X, fit_data.y, color='blue',label='Model Value', linewidth=3)
            plt.legend(loc='best')
            plt.show()

    def get_parameters(self, pr=True):
        """
        get parameters, standard errors, t-values, p-values
        """
        if pr:
            print('Coeficients: ', self.reg.params)
            print('Standard error: ', self.reg.bse)
            print('T-values: ', self.reg.tvalues)
            print('p-values: ', self.reg.pvalues)
        return self.reg.params, self.reg.bse, self.reg.tvalues, self.reg.pvalues

    def get_confidence_intervals(self, alpha=0.05):
        return self.reg.conf_int(alpha=alpha)

    def print_latex(self):
        return self.reg.summary().as_latex()

    def get_residuals(self, normalize=False):
        """
        normalize: Default False, return normalized residuals to have unit variance if True
        """
        if normalize:
            return self.reg.resid_pearson
        else:
            return self.reg.resid

    def get_r_square(self):
        """
        return R2 and Adj-R2
        """
        return self.reg.rsquared, self.reg.rsquared_adj

    def panel_regression(self, X, y, entity_col, time_col, entity_effects=False, time_effects=False, other_effects=None, add_const=True, drop_absorbed=True):
        """
        other_effects (array-like) – Category codes to use for any effects that are not entity or time effects. Each variable is treated as an effect
        return fitted res
        """

        X = X.set_index([entity_col, time_col])
        y.index = X.index
        if add_const:
            X = sm.add_constant(X)
        if other_effects is None:
            mod = PanelOLS(y, X, entity_effects=entity_effects, time_effects=time_effects)#, endog_names=['intercept'] + X.columns)
        else:
            mod = PanelOLS(y, X, entity_effects=entity_effects, time_effects=time_effects, other_effects=X[other_effects])
        res = mod.fit()
        print(res.summary)
        return res

    def glm_regression(self, X, y, mod_family, add_const=True):
        """
        mod_family: sm.families.Binomial([link]), Gamma(), Gaussian(),InverseGaussian(), NegativeBinomial(), Poisson(), Tweedie()
        link: CDFlink, CLogLog, Log, Logit, NegativeBinomial([alpha]), Power([power]), cauthy(), cloglog, identity(), inverse_power(), inverse_sqared(), log, logit, nbinom([alphal]), probit([dbn])
        return fitted res
        """
        if add_const:
            X = sm.add_constant(X)
        mod = sm.GLM(y, X, family=mod_family)
        res = mod.fit()
        print(res.summary())
        return res

import statsmodels.api as sm
from scipy.special import expit

class logistic_regression:
    def __init__(self):
        self.reg = None

    def fit(self, X, y, add_const=True, plot=None, method='bfgs', if_print=True):
        """
        print summary table, prediction table (pred_table[i,j] refers to the number of times “i” was observed and the model predicted “j”. Correct predictions are along the diagonal.)
        """
        if add_const:
            X_ = sm.add_constant(X)
        mod = sm.Logit(y, X_)
        res = mod.fit(method=method)
        self.reg = res
        if if_print:
            print(res.summary())
            print('Prediction Table:')
            print(pd.DataFrame(res.pred_table()))

        if plot is not None:
            plt.clf()
            plt.figure(figsize=(10,6))
            plt.scatter(X[y == 1][plot], y[y == 1], color='r',label='Positive Sample', zorder=1)
            plt.scatter(X[y == 0][plot], y[y == 0], color='b',label='Negative Sample', zorder=1)
            X_ = X_.sort_values(by=plot)
            logit = expit(np.dot(X_, res.params))
            plt.plot(X_[plot], logit, 'k--',label='Fitted Model', linewidth=1)
            plt.legend(loc='best')
            plt.show()

    def get_predict(self, X_test, add_const=True):
        if add_const:
            X_test_ = sm.add_constant(X_test)
        return self.reg.predict(X_test_)

    def test_performance(self, X_test, y_test, plot=None, add_const=True):
        """
        print out precision, accuracy, recall, f1
        """
        y_pred = self.get_predict(X_test, add_const=add_const)
        tp = np.count_nonzero(np.all([y_pred == 1, y_test == 1], axis=0))
        fp = np.count_nonzero(np.all([y_pred == 1, y_test == 0], axis=0))
        tn = np.count_nonzero(np.all([y_pred == 0, y_test == 0], axis=0))
        fn = np.count_nonzero(np.all([y_pred == 0, y_test == 1], axis=0))
        if tp+fp > 0:
            precision = tp / (tp+fp)
            print('Precision: ', precision)
        if tp+fn > 0:
            recall = tp / (tp + fn)
            print('Recall: ', recall)
        accuracy = (tp + tn) / (tp + tn + fp + fn)
        print('Accuracy: ', accuracy)
        if tp+fp > 0 and tp + fn >0:
            print('F-measure: ', 2 * precision * recall / (precision + recall))

        if plot is not None:
            plt.clf()
            plt.figure(figsize=(10,6))
            plt.scatter(X_test[y_test == 1][plot], y_test[y_test == 1], color='r',label='Positive Sample', zorder=1)
            plt.scatter(X_test[y_test == 0][plot], y_test[y_test == 0], color='b',label='Negative Sample', zorder=1)
            if add_const:
                X_test_ = sm.add_constant(X_test)
            X_test_ = X_test_.sort_values(by=plot)
            logit = expit(np.dot(X_test_, self.reg.params))
            plt.plot(X_test_[plot], logit, 'k--',label='Fitted Model', linewidth=1)
            plt.legend(loc='best')
            plt.show()

    def get_coefficients(self):
        return self.reg.params

    def get_confidence_intervals(self, alpha=0.05):
        return self.reg.conf_int(alpha=alpha)

    def print_latex(self):
        return self.reg.summary().as_latex()

    def get_r_square(self):
        """
        McFadden’s pseudo-R-squared
        """
        return self.reg.prsquared

import numpy,from scipy,import statsmodels,from statsmodels


# polyfit.py
#
# Author: Daniel Clark, 2015

'''
This module contains functions to fit a polynomial to input data

Usage: python polyfit.py [-m <order_of_poly>] [-i <path_to_input_npy>]
                         [-t <path_to_target_npy>]
'''

# Polynomial fit function
def calc_coeffs(input_data, target_data, poly_order):
    '''
    Function to calculate the polynomial equation coefficients of the
    best-fit line (using a sum-of-squares error function) from the
    input data and target data using matrix algebra

    Parameters
    ----------
    input_data : numpy.ndarray
        array of input data
    target_data : numpy.ndarray
        array of target data for corresponding input data
    poly_order : integer
        the order of the polynomial equation

    Returns
    -------
    w_star : numpy.ndarray
        the polynomial equation weights of the best-fit line; this is a
        column vector of (poly_order+1 x 1) dimensions
    x_mat : numpy.ndarray
        the tile'd input matrix where the input data is repeated
        poly_order+1 times (columns), where each column is raised to
        the power of its column index
        (e.g. 0th col, x_n^0, ..., Mth col x_n^M)
    '''

    # Import packages
    import numpy as np

    # Init variables
    x_mat = np.tile(input_data.transpose(), (1, poly_order+1))

    # Populate matrix X
    for col in range(poly_order+1):
        x_mat[:, col] = x_mat[:, col]**col

    # Create A = X^T*X
    a_mat = np.dot(x_mat.transpose(), x_mat)

    # Calculate b
    b_vec = np.dot(target_data, x_mat).transpose()

    # w* = A^-1 * b
    amat_inv = np.linalg.inv(a_mat)
    w_star = np.dot(amat_inv, b_vec)

    # Return w* and X
    return w_star, x_mat

# Generate input toy data
def generate_input(num_pts, rnd_seed=0, show_plot=False):
    '''
    Function to generate noisy input data using the function
    f(x) = 7*sin(2*PI*x) + N(0, 1) where N(0,1) is the standard normal
    distribution

    Parameters
    ----------
    num_pts : integer
        the number of time points to sample the data
    rnd_seed : integer (optional); defualt=0
        seed for random number generator; if this is set to any number
        that is non-zero, the noisy data will be from a fixed pdf
    show_plot : boolean (optional); default=False
        flag to indicate whether to display the generated data

    Returns
    -------
    input_data : numpy.ndarray
        real-valued observed input data, x_1...x_n
    target_data : numpy.ndarray
        target values from the using f(x) on the input data, t_1...t_n
    '''

    # Import packages
    import matplotlib.pyplot as plt
    import numpy as np

    # Init variables
    PI = np.pi
    input_data = np.arange(1, step=1.0/num_pts)

    # Seed random number generator
    if rnd_seed != 0:
        np.random.seed(rnd_seed)

    # Ensure row vector
    input_data = np.reshape(input_data, (1, num_pts))

    # Generate noisy data
    target_data = 7*np.sin(2*PI*input_data) + np.random.randn(num_pts)

    # Check whether to plot data
    if show_plot:
        plt.plot(input_data, target_data, 'bo')
        plt.xlabel('time points, t')
        plt.ylabel('f(t)')
        plt.show()

    # Return the input data
    return input_data, target_data

# Plot data
def plot_data(x_mat, target_data, w_star):
    '''
    Function to plot input, target, and best-fit data

    Parameters
    ----------
    x_mat : numpy.ndarray
        a (N x poly_order+1) matrix with the input data tiled along
        the column axis (used in calculating w_star via matrix algebra)
    target_data : numpy.ndarray
        a vector of length N of the target data
    '''

    # Import packages
    import matplotlib.pyplot as plt
    import numpy as np

    # Init variables
    num_pts, poly_order = np.shape(x_mat)
    input_data = x_mat[:, 1]

    # Calculate the best-fit line
    best_fit = np.dot(x_mat, w_star)

    # Plot
    plt.plot(input_data.flatten(), target_data.flatten(), 'bo',
             input_data.flatten(), best_fit.flatten(), 'r')
    plt.xlabel('input data, x')
    plt.ylabel('target data, y')
    plt.title('Polynomial fit, M = %d, N = %d' % (poly_order-1, num_pts))
    plt.show()

# Calculate polynomial weights and plot best fit
def main(poly_order, input_data, target_data):
    '''
    Function to take input and target data, fit a polynomial line to
    the data and plot the result

    Parameters
    ----------
    poly_order : integer
        the order of the polynomial to fit to the data
    input_data : numpy.ndarray
        input data points to use
    target_data : numpy.ndarray
        target data points to use

    Returns
    -------
    None
        this function finds the best-fit and shows a plot
    '''

    # Init variables
    num_pts = 100

    # If input is not specified, generate noisy input
    if not input_data:
        input_data, target_data = generate_input(num_pts, show_plot=False)

    # Calculate w* and X
    w_star, x_mat = calc_coeffs(input_data, target_data, poly_order)

    # Print weight vector
    print 'Weight vector for polynomial:'
    print w_star

    # Plot the data
    plot_data(x_mat, target_data, w_star)

    # Return the weights vector
    return w_star

# Make executable
if __name__ == '__main__':

    # Import packages
    import argparse
    import numpy as np
    import sys

    # Init argparser
    parser = argparse.ArgumentParser(description=__doc__)

    # Init arguments
    parser.add_argument('-m', '--order', nargs=1, required=False,
                        help='Order of polynomial to fit to data')
    parser.add_argument('-i', '--input', nargs=1, required=False,
                        help='Path to .npy file with input data to fit')
    parser.add_argument('-t', '--target', nargs=1, required=False,
                        help='Path to .npy file with target data to fit')

    # Parse arguments
    args = parser.parse_args()

    # Init argument variables
    try:
        poly_order = int(args.order[0])
        if poly_order < 1:
            print 'Polynomial order must be >= 1!\nSpecified: %d' % poly_order
            sys.exit()
    except TypeError as exc:
        poly_order = 5
        print 'No variable specified for polynomial order; default = %d' \
              % poly_order
    try:
        input_data = np.load(args.input[0])
        target_data = np.load(args.target[0])
    except IOError as exc:
        print 'Could not import data.\nError: %s' % exc
    except TypeError as exc:
        input_data = None
        target_data = None
        print 'Data files were not specified; generating default input = '\
              't = 7*sin(2*pi*x) + N(0,1)'

    # Run main routine
    w_star = main(poly_order, input_data, target_data)


import numpy as np

'''
Implementation of the continuous Dice Coefficient (https://www.biorxiv.org/content/10.1101/306977v1.full.pdf)
"Continuous Dice Coefficient: a Method for Evaluating Probabilistic Segmentations"
Reuben R Shamir,Yuval Duchin, Jinyoung Kim, Guillermo Sapiro, and Noam Harel

Input:
A - ground-truth or gold-standard binary segmentation (expert labeled data; assumes values are 0 or 1)
B - segmentation probabilistic map (your algorithm's output; assumes values are between 0-1)

Author: Ruby Shamir (feedback is welcome at shamir.ruby at gmail)
'''

def continous_Dice_coefficient(A_binary, B_probability_map):

    AB = A_binary * B_probability_map
    c = np.sum(AB)/max(np.size(AB[AB>0]), 1)
    cDC = 2*(np.sum(AB))/(c*np.sum(A_binary) + np.sum(B_probability_map))

    return cDC

def Dice_coefficient(A_binary, B_binary):

    AB = A_binary * B_binary
    DC = 2*(np.sum(AB))/(np.sum(A_binary) + np.sum(B_binary))

    return DC

def simulate_probablistic_segmentation (start, end):
    x, y = np.meshgrid(np.linspace(start, end, 100), np.linspace(start, end, 100))
    d = np.array(np.sqrt(x * x + y * y))
    mu = 0.0
    sigma = 2.0
    segmentation_result = np.exp(-((d - mu) * (d - mu) / (2.0 * sigma * sigma)))
    segmentation_result[segmentation_result<0.01] = 0
    return segmentation_result

## compare Dice and continous Dice under simulated error #########
if __name__ == '__main__':

    # in this example we simulate a ground truth segmentation (circle) and probabilistic segmentation results (gaussian)
    # the we demonstrate the cDC is less sensitive for shifts in segmentation than Dice Coefficient

    all_cDice = list()
    all_Dice = list()

    start = -10
    end = 10
    segmentation_result = simulate_probablistic_segmentation (start, end)

    ground_truth_simulated = np.ones_like(segmentation_result)
    ground_truth_simulated[segmentation_result < 0.01] = 0

    cDC = continous_Dice_coefficient(ground_truth_simulated, segmentation_result)
    all_cDice.append(cDC)
    binary_segmentation_result = np.zeros_like(segmentation_result)
    binary_segmentation_result[segmentation_result > 0.01] = 1
    DC = Dice_coefficient(ground_truth_simulated, binary_segmentation_result)
    all_Dice.append(DC)
    step = 2
    for shift in range(0, 4):
        segmentation_result = np.hstack((segmentation_result, np.zeros((segmentation_result.shape[0],step))))
        segmentation_result = np.delete(segmentation_result, range(0, step),1)

        cDC = continous_Dice_coefficient(ground_truth_simulated, segmentation_result)
        all_cDice.append(cDC)

        binary_segmentation_result = np.zeros_like(segmentation_result)
        binary_segmentation_result[segmentation_result>0.01] = 1
        # when the input is binary continues Dice Coefficient returns the Dice Coefficient.
        DC = Dice_coefficient(ground_truth_simulated, binary_segmentation_result)
        all_Dice.append(DC)

    all_cDice = [str(round(val,2)) for val in all_cDice]
    all_Dice = [str(round(val, 2)) for val in all_Dice]

    print ('Shift errors of: (mm)')
    print ([str(round(i,2)) for i in range(0, 10, 2)])
    print('Reduced the continues Dice:')
    print (all_cDice)
    print('And the original Dice is:')
    print (all_Dice)


# =============================================================================
#  ***** num_derivatives.py *****
#  Python script to plot numerical derivative information.
#
#  Author:     Ryan Clement
#  Created:    July 2021
#
#  Change Log:
#  Who:
#  Date:       MM/DD/YYY
#  What:
#
#  Who:
#  Date:       MM/DD/YYYY
#  What:
# =============================================================================

import numpy as np
from matplotlib import pyplot as plt

### Functions
def parabola(x):
    return 10*x - x**2
def paraDeriv(x):
    return 10 - 2*x
def slope(xa,xb,ya,yb):
    return (yb-ya)/(xb-xa)
def getYN(xa,xb,xd,ya,yb,yd,x):
    return yd + slope(xa,xb,ya,yb)*(x-xd)
def getYE(xd,yd,x):
    return yd + paraDeriv(xd)*(x-xd)
def nD(xo,dx):
    return (np.sin(xo+dx) - np.sin(xo))/dx
def nD2(xo,dx):
    return (np.sin(xo+dx) - np.sin(xo-dx))/(2.0*dx)

a = 0
b = 10
xPoints = np.linspace(a,b,11)
yPoints = parabola(xPoints)
xSmooth = np.linspace(a,b,101)
ySmooth = parabola(xSmooth)
x3 = xPoints[3]
x4 = xPoints[4]
x5 = xPoints[5]
x6 = xPoints[6]
x7 = xPoints[7]
y3 = yPoints[3]
y4 = yPoints[4]
y5 = yPoints[5]
y6 = yPoints[6]
y7 = yPoints[7]
xDers = np.array([x3,x7])

# Exact
ya = getYE(x5,y5,x3)
yb = getYE(x5,y5,x7)
yExact = np.array([ya,yb])

# Centered
ya = getYN(x4,x6,x5,y4,y4,y5,x3)
yb = getYN(x4,x6,x5,y4,y4,y5,x7)
yCent = np.array([ya,yb])
# print('Centered: ',(yb-ya)/(x6-x4))

# Left
ya = getYN(x4,x5,x5,y4,y5,y5,x3)
yb = getYN(x4,x5,x5,y4,y5,y5,x7)
yLeft = np.array([ya,yb])
# print('Left: ',(yb-ya)/(x5-x4))

# Right
ya = getYN(x5,x6,x5,y5,y4,y5,x3)
yb = getYN(x5,x6,x5,y5,y4,y5,x7)
yRight = np.array([ya,yb])
# print('Right: ',(yb-ya)/(x6-x5))

# Plot
figP, axP = plt.subplots()
axP.set_title(r'Parabola: $y = 10x - x^2$')
axP.set_xlabel('x')
axP.set_ylabel('y')
axP.set_xticks(xPoints)
axP.plot(xSmooth, ySmooth, 'k')
axP.plot(xPoints, yPoints, 'ko')
axP.plot([x4,x4],[0,y4],'--b')
axP.plot([x5,x5],[0,y5],'--',color='purple')
axP.plot([x6,x6],[0,y6],'--r')
axP.plot(xDers,yCent,'y',label='Centered')
axP.plot(xDers,yLeft,'b',label='Left')
axP.plot(xDers,yRight,'r',label='Right')
axP.plot(xDers,yExact,'--k',label='Exact')
axP.legend()
# figP.savefig('parabola_derivative.png')

### Sine Error
x0 = 1.0
exact = np.cos(x0)
dx = np.logspace(-16,-1,1000)
diff = np.abs(nD(x0,dx) - exact)
# Plot
figC, axC = plt.subplots()
axC.set_xlabel(r'$\Delta x$')
axC.set_ylabel('Error')
# axC.set_xlim([1e-16,1e-1])
axC.loglog(dx,diff,'r')
# figC.savefig('sine_derivative.png')

### Sine Order
dx = np.logspace(-9,-1,1000)
diffFO = np.abs(nD(x0,dx) - exact)
diffSO = np.abs(nD2(x0,dx) - exact)
# Plot
figS, axS = plt.subplots()
axS.set_xlabel(r'$\Delta x$')
axS.set_ylabel('Error')
axS.loglog(dx,diffFO,'b',label=r'$O(\Delta x)$')
axS.loglog(dx,diffSO,'r',label=r'$O(\Delta x^2)$')
axS.legend()
# figS.savefig('sine_derivative_order.png')


"""
This is a simple test that differentiates the Moore-Penrose pseudo-inverse
computation. Explicitly:

..math::
    A^\dagger = (A^T A)^{-1} A^T

Two methods are compared. First the naive approach by first computing A^T A,
then invert it and then multiplication with A^T.
Then the QR approach.

"""

import numpy
from algopy import CGraph, Function, UTPM, dot, qr, eigh, inv, solve

D,P,M,N = 2,1,5,2

# generate badly conditioned matrix A
A = UTPM(numpy.zeros((D,P,M,N)))
x = UTPM(numpy.zeros((D,P,M,1)))
y = UTPM(numpy.zeros((D,P,M,1)))

x.data[0,0,:,0] = [1,1,1,1,1]
x.data[1,0,:,0] = [1,1,1,1,1]

y.data[0,0,:,0] = [1,2,1,2,1]
y.data[1,0,:,0] = [1,2,1,2,1]

alpha = 10**-5
A = dot(x,x.T) + alpha*dot(y,y.T)
A = A[:,:2]

# Method 1: Naive approach
Apinv = dot(inv(dot(A.T,A)),A.T)

print('naive approach: A Apinv A - A = 0 \n', dot(dot(A, Apinv),A) - A)
print('naive approach: Apinv A Apinv - Apinv = 0 \n', dot(dot(Apinv, A),Apinv) - Apinv)
print('naive approach: (Apinv A)^T - Apinv A = 0 \n', dot(Apinv, A).T  - dot(Apinv, A))
print('naive approach: (A Apinv)^T - A Apinv = 0 \n', dot(A, Apinv).T  - dot(A, Apinv))

# Method 2: Using the differentiated QR decomposition
Q,R = qr(A)
tmp1 = solve(R.T, A.T)
tmp2 = solve(R, tmp1)
Apinv = tmp2

print('QR approach: A Apinv A - A = 0 \n',  dot(dot(A, Apinv),A) - A)
print('QR approach: Apinv A Apinv - Apinv = 0 \n', dot(dot(Apinv, A),Apinv) - Apinv)
print('QR approach: (Apinv A)^T - Apinv A = 0 \n', dot(Apinv, A).T  - dot(Apinv, A))
print('QR approach: (A Apinv)^T - A Apinv = 0 \n', dot(A, Apinv).T  - dot(A, Apinv))


"""
Modify model_g, change the prior for the mean to a Gaussian distribution
centered at the empirical mean, and play with a couple of reasonable values for
the standard deviation of this prior. How robust/sensitive are the inferences
to these changes? What do you think of using a Gaussian, which is an unbounded
distribution (goes from -∞ to ∞), to model bounded data such as this? Remember
that we said it is not possible to get values below 0 or above 100.

- doesn't appear very sensitive at all
"""

import pymc3 as pm
import pandas as pd
from pandas import DataFrame, Series
import numpy as np

data = Series(np.loadtxt('../data/chemical_shifts.csv'), name='shift')
data_mean = data.mean()

#########
# in book
#########
with pm.Model() as model_g:
  mu = pm.Uniform('mu', lower=40, upper=70)
  sigma = pm.HalfNormal('sigma', sd=10)
  y = pm.Normal('y', mu=mu, sd=sigma, observed=data)

  trace_g = pm.sample(1000)

pm.plot_trace(trace_g)

################
# modification 1
################
with pm.Model() as model1:
  mu = pm.Normal('mu', data_mean, 5)
  sigma = pm.HalfNormal('sigma', sd=10)
  y = pm.Normal('y', mu=mu, sd=sigma, observed=data)
  trace1 = pm.sample(1000)

pm.plot_trace(trace1)

################
# modification 2
################
with pm.Model() as model1:
  mu = pm.Normal('mu', data_mean, 10)
  sigma = pm.HalfNormal('sigma', sd=10)
  y = pm.Normal('y', mu=mu, sd=sigma, observed=data)
  trace2 = pm.sample(1000)

pm.plot_trace(trace2)
pm.plot_posterior(trace2)


import numpy as np
import matplotlib.pyplot as plt
from scipy.stats import multivariate_normal

##########################################################################################################
################################################### Part a ###############################################
##########################################################################################################
# Loading Dataset
x = []
f =  open("data/data/faithful/faithful.txt",'r')
for line in f.readlines():
    x.append([float(i) for i in line.strip().split(" ")])
x = np.array(x)

#Normalise the data within [-1, 1]
x =  (x - np.mean(x, axis=0))*(1/(np.max(x,axis=0) - np.min(x, axis=0)))
plt.scatter(x[:,0], x[:,1])
plt.show()

# Number of clusters
k = 2

# Initialising Cluster parameters
mean = np.array([[-1, 1],[1, -1]])
covariance = np.tile(0.1 * np.eye(2), (k,1,1))
mix = np.ones((k,1))/k

print("Initialisation done. \n mean = \n", mean, "\n covariance = \n", covariance, "\n mixing coefficients = \n", mix)

def e_step(x, k, mean, covariance, mix):
    gamma = np.zeros((x.shape[0], k))
    for i in range(k):
        gamma[:,i] = mix[i]*multivariate_normal.pdf(x=x, mean=mean[i], cov=covariance[i])
    temp = np.tile(1/np.sum(gamma, axis=1), (2,1)).transpose()
    return(gamma*temp)

def m_step(x, k, gamma):
    mix = np.sum(gamma, axis=0)/np.sum(gamma)
    mean = np.zeros((k,x.shape[1]))
    covariance = np.zeros((k, x.shape[1], x.shape[1]))
    for i in range(k):
        temp1 = gamma[:,i].reshape(gamma.shape[0],1)
        mean[i] = np.sum(x*temp1, axis=0)/np.sum(gamma[:,i])
        temp2 = x - mean[i]
        temp3 = 0
        for j in range(x.shape[0]):
            temp3 = temp3+gamma[j,i]*np.matmul(temp2[j].reshape(-1,1), temp2[j].reshape(-1,1).transpose())
        covariance[i] = temp3/np.sum(gamma[:,i])
    return mean, covariance, mix

# Performing Updates
max_iter = 100
plot_iter = [1,2,5,100]
for i in range(1, max_iter+1):
    gamma = e_step(x, k, mean, covariance, mix)
    mean, covariance, mix = m_step(x, k, gamma)

    # Plotting at desired locations
    if(i in plot_iter):
        cluster_id = np.argmax(gamma, axis=1)
        colours = 'rb'
        plt.clf()
        for j in range(k):
            temp = (cluster_id==j)
            plt.scatter(x[temp,0], x[temp,1])
        plt.plot(mean[:,0], mean[:,1],'bD')
        plt.title("Iteration {}".format(i))
        plt.show()

##########################################################################################################
################################################### Part b ###############################################
##########################################################################################################
# Loading Dataset
x = []
f =  open("data/data/faithful/faithful.txt",'r')
for line in f.readlines():
    x.append([float(i) for i in line.strip().split(" ")])
x = np.array(x)

#Normalise the data within [-1, 1]
x =  (x - np.mean(x, axis=0))*(1/(np.max(x,axis=0) - np.min(x, axis=0)))

# Number of clusters
k = 2

# Initialising Cluster parameters
mean = np.array([[-1, -1],[1, 1]])
covariance = np.tile(0.5 * np.eye(2), (k,1,1))
mix = np.ones((k,1))/k

print("Initialisation done. \n mean = \n", mean, "\n covariance = \n", covariance, "\n mixing coefficients = \n", mix)

# Performing Updates
max_iter = 100
plot_iter = [1,2,5,100]
for i in range(1, max_iter+1):
    gamma = e_step(x, k, mean, covariance, mix)
    mean, covariance, mix = m_step(x, k, gamma)

    # Plotting at desired locations
    if(i in plot_iter):
        cluster_id = np.argmax(gamma, axis=1)
        colours = 'rb'
        plt.clf()
        for j in range(k):
            temp = (cluster_id==j)
            plt.scatter(x[temp,0], x[temp,1])
        plt.plot(mean[:,0], mean[:,1],'bD')
        plt.title("Iteration {}".format(i))
        plt.show()

##########################################################################################################
################################################### Part c ###############################################
##########################################################################################################
# Loading Dataset
x = []
f =  open("data/data/faithful/faithful.txt",'r')
for line in f.readlines():
    x.append([float(i) for i in line.strip().split(" ")])
x = np.array(x)

#Normalise the data within [-1, 1]
x =  (x - np.mean(x, axis=0))*(1/(np.max(x,axis=0) - np.min(x, axis=0)))

# Number of clusters
k = 2

# Initialising Cluster parameters
mean = np.array([[0, 0],[0, 0]])
covariance = np.tile(0.5 * np.eye(2), (k,1,1))
mix = np.ones((k,1))/k

print("Initialisation done. \n mean = \n", mean, "\n covariance = \n", covariance, "\n mixing coefficients = \n", mix)

# Performing Updates
max_iter = 100
plot_iter = [1,2,5,100]
for i in range(1, max_iter+1):
    gamma = e_step(x, k, mean, covariance, mix)
    mean, covariance, mix = m_step(x, k, gamma)

    # Plotting at desired locations
    if(i in plot_iter):
        cluster_id = np.argmax(gamma, axis=1)
        colours = 'rb'
        plt.clf()
        for j in range(k):
            temp = (cluster_id==j)
            plt.scatter(x[temp,0], x[temp,1])
        plt.plot(mean[:,0], mean[:,1],'bD')
        plt.title("Iteration {}".format(i))
        plt.show()


from scipy.stats import norm
import matplotlib.pyplot as plt
import numpy as np

x = np.linspace(-5, 5, 100)
zoomed_x = np.linspace(0, 5, 100)

data = [
    { 'x': x, 'fn': norm.pdf, 'c': 'red',  'ls': '-', 'label': 'pdf', 'title': 'Probability density function (pdf)' },
    { 'x': x, 'fn': norm.cdf, 'c': 'blue', 'ls': ':', 'label': 'cdf', 'title': 'Cumulative distribution function (cdf)' },
    { 'x': x, 'fn': norm.sf, 'c': 'black', 'ls': '-.', 'label': 'sf', 'title': 'Survival function (1-cdf)' },
 #   { 'x': x, 'fn': norm.sf, 'c': 'darkgreen', 'ls': '-.', 'label': 'log sf', 'title': 'Survival function (log)', 'yscale': 'log' },
    { 'x': zoomed_x, 'fn': norm.sf, 'c': 'black', 'ls': '-.', 'label': 'survival', 'title': 'Survival zoom [0,5]', 'yscale': 'log' },
]

# plots
factor = 1.7
rows,cols=2,2
fig, axes = plt.subplots(rows, cols,
    figsize=(cols*3*factor, rows*2*factor),
    tight_layout=False)
fig.suptitle('Gaussian things')

ax = axes.flatten()

for i,d in enumerate(data):
    ax[i].plot(d['x'], d['fn'](d['x']), color=d['c'], linestyle=d['ls'], label=d['label'])
    ax[i].set_title(d['title'])
    if 'yscale' in d:
        ax[i].set_yscale(d['yscale'])

# sigma on pdf
for i in range(1,6):
    for j in -i, i:
        ax[0].plot((j,j), (0, norm.pdf(j)), color='gray', linestyle='-', alpha=0.5)

    ax[0].text(x=2, y=norm.pdf(0)*0.6,
        s='1σ: {:.4%}\n'.format(norm.cdf(1)-norm.cdf(-1)) \
         +'2σ: {:.4%}\n'.format(norm.cdf(2)-norm.cdf(-2)) \
         +'3σ: {:.4%}\n'.format(norm.cdf(3)-norm.cdf(-3)) \
         +'4σ: {:.4%}\n'.format(norm.cdf(4)-norm.cdf(-4)) \
         +'5σ: {:.4%}'.format(norm.cdf(5)-norm.cdf(-5)))

# annotation on cumulative
for i in range(0,6):
    if i < 3:
        text_pos = (-5, 0.4+i*0.25)
    else:
        text_pos = ((i-3)*1.4, 0.3+(i-3)*-0.12)
    fmt='{:.'+f'{i}'+'%}'
    ax[1].annotate(fmt.format(norm.cdf(i)),
        xy=(i, norm.cdf(i)),
        xytext=text_pos,
        arrowprops={ 'arrowstyle': '->' },
        bbox={ 'boxstyle': 'round', 'fc': 'white', 'alpha': 0.9 })
    ax[1].plot((i,i), (0, norm.cdf(i)), color='red', linestyle='--', alpha=0.5)

# annotation on survival zoom
for i in range(6):
    if i < 3:
        text_pos = (i+0.1, 0.5*10**-(i+2))
    else:
        text_pos = (i-1.0, 0.5*10**-(i-2))
    fmt='{:.'+f'{i}'+'%}'
    ax[3].annotate(fmt.format(norm.sf(i)),
        xy=(i, norm.sf(i)),
        xytext=text_pos,
        arrowprops={ 'arrowstyle': '->' })
    # vertical lines
    ax[3].plot((i,i), (0, norm.sf(i)), color='red', linestyle='--', alpha=0.5)

plt.show()


import dislog
from dislog.util import debug
from sympy.ntheory.modular import crt

def pohlighellman(alpha, beta, n, n_factors):
    """Computes discrete logarithm using Pohlig-Hellman algorithm.

    Given a generator alpha of a cyclic group G, another element beta of the
    group G, the order n of the group G and the prime factorization of n,
    computes the discrete logarithm of beta to the base of alpha using the
    Pohlig-Hellman algorithm.

    Args:
        alpha: logarithm base, must support internal equality and
            multiplication and integer exponentiation; should be a generator to
            guarantee the existence of the logarithm
        beta: logarithm argument, must support internal equality and
            multiplication
        n: order of the group containing alpha and beta
        n_factors: dictionary containing the prime factors of n as keys and
            their multiplicity as values

    Returns:
        The discrete logarithm log_{alpha}(beta) (the integer x such that alpha
        to the power of x equals beta) if it exists, None otherwise
    """
    debug(
        pohlighellman,
        "alpha={} beta={} n={} factors={}", alpha, beta, n, n_factors
    )

    # List of remainder values and moduli to be solved with C.r.t.
    remainders = []
    moduli = []

    # For each factor p, store x mod (p ^ e) together with the modulus
    for p, e in n_factors.items():
        debug(pohlighellman, "Factor: {}^{}", p, e)

        # Initialize reduced logarithm base, constant
        base = alpha ** (n // p)

        # Initialize helper variables
        arg_base = beta
        cur_pow = 0
        next_pow = 1
        l = 0
        rem = 0

        # Compute expansion coefficients
        for j in range(e):
            debug(pohlighellman, "Calculating l_({})", j)

            prev_l = l  # l_{j - 1}

            prev_pow = cur_pow       # p ^ (j - 1) for j > 0
            cur_pow = next_pow       # p ^ (j)
            next_pow *= p            # p ^ (j + 1)

            arg_base *= alpha ** (- prev_l * prev_pow)
            arg_exp = n // next_pow
            arg = arg_base ** arg_exp

            l = dislog.exhaustive(base, arg, n)

            if l is None:
                debug(pohlighellman, "Could not calculate reduced logarithm")
                return None
            debug(pohlighellman, "l_({})={}", j, l)

            rem += l * cur_pow

        debug(pohlighellman, "Found congruence: x = {} mod ({}^{})", rem, p, e)

        remainders.append(rem)
        moduli.append(p ** e)

    debug(pohlighellman, "Applying Chinese remainder theorem")
    ret = crt(moduli, remainders)

    if ret is not None:
        ret = ret[0]
        debug(pohlighellman, "Logarithm={}", ret)
    else:
        debug(pohlighellman, "No solution found")

    return int(ret)


import numpy as np

def mean_square_error(y_true: np.ndarray, y_pred: np.ndarray) -> float:
    """
    Calculate MSE loss

    Parameters
    ----------
    y_true: ndarray of shape (n_samples, )
        True response values
    y_pred: ndarray of shape (n_samples, )
        Predicted response values

    Returns
    -------
    MSE of given predictions
    """
    __check_labels_sizes(y_pred, y_true)
    sum_squared_error = np.sum(np.power(y_true - y_pred, 2))
    return sum_squared_error / len(y_true)

def misclassification_error(y_true: np.ndarray, y_pred: np.ndarray, normalize: bool = True) -> float:
    """
    Calculate misclassification loss

    Parameters
    ----------
    y_true: ndarray of shape (n_samples, )
        True response values
    y_pred: ndarray of shape (n_samples, )
        Predicted response values
    normalize: bool, default = True
        Normalize by number of samples or not

    Returns
    -------
    Misclassification of given predictions
    """
    __check_labels_sizes(y_pred, y_true)
    misses = np.sum([True if y_true[index] != y_pred[index] else False for index in range(len(y_true))])
    return misses / len(y_true) if normalize else misses

def accuracy(y_true: np.ndarray, y_pred: np.ndarray) -> float:
    """
    Calculate accuracy of given predictions

    Parameters
    ----------
    y_true: ndarray of shape (n_samples, )
        True response values
    y_pred: ndarray of shape (n_samples, )
        Predicted response values

    Returns
    -------
    Accuracy of given predictions
    """
    __check_labels_sizes(y_pred, y_true)
    misses = np.sum([True if y_true[index] == y_pred[index] else False for index in range(len(y_true))])
    return misses / len(y_true)

def __check_labels_sizes(y_pred, y_true):
    if len(y_true) != len(y_pred):
        raise Exception(f"predicted labels and true label are different in their sizes."
                        f" func: {misclassification_error.__name__}")

def cross_entropy(y_true: np.ndarray, y_pred: np.ndarray) -> float:
    """
    Calculate the cross entropy of given predictions

    Parameters
    ----------
    y_true: ndarray of shape (n_samples, )
        True response values
    y_pred: ndarray of shape (n_samples, )
        Predicted response values

    Returns
    -------
    Cross entropy of given predictions
    """
    raise NotImplementedError()


# Python > Classes > Class 2 - Find the Torsional Angle
# Find the angle between two planes.
#
# https://www.hackerrank.com/challenges/class-2-find-the-torsional-angle/problem
#

import math
# (skeliton_head) ----------------------------------------------------------------------

class Points(object):
    def __init__(self, x, y, z):
        self.x, self.y, self.z = x, y, z

    def __sub__(self, no):
        return Points(self.x - no.x, self.y - no.y, self.z - no.z)

    def dot(self, no):
        """ produit scalaire """
        return self.x * no.x + self.y * no.y + self.z * no.z

    def cross(self, no):
        """ produit vectoriel """
        return Points(self.y * no.z - self.z * no.y,
                      self.z * no.x - self.x * no.z,
                      self.x * no.y - self.y * no.x)

    def absolute(self):
        return pow((self.x ** 2 + self.y ** 2 + self.z ** 2), 0.5)

    def __str__(self):
        return "[{} {} {}]".format(self.x, self.y, self.z)

# (skeliton_tail) ----------------------------------------------------------------------
if __name__ == '__main__':
    points = list()
    for i in range(4):
        a = list(map(float, input().split()))
        points.append(a)

    a, b, c, d = Points(*points[0]), Points(*points[1]), Points(*points[2]), Points(*points[3])
    x = (b - a).cross(c - b)
    y = (c - b).cross(d - c)
    angle = math.acos(x.dot(y) / (x.absolute() * y.absolute()))
    print("%.2f" % math.degrees(angle))

    if False:
        import numpy as np
        a = np.array(points[0])
        b = np.array(points[1])
        c = np.array(points[2])
        d = np.array(points[3])
        x = np.cross(b - a, c - b)
        y = np.cross(c - b, d - c)
        angle = math.acos(x.dot(y) / (np.linalg.norm(x) * np.linalg.norm(y)))
        print("%.2f" % math.degrees(angle))


# Find highest positive y velocity that gets us into the rectangle
# having the following bounding coordinates

# %%
import numpy as np

# %%
# target area: x=137..171, y=-98..-73
x1, x2 = (137, 171)  # inclusive
y1, y2 = (-98, -73)

#x1,x2=(20,30)
#y1,y2=(-10,-5)

# Let's find the starting velocity `vx` range.
# Upper bound is overshooting `x2` in the first step.
# Lower bound is when 1+2+...+vx = x1
# or vx*(vx+1)/2 < x1 ==> vx^2+vx-2*x1 = 0
# vx = (-1 + sqrt(1 + 8*x1))/2

assert(x1 > 0)
vx_upper_bound = x2  # single step
vx_lower_bound = int((np.sqrt(1 + 8*x1)-1)/2)  # also min number of steps

# Let's take care of Y coordinate.
# Suppose as an example that dy_0 = 4. We'll get
# 4, 7, 9, 10, 10, 9, 7, 4, 0, -5, ...
# So, after 2*dy_0+1 steps we're back to y=0 with dy=-dy_0-1
# So: dy_0 = dy+1 when back at y=0, and dy <= y1 in order not to overshoot in one step
# ==> dy_0 <= -y1+1

vy_lower_bound = y1-1
vy_upper_bound = -y1+2

# %%
def simulate(dx, dy, x1, x2, y1, y2):
    x, y = (0, 0)
    while True:
        if x1 <= x <= x2 and y1 <= y <= y2:
            return True
        if x > x2 or y < y1:
            return False
        x += dx
        y += dy
        dx = dx-1 if dx > 0 else 0
        dy = dy-1
# %%
best_dy = -1000000
best_dx = 0
num_solutions = 0
for dx in range(vx_lower_bound, vx_upper_bound+1):
    for dy in range(vy_upper_bound, vy_lower_bound-1, -1):
        if simulate(dx, dy, x1, x2, y1, y2):
            num_solutions += 1
            if dy > best_dy:
                best_dx, best_dy = dx, dy
                best_dy = dy
            # break   # commented due to part 2
max_height = best_dy * (best_dy+1) // 2
assert(best_dy > 0)
print('Best dx,dy:',best_dx, best_dy)
print('Max height (part 1):', max_height)
print('Number of solutions (part 2):', num_solutions)
# %%


def dr(prox_f,prox_g,dim,gamma = 1,rho = 1,Z0 = None, tol = None):
    """
    Douglas-Rachford proximal splitting algorithm with two proximal
    mappings as inputs

    DR(prox_f,prox_g,dim) runs the Douglas-Rachford splitting algorithm for the objective function
    f + g, where:
        1. prox_f(Z,gamma) and prox_g(Z,gamma) are funcionts that derive the proximal mappings of
        gamma*f and gamma*g evalutaed in Z.
        2. dim is the dimension of the output of f and g.

    Note: If f and g are convex, then X = argmin(f(X) + g(X)).

    X,Z_fix,iter,D = DR(prox_f,prox_g,dim) returns:
        1. Solution X
        2. Z_fix = fix point of that Douglas-Rachford iterations
        3. iter = total number of Douglas-Rachford iterations
        4. D = Z_fix - X, which is the solution of the dual problem if f
        and g are convex.

    ... = DR(prox_f,prox_g,dim,option) allows to specify further options:
        1. ... = DR(prox_f,prox_g,dim,...,rho = rho_val,...) set the step
        length update of the fix-point update, i.e.,
        Z_{k+1} = Z_k + rho*(Y_k - X_k), where 0 < rho < 2.
        The default value is rho = 1.
        2. ... = DR(prox_f,prox_g,dim,...,gamma = gamma_val,...) sets gamma
        to another value than the default gamma = 1.
        3. ... = DR(prox_f,prox_g,dim,...,Z0 = Z0_val,...) sets the
        initial value of the fix-point iteration.
        The default choice is Z0 = 0.
        4. ... = DR(prox_f,prox_g,dim,...,tol=tol_val,...)
        sets the tolerance for zero entries as well as to stop the
        iteratrions once norm(Y_k-X_k,'fro') < tol
        The default tol-value is sqrt(eps).
    """

    import numpy as np

    # Standard value for Z0 and tol
    if Z0 is None:
        Z = np.zeros(dim)
    else:
        Z = Z0

    if tol is None:
        tol = np.sqrt(np.finfo(float).eps)

    # Initialize iteration counter
    iter = 0

    # Initialize X, Y and error between X and Y
    X = np.zeros(dim)
    Y = np.zeros(dim)
    err_XY = tol+1 # Larger than the tol so that loop starts

    # Set step-size for printing err_XY
    err_bound = 1e-1 # Error bound step-size for display

    # Compute the Douglas-Rachford iterations

    while err_XY >= tol:

        iter += 1 # Increase counter

        # Display error between X_iter and Y_iter
        if err_XY <= err_bound:
            print('Error between X_'+str(iter)+' and '+'Y_'+str(iter)+' <= '+str(err_bound))
            err_bound /= 10

        ## Compute Douglas-Rachford steps
        X = prox_f(Z=Z,gamma=gamma)
        # Check if X is tuple in case of multiple outputs
        if isinstance(X, tuple):
            X = X[0]
        # Check if Y is tuple in case of multiple outputs
        Y = prox_g(Z=2*X-Z,gamma=gamma)
        if isinstance(Y, tuple):
            Y = Y[0]
        Z = Z+rho*(Y-X)

        # Update iteration error ||X_k - Y_k||_F
        err_XY = np.linalg.norm(X-Y,ord = 'fro')

    # Set the final solution
    X = Y

    # Compute dual variable D
    D = (Z-X)/gamma
    Z[(np.absolute(Z) < 0)] = 0

    # Set fix point
    if iter > 0:
        Z_fix = Z
    else:
        # Set fix point if iter = 0 and D = 0
        Z_fix = X

    X[np.absolute(X) < tol] = 0

    return X,Z_fix,iter,D


"""Tool for solving the ordinary differential equation of order 2
by the tridiagonal matrix algorithm and finite difference method.

Functions:
    solve_TDMA(below, main, above, vector) -> ndarray
    solve_diffeq(px, qx, fx, alpha1, alpha2, A,
                 beta1, beta2, B, h, a, b) -> tuple(ndarray, ndarray)
"""
from numpy import zeros, array
from sympy import sympify, symbols

def solve_diffeq(px, qx, fx, alpha1, alpha2, A, beta1, beta2, B, h, a, b):
    """solve differential equation of 2nd order:

    Arguments from the equation below:
    y'' + p(x) y' + q(x) y = f(x), x in [a, b]
    alpha1 * y(a) + alpha2 * y'(a) = A
    beta1 * y(b) + beta2 * y'(b) = B
    """
    x = symbols('x')
    px, qx, fx = sympify(px), sympify(qx), sympify(fx)
    if (alpha1 ** 2 + alpha2 ** 2 == 0 or
        beta1 ** 2 + beta2 ** 2 == 0 or
        alpha1 * alpha2 > 0 or
        beta1 * beta2 < 0):
        print('the algorithm is unstable for these boundary conditions')

    n = int((b - a) / h)
    ac, bc, cc, gc = zeros(n), zeros(n + 1), zeros(n), zeros(n + 1)
    for i in range(1, n):
        ac[i] = 1 + h / 2 * px.subs({x: a + i * h})
        bc[i] = - 2 + h ** 2 * qx.subs({x: a + i * h})
        cc[i - 1] = 1 - h / 2 * px.subs({x: a + i * h})
        gc[i] = h ** 2 * fx.subs({x: a + i * h})

    fraction = (cc[0] * alpha2 + 2 * h * ac[1] * alpha1 - 3 * alpha2 * ac[1])
    kappa1 = alpha2 * (-bc[1] - 4 * ac[1]) / fraction
    nu1 = (2 * h * ac[1] * A + alpha2 * gc[1]) / fraction

    fraction = beta2 * (ac[n - 1] - cc[n - 2] * (beta1 * 2 * h + 3 * beta2))
    kappa2 = beta2 * (-bc[n - 1] - 4 * cc[n - 2]) / fraction
    nu2 = (gc[n - 1] * beta2 - 2 * h * B * cc[n - 2]) / fraction

    ac[0] = - kappa1
    cc[n - 1] = - kappa2
    bc[0], bc[n] = 1, 1
    gc[0], gc[n] = nu1, nu2
    ys = solve_TDMA(cc, bc, ac, gc)
    return array([a + i * h for i in range(n + 1)]), ys

def solve_TDMA(below, main, above, vector):
    '''solve SoLE Ax = b by Tri Diagonal Matrix Algorithm

    Arguments:
    below -- the first diagonal below the main diagonal
    main -- main diagonal
    above --  the first diagonal above the main diagonal
    vector -- vector b
    '''
    n_equations = len(vector)
    for i in range(1, n_equations):
        mc = below[i - 1] / main[i - 1]
        main[i] = main[i] - mc * above[i - 1]
        vector[i] = vector[i] - mc * vector[i - 1]

    xs = main
    xs[-1] = vector[-1] / main[-1]
    for i in range(n_equations - 2, -1, -1):
        xs[i] = (vector[i] - above[i] * xs[i + 1]) / main[i]

    return xs

if __name__ == '__main__':
    px = 'sin(x) / (1 + x**2)**0.5'
    qx = '-(1 + x + x * cos(x ** 2))'
    fx = '1'
    alpha1, alpha2, A = -0.1, 1, 0.4
    beta1, beta2, B = 0.2, 1, 0.5
    a, b = 0, 1

    h = 0.1
    xs1, ys1 = solve_diffeq(px, qx, fx, alpha1, alpha2, A,
                                beta1, beta2, B, h, a, b)

    h = 0.01
    xs01, ys01 = solve_diffeq(px, qx, fx, alpha1, alpha2, A,
                                beta1, beta2, B, h, a, b)

    h = 0.001
    xs001, ys001 = solve_diffeq(px, qx, fx, alpha1, alpha2, A,
                                beta1, beta2, B, h, a, b)

    print('\th = 0.1\t\t\t\t', 'h = 0.01\t\t\t', 'h = 0.001')
    for i in range(11):
        print(ys1[i], '\t\t', ys01[i * 10], '\t\t', ys001[i * 100])


"""ЛР 1.3, Ларькин Владимир, М8О-303Б-18"""

import numpy as np
import fire  # CLI
import matplotlib.pyplot as plt

from utilities import parse_matrix  # парсинг матрицы из файла
from sem1.lab1_1.gauss import lu_det, lu_decomposition

def diag_dominance(a: np.ndarray) -> bool:
    """Проверка матрицы на диагональное преобладание"""
    assert a.shape[0] == a.shape[1] and len(a.shape) == 2
    for i in range(a.shape[0]):
        s = sum(np.abs(a[i, :])) - 2 * np.abs(a[i, i])
        if s >= 0:
            return False
    return True

def matrix_norm(a: np.ndarray) -> float:
    """C-норма матрицы"""
    return max([sum(np.abs(row)) for row in a])

def vector_norm(v: np.ndarray) -> float:
    """C-норма вектора"""
    return max(np.abs(v))

def make_alpha_beta(a: np.ndarray, b: np.ndarray) -> (np.ndarray, np.ndarray):
    """Матрица alpha и вектор beta для метода Якоби"""

    alpha = np.copy(a)
    beta = np.copy(b)

    n = a.shape[0]

    for j in range(n):
        m = np.abs(alpha[j:, j]).argmax() + j
        alpha[[j, m]] = alpha[[m, j]]
        beta[[j, m]] = beta[[m, j]]

    diag = np.copy(np.diag(alpha))

    for j in range(n):
        alpha[j, :] /= -diag[j]
        alpha[j, j] = 0
        beta[j] /= diag[j]

    return alpha, beta

def fixed_point(a: np.ndarray, b: np.ndarray, eps: float = 0.01, count_it=False) -> np.ndarray:
    """Метод простой итерации

    :param a: матрица коэффициентов
    :param b: вектор правых частей
    :param eps: точность
    :param count_it: если True, то будет возвращено количество проделанных итераций
    :return: решение системы, а если count_it=True, то кортеж из решения системы и числа итераций
    """

    assert a.shape[0] == a.shape[1] == b.shape[0] and len(a.shape) == 2
    assert lu_det(lu_decomposition(a)[2], drop_sign=True) != 0

    alpha, beta = make_alpha_beta(a, b)

    c = matrix_norm(alpha) / (1 - matrix_norm(alpha))
    x = np.copy(beta)
    eps_k = eps + 1
    count = 0
    while eps_k >= eps:
        x_new = beta + alpha @ x
        eps_k = c * vector_norm(x - x_new)
        x = x_new
        count += 1

    return x if not count_it else (x, count)

def seidel(a: np.ndarray, b: np.ndarray, eps: float = 0.01, count_it=False) -> np.ndarray:
    """Метод Зейделя

    :param a: матрица коэффициентов
    :param b: вектор правых частей
    :param eps: точность
    :param count_it: если True, то будет возвращено количество проделанных итераций
    :return: решение системы, а если count_it=True, то кортеж из решения системы и числа итераций
    """

    assert a.shape[0] == a.shape[1] == b.shape[0] and len(a.shape) == 2
    assert lu_det(lu_decomposition(a)[2], drop_sign=True) != 0

    alpha, beta = make_alpha_beta(a, b)

    x = np.copy(beta)
    eps_k = eps + 1
    count = 1
    while eps_k >= eps:
        x_new = np.copy(x)
        for i in range(len(x)):
            x_new[i] = beta[i] + alpha[i, :] @ x_new

        eps_k = vector_norm(x - x_new)
        x = x_new
        count += 1

    return x if not count_it else (x, count)

def main(src, test=False, eps=0.01):
    """Решение СЛАУ методом простых итераций и методом Зейделя

    :param src: путь к текстовому файлу с матрицей
    :param test: флаг, запускающий тестирование
    :param eps: точность вычисления
    """

    np.random.seed(42)

    # чтение файла
    with open(src, "r") as file:
        s = file.readlines()

    matrix = parse_matrix(s)

    a = matrix[:, :-1]
    b = matrix[:, -1]

    print(f"A:\n{a}")
    print(f"b:\n{b}")

    print(f"eps={eps}")

    print(f"Метод простой итерации:\n{fixed_point(a, b, eps)}")
    print(f"Метод Зейделя:\n{seidel(a, b, eps)}")
    print(f"Реальное решение:\n{np.linalg.solve(a, b)}")

    # анализ влияния точности на число итераций
    if test:
        powers = np.arange(0, 11, .5)  # порядки эпсилонов (eps = 10 ** -p)
        fp = []
        z = []
        for p in powers:
            fp.append(fixed_point(a, b, 10 ** -p, count_it=True)[1])
            z.append(seidel(a, b, 10 ** -p, count_it=True)[1])

        plt.plot(powers, fp, label="Метод простых итераций")
        plt.plot(powers, z, label="Метод Зейделя")
        plt.legend()
        plt.title("Зависимость числа итераций от точности")
        plt.xlabel("Порядок точности ($\\epsilon = 10^{-порядок}}$)")
        plt.ylabel("Число итераций")
        plt.xticks(powers[::2])
        plt.yticks(range(0, max(fp[-1], z[-1]), 5))
        plt.grid(True)
        plt.savefig("benchmark.jpg", dpi=300)
        plt.show()

if __name__ == "__main__":
    fire.Fire(main)


### Author : Piyush Sharma
# --> Implementation of **K-MEANS** algorithim.
"""

#Importing the Libraries
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import random

#Reading the dataset
iris = pd.read_csv('https://raw.githubusercontent.com/Piyush9323/NaiveBayes_in_Python/main/iris.csv')
#iris.head()
iris.tail()

iris.describe()

# Preparing input with 4 features from dataset
X = iris.iloc[:,1:5].values
X.shape

# Preparing Expected output for comparison
Y = iris.iloc[:,-1].values
for i in range(150):
    if i < 50 :
        Y[i] = 0
    elif i < 100 :
        Y[i] = 1
    else :
        Y[i] = 2
#Y

#number of training datapoints
m = X.shape[0]

#number of features
n = X.shape[1]

# number of iterations
n_iterations = 100

# number of clusters
k = 3
print(m,n,k)

data = iris.iloc[:,1:3].values
plt.scatter(data[:,0], data[:,1], c = 'black', label = 'Unclustered Data')
plt.xlabel('Sepal Length')
plt.ylabel('Sepal Width')
plt.legend()
plt.show()

# It claculates Eucledian distance between two points
def E_distance(X1,X2):
    d = sum((X1 - X2)**2)**0.5
    return d

# K-Means Algorithm
def K_Means(data):

    # choosing centroids randomly
    random.seed(121)
    centroids = {}
    for i in range(k):
        rand = random.randint(0,m-1)
        centroids[i] = data[rand]

    # creating output dictionary
    for iteration in range(n_iterations):
        classes = {}

        # classes are initialising with classKey as indices.
        for class_key in range(k):
            classes[class_key] = []

        # finding the distance of each data point with 3 centroids assigned recently.
        p = 0
        for data_point in data:
            distance = []
            for centroid in centroids:
                temp_dis = E_distance(data_point, centroids[centroid])
                distance.append(temp_dis)

            # finding the centroid with minimum distance from the point and append it into that centroid class.
            min_dis = min(distance)
            min_dis_index = distance.index(min_dis)
            classes[min_dis_index].append(data_point)
            Y[p] = min_dis_index
            p += 1

        # new centroids are formed by taking the mean of the data in each class.
        for class_key in classes:
            class_data = classes[class_key]
            new_centroids = np.mean(class_data, axis = 0)
            centroids[class_key] = list(new_centroids)

    return classes, centroids

# Running K-Means algorithm
classes, centroids = K_Means(X)
classes

# plotting the clustered data
color = ['red','blue','green']
labels = ['Iris-setosa','Iris-versicolour','Iris-virginica']

for i in range(k):
    x = list(list(zip(*classes[i]))[0])
    y = list(list(zip(*classes[i]))[1])
    plt.scatter(x, y, c = color[i], label = labels[i])

# plotting centroids of clusters
cv = centroids.values()
plt.scatter(list(list(zip(*cv))[0]), list(list(zip(*cv))[1]), s = 100, c = 'cyan',label= 'centroids')
plt.xlabel('Sepal Length')
plt.ylabel('Sepal Width')
plt.legend()
plt.show()

"""# **K-MEANS** algorithm using inbuilt-function."""

from sklearn.cluster import KMeans

k_means = KMeans(n_clusters = 3, max_iter = 100, random_state = 15)
predicted = k_means.fit_predict(X)
predicted

# plotting the clusters
plt.scatter(X[predicted == 0, 0], X[predicted == 0, 1], s = 100, c = 'red', label = 'Iris-setosa')
plt.scatter(X[predicted == 1, 0], X[predicted == 1, 1], s = 100, c = 'blue', label = 'Iris-versicolour')
plt.scatter(X[predicted == 2, 0], X[predicted == 2, 1], s = 100, c = 'green', label = 'Iris-virginica')

# plotting the centroids of the clusters
plt.scatter(k_means.cluster_centers_[:, 0], k_means.cluster_centers_[:,1], s = 100, c = 'cyan', label = 'Centroids')

plt.legend()

"""# **Summary** : Both clusters using my implementation and using Kmeans function from sklearn library gives nearly same results."""


"""数组合并
维度为0的合并是行数改变但是列数不变
维度为1的合并是列数改变但是行数不变"""
import numpy as np
# A = np.array([1, 1, 1])
# # print(A)
# B = np.array([2, 2, 2])
# """行合并"""
# print(np.vstack((A, B)))  # vertical stack
# """[[1 1 1]
#  [2 2 2]]"""
# """列合并"""
# print(np.hstack((A, B)))
# """[1 1 1 2 2 2]"""
# print(A[np.newaxis, :])  # 对A在行方向上增加一个维度[[1 1 1]]
# print(A[:, np.newaxis])
# """[[1]
#    [1]
#    [1]]"""
# """纵向合并"""
# 将以上结合起来表示就是
A = np.array([1, 1, 1])[:, np.newaxis]
"""
[[1]
 [1]
 [1]]
 """
B = np.array([2, 2, 2])[:, np.newaxis]
"""
[[2]
 [2]
 [2]]
"""
# print(A)
# print(B)
# C = np.vstack((A, B))
"""
[[1]
 [1]
 [1]
 [2]
 [2]
 [2]]
 """
# D = np.hstack((A, B))
"""
[[1 2]
 [1 2]
 [1 2]]
 """
# np.hstack和np.vastack的方法也可以进行三个或者三个元素以上的合并
C = np.concatenate((A, B, B, A), axis=0)  # 这个方法的好处是可以定义axis进行不同维数的合并
print(C)
# [[1]
#  [1]
#  [1]
#  [2]
#  [2]
#  [2]
#  [2]
#  [2]
#  [2]
#  [1]
#  [1]
#  [1]]
D = np.concatenate((A, B, B, A), axis=1)
print(D)
# [[1 2 2 1]
#  [1 2 2 1]
#  [1 2 2 1]]


#!/usr/bin/env python

"""
Bevington & Robinson's model of dual exponential decay

References::
    [5] Bevington & Robinson (1992).
    Data Reduction and Error Analysis for the Physical Sciences,
    Second Edition, McGraw-Hill, Inc., New York.
"""

from numpy import exp, sqrt, vstack, array, asarray

def dual_exponential(t, A, B, C, tauA, tauB):
    """
    Computes dual exponential decay.

        y = A exp(-t/tauA) + B exp(-t/tauB) + C
    """
    t = asarray(t)
    return C + A*exp(-t/tauA) + B*exp(-t/tauB)

# data from Chapter 8 of [5].
data = array([[15, 775], [30, 479], [45, 380], [60, 302],
[75, 185], [90, 157], [105,137], [120, 119], [135, 110],
[150, 89], [165, 74], [180, 61], [195, 66], [210, 68],
[225, 48], [240, 54], [255, 51], [270, 46], [285, 55],
[300, 29], [315, 28], [330, 37], [345, 49], [360, 26],
[375, 35], [390, 29], [405, 31], [420, 24], [435, 25],
[450, 35], [465, 24], [480, 30], [495, 26], [510, 28],
[525, 21], [540, 18], [555, 20], [570, 27], [585, 17],
[600, 17], [615, 14], [630, 17], [645, 24], [660, 11],
[675, 22], [690, 17], [705, 12], [720, 10], [735, 13],
[750, 16], [765, 9], [780, 9], [795, 14], [810, 21],
[825, 17], [840, 13], [855, 12], [870, 18], [885, 10]])

# Set uncertainty to sqrt(counts)
data = { 'x': data[0], 'y': data[1], 'dy': sqrt(data[1]) }

#coeff = {'A': 1, 'B': 1, 'C': 1, 'tauA': 1, 'tauB': 1}


import numpy as np

class Quaternion:

    def __init__(self, w,x,y,z):

        self.x = x
        self.y = y
        self.z = z
        self.w = w

    def to_euler_angles(self):

        sinr_cosp = 2 * (self.w * self.x + self.y * self.z)
        cosr_cosp = 1 - 2 * (self.x * self.x + self.y * self.y)
        x=np.arctan2(sinr_cosp, cosr_cosp)

        sinp = 2 * (self.w * self.y - self.z * self.x);
        if  np.abs(sinp) >= 1:
            y = np.copysign(np.pi/2, sinp)
        else:
            y = np.arcsin(sinp)

        siny_cosp = 2 * (self.w * self.z + self.x * self.y)
        cosy_cosp = 1 - 2 * (self.y * self.y + self.z * self.z)
        z = np.arctan2(siny_cosp, cosy_cosp)

        return np.array([x,y,z], dtype=np.float64)

    @staticmethod
    def from_euler_angles(x,y,z):

        cy = np.cos(z*0.5)
        sy = np.sin(z*0.5)
        cp = np.cos(y*0.5)
        sp = np.sin(y*0.5)
        cr = np.cos(x*0.5)
        sr = np.sin(x*0.5)

        w = cr * cp * cy + sr * sp * sy
        x = sr * cp * cy - cr * sp * sy
        y = cr * sp * cy + sr * cp * sy
        z = cr * cp * sy - sr * sp * cy

        return Quaternion(w,x,y,z)

    @property
    def array(self):
        return np.array([self.w, self.x, self.y, self.z], dtype=np.float64)

    def __repr__(self):
        return f"w: {self.w} x:{self.x} y:{self.y} z:{self.z}"


import numpy as np

import src as ya

def getEntropy(data: np.ndarray) -> float:
    """Get entropy of data.

    Parameters
    ----------
    data: numpy.ndarray
        Data for processing

    Returns
    -------
    entropy: float
        Entropy of data
    """
    labels = data[:, -1]
    cdist = ya.util.histc(labels)
    cdist_norm = cdist/np.sum(cdist)
    return -np.sum(cdist_norm * np.log(cdist_norm))

def getInformationGain(data: np.ndarray, idx: np.ndarray) -> float:
    """Information Gain - the 'purity' of data labels in
    both child nodes after split. The higher the purer.

    Parameters
    ----------
    data: numpy.ndarray
        Data to split
    idx: numpy.ndarray
        Indexes of left child

    Returns
    -------
    information_gain: float
        Information Gain - Purity
    """
    L = data[idx, :]
    R = data[~idx, :]
    H = getEntropy(data)
    HL = getEntropy(L)
    HR = getEntropy(R)
    return H - np.sum(idx)/len(idx)*HL - sum(~idx)/len(idx)*HR

def updateInformationGain(node, ig_best, ig, t, idx, dim, idx_best):
    """"""
    if ig > ig_best:
        ig_best = ig
        node.t = t
        node.dim = dim
        idx_best = idx
    return node, ig_best, idx_best


from __future__ import division
import numpy as np

def conv_flops(input_shape, filters, kernel_size, stride=1, padding=0, relu=True):
    n_ch, height, width = input_shape
    n = kernel_size**2 * n_ch
    flops_per_instance = n + (n-1)

    out_height = (height-kernel_size+2*padding)/stride + 1
    out_width = (width-kernel_size+2*padding)/stride + 1
    total_flops_per_layer = filters * out_height * out_width * flops_per_instance

    # bias_add
    total_flops_per_layer += filters

    if relu:
        total_flops_per_layer += filters * out_height * out_width

    return total_flops_per_layer

def mpool_flops(input_shape, stride=2):
    return np.prod(input_shape)/(stride**2)

def fc_flops(n_in, n_out, relu=True):
    total_flops_per_layer = n_in*n_out + n_out
    if relu:
        total_flops_per_layer += n_out
    return total_flops_per_layer

def lenet_dense_flops(n_active):
    return fc_flops(n_active[0], n_active[1]) \
            + fc_flops(n_active[1], n_active[2]) \
            + fc_flops(n_active[2], 10, relu=False)

# n_active1: BB only
# n_active2: DBB
def lenet_dense_flops_dbb(n_active1, n_active2):
    return 4*n_active1[0] + fc_flops(n_active2[0], n_active1[1]) \
            + 4*n_active1[1] + fc_flops(n_active2[1], n_active1[2]) \
            + 4*n_active1[2] + fc_flops(n_active2[2], 10, relu=False)

def lenet_conv_flops(n_active):
    return conv_flops([1, 28, 28], n_active[0], 5) \
            + mpool_flops([n_active[0], 24, 24]) \
            + conv_flops([n_active[0], 12, 12], n_active[1], 5) \
            + mpool_flops([n_active[1], 8, 8]) \
            + fc_flops(n_active[2], n_active[3]) \
            + fc_flops(n_active[3], 10, relu=False)

def lenet_conv_flops_dbb(n_active1, n_active2):
    return conv_flops([1, 28, 28], n_active1[0], 5) + 4*n_active1[0] \
            + mpool_flops([n_active2[0], 24, 24]) \
            + conv_flops([n_active2[0], 12, 12], n_active1[1], 5) + 4*n_active1[1] \
            + mpool_flops([n_active2[1], 8, 8]) \
            + 4*n_active1[2] + fc_flops(n_active2[2], n_active1[3]) \
            + 4*n_active1[3] + fc_flops(n_active2[3], 10, relu=False)

nets = [
        ('Original ', [20, 50, 800, 500]),

        ('SSL(2e-3)', [4, 5, 68,  10]),
        ('SSL(1e-3)', [4, 7, 102, 10]),
        ('SSL(5e-4)', [6, 8, 107, 10]),
        ('SSL(1e-4)', [7, 14, 189, 13]),
        ('SSL(5e-5)', [8, 15, 218, 12]),

        ('SVD(5e-2)', [20,  9, 121, 14]),
        ('SVD(2e-2)', [12,  13,  192, 20]),
        ('SVD(1e-2)', [10,  16,  238, 29]),
        ('SVD(5e-3)', [10,  18,  285, 35]),
        ('SVD(1e-3)', [13,  25,  362, 48]),

        ('SBP(2e-2)', [2, 15,  123, 46]),
        ('SBP(1.5e-2)', [10,  18,  112, 33]),
        ('SBP(1e-2)', [10,  18,  128, 38]),
        ('SBP(5e-3)', [11,  20,  143, 46]),
        ('SBP(1e-3)', [14,  23,  204, 52]),

        ('BB(5e-2)', [13,  20,  99,  30]),
        ('BB(2e-2)', [12,  23,  128, 40]),
        ('BB(1e-2)', [13,  25,  156, 54]),
        ('BB(5e-3)', [14,  25,  198, 62]),
        ('BB(1e-3)', [14,  27,  403, 260]),

        ('DBB(5e-2)', [10,  15,  33,  10]),
        ('DBB(2e-2)', [11,  20,  40,  18]),
        ('DBB(1e-2)', [13,  24,  53,  27]),
        ('DBB(5e-3)', [13,  24,  80,  42]),
        ('DBB(1e-3)', [14,  26,  349, 164]),

        ('VIB(5e-2)', [12,  15,  69,  23]),
        ('VIB(2e-2)', [12,  17,  78,  29]),
        ('VIB(1e-2)', [12,  18,  82,  34]),
        ('VIB(5e-3)', [12,  19,  90,  37]),
        ('VIB(1e-3)', [13,  22,  120, 42]),

        ('GD(5e-2)', [14,  25,  249, 70]),
        ('GD(2e-2)', [14,  26,  325, 116]),
        ('GD(1e-2)', [14,  29,  368, 174]),
        ('GD(5e-3)', [14,  32,  378, 268]),
        ('GD(1e-3)', [14,  34,  516, 484])
]

orig_flops = lenet_conv_flops([20, 50, 800, 500])
print(orig_flops)
print("=================================")

for idx, net in enumerate(nets):
    if idx == 0:
        continue
    name, structure = net
    if 'DBB' not in name:
        flops_ = lenet_conv_flops(structure)
    else:
        flops_ = lenet_conv_flops_dbb([13, 25, 156, 54], structure)

    print('{}, {}, {:2.2f}'.format(name, flops_, float(orig_flops)/float(flops_)))

'''
import sys
if __name__ == '__main__':
    lenet_dense_base = lenet_dense_flops([784, 500, 300])
    lenet_conv_base = lenet_conv_flops([20, 50, 800, 500])

    if len(sys.argv) == 4:
        n_active = [int(x) for x in sys.argv[1:]]
        print lenet_dense_base/lenet_dense_flops(n_active)
    elif len(sys.argv) == 5:
        n_active = [int(x) for x in sys.argv[1:]]
        print lenet_conv_base/lenet_conv_flops(n_active)
    elif len(sys.argv) == 7:
        n_active1 = [int(x) for x in sys.argv[1:4]]
        n_active2 = [int(x) for x in sys.argv[4:]]
        print lenet_dense_base/lenet_dense_flops_dbb(n_active1, n_active2)
    elif len(sys.argv) == 9:
        n_active1 = [int(x) for x in sys.argv[1:5]]
        n_active2 = [int(x) for x in sys.argv[5:]]
        print lenet_conv_base/lenet_conv_flops_dbb(n_active1, n_active2)
'''


import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
# pd.set_option('display.max_columns', None)
# pd.set_option('display.max_rows', None)

import warnings
warnings.filterwarnings('ignore')

from statsmodels.tsa.api import VAR
from statsmodels.tsa.stattools import adfuller
from statsmodels.tools.eval_measures import rmse

#------------------------------------------

#load datasets (from FRED)
# ----------
df = pd.read_csv('data/M2SLMoneyStock.csv', index_col=0, parse_dates=True)
df.index.freq='MS'

sp = pd.read_csv('data/PCEPersonalSpending.csv', index_col=0, parse_dates=True)
df.index.freq='MS'

#combine them into one dataframe
df = df.join(sp)
df = df.dropna()

#first, show the data
# ----------
df.plot()
plt.show()

#second, run tests
# ----------
#adf1uller
def adf_test(series,title=''):
    print('*******************************')
    print(f'Augmented Dickey-Fuller Test: {title}')
    result = adfuller(series.dropna(),autolag='AIC') # .dropna() handles differenced data

    labels = ['ADF test statistic','p-value','# lags used','# observations']
    out = pd.Series(result[0:4],index=labels)

    for key,val in result[4].items():
        out[f'critical value ({key})']=val

    print(out.to_string())          # .to_string() removes the line "dtype: float64"

    if result[1] <= 0.05:
        print("Strong evidence against the null hypothesis - reject null")
        print("Data is stationary")
    else:
        print("Weak evidence against the null hypothesis - accept null")
        print("Data is NON-stationary")

    print('*******************************')

adf1 = adf_test(df['Money'])
# => data is NON-stationary
adf2 = adf_test(df['Spending'])
# => data is NON-stationary

# second-bis, run differencing if data NON - stationary
# ----------
# pandas .diff method will run differencing in the entire dataframe
df_transformed = df.diff()
# repeat the adf tests to check if one differencing ha solved it
adf3 = adf_test(df_transformed['Money'])
adf4 = adf_test(df_transformed['Spending'])
# as one of the two series is not yet stationary, we run another differencing round on both
df_transformed = df_transformed.diff().dropna()
adf5 = adf_test(df_transformed['Money'])
adf6 = adf_test(df_transformed['Spending'])

# third, split data set
# ----------
nobs = 12   #number of observations for the test set
train = df_transformed.iloc[:-nobs] #start = beginning of dataframe, or 0, to -12 from the end
test = df_transformed.iloc[-nobs:] #start at -12 from the end to go to the end

#fourth, run the grid search manually via a loop
# ----------
model = VAR(train)
lags = [1,2,3,4,5,6,7]

for p in lags:
    results = model.fit(p)
    print(f'order {p}')
    print(f'AIC {results.aic}')
    print('--------')

#best order is p=5

#fifth, run the model with the chosen order p
# ----------
p = 5
results = model.fit(p)
# print(results.summary())

#sixth, run the temporary forecast vs test set
# ----------
# forecast via .forecast:
# (1) y = nparray p * k, where k is the series
# (2) the next steps to predict in the future (= to train)
lagged_values = train.values[-p:]    #this returns the dataframe as nparray for the last p lines in p * k shape
z = results.forecast(y=lagged_values, steps=nobs)
#transform z back into a dataframe
test_start = '2015-01-01'
index = pd.date_range(test_start, periods=nobs, freq='MS')
df_forecast = pd.DataFrame(data=z, index=index, columns=['Money_diff2x', 'Spending_diff2x'])

# sixth-bis, reverse the differencing to make values turn normal, if needed
# ----------
# we transform the differences by taking the cumulative sum of a column and adding it to a one line difference to revert it
df_forecast['Money_diff1x'] = (df['Money'].iloc[-nobs-1]-df['Money'].iloc[-nobs-2]) + df_forecast['Money_diff2x'].cumsum()
df_forecast['Money_Fcst'] = df['Money'].iloc[-nobs-1] + df_forecast['Money_diff1x'].cumsum()

df_forecast['Spending_diff1x'] = (df['Spending'].iloc[-nobs-1]-df['Spending'].iloc[-nobs-2]) + df_forecast['Spending_diff2x'].cumsum()
df_forecast['Spending_Fcst'] = df['Spending'].iloc[-nobs-1] + df_forecast['Spending_diff1x'].cumsum()

#seventh, plot temp forecast vs test
# ----------
#define the original test range, then plot the forecast
test_range = df[-nobs:]
test_range[['Money', 'Spending']].plot()
df_forecast[['Money_Fcst', 'Spending_Fcst']].plot()
# test_range[['Money']].plot(legend=True)
# df_forecast[['Money_Fcst']].plot(legend=True)
# test_range[['Spending']].plot(legend=True)
# df_forecast[['Spending_Fcst']].plot(legend=True)
plt.show()

#eight, measure the error via rmse
# ----------
error1 = rmse(test_range['Money'], df_forecast['Money_Fcst'])
error2 = rmse(test_range['Spending'], df_forecast['Spending_Fcst'])
std1 = test_range['Money'].std()
std2= test_range['Spending'].std()
error_result1 = error1/std1 * 100
error_result2 = error2/std2 * 100
print('rmse error is the following percentage out of standard dev: ')
print(error_result1)
print(error_result2)

# ninth, run full forecast
# ----------


"""Kernel zoo."""

from functools import partial

import jax
import jax.numpy as jnp

from probfindiff.typing import ArrayLike

@jax.jit
def exponentiated_quadratic(
    x: ArrayLike, y: ArrayLike, input_scale: float = 1.0, output_scale: float = 1.0
) -> ArrayLike:
    r"""Exponentiated quadratic kernel.

    The kernel is defined as

    .. math:: k(x,y) = \sigma \exp(-\epsilon \|x-y\|^2/2)

    Parameters
    ----------
    x
        Input variable.
    y
        Input variable.
    input_scale
        Input scale :math:`\epsilon` of the kernel.
    output_scale
        Output scale :math:`\sigma` of the kernel.

    Returns
    -------
    :
        Evaluation :math:`k(x,y)`.
    """
    return output_scale * jnp.exp(-input_scale * (x - y).dot(x - y) / 2.0)

@partial(jax.jit, static_argnames=("scale", "order", "bias"))
def polynomial(
    x: ArrayLike,
    y: ArrayLike,
    *,
    p: ArrayLike,
) -> ArrayLike:
    r"""Polynomial kernels.

    The kernel is defined as

    .. math::
        k(x,y) = p[0]*\langle x, y\rangle^(N-1) + p[1]*\langle x, y\rangle^(N-2) + ... + p[N-2]*\langle x, y\rangle + p[N-1]

    Parameters
    ----------
    x
        Input variable.
    y
        Input variable.
    p
        Coeficients of the polynomial

    Returns
    -------
    :
        Evaluation :math:`k(x,y)`.
    """
    return jnp.polyval(p, jnp.dot(x, y)) + 1


# =============================================================================
# Created By: Brett Patterson
# Updated By: Brett Patterson
# Created On: 9/20/2018
# Updated On: 9/20/2018
# Purpose: Perform unconstrained gradient descent on convex functions
# =============================================================================

# if you want to calc the Hessian you'll need to use
# from autograd import hessian
# and call it twice on your function

from autograd import grad
from autograd import numpy as np
from nonlinear_optimization.search import backtrack_line_search as bls

def example_function(x):
    y = x[0]**2 + 3 * x[1]**2 + 2*x[0]*x[1] + x[0] + 3
    return(y)

def gradient_descent(x, f_x, grad_f_x, epsilon=.000001):
    delta_x = -1 * grad_f_x(x)

    while(np.linalg.norm(delta_x)>epsilon):
        t = bls.backtracking_line_search(delta_x=delta_x, f_x=f_x, x=x)

        x = x + t * delta_x
        delta_x = -1 * grad_f_x(x)

    return(x)

grad_f_x = grad(example_function)
x = np.array([0.75,2.0])

x_star = np.array([-0.75, 0.25])
x_opt_grad = gradient_descent(x, example_function, grad_f_x)

print(x_star)
print(x_opt_grad)
print(np.sum((x_star - x_opt_grad)**2))


# Reference for discrete equations:
# http://physics.bu.edu/py502/lectures3/cmotion.pdf

from numpy import zeros_like, array
from numpy import sum as npsum
from numba import njit, prange

# Constants
G     = 6.67430e-11     # Netwon's gravitational constant in SI units
m2AU  = 6.684587122e-12 # Metre to astronomical units
kg2ME = 1.6744e-25      # Kilogram to Earth masses
s2dy  = 1.157407407e-5  # Second to days (ass. 24hr/day)

convG = G * m2AU**3 / (kg2ME * s2dy**2) # G in unit [au^3 ME^-1 dy^-2]

@njit(parallel=True)
def make_step(stepSize, numBodies, bodyMasses, currentPosits, currentVels):
    """Perform a forward step using the fourth-order Runge-Kutta method
    in parallel over all bodies in the simulation using Numba.

    This function takes the current positions and velocities of a number of
    bodies and performs a single forward time step of Netwon's gravitational
    equations of motion using the 4th order Runge-Kutta method for all bodies.

    Arguments:
    stepSize      -- the size of the time step in days.
    numBodies     -- the number of bodies equal to len(bodyMasses).
    bodyMasses    -- a list of masses for each body.
    currentPosits -- a NumPy ndarray of the current positions of all bodies,
                     each element of which has in turn two dimensions: [x, y].
    currentVels   -- a NumPy ndarray of the current velocities of all bodies,
                     each element of which has in turn two dimensions: [x, y].

    Returns:
    nextPosits  -- a NumPy ndarray of the next positions of all bodies, each
                   element of which has in turn two dimensions: [x, y].
    currentVels -- a NumPy ndarray of the next velocities of all bodies, each
                   element of which has in turn two dimensions: [x, y].

    Note, the order of the position and velocities vectors for each mass in the
    arguments 'currentPosits' and 'currentVels' must be the same as the order
    of masses in 'bodyMasses'. The return arrays also have the same ordering.
    """

    nextPosit = zeros_like(currentPosits)
    nextVel   = zeros_like(currentVels)

    for i in prange(numBodies):

        iMass  = bodyMasses[i]
        iPosit = currentPosits[i]
        iVel   = currentVels[i]

        l1 = stepSize*iVel
        k1 = array([0., 0.])
        for j in prange(numBodies):
            if i == j: continue

            jMass  = bodyMasses[j]
            jPosit = currentPosits[j]

            rVec   = jPosit - iPosit
            rSumSq = npsum(rVec**2)
            k1    += stepSize * convG * jMass * (rVec/rSumSq**1.5)

        l2 = stepSize * (iVel + k1/2)
        k2 = array([0., 0.])
        for j in prange(numBodies):
            if i == j: continue

            jMass  = bodyMasses[j]
            jPosit = currentPosits[j]

            rVec   = jPosit - (iPosit + l1/2)
            rSumSq = npsum(rVec**2)
            k2    += stepSize * convG * jMass * (rVec/rSumSq**1.5)

        l3 = stepSize * (iVel + k2/2)
        k3 = array([0., 0.])
        for j in prange(numBodies):
            if i == j: continue

            jMass  = bodyMasses[j]
            jPosit = currentPosits[j]

            rVec   = jPosit - (iPosit + l2/2)
            rSumSq = npsum(rVec**2)
            k3    += stepSize * convG * jMass * (rVec/rSumSq**1.5)

        l4 = stepSize * (iVel + k3)
        k4 = array([0., 0.])
        for j in prange(numBodies):
            if i == j: continue

            jMass  = bodyMasses[j]
            jPosit = currentPosits[j]

            rVec   = jPosit - (iPosit + l3)
            rSumSq = npsum(rVec**2)
            k4    += stepSize * convG * jMass * (rVec/rSumSq**1.5)

        nextPosit[i] = iPosit + (l1 + 2*l2 + 2*l3 + l4)/6
        nextVel[i]   = iVel   + (k1 + 2*k2 + 2*k3 + k4)/6

    return nextPosit, nextVel


import numpy as np
import scipy as sp
from numpy.random import randn, rand
import matplotlib.pyplot as plt
import seaborn as sns
from scipy.stats import norm

def em_gauss_sim(n_sample=1000, n_component=5, iter_lim=20, seed=7):
    # set seed
    np.random.seed(seed)

    x = randn(n_sample) + (rand(n_sample) > 0.3) * 4 - 2

    L = - np.inf

    # initial values
    weights = np.ones(n_component) / n_component
    means = np.linspace(np.min(x), np.max(x), n_component)
    covs = np.ones(n_component)/10
    eta = np.zeros([n_sample, n_component])

    # main loop
    n_iter = 0
    while n_iter < iter_lim:
        # E step
        for i in range(n_sample):
            for l in range(n_component):
                eta[i][l] = (weights[l] * norm.pdf(x=x[i], loc=means[l], scale=np.sqrt(covs[l]))) / np.dot(weights, norm.pdf(x=x[i], loc=means, scale=np.sqrt(covs)))

        # M step
        means_prev = means.copy()
        weights_prev = weights.copy()
        covs_prev = covs.copy()
        for l in range(n_component):
            weights[l] = np.sum(eta[:,l]) / n_sample
            covs[l] = np.dot(eta[:,l], (x - means_prev[l])**2) / (1 * np.sum(eta[:,l]))
            means[l] = np.dot(eta[:,l], x) / np.sum(eta[:, l])

        # stop condition
        if np.sum((weights - weights_prev)**2) < 0.00001:
            break

        n_iter += 1

    # for debugging
    print(n_iter)

    sim_x = np.linspace(np.min(x), np.max(x), 1000)
    y = np.zeros_like(sim_x)
    for l in range(n_component):
        tmp = weights[l] * norm.pdf(x=sim_x, loc=means[l], scale=np.sqrt(covs[l]))
        y += tmp

    return x, sim_x, y

import numpy,from numpy,import scipy,from scipy


import numpy as np

def Sigmoid(feature,derivative=False):
  if derivative==False:
    return 1/(1+np.exp(-feature))
  else :
    return np.exp(-feature)/((1+ np.exp(-feature)))**2

def ReLu(feature,derivative=False):
  if derivative==False:
    return feature*(feature>0)
  else :
    return 1*(feature>0)

def Tanh(feature,derivative=False):
  if derivative==False:
    return np.tanh(feature)
  else :
    return 1/(np.cosh(feature))**2


from sage.all import QQ
from sage.all import ZZ
from sage.all import matrix
from sage.all import vector

def attack(y, k, s, m, a, c):
    """
    Recovers the states associated with the outputs from a truncated linear congruential generator.
    More information: Frieze, A. et al., "Reconstructing Truncated Integer Variables Satisfying Linear Congruences"
    :param y: the sequential output values obtained from the truncated LCG (the states truncated to s most significant bits)
    :param k: the size in bits of the states
    :param s: the size in bits of the outputs
    :param m: the modulus of the LCG
    :param a: the multiplier of the LCG
    :param c: the increment of the LCG
    :return: a list containing the states associated with the provided outputs
    """
    diff_bitsize = k - s

    # Preparing for the lattice reduction.
    delta = c % m
    y = vector(ZZ, y)
    for i in range(len(y)):
        # Shift output value to the MSBs and remove the increment.
        y[i] = (y[i] << diff_bitsize) - delta
        delta = (a * delta + c) % m

    # This lattice only works for increment = 0.
    B = matrix(ZZ, len(y), len(y))
    B[0, 0] = m
    for i in range(1, len(y)):
        B[i, 0] = a ** i
        B[i, i] = -1

    B = B.LLL()

    # Finding the target value to solve the equation for the states.
    b = B * y
    for i in range(len(b)):
        b[i] = round(QQ(b[i]) / m) * m - b[i]

    # Recovering the states
    delta = c % m
    x = list(B.solve_right(b))
    for i, state in enumerate(x):
        # Adding the MSBs and the increment back again.
        x[i] = int(y[i] + state + delta)
        delta = (a * delta + c) % m

    return x


import math
import numpy as np

class ProjectionVector(object):
	"""
	Computes the projection vector from an input segment
	Input must be in the form: ((x0,y0),(x1,y1))
	"""
	def __init__(self, lineA):
		self.lineA = lineA
		self.vector = self.find_projection()
		self.slope = self.find_slope()
		self.intercept = self.find_intercept()

	def find_projection(self):
		return (self.lineA[1], (2*self.lineA[1][0]-self.lineA[0][0], 2*self.lineA[1][1]-self.lineA[0][1]))

	def find_slope(self):
		a =  self.vector[1][1]- self.vector[0][1]
		if  self.vector[1][0] !=  self.vector[0][0]: # protect from zero division error
			return (a / ( self.vector[1][0]- self.vector[0][0]))
		else:
			return a

	def find_intercept(self):
		return  self.vector[1][1] - self.slope* self.vector[1][0]

def find_angle(lineB,VectorP):
	"""
	Line A (not present in function) is the starting segment
	Line B is the following segment
	VectorP is an object of class ProjectionVector and it is derived from Line A
	"""
	vB = [(lineB[0][0]-lineB[1][0]), (lineB[0][1]-lineB[1][1])]
	vP = [(VectorP.vector[0][0]-VectorP.vector[1][0]), (VectorP.vector[0][1]-VectorP.vector[1][1])]
	dot_prod = np.dot(vB,vP)
	magA = np.sqrt(np.dot(vB, vB))
	magB = np.sqrt(np.dot(vP, vP))
	cos_ = dot_prod/(magA*magB)
	ang = np.arccos(cos_)
	ang_deg = math.degrees(ang)%360
	if lineB[1][1]<VectorP.slope*lineB[1][0] + VectorP.intercept:
	    ang_deg = -ang_deg
#	print(ang_deg)
	return -1*ang_deg

if __name__ == '__main__':
	#input lines
	lineA = ((1, 1),(1, 2))
	# lineA = np.array([[-2, 2],[-4, 1.5]])
	lineB = ((1, 2),(2, 1))

	#projected vector
	vectorP = ProjectionVector(lineA)

	a = find_angle(lineB,vectorP)

# for i in range(0,len(a), 12):
# 	vP = ProjectionVector([a[i], a[i+1]])
# 	find_angle([a[i+1], a[i+2]],vP)


import numpy as np
import math

class NaiveBayes():
    def __init__(self):
        self.partial_vector_0 = []
        self.partial_vector_1 = []
        self.prob_class_1 = 0

    def train_naive_bayes(self, matrix, categories):
        '''Assume there are only two class.
        @param: categories is a list of 1's and 0's
        '''
        self.prob_class_1 = sum(categories) / len(categories)
        norm_features_0 = np.ones(len(matrix[0]))
        norm_features_1 = np.ones(len(matrix[0]))
        cnt0 = cnt1 = len(matrix[0])
        for i in range(len(categories)):
            if categories[i] == 1:
                norm_features_1 += matrix[i]
                cnt1 += sum(matrix[i])
            else:
                norm_features_0 += matrix[i]
                cnt0 += sum(matrix[i])
        self.partial_vector_0 = np.log(norm_features_0 / cnt0)
        self.partial_vector_1 = np.log(norm_features_1 / cnt1)

    def predict(self, test_matrix):
        '''@param: test_matrix: vector of vectors
        '''
        return [1 if sum(vec * self.partial_vector_1) + math.log(self.prob_class_1) > \
            sum(vec * self.partial_vector_0) + math.log(1 - self.prob_class_1) else 0 \
                for vec in test_matrix]

if __name__ == "__main__":
    matrix = [[0, 1, 1, 0], [1, 0, 0, 0], [1, 1, 1, 0]]
    print(NaiveBayes().train_naive_bayes(matrix, [0, 0, 1]))
    # x = np.asarray([1,2,3,4])
    # print(np.log(x))
    # print(math.log(2))

    # y = np.asarray([1,2,3,4])
    # z = x + y
    # print(z)
    # print(np.asarray(x) / 3)


import numpy as np

class AffineLayer:
    """
    forward/backward propagation for Affine transformation: Y = XW + B

    In the textbook, `doutdW` and `doutdb` are fields.
    """

    def __init__(self):
        self.x = None
        self.W = None
        self.b = None

    def forward(self, x, W, b):
        """
        return y = xW + b

        x: (batch_size, n) matrix
        W: (n, m) matrix
        b: (1, m) matrix. This is NOT (batch_size, m) matrix because bias is common for all data in `x`.
        """
        self.x = x
        self.W = W
        self.b = b
        return np.dot(self.x, self.W) + self.b

    def backward(self, doutdy):
        """
        return d(out)/dx, d(out)/dW

        d(out)/dx = d(out)/dy * [W]T
        d(out)/dW = [x]T * d(out)/dy
        d(out)/db = d(out)/dy
        ([A]T means the transpose matrix of A.)

        doutdy: (batch_size, m) matrix (`batch_size` and `m` must be consistent with those of `forward`.)
        """
        doutdx = np.dot(doutdy, self.W.T)
        doutdW = np.dot(self.x.T, doutdy)
        doutdb = np.sum(doutdy, axis=0)
        return doutdx, doutdW, doutdb

if __name__ == '__main__':
    affine_layer = AffineLayer()

    # non-batch
    # Be aware of '[[1, 2, 0]]' not '[1, 2, 0]'.
    # np.array([1,2,3]).T = np.array([1,2,3]), not np.array([[1],[2],[3]]).
    x = np.array([[1, 2, 0]])  # (1, 3) matrix
    W = np.array([[1, 0], [0, 1], [0, 0]])  # (3, 2) matrix
    b = np.array([[1, 1]])  # (1, 2) matrix
    y = affine_layer.forward(x, W, b)
    print('y = {0}'.format(y))
    doutdy = np.array([[1, 0]])  # (1, 2) matrix
    doutdx, doutdW, doutdb = affine_layer.backward(doutdy)
    print('doutdx = {0}, doutdW = {1}, doutdb = {2}'.format(
        doutdx, doutdW, doutdb))

    # batch
    x = np.array([[1, 2, 0], [0, 1, 2]])  # (2, 3) matrix
    W = np.array([[1, 0], [0, 1], [0, 0]])  # (3, 2) matrix
    b = np.array([[1, 1]])  # (1, 2) matrix
    y = affine_layer.forward(x, W, b)
    print('y = {0}'.format(y))
    doutdy = np.array([[1, 0], [0, 1]])  # (2, 2) matrix
    doutdx, doutdW, doutdb = affine_layer.backward(doutdy)
    print('doutdx = {0}, doutdW = {1}, doutdb = {2}'.format(
        doutdx, doutdW, doutdb))


from math import factorial

import matplotlib.pyplot as plt
import numpy as np

from numerical_analysis.dependencies import GeometricalPlace
from numerical_analysis.dependencies import Polynomial
from numerical_analysis.root_finding import newton_raphson

class Bezier(GeometricalPlace):

    def __init__(self, control_points: np.array, datatype=np.float):
        super().__init__()
        self.datatype = datatype
        self.cp = control_points
        self.n = len(self.cp) - 1
        self.m = self.matrix(self.n, self.datatype)
        self.c = self.polynomial_coefficients()
        self.polynomials = {"x": Polynomial(self.c[0]),
                            "y": Polynomial(self.c[1]),
                            "dx/dt": Polynomial(self.c[0]).derivative(),
                            "dy/dt": Polynomial(self.c[1]).derivative(),
                            "p": [Polynomial(self.m[i]) for i in range(self.n + 1)]}

    def polynomial_coefficients(self):
        c = np.empty([2, self.n + 1], dtype=self.datatype)
        for k in range(2):
            c[k] = np.matmul(np.transpose(self.m), self.cp[:, k])
        return c

    def y_x(self, x, error=1e-14):
        t0 = newton_raphson(lambda t: self.x_t(t) - x, self.dx_dt, 0.5, error)
        return self.y_t(t0)

    def x_t(self, t):
        return self.polynomials["x"].value(t)

    def y_t(self, t):
        return self.polynomials["y"].value(t)

    def dx_dt(self, t):
        return self.polynomials["dx/dt"].value(t)

    def dy_dt(self, t):
        return self.polynomials["dy/dt"].value(t)

    def p_i(self, i, t):
        return self.polynomials["p"][i].value(t)

    def graph_cp(self):
        return [[self.cp[j, i] for j in range(len(self.cp))] for i in range(len(self.cp[0]))]

    def plot(self, dt):
        graph = self.graph(dt)
        # noinspection PyUnresolvedReferences
        plt.plot(graph[0], graph[1], self.graph_cp()[0], self.graph_cp()[1])
        plt.show()

    def update(self):
        self.c = self.polynomial_coefficients()
        self.polynomials = {"x": Polynomial(self.c[0]),
                            "y": Polynomial(self.c[1]),
                            "dx/dt": Polynomial(self.c[0]).derivative(),
                            "dy/dt": Polynomial(self.c[1]).derivative(),
                            "p": [Polynomial(self.m[i]) for i in range(self.n + 1)]}

    def modify_control_point(self, i, new_control_point):
        self.cp[i][0] = new_control_point[0]
        self.cp[i][1] = new_control_point[1]
        self.update()

    def modify_control_point_coordinate(self, i, k, new_control_point_coordinate):
        self.cp[i, k] = new_control_point_coordinate
        self.update()

    def refresh_control_points(self, control_points):
        self.cp = control_points
        self.update()

    def append_control_point(self, new_control_point):
        self.cp = np.append(self.cp, [new_control_point], axis=0)
        self.update()

    def change_datatype(self, datatype):
        self.datatype = datatype
        self.cp = self.cp.astype(datatype)
        self.m = self.m.astype(datatype)
        self.c = self.c.astype(datatype)

    @classmethod
    def matrix(cls, n, datatype=np.int):
        return np.array([[((-1) ** (j - i) * factorial(n)) /
                          (factorial(i) * factorial(j - i) * factorial(n - j))
                          if j >= i else 0. for j in range(n + 1)] for i in range(n + 1)], dtype=datatype)


import numpy

class DiagGaussianKernel(object):
    def __init__(self, sigma=1.0):
        """
        Initialise object with given value of sigma >= 0
        :param sigma: kernel width parameter.
        :type sigma: :class:`float`
        """
        self.sigma = sigma

    def __call__(self, X1, X2):
        # if X1.shape[1] != X2.shape[1]:
        # raise ValueError("Invalid matrix dimentions: " + str(X1.shape) + " " + str(X2.shape))

        K = numpy.exp(- numpy.sum((X1 - X2) ** 2, 1) / (2 * self.sigma ** 2))
        K = numpy.array(K, ndmin=2).T
        return K

class GaussianKernel(object):
    """
    A class to find gaussian kernel evaluations k(x, y) = exp (-||x - y||^2/2 sigma^2)
    """

    def __init__(self, sigma=1.0):
        """
        Initialise object with given value of sigma >= 0
        :param sigma: kernel width parameter.
        :type sigma: :class:`float`
        """
        self.sigma = sigma

    def __call__(self, X1, X2):
        """
        Find kernel evaluation between two matrices X1 and X2 whose rows are
        examples and have an identical number of columns.
        :param X1: First set of examples.
        :type X1: :class:`numpy.ndarray`
        :param X2: Second set of examples.
        :type X2: :class:`numpy.ndarray`
        """

        if X1.shape[1] != X2.shape[1]:
            raise ValueError("Invalid matrix dimentions: " + str(X1.shape) + " " + str(X2.shape))

        j1 = numpy.ones((X1.shape[0], 1))
        j2 = numpy.ones((X2.shape[0], 1))

        diagK1 = numpy.sum(X1 ** 2, 1)
        diagK2 = numpy.sum(X2 ** 2, 1)

        X1X2 = numpy.dot(X1, X2.T)

        Q = (2 * X1X2 - numpy.outer(diagK1, j2) - numpy.outer(j1, diagK2)) / (2 * self.sigma ** 2)

        return numpy.exp(Q)

    def __str__(self):
        return "GaussianKernel: sigma = " + str(self.sigma)

class PolyKernel(object):
    """
    A class to find linear kernel evaluations k(x, y) = <x, y>
    """

    def __init__(self, c, p):
        """
        Intialise class.
        """
        self.c = c
        self.p = p

    def __call__(self, X1, X2):
        """
        Find kernel evaluation between two matrices X1 and X2 whose rows are
        examples and have an identical number of columns.
        :param X1: First set of examples.
        :type X1: :class:`numpy.ndarray`
        :param X2: Second set of examples.
        :type X2: :class:`numpy.ndarray`
        """

        if X1.shape[1] != X2.shape[1]:
            raise ValueError("Invalid matrix dimentions: " + str(X1.shape) + " " + str(X2.shape))

        return (numpy.dot(X1, X2.T) + self.c) ** self.p

class LinearKernel(PolyKernel):
    def __init__(self):
        super(LinearKernel, self).__init__(0, 1)


import numpy as np

def l2_regularization(W, reg_strength):
    '''
    Computes L2 regularization loss on weights and its gradient
    Arguments:
      W, np array - weights
      reg_strength - float value
    Returns:
      loss, single value - l2 regularization loss
      gradient, np.array same shape as W - gradient of weight by l2 loss
    '''
    loss = reg_strength * np.sum(W**2)
    grad = 2*reg_strength*W

    return loss, grad

def softmax_with_cross_entropy(predictions, target_index):
    '''
    Computes softmax and cross-entropy loss for model predictions,
    including the gradient
    Arguments:
      predictions, np array, shape is either (N) or (N, batch_size) -
        classifier output
      target_index: np array of int, shape is (1) or (batch_size) -
        index of the true class for given sample(s)
    Returns:
      loss, single value - cross-entropy loss
      dprediction, np array same shape as predictions - gradient of predictions by loss value
    '''
    if not isinstance(target_index, np.ndarray):
        target_index = np.array([[target_index]])
    target_index.shape = (-1,)

    f_num = predictions.shape[-1]
    batch_size = target_index.shape[0]

    # Make correct shape
    pred_copy = predictions.copy().reshape(batch_size, f_num)
    pred_copy -= np.amax(pred_copy, axis=1).reshape(batch_size, 1)

    # Compute softmax
    e = np.exp(pred_copy)
    probs = e / np.sum(e, axis=1).reshape(batch_size, 1)

    # Compute loss
    s = np.arange(batch_size) # make array of [0, 1, 2, ..., (batch_size-1)]
    target_probs = probs[s, target_index] # using multiple array indexing

    loss = -np.average(np.log(target_probs))

    # Compute gradient
    dprediction = np.copy(probs)
    dprediction[s, target_index] -= 1
    dprediction /= batch_size # normalize gradient

    dprediction.shape = predictions.shape

    return loss, dprediction

class Param:
    """
    Trainable parameter of the model
    Captures both parameter value and the gradient
    """

    def __init__(self, value):
        self.value = value
        self.grad = np.zeros_like(value)

class ReLULayer:
    def __init__(self):
        pass

    def forward(self, X):
        self.X = X # Will be used in the back pass
        return np.maximum(0, X)

    def backward(self, d_out):
        """
        Backward pass

        Arguments:
        d_out, np array (batch_size, num_features) - gradient
           of loss function with respect to output

        Returns:
        d_result: np array (batch_size, num_features) - gradient
          with respect to input
        """
        d_input = np.ones(self.X.shape)
        d_input[np.where(self.X < 0)] = 0
        d_result = d_out * d_input
        return d_result

    def params(self):
        # Doesn't have any parameters
        return {}

class FullyConnectedLayer:
    def __init__(self, n_input, n_output):
        self.W = Param(0.001 * np.random.randn(n_input, n_output))
        self.B = Param(0.001 * np.random.randn(1, n_output))
        self.X = None

    def forward(self, X):
        self.X = X
        result = X @ (self.W.value) + self.B.value
        return result

    def backward(self, d_out):
        """
        Backward pass
        Computes gradient with respect to input and
        accumulates gradients within self.W and self.B

        Arguments:
        d_out, np array (batch_size, n_output) - gradient
           of loss function with respect to output

        Returns:
        d_result: np array (batch_size, n_input) - gradient
          with respect to input
        """
        d_result = d_out @ (self.W.value).T
        self.W.grad = self.X.T @ d_out
        self.B.grad = np.sum(d_out, axis=0, keepdims=True)
        return d_result

    def params(self):
        return {'W': self.W, 'B': self.B}