import math as m

import numpy as np
from scipy import integrate
from scipy.special import beta
import matplotlib.pyplot as plt
plt.style.use('seaborn-whitegrid')

class Norm_rv:

    """
    Class to initialize a normal random variable with mean mu and variance
    sigma^2

    Norm_rv(mean, variance, crit_value=0.0)

    References
    ----------
    [1] Sahoo, Prasanna. "Probability and Mathematical Statistics",
    pp 166 - 169 (2008).
    """

    def __init__(self, mean, variance, crit_value=0.0):
        self.mean = float(mean)
        self.sigma = float(m.sqrt(variance))
        if variance >0 and variance < np.inf:
            self.variance = variance
        else:
            raise ValueError('Enter a variance between 0 and infinity')
        self.crit_value = float(crit_value)
        self.x_range = np.linspace(-4*self.variance, 4*self.variance, 500*self.variance)

    def __repr__(self):
        return f"Normal distribution with mean {self.mean}, variance {self.variance}, and critical value {self.crit_value}"

    def pdf(self):

        """
        this is the probability density function (pdf) of a normal distribution
        with mean mu and variance sigma^2. To check that it is, in fact, a pdf,
        the y values must sum to 1. This would theoretically be the integral
        from -infty to infty but is approximated here with a sum.
        """

        return (1/(self.sigma*m.sqrt(2*m.pi)))*m.e**((-1/2)*((self.x_range-self.mean)/self.sigma)**2)

    def plot_pdf(self, cv_probability=False):

        """
        this function takes a given normal random variable, uses the pdf that
        was previously calculated, and plots it.
        """

        plt.title(self.__repr__())
        plt.plot(self.x_range, self.pdf(),linestyle='dashed', color='blue',linewidth=3)
        if cv_probability==False:
            plt.fill_betweenx(self.pdf(), self.x_range, x2=self.mean,
                          where=(self.x_range<self.mean), color='navy', alpha=0.3)
        else:
            plt.fill_betweenx(self.pdf(), self.x_range, x2=self.crit_value,
                          where=(self.x_range<self.crit_value), color='navy', alpha=0.3)
        plt.tight_layout()
        plt.show()

    def probability_calc(self):

        """
        This calculates the probability to the LEFT of the critical value by
        integrating the area under the distribution from negative infinity
        to the critical value.

        Because the integration function needs a general x variable, and since
        the pdf from Norm_rv.pdf is evaluated over a range of x-values to plot,
        the normal pdf needs to be redefined in this method.This is true for
        all the other distributions as well.
        """

        f = lambda x: (1/(self.sigma*m.sqrt(2*m.pi)))*m.e**((-1/2)*((x-self.mean)/self.sigma)**2)
        self.probability, self.error_est = integrate.quad(f,-np.inf,self.crit_value)
        return f"P(X<crit_val) is {round(self.probability,5)} with an error estimate of {round(self.error_est,5)}"

class ChiSq_rv:

    """
    Class for a Chi-squared random variable with k degrees of freedom

    ChiSq_rv(deg_freedom, crit_value=0.0)

    As degrees of freedom increases to infinity, the Chi-squared distribution
    approximates a normal distribution. You may notice that with >171 degrees of
    freedom, the math.gamma function returns a range error as this is a very
    large number and exceeds the Python-allowed data type limit.

    References
    ----------
    [1] Sahoo, Prasanna. "Probability and Mathematical Statistics",
    pp 392 (2008).
    """

    def __init__(self, df, crit_value=0.0):
        if df > 0:
            self.df = df
        else:
            raise ValueError('Degrees of freedom must be > 0')
        self.mean = self.df
        self.variance = 2*self.df
        self.crit_value = float(crit_value)
        self.x_range = np.linspace(0, 5*self.df, 2000)

    def __repr__(self):
        return f"Chi-squared distribution with {self.df} degrees of freedom and critical value {self.crit_value}"

    def pdf(self):

        """
        this is the probability density function (pdf) of a chi squared
        distribution with k degrees of freedom.To check that it is, in fact,
        a pdf, the y values must integrate to 1.
        """

        #TODO: add integration rather than sum approximation
        return (1/(m.gamma(self.df/2)*2**(self.df/2)))*self.x_range**((self.df/2)-1)*m.e**(-self.x_range/2)

    def plot_pdf(self, cv_probability=False):

        """
        this function takes a given Chi-squared random variable, uses the pdf
        that was previously calculated, and plots it.
        """

        plt.title(self.__repr__())
        plt.plot(self.x_range, self.pdf(),linestyle='dashed', color='red',linewidth=3)
        if cv_probability==False:
            plt.fill_betweenx(self.pdf(), self.x_range, x2=self.df,
                          where=(self.x_range>self.df), color='red', alpha=0.3)
        else:
            plt.fill_betweenx(self.pdf(), self.x_range, x2=self.crit_value,
                          where=(self.x_range>self.crit_value), color='red', alpha=0.3)
        plt.tight_layout()
        plt.show()

    def probability_calc(self):

        """
        This calculates the probability to the RIGHT of the critical value by
        integrating the area under the distribution from the critical value to
        infinity.
        """

        f = lambda x: (1/(m.gamma(self.df/2)*2**(self.df/2)))*x**((self.df/2)-1)*m.e**(-x/2)
        self.probability, self.error_est = integrate.quad(f,self.crit_value,np.inf)
        return f"P(X>crit_val) is {round(self.probability,5)} with an error estimate of {round(self.error_est,5)}"

class t_rv:

    """
    Class for a random variable with a t-distribution and v degrees of freedom

    t_rv(deg_freedom, crit_value=0.0)

    As degrees of freedom increases to infinity, the t distribution approximates
    a standard normal distribution. You may notice that with >171 degrees of
    freedom, the math.gamma function returns a range error as this is a very
    large number and exceeds the Python-allowed data type limit.

    References
    ----------
    [1] Sahoo, Prasanna. "Probability and Mathematical Statistics",
    pp 396 (2008).
    """

    def __init__(self, df, crit_value=0.0):
        if df > 0:
            self.df = df
        else:
            raise ValueError('Degrees of freedom must be > 0')

        if df >= 3:
            self.mean = 0
            self.variance = (self.df / (self.df - 2))
        else:
            raise ValueError('E(X) DNE for df = 1 and Var(X) DNE for df = {1,2}')

        self.crit_value = float(crit_value)
        self.x_range = np.linspace(-2*self.df, 2*self.df, 2000)

    def __repr__(self):
        return f"t distribution with {self.df} degrees of freedom and critical value {self.crit_value}"

    def pdf(self):

        """
        this is the probability density function (pdf) of a t distribution with
        v degrees of freedom. To check that it is, in fact, a pdf, the y values
        must integrate to 1.
        """

        return m.gamma((self.df+1)/2) / (m.sqrt(m.pi * self.df) * m.gamma(self.df / 2) * (1 + ((self.x_range**2)/self.df))**((self.df + 1) / 2))

    def plot_pdf(self, cv_probability=False):

        """
        this function takes a given t random variable, uses the pdf that
        was previously calculated, and plots it.
        """

        plt.title(self.__repr__())
        plt.plot(self.x_range, self.pdf(),linestyle='dashed', color='purple',linewidth=3)
        if cv_probability==False:
            plt.fill_betweenx(self.pdf(), self.x_range, x2=self.df,
                          where=(self.x_range>self.df), color='purple', alpha=0.3)
        else:
            plt.fill_betweenx(self.pdf(), self.x_range, x2=self.crit_value,
                          where=(self.x_range>self.crit_value), color='purple', alpha=0.3)
        plt.tight_layout()
        plt.show()

    def probability_calc(self):

        """
        This calculates the probability to the RIGHT of the critical value by
        integrating the area under the distribution from the critical value
        to infinity.
        """

        f = lambda x: m.gamma((self.df+1)/2) / (m.sqrt(m.pi * self.df) * m.gamma(self.df / 2) * (1 + ((x**2)/self.df))**((self.df + 1) / 2))
        self.probability, self.error_est = integrate.quad(f,self.crit_value,np.inf)
        return f"P(X>crit_val) is {round(self.probability,5)} with an error estimate of {round(self.error_est,5)}"

class F_rv:

    """
    Class for a random variable with an F distribution with v_1 and v_2
    degrees of freedom with x > 0, v_2 >= 5. If v < 5, the Var(X) DNE and if
    v_2 < 3, E(X) DNE.

    F_rv(v_1, v_2, crit_value=0.0)

    As degrees of freedom increases to infinity, the F distribution approximates
    a standard normal distribution. You may notice that as degrees of
    freedom grows large, the math.gamma function returns a range error
    as this is a very large number and exceeds the Python-allowed data type
    limit.

    References
    ----------
    [1] Sahoo, Prasanna. "Probability and Mathematical Statistics",
    pp 401 (2008).
    """

    def __init__(self, v_1, v_2, crit_value=0.0):

        if v_1 >0:
            self.v_1 = v_1
        else:
            raise ValueError('v_1 must be > 0')

        if v_2 >= 5:
            self.v_2 = v_2
            self.mean = self.v_2 / (self.v_2 - 2)
            self.variance = (2*self.v_2**2 * (self.v_1 + self.v_2 -2)) \
                            /((self.v_1)*(self.v_2 - 2)^2 * (self.v_2 -4))
        else:
            if v_2 < 3:
                raise ValueError('with v_2 < 3, E(X) DNE')
            else:
                raise ValueError('with v_2 < 5, Var(X) DNE')

        self.crit_value = float(crit_value)
        self.x_range = np.linspace(0, 2*self.v_2, 2000)

    def __repr__(self):
        return f"""F(v_1,v_2) distribution with v_1={self.v_1} , v_2={self.v_2}
        degrees of freedom, mean {round(self.mean,2)}, and critical value
        {self.crit_value}"""

    def pdf(self):

        """
        this is the probability density function (pdf) of an F distribution with
        v_1 and v_2 degrees of freedom. To check that it is, in fact, a pdf,
        the y values must integrate to 1.
        """

        return (m.gamma((self.v_1 + self.v_2) / 2) * (self.v_1 / self.v_2)**(self.v_1 / 2) * self.x_range**((self.v_1 /2) -1)) \
        / (m.gamma(self.v_1 / 2) * m.gamma(self.v_2 / 2) * (1 + (self.v_1 /self.v_2)*self.x_range)**((self.v_1 + self.v_2) / 2))

    def plot_pdf(self, cv_probability=False):

        """
        this function takes a given F random variable, uses the pdf that
        was previously calculated, and plots it.
        """

        plt.title(self.__repr__())
        plt.plot(self.x_range, self.pdf(),linestyle='dashed', color='forestgreen',linewidth=3)
        if cv_probability==False:
            plt.fill_betweenx(self.pdf(), self.x_range, x2=self.mean,
                          where=(self.x_range>self.mean), color='forestgreen', alpha=0.3)
        else:
            plt.fill_betweenx(self.pdf(), self.x_range, x2=self.crit_value,
                          where=(self.x_range>self.crit_value), color='forestgreen', alpha=0.3)
        plt.tight_layout()
        plt.show()

    def probability_calc(self):

        """
        This calculates the probability to the RIGHT of the critical value by
        integrating the area under the distribution from the critical value
        to infinity.
        """

        f =  lambda x: ((self.v_2**(self.v_2/2) * self.v_1**(self.v_1/2)
                         * x**(self.v_1/2 -1))/
                        ((self.v_2 +self.v_1*x)**((self.v_1 + self.v_2)/2) *
                        beta(self.v_1/2, self.v_2/2)))
        self.probability, self.error_est = integrate.quad(f,self.crit_value,np.inf)
        return f"P(X>crit_val) is {round(self.probability,5)} with an error estimate of {round(self.error_est,5)}"


import numpy as np

def diamondarray(dimension=1,fill=1,unfill=0):
    """ Create a diamond array using a square dimension.
    Fill and unfill values can be integer or float. """
    nullresult=np.zeros(1)

    #// verify inputs
    try:
        if not isinstance(dimension, (int, np.integer)):
            dimesion=int(dimension)
        if not isinstance(fill, (int, float, np.integer)):
            fill=int(fill)
        if not isinstance(unfill, (int, float, np.integer)):
            unfill=int(unfill)
    except:
        return nullresult

    #// check if odd

        return nullresult

    #// initialize 2d array
    a=np.zeros((dimension,dimension))

    for row in range(dimension):
        for col in range(dimension):
            a[row,col]=unfill

    #// find the middle of the array
    midpoint=(dimension-1)/2

    #// initialize an offset
    offset=-1
    offsetstep=1

    #// loop through rows and columns
    for row in range(dimension):
        if dimension%2 == 0 and row == np.ceil(midpoint):
            #// repeat offset for second midpoint row
            offset=offset
        else:
            if row <= np.ceil(midpoint):
                #// increase offset for each row for top
                offset=offset+offsetstep
            else:
                #// decrease offset for each row for bottom
                offset=offset-offsetstep

        for col in range(dimension):
            #// set value to one
            if dimension%2 == 0:
                if col <= np.floor(midpoint):
                    if col == np.floor(midpoint)-offset:
                        a[row,col]=fill
                if col >= np.ceil(midpoint):
                    if col == int(midpoint)+offset+1:
                        a[row,col]=fill
            else:
                if col == midpoint+offset or col == midpoint-offset:
                    pass
                    a[row,col]=fill
    return a

def bisectorarray(dimension=1,vertical=True,horizontal=True,fill=1,unfill=0):
    """ Create an array using square dimension with the midpoint column
    filled. Fill and unfill values can be integer or float. """

    nullresult=np.zeros(1)

    #// verify inputs
    try:
        if not isinstance(dimension, (int, np.integer)):
            dimesion=int(dimension)
        if not isinstance(fill, (int, float, np.integer)):
            fill=int(fill)
        if not isinstance(unfill, (int, float, np.integer)):
            unfill=int(unfill)
    except:
        return nullresult

    #// initialize 2d array
    a=np.zeros((dimension,dimension))

    for row in range(dimension):
        for col in range(dimension):
            a[row,col]=unfill

    #// find the middle of the array
    midpoint=(dimension-1)/2

    #// loop through rows and columns
    for row in range(dimension):
        for col in range(dimension):
            #// set value to one
            if (col == np.floor(midpoint) or col == np.ceil(midpoint)) and vertical==True:
                a[row,col]=fill
            if (row == np.floor(midpoint) or row == np.ceil(midpoint)) and horizontal==True:
                a[row,col]=fill
    return a


import numpy as np

def shuffle_data(X, y, seed=None):
    """ Randomly shuffle data samples """
    if seed:
        np.random.seed(seed)
    idx = np.arange(X.shape[0])
    np.random.shuffle(idx)
    return X[idx], y[idx]

def train_test_split(X, y, test_size=0.5, shuffle=True, seed=None):
    """ Split the data into train and test sets """
    if shuffle:
        X, y = shuffle_data(X, y, seed)
    # Split the training data from test data in the ratio specified in
    # test_size
    split_i = int(len(y) * (1 - test_size))
    X_train, X_test = X[:split_i], X[split_i:]
    y_train, y_test = y[:split_i], y[split_i:]

    return X_train, X_test, y_train, y_test

def mean_squared_error(y_true, y_pred):
    """ Returns the mean squared error between y_pred and y_true """
    mse = np.mean((y_true - y_pred)**2)
    return mse

def accuracy_score(y_true, y_pred):
    """ Compare y_true to y_pred and return the accuracy """
    accuracy = np.sum(y_true == y_pred, axis=0) / len(y_true)
    return accuracy

def euclidean_distance(x1, x2):
    """ Calculates the l2 distance between two Numpy arrays  """
    return np.sqrt(np.sum(np.power(x1 - x2, 2)))

def calculate_covariance_matrix(X, Y=None):
    """ Calculate the covariance matrix for the dataset X """
    if Y is None:
        Y = X
    n_samples = np.shape(X)[0]
    covariance_matrix = (1 / (n_samples - 1)) * (
        X - X.mean(axis=0)).T.dot(Y - Y.mean(axis=0))

    return np.array(covariance_matrix, dtype=float)


""" Collection of random number generation methodologies.
"""

#%% imports
import numpy as np

# make output stable across runs
np.random.seed(42)

# to plot pretty figures
import matplotlib.pyplot as plt
plt.rcParams['axes.labelsize'] = 14
plt.rcParams['xtick.labelsize'] = 12
plt.rcParams['ytick.labelsize'] = 12

#%% exponential continuous random variable

# use exponential distribution when you have some idea what the scale of the hyperparameter should be
# log of distribution shows most values aroung exp(-2) to exp(+2), i.e. ~ 0.1 to 7.4

from scipy.stats import expon, reciprocal

expon_distrib = expon(scale=1.)
samples = expon_distrib.rvs(10000, random_state=42)

plt.figure(figsize=(10, 4))

plt.subplot(121)
plt.title("Exponential distribution (scale=1.0)")
plt.hist(samples, bins=50)

plt.subplot(122)
plt.title("Log of this distribution")
plt.hist(np.log(samples), bins=50)

plt.show()

#%% reciprocal continuous random variable

# use reciprocal distribution when you have no idea what the scale of the hyperparameter should be
# log of the samples roughly constant as scale of the samples picked from a uniform distribution

reciprocal_distrib = reciprocal(20, 200000)
samples = reciprocal_distrib.rvs(10000, random_state=42)

plt.figure(figsize=(10, 4))

plt.subplot(121)
plt.title("Reciprocal distribution (scale=1.0)")
plt.hist(samples, bins=50)

plt.subplot(122)
plt.title("Log of this distribution")
plt.hist(np.log(samples), bins=50)

plt.show()

#%% linear looking data

import numpy as np

X = 2 * np.random.rand(100, 1)
y = 4 + 3 * X + np.random.rand(100,  1)

# nonlinear and noisy dataset generated from a quadratic equation

m = 100
X = 6 * np.random.rand(m, 1) - 3
y = 0.5 * X**2 + X + 2 + np.random.randn(m, 1)


### Explicit module import
import math
print(math.cos(math.pi))

### explicit module import by alias
import numpy as np
print(np.cos(np.pi))

### explicit import of module contents (to local namespace)
from math import cos, pi
print(cos(pi))

### implicit import of module contents (to local namespace)
from math import *
print(sin(pi) ** 2 + cos(pi) ** 2)

    # the above is not a recommended method of import.
    # there are 50 funcs in math module imported to local namespace, of which only 3 are used
    # further, this method of import overwrites previously declared funcs of same name


# This module contains the boundary value solver code, which we call in the Jupyter Notebook.
import numpy as np
from scipy.optimize import root
from matplotlib import pyplot as plt

def get_cheb_extrema(a, b, n):
    """Creates n Chebyshev extrema between a and b."""

    # nodes (Chebyshev extrema)
    x = np.array(list(reversed(np.cos((np.pi * np.arange(n)) / n))))
    # scale to (a, b)
    return (x * (b - a) + b + a) / 2

def fun(cf, a, b, dim, deg, ode, bc):
    cf = cf.reshape((dim, deg+1))
    cf = np.array(cf)

    # nodes (Chebyshev extrema)
    x = get_cheb_extrema(a, b, deg + 1)

    # polynomial evaluated at nodes
    cheb_poly = [np.polynomial.chebyshev.Chebyshev(coeffs, (a, b)) for coeffs in cf]

    p = np.array([cheb_poly[i](x) for i in range(dim)])

    # polynomial at the end points and middle
    ya = p[:, 0]
    yc = p[:, len(p) // 2]
    yb = p[:, -1]

    # coefficients of the derivative of the polynomial
    cheb_poly_der = [cheb_poly[i].deriv() for i in range(dim)]

    # derivative of polynomial evaluated at the nodes
    p_der = np.array([cheb_poly_der[i](x) for i in range(dim)])

    ya_prime = p_der[:, 0]
    yc_prime = p_der[:, len(p_der) // 2]
    yb_prime = p_der[:, -1]

    # output for fsolve
    return [
        *np.ravel(bc([ya, ya_prime], [yc, yc_prime], [yb, yb_prime])),  # boundary conditions
        *np.ravel((p_der - ode(x,p)))  # ODE conditions
    ]

def our_own_bvp_solve(f, a, b, n, y0, dim, bc, tol=1e-2):
    """Solves a boundary value problem using Chebyshev collocation. Returns a list of functions that form the solution to
    the problem."""

    # interpolate the initial guess function y0 on Chebyshev points of the first kind
    cf0 = []
    for y0_i in y0:
        for thing in np.polynomial.chebyshev.Chebyshev(np.zeros(n), (a, b)).interpolate(y0_i, n, (a, b)):
            cf0.append(thing)

    solution = root(lambda u: fun(u, a, b, dim, n, f, bc), cf0, method='lm', tol=tol)
    if not solution.success:
        print('root finding failed')

    cf = solution.x
    cf = cf.reshape((dim, cf.size // dim))

    return [np.polynomial.chebyshev.Chebyshev(cf[i], (a, b)) for i in range(dim)]


import numpy as np
import matplotlib.pyplot as plt

plt.style.use('seaborn')

# make fake data
x_orig = np.linspace(0, 4, 20) # points between 0 and 4
noise = 0.025*np.random.normal(size=len(x_orig)) # random numbers
y_orig = np.exp(-x_orig) + noise # data is theory plus noise

# create theoretical curve to compare with "data"
x_pred = np.linspace(min(x_orig),max(x_orig), 200) # use more values to get smooth curve
y_pred = np.exp(-x_pred)

# setup the plots: both points and smooth curve
fig= plt.figure()
plt.plot(x_orig, y_orig, 'bo', label='data', lw=3) # points
plt.plot(x_pred, y_pred, color='green', label='pred', lw=3) # line
# plt.grid() #can use this if the style is not imported
plt.legend()
plt.xlabel('x')
plt.ylabel('Original vs Predicted')
plt.title("Results")
# plt.yscale('log') # make the y axis (ordinate) log; that is, log-linear

plt.savefig('simple_plot_non_log.png',dpi=300,bbox_inches='tight')
plt.close('all')

### Plot 2
fig= plt.figure()
# setup the plots: both points and smooth curve
plt.plot(x_orig, y_orig, 'bo', label='data', lw=3) # points
plt.plot(x_pred, y_pred, color='green', label='pred', lw=3) # line
# plt.grid() #can use this if the style is not imported
plt.legend()
plt.xlabel('x')
plt.ylabel('Original vs Predicted')
plt.title("Results")
plt.yscale('log') # make the y axis (ordinate) log; that is, log-linear

plt.savefig('simple_plot.png',dpi=300,bbox_inches='tight')
plt.close('all')


import numpy as np
from numpy import diag, sqrt, hstack, ones, eye
from numpy.linalg import inv
from scipy.linalg import qr, solve_triangular, cholesky, svd

def ols(A, b):
    """
    Ordinary Least-Squares Regression
    Coefficients Estimation.

    This is the linear algebra solution to
    OLS; minimising |Ax-b| means we want the
    values where the perpendicular is zero.
    This means we want columnwise-multiplication
    with A, giving A.T(Ax-b), therefore
    0 = A.T*Ax - A.T*b, therefore
    x = A.T*b / A.T*A. QED.
    """
    return np.linalg.inv(A.T @ A) @ A.T @ b

def ols_qr(A, b):
    """
    Ordinary Least-Squares Regression
    Coefficients Estimation.

    This is the QR Factorization solution
    to OLS; Ax = b, A = QR, QRx = b,
    therefore Rx = Q.T * b
    """
    Q, R = qr(A, mode='economic')
    return solve_triangular(R, Q.T @ b)

def ols_cholesky(A, b):
    """
    Ordinary Least-Squares Regression Coefficients
    Estimation.

    If (A.T @ A) @ x = A.T @ b and A is full rank
    then there exists an upper triangular matrix
    R such that:

    (R.T @ R) @ x = A.T @ b
    R.T @ w = A.T @ b
    R @ x = w

    Find R using Cholesky decomposition.
    """
    R = cholesky(A.T @ A)
    w = solve_triangular(R, A.T @ b, trans='T')
    return solve_triangular(R, w)

def ols_svd(A, b):
    """
    Ordinary Least-Squares Regression Coefficients
    Estimation.

    A @ x = b
    A = U @ Σ @ Vh (singular value decomposition of A)
    Σ @ Vh @ x = U.T @ b
    Σ @ w = U.T @ b
    x = Vh.T @ w
    """
    U, sigma, Vh = svd(A, full_matrices=False)
    w = (U.T @ b) / sigma
    return Vh.T @ w

def add_intercept(A):
    """
    Adds an intercept (column of ones) to a supplied array A
    such that the intercept is the first (leftmost) column
    """
    n = A.shape[0]
    intercept = ones(n).reshape(n, 1)
    return hstack((intercept, A))

def _hat(A):
    """
    The 'hat' matrix for an array A
    """
    return A @ np.linalg.inv(A.T @ A) @ A.T

def _m_matrix(A):
    """
    The 'hat' matrix for a vector of ones whose size is
    equal to the first dimension of the supplied array A
    """
    n = A.shape[0]
    l = np.ones(n).reshape(n, 1)
    return _hat(l)

def sum_of_squared_residuals(A, b):
    """
    The sum of squared residuals
    """
    n = A.shape[0]
    I = np.eye(n)
    H = _hat(A)
    return b.T @ (I - H) @ b

def total_sum_of_squares(A, b):
    """
    The total sum of squares
    """
    n = A.shape[0]
    I = np.eye(n)
    M = _m_matrix(A)
    return b.T @ (I - M) @ b

def r_squared(A, b):
    """
    The r-squared value (a.k.a. coefficient of determination)
    """
    ssr = sum_of_squared_residuals(A, b)
    sst = total_sum_of_squares(A, b)
    return 1.0 - ssr / sst

def r_squared_no_intercept(A, b):
    """
    The r-squared value (a.k.a. coefficient of determination) in the
    case where A has no intercept, per statsmodels - compare the
    slope-only model to a model that simply makes a constant
    prediction of 0 for all observations
    """
    fitted = fitted_values(A, b)
    return (fitted.T @ fitted) / (b.T @ b)

def adjusted_r_squared(A, b):
    """
    The adjusted r-squared value
    """
    n, k = A.shape
    return 1 - (n - 1) / (n - k) * (1 - r_squared(A, b))

def adjusted_r_squared_no_intercept(A, b):
    """
    The adjusted r-squared value in the case where no intercept term is present
    """
    n, k = A.shape
    return 1 - n / (n - k) * (1 - r_squared_no_intercept(A, b))

def fitted_values(A, b):
    """
    The predicted values for b
    """
    return _hat(A) @ b

def _residual_maker(A, b):
    """
    Returns a matrix which can be used to make residuals from b
    """
    n = A.shape[0]
    I = eye(n)
    return I - _hat(A)

def residuals(A, b):
    """
    The residuals of the model
    """
    return _residual_maker(A, b) @ b

def mean_standard_error_residuals(A, b):
    """
    Mean squared error of the residuals. The sum of squared residuals
    divided by the residual degrees of freedom.
    """
    n, k = A.shape
    ssr = sum_of_squared_residuals(A, b)
    return ssr / (n - k)

def mean_standard_error_model(A, b):
    """
    Mean squared error the model.
    """
    k = A.shape[1]

    sst = total_sum_of_squares(A, b)
    ssr = sum_of_squared_residuals(A, b)
    sse = sst - ssr

    return sse / (k - 1)

def mean_standard_error_model_no_intercept(A, b):
    """
    Mean squared error the model in the case where A has no
    intercept and the model is slope-only.
    """
    k = A.shape[1]

    fitted = fitted_values(A, b)
    sse = fitted.T @ fitted

    return sse / k

def standard_error(A, b):
    """
    The standard errors of the parameter estimates.
    """
    mse = mean_standard_error_residuals(A, b)
    C = inv(A.T @ A)
    return sqrt(diag(C) * mse)

def t_statistic(A, b):
    """
    t-statistics for the model.
    """
    betas = ols(A, b)
    se = standard_error(A, b)
    return betas / se

def f_statistic(A, b):
    """
    F-statistic of the fully specified model.
    """
    mse_model = mean_standard_error_model(A, b)
    mse_resid = mean_standard_error_residuals(A, b)
    return mse_model / mse_resid

def f_statistic_no_intercept(A, b):
    """
    F-statistic of the fully specified model in the case where
    A has no intercept (a slope-only model).
    """
    mse_model = mean_standard_error_model_no_intercept(A, b)
    mse_resid = mean_standard_error_residuals(A, b)
    return mse_model / mse_resid

def drop_missing(A, b):
    """
    Returns a filtration of A (features) and b (targets) where all
    values are finite, with the intention that OLS can then be
    performed on dense / 'complete' observations.

    This is analogous to the statsmodels missing='drop' mechanism.

    >>> import numpy as np
    >>> A = np.array([[1, 1.1], [2, np.nan], [3, 3.1], [4, 4.1]])
    >>> b = np.array([5, 6, 7, 8])

    >>> drop_missing(A, b)
    (array([[1. , 1.1],
            [3. , 3.1],
            [4. , 4.1]]), array([5, 7, 8]))
    """
    m = A.shape[0]
    assert m == len(b)

    A_bar = np.empty_like(A)
    b_bar = np.empty_like(b)

    ctr = 0

    for i in range(m):
        feature_row = A[i, :]
        target = b[i]
        if np.all(np.isfinite(feature_row)) and np.isfinite(target):
            A_bar[ctr, :] = feature_row
            b_bar[ctr] = target
            ctr += 1

    return A_bar[:ctr, :], b_bar[:ctr]

if __name__ == '__main__':
    import pytest
    pytest.main([__file__])


# A neural network which approximates linear function y = 2x + 3.
# The network has 1 layer with 1 node, which has 1 input (and a bias).
# As there is no activation effectively this node is a linear function.
# After +/- 10.000 iterations W should be close to 2 and B should be close to 3.

import matplotlib.pyplot as plt
import numpy as np

np.set_printoptions(formatter={"float": "{: 0.3f}".format}, linewidth=np.inf)
np.random.seed(1)

X = np.array([[0], [1], [2], [3], [4]])  # X = input (here: 5 values)
Y = 2 * X + 3  # Y = output: y = 2x + 3 (as many values as there are X's)

W = np.random.normal(scale=0.1, size=(1, 1))  # layer: (1, 1) = 1 node with 1 input
B = np.random.normal(scale=0.1, size=(1, 1))  # bias: (1, 1) = for 1 node (and by definition only 1 bias value per node)

learning_rate = 0.001
iterations = 10000
error = []

print("initial :", "W =", W, "B =", B, "(random initialization)")

m = X.shape[0]

for _ in range(iterations):
    # forward pass
    a = W.dot(X.T) + B

    # back propagation
    da = a - Y.T  # da = error
    dz = da  # no activation
    dw = dz.dot(X) / m
    db = np.sum(dz, axis=1, keepdims=True) / m

    W -= learning_rate * dw
    B -= learning_rate * db

    error.append(np.average(da ** 2))

print("result  :", "W =", W, "B =", B, "(after {} iterations)".format(iterations))
print("expected: W = 2, B = 3")

plt.plot(range(iterations), error)
plt.title("MSE (mean squared error)")
plt.xlabel("training iterations")
plt.ylabel("mse")
plt.show()


#!/usr/bin/env python
# coding: utf-8

import numpy as np
import matplotlib.pyplot as plt

# Used libs: pda_utils for manipulating with PDF/CDF and their statistics, maxent_appx is realization of maximum entropy optimization procedure that leads to probability distribution function estimation.

import pdf_appx.pda_utils as cd
import pdf_appx.maxent_appx as me

# Define points of approximation in (0;1) and number of moments (by power, excluding zero power moment ==1) will be used for approximation:

N = 500
t= np.linspace(0,1,N)
num_moments=9

# We have function that creates casual (nearly random) increasing fucntion that determines some CDF and PDF:

getcdf = cd.casual_increasing_function()
print(type(getcdf))

Mycdf = getcdf(t)
plt.plot(t, Mycdf, label='casual cdf')
plt.legend()
plt.show()

getpdf = getcdf.derivative(1)
Mypdf = getpdf(t)

plt.plot(t, Mypdf, label='casual pdf')
plt.legend()
plt.show()

# Get moments of rhis function and use them in maxent-optimization to estimate PDF

moments = cd.moments_from_cdf(getcdf,num_moments)
maxent_approx_1 =  me.pdf_approx(moments)
pdf_hat = maxent_approx_1(t)
plt.plot(t, Mypdf, label='some casual pdf')
plt.plot(t,pdf_hat, label='pdf approximation')
plt.legend()
plt.show()

#  Now create pdf approximation for the random data sample. We will use uniform distributed samples transformed by casual increasing function.

gettransform = cd.casual_increasing_function()
Mytransform = gettransform(t)

plt.plot(t, Mytransform, label='some transformation function')
plt.legend()
plt.show()

samples_uni = np.random.rand(10000)
samples = gettransform(samples_uni)

# Lets get optimization from estimation of moments

moments_star = cd.moments_from_samples(samples, num_moments)
maxent_approx_2 =  me.pdf_approx(moments_star)

pdf_star_hat = maxent_approx_2(t)

plt.hist(samples, bins=50, label = 'histogram', density = True)
plt.plot(t,pdf_star_hat, label='pdf approximation')
plt.legend()
plt.show()

# Sometimes we can catch instabilities during optimization procedure due to moments of high order.


import numpy as np

n = 10  # dimension of array
x = np.empty((n, 1))
w = np.empty((1, n))

# OR implementation with perceptron

x.fill(1)
w.fill(1)
# x[0][0] = 0

# conditions on first element ("b")
# x[0][0] = 1
x_0 = [[1]]
# print("x_0 shape: {}".format(np.shape(x_0)))
# w[0][0] = -0.5
w_0 = [[-0.5]]

x = np.append(x, x_0)
w = np.append(w, w_0)
H = np.sign(np.dot(w, x))
print("X: {}".format(x))
print("w: {}".format(w))
print("OR between elements: {}".format(H))

# -------------------------------------------

# AND implementation with perceptron
x = np.empty((n, 1))
w = np.empty((1, n))

x.fill(1)
w.fill(1)
x[0][0] = 0  # makes condition false

# conditions on first element ("b")
# x[0][0] = 1
x_0 = np.ones((1, 1))
w_0 = np.empty((1, 1))
w_0[0][0] = - np.shape(x)[0] + 0.5

# w[0][0] = - np.shape(x)[0] + 1 + 0.5  # because x-shape is n+1
x = np.append(x, x_0)
w = np.append(w, w_0)

H = np.sign(np.dot(w, x))
print("X: {}".format(x))
print("w: {}".format(w))
print("AND between elements: {}".format(H))

# ------------------------------------------

# NOT implementation with perceptron
x = np.ones((1, 1))
w = np.empty((1, 1))

w[0][0] = -2

# conditions on "b"
x_0 = np.ones((1, 1))
w_0 = np.ones((1, 1))

x = np.append(x, x_0)
w = np.append(w, w_0)

H = np.sign(np.dot(w, x))
print("X: {}".format(x))
print("w: {}".format(w))
print("NOT between elements: {}".format(H))

# ------------------------------------------------

# A AND (NOT B):
# A | B | NOT B | A AND (NOT B)
# 0 | 0 |   1   |  0
# 1 | 0 |   1   |  1
# 0 | 1 |   0   |  0
# 1 | 1 |   0   |  0

x = np.empty((2, 1))
w = np.empty((1, 2))

A = x[0][0] = 1  # A
B = x[1][0] = 0  # B

# first we have to make not of B. First perceptron
# we treat only B input
w_B = w[0][1] = -2
B_0 = np.ones((1, 1))
w_B_0 = np.ones((1, 1))
B_1 = np.append(B, B_0)
w_1 = np.append(w_B, w_B_0)
not_B = np.sign(np.dot(B_1, w_1))
print("A: {a}\nB: {b}".format(a=A, b=B))
print("NOT B: {}".format(not_B))

# then we have to make A AND NOT B
# rewrite x and w with new values
x = np.empty((2, 1))
x[0][0] = A
x[1][0] = not_B
w.fill(1)

x_0 = np.ones((1, 1))
w_0 = np.empty((1, 1))
w_0[0][0] = - np.shape(x)[0] + 0.5

# w[0][0] = - np.shape(x)[0] + 1 + 0.5  # because x-shape is n+1
x = np.append(x, x_0)
w = np.append(w, w_0)

H = np.sign(np.dot(w, x))
print("A AND (NOT B)): {}".format(H))

# ----------------------------------------------------

# XOR: (A AND (NOT B)) OR ((NOT A) AND B)
# A | B | NOT A | NOT B | XOR
# 0 | 0 |   1   |   1   |  0
# 1 | 0 |   0   |   1   |  1
# 0 | 1 |   1   |   0   |  1
# 1 | 1 |   0   |   0   |  0

# much simpler with functions!


#!/usr/bin/env python3
"""
Clustering Module
"""
import numpy as np
expectation_maximization = __import__('8-EM').expectation_maximization

def BIC(X, kmin=1, kmax=None, iterations=1000, tol=1e-5, verbose=False):
    """
    Finds the best number of clusters for a GMM using the Bayesian Information
    Criterion:

    X is a numpy.ndarray of shape (n, d) containing the data set
    kmin is a positive integer containing the minimum number of clusters to
    check for (inclusive)
    kmax is a positive integer containing the maximum number of clusters to
    check for (inclusive)
        If kmax is None, kmax should be set to the maximum number of clusters
        possible
    iterations is a positive integer containing the maximum number of
    iterations for the EM algorithm
    tol is a non-negative float containing the tolerance for the EM algorithm
    verbose is a boolean that determines if the EM algorithm should print
    information to the standard output

    Returns: best_k, best_result, l, b, or None, None, None, None on failure
        best_k is the best value for k based on its BIC
        best_result is tuple containing pi, m, S
            pi is a numpy.ndarray of shape (k,) containing the cluster priors
            for the best number of clusters
            m is a numpy.ndarray of shape (k, d) containing the centroid means
            for the best number of clusters
            S is a numpy.ndarray of shape (k, d, d) containing the covariance
            matrices for the best number of clusters
        l is a numpy.ndarray of shape (kmax - kmin + 1) containing the log
        likelihood for each cluster size tested
        b is a numpy.ndarray of shape (kmax - kmin + 1) containing the BIC
        value for each cluster size tested
            Use: BIC = p * ln(n) - 2 * l
            p is the number of parameters required for the model
            n is the number of data points used to create the model
            l is the log likelihood of the model
    """
    if not isinstance(X, np.ndarray) or len(X.shape) != 2:
        return (None, None, None, None)
    if type(kmin) != int or kmin <= 0 or kmin >= X.shape[0]:
        return (None, None, None, None)
    if type(kmax) != int or kmax <= 0 or kmax >= X.shape[0]:
        return (None, None, None, None)
    if kmin >= kmax:
        return (None, None, None, None)
    if type(iterations) != int or iterations <= 0:
        return (None, None, None, None)
    if type(tol) != float or tol <= 0:
        return (None, None, None, None)
    if type(verbose) != bool:
        return None, None, None, None

    krslt, rslts, lttls, bttls = [], [], [], []
    n, d = X.shape
    for k in range(kmin, kmax + 1):
        pi, m, S, g, lkhd = expectation_maximization(
            X, k, iterations, tol, verbose)
        krslt.append(k)
        rslts.append((pi, m, S))
        lttls.append(lkhd)
        p = (k * d * (d + 1) / 2) + (d * k) + k - 1
        bic = p * np.log(n) - 2 * lkhd
        bttls.append(bic)
    bttls = np.asarray(bttls)
    topb = np.argmin(bttls)
    lttls = np.asarray(lttls)

    return (krslt[topb], rslts[topb], lttls, bttls)


import numpy as np

class ActivationFns:
    @staticmethod
    def sigmoid(Z):
        A = 1 / (1 + np.exp(-Z))

        return A

    @staticmethod
    def relu(Z):
        A = np.maximum(0, Z)
        return A

    @staticmethod
    def leaky_relu(Z, slope=0.01):
        A = np.maximum(Z * slope, Z)

        return A

    @staticmethod
    def tanh(Z):
        A = np.tanh(Z)

        return A

class ActivationFnsDerivatives:
    @staticmethod
    def sigmoid(dA, Z):
        s = 1 / (1 + np.exp(-Z))
        dZ = dA * s * (1 - s)

        assert (dZ.shape == Z.shape)

        return dZ

    @staticmethod
    def relu(dA, Z):
        dZ = np.array(dA, copy=True)
        dZ[Z <= 0] = 0

        assert (dZ.shape == Z.shape)

        return dZ

    @staticmethod
    def leaky_relu(dA, Z, slope=0.01):
        dZ = np.array(dA, copy=True)
        dZ[Z <= 0] = slope

        assert (dZ.shape == Z.shape)

        return dZ

    @staticmethod
    def tanh(dA, Z):
        dZ = 1 / np.power(np.cosh(dA), 2)
        return dZ


import numpy as np
import matplotlib.pyplot as plt
from math import *
import scipy.integrate as integr

def Iteres_1(n):
    res=np.arange(n+1)+1
    while n>0:
        res=np.sin(res)
        n-=1
    return res

def Iteres_2(n):
    res=(n+1)*[1.]
    for i in range(n):
        res[i+1]=np.sin(res[i])
    return res

def x(n):
    res=1
    while n>0:
        res=np.sin(res)
        n-=1
    return res

def Iteres_3(n):
    res=list(range(n+1))
    for i in res:
        res[i]=x(i)
    return res

def Draw(n):
    x=1
    List_X=[x]
    List_Y=[0]
    for k in range(n):
        y=np.sin(x)
        List_X.append(x)
        List_Y.append(y)
        List_Y.append(y)
        List_X.append(y)
        x=y
    plt.plot(List_X,List_Y,"r")
    x=np.arange(0,1,0.01)
    y=[np.sin(k) for k in x]
    plt.plot(x,y,"r")
    plt.plot(x, x, "g")
    plt.show()

def Draw_Serie(n):
    x=1
    List_X=[x]
    List_Y=[0]
    for k in range(n):
        y=(np.sin(x)+x)*(-1)
        List_X.append(x)
        List_Y.append(y)
        List_Y.append(y)
        List_X.append(y)
        x=y
    plt.plot(List_X,List_Y,"r")
    x=np.arange(-max([max(List_Y),max(List_X)]),max([max(List_Y),max(List_X)]),0.01)
    plt.plot(x, x, "g")
    plt.show()

L=[1,3,3,-1]

def Formatage(L):
    n=len(L)
    M=[]
    i=0
    while(i<n-1):
        if (L[i]==L[i+1]):
            M.append(0)
        else:
            if (L[i]<L[i+1]):
                M = M + (L[i+1]-L[i])*[1]
            else:
                M = M + (L[i]-L[i+1])*[-1]
        i+=1
    return M

def Trace(L):
     M=Formatage(L)
     X=range(len(M)+1)
     Y=[L[0]]
     for i in range(1,len(M)):
         Y.append(Y[i-1]+M[i-1])
     Y.append(Y[-1]+M[-1])
     plt.plot(X, Y, "g")
     plt.show()

def Sol():
    def f(x,t):
        return -x+cos(10*t)
    T = np.arange(0,2.01,0.01)
    X=integr.odeint(f,3,T)
    plt.plot(T,X)
    plt.show()

def Resolution():
    def f(x,t):
        return np.array([x[0]*(2-x[1]),x[1]*(x[0]-1)])
    T = np.arange(-10,10,0.01)
    x0=np.array([2,1])
    X=integr.odeint(f,x0,T)

    plt.plot(T,X[:,0], "b")
    plt.plot(T,X[:,1], "r")
    plt.grid()
    plt.title("un titre")

    plt.show()


import numpy as np
import itertools as it

class PolynomialTransformation(object):
    def __init__(self, degree: int = 2, include_bias: bool = False):
        self.degree = degree
        self.include_bias = include_bias

    def transform(self, X: np.array):
        self.n_samples, self.n_features = X.shape
        self.__create_combinations()

        poly_features = X.copy()
        for combination in self.combinations:
            poly_features = np.append(poly_features, np.prod(X[:, combination], axis=1, keepdims=True), axis=1)

        if self.include_bias:
            poly_features = np.append(np.ones(shape=(self.n_samples, 1)), poly_features, axis=1)
        return poly_features

    def __create_combinations(self):
        combinations = [it.combinations_with_replacement(range(self.n_features), i) for i in range(2, self.degree + 1)]
        self.combinations = list(it.chain.from_iterable(combinations))


"""Utilities for geometry defined by tuples or lists of numbers"""

from math import pi, sin, cos, tan, atan, atan2, asin, acos, exp, log, log10
from functools import reduce
from operator import mul

def rad_to_deg(ang):
    """Converts radians to degrees"""
    return ang * 180 / pi

def deg_to_rad(deg):
    """Converts degrees to radians"""
    return deg / 180 * pi

def dist2D(pt1, pt2):
    """Returns distance between two 2D points (as two 2-tuples)"""
    return ((pt2[0]-pt1[0])**2 + (pt2[1]-pt1[1])**2)**0.5

def mag2D(v2D):
    """Returns the magnitude of the vector"""
    return (v2D[0]**2 + v2D[1]**2)**0.5

def neg2D(v2D):
    """Returns the negative of the vector"""
    return -v2D[0], -v2D[1]

def unit2D(v):
    """Returns the unit 2D vector in the same direction
    as the input 2D vector (as one 2-tuple)"""
    if mag2D(v) == 0.0 :
        raise ValueError('Vector should have non-zero magnitude')
    else:
        return scale2D(v, 1.0 / mag2D(v))

def ang2D(pt):
    """Returns the anti-clockwise angle of the
    vector from the x-axis (in radians)"""
    return atan2(pt[1],pt[0])

def scale2D(v,scale):
    """Returns a scaled 2D vector"""
    return (v[0] * scale, v[1] * scale)

def add2D(v1,v2):
    """Vector addition of v1 to v2"""
    return (v1[0] + v2[0], v1[1] + v2[1])

def sub2D(v1,v2):
    """Vector subtraction of v2 from v1"""
    return (v1[0] - v2[0], v1[1] - v2[1])

def mul2D(v1,v2):
    """Elementwise multiplication of vector v1 with v2"""
    return (v1[0] * v2[0], v1[1] * v2[1])

def div2D(v1,v2):
    """Elementwise division of vector v1 by v2"""
    return (v1[0] / v2[0], v1[1] / v2[1])

def cross2D(v1,v2):
    """calculates the scalar cross product magnitude of two 2D vectors, v1 and v2"""
    return v1[0]*v2[1] - v1[1]*v2[0]

def outer2D(v1, v2):
    """Calculates the magnitude of the outer product of two 2D vectors, v1 and v2"""
    return v1[0]*v2[1] - v1[1]*v2[0]

def dot2D(v1,v2):
    """calculates the scalar dot product of two 2D vectors, v1 and v2"""
    return v1[0]*v2[0] + v1[1]*v2[1]

def inner2D(v1, v2):
    """Calculates the inner product of two 2D vectors, v1 and v2"""
    return v1[0]*v2[0] + v1[1]*v2[1]

def sin2D(v1, v2):
    """calculates the sine of the angle between two 2D vectors, v1 and v2
    using the magnitude of the cross product of the normalised vectors."""
    if mag2D(v1) * mag2D(v2) == 0.0 :
        raise ValueError('Both vectors should have non-zero magnitude')
    else:
        scale = 1 / mag2D(v1) / mag2D(v2)
        return cross2D(v1, v2) * scale

def cos_sim2D(v1, v2):
    """returns the cosine of the angle between two vectors (v1, v2) based on the dot product
    v1 . v2 = |v1|  |v2| cos (<v1,v2>)
    A result of 1 means they are parallel, -1 that they are anti-parallel,
    and 0 that they are perpendicular."""
    if mag2D(v1) * mag2D(v2) == 0.0 :
        #raise ValueError('Both vectors should have non-zero magnitude')
        return 1
    else:
        return inner2D(v1, v2) / mag2D(v1) / mag2D(v2)

def rotate2D(pt, ang):
    """Rotates a point (x, y) about the origin by an angle in radians"""
    return mag2D(pt) * cos(ang + ang2D(pt)), mag2D(pt) * sin(ang + ang2D(pt))

def planar_angle2D(v1, v2):
    """returns the angle of one vector relative to the other in the
    plane defined by the normal (default is in the XY plane)
    NB This algorithm avoids carrying out a coordinate transformation
    of both vectors. However, it only works if both vectors are in that
    plane to start with. """
    return atan2(sin2D(v1, v2), cos_sim2D(v1, v2))

# ===========================
# ========== 3 D ============
# ===========================

def cart2cyl(vec3D, default_ang=0):
    """Converts a 3D cartesian vector (x, y, z) to
    cylindrical coordinates (r, theta, z).

    Note that the third coordinate is simply passed through
    (and in fact can be any type and any length - in other words
    it can also handle 2D and nD where n > 1)."""
    x, y, *z = vec3D
    r = (x**2 + y**2)**0.5
    ang = default_ang if x == y == 0 else atan2(y,x)
    return (r, ang, *z)

def cyl2cart(vec3D):
    """Converts a 3D cylindrical vector (r, theta, z) to
    cartesian coordinates (x, y, z).

    Note that the third coordinate is simply passed through
    (and in fact can be any type and any length - in other words
    it can also handle 2D and nD where n > 1)."""
    r, theta, *z = vec3D
    return (r * cos(theta), r * sin(theta), *z)

def angfix(ang):
    """Keeps an angle in radians within -pi > angle > pi"""
    return ((ang + pi) % (2*pi)) - pi

def cyl_rot3D(v, ang):
    """Rotates 3D cylindrical coordinates (v) by an angle (ang) in radians"""
    r, theta, z = v
    return r, angfix(theta + ang), z

def dist3D(pt1, pt2):
    """Returns distance between two 3D points (as two 3-tuples)"""
    return ((pt2[0]-pt1[0])**2 + (pt2[1]-pt1[1])**2 + (pt2[2]-pt1[2])**2)**0.5

def mag3D(v3D):
    """Magnitude of the point or vector"""
    return (v3D[0]**2 + v3D[1]**2 + v3D[2]**2)**0.5

def neg3D(v3D):
    """Returns the negative of the 3D vector"""
    return -v3D[0], -v3D[1], -v3D[2]

def unit3D(v):
    """Returns the unit 3D vector in the same direction
    as the input 3D vector (as one 3-tuple)"""
    if mag3D(v) == 0.0 :
        raise ValueError('Vector should have non-zero magnitude')
    else:
        return scale3D(v, 1.0 / mag3D(v))

def add3D(v1,v2):
    """Vector subtraction of v2 from v1"""
    return (v1[0] + v2[0], v1[1] + v2[1], v1[2] + v2[2])

def sub3D(v1,v2):
    """Vector subtraction of v2 from v1"""
    return (v1[0] - v2[0], v1[1] - v2[1], v1[2] - v2[2])

def scale3D(v,scale):
    """Returns a scaled 3D vector"""
    return (v[0] * scale, v[1] * scale, v[2] * scale)

def dot3D(v1, v2):
    """Calculates the scalar dot product of two 3D vectors, v1 and v2"""
    return v1[0]*v2[0] + v1[1]*v2[1] + v1[2]*v2[2]

def cross3D(v1, v2):
    """Calculates the vector cross product of two 3D vectors, v1 and v2"""
    return (v1[1]*v2[2] - v1[2]*v2[1], v1[2]*v2[0] - v1[0]*v2[2], v1[0]*v2[1] - v1[1]*v2[0])

def sin3D(v1, v2, as_scalar = True):
    """calculates the sine of the angle between two 3D vectors, v1 and v2
    using the magnitude of the cross product of the normalised vectors."""
    if mag3D(v1) * mag3D(v2) == 0.0 :
        raise ValueError('Both vectors should have non-zero magnitude')
    else:
        scale = 1 / mag3D(v1) / mag3D(v2)
        vec = scale3D(cross3D(v1, v2), scale)
        return mag3D(vec) if as_scalar else vec

def cos_sim3D(v1, v2):
    """returns the cosine of the angle between two vectors (v1, v2) based on the dot product
    v1 . v2 = |v1|  |v2| cos (<v1,v2>)"""
    if mag3D(v1) * mag3D(v2) == 0.0 :
        raise ValueError('Both vectors should have non-zero magnitude')
    else:
        return dot3D(v1, v2) / mag3D(v1) / mag3D(v2)

#def planar_angle(v1, v2, normal=(0,0,1)):
#    """returns the angle of one vector relative to the other in the
#    plane defined by the normal (default is in the XY plane)
#    NB This algorithm avoids carrying out a coordinate transformation
#    of both vectors. However, it only works if both vectors are in that
#    plane to start with. """
#    return atan2(dot3D(normal, sin3D(v1, v2, False)), cos_sim3D(v1, v2))

def rotQ(self, ang, other):
    """Returns the vector rotated by an angle of 'ang' radians about the axis
    vector (other) by means of quaternions"""
    q_w = cos(0.5*ang)
    q_xyz = scale3D(other, sin(0.5*ang))
    t = cross3D(q_xyz, self)
    #print(self, scale3D(t, 2.0 * q_w), cross3D(q_xyz,scale3D(t, 2.0)), sep=' : ')
    #return self + scale3D(t, 2.0 * q_w) + cross3D(q_xyz,scale3D(t, 2.0))
    return add3D(add3D(self, scale3D(t, 2.0 * q_w)), cross3D(q_xyz,scale3D(t, 2.0)))

def rotR(self, ang, other):
    """Returns the vector rotated by an angle of 'ang' radians about the axis
    vector (other) by means of Rodrigues' formula"""
    #print(self, scale3D(self, cos(ang)), scale3D(cross3D(other, self), sin(ang)),
    #      scale3D(other, (1.0 - cos(ang)) * (dot3D(self, other))), sep=' : ')
    #return scale3D(self, cos(ang)) + scale3D(cross3D(other, self), sin(ang)) + \
    #    scale3D(other, (1.0 - cos(ang)) * (dot3D(self, other)))
    return add3D(add3D(scale3D(self, cos(ang)), scale3D(cross3D(other, self), sin(ang))),
        scale3D(other, (1.0 - cos(ang)) * (dot3D(self, other))))

def fmt_3x3(dc, fmt='7.4f'):
    """For pretty-printing a 3x3 matrix of unit vectors (3-tuple of 3-tuples)
    NB This is used by test functions, so do not change in isolation"""
    #return ', '.join([f'({x:6.3f}, {y:6.3f}, {z:6.3f})' for x, y, z in dc])
    return ', '.join([f'({x:{fmt}}, {y:{fmt}}, {z:{fmt}})' for x, y, z in dc])

# ===========================
# ========== n D ============
# ===========================

def magND(v):
    """Returns magnitude of an nD vector"""
    return sum(vv**2 for vv in v) ** 0.5

def magNDx(v, limit=0):
    """Returns magnitude of an nD vector,
    ignoring items if they are not numeric, with
    an option to limit length of the tuple to a certain
    length defined by the `limit` argument"""
    if limit > 0:
        return sum(vv**2  for i, vv in enumerate(v)
        if (isinstance(vv, (int, float))
            and i < limit))**0.5
    else:
        return sum(vv**2  for i, vv in enumerate(v)
        if (isinstance(vv, (int, float))))**0.5

def negND(v):
    """Returns negative of an nD vector"""
    return [-vv for vv in v]

def negNDx(v, limit=0):
    """Returns negative of an nD vector,
    ignoring items if they are not numeric, with
    an option to limit length of the tuple to a certain
    length defined by the `limit` argument"""
    if limit > 0:
        return [-vv  for i, vv in enumerate(v)
        if (isinstance(vv, (int, float))
            and i < limit)]
    else:
        return [-vv  for i, vv in enumerate(v)
        if (isinstance(vv, (int, float)))]

def unitND(v):
    """Returns the unit nD vector in the same direction
    as the input nD vector (as one 3-tuple)"""
    if magND(v) == 0.0 :
        raise ValueError('Vector should have non-zero magnitude')
    else:
        return scaleND(v, 1.0 / magND(v))

def addND(v1, v2):
    """Adds two nD vectors together, itemwise"""
    return [vv1 + vv2 for vv1, vv2 in zip(v1, v2)]

def addNDx(v1, v2, limit=0):
    """Adds two nD vectors together, itemwise,
    ignoring items if they are not numeric, with
    an option to limit length of tuples to a certain
    length defined by the `limit` argument"""
    if limit > 0:
        return [vv1 + vv2 for i, (vv1, vv2) in enumerate(zip(v1, v2))
        if (isinstance(vv1, (int, float))
            and isinstance(vv2, (int, float))
            and i < limit)]
    else:
        return [vv1 + vv2 for vv1, vv2 in zip(v1, v2)
        if (isinstance(vv1, (int, float)) and isinstance(vv2, (int, float)))]

def subND(v1, v2):
    """Subtracts two nD vectors together, itemwise"""
    return [vv1 - vv2 for vv1, vv2 in zip(v1, v2)]

def subNDx(v1, v2, limit=0):
    """Subtracts two nD vectors together, itemwise,
    ignoring items if they are not numeric, with
    an option to limit length of tuples to a certain
    length defined by the `limit` argument

    e.g.    subNDx((2,4,1,'a'),(7,2,8,'b'))
        and
            subNDx((2,4,1,'a'),(7,2,8,'b'), limit=3)
        will both return [-5,2,-7]
    """
    if limit > 0:
        return [vv1 - vv2 for i, (vv1, vv2) in enumerate(zip(v1, v2))
        if (isinstance(vv1, (int, float))
            and isinstance(vv2, (int, float))
            and i < limit)]
    else:
        return [vv1 - vv2 for vv1, vv2 in zip(v1, v2)
        if (isinstance(vv1, (int, float)) and isinstance(vv2, (int, float)))]

def mulND(v1, v2):
    """Returns product of two nD vectors
    (same as itemwise multiplication)"""
    return [vv1 * vv2 for vv1, vv2 in zip(v1, v2)]

def mulNDx(v1, v2, limit=0):
    """Multiplies two nD vectors together, itemwise,
    ignoring items if they are not numeric, with
    an option to limit length of tuples to a certain
    length defined by the `limit` argument"""
    if limit > 0:
        return [vv1 * vv2 for i, (vv1, vv2) in enumerate(zip(v1, v2))
        if (isinstance(vv1, (int, float))
            and isinstance(vv2, (int, float))
            and i < limit)]
    else:
        return [vv1 * vv2 for vv1, vv2 in zip(v1, v2)
        if (isinstance(vv1, (int, float)) and isinstance(vv2, (int, float)))]

def divND(v1, v2):
    """Returns divisor of two nD vectors
    (same as itemwise multiplication)
    Note that this does not catch cases
    where an element of the divisor is zero."""
    return [vv1 / vv2 for vv1, vv2 in zip(v1, v2)]

def divNDx(v1, v2, limit=0):
    """Returns itemwise divisor of two nD vectors,
    ignoring items if one or more are not numeric, with
    an option to limit length of tuples to a certain
    length defined by the `limit` argument
    Note that this does not catch cases
    where an element of the divisor is zero."""
    if limit > 0:
        return [vv1 * vv2 for i, (vv1, vv2) in enumerate(zip(v1, v2))
        if (isinstance(vv1, (int, float))
            and isinstance(vv2, (int, float))
            and i < limit)]
    else:
        return [vv1 * vv2 for vv1, vv2 in zip(v1, v2)
        if (isinstance(vv1, (int, float)) and isinstance(vv2, (int, float)))]

def dotND(v1, v2):
    """Returns dot product of two nD vectors
    (same as itemwise multiplication followed by summation)."""
    return sum(vv1 * vv2 for vv1, vv2 in zip(v1, v2))

def distND(pt1, pt2):
    """Returns distance between two nD points (as two n-tuples)"""
    return (sum((vv2 - vv1)**2.0 for vv1, vv2 in zip(pt1, pt2)))**0.5

def distNDx(pt1, pt2, limit=0):
    """Returns distance between two nD points (as two n-tuples)
    It ignores items if they are not numeric, and also has
    an option to limit length of tuples to a certain
    length defined by the `limit` argument"""
    if limit > 0:
        return (sum((vv2 - vv1)**2.0 for i, (vv1, vv2) in enumerate(zip(pt1, pt2))
        if isinstance(vv1, (int, float))
        and isinstance(vv2, (int, float))
        and i < limit))**0.5
    else:
        return (sum((vv2 - vv1)**2.0 for vv1, vv2 in zip(pt1, pt2)
        if isinstance(vv1, (int, float))
        and isinstance(vv2, (int, float))))**0.5

def scaleND(v, s):
    """Scales an nD vector by a factor s."""
    return [s * vv for vv in v]

def scaleNDx(v, s, limit=0):
    """Multiplies two nD vectors together, itemwise,
    ignoring items if they are not numeric, and also
    an option to limit length of tuples to a certain
    length defined by the `limit` argument"""
    if limit > 0:
        return [s * vv for i, vv in enumerate(v)
        if (isinstance(v, (int, float))
            and i < limit)]
    else:
        return [s * vv for vv in v
        if isinstance(v, (int, float))]

def cos_simND(v1, v2):
    """returns the cosine of the angle between two vectors (v1, v2) based on the dot product
    v1 . v2 = |v1|  |v2| cos (<v1,v2>)"""
    m1, m2 = magND(v1), magND(v2)
    if m1 * m2 == 0.0 :
        raise ValueError('Both vectors should have non-zero magnitude')
    else:
        return dotND(v1, v2) / m1 / m2

# ================================
# ==== Matrix Manipulations ======
# ================================

def Madd(M1, M2):
    """Matrix addition (elementwise)"""
    return [[a+b for a, b in zip(c, d)] for c, d in zip(M1, M2)]

def Msub(M1, M2):
    """Matrix subtraction (elementwise)"""
    return [[a-b for a, b in zip(c, d)] for c, d in zip(M1, M2)]

def Mtranspose(M):
    """Matrix transposition"""
    return list(zip(*M))

def Mmult(M1, M2):
    """Matrix multiplication (no checks)"""
    return [[sum(a*b for a, b in zip(c, d)) for c in Mtranspose(M2)] for d in M1]

def MI(n: int):
    """nxn identity matrix"""
    return [[1 if i==j else 0 for i in range(n)] for j in range(n)]

def Minv(matrix):
    """Returns the inverse of a square matrix using Gaussian elimination
    Raises a ValueError if matrix is non-square or singular"""
    n = len(matrix)
    if sum(len(matrix)!=n for m in matrix):
        raise ValueError('Matrix is not square.')

    Mm = matrix.copy() # otherwise original will be modified
    Im = MI(n)

    for i in range(n):
        k = 1
        s = Mm[i][i]
        while s==0 and i+k < n:
            Mm[i] = addND(Mm[i], Mm[i+k])
            Im[i] = addND(Im[i], Im[i+k])
            k+=1
            s = Mm[i][i]
        if s == 0:
            err_msg = 'Singular Matrix\n'
            err_msg += f'Processing row {i+1} of {n}:\n'
            err_msg +=  repr(Mm)
            raise ValueError(err_msg)

        Mm[i] = scaleND(Mm[i], 1 / s)
        Im[i] = scaleND(Im[i], 1 / s)
        for j in range(n):
            if j!=i and Mm[j][i] != 0:
                s = Mm[j][i]
                Mm[j] = subND(Mm[j], scaleND(Mm[i], s))
                Im[j] = subND(Im[j], scaleND(Im[i], s))
    return Im #, Mm

def Mdet(matrix):
    """Returns the determinant of a square matrix using Gaussian elimination

    Determinant is calculated by Gaussian elimination method
    A ValueError is raised if matrix is non-square or singular

    Args:
        matrix:

    """
    n = len(matrix)
    if sum(len(m)!=n for m in matrix):
        raise ValueError('Matrix is not square.')

    M = matrix.copy() # otherwise original will be modified

    det_sign = 1

    for i in range(n):
        k=1
        s = M[i][i]
        # Switch rows if diagonal is zero
        while s==0 and i+k < n:
            M[i], M[i+k] = M[i+k], M[i]
            det_sign *= -1
            k+=1
            s = M[i][i]
        if s == 0:
            err_msg = 'Singular Matrix\n'
            err_msg += f'Processing row {i+1} of {n}:\n'
            err_msg +=  repr(M)
            raise ValueError(err_msg)

        # eliminate value in i-th column in remainimg rows
        for j in range(i+1, n):
            if M[j][i] != 0:
                s = M[j][i] / M[i][i]
                M[j] = subND(M[j], scaleND(M[i], s))

    #print(M)
    return det_sign * reduce(mul, [M[i][i] for i in range(n)], 1)

def kron():
    """
    TO DO - Kronecker product"""
    pass

def main():
    #import numpy as np
    print("Hello World!")
    A = [[1, 2], [4, -2]]
    B = [[4, -1], [-3, 5]]
    C = [[0, 3], [1, 5]]
    print('A', A)
    print('B', B)
    print('C', C)
    print('A+B', Madd(A,B))
    print('A*B', Mmult(A,B))
    print('det(A)', Mdet(A))

    A = [[1, 3, 2], [4, -1, 2], [2, 4, -1]]
    B = [[1, 3, 2], [4, -1, 2], [-3, 1, 5]]
    C = [[0, 3, 2], [0, 0, 2], [-3, 1, 5]]
    D = [[0, 0, 2], [0, 3, 0], [-5, 0, 0]]
    A[0], A[1] = A[1], A[0]
    print('A', A)
    print('B', B)
    print('C', C)
    print('A+B', Madd(A,B))
    print('A*B', Mmult(A,B))
    print('det(A)', Mdet(A))
    print('det(B)', Mdet(B))
    print('det(C)', Mdet(C))
    print('det(D)', Mdet(D))
    #print('C', C)
    print('inv(A)', Minv(A))
    print('inv(D)', Minv(D))

    #print('det(A)', np.linalg.det(np.array(A)))
    #print('det(B)', np.linalg.det(np.array(B)))
    #print('det(C)', np.linalg.det(np.array(C)))
    #print('det(D)', np.linalg.det(np.array(D)))
    #print('inv(A)', np.linalg.inv(np.array(A)))
    #print('inv(D)', np.linalg.inv(np.array(D)))

if __name__ == "__main__":
    main()


import numpy as np

# It's possible, given the mistakes of each training data, to calculate the parameters when using Kernel Perceptron Algorithm as follows:
#
# [Writen in LaTeX]
# \theta := theta_{initial} + \sum_{i=1}^{n} \beta^{i}y^{i}\phi(x^{i})
# \theta_{0} := theta_{0, initial} + \sum_{i=1}^{n} \beta^{i}y^{i}
#
# where \beta^{i} is the number of mistakes of ith training point

# Initialize the parameters as 0
theta_exam, theta_0_exam = np.zeros(3), 0

# Number os mistakes
history_exam = {'x1': 1, 'x2': 65, 'x3': 11, 'x4': 31,
                'x5': 72, 'x6': 30, 'x7': 0, 'x8': 21, 'x9': 4, 'x10': 15}

# Training set
x = np.array([[0, 0],
              [2, 0],
              [1, 1],
              [0, 2],
              [3, 3],
              [4, 1],
              [5, 2],
              [1, 4],
              [4, 4],
              [5, 5]])

y = np.array([[-1],
              [-1],
              [-1],
              [-1],
              [-1],
              [1],
              [1],
              [1],
              [1],
              [1]])

def Kernel(x):
    return np.array([x[0]**2, np.sqrt(2)*x[0]*x[1], x[1]**2])

# Evaluate the parameters
for i in range(len(history_exam)):
    theta_exam += history_exam.get('x' + str(i + 1))*y[i]*Kernel(x[i])
    theta_0_exam += history_exam.get('x' + str(i + 1))*y[i]

print(
    f"Kernel Vector:\n {np.array([Kernel(x[i]) for i in range(len(history_exam))])}\n")
print(f"Theta Exam: {theta_exam} | Theta 0 Exam: {theta_0_exam} \n")

# Classify the Training Points considering the Parameters above:
prediction = np.empty((y.shape))
for i, point in enumerate(x):
    phi = Kernel(x[i])
    prediction[i] = np.where(np.dot(theta_exam, phi) + theta_0_exam < 0, -1, 1)

if (y == prediction).sum() == y.shape[0]:
    print(
        f"The Kernel Perceptron correctly classified all points")
else:
    print(
        f"The Kernel Perceptron incorrectly classified {(y != prediction).sum()} points")


# Method of manufactured solutions (MMS)

from sympy import *
x, t, rho, dt = symbols('x t rho dt')

def a(u):
    return 1 + u**2

def u_simple(x, t):
    return x**2*(Rational(1,2) - x/3)*t

# Show that u_simple satisfies the BCs
for x_point in 0, 1:
    print 'u_x(%s,t):' % x_point,
    print diff(u_simple(x, t), x).subs(x, x_point).simplify()

print 'Initial condition:', u_simple(x, 0)

# MMS: full nonlinear problem
u = u_simple(x, t)
f = rho*diff(u, t) - diff(a(u)*diff(u, x), x)
print 'f: nonlinear problem:'
print f.simplify()

# MMS: one-Picard-iteration linearized problem
u_1 = u_simple(x, t-dt)
f = rho*diff(u, t) - diff(a(u_1)*diff(u, x), x)
print 'f: 1-Picard-iteration linearized problem:'
print f.simplify()

# MMS: Backward Euler discretization in time +
# one-Picard-iteration linearized problem
def D_t_backward(u):
    return (u(x, t) - u(x, t - dt))/dt

u_1 = u_simple(x, t-dt)
f = rho*D_t_backward(u_simple) - diff(a(u_1)*diff(u, x), x)
print 'f: BE in time + 1-Picard-iteration linearized problem:'
print f.simplify()


import numpy as np
def gradient_descent(grad_func, x_set, y_set, w0,
					learning_rate=0.01, MaxIter=10):
	path = []
	path.append(w0)
	for i in range(MaxIter):
		# compute gradient
		grad = grad_func(w0, x_set, y_set)
		# stopping criteria
		if np.linalg.norm(grad) < 1E-7:
			break
		# next step
		w1 = w0 - learning_rate * grad
		# write history of w0
		path.append(w1)
		# update
		w0 = w1
	return w0, path

def newton_method(grad_func, hessian_func, x_set, y_set, w0,
					learning_rate=0.01, MaxIter=10):
	path = []
	path.append(w0)
	for i in range(MaxIter):
		# compute gradient
		grad = grad_func(w0, x_set, y_set)
		# compute hessian
		hessian = hessian_func(w0, x_set, y_set)
		# stopping criteria
		if np.linalg.norm(grad) < 1E-7:
			break
		# set serach direction by Newton method
		dx = np.linalg.solve(hessian, grad)
		# next step
		w1 = w0 - learning_rate * dx
		# write history of w0
		path.append(w1)
		# update
		w0 = w1
	return w0, path

def bfgs_method(grad_func, x_set, y_set, w0,
					learning_rate=0.01, MaxIter=10):
	path = []
	path.append(w0)
	B0 = np.eye(len(w0))
	for i in range(MaxIter):
		grad = grad_func(w0,x_set, y_set)
		if np.linalg.norm(grad) < 1E-7:
			break
		p0 = -np.linalg.solve(B0, grad)
		s0 = learning_rate * p0
		w1 = w0 + s0
		y0 = (grad_func(w1) - grad).reshape(-1,1) # convert to a column vector
		B1 = B0 + np.dot(y0, y0.T) / np.dot(y0.T, s0) \
				- (np.dot(np.dot(B0, s0).reshape(-1,1), np.dot(s0, B0).reshape(-1,1).T)) / np.dot(np.dot(B0, s0), s0)
		# write history of w0
		path.append(w1)
		# update
		w0 = w1
		B0 = B1
	return w0, path

def nelder_mead(func, x0):
    maxiter = 2000
    maxfun = None

    rho = 1
    chi = 2
    psi = 0.5
    sigma = 0.5
    nonzdelt = 0.05
    zdelt = 0.00025

    dim = 5

    N = len(x0)

    sim = np.zeros((N + 1, N), dtype=x0.dtype)
    sim[0] = x0
    for k in range(N):
        y = np.array(x0, copy=True)
        if y[k] != 0:
            y[k] = (1 + nonzdelt)*y[k]
        else:
            y[k] = zdelt
        sim[k + 1] = y

    fsim = np.zeros((N + 1,), float)

    for k in range(N + 1):
        fsim[k] = func(sim[k])

    ind = np.argsort(fsim)
    fsim = np.take(fsim, ind, 0)
    # sort so sim[0,:] has the lowest function value
    sim = np.take(sim, ind, 0)

    iterations = 1

    while iterations < maxiter :
        xbar = np.add.reduce(sim[:-1], 0) / N
        xr = (1 + rho) * xbar - rho * sim[-1]
        fxr = func(xr)
        doshrink = 0

        if fxr < fsim[0]:
            xe = (1 + rho * chi) * xbar - rho * chi * sim[-1]
            fxe = func(xe)

            if fxe < fxr:
                sim[-1] = xe
                fsim[-1] = fxe
            else:
                sim[-1] = xr
                fsim[-1] = fxr
        else:  # fsim[0] <= fxr
            if fxr < fsim[-2]:
                sim[-1] = xr
                fsim[-1] = fxr
            else:  # fxr >= fsim[-2]
                # Perform contraction
                if fxr < fsim[-1]:
                    xc = (1 + psi * rho) * xbar - psi * rho * sim[-1]
                    fxc = func(xc)

                    if fxc <= fxr:
                        sim[-1] = xc
                        fsim[-1] = fxc
                    else:
                        doshrink = 1
                else:
                    # Perform an inside contraction
                    xcc = (1 - psi) * xbar + psi * sim[-1]
                    fxcc = func(xcc)

                    if fxcc < fsim[-1]:
                        sim[-1] = xcc
                        fsim[-1] = fxcc
                    else:
                        doshrink = 1

                if doshrink:
                    for j in range(1, N + 1):
                        sim[j] = sim[0] + sigma * (sim[j] - sim[0])
                        fsim[j] = func(sim[j])

        ind = np.argsort(fsim)
        sim = np.take(sim, ind, 0)
        fsim = np.take(fsim, ind, 0)
        iterations += 1

    x = sim[0]
    fval = np.min(fsim)
    return x

def momentum_method(grad_func, x_set, y_set, w0,
					learning_rate=0.01, alpha=0.9, MaxIter=10):
	path = []
	path.append(w0)
	velocity = np.zeros_like(w0)
	for i in range(MaxIter):
		# compute gradient
		grad = grad_func(w0, x_set, y_set)
		velocity = alpha * velocity  - learning_rate * grad
		# stopping criteria
		if np.linalg.norm(grad) < 1E-7:
			break
		# next step
		w1 = w0 + velocity
		# write history of w0
		path.append(w1)
		# update
		w0 = w1
	return w0, path

def nesterov_method(grad_func, x_set, y_set, w0,
					learning_rate=0.01, alpha=0.9, MaxIter=10):
	path = []
	path.append(w0)
	velocity = np.zeros_like(w0)
	for i in range(MaxIter):
		# compute gradient
		grad = grad_func(w0 + alpha * velocity, x_set, y_set)
		velocity = alpha * velocity  - learning_rate * grad
		# stopping criteria
		if np.linalg.norm(grad) < 1E-7:
			break
		# next step
		w1 = w0 + velocity
		# write history of w0
		path.append(w1)
		# update
		w0 = w1
	return w0, path

def adagrad_method(grad_func, x_set, y_set, w0,
					learning_rate=0.01, delta=1E-7, MaxIter=10):
	path = []
	path.append(w0)
	r = np.zeros_like(w0)
	for i in range(MaxIter):
		# compute gradient
		grad = grad_func(w0, x_set, y_set)
		r = r  + grad * grad
		# stopping criteria
		if np.linalg.norm(grad) < 1E-7:
			break
		# next step
		w1 = w0 - learning_rate * grad / ( delta + np.sqrt(r) )
		# write history of w0
		path.append(w1)
		# update
		w0 = w1
	return w0, path

def rmsprop_method(grad_func, x_set, y_set, w0,
					learning_rate=0.01, delta=1E-6, rho=0.9 , MaxIter=10):
	path = []
	path.append(w0)
	r = np.zeros_like(w0)
	for i in range(MaxIter):
		# compute gradient
		grad = grad_func(w0, x_set, y_set)
		r = rho * r  + (1 - rho) * (grad * grad)
		# stopping criteria
		if np.linalg.norm(grad) < 1E-7:
			break
		# next step
		w1 = w0 - learning_rate  * grad / np.sqrt(delta + r)
		# write history of w0
		path.append(w1)
		# update
		w0 = w1
	return w0, path

def adam_method(grad_func, x_set, y_set, w0,
					learning_rate=0.001, delta=1E-8, rho1=0.9, rho2=0.999, MaxIter=10):
	path = []
	path.append(w0)
	s = np.zeros_like(w0)
	r = np.zeros_like(w0)
	for i in range(MaxIter):
		# compute gradient
		grad = grad_func(w0, x_set, y_set)
		s = rho1 * s  + (1 - rho1) * grad
		r = rho2 * r  + (1 - rho2) * (grad * grad)
		# stopping criteria
		if np.linalg.norm(grad) < 1E-7:
			break
		shat = s / (1. - rho1 ** (i+1))
		rhat = r / (1. - rho2 ** (i+1))
		# next step
		w1 = w0 - learning_rate  * shat / (delta + np.sqrt(rhat))
		# write history of w0
		path.append(w1)
		# update
		w0 = w1
	return w0, path

def generate_batches(batch_size, features, labels):
	"""
	Create batches of features and labels
	:param batch_size: The batch size
	:param features: List of features
	:param labels: List of labels
	:return: Batches of (Features, Labels)
	"""
	assert len(features) == len(labels)
	outout_batches = []

	sample_size = len(features)
	for start_i in range(0, sample_size, batch_size):
		end_i = start_i + batch_size
		batch = [features[start_i:end_i], labels[start_i:end_i]]
		outout_batches.append(batch)

	return outout_batches


import numpy as np
#
# Geographic Centroid
#
def get_centroid(in_lat, in_lon):
    """Given collection of latitudes and longitudes,
       return the location (lon, lat) of the geographic centroid.

       in_lat :: list or array of latitudes in DEGREES.
       in_lon :: list or array of longitudes in DEGREES.

       return :: tuple (lat, lon) of the centroid in DEGREES
    """
    assert len(in_lat) == len(in_lon)
    if isinstance(in_lat, list):
        in_lat = np.array(in_lat)
    if isinstance(in_lon, list):
        in_lon = np.array(in_lon)
    # convert to spherical:
    xpts = np.cos(np.radians(in_lat)) * np.cos(np.radians(in_lon))
    ypts = np.cos(np.radians(in_lat)) * np.sin(np.radians(in_lon))
    zpts = np.sin(np.radians(in_lat))

    # average (not weighted):
    xctr = xpts.mean()
    yctr = ypts.mean()
    zctr = zpts.mean()

    # convert to lat lon
    ctr_lon = np.arctan2(yctr, xctr)
    hyp = np.sqrt(xctr**2 + yctr**2)
    ctr_lat = np.arctan2(zctr, hyp)
    return np.degrees(ctr_lat), np.degrees(ctr_lon)

#
# Haversine Distance (note there is a package called haversine)
#
def distance(lat1, lon1, lat2, lon2, r=None):
    """
    Using Haversine function, get distance between
    (lat1, lon1) and (lat2, lon2) in meters.

    lon1, lat1, lon2, lat2 :: coordinates in DEGREES
    r :: is the radius of the sphere; if None use Earth radius
    return :: distance in meters
    """
    lat1 = np.radians(lat1)
    lon1 = np.radians(lon1)
    lat2 = np.radians(lat2)
    lon2 = np.radians(lon2)
    if r is None:
        r = 6378.1 * 1000. # km * (1000m/km) = meters
    term1 = (np.sin((lat2 - lat1)/2))**2
    term2 = np.cos(lat1)*np.cos(lat2)*(np.sin((lon2 - lon1) / 2))**2
    return 2 * r * np.arcsin(np.sqrt(term1 + term2))


import numpy as np
import matplotlib.pyplot as plt

# Lloyds algorithm for k-means
def kmeans(X, k, max_iter = 100, tolerance = 10**(-3)):
  n_samples = X.shape[0]
  n_features = X.shape[1]
  classifications = np.zeros(n_samples, dtype = np.int64)

  # Choose initial cluster centroids randomly
  I = np.random.choice(n_samples, k)
  centroids = X[I, :]

  loss = 0
  for m in range(0, max_iter):
    # Compute the classifications
    for i in range(0, n_samples):
      distances = np.zeros(k)
      for j in range(0, k):
        distances[j] = np.sqrt(np.sum(np.power(X[i, :] - centroids[j], 2)))
      classifications[i] = np.argmin(distances)

    # Compute the new centroids and new loss
    new_centroids = np.zeros((k, n_features))
    new_loss = 0
    for j in range(0, k):
      # compute centroids
      J = np.where(classifications == j)
      X_C = X[J]
      new_centroids[j] = X_C.mean(axis = 0)

      # Compute loss
      for i in range(0, X_C.shape[0]):
        new_loss += np.sum(np.power(X_C[i, :] - centroids[j], 2))

    # Stopping criterion
    if np.abs(loss - new_loss) < tolerance:
      return new_centroids, classifications, new_loss

    centroids = new_centroids
    loss = new_loss

  print("Failed to converge!")
  return centroids, classifications, loss


import numpy as np

array1 = np.random.randint(10, size=(4, 3))
array2 = np.random.randint(10, size=(3, 5))
result = np.ones((4,5))

result = np.dot(array1, array2)

print('first_array\n',array1);
print('second_array\n',array2);
print('result_array\n',result);


import numpy as np
import matplotlib.pyplot as plt
from sklearn.cluster import SpectralClustering
from sklearn.datasets import make_blobs

n_samples = 200
random_state = 10

X, y = make_blobs(n_samples=n_samples, random_state=random_state)
y_predicted = SpectralClustering(n_clusters=3, random_state=random_state).fit_predict(X)
plt.scatter(X[:, 0], X[:, 1], c=y_predicted)
plt.title("Spectral Clustering")
plt.show()

X_1, y_1 = make_blobs(n_samples=n_samples, cluster_std=[1, 0.5, 3.0], random_state=random_state)
y_predicted = SpectralClustering(n_clusters=3, random_state=random_state).fit_predict(X_1)
plt.scatter(X_1[:, 0], X_1[:, 1], c=y_predicted)
plt.title("Spectral Clustering II")
plt.show()

X_2, y_2 = make_blobs(n_samples=n_samples, cluster_std=[1, 1.5, 2.5], random_state=random_state)
y_pred = SpectralClustering(n_clusters=3, random_state=random_state).fit_predict(X_2)
plt.scatter(X_2[:, 0], X_2[:, 1], c=y_predicted)
plt.title("Spectral Clustring III")
plt.show()

X_not_balanced = np.vstack((X[y == 0][:500], X[y == 1][:200], X[y == 2][:10]))
y_predicted= SpectralClustering(n_clusters=3,random_state=random_state).fit_predict(X_not_balanced)
plt.scatter(X_not_balanced[:, 0], X_not_balanced[:, 1], c=y_predicted)
plt.title("Blobs having unequal number of elements in each cluster")
plt.show()


import math
import numpy as np
import scipy.io as sio
import time
import torch

from numpy import dot
from torch import tensor

# computing curvature
def kappa(v,a):
    return (dot(v,v))**(-3/2)*(dot(v,v)*dot(a,a) - dot(v,a)**2)

# returned as torch.tensor
def gbasis(u0,u1,theta_ls,N_0,q):

    assert len(u0) == N_0 and len(u1) == N_0, "u_0 or u_1 dimension doesn't match N_0"

    u0, u1 = u0.reshape((-1,1)), u1.reshape((-1,1))
    #U0,U1 = np.repeat(u0, len(theta_ls), axis=1), np.repeat(u1, len(theta_ls), axis=1)
    U0, U1 = u0.repeat((1,len(theta_ls))), u1.repeat((1,len(theta_ls)))

    # just watch this part if torch diff doesn't work (dtype=torch.float32)
    return torch.sqrt( tensor([float(N_0*q)]) )*(torch.cos(theta_ls)*U0 + torch.sin(theta_ls)*U1)
    #return torch.sqrt( tensor([N_0*q]) )*(torch.cos(theta_ls)*U0 + torch.sin(theta_ls)*U1, dtype=torch.float32)

# wm_ls,u0,u1 fixed
def hidden_fixed(N_0,q,theta,wm_ls,u0,u1,post=True):

    L = len(wm_ls)
    #gcirc = np.sqrt(N_0*q)*(np.cos(angles[0])*u0 + np.sin(angles[0])*u1)
    gcirc = np.sqrt(N_0*q)*(np.cos(theta)*u0 + np.sin(theta)*u1)
    hidden_ls = [gcirc - gcirc.mean(0,keepdims=True)] # includes input layer

    if post == True:
        for l in range(L):
            W_l = wm_ls[l]
            x_l = np.tanh(np.matmul(hidden_ls[l],W_l))
            hidden_ls.append(x_l)
    else:
        for l in range(L):
            W_l = wm_ls[l]
            x_l = np.matmul(hidden_ls[l],W_l)
            preact_layers.append(x_l)
            x_l = np.tanh(x_l)

    return np.array(hidden_ls)

# velocity and acceleration (first and second order central)
def vel_acc(N_0,q,theta_ls,wm_ls,u0,u1,post=True):

    h = 1e-15 # error
    vel_ls = []
    acc_ls = []

    for i in range(len(theta_ls)):
        theta = theta_ls[i]
        hidden_ls = hidden_fixed(N_0,q,theta,wm_ls,u0,u1,post)

        vel = (hidden_fixed(N_0,q,theta + h,wm_ls,u0,u1,post) - hidden_fixed(N_0,q,theta - h,wm_ls,u0,u1,post))/(2*h)
        acc = (hidden_fixed(N_0,q,theta + h,wm_ls,u0,u1,post) + hidden_fixed(N_0,q,theta,wm_ls,u0,u1,post) - hidden_fixed(N_0,q,theta - h,wm_ls,u0,u1,post))/(h**2)

        vel_ls.append(vel)
        acc_ls.append(acc)

    return np.array(vel_ls), np.array(acc_ls)


import numpy as np

def get_onehot(y):
    classes = np.unique(y)
    one_hot = np.zeros((len(y),len(classes)))
    for i in range(len(y)):
        one_hot[i][y[i]] = 1
    return one_hot

class activater():
    def softmax(self, X):
        return np.exp(X)/np.sum(np.exp(X),1).reshape(-1,1)

    def sigmoid(self, X):
        return 1 / (1 + np.exp(-X))

    def is_multi_class(self, multi_class = True):
        if multi_class:
            return self.softmax
        else:
            return self.sigmoid

class LogisticRegression():
    np.random.seed(0)
    def __init__(self, learning_rate, iterations, multi_class = True, fit_intercept=True):
        self.learning_rate = learning_rate
        self.iterations = iterations
        self.activater_fun = activater().is_multi_class(multi_class)
        self.multi_class = multi_class
        self.fit_intercept = fit_intercept

    def add_intercept(self, X):
        intercept = np.ones((X.shape[0], 1))
        return np.concatenate((intercept, X), axis=1)

    def fit(self, X, t):
        if self.fit_intercept:
            X = self.add_intercept(X)
        self.X = X
        self.t = t
        self.classes = t.shape[1] if self.multi_class else 1
        self.m, self.n = X.shape
        if self.multi_class:
            self.w = np.random.randn(self.classes,self.n)
        else:
            self.w = np.random.randn(self.n)
        for i in range(self.iterations):
            error = self.cross_entropy(self.X, self.t)
            if (i+1) % 10 == 0:
                print('Iteration{}/{}, Cross Entropy Error: {:.3f}'.format(i+1,self.iterations,error))
            self.update_weights()

    def update_weights( self ) : #gradient decent
        scale = self.activater_fun(self.y(self.X))-self.t
        dw = scale.T.dot(self.X)/self.m
        self.w -= self.learning_rate * dw

    def y(self, X):
        if self.multi_class:
            return self.w.dot(X.T).T
        else:
            return self.w.dot(X.T)

    def cross_entropy(self, X, t):
        s = self.activater_fun(self.y(X))
        if self.multi_class:
            error = -np.sum(np.log(s)*t)
        else:
            error = -np.sum(t * np.log(s) + (1 - t) * np.log(1-s))
        return error/self.m

    def predict(self, X):
        Y = self.activater_fun(self.y(X))
        if self.multi_class:
            pre = np.argmax(Y,1)
        else:
            pre = Y>=1/2
            pre = pre*1
        return pre


import matplotlib.pyplot as plt
from numpy.ma import arange

from probability.distributions.continuous.gamma import Gamma
from mpl_format.axes.axis_utils import new_axes

# https://en.wikipedia.org/wiki/Gamma_distribution

x = arange(0, 20.1, 0.1)

def plot_wikipedia_pdfs():
    """
    https://en.wikipedia.org/wiki/Gamma_distribution#/media/
    File:Gamma_distribution_pdf.svg
    """
    ax = new_axes()
    Gamma.from_k_theta(k=1, theta=2).plot(x=x, color='red', ax=ax)
    Gamma.from_k_theta(k=2, theta=2).plot(x=x, color='orange', ax=ax)
    Gamma.from_k_theta(k=3, theta=2).plot(x=x, color='yellow', ax=ax)
    Gamma.from_k_theta(k=5, theta=1).plot(x=x, color='green', ax=ax)
    Gamma.from_k_theta(k=9, theta=0.5).plot(x=x, color='black', ax=ax)
    Gamma.from_k_theta(k=7.5, theta=1).plot(x=x, color='blue', ax=ax)
    Gamma.from_k_theta(k=0.5, theta=1).plot(x=x, color='purple', ax=ax)
    ax.set_ylim(0, 0.5)
    ax.legend(loc='upper right')
    plt.show()

def plot_wikipedia_cdfs():
    """
    https://en.wikipedia.org/wiki/Gamma_distribution#/media/
    File:Gamma_distribution_cdf.svg
    """
    ax = new_axes()
    Gamma.from_k_theta(k=1, theta=2).cdf().plot(x=x, color='red', ax=ax)
    Gamma.from_k_theta(k=2, theta=2).cdf().plot(x=x, color='orange', ax=ax)
    Gamma.from_k_theta(k=3, theta=2).cdf().plot(x=x, color='yellow', ax=ax)
    Gamma.from_k_theta(k=5, theta=1).cdf().plot(x=x, color='green', ax=ax)
    Gamma.from_k_theta(k=9, theta=0.5).cdf().plot(x=x, color='black', ax=ax)
    Gamma.from_k_theta(k=7.5, theta=1).cdf().plot(x=x, color='blue', ax=ax)
    Gamma.from_k_theta(k=0.5, theta=1).cdf().plot(x=x, color='purple', ax=ax)
    ax.set_ylim(0, 1)
    ax.legend(loc='lower right')
    plt.show()

def plot_increasing_alpha_beta():

    ax = new_axes()
    x = arange(0, 1.001, 0.001)
    Gamma(alpha=4, beta=10).pdf().plot(x=x, color='red', ax=ax)
    Gamma(alpha=8, beta=20).pdf().plot(x=x, color='red', ax=ax)
    Gamma(alpha=16, beta=40).pdf().plot(x=x, color='orange', ax=ax)
    Gamma(alpha=32, beta=80).pdf().plot(x=x, color='yellow', ax=ax)
    Gamma(alpha=64, beta=160).pdf().plot(x=x, color='green', ax=ax)
    Gamma(alpha=128, beta=320).pdf().plot(x=x, color='black', ax=ax)
    Gamma(alpha=256, beta=640).pdf().plot(x=x, color='blue', ax=ax)
    Gamma(alpha=512, beta=1280).pdf().plot(x=x, color='purple', ax=ax)
    ax.set_xlim(0, 1)
    ax.legend(loc='lower right')
    plt.show()

if __name__ == '__main__':

    # plot_wikipedia_pdfs()
    # plot_wikipedia_cdfs()
    plot_increasing_alpha_beta()


# 퍼셉트론이란 다수의 신호를 받아 하나의 신호를 출력한다.
# 입력신호가 노드에 보내지고 각각에 가중치가 곱해진다.
# 거기에 고유한 편향(bias)가 더해져서 정해진 값을 넘어서면 1, 넘지 못하면 0이 출력된다.

# y = {0 (w1*x1 + w2*x2) <= theta}
# y = {1 (w1*x1 + w2*x2) > theta}
# 여기서 b는 편향을 의미하며 w는 가중치

# 단순한 논리회로
# AND, OR, NAND 게이트
# AND, OR, NAND 게이트는 입력이 둘이고 출력은 하나, 입력이 모두 1일때 1을 출력하고 그 외에는 모두 0을 출력
# AND, OR, NAND 게이트를 진리표대로 작동하도록 하는 가중치를 구하자!
# 퍼셉트론 구현하기
def AND(x1, x2):
    w1, w2, theta = 0.5, 0.5, 0.7
    tmp = x1*w1 + x2*w2
    if tmp <= theta:
        return 0
    elif tmp > theta:
        return 1
AND(0, 0)
AND(0, 1)
AND(1, 0)
AND(1, 1)

# 가중치와 편향도입
# y = {0 (b + w1*x1 + w2*x2) <= 0}
# y = {1 (b + w1*x1 + w2*x2) > 0}

# 위의 결과를 넘파이로 구현
import numpy as np
x = np.array([0, 1])
w = np.array([.5, .5])
b = -.7
w * x
np.sum(w*x)      # 각 노드로 들어가는 가중치 다 더해줌
np.sum(w*x) + b  # 편향 더함, -0.19999999999999996

# 가중치와 편향 구현하기
def AND(x1, x2):
    x = np.array([x1, x2])
    w = np.array([0.5, 0.5])  # 여기를 조정
    b = -0.7
    tmp = np.sum(w*x) + b
    if tmp <= 0:
        return 0
    else:
        return 1

def NAND(x1, x2):
    x = np.array([x1, x2])
    w = np.array([-0.5, -0.5])  # 여기를 조정
    b = 0.7
    tmp = np.sum(w*x) + b
    if tmp <= 0:
        return 0
    else:
        return 1

def OR(x1, x2):
    x = np.array([x1, x2])
    w = np.array([0.5, 0.5])  # 여기를 조정
    b = -0.2
    tmp = np.sum(w*x) + b
    if tmp <= 0:
        return 0
    else:
        return 1

# 퍼셉트론의 한계
# XOR을 구현하지 못한다.
# 퍼셉트론은 직선 하나로만 나눈 영역만 표현할 수 있다는 한계가 있다.
# x1 + x2 <= 0.5처럼

# 다층 퍼셉트론을 사용하여 XOR구현하기
# 기존 게이트 조합하기
# (NAND + OR) + AND ==> XOR

def XOR(x1, x2):
    s1 = NAND(x1, x2)
    s2 = OR(x1, x2)
    y = AND(s1, s2)
    return y
XOR(0,0)
XOR(1,0)
XOR(0,1)
XOR(1,1)


import numpy as np

def get_PCA(X, n_components=5):
    """
    Calculate PCA for a given matrix

    Parameters
    ----------

    X : array_like, (var, obs)
        Input matrix
    n_components : int
        Number of component to return; should be <= X.shape[1]
    """
    from sklearn.decomposition import PCA

    n_comp = min(X.shape[0], n_components)

    pca = PCA(n_components=n_comp)
    pca.fit(X)
    PC_mat = pca.components_

    return PC_mat

# , min_sum=None
#     n_comp = min(X.shape[0], n_components)
#     PC_mat = np.zeros((n_components, X.shape[1]))
#     PC_mat[:, :] = None

#     if min_sum is not None:
#         idx = np.where(np.sum(X, axis=0) >= min_sum)[0]
#     else:
#         idx = range(X.shape[1])

#     pca = PCA(n_components=n_comp)
#     pca.fit(X[:, idx])
#     PC_mat[:, idx] = pca.components_

#     return PC_mat


import numpy as np
from _decimal import Decimal
from numpy import linalg
'''
SOR iterative method.

Passes in Matrix A, b, uses zero vector for x as starting point.

Requires an error to be passed in, in our case 10^-8.

Requires an actual solution to be be passed in to calculate norm.

Norm is calculated using built in linalg function.

The default w is 1.0 if the user does not input a factor.

SOR method in terms of matrix multiplication:
((D-wL)^-1) * Ux + ((D-wL)^-1) * b

This function modifies the above to:
w * L^-1 * (b - Ux),
where L is a lower triangular matrix containing the diagonal,
and w is the SOR factor.
'''

def sor(A, b, x, error, actual_soln, max_iterations=9999, w=1.0):
    if error and actual_soln.any():
        # 'np.tril' returns a lower triangular matrix, containing the diagonal
        L = np.tril(A)
        # Multiply by a SOR factor
        L *= w
        U = A - L
        n = 0
        real_norm = linalg.norm(actual_soln)
        while True:
            x = np.dot(np.linalg.inv(L), b - np.dot(U, x))
            norm = linalg.norm(x)
            n += 1
            if abs(Decimal(real_norm) - Decimal(norm)) <= error or n >= max_iterations:
                break
    else:
        raise SyntaxError("Must pass in an error term and actual solution.")

    # Returns an array of the solution vector and number of iterations
    return [x, n]


import numpy as np

def trapezoidal_rule(xs, ys):
    h = (xs[len(xs)-1] - xs[0]) / len(xs)

    integral = ys[0] / 2
    for i in range(0, len(xs)-1):
        integral += h/2 * (ys[i] + ys[i+1])
    integral += ys[len(ys) - 1] / 2

    integral /= 1000  # change m to km
    return integral

xs = list(np.arange(0, 100, 0.1))
ys = [10 * x for x in xs]
ys = [5/18 * y for y in ys]

custom_int = trapezoid_rule(xs, ys)
lib_int = np.trapz(ys, xs) / 1000

print("Custom: " + str(custom_int))
print("Library: " + str(lib_int))


import numpy as np
from math import pi, exp
from scipy.special import erf

"""
Reference:: Modern Quantum Chemistry by Szabo & Ostlund
"""

def _EF(t):
    if t == 0: return 1.
    return (0.5*(pi/t)**0.5)*erf(t**0.5)

def S(g1, g2):
    """
    Calculate (g1|g2)
    """
    a, R_a = g1.a, g1.R
    b, R_b = g2.a, g2.R
    norm = np.linalg.norm(R_a - R_b)**2
    retval = (pi/(a+b))**1.5
    retval *= (4*a*b/pi**2)**0.75
    retval *= exp(-norm*a*b/(a+b))
    return retval

def T(g1, g2):
    """
    Calculate (g1|-0.5*\nabla^2|g2)
    """
    a, R_a = g1.a, g1.R
    b, R_b = g2.a, g2.R
    norm = np.linalg.norm(R_a - R_b)**2
    retval = (pi/(a+b))**1.5
    retval *= a*b/(a+b)
    retval *= 3-2*norm*a*b/(a+b)
    retval *= (4*a*b/pi**2)**0.75
    retval *= exp(-norm*a*b/(a+b))
    return retval

def V(g1, g2, Z_c, R_c):
    """
    Calculate (g1|-Z_c/R_c|g2)
    """
    a, R_a = g1.a, g1.R
    b, R_b = g2.a, g2.R
    _, p, R_p = g1.prod(g2)
    norm = np.linalg.norm(R_a - R_b)**2
    norm2 = np.linalg.norm(R_p - R_c)**2
    retval = -2*pi*Z_c/(a+b)
    retval *= (4*a*b/pi**2)**0.75
    retval *= exp(-norm*a*b/(a+b))
    retval *= _EF(norm2*(a+b))
    return retval

def two_electron(g1, g2, g3, g4):
    """
    Calculate (g1,g2|g3,g4)
    """
    a, R_a = g1.a, g1.R
    b, R_b = g2.a, g2.R
    c, R_c = g3.a, g3.R
    d, R_d = g4.a, g4.R
    _, p, R_p = g1.prod(g2)
    _, q, R_q = g3.prod(g4)
    norm = np.linalg.norm(R_a - R_b)**2
    norm2 = np.linalg.norm(R_c - R_d)**2
    norm3 = np.linalg.norm(R_p - R_q)**2
    retval = 2*pi**2.5/((a+b)*(c+d)*(a+b+c+d)**0.5)
    retval *= (4*a*b/pi**2)**0.75
    retval *= (4*c*d/pi**2)**0.75
    retval *= exp(-norm*a*b/(a+b))
    retval *= exp(-norm2*c*d/(c+d))
    retval *= _EF(norm3*(a+b)*(c+d)/(a+b+c+d))
    return retval


from numpy import exp,sqrt

def trapezoidal(f,a,b,n):
    h = (b-a)/n
    trap = 0.0
    for i in range(1,n):
        x = a + i*h
        trap = trap + 2.0 * f(x)
    trap = h * (f(a) + trap + f(b)) / 2.0
    return trap

def f(x):
    return x**2 * exp(-x)/sqrt(1.0 + x**2)

a = -1.0
b = 1.0
n = 512
T = trapezoidal(f,a,b,n)
print('T = %8.4f' % T)


import numpy as np

def squared_error(pred_y, expec_y):
    return (pred_y - expec_y) ** 2

def absolute_error(pred_y, expec_y):
    return np.abs(pred_y - expec_y)

def mean_squared_error(pred_y, expec_y):
    return np.average(squared_error(pred_y, expec_y))

def mean_absolute_error(pred_y, expec_y):
    return np.average(absolute_error(pred_y, expec_y))

def r2 (pred_y, expec_y):
    var_a = sum([ (y_i - y_t) ** 2 for y_i, y_t in zip(pred_y, expec_y) ])
    var_b = sum([ (y_i - expec_y.mean()) ** 2 for y_i in expec_y ])

    return 1 - (var_a/var_b)

def cross_entropy(y_pred, y_expec):
    return (-y_expec*np.log(y_pred) - (1 - y_expec)*np.log(1 - y_pred)).mean()

def accuracy(y_pred, y_expec):
    return (y_pred == y_expec).mean()


from math import sqrt
from typing import Tuple

import numpy as np
from scipy.stats import rankdata

def math_round(x: float, digit: int = 0) -> int:
    # In python there is no MATH round in std lib, so here's a hack
    return int(np.round(x, digit))

def calc_rank(arr: np.ndarray) -> Tuple[int, int, float]:
    arr = arr[np.argsort(arr[:, 0])]

    ranks = rankdata(arr[:, 1])
    # Now rank is from 1 to N
    # We need from N to 1
    ranks -= ranks.max() + 1
    ranks *= -1

    n = arr.shape[0]
    p = math_round(n / 3)

    r1 = ranks[:p].sum()
    r2 = ranks[-p:].sum()
    diff = r1 - r2
    error = math_round((n + 0.5) * sqrt(p / 6))
    corr = diff / (p * (n - p))

    return math_round(diff), error, np.round(corr, 2)

if __name__ == '__main__':
    def main():
        arr = np.loadtxt("in.txt", delimiter=" ")
        assert arr.ndim == 2 and arr.shape[1] == 2
        if arr.shape[0] < 9:
            print(f"There's just {arr.shape[0]} elements which is less than 9 and isn't enough for calculation")

        # Sort by x
        diff, error, corr = calc_rank(arr)

        with open("out.txt", "w") as f:
            f.write(f"{diff} {error} {corr}\n")

    main()


import numpy as np

def gauss_beam(sigma, ny, nx, FWHM=False):
	'''
	Return a Gaussian beam at the center of an image

	Parameters
	----------
	sigma: float
		Standard deviation (sigma) of the Gaussian function
	nx, ny: integers
		Size of the output image
	FWHM: bool
		If True, the sigma parameter becomes the Full Width Half Max

	Return
	----------
	Gaussian beam at the center of the image
	'''

	X=np.arange(nx)
	Y=np.arange(ny)
	xmap,ymap=np.meshgrid(X,Y)

	if (nx % 2) == 0:
		xmap = xmap - (nx)/2.
	else:
		xmap = xmap - (nx-1.)/2.

	if (ny % 2) == 0:
		ymap = ymap - (ny)/2.
	else:
		ymap = ymap - (ny-1.)/2.

	map = np.sqrt(xmap**2.+ymap**2.)

	if FWHM == True:

		sigma = sigma / (2.*np.sqrt(2.*np.log(2.)))

	gauss = np.exp(-0.5*(map)**2./sigma**2.)

	return gauss


import numpy as np

def gen_mean(vals, p = 3):
    p = float(p)
    return np.power(
        np.mean(
            np.power(
                np.array(vals, dtype=np.float64),
                p),
            axis=0),
        1 / p
    )

def get_pmeans(wordembeddings):
    """give wordembeddings"""
    pmean_embedding = []

    # ('max', (lambda word_embeddings: [np.max(word_embeddings, axis=0)], lambda embeddings_size: embeddings_size)),
    # ('min', (lambda word_embeddings: [np.min(word_embeddings, axis=0)], lambda embeddings_size: embeddings_size)),

    pmean_embedding.append(gen_mean(wordembeddings,1))
    pmean_embedding.append(np.max(wordembeddings,axis= 0))
    pmean_embedding.append(np.min(wordembeddings, axis= 0))
    # pmean_embedding.append(gen_mean(wordembeddings,0.5))
    pmean_embedding.append(gen_mean(wordembeddings,2))
    pmean_embedding.append(gen_mean(wordembeddings,4))

    # pmean_embedding.append(gen_mean(wordembeddings,4))
    # pmean_embedding.append(gen_mean(wordembeddings,5))

    return np.concatenate(pmean_embedding, axis=0)


from PIL import Image
import numpy as np
import math
from scipy import signal

#returns true if an integer is odd
def isOdd(n: int) -> bool:
    return n % 2 == 1

#returns a boxfilter of size n*n
def boxfilter(n: int) -> np.ndarray:
    assert isOdd(n)
    valueToFill = 1 / (n * n)
    return np.full((n, n), valueToFill)

def nextOdd(n: float) -> int:
    return int(np.ceil(n) // 2 * 2 + 1)

# returns a 1D Gaussian filter for a given value of sigma
def gauss1d(sigma: float) -> np.ndarray:
    length = nextOdd(6 * sigma)
    distFromCenterList: list = [x for x in range(-(length // 2), (length // 2) + 1)]
    unNormalized = [math.exp(- (x ** 2) / (2 * (sigma ** 2))) for x in distFromCenterList]
    listSum = sum(unNormalized)
    return np.asarray([x / listSum for x in unNormalized])

# returns a 2D Gaussian filter for a given value of sigma
def gauss2d(sigma: float) -> np.ndarray:
    f = gauss1d(sigma)
    f = f[np.newaxis]  # make 2d
    fT = f.transpose()
    return np.asarray(signal.convolve(f, fT))

# part 1-4a
# applies Gaussian convolution to a 2D array for the given value of sigma
def gaussconvolve2d(array: np.ndarray, sigma: float) -> np.ndarray:
    filter = gauss2d(sigma)
    return signal.convolve2d(array, filter, 'same')

# part 1-4b
#shows the dog picture as original, and as blurred using gaussian filter
def demo() -> None:
    im = Image.open('dog.jpg')
    im = im.convert('L')  # make grayscale
    im.show()  # display original
    im_array = np.asarray(im)

    im2_array = gaussconvolve2d(im_array, 3)
    im2 = Image.fromarray(im2_array)
    im2.show() #display blurred dog
    return

# part 2-1
# returns a image that is either a high or low pass of a source image
# if hipass==false, then returns a low pass version
def hilowpassrgb(filename: str, sigma: float, hipass=True) -> np.ndarray:
    im = Image.open(filename)
    im = im.convert('RGB')
    # im.show()  # display
    im_array = np.asarray(im)

    im2_array_r = gaussconvolve2d(im_array[:, :, 0], sigma)
    im2_array_g = gaussconvolve2d(im_array[:, :, 1], sigma)
    im2_array_b = gaussconvolve2d(im_array[:, :, 2], sigma)

    im2_array = np.dstack((im2_array_r, im2_array_g, im2_array_b))
    # print('Before:', filename, ' ', np.mean(im2_array))

    if hipass:
        im2_array = np.subtract(im_array, im2_array)
        # im2_array[:,:] += 128
        # print(im2_array)

    # print('After:', filename, ' ', np.mean(im2_array))

    # im2 = Image.fromarray(np.uint8(im2_array))
    # im2.show()
    return im2_array

#shows the hybridized images for part 2-3
def hybrid_demo() -> None:

    #catdog
    hi = hilowpassrgb('images/0a_cat.bmp', 4, hipass=True)
    lo = hilowpassrgb('images/0b_dog.bmp', 8, hipass=False)
    im = Image.fromarray(np.uint8(np.clip(np.add(hi, lo), 0, 255)))
    im.show()

    #marilynstein
    hi = hilowpassrgb('images/2a_einstein.bmp', 5, hipass=True)
    lo = hilowpassrgb('images/2b_marilyn.bmp', 2, hipass=False)
    im = Image.fromarray(np.uint8(np.clip(np.add(hi, lo), 0, 255)))
    im.show()

    #motobike
    hi = hilowpassrgb('images/1b_motorcycle.bmp', 3, hipass=True)
    lo = hilowpassrgb('images/1a_bicycle.bmp', 9, hipass=False)
    im = Image.fromarray(np.uint8(np.clip(np.add(hi, lo), 0, 255)))
    im.show()