import numpy as np

def f(x):
    return (np.sin(np.sqrt(100 * x))) ** 2

def adaptive_simpson(a, b, e):
    n = 2
    h = []
    er = []
    S = []
    T = []
    i = []
    h.append((b - a) / n)
    s = (f(a) + f(b)) * (1 / 3)
    t = (2 / 3) * f(a + h[0])
    S.append(s)
    T.append(t)
    i.append(h[0] * (S[0] + (2 * T[0])))
    er.append("   ------------")
    for m in range(1, 1000, 1):
        sum2 = 0
        n = 2 * n
        h.append((b - a) / n)
        S.append(S[m - 1] + T[m - 1])
        for k in range(1, n, 2):
            sum2 = sum2 + f(a + (k * h[m]))
        T.append((2 / 3) * sum2)
        i.append(h[m] * (S[m] + (2 * T[m])))
        er.append((1 / 15) * abs(i[m] - i[m - 1]))
        if er[m] <= e:
            break
    return i, er

i1, error = adaptive_simpson(0, 1, 1e-6)
p = 1
print("n    I                           error")
for j in range(0, len(i1)):
    p = 2 * p
    print(p, " ", i1[j], "  ", error[j])

Ankit Khandelwal_HW2/exercise 3/exercise 3c.py


'''
plot the beta distribution of posterior probability
'''

import matplotlib.pyplot as plt
import numpy as np
import scipy.stats as st

def plot_beta(a, b):
    x = np.linspace(0, 1, 10000)
    y = st.beta.pdf(x, a, b)
    plt.plot(x, y, 'r-', lw=5, alpha=0.6, label='beta pdf')
    plt.show()
    return x, y

def interval_estimate(prob, a, b):
    print('The {} interval estimate is {}'.format(prob, st.beta.interval(prob, a, b)))

def beta_low(prob, a, b):
    print('The probability that the proportion of monitoring sites with detectable algae levels'
          ' π is smaller than {} is {}.'.format(prob, round(st.beta.cdf(prob, a, b), 3)))

def get_mean(a, b):
    print(st.beta.mean(a, b))

if __name__ == '__main__':
    # plot_beta(2, 10)
    interval_estimate(0.95, 74, 380)
    # beta_low(0.19999, 46, 239)
    # print(st.beta.mean(46, 239))
    get_mean(74, 380)


from __future__ import division
import numpy as np
import scipy as sp
from scipy.io import loadmat
from dml.tool import pca,featurenormal,projectData,recoverData,displayData
import matplotlib.pyplot as plt
data = loadmat("../data/face/ex7data1.mat")
X = data['X'].transpose()
plt.axis([0,8,0,8])

'''
	simple test for PCA
'''

X_norm,mu,sigma=featurenormal(X)
plt.plot(X[0,:],X[1,:],'o')
#print mu,sigma,'======='
U,S = pca(X_norm)
#print U,S
tt=mu+U[:,0]
tp=mu+U[:,1]

plt.plot([mu[0],tt[0]] ,[mu[1],tt[1]])
plt.plot([mu[0],tp[0]],[mu[1],tp[1]])
plt.show()
plt.axis([-3,3,-3,3])

plt.plot(X_norm[0,:],X_norm[1,:],'o')

Z=projectData(X_norm, U, 1);
X_rec=recoverData(Z, U, 1);
plt.plot(X_rec[0,:],X_rec[1,:],'o')
for i in range(X.shape[1]):
	plt.plot([X_norm[0,i],X_rec[0,i]],[X_norm[1,i],X_rec[1,i]],'r')
plt.show()

'''
   face images dimension reduction
'''
data = loadmat("../data/face/ex7faces.mat")
X = data['X'].transpose()

X_norm,mu,sigma=featurenormal(X)
fig = plt.figure()
fig.add_subplot(1,2, 1)
plt.imshow(displayData(X_norm[:,:100]) , cmap='gray')

#PCA STEPs
[U, S] = pca(X_norm);
print S
K = 100;
Z = projectData(X_norm, U, K);
X_rec  = recoverData(Z, U, K);

fig.add_subplot(1,2, 2)
plt.imshow(displayData(X_rec[:,:100]) , cmap='gray')
plt.show()


#%%
"""
Created on August 25 2021
The Heston Hull-White model used for pricing of European type of FX options using
the COS method and comparisons with the Monte Carlo simulation.

This code is purely educational and comes from "Financial Engineering" course by L.A. Grzelak
The course is based on the book “Mathematical Modeling and Computation
in Finance: With Exercises and Python and MATLAB Computer Codes”,
by C.W. Oosterlee and L.A. Grzelak, World Scientific Publishing Europe Ltd, 2019.
@author: Lech A. Grzelak
"""
import numpy as np
import matplotlib.pyplot as plt
import scipy.stats as st
import scipy.special as sp
import scipy.integrate as integrate
import scipy.optimize as optimize
import enum

# This class defines puts and calls
class OptionType(enum.Enum):
    CALL = 1.0
    PUT = -1.0

# Black-Scholes Call option price
def BS_Call_Put_Option_Price(CP,S_0,K,sigma,tau,r):
    if K is list:
        K = np.array(K).reshape([len(K),1])
    d1    = (np.log(S_0 / K) + (r + 0.5 * np.power(sigma,2.0))
    * tau) / (sigma * np.sqrt(tau))
    d2    = d1 - sigma * np.sqrt(tau)
    if CP == OptionType.CALL:
        value = st.norm.cdf(d1) * S_0 - st.norm.cdf(d2) * K * np.exp(-r * tau)
    elif CP == OptionType.PUT:
        value = st.norm.cdf(-d2) * K * np.exp(-r * tau) - st.norm.cdf(-d1)*S_0
    return value

# Implied volatility method
def ImpliedVolatilityBlack76(CP,marketPrice,K,T,S_0):
    # To determine initial volatility we interpolate define a grid for sigma
    # and interpolate on the inverse
    sigmaGrid = np.linspace(0.0,5.0,5000)
    optPriceGrid = BS_Call_Put_Option_Price(CP,S_0,K,sigmaGrid,T,0.0)
    sigmaInitial = np.interp(marketPrice,optPriceGrid,sigmaGrid)
    print("Strike = {0}".format(K))
    print("Initial volatility = {0}".format(sigmaInitial))

    # Use determined input for the local-search (final tuning)
    func = lambda sigma: np.power(BS_Call_Put_Option_Price(CP,S_0,K,sigma,T,0.0) - marketPrice, 1.0)
    impliedVol = optimize.newton(func, sigmaInitial, tol=1e-15)
    print("Final volatility = {0}".format(impliedVol))
    if impliedVol > 2.0:
        impliedVol = 0.0
    return impliedVol

def CallPutOptionPriceCOSMthd_StochIR(cf,CP,S0,tau,K,N,L,P0T):
    # cf   - characteristic function as a functon, in the book denoted as \varphi
    # CP   - C for call and P for put
    # S0   - Initial stock price
    # tau  - time to maturity
    # K    - list of strikes
    # N    - Number of expansion terms
    # L    - size of truncation domain (typ.:L=8 or L=10)
    # P0T  - zero-coupon bond for maturity T.

    # reshape K to a column vector
    if K is not np.array:
        K = np.array(K).reshape([len(K),1])

    #assigning i=sqrt(-1)
    i = np.complex(0.0,1.0)
    x0 = np.log(S0 / K)

    # truncation domain
    a = 0.0 - L * np.sqrt(tau)
    b = 0.0 + L * np.sqrt(tau)

    # sumation from k = 0 to k=N-1
    k = np.linspace(0,N-1,N).reshape([N,1])
    u = k * np.pi / (b - a)

    # Determine coefficients for Put Prices
    H_k = CallPutCoefficients(OptionType.PUT,a,b,k)
    mat = np.exp(i * np.outer((x0 - a) , u))
    temp = cf(u) * H_k
    temp[0] = 0.5 * temp[0]
    value = K * np.real(mat.dot(temp))

    # we use call-put parity for call options
    if CP == OptionType.CALL:
        value = value + S0 - K * P0T

    return value

# Determine coefficients for Put Prices
def CallPutCoefficients(CP,a,b,k):
    if CP==OptionType.CALL:
        c = 0.0
        d = b
        coef = Chi_Psi(a,b,c,d,k)
        Chi_k = coef["chi"]
        Psi_k = coef["psi"]
        if a < b and b < 0.0:
            H_k = np.zeros([len(k),1])
        else:
            H_k      = 2.0 / (b - a) * (Chi_k - Psi_k)
    elif CP==OptionType.PUT:
        c = a
        d = 0.0
        coef = Chi_Psi(a,b,c,d,k)
        Chi_k = coef["chi"]
        Psi_k = coef["psi"]
        H_k      = 2.0 / (b - a) * (- Chi_k + Psi_k)

    return H_k

def Chi_Psi(a,b,c,d,k):
    psi = np.sin(k * np.pi * (d - a) / (b - a)) - np.sin(k * np.pi * (c - a)/(b - a))
    psi[1:] = psi[1:] * (b - a) / (k[1:] * np.pi)
    psi[0] = d - c

    chi = 1.0 / (1.0 + np.power((k * np.pi / (b - a)) , 2.0))
    expr1 = np.cos(k * np.pi * (d - a)/(b - a)) * np.exp(d)  - np.cos(k * np.pi
                  * (c - a) / (b - a)) * np.exp(c)
    expr2 = k * np.pi / (b - a) * np.sin(k * np.pi *
                        (d - a) / (b - a))   - k * np.pi / (b - a) * np.sin(k
                        * np.pi * (c - a) / (b - a)) * np.exp(c)
    chi = chi * (expr1 + expr2)

    value = {"chi":chi,"psi":psi }
    return value

def EUOptionPriceFromMCPathsGeneralizedFXFrwd(CP,S,K):
    # S is a vector of Monte Carlo samples at T
    result = np.zeros([len(K),1])
    if CP == OptionType.CALL:
        for (idx,k) in enumerate(K):
            result[idx] = np.mean(np.maximum(S-k,0.0))
    elif CP == OptionType.PUT:
        for (idx,k) in enumerate(K):
            result[idx] = np.mean(np.maximum(k-S,0.0))
    return result

def GeneratePathsHHWFXHWEuler(NoOfPaths,NoOfSteps,T,frwdFX,v0,vbar,kappa,gamma,lambdd,lambdf,etad,etaf,rhoxv,rhoxrd,rhoxrf,rhovrd,rhovrf,rhordrf):
    Wx = np.zeros([NoOfPaths, NoOfSteps+1])
    Wv = np.zeros([NoOfPaths, NoOfSteps+1])
    Wrd = np.zeros([NoOfPaths, NoOfSteps+1])
    Wrf = np.zeros([NoOfPaths, NoOfSteps+1])

    V = np.zeros([NoOfPaths, NoOfSteps+1])
    FX = np.zeros([NoOfPaths, NoOfSteps+1])
    V[:,0] = v0
    FX[:,0] = frwdFX

    dt = T / float(NoOfSteps)
    Bd = lambda t,T: 1.0/lambdd*(np.exp(-lambdd*(T-t))-1.0)
    Bf = lambda t,T: 1.0/lambdf*(np.exp(-lambdf*(T-t))-1.0)

    cov = np.array([[1.0, rhoxv,rhoxrd,rhoxrf],[rhoxv,1.0,rhovrd,rhovrf],\
                        [rhoxrd,rhovrd,1.0,rhordrf],[rhoxrf,rhovrf,rhordrf,1.0]])

    time = np.zeros([NoOfSteps+1])

    for i in range(0,NoOfSteps):
        Z = np.random.multivariate_normal([.0,.0,.0,.0],cov,NoOfPaths)
        if NoOfPaths > 1:
            Z[:,0] = (Z[:,0] - np.mean(Z[:,0])) / np.std(Z[:,0])
            Z[:,1] = (Z[:,1] - np.mean(Z[:,1])) / np.std(Z[:,1])
            Z[:,2] = (Z[:,2] - np.mean(Z[:,2])) / np.std(Z[:,2])
            Z[:,3] = (Z[:,3] - np.mean(Z[:,3])) / np.std(Z[:,3])

        Wx[:,i+1] = Wx[:,i] + np.power(dt, 0.5)*Z[:,0]
        Wv[:,i+1] = Wv[:,i] + np.power(dt, 0.5)*Z[:,1]
        Wrd[:,i+1] = Wrd[:,i] + np.power(dt, 0.5)*Z[:,2]
        Wrf[:,i+1] = Wrf[:,i] + np.power(dt, 0.5)*Z[:,3]

        # Variance process- Euler discretization
        V[:,i+1] = V[:,i] + kappa*(vbar - V[:,i])*dt \
                          + gamma*rhovrd*etad*Bd(time[i],T)*np.sqrt(V[:,i]) * dt \
                          + gamma* np.sqrt(V[:,i]) * (Wv[:,i+1]-Wv[:,i])
        V[:,i+1] = np.maximum(V[:,i+1],0.0)

        # FX process under the forward measure
        FX[:,i+1] = FX[:,i] *(1.0 + np.sqrt(V[:,i])*(Wx[:,i+1]-Wx[:,i]) \
                   -etad*Bd(time[i],T)*(Wrd[:,i+1]-Wrd[:,i])\
                   +etaf*Bf(time[i],T)*(Wrf[:,i+1]-Wrf[:,i]))
        time[i+1] = time[i] +dt

    paths = {"time":time,"FX":FX}
    return paths

# Exact expectation E(sqrt(V(t)))
def meanSqrtV_3(kappa,v0,vbar,gamma):
    delta = 4.0 *kappa*vbar/gamma/gamma
    c= lambda t: 1.0/(4.0*kappa)*gamma*gamma*(1.0-np.exp(-kappa*(t)))
    kappaBar = lambda t: 4.0*kappa*v0*np.exp(-kappa*t)/(gamma*gamma*(1.0-np.exp(-kappa*t)))
    temp1 = lambda t: np.sqrt(2.0*c(t))* sp.gamma((1.0+delta)/2.0)/sp.gamma(delta/2.0)*sp.hyp1f1(-0.5,delta/2.0,-kappaBar(t)/2.0)
    return temp1

def C_H1HW_FX(u,tau,kappa,gamma,rhoxv):
    i = np.complex(0.0,1.0)

    D1 = np.sqrt(np.power(kappa-gamma*rhoxv*i*u,2.0)+(u*u+i*u)*gamma*gamma)
    g  = (kappa-gamma*rhoxv*i*u-D1)/(kappa-gamma*rhoxv*i*u+D1)
    C  = (1.0-np.exp(-D1*tau))/(gamma*gamma*(1.0-g*np.exp(-D1*tau)))\
        *(kappa-gamma*rhoxv*i*u-D1)
    return C

def ChFH1HW_FX(u,tau,gamma,Rxv,Rxrd,Rxrf,Rrdrf,Rvrd,Rvrf,lambdd,etad,lambdf,etaf,kappa,vBar,v0):
    i  = np.complex(0.0,1.0)
    C = lambda u,tau: C_H1HW_FX(u,tau,kappa,gamma,Rxv)
    Bd = lambda t,T: 1.0/lambdd*(np.exp(-lambdd*(T-t))-1.0)
    Bf = lambda t,T: 1.0/lambdf*(np.exp(-lambdf*(T-t))-1.0)
    G = meanSqrtV_3(kappa,v0,vBar,gamma)

    zeta = lambda t: (Rxrd*etad*Bd(t,tau) - Rxrf*etaf*Bf(t,tau))*G(t) + \
                    Rrdrf*etad*etaf*Bd(t,tau)*Bf(t,tau) - 0.5*(etad**2.0*Bd(t,tau)**2.0+etaf**2.0*Bf(t,tau)**2.0)

    # Integration in the function A(u,tau)
    int1=np.zeros([len(u),1],dtype=np.complex_)
    N = 500
    z=np.linspace(0.0+1e-10,tau-1e-10,N)

    temp1 =lambda z1: kappa*vBar + Rvrd*gamma*etad*G(tau-z1)*Bd(tau-z1,tau)
    temp2 =lambda z1, u: -Rvrd*gamma*etad*G(tau-z1)*Bd(tau-z1,tau)*i*u
    temp3 =lambda z1, u:  Rvrf*gamma*etaf*G(tau-z1)*Bf(tau-z1,tau)*i*u
    f = lambda z1,u: (temp1(z1)+temp2(z1,u)+temp3(z1,u))*C(u,z1)

    value1 =integrate.trapz(np.real(f(z,u)),z).reshape(u.size,1)
    value2 =integrate.trapz(np.imag(f(z,u)),z).reshape(u.size,1)
    int1=(value1 + value2*i)

    """
    for k in range(0,len(u)):
        temp1 = kappa*vBar + Rvrd*gamma*etad*G(tau-z)*Bd(tau-z,tau)
        temp2 = -Rvrd*gamma*etad*G(tau-z)*Bd(tau-z,tau)*i*u[k]
        temp3 = Rvrf*gamma*etaf*G(tau-z)*Bf(tau-z,tau)*i*u[k]
        f = (temp1+temp2+temp3)*C(u[k],z)
        value1 =integrate.trapz(np.real(f),z)
        value2 =integrate.trapz(np.imag(f),z)
        int1[k]=(value1 + value2*i)
    """
    int2 = (u**2.0 + i*u)*integrate.trapz(zeta(tau-z),z)
    A = int1 + int2

    cf = np.exp(A + v0*C(u,tau))
    return cf

def GenerateStrikes(frwd,Ti):
    c_n = np.array([-1.5, -1.0, -0.5,0.0, 0.5, 1.0, 1.5])
    return frwd * np.exp(0.1 * c_n * np.sqrt(Ti))

def mainCalculation():
    CP  = OptionType.CALL
    T       = 5.0

    NoOfPaths = 1000
    NoOfSteps = (int)(T*50)

    # Settings for the COS method
    N = 500
    L = 8

    # Market Settings
    P0Td = lambda t: np.exp(-0.02*t)
    P0Tf = lambda t: np.exp(-0.05*t)
    y0      = 1.35
    frwdFX  = y0*P0Tf(T)/P0Td(T)
    kappa   = 0.5
    gamma   = 0.3
    vbar    = 0.1
    v0      = 0.1

    # HW model settings
    lambdd  = 0.01
    lambdf  = 0.05
    etad    = 0.007
    etaf    = 0.012

    # correlations
    Rxv   = -0.4
    Rxrd  = -0.15
    Rxrf  = -0.15
    Rvrd  = 0.3
    Rvrf  = 0.3
    Rrdrf = 0.25

    # Strikes
    K = GenerateStrikes(frwdFX,T)
    K = np.array(K).reshape([len(K),1])

    # number of repeated simulations for different Monte Carlo seed
    SeedV = range(0,20)
    optMCM = np.zeros([len(SeedV),len(K)])
    for (idx,seed) in enumerate(SeedV):
        print('Seed number = {0} out of {1}'.format(idx,len(SeedV)))
        np.random.seed(seed)
        paths = GeneratePathsHHWFXHWEuler(NoOfPaths,NoOfSteps,T,frwdFX,v0,vbar,kappa,gamma,lambdd,lambdf,etad,etaf,Rxv,Rxrd,Rxrf,Rvrd,Rvrf,Rrdrf)
        frwdfxT = paths["FX"]
        optMC = P0Td(T)* EUOptionPriceFromMCPathsGeneralizedFXFrwd(CP,frwdfxT[:,-1],K)
        optMCM[idx,:]= np.squeeze(optMC)

    # Average of the runs + standard deviation
    optionMC_E = np.zeros([len(K)])
    optionMC_StDev = np.zeros([len(K)])
    for (idx,k) in enumerate(K):
        optionMC_E[idx] = np.mean(optMCM[:,idx])
        optionMC_StDev[idx]  = np.std(optMCM[:,idx])

    # Value from the COS method
    cf = lambda u: ChFH1HW_FX(u,T,gamma,Rxv,Rxrd,Rxrf,Rrdrf,Rvrd,Rvrf,lambdd,etad,lambdf,etaf,kappa,vbar,v0)
    valCOS_H1HW = P0Td(T)*CallPutOptionPriceCOSMthd_StochIR(cf, CP, frwdFX, T, K, N, L,1.0)

    # checking martingale property
    EyT = P0Td(T)/P0Tf(T)*EUOptionPriceFromMCPathsGeneralizedFXFrwd(CP,frwdfxT[:,-1],[0.0])
    print("Martingale check: P_d(T)/P_f(T)*E[FX(T)] ={0:.4f} and y0 ={1}".format(EyT[0][0],y0))
    print("Maturity chosen to T={0}".format(T))
    for (idx,k) in enumerate(K):
        print("Option price for strike K={0:.4f} is equal to: COS method = {1:.4f} and MC = {2:.4f} with stdDev = {3:.4f}".format(k[0],valCOS_H1HW[idx][0],optionMC_E[idx],optionMC_StDev[idx]))

    plt.figure(1)
    plt.plot(K,optionMC_E)
    plt.plot(K,valCOS_H1HW,'--r')
    plt.grid()
    plt.legend(['Monte Carlo Option Price','COS method'])
    plt.title("Fx Option prices")

    # implied volatilities
    IVCos =np.zeros([len(K),1])
    IVMC =np.zeros([len(K),1])
    for (idx,k) in enumerate(K):
        priceCOS = valCOS_H1HW[idx]/P0Td(T)
        IVCos[idx] = ImpliedVolatilityBlack76(CP,priceCOS ,k,T,frwdFX)*100.0
        priceMC = optionMC_E[idx]/P0Td(T)
        IVMC[idx] = ImpliedVolatilityBlack76(CP,priceMC ,k,T,frwdFX)*100.0

    plt.figure(2)
    plt.plot(K,IVMC)
    plt.plot(K,IVCos,'--r')

    plt.grid()
    plt.legend(['IV-COS','IV-MC'])
    plt.title("Fx Implied volatilities")

mainCalculation()

Lecture 10-Foreign Exchange (FX) and Inflation/Materials/H1_HW_COS_vs_MC_FX.py


import numpy as np
from collections import Counter
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import scipy.stats as stats
from pandas.api.types import is_object_dtype, is_numeric_dtype
from scipy import stats

def grubbs_remove_outlier(X, alpha):
    '''
    This function runs Grubbs’ Statistical Test to detect and remove the outlier based on significance level
    and grubbs value & critical value.

    Parameters
    ----------
    X : ndarray
        A python list to be tested for outliers.

    alpha : float
        The significance level.

    Returns
    -------
    X : ndarray
        The original array with one outlier removed.
    '''

    # Calculte Grubbs Statistics Value
    std_dev = np.std(X)
    avg_X = np.mean(X)
    abs_val_minus_avg = abs(X - avg_X)
    max_of_deviations = max(abs_val_minus_avg)
    grubbs_value = max_of_deviations / std_dev

    # Find Max Index
    max_index = np.argmax(abs_val_minus_avg)
    outlier = X[max_index]

    # Calculte Grubbs Critical Value
    size = len(X)
    t_dist = stats.t.ppf(1 - alpha / (2 * size), size - 2)
    numerator = (size - 1) * np.sqrt(np.square(t_dist))
    denominator = np.sqrt(size) * np.sqrt(size - 2 + np.square(t_dist))
    grubbs_critical_value = numerator / denominator

    if grubbs_value > grubbs_critical_value:
        print('{} is an outlier. Grubbs value  > Grubbs critical: {:.4f} > {:.4f} \n'.format(
            outlier, grubbs_value, grubbs_critical_value))
        #X = np.delete(X, np.where(X == outlier) )
        X.remove(outlier)

    return X

def grubbs(X, max_outliers,  alpha=0.05, name=''):
    '''
    Grubbs’ Test is also known as the maximum normalized residual test or extreme studentized deviate test is a test used to detect outliers in a univariate data set assumed to come from a normally distributed population. This test is defined for the hypothesis:

    Ho: There are no outliers in the dataset
    Ha: There is exactly one outlier in the dataset

    Parameters
    ----------
    X : ndarray
        A python list to be tested for outliers.

    alpha : float
        The significance level.

    max_outliers: int
    The maximum  number of iteration that will be deleted by this method

    Returns
    -------
    X : ndarray
        The original array with outliers removed.

    outliers: the list of all outliers
    '''

    init_x_len = len(X)
    reach_end = False

    clean_X = X.copy().tolist()

    for i in range(max_outliers):
        clean_X = grubbs_remove_outlier(clean_X, alpha)
        if len(X) == len(clean_X):
            reach_end = True
            break

    if name != "":
        name = " in " + name + " dataset"
    else:
        name = " in the dataset "

    outlisers = []

    if (len(X)-len(clean_X) == 0):
        print("we can not reject the null hypothesis and there are no outliers " + name)

    else:
        print("we reject the null hypothesis and there are outliers "+name)
        outlisers = [value for value in X if value not in clean_X]

        if reach_end:
            print("Found all Grubbs outliers, " +
                  str(len(X)-len(clean_X)) + " outliers")
        else:
            print("Found some Grubbs outliers, " + str(len(X)-len(clean_X)
                                                       ) + " outliers. increase the max itration to find more")

    return clean_X, outlisers

def detect_outliers(X, factor=1.5):

    # list to store outlier indices
    outlier_indices = []

    # Get the 1st quartile (25%)
    Q1 = np.percentile(X, 25)

    # Get the 3rd quartile (75%)
    Q3 = np.percentile(X, 75)

    # Get the Interquartile range (IQR)
    IQR = Q3 - Q1

    # Define our outlier step
    outlier_step = 1.5 * IQR

    return X[(X < Q1 - outlier_step) | (X > Q3 + outlier_step)]

def missing_values_table(df):
    """
    Returns a pandas DataFrame containing the counts of missing values by column in the input DataFrame

    """
    mis_val = df.isnull().sum()
    mis_val_percent = 100 * df.isnull().sum() / len(df)
    mis_val_table = pd.concat([mis_val, mis_val_percent], axis=1)
    mis_val_table_ren_columns = mis_val_table.rename(
        columns={0: 'Missing Values', 1: '% of Total Values'})
    mis_val_table_ren_columns = mis_val_table_ren_columns[
        mis_val_table_ren_columns.iloc[:, 1] != -1].sort_values(
        '% of Total Values', ascending=False).round(1)
    # print("Your selected dataframe has " + str(df.shape[1]) + " column/s and " + str(df.shape[0]) + " row/s.\n"
    #       "There are " + str(mis_val_table_ren_columns.shape[0]) +
        #   " columns that have missing values.")
    return mis_val_table_ren_columns

#logic preparation
def logic_preparation(x_df, no_treatment_columns = [] ):
    """
    This function provides a recommendation on how to deal with each feauture's missing values based on the missing values' percentage.

    """
    missing = missing_values_table(x_df).iloc[:,1]
    df_missing = pd.DataFrame(columns=['name', 'missing','treatment', 'comment' ])

    df_missing["missing"] = missing.tolist()
    df_missing["name"] = missing.index.values.tolist()
    df_missing.treatment = "Treat"
    df_missing.comment = "Treat missing values based on statistical distribution (mean or median) or mode"
    df_missing.treatment.loc[df_missing["missing"] >= 70] = "Drop"
    df_missing.treatment.loc[df_missing["missing"] < 5] = "Omit"
    df_missing.treatment.loc[df_missing["missing"] == 0] = "Do Nothing"
    df_missing.comment.loc[df_missing["missing"] == 0] = "Feature has no missing values"
    df_missing.comment.loc[df_missing["missing"] >= 70] = "Drop the feature as it is missing more than 70 pecent of the values"
    df_missing.comment.loc[df_missing["missing"] < 5] = "Omit observations with this feature missing (less than 5 percent of total records)"
    df_missing.comment.loc[df_missing["missing"] == 0] = "Feature has no missing values"

    columns_to_drop = []
    columns_to_drop = df_missing.name.loc[df_missing["treatment"] == "Drop"].tolist()
    columns_to_drop = list(dict.fromkeys(columns_to_drop))

    for i in range(len(df_missing.index)):
        if (df_missing.iloc[i]["name"] in no_treatment_columns and
                df_missing.iloc[i]["treatment"] == "Treat"):
            df_missing.treatment.iloc[i] = "Omit"

    columns_to_omit_missing_values = []
    columns_to_omit_missing_values = df_missing.name.loc[df_missing["treatment"] == "Omit"].tolist()
    columns_to_omit_missing_values = list(dict.fromkeys(columns_to_omit_missing_values))

    return df_missing

def eval_df(df_o,
            normal_test_alpha = 0.05,
            grubbs_alpha = 0.05,
            grubbs_max_outliers = 20,
            iqr_factor = 2.5 ):
    df = df_o.copy()

    output = {}

    output['name']          = []
    output['count']         = []
    output['missing']       = []
    output['missing_prc']   = []
    output['types']         = []
    output['q1']            = []
    output['q2']            = []
    output['q3']            = []
    output['is_normal']     = []
    output['unique']        = []
    output['top']           = []
    output['freq']          = []
    output['min']           = []
    output['max']           = []
    output['mean']          = []
    output['sd']            = []
    output['variance']      = []
    output['skewness']      = []
    output['kurtosis']      = []
    output['iqrs']          = []
    output['has_outliers']  = []
    outliers   = {}

    def get_stat(name):
        """

        This function outputs feature statistics such as IQR, min, max, mean, variance, skewness, kurtosis, etc..

        """
        if(name in output['name']):
            return

        mis_val         = df.isnull().sum()
        mis_val_percent = 100 * df.isnull().sum() / len(df)

        df_describe = df.describe(include='all', datetime_is_numeric=True)

        output['name'].append(name)
        output['count'].append(df_describe[name]['count'])
        output['missing'].append( mis_val[name])
        output['missing_prc'].append(mis_val_percent[name])

        if is_numeric_dtype(df[name]):
            output['types'].append('Numerical')

            num_item_describe  =stats.describe(df[name])
            output['q1'].append(df_describe[name]['25%'])
            output['q3'].append(df_describe[name]['75%'])

            iqr = df_describe[name]['75%'] - df_describe[name]['25%']

            _, p = stats.normaltest(df[name])

            is_normal = True
            if (p < normal_test_alpha):
                is_normal = False
                output['is_normal'].append("Non-normal")
            else:
                is_normal = True
                output['is_normal'].append("Normal")

            output['unique'].append(np.nan)
            output['top'].append(np.nan)
            output['freq'].append(np.nan)
            output['min'].append(df_describe[name]['min'])
            output['max'].append(df_describe[name]['max'])
            output['mean'].append(df_describe[name]['mean'])
            output['sd'].append(df_describe[name]['std'])
            output['q2'].append(df_describe[name]['50%'])
            output['variance'].append(num_item_describe.variance)
            output['skewness'].append(num_item_describe.skewness)
            output['kurtosis'].append(num_item_describe.kurtosis)
            output['iqrs'].append(iqr)
            #to skip outlier detection in binary vars or very small range features
            if(df_describe[name]['max'] -  df_describe[name]['min'] <= 1):
                pass

            elif (is_normal):
                _ , grubbs_outliers = grubbs( df[name].dropna() , grubbs_max_outliers, grubbs_alpha,name)

                if len(grubbs_outliers) > 0:
                    outliers[name]  = grubbs_outliers
            else:
                iqr_outliers =  detect_outliers(df[name],iqr_factor)

                if len(iqr_outliers) > 0:
                    outliers[name]  = iqr_outliers

            if name in outliers:
                output['has_outliers'].append('True')
            else:
                output['has_outliers'].append('False')

        else:
            output['types'].append('Categorical')
            output['unique'].append(df_describe[name]['unique'])
            output['top'].append(df_describe[name]['top'])
            output['freq'].append(df_describe[name]['freq'])
            output['min'].append(np.nan)
            output['max'].append(np.nan)
            output['mean'].append(np.nan)
            output['sd'].append(np.nan)
            output['variance'].append(np.nan)
            output['skewness'].append(np.nan)
            output['kurtosis'].append(np.nan)
            output['q1'].append(np.nan)
            output['q2'].append(np.nan)
            output['q3'].append(np.nan)
            output['iqrs'].append(np.nan)
            output['is_normal'].append(np.nan)
            output['has_outliers'].append(np.nan)

    vfunc = np.vectorize(get_stat)
    vfunc(df.columns)

    descriptive_statistics = pd.DataFrame({'Name'     : output['name'],
                                           'Type'     : output['types'] ,
                                           'Count'    : output['count'],
                                           'Unique'   : output['unique'],
                                           'Top'      : output['top'],
                                           'Freq'     : output['freq'] ,
                                           'Min'      : output['min'],
                                           'Max'      : output['max'],
                                           'Mean'     : output['mean'],
                                           'SD'       : output['sd'],
                                           'Variance' : output['variance'] ,
                                           'Skewness' : output['skewness'],
                                           'Kurtosis' : output['kurtosis'],
                                           '25%'      : output['q1'],
                                           '50%'      : output['q2'],
                                           '75%'      : output['q3'] ,
                                           'IQR'      : output['iqrs'] ,
                                           '# Missing Values' : output['missing'],
                                           '% Missing Values' : output['missing_prc'],
                                           'Is Normal': output['is_normal'],
                                           'Has Outliers':output['has_outliers']
                                           })

    return descriptive_statistics , logic_preparation(df) , outliers

def clean_data(df_train, df_test):
    """
    This function is used to handle outliers and missing data, for outliers, it will clean the outliers in the training and testing datasets
    based on the output of eval_df function. For missing values, the function is also used for imputation of missing values by Mean/Median/Mode.

    """

    df_clean = df_train.copy()
    descriptive_statistics, df_treatment_logic, outliers = eval_df(df_clean)

    df_test_clean = df_test.copy()
    descriptive_statistics_test, df_treatment_logic_test, outliers_test = eval_df(
        df_test_clean.copy())

    print("Number of rows before handling the outliers values in training " +
          str(df_clean.shape[0])+"\n")
    print("Number of rows before handling the outliers values in testing " +
          str(df_test_clean.shape[0])+"\n")

    # clean all outliers in training data set
    for item in df_clean.columns:
        if item in outliers:
            df_clean = df_clean.loc[~df_clean[item].isin(outliers[item])]

    # clean all outliers in test data set
    for item in df_test_clean.columns:
        if item in outliers_test:
            df_test_clean = df_test_clean.loc[~df_test_clean[item].isin(
                outliers_test[item])]

    print("Number of rows after handling outliers (before handling missing data) in training " +
          str(df_clean.shape[0])+"\n")
    print("Number of rows after handling outliers (before handling missing data) in testing " +
          str(df_test_clean.shape[0])+"\n")

    for column in df_clean.columns:
        if(df_treatment_logic.loc[df_treatment_logic.name == column, :].treatment.tolist()[0] == "Drop"):
            df_clean = df_clean.drop(column, axis=1)
            df_test_clean = df_test_clean.drop(column, axis=1)
        elif(df_treatment_logic.loc[df_treatment_logic.name == column, :].treatment.tolist()[0] == "Omit"):
            df_clean = df_clean.dropna(subset=[column])
        elif(df_treatment_logic_test.loc[df_treatment_logic_test.name == column, :].treatment.tolist()[0] == "Omit"):
            df_test_clean = df_test_clean.dropna(subset=[column])

    # Outliers Treatment

    # impute based if the data is normal or not
    for item in df_clean.columns:
        row = descriptive_statistics[descriptive_statistics['Name'] == item]
        is_norm = str(row['Is Normal'].to_string(index=False))

        if (is_norm == 'Normal'):
            df_clean[item].fillna((df_clean[item].mean()), inplace=True)
            df_test_clean[item].fillna((df_clean[item].mean()), inplace=True)

        elif (is_norm == 'Non-normal'):
            df_clean[item].fillna((df_clean[item].median()), inplace=True)
            df_test_clean[item].fillna((df_clean[item].median()), inplace=True)
        else:
            df_clean[item].fillna((df_clean[item].mode()[0]), inplace=True)
            df_test_clean[item].fillna((df_clean[item].mode()[0]), inplace=True)

    print("Number of rows after handling the missing values in training " +
          str(df_clean.shape[0])+"\n")
    print("Number of rows after handling the missing values in testing " +
          str(df_test_clean.shape[0])+"\n")

    return df_clean, df_test_clean


# ## The `numpy` library
#
# `numpy` is a Python library designed to manipulate multi-dimensional arrays.
# It is often used together with the `scipy` library, containing tools for
# linear algebra, integration, optimisation, signal analysis and more.
#
# We present here very simple Numpy functionalities. For more details, refer to
# the official documentation:
# https://docs.scipy.org/doc/numpy/reference/

# %%
import numpy as np

# ## Types of embedded arrays
#
# A Numpy array can be made of an iterable structure (table, tuple, list). Numpy
# becomes powerful because all elements in the array have the same type
# (called `dtype`)
# %%
# Types in a Python array
table = [2, 7.3, 4]
type(table), [(value, type(value)) for value in table]

# %%
# Types in a Numpy array
np_table = np.array(table)
(type(np_table), np_table.dtype, [(value, type(value)) for value in np_table])

# ## Performance
#
# Python is commonly blamed for being slow at runtime. This comes for many
# factors, in particular the flexibility of the language, the numerous checks
# made at runtime (Python cannot assume anything about your data) and dynamic
# typing. Now with Numpy, we know with the `dtype`, the types of all elements in
# the array: arithmetic operations can be processed and accelerated in C.
# %%
table = [i for i in range(1, 10000000)]
np_array = np.array(table)

# %%
%%timeit
# Note: the timeit operator is specific to IPython and Jupyter. It is not
# valid Python syntax but will be interpreted correctly in the console if
# put at the very beginning of a cell.
double = [x * 2 for x in table]

# %%
%%timeit
np_double = 2 * np_array

# ## Views on Numpy array
#
# It is possible to work on views of a previously built array. We use the word
# "view" because modifying data in the "view" also modifies data in the original
# array.
# %%
np_array = np.array([[i + 2 * j for i in range(5)] for j in range(4)])
np_array

# %%
sub = np_array[0:2, 2:4]
sub

# %%
sub = np_array[:3, 2:]
sub

# %%
# Modifying sub values also modifies the original array
sub[:] = 0
np_array

# ## Matrix operations
#
# %%
a = np.array([[4, 6, 7, 6]])
b = np.array([[i + j for i in range(4)] for j in range(4)])

print(f"{a}\n with dimensions {a.shape}")
print(f"{b}\n with dimensions {b.shape}")

# %%
# Dot product (or matrix product)
np.dot(a, b)

# %%
# Term by term addition, multiplication, etc.
2 * a

# %%
a + a

# %%
a * a

# %%
# Dimensions are broadcast when needed:
# (details in https://www.xoolive.org/python/numpy/)

a * b

# This is NOT a dot product!!

# %%
import numpy.linalg  # noqa: E402

a = np.array([[abs(i - j) for i in range(5)] for j in range(5)])
inv_a = numpy.linalg.inv(a)

a * inv_a

# %%
inv_a @ a  # @ operator (called `matmul`), equivalent to np.dot(inv_a, a)

# ## Further references
#
# - Numpy tutorial:
#   https://github.com/rougier/numpy-tutorial
# - From Python to Numpy:
#   https://www.labri.fr/perso/nrougier/from-python-to-numpy/
# - 100 Numpy exercices:
#   https://github.com/rougier/numpy-100/
# - The dedicated chapter in https://www.xoolive.org/python/numpy/


import numpy as np

class Logistic_Regression:
    def __init__(self):
        self.w = []

    # sigmoid function
    def sigmoid(self, z):
        return 1 / (1 + np.exp(-z))

    # logistic function
    def hx(self, w, X):
        weights_num = X.shape[1] + 1
        z_arrays = [1] * weights_num
        z_arrays[0] = w[0]
        for i in range(weights_num - 1):
            z_arrays[i + 1] = w[i + 1] * np.array(X[:, i])
        z = np.array(sum(z_arrays[j] for j in range(weights_num)))
        return self.sigmoid(z)

    # cost function - binary cross entropy
    def cost(self, w, X, Y):
        y_pred = self.hx(w, X)
        return -1 * sum(Y * np.log(y_pred) + (1 - Y) * np.log(1 - y_pred))

    # gradient descent
    def grad_descent(self, w, X, Y):
        y_pred = self.hx(w, X)
        weights_num = X.shape[1] + 1
        grad = [0] * weights_num
        grad[0] = -1 * sum(Y * (1 - y_pred) - (1 - Y) * y_pred)
        for i in range(weights_num - 1):
            grad[i + 1] = -1 * sum(Y * (1 - y_pred) * X[:, i] - (1 - Y) * y_pred * X[:, i])
        return grad

    def fit(self, X, Y, w, lr, max_iter):
        weights_num = X.shape[1] + 1
        iter = 0
        while iter < max_iter:
            w_prev = w.copy()
            grad = self.grad_descent(w_prev, X, Y)
            for index in range(weights_num):
                w[index] = w[index] - lr * grad[index]
            iter += 1
        self.w = w.copy()

    def predict(self, X):
        predictions = self.hx(self.w, X)
        return np.around(predictions).astype(int)


#! /usr/bin/env python
# -*- coding: utf-8 -*-
#
# Distributed under terms of the MIT license.
# Copyright 2021 Chao Pan.

import numpy as np

# short for norm
def norm(x, axis=None):
    return np.linalg.norm(x, axis=axis)

# distance in poincare disk
def poincare_dist(u, v, eps=1e-5):
    d = 1 + 2 * norm(u-v)**2 / ((1 - norm(u)**2) * (1 - norm(v)**2) + eps)
    return np.arccosh(d)

# compute symmetric poincare distance matrix
def poincare_distances(embedding):
    n = embedding.shape[0]
    dist_matrix = np.zeros((n, n))
    for i in range(n):
        for j in range(i+1, n):
            dist_matrix[i][j] = poincare_dist(embedding[i], embedding[j])
    return dist_matrix

# convert array from poincare disk to hyperboloid
def poincare_pts_to_hyperboloid(Y, eps=0, metric='minkowski'):
    X = Y.T
    d, n = np.shape(X)
    Z = np.zeros((d+1, n))
    if metric == "minkowski":
        for i in range(n):
            Z[0, i] = (1 + np.sum(X[:, i]**2)) / (1 - np.sum(X[:, i]**2) + eps)
            Z[1:, i] = 2 * X[:, i] / (1 - np.sum(X[:, i]**2) + eps)
    elif metric == "lorentz":
        for i in range(n):
            Z[d, i] = (1 + np.sum(X[:, i]**2)) / (1 - np.sum(X[:, i]**2) + eps)
            Z[0:d, i] = 2 * X[:, i] / (1 - np.sum(X[:, i]**2) + eps)
    else:
        raise NotImplementedError
    return Z.T

# define hyperboloid bilinear form
def hyperboloid_dot(u, v):
    return np.dot(u[:-1], v[:-1]) - u[-1]*v[-1]

# define alternate minkowski/hyperboloid bilinear form
def minkowski_dot(u, v):
    return u[0]*v[0] - np.dot(u[1:], v[1:])

# hyperboloid distance function
def hyperboloid_dist(u, v, eps=1e-6, metric='lorentz'):
    if metric == 'minkowski':
        dist = np.arccosh(-1*minkowski_dot(u, v))
    else:
        dist = np.arccosh(-1*hyperboloid_dot(u, v))
    if np.isnan(dist):
        # print('Hyperboloid dist returned nan value')
        return eps
    else:
        return dist

# compute symmetric hyperboloid distance matrix
def hyperboloid_distances(embedding):
    n = embedding.shape[0]
    dist_matrix = np.zeros((n, n))
    for i in range(n):
        for j in range(i+1, n):
            dist_matrix[i][j] = hyperboloid_dist(embedding[i], embedding[j])
    return dist_matrix

# project within disk
def proj(theta, eps=0.1):
    if norm(theta) >= 1:
        theta = theta/norm(theta) - eps
    return theta

# helper function to generate samples
def generate_data(n, radius=0.7, hyperboloid=False):
    theta = np.random.uniform(0, 2*np.pi, n)
    u = np.random.uniform(0, radius, n)
    r = np.sqrt(u)
    x = r * np.cos(theta)
    y = r * np.sin(theta)
    init_data = np.hstack((x.reshape(-1, 1), y.reshape(-1, 1)))
    if hyperboloid:
        return poincare_pts_to_hyperboloid(init_data)
    else:
        return init_data


import numpy as np
import random
import math

def ackley_function(transposed_decimal_pop_input):
    """Ackley Test Function"""

    y = []
    for individual in transposed_decimal_pop_input:

        ackley = 0
        sum_1 = 0
        sum_2 = 0
        for xi in individual:
            sum_1 = sum_1 + xi**2
            sum_2 = sum_2 + math.cos(2*math.pi*xi)
        term_1 = -20*math.exp(-0.2*(math.sqrt(1/len(individual)*sum_1)))
        term_2 = - math.exp((1/len(individual))*sum_2)
        ackley = term_1 + term_2 + 20 + math.exp(1)
        y.append(ackley)
    return y

def bohachevsky_function(transposed_decimal_pop_input):
    """Bohachevsky Test Function"""

    y = []
    for individual in transposed_decimal_pop_input:

        bohachevsky = 0
        term_1 = 0
        term_2 = 0

        for idx, xi in enumerate(individual):
            if idx == (len(individual)-1):
                break
            term_1 = (xi**2 + 2*((individual[idx + 1])**2)-0.3*math.cos(3*math.pi*xi))
            term_2 = (-0.4*math.cos(4*math.pi*individual[idx + 1]) + 0.7)
            bohachevsky = bohachevsky + (term_1 + term_2)

        y.append(bohachevsky)
    return y

def de_jong_step_a_function(transposed_decimal_pop_input):
    """De Jong Step - A Test Function"""

    y = []
    for individual in transposed_decimal_pop_input:

        de_jong_step = 0

        for xi in individual:
            de_jong_step = de_jong_step + abs(round(xi))  # (abs) was used to adjust the function for minimization
        y.append(de_jong_step)
    return y

def de_jong_step_b_function(transposed_decimal_pop_input):
    """De Jong Step - B Test Function"""

    y = []
    for individual in transposed_decimal_pop_input:

        de_jong_step_func_4 = 0
        term_1 = 0
        term_2 = random.gauss(0, 1)

        for idx, xi in enumerate(individual):
            term_1 = term_1 + idx*(xi**4)
        de_jong_step_func_4 = term_1 + term_2
        y.append(de_jong_step_func_4)
    return y

def griewangk_function(transposed_decimal_pop_input):
    """Griewangk Test Function"""

    y = []
    for individual in transposed_decimal_pop_input:

        griewangk = 0
        term_1 = 0
        term_2 = 1

        for idx, xi in enumerate(individual):
            term_1 = term_1 + (xi**2)/4000
            term_2 = term_2 * math.cos(xi/(math.sqrt(idx + 1)))
        griewangk = 1 + term_1 - term_2
        y.append(griewangk)
    return y

def rastrigin_function(transposed_decimal_pop_input):
    """Rastrigin Test Function"""

    y = []
    for individual in transposed_decimal_pop_input:
        rastrigin = 0
        term_1 = 10*len(individual)
        term_2 = 0

        for xi in individual:
            term_2 = term_2 + (xi**2)-10*(math.cos(2*xi*(math.pi)))
        rastrigin = term_1 + term_2
        y.append(rastrigin)
    return y

def rosenbrock_function(transposed_decimal_pop_input):
    """Rosenbrock Test Function"""

    y = []
    for individual in transposed_decimal_pop_input:
        rosenbrock = 0
        for idx, xi in enumerate(individual):
            if idx == (len(individual)-1):
                break
            rosenbrock = rosenbrock + (100*(((xi**2)-individual[idx+1])**2)+((1-xi)**2))
        y.append(rosenbrock)
    return y

def schaffer_function(transposed_decimal_pop_input):
    """Schaffer Test Function"""

    y = []
    for individual in transposed_decimal_pop_input:

        shaffer = 0
        parentheses = 0

        for xi in individual:
            parentheses = parentheses + xi**2
        term_1 = parentheses**0.25
        term_2 = (((math.sin(50*(parentheses**0.1)))**2)+1)
        schaffer = (term_1 * term_2)
        y.append(schaffer)
    return y

def schwefel_a_function(transposed_decimal_pop_input):
    """Schwefel - A Test Function"""

    y = []
    for individual in transposed_decimal_pop_input:
        schwefel = 0
        for xi in individual:
            schwefel = schwefel + ((xi + 0.5)**2)
        y.append(schwefel)
    return y

def schwefel_b_function(transposed_decimal_pop_input):
    """Schwefel - B Test Function"""

    y = []
    for individual in transposed_decimal_pop_input:

        schwefel = 0

        for idx, xi in enumerate(individual):
            schwefel = schwefel + (-xi*(math.sin(math.sqrt(abs(xi)))))
#        y.append(schwefel/len(individual)) # The division is not part of the function. However, it might be helpful for some. It enables the same result, independent of the number of inputs.
        y.append(schwefel)
    return y

def schwefels_pro_function(transposed_decimal_pop_input):
    """Schwefel's Pro Test Function"""

    y = []
    schwefels = 0
    for individual in transposed_decimal_pop_input:
        schwefels = 0
        term_2 = 0
        for idx, xi in enumerate(individual):
            j = 0
            term_1 = 0
            while j <= idx:
                term_1 = term_1 + individual[j]
                j = j + 1
            term_2 = term_1**2
            schwefels = schwefels + term_2
        y.append(schwefels)
    return y

def sphere_function(transposed_decimal_pop_input):
    """Sphere Test Function"""

    y = []
    for individual in transposed_decimal_pop_input:
        y.append(sum(np.square(individual)))
    return y

def untitled_function(transposed_decimal_pop_input):
    """Untitled Test Function"""

    y = []
    for individual in transposed_decimal_pop_input:
        untitled = 0
        parentheses = 0
        term_1 = 0
        term_2 = 1

        for xi in individual:
            parentheses = parentheses + xi**2
            term_2 = term_2*(math.cos(20*math.pi*xi))
        term_1 = parentheses/2
        untitled = term_1 - term_2 + 2
        y.append(untitled)
    return y


import numpy as np

# Generates a 2D bit array of dimensions `h` by `w`, with a
# disk of ones with radius `r`, centered at point (x, y).
# All other points in the array are zero valued.
def disk(r, x, y, h, w):
    ind = np.indices((h, w))
    return r ** 2.0 >= (ind[0] - y) ** 2.0 + (ind[1] - x) ** 2.0

with open("input.txt", "r") as f:
    h, w, r = map(int, f.readline().split())
    # Convert x to 1 and everything else (.) to 0.
    grid = np.array([map(lambda c: 1 if c == 'x' else 0, f.readline()[:-1]) for j in range(h)])

max = 0
for i in range(h):
    for j in range(w):
        sum = (grid * disk(r, j, i, h, w)).sum()
        if sum >= max:
            max = sum
            x, y = j, i

print(x, y)


import numpy as np
import pandas as pd
import pickle as pkl
import math
from random import randint
import matplotlib.pyplot as plt
import sys
import os
from PIL import Image

__set_file = "images.csv"

__training_percent = 75

__params = ["image_name"]

__output = ["Output"]

__neural_network_model = "neural_network_model.pickle"

__images_folder = "handwritten_digit_corpus"

__number_mapping = {
	0: [1,0,0,0,0,0,0,0,0,0],
	1: [0,1,0,0,0,0,0,0,0,0],
	2: [0,0,1,0,0,0,0,0,0,0],
	3: [0,0,0,1,0,0,0,0,0,0],
	4: [0,0,0,0,1,0,0,0,0,0],
	5: [0,0,0,0,0,1,0,0,0,0],
	6: [0,0,0,0,0,0,1,0,0,0],
	7: [0,0,0,0,0,0,0,1,0,0],
	8: [0,0,0,0,0,0,0,0,1,0],
	9: [0,0,0,0,0,0,0,0,0,1]
}

__neural_network = {
	"layer_1": {
		"number_of_neurons": 16,
		"number_of_inputs": 784
	},
	"layer_2":{
		"number_of_neurons": 12,
		"number_of_inputs": 16
	},
	"layer_3":{
		"number_of_neurons": 10,
		"number_of_inputs": 12
	},
	"layers_count": 3
}

def sigmoid(input_vec):
	return 1/(1 + np.exp(-input_vec))

def sigmoid_diff(input_vec):
	return sigmoid(input_vec) * sigmoid(1 - input_vec)

def summation(weights, input_vec, bias_vec):
	intermediate_vec = np.matmul(weights, input_vec) + bias_vec
	return intermediate_vec

def initialize_weights(number_of_inputs, neurons_count):
	return np.random.randn(neurons_count, number_of_inputs)

def initialize_bias(neurons_count):
	return np.random.randn(neurons_count, 1)

def get_last_index(length_of_df):
	return math.floor(length_of_df * (__training_percent/100))

def calculate_error(actual_output, desired_output, last_neuron_layer_count):
	diff_error = actual_output - desired_output
	squared_error = diff_error ** 2
	squared_sum_error = np.sum(squared_error)
	avg_squared_sum_error = squared_sum_error / last_neuron_layer_count
	return avg_squared_sum_error

def error_derivative(actual_output, desired_output, last_neuron_layer_count):
	return ((actual_output - desired_output)/last_neuron_layer_count)

def store_model(model_perceptron):
	with open(__neural_network_model, 'wb') as handle:
		pkl.dump(model_perceptron, handle, protocol=pkl.HIGHEST_PROTOCOL)

def load_model():
	with open(__neural_network_model, 'rb') as handle:
		model_perceptron = pkl.load(handle)
	return model_perceptron

def get_data_set(set_type="train"):
	dataframe = pd.read_csv(__set_file)

	dataframe["Output"] = dataframe[["label"]].apply(lambda x: np.array([__number_mapping.get(x.to_frame().T.reset_index().label[0])]).T, axis=1)
	# dataframe = dataframe.sample(frac=1).reset_index(drop=True)

	__last_idx = get_last_index(len(dataframe))

	if set_type == "train":
		chuncked_dataframe = dataframe[:__last_idx]
	elif set_type == "test":
		chuncked_dataframe = dataframe[__last_idx:]

	return chuncked_dataframe[__params].as_matrix(), chuncked_dataframe[__output].as_matrix()

def confusion_matrix(actual_output, desired_output):
	y_actu = pd.Series(actual_output, name='Actual')
	y_pred = pd.Series(desired_output, name='Predicted')
	df_confusion = pd.crosstab(y_actu, y_pred)

	return df_confusion

def get_output_value(output_vector):
	return (output_vector.index(max(output_vector)) + 1)

def get_input_image_as_vec(image_file_name):
	file_path = os.path.join(__images_folder, image_file_name)
	image_buffer = Image.open(file_path).convert('L')
	image_array = np.array(image_buffer)
	flattened_array = image_array.ravel()
	return flattened_array

def train_nn():
	input_params, desired_output = get_data_set("train")
	forward_neuron_layer_order = list(range(1, __neural_network.get("layers_count") + 1))
	backward_neuron_layer_order = forward_neurons_order[::-1]

	# INITIALIZATION OF WEIGHTS AND BIAS
	# IT IS REQUIRED BEFORE TRAINING OF NEURAL NETWORK
	for each_layer in forward_neuron_layer_order:
		layer_name = "_".join(["layer", each_layer])
		__neural_network[layer_name]["weights"] = initialize_weights(__neural_network.get(layer_name).get("number_of_inputs"),
									 								 __neural_network.get(layer_name).get("number_of_neurons"))
		__neural_network[layer_name]["bias"] = initialize_bias(__neural_network.get(layer_name).get("number_of_neurons"))

	# TODO:
	# STOCHASTIC GRADIENT DESCENT WOULD BE USED,
	# BECAUSE DATASET IS TOO LARGE FOR NORMAL TRAINING
	# THUS IT NEEDS TO BE LEARNT FIRST
	__learning_rate = 0.2

	last_layer = "_".join(["layer", model_neural_network.get("layers_count")])

	__n_times = 5
# 	__batch_size = 100
	__total_dataset_size = input_params.shape[0]
	__iteration_times = __n_times * math.ceil(__total_dataset_size)

	__start_idx = 0
	__end_idx = __total_dataset_size

	__errors = []

	for each_time in range(__iteration_times):
		# UNCOMMENT THIS OUT FOR BATCH SIZED GRADIENT DESCENT
		# for each_time in range(__batch_size):
		# FEEDFORWARD NETWORK
		random_index = randint(__start_idx, __end_idx)
		input_vec = np.array([get_input_image_as_vec(input_params[random_index][0])])
		desired_output_vec = np.array([desired_output[random_index][0]])

		# FEEDING FORWARD STARTS HERE
		# FOR EACH NEURON LAYERS
		__net_output = []
		__net_input = []

		# FEED FORWARDING
		for each_layer in forward_neuron_layer_order:
			layer_name = "_".join(["layer", each_layer])
			# INPUT FOR EACH LAYER IN INDEXING ORDER
			__net_input.append(input_vec)
			net_output = summation(
				__neural_network[layer_name]["weights"],
				input_vec.T,
				__neural_network[layer_name]["bias"]
			)
			# OUTPUT FOR EACH LAYER IN INDEXING ORDER
			__net_output.append(net_output)
			synaptic_output = sigmoid(net_output)

			# ASSIGNING SYNAPTIC OUTPUT TO INPUT
			input_vec = synaptic_output

		# BACKWARD PROPAGATION
		delta_in_weights = []
		delta_in_bias = []

		for each_layer in backward_neuron_layer_order:
			layer_name = "_".join(["layer", each_layer])
			if layer_name == last_layer:
				error_to_backpropagate = error_derivative(synaptic_output, desired_output_vec,
					__neural_network.get(layer_name).get("number_of_neurons"))
			else:
				prior_layer = "_".join(["layer", each_layer + 1])
				error_to_backpropagate = np.matmul(__neural_network[prior_layer]["weights"],
					error_to_backpropagate.T)
			# MINIMIZING ERROR
			derivative_of_sig = sigmoid_diff(__net_output[each_layer])
			input_vec = __net_input[each_layer]

			error_to_backpropagate = np.multiply(error_to_backpropagate, derivative_of_sig)
			weights_delta = np.matmul(input_vec.T, error_to_backpropagate.T).T
			bais_delta = error_to_backpropagate

			delta_in_weights.append(weights_delta)
			delta_in_bias.append(bais_delta)

		# UPDATE WEIGHTS AFTER
		# EACH ITERATION
		for each_layer in backward_neuron_layer_order:
			layer_name = "_".join(["layer", each_layer])
			__neural_network[layer_name]["weights"] -= __learning_rate * delta_in_weights[each_layer]
			__neural_network[layer_name]["bias"] -= __learning_rate * delta_in_bias[each_layer]

		calculated_error = calculate_error(synaptic_output, desired_output_vec, __neural_network.get(last_layer).get("number_of_neurons"))
		__errors.append(calculated_error)

	plt.plot(__errors)
	plt.ylabel('Error plotting')
	plt.show()

	return __neural_network

def test_nn(model_neural_network):
	__error_threshold = 0.01

	test_params, desired_output = get_data_set("test")

	test_dataset_size = len(test_params)

	calculated_output_arr = []
	desired_output_arr = []
	correct_classification = 0

	last_layer = "_".join(["layer", model_neural_network.get("layers_count")])

	forward_neuron_layer_order = list(range(1, model_neural_network.get("layers_count") + 1))

	for idx in range(test_dataset_size):
		# FEED FORWARDING
		desired_output_vec = np.array([desired_output[idx][0]])
		input_vec = np.array([get_input_image_as_vec(input_params[random_index][0])])

		for each_layer in forward_neuron_layer_order:
			layer_name = "_".join(["layer", each_layer])
			# INPUT FOR EACH LAYER IN INDEXING ORDER
			__net_input.append(input_vec)
			net_output = summation(
				model_neural_network[layer_name]["weights"],
				input_vec.T,
				model_neural_network[layer_name]["bias"]
			)
			# OUTPUT FOR EACH LAYER IN INDEXING ORDER
			__net_output.append(net_output)
			synaptic_output = sigmoid(net_output)

			# ASSIGNING SYNAPTIC OUTPUT TO INPUT
			input_vec = synaptic_output

		calculated_error = calculate_error(synaptic_output, desired_output_vec, model_neural_network.get(last_layer).get("number_of_neurons"))

		calculated_output = get_output_value(desired_output_vec)
		desired_output = get_output_value(synaptic_output)

		if calculated_output == desired_output:
			correct_classification += 1

		calculated_output_arr.append(get_output_value(calculated_output))
		desired_output_arr.append(get_output_value(desired_output))

	accuracy = (correct_classification / test_dataset_size) * 100.0
	print("Model accuracy: ",accuracy)

	return confusion_matrix(calculated_output_arr, desired_output_arr)

if __name__ == "__main__":
	try:
		command = sys.argv[1]
	except Exception as ex:
		command = "empty"

	if command == "train":
		model = train_nn()
		store_model(model)
	elif command == "test":
		model = load_model()
		result = test_nn(model)
		print("-- Confusion matrix --")
		print(result)
	else:
		print("Please provide command")
		print("Example: python perceptron.py train")
		print("Example: python perceptron.py test")


import scipy.stats as st
import scipy.optimize
import warnings
import numpy as np

def log_like_iid_gamma(params, n):
    """Log likelihood for i.i.d. Gamma measurements, parametrized
    by x, a"""

    beta, alpha = params
    if n.any() <= 0:
        return -np.inf
    if beta <= 0:
        return -np.inf
    if alpha<=0:
        return -np.inf

    return st.gamma.logpdf(n , alpha, loc=0, scale=1/beta).sum()

#Code based on Bois (2020)

def mle_iid_gamma(n):
    """Perform maximum likelihood estimates for parameters for i.i.d.
   Gamma measurements, parametrized by x, a (cov matrix)"""
    with warnings.catch_warnings():
        warnings.simplefilter("ignore")

        res = scipy.optimize.minimize(
            fun=lambda params, n: -log_like_iid_gamma(params, n),
            x0=np.array([2.00, 0.005]),
            args=(n,),
            method='Powell'
        )

    if res.success:
        return res.x
    else:
        raise RuntimeError('Convergence failed with message', res.message)

def log_like_iid_bespoke(params, n):
    beta, dbeta = params
    if n.any() <= 0:
        return -np.inf
    if beta <= 0:
        return -np.inf
    if dbeta <= 0:
        return -np.inf

    return np.sum(np.log(beta)+np.log(beta+dbeta)-np.log(dbeta)-beta*n+np.log(1-np.exp(-dbeta*n)))

def mle_iid_bespoke(n):
    with warnings.catch_warnings():
        warnings.simplefilter("ignore")

        res = scipy.optimize.minimize(
            fun=lambda params, n: -log_like_iid_bespoke(params, n),
            x0=np.array([1.00, 0.5]),
            args=(n,),
            method='Powell'
        )

    if res.success:
        return res.x
    else:
        raise RuntimeError('Convergence failed with message', res.message)


from vector_calculus.containers import *
from sympy import symbols, Expr, S
from linalg import tr, dot

# These are cannonical variables of cartesian coordinate system
xyz = symbols('x, y, z')

def div(u):
    '''Divergence of vector --> scalar. Divergence of tensor --> vector.'''
    # Vector
    if isinstance(u, Vector):
        return sum((Dx(ui, vari) for ui, vari in zip(u, xyz)), S(0))
    # Tensor, recurse rows
    elif isinstance(u, Tensor):
        return Vector([div(ui) for ui in u])
    else:
        raise TypeError('Only divergence of vector or tensor allowed.')

def grad(u, dim=None):
    '''Gradient of vector --> tensor. Gradient of scalar --> vector.'''
    # Scalar
    if isinstance(u, Expr):
        # Infer dim from arguments of u
        if dim is None:
            # If there is no z dependence this is most likely a 2d scalar
            if xyz[2] not in u.atoms():
                dim = 2
            else:
                dim = 3

        return Vector([Dx(u, var) for var in xyz[:dim]])
    # Vector, recurse rows
    elif isinstance(u, Vector):
        dim = len(u)
        return Tensor([grad(ui, dim) for ui in u])
    else:
        raise ValueError('Only gradient of scalar or vector allowed.')

def curl(u):
    '''Curl of 3d vector --> vector. Curl 2d vecror --> scalar.'''
    assert isinstance(u, Vector), 'Need vector for curl'
    if len(u) == 3:
        return -Vector([Dx(u[1], xyz[2]) - Dx(u[2], xyz[1]),
                        Dx(u[2], xyz[0]) - Dx(u[0], xyz[2]),
                        Dx(u[0], xyz[1]) - Dx(u[1], xyz[0])])
    else:
        R = Tensor([[0, 1], [-1, 0]])
        return div(dot(R, u))

def rot(u, orientation='+'):
    '''Rotation of 2d scalar --> 2d vector. Default is counter-clockwise rot.'''
    assert isinstance(u, Expr), 'Can only take rot of scalar'
    assert xyz[2] not in u.atoms(), 'Scalar must be function of x, y only'

    R = Tensor([[0, -1], [1, 0]])
    R = R if orientation == '+' else -R
    return dot(R, grad(u))

def Dx(u, var):
    '''Partial derivative of u w.r.t to var. Shape is maintained.'''
    if isinstance(u, (int, float)):
        u = S(u)
    # Scalar
    if isinstance(u, Expr):
        return u.diff(var, 1)
    # Vector
    elif isinstance(u, Vector):
        return Vector([Dx(ui, var) for ui in u])
    # Tensor
    elif isinstance(u, Tensor):
        return Tensor([Dx(ui, var) for ui in u])
    else:
        raise TypeError('Cannot take derivatieve of type %s' % type(u))


import numpy as np
import scipy.linalg as la
from scipy.stats import bernoulli

def acceptance(x, y, mu, sigma):
    p = min(1, np.exp(-0.5 * (np.dot(x - mu, la.solve(
        sigma, x - mu)) - np.dot(y - mu, la.solve(sigma, y - mu)))))
    return bernoulli.rvs(p)

def nextState(x, mu, sigma):
    K = len(x)
    y = np.random.multivariate_normal(x, np.eye(K))
    accept = acceptance(y, x, mu, sigma)
    if accept:
        return y
    else:
        return x

def lmvnorm(x, mu, sigma):
    return -0.5 * (np.dot(x - mu, la.solve(sigma, x - mu)) - len(x) * np.log(2 * np.pi) - np.log(la.det(sigma)))

def metropolis(x, mu, sigma, n_samples=1000):
    logprobs = np.zeros(n_samples)
    x_samples = np.zeros((n_samples, len(x)))
    for i in xrange(n_samples):
        logprobs[i] = lmvnorm(x, mu, sigma)
        x = nextState(x, mu, sigma)
        x_samples[i, :] = x.copy()
    return x_samples, logprobs


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Sat Apr 27 18:57:47 2019

@author: khaledghobashy
"""

# Third party imports.
import numpy as np

def skew_matrix(v):
    x, y, z = v.flatten()
    vs = np.array([[0, -z, y], [z, 0, -x], [-y, x, 0]])
    return vs

def orthogonal_vector(v):
    dummy = np.ones((3,1))
    i = np.argmax(np.abs(v))
    dummy[i] = 0
    m = np.linalg.multi_dot([skew_matrix(v), dummy])
    return m

def triad(v1, v2=None):
    k = v1/np.linalg.norm(v1)
    if v2 is not None:
        i = v2/np.linalg.norm(v2)
    else:
        i = orthogonal_vector(k)
        i = i/np.linalg.norm(i)
    j = np.linalg.multi_dot([skew_matrix(k), i])
    j = j/np.linalg.norm(j)
    m = np.concatenate([i, j, k], axis=1)
    return m

def mirrored(v):
    if v.shape != (3, 1):
        return v
    else:
        m = np.array([[1, 0, 0],
                      [0,-1, 0],
                      [0, 0, 1]], dtype=np.float64)
        return m.dot(v)

def centered(*args):
    p = np.sum(args, 0)
    p = p/len(args)
    return p

def oriented(*args):
    if len(args) == 2:
        v = args[1] - args[0]
    elif len(args) == 3:
        a1 = args[1] - args[0]
        a2 = args[2] - args[0]
        v = np.cross(a2, a1, axisa=0, axisb=0)
        v = np.reshape(v, (3, 1))
    v = v/np.linalg.norm(v)
    return v


# -*- coding: utf-8 -*-

#-------------------------------------- DATA PREPROCESSING ---------------------------------#

# Imports
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import math

from matplotlib.animation import FuncAnimation

# Reading the dataset from data
dataset = pd.read_csv(r'..\\data\\auto_insurance.csv')

# Creating Dependent and Independent variables
X = dataset['X'].values
y = dataset['Y'].values

# Visualizing the data
title='Linear Regression on <Dataset>'
x_axis_label = 'X-value < The corresponding attribute of X in dataset >'
y_axis_label = 'y-value < The corresponding attribute of X in dataset >'
title='Linear Regression on Auto Insurance Sweden Dataset'
x_axis_label = "Total Payment"
y_axis_label = "Number of Claims"

plt.scatter(X,y)
plt.title(title)
plt.xlabel(x_axis_label)
plt.ylabel(y_axis_label)
plt.show()

# Splitting the data into training set and test set
X_train,X_test = np.split(X,indices_or_sections = [int(len(X)*0.8)])
y_train,y_test = np.split(y,indices_or_sections = [int(len(X)*0.8)])

#------------------------------------ DATA PREPROCESSING ENDS -----------------------------#

#--------------------------------------- TRAINING   ---------------------------------------#

# Functions for caluculation of gradient descent

# Function to caluculate mean square error
def mean_square_error(slope,intercept,x_data,y_data):
    n = len(x_data)
    sum_of_squares = 0
    for i in range(n):
        y_pred = x_data[i]*slope+intercept
        y_actual = y_data[i]
        sum_of_squares += (y_actual - y_pred)**2

    mean_square_error = math.sqrt(sum_of_squares/n)
    return mean_square_error

# Function to caluculate Gardient of slope at each intermidiatory iteration
def gradient_slope(slope,intercept,x_data,y_data):
    n = len(x_data)
    sum_of_gradients = 0
    for i in range(n):
        y_pred = x_data[i]*slope + intercept
        y_actual = y_data[i]
        sum_of_gradients += x_data[i]*(y_actual-y_pred)
    sum_of_gradients = -1*sum_of_gradients
    gradient_slope = sum_of_gradients*(2/n)
    return gradient_slope

# Function to caluculate Gardient of intercept at each intermidiatory iteration
def gradient_intercept(slope,intercept,x_data,y_data):
    n = len(x_data)
    sum_of_gradients = 0
    for i in range(n):
        y_pred = x_data[i]*slope +intercept
        y_actual = y_data[i]
        sum_of_gradients += -1 *(y_actual-y_pred)
    sum_of_gradients = -1*sum_of_gradients
    gradient_intercept = sum_of_gradients*(2/n)
    return gradient_intercept

# Gradient Decent function
slope_history = []
intercept_history = []
loss_history = []
def gradient_decent(x_data,y_data,epochs,learning_rate,initial_slope,initial_intercept):
    slope = initial_slope
    intercept = initial_intercept
    for epoch in range(epochs):
        slope  = slope - learning_rate*(gradient_slope(slope,intercept,x_data,y_data))
        intercept = intercept - learning_rate*(gradient_intercept(slope,intercept,x_data,y_data))
        loss = mean_square_error(slope,intercept,x_data,y_data)
        print("The loss obtained is %0.2f "%(loss))
        loss_history.append(loss)
        slope_history.append(slope)
        intercept_history.append(intercept)
    return (slope,intercept)

# Training
# Hyperparameters
epochs = 500
learning_rate = 0.00001
initial_slope = 0
initial_intercept = 0

(slope,intercept) = gradient_decent(X_train,y_train,epochs = epochs,learning_rate=learning_rate,initial_slope = initial_slope,initial_intercept =initial_intercept)

# The plot to inspect the decrease in loss
plt.title("Loss Function")
plt.plot(loss_history)
plt.show()

print('gif is created')

# The plot to inspect the variation in slope and intercept
plt.plot(slope_history)
plt.plot(intercept_history)
plt.show()

#-------------------------------------- TRAINING ENDS  ------------------------------------#

#------------------------------- PREDICTION AND PLOTING -----------------------------------#

m = slope
c = intercept

# Predicting the Results
y_pred = X_test*m + c

# Visualizing the Results
plt.scatter(X_test,y_test,c='red')
plt.plot(X_test,y_pred)

plt.title(title)
plt.xlabel(x_axis_label)
plt.ylabel(y_axis_label)
plt.show()

# Animation of Gradient Descent

fig, ax = plt.subplots()
xdata, ydata = [], []
ln, = ax.plot([], [], 'b', animated=True)
sc = plt.scatter(X_test,y_test,c='red',animated=True)

def init():
    ax.set_title(title)
    plt.xlabel(x_axis_label)
    plt.ylabel(y_axis_label)
    ax.set_xlim(min(X_test)-10, max(X_test)+10)
    ax.set_ylim(min(y_test)-10, max(y_test)+10)
    return ln,

def update(frame):
    i = frame
    m = slope_history[i]
    c = intercept_history[i]
    y_pred = X_test*m + c
    loss = mean_square_error(m,c,X_test,y_pred)
    ax.text(2,3,loss)
    ln.set_data(X_test, y_pred)

anim = FuncAnimation(fig, update, frames=range(len(slope_history)),init_func=init)
anim.save('gradient_descent.gif', fps=30)
plt.show()

#------------------------------ PREDICTION AND PLOTING ENDS--------------------------------#


# -*- coding: utf-8 -*-
#

from sympy.core import Symbol
from sympy import lambdify

from numpy import linspace, pi
from matplotlib import pyplot as plt

from .glt import Stiffness

def plot_stiffness_symbols(degrees=[2, 3], nx=100):
    """
    Plots the stiffness symbol for different degrees on a given number of elements.

    degrees: list, tuple
        a list/tuple of spline degrees

    nx: int
        number of points used for plot
    """

    colors = ["r", "b", "g", "y", "m", "k", "c"]

    for i,p in enumerate(degrees):

        t = Symbol('t')

        symbol = Stiffness(p, t)

        # ... make the symbol a numeric function, that can be evaluated
        f = lambdify(t, symbol, "numpy")

        t1 = linspace(-pi,pi, nx)
        w = f(t1)
        # ...

        plt.plot(t1, w.real, "-"+colors[i], label="$p=" + str(p) + "$")

    plt.xlabel('fequencies')
    plt.ylabel('$\mathfrak{s}_p$')
    plt.legend(loc=9)


import math
import numpy as np

# ---- convention for accessing all 4 pi sr by just two angles, theta and phi ----

def gamma_convention():
    return 0

def beta_convention(theta):
    return -np.pi/2 + theta  # theta \in [0, pi]

def alpha_convention(phi):
    return phi  # phi \in [0, 2*pi]
# ----

def get_zxz_rot(alpha, beta, gamma):
    """
    calculate active rotation R_z(gamma) R_x(beta) R_z(alpha)
    think like this: the application is in the reverse order,
    and the order of arguments is the order of application,
    so, again reversed!
    and the index namings can also be flipped around without change
    """
    return np.einsum("ij,jk",
                     np.einsum("ij,jk", get_R_z(alpha), get_R_x(beta)),
                     get_R_z(gamma))

    # return np.dot(get_R_x(beta), get_R_z(alpha))

    # return np.dot(get_R_z(gamma), np.dot(get_R_x(beta), get_R_z(alpha)))  # this produces the wrong result
    # import ipdb; ipdb.set_trace()  # noqa BREAKPOINT
    # return np.einsum("ij,jk,kl", get_R_z(alpha), get_R_x(beta), get_R_z(gamma))

def get_R_x(angle):
    return np.array([[1, 0, 0],
                     [0, math.cos(angle), -math.sin(angle)],
                     [0, math.sin(angle), math.cos(angle)]])

def get_R_y(angle):
    return np.array([[math.cos(angle), 0, math.sin(angle)],
                     [0, 1, 0],
                     [-math.sin(angle), 0, math.cos(angle)]])

def get_R_z(angle):
    # import ipdb; ipdb.set_trace()  # noqa BREAKPOINT
    return np.array([[math.cos(angle), -math.sin(angle), 0],
                     [math.sin(angle), math.cos(angle), 0],
                     [0, 0, 1]])


import numpy as np
from nn_utils import read_dataset

size = 1000

def sigmoid(x, deriv=False):
    if (deriv == True):
        return x * (1 - x)
    return 1 / (1 + np.exp(-x))

X, y = read_dataset(180, 1000)

np.random.seed(5)
# synapses
w0 = 2 * np.random.random((X.size / X.__len__(), X.__len__())) - 1
w1 = 2 * np.random.random((X.__len__(), X.__len__())) - 1
w2 = 2 * np.random.random((X.__len__(), 1)) - 1

# training step
for j in xrange(60000):

    # Calculate forward through the network.
    l0 = X
    l1 = sigmoid(np.dot(l0, w0))
    l2 = sigmoid(np.dot(l1, w1))
    l3 = sigmoid(np.dot(l2, w2))

    # Error back propagation of errors using the chain rule.
    l3_error = y - l3
    if (j % 10) == 0:
        print("Error: " + str(np.mean(np.abs(l3_error))))

    l3_adjustment = l3_error * sigmoid(l3, deriv=True)  # (y-a).d/dw(-a), a = sigmoid(Sum Xi*Wi)
    l2_error = l3_adjustment.dot(w2.T)

    l2_adjustment = l2_error * sigmoid(l2, deriv=True)  # (y-a).d/dw(-a), a = sigmoid(Sum Xi*Wi)
    l1_error = l2_adjustment.dot(w1.T)

    l1_adjustment = l1_error * sigmoid(l1, deriv=True)  # (y-a).d/dw(-a), a = sigmoid(Sum Xi*Wi)

    # update weights for all the synapses (no learning rate term)
    w2 += l2.T.dot(l3_adjustment)
    w1 += l1.T.dot(l2_adjustment)
    w0 += l0.T.dot(l1_adjustment)

print("Output after training")
print(l3)

def predict(X1):
    l0 = np.zeros((X.__len__(), X.size / X.__len__()))
    max = np.matrix(X1).max()
    l0[0] = 2 * np.asanyarray(X1, dtype=np.float32) / max - 1
    l1 = sigmoid(np.dot(l0, w0))
    l2 = sigmoid(np.dot(l1, w1))
    l3 = sigmoid(np.dot(l2, w2))
    return l3[0][0]  # since process X1[0] output would be l2[0]

test_dataset = [1, 9, 19, 33, 16, 2, 1]
result = predict(test_dataset)
print("expected output 1, predicted output " + repr(result))
assert (result > 0.95), "Test Failed. Exepected result > 0.95"

test_dataset = [1, 0, 1, 4, 1, 3, 1]
result = predict(test_dataset)
print("expected output 0, predicted output " + repr(result))
assert (result < 0.95), "Test Failed. Exepected result < 0.95"


import numpy as np

def mad_based_outlier(points, thresh=3.5, two_sided=False):
    '''Identifies outliers using Median Absolute Deviation.

    By default, returns outliers who are greater than the main distribution.

    Args:
        points (numpy.array): Data points.
        thresh (float): Modified Z-score threshold.
        two_sided (bool, default=False): If True, returns outliers both greater
            than and less than the main distribution.
    Returns:
        numpy.array: A boolean array of outliers.
    '''
    if len(points.shape) == 1:
        points = np.array(points)[:, None]
    median = np.median(points, axis=0)
    diff = np.sum((points - median)**2, axis=-1)
    diff = np.sqrt(diff)
    med_abs_deviation = np.median(diff)
    signed_diff = np.sum((points - median), axis=-1)
    modified_z_score = 0.6745 * signed_diff / med_abs_deviation
    return modified_z_score > thresh


# -*- coding: utf-8 -*-
# math_helpers.py

def simple_linear_regression(x, y):
    import numpy as np

    x = np.array(x)
    y = np.array(y)

    n = x.shape[0]
    assert n > 2, "not enough data points"

    sum_x = np.sum(x)
    sum_xx = np.sum(x * x)

    slope = (n * np.sum(x * y) - sum_x * np.sum(y)) / (n * sum_xx - sum_x ** 2)
    bias = y.mean() - slope * x.mean()

    residuals_sq = ((y - (slope * x + bias)) ** 2).sum()
    std_error_slope = np.sqrt(residuals_sq / (n - 2) / (np.sum((x - x.mean()) ** 2)))

    std_error_bias = np.sqrt(
        residuals_sq / (n - 2) / n * sum_xx / (np.sum((x - x.mean()) ** 2))
    )

    return (slope, std_error_slope), (bias, std_error_bias)

def correlation(x, y) -> float:
    from statistics import stdev, mean

    mean_x, std_x = mean(x), stdev(x)
    mean_y, std_y = mean(y), stdev(y)

    if (std_y == 0) or (std_x == 0):
        return 0

    running_sum = 0
    running_count = 0
    for (x_, y_) in zip(x, y):
        running_sum += (x_ - mean_x) * (y_ - mean_y)
        running_count += 1

    if running_count < 1:
        return 0

    return (running_sum / (running_count - 1)) / std_y / std_x


import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from sklearn.preprocessing import StandardScaler

# Data preprocessing
data = pd.read_csv("data.csv")

# Converting Pandas dataframe to numpy array
X = data.x.values.reshape(-1, 1)
Y = data.y.values.reshape(-1, 1)

# Add bias
m = X.shape[0]  # sample count
bias = np.ones((m, 1))
X_train = np.append(bias, X, axis=1)

# Add x^2 values
x2 = X**2
X_train = np.append(X_train, x2, axis=1)
Y_train = np.array(Y)

# Calculating theta with Normal Equation formula
def normalEquation(X, Y):
    theta = []

    # Normal Equation formula is :
    # inverse(X^T * X) * X^T * y
    X_transpose = np.transpose(X)
    theta = np.linalg.inv(X_transpose.dot(X)).dot(X_transpose).dot(Y)

    return theta

# Get theta
theta = normalEquation(X_train, Y_train)
print("Theta = ", theta)

# Predicted Values
Y_pred = X_train.dot(theta)

# Model Evaluation
def rmse(Y, Y_pred):
    rmse = np.sqrt(sum((Y - Y_pred) ** 2) / Y.shape[0])
    return rmse

def r2_score(Y, Y_pred):
    mean_y = np.mean(Y)
    ss_tot = sum((Y - mean_y) ** 2)
    ss_res = sum((Y - Y_pred) ** 2)
    r2 = 1 - (ss_res / ss_tot)
    return r2

# Print Scores
print("RMSE = ", rmse(Y_train, Y_pred))
print("R2 Score = ", r2_score(Y_train, Y_pred))

# Visualization
# Ploting Regression Line
plt.plot(X, Y_pred, color='#c93e4e', label='Regression Line')
# Ploting Scatter Points
plt.scatter(X, Y, c='#54a774', label='Scatter Plot')
plt.xlabel('Size')
plt.ylabel('Price')
plt.legend()
plt.show()


"""
This code includes functions for computing
the eigenvalues of large sparse matrices.

The main eigenvalue of interest is the second
smallest eigenvalue of the graph laplacian
(known as the algerbaic connectivity).

"""

#Import packages:
from scipy.sparse.linalg import LinearOperator, eigsh, minres
import numpy as np

def Largest_EV(M,return_eigenvector = False) :

	"""

	Computes the largest eigenvalue of a sparse matrix M

	Parameters
	------------

	M : scipy sparse matrix

	Returns
	----------

	EV : double

	The largest eigenvalue of the matrix M

	"""
	M = M.asfptype()

	if return_eigenvector == True:
		evals_large , evecs_large = eigsh(M, 1, which='LM', tol=1E-6, return_eigenvectors=True)
		return evals_large[0], np.transpose(evecs_large)[-1]
	else:
		evals_large = eigsh(M, 1, which='LM', tol=1E-6, return_eigenvectors=False)
		return evals_large[0]

def Smallest_Non_Zero(M, return_eigenvector = False , v0 = None , Threshold = 1E-8 , zero_eig_limit = 100) :

		"""

		Method to find the smallest non zero eigenvalue of a positive semidefinite matrix.

		-This method can be used when we want to estimate the mu2 value
		without having to assume that we have initially extracted the LCC.

		-Recommend using this even for matrices which we expect to have a single
		zero eigenvalue as the eigsh method can sometimes return multiple values
		close to zero due to numerical error in the eigensolver.

		Terminates if the first 100 eigenvalues are found to be below
		the threshold.

		From M we can construct a linear operator which is used in the eigenvalue computation:
		(See https://github.com/scipy/scipy/issues/4170 or https://docs.scipy.org/doc/scipy-0.18.1/reference/generated/scipy.sparse.linalg.LinearOperator.html for details )

		Parameters
		---------------

		M : scipy sparse matrix

		matrix to find eigenvalues of (This code has been desigend for this to be a
		graph laplacian matrix).

		return_eigenvector : bool (optional)

		option to return the eigenvector associated with the smallest non-zero eigenvalue.

		v0 : vector/array (optional)

		Vector to use as a starting point for iteration.

		Threshold : float (optional)

		Threshold for eigenvalues to be assumed to be zero.
		Set to 1E-8 by default. Has to be non-zero otherwise
		we will incorrectly return zero eigenvalues as non-zero
		due to numerical error.

		zero_eig_limit : int

		Terminate if we find more than 'zero_eig_limit' eigenvalues close to zero.

		"""

		Found_Non_Zero = False
		k = 0

		#Construct the linear operator to use in the solver:
		OPinv = LinearOperator(matvec=lambda v: minres(M, v, tol=1e-6)[0], shape=M.shape, dtype=M.dtype)

		"""Compute eigenvalues and eigenvectors until we find one
		which is greater than a certain threshold."""
		while Found_Non_Zero == False and k < zero_eig_limit :

			#Compute eigenvalues and eigenvectors: (looks for at least two eigenvalues)
			evals_small , evecs_small = eigsh(M, k+2, sigma= 0, which='LM', tol = 1E-6, return_eigenvectors = True , OPinv = OPinv , v0 = v0)

			#Check if any eigenvalues are above the threshold:
			if max(evals_small) > Threshold :
				Found_Non_Zero = True
				Mu2 = max(evals_small)
			k = k + 1

		if return_eigenvector == True :
			return Mu2 , np.transpose(evecs_small)[-1]
		else:
			return Mu2


import numpy as np
from numba import njit
from time import time
from tensiga.iga.auxkv import auxkv
from tensiga.iga.Bspline import Bspline
from tensiga.quadrature.afftf import afftf

def glint(kv, p):
    """Univariate, global Gauss-Legendre quadrature rule

    Parameters:
        n (int) : quadrature order n > 0

    Returns:
        ip (np.array(float)) : array integration point coordinates
        W (numpy.array(float)) : weights W[k] corresponding to ip[k]
    """
    dim = len(kv)
    aux, _ = auxkv(kv)
    nelem = len(aux) - 1

    ip, W = proto_rule = np.polynomial.legendre.leggauss(p)

    glob_ip = [None] * ip.size * nelem
    glob_W = [None] * ip.size * nelem
    for idx in range(nelem):
        lbound = aux[idx]
        rbound = aux[idx+1]

        x, dx = afftf(ip, [lbound, rbound])
        glob_ip[(idx*ip.size):(idx*ip.size + ip.size)] = x
        glob_W[(idx*ip.size):(idx*ip.size + ip.size)] = W * dx

    return np.array(glob_ip), np.array(glob_W)

if __name__ == '__main__':
    def f1(x):
        return x**2 + 3*x - 10

    kv = np.array([0, 1, 1, 1, 2, 3, 4, 4, 5, 6, 6, 6])
    kv = np.linspace(0,6,1000)

    tstart = time()
    ip, W = glint(kv, 2)
    print('time:', time()-tstart)

    res = 0
    for (ip_k, W_k) in zip(ip, W):
        res += W_k * f1(ip_k)

    print(res) # exact: 66


import numpy as np
'''
ベクトル[の|][演算|計算]を[する|行う]
行列[の|][演算|計算]を[する|行う]
numpyを[使う|入れる|インポートする]
@prefix(aArray;配列)
'''

iterable = np.array([0, 1, 2, 3])
aArray = np.array([1, 2, 3, 4])
aArray2 = iterable
aList = [1, 2]

np.array(aList)
'''
aListから配列を[作る]
'''

np.array(iterable)
'''
iterableから配列を作る
'''

aArray.shape()
'''
aArrayの[形状|形]を調べる
'''

aArray.dtype()
'''
aArrayの[データ型|型]を調べる
'''

np.ndim(aArray)
'''
aArrayの[次元数|次元の数]を調べる
'''

np.arange(10)
'''
0から9までの配列を作る
'''

aArray.reshape(3, 3)
'''
aArrayを3×3の多次元配列に変形する
'''

np.concatenate([aArray, aArray2])
'''
aArrayと@aArray2を[列方向|縦方向]に連結する
'''

np.concatenate([aArray, aArray2], axis=0)
'''
aArrayと@aArray2を[列方向|縦方向]に連結する
'''

np.concatenate([aArray, aArray2], axis=1)
'''
aArrayと@aArray2を[行方向|横方向]に連結する
'''

np.sum(aArray)
'''
aArrayの[合計値|合計]を調べる
'''

np.sum(aArray, axis=0)
'''
aArrayの列ごとの[合計値|合計]を調べる
'''

np.sum(aArray, axis=1)
'''
aArrayの行ごとの[合計値|合計]を調べる
'''

np.mean(aArray)
'''
aArrayの[平均値|平均]を調べる
'''

np.mean(aArray, axis=0)
'''
aArrayの列ごとの[平均値|平均]を調べる
'''

np.mean(aArray, axis=1)
'''
aArrayの行ごとの[平均値|平均]を調べる
'''

np.min(aArray)
'''
aArrayの[最小値|最小]を調べる
'''

np.min(aArray, axis=0)
'''
aArrayの列ごとの[最小値|最小]を調べる
'''

np.min(aArray, axis=1)
'''
aArrayの行ごとの[最小値|最小]を調べる
'''

np.max(aArray)
'''
aArrayの[最大値|最大]を調べる
'''

np.max(aArray, axis=0)
'''
aArrayの列ごとの[最大値|最大]を調べる
'''

np.max(aArray, axis=1)
'''
aArrayの行ごとの[最大値|最大]を調べる
'''

np.std(aArray)
'''
aArrayの標準偏差を調べる
'''

np.std(aArray, axis=0)
'''
aArrayの列ごとの標準偏差を調べる
'''

np.std(aArray, axis=1)
'''
aArrayの行ごとの標準偏差を調べる
'''

np.var(aArray)
'''
aArrayの分散を調べる
'''

np.var(aArray, axis=0)
'''
aArrayの列ごとの分散を調べる
'''

np.var(aArray, axis=1)
'''
aArrayの行ごとの分散を調べる
'''

np.eye(3)
'''
3×3の単位行列を作る
'''

np.identity(3)
'''
3×3の単位行列を作る
'''

np.empty(5)
'''
要素数5の[空配列|空の配列]を作る
'''

np.empty((2, 3))
'''
2×3の[空配列|空の配列]を作る
'''

np.empty_like(aArray)
'''
aArrayと同じ大きさの[空配列|空の配列]を作る
'''

np.gcd(aArray, aArray2)
'''
aArrayと@aArray2の要素ごとの最大公約数を調べる
'''

np.lcm(aArray, aArray2)
'''
aArrayと@aArray2の要素ごとの最小公倍数を調べる
'''

np.unique(aArray)
'''
aArrayから重複を除いた配列を作る
aArrayのユニークな要素を調べる
'''

u, counts = np.unique(aArray, return_counts=True)
'''
aArrayのユニークな要素とその個数を調べる
'''

u, indices = np.unique(aArray, return_index=True)
'''
aArrayのユニークな要素とその位置を調べる
'''

np.cumsum(aArray)
'''
aArrayの累積和を調べる
'''

np.cumprod(aArray)
'''
aArrayの累積積を調べる
'''

aArray.flatten()
'''
aArrayを[一次元にする|一次元化|平坦化]
'''


# -*-coding: utf-8 -*-

from numpy import*
# 将文本信息转换为矩阵返回
def loadDataSet(fileName):
    fr = open(fileName)
    stringArr = [line.strip().split('\t') for line in fr.readlines()]
    datArr = [map(float,line) for line in stringArr]     #map(float,line)第一个参数是一个函数，第二个参数是输入的数据，函数作用于第二个输入的每一个对象并返回list
    return mat(datArr)

# PCA算法实现
def pca(dataMat,topNfeat):
    meanVals = mean(dataMat ,axis = 0)  #对矩阵的每一列取平均值
    meanRemoved = dataMat - meanVals   #去均值
    covMat = cov(meanRemoved,rowvar = 0)
    eigVals,eigVects = linalg.eig(mat(covMat))   #算得特征值(是个数组)和特征向量（是个矩阵）
    eigValInd = argsort(eigVals)               #对特征值进行从小到大的排序
    eigValInd = eigValInd[:-(topNfeat+1):-1]
    redEigVects = eigVects[:,eigValInd]
    loadDataMat = meanRemoved * redEigVects  #将去除均值后的数据矩阵*压缩矩阵，转换到新的空间，使维度降低为N
    reconMat = (loadDataMat * redEigVects.T) + meanVals  #利用降维后的矩阵反构出原数据矩阵(用作测试，可跟未压缩的原矩阵比对)
    return loadDataMat,reconMat

# 自适应PCA算法实现
def zsypca(dataMat):
    meanVals = mean(dataMat ,axis = 0)  #对矩阵的每一列取平均值
    meanRemoved = dataMat - meanVals   #去均值
    covMat = cov(meanRemoved,rowvar = 0)
    eigVals,eigVects = linalg.eig(mat(covMat))   #算得特征值(是个数组)和特征向量（是个矩阵）
    eigVals = real(eigVals)
    eigValInd = argsort(eigVals)               #对特征值进行从小到大的排序
    N = len(eigValInd)                         #特征值的个数
    eigValInd = eigValInd[:-(N+1):-1]
    newEigValInd = []
    sumEigVals = sum(eigVals)
    i = 0
    sumOfEigVls= 0
    while sumOfEigVls/float(sumEigVals) < 0.98:  # 压缩比约为98%
        sumOfEigVls += eigVals[i]
        newEigValInd.append(i)
        i += 1
    demension = i
    compressRatio = sumOfEigVls/float(sumEigVals)
    newEigValInd = array(newEigValInd)
    redEigVects = eigVects[:,newEigValInd]
    redEigVects = real(redEigVects)
    loadDataMat = meanRemoved * redEigVects  #将去除均值后的数据矩阵*压缩矩阵，转换到新的空间，使维度降低为N
    reconMat = (loadDataMat * redEigVects.T) + meanVals  #利用降维后的矩阵反构出原数据矩阵(用作测试，可跟未压缩的原矩阵比对)
    return loadDataMat,reconMat,demension ,compressRatio,redEigVects,meanVals


# Starting in the top left corner of a 2×2 grid,
# and only being able to move to the right and down,
# there are exactly 6 routes to the bottom right corner.
#
# How many such routes are there through a 20×20 grid?

from scipy.special import binom

def solution():
    return binom(40, 20)

if __name__ == '__main__':
    print(solution())


##%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
##                          K-Means clustering of Etch data
## %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

#%% import required packages
import numpy as np
import matplotlib.pyplot as plt
from sklearn.cluster import KMeans

#%% fetch data
import scipy.io

matlab_data = scipy.io.loadmat('MACHINE_Data.mat', struct_as_record = False)
Etch_data = matlab_data['LAMDATA']
calibration_dataAll = Etch_data[0,0].calibration # calibration_dataAll[i,0] corresponds to a 2D data from ith batch where columns correspond to different variables

variable_names = Etch_data[0,0].variables

##%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
##                          perform Multiway PCA
## %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

#%% generate unfolded data matrix
n_vars = variable_names.size - 2 # first 2 columns are not process variables
n_samples = 85 # following the work of He et al.

unfolded_dataMatrix = np.empty((1,n_vars*n_samples))
for expt in range(calibration_dataAll.size):
    calibration_expt = calibration_dataAll[expt,0][5:90,2:] # removing first 5 measurements as done in He et al.

    if calibration_expt.shape[0] < 85:
        continue

    unfolded_row = np.ravel(calibration_expt, order='F')[np.newaxis,:]
    unfolded_dataMatrix = np.vstack((unfolded_dataMatrix, unfolded_row))

unfolded_dataMatrix = unfolded_dataMatrix[1:,:]

#%% scale data
from sklearn.preprocessing import StandardScaler

scaler = StandardScaler()
data_train_normal = scaler.fit_transform(unfolded_dataMatrix)

#%% PCA
from sklearn.decomposition import PCA

pca = PCA(n_components = 3) # following the work of He et al.
score_train = pca.fit_transform(data_train_normal)

#%% visualize in 2D
plt.figure()
plt.scatter(score_train[:,0],score_train[:,1])
plt.xlabel('PC1 scores')
plt.ylabel('PC2 scores')

##%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
##                          K-Means on PCA scores
## %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

#%% determining number of clusters via elbow method
SSEs = []
for n_cluster in range(1, 10):
    kmeans = KMeans(n_clusters = n_cluster, random_state = 100).fit(score_train)
    SSEs.append(kmeans.inertia_)

plt.figure()
plt.plot(range(1,10), SSEs, marker='o')
plt.xlabel('Number of clusters')
plt.ylabel('SSEs')
plt.show()

#%% fit k-means model
n_cluster = 3
kmeans = KMeans(n_clusters = n_cluster, random_state = 100).fit(score_train)
cluster_label = kmeans.predict(score_train) # can also use kmeans.labels_

plt.figure()
plt.scatter(score_train[:, 0], score_train[:, 1], c = cluster_label, s = 20, cmap = 'viridis')
plt.xlabel('PC1 scores')
plt.ylabel('PC2 scores')

cluster_centers = kmeans.cluster_centers_
cluster_plot_labels = ['Cluster ' + str(i+1) for i in range(n_cluster)]
for i in range(n_cluster):
    plt.scatter(cluster_centers[i,0], cluster_centers[i,1], c = 'red', s = 40, marker = '*', alpha = 0.9)
    plt.annotate(cluster_plot_labels[i], (cluster_centers[i,0], cluster_centers[i,1]))

##%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
##                          Analyzing cluster quality via silhouette plots
## %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

#%% average silhouette score
from sklearn.metrics import silhouette_samples, silhouette_score

silhouette_avgValue = silhouette_score(score_train, cluster_label)
print('Average silhouette score is :', silhouette_avgValue)

#%% silhouette plot
from matplotlib import cm

plt.figure()
silhouette_values = silhouette_samples(score_train, cluster_label)
y_lower, y_upper = 0, 0
yticks = []
for i in range(n_cluster):
    cluster_silhouette_vals = silhouette_values[cluster_label == i]
    cluster_silhouette_vals.sort()

    y_upper += len(cluster_silhouette_vals)
    color = cm.nipy_spectral(i / n_cluster)
    plt.barh(range(y_lower, y_upper),cluster_silhouette_vals,height=1.0,edgecolor='none',color=color)

    yticks.append((y_lower + y_upper) / 2)
    y_lower += len(cluster_silhouette_vals)

plt.axvline(silhouette_avgValue, color="red", linestyle="--")
plt.yticks(yticks, np.arange(n_cluster)+1)
plt.xlabel('Silhouette coefficient values')
plt.ylabel('Cluster')


# Exercise 5.21
# Author: Noah Waterfield Price

import numpy as np
import matplotlib.pyplot as plt

def factorial(n):
    if n == 0:
        return 1
    else:
        return n * factorial(n - 1)

def S(x, n):
    ans = 0
    for j in range(n + 1):
        ans += (-1) ** j * x ** (2 * j + 1) / factorial(2 * j + 1)
    return ans

x = np.linspace(0, 4 * np.pi, 1001)

plt.plot(x, np.sin(x), '--k', linewidth=2)
for n in [1, 2, 3, 6, 12]:
    plt.plot(x, S(x, n))

plt.ylim([-1.1, 1.1])
plt.legend(['n = 1', 'n = 2', 'n = 3', 'n = 6', 'n = 12', 'sin(x)'])
plt.xlabel('x')
plt.ylabel('f(x)')
plt.show()


# -*- coding: utf-8 -*-
"""
Created on Thu Feb 14 19:00:16 2019

@author: z
"""

import numpy as np
import scipy.stats as ss
import statsmodels.api as sm
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D
from sklearn.linear_model import LinearRegression
from sklearn.model_selection import train_test_split

#生成数据
n = 100
beta_0 = 5
beta_1 = 2
beta_2 = -1
np.random.seed(1)
x_1 = 10 * ss.uniform.rvs(size=n)
x_2 = 10 * ss.uniform.rvs(size=n)
X = np.stack([x_1,x_2], axis=1)
y_sci = beta_0 + beta_1 * x_1 + beta_2 * x_2 + ss.norm.rvs(loc=0, scale=1, size=n)

x = 10 * ss.uniform.rvs(size=n)
y = beta_0 + beta_1 * x + ss.norm.rvs(loc=0, scale=1, size=n)

plt.figure()
plt.plot(x,y,"o",ms=5)
xx = np.array([0,10])
plt.plot(xx,beta_0 + beta_1 * xx)
plt.xlabel("x")
plt.ylabel("y");

fig = plt.figure()
ax = fig.add_subplot(111, projection='3d')
ax.scatter(X[:,0], X[:,1], y_sci, c=y_sci)
ax.set_xlabel("$x_1$")
ax.set_ylabel("$x_2$")
ax.set_zlabel("$y_sci$");

#计算方差
def compute_rss(y_estimate, y):
  return sum(np.power(y-y_estimate, 2))
def estimate_y(x, b_0, b_1):
  return b_0 + b_1 * x
rss = compute_rss(estimate_y(x, beta_0, beta_1), y)

#最小二乘估计
rss = []
slopes = np.arange(-10,15,0.01)
for slope in slopes:
    rss.append(np.sum((y - beta_0 - slope * x)**2))
ind_min = np.argmin(rss)

plt.figure()
plt.plot(slopes,rss)
plt.xlabel("Slope")
plt.ylabel("RSS")
print("Estimate for the slope: ", slopes[ind_min])

#使用Statsmodel模型库
X_sm = sm.add_constant(x)
mod = sm.OLS(y,X_sm)
est = mod.fit()
print(est.summary())

#使用Scikit_Learn机器学习库
lm = LinearRegression(fit_intercept=True)
lm.fit(X,y_sci)
print("beta 0: ",lm.intercept_)    #截距
print("beta 1&2: ",lm.coef_)         #变量数组
X_0 = np.array([2,4])
print("预测 [%d,%d]处y值: %.4f" % (X_0[0], X_0[1], lm.predict(X_0.reshape(1, -1))))
print("R2 :",lm.score(X, y_sci))

#测试数据
X_train, X_test, y_train, y_test = train_test_split(X, y_sci, train_size=0.5, random_state=1)
lm = LinearRegression(fit_intercept=True)
lm.fit(X_train, y_train)
lm.score(X_test, y_test)

import numpy,import scipy,import statsmodels


import logging
from math import pi
import numpy as np

log = logging.getLogger(__name__)

def linear(N):
    return (np.linspace(-1, 1, N), np.ones(N))

def trapezoid(N):
    points = np.linspace(-1.0, 1.0, N)
    w = 1.0 / float(N)
    weights = np.ones(N) * 2.0 * w
    weights[0] = weights[-1] = w
    return (points, weights)

def periodic_trapezoid(N):
    shift = 1.0 / float(N)
    points = np.linspace(-1 + shift, 1 - shift, N)
    weights = np.ones(N) * (2.0 / float(N))
    return (points, weights)

def legendre_gauss_lobatto(N):
    # Use the Chebyshev-Gauss-Lobatto nodes as the first guess
    x = np.cos(np.linspace(0, pi, N))
    # The Legendre Vandermonde Matrix
    P = np.zeros((N, N))
    # Compute P_(N) using the recursion relation
    # Compute its first and second derivatives and
    # update x using the Newton-Raphson method.
    xold = 2
    while np.amax(np.abs(x - xold)) > (1e-15):
        xold = x
        P[:, 0] = 1
        P[:, 1] = x
        for k in range(2, N):
            P[:, k] = ((2 * k - 1) * x * P[:, k - 1] -
                       (k - 1) * P[:, k - 2]) / k
        x = xold - (x * P[:, N - 1] - P[:, N - 2]) / (N * P[:, N - 1])

    w = 2.0 / ((N - 1) * N * np.square(P[:, N - 1]))

    return (np.array(x[::-1]), np.array(w[::-1]))


import numpy as np

def beta_model(So, r, rc, beta):
    ''' Beta model with 3 parameters.
    So -- normalization
    rc -- core radius
    beta -- powerlaw slope

    returns the flux (Cnts/pixel)

    '''

    return So * ( 1.0 + (r / rc)**2)**(-3.0 * beta + 0.5)

def integ_beta_model(r, rc, beta, r_start=0):
    ''' 2pi*r integral of the above Beta model with 3 parameters.
    r -- r to integrate to
    rc -- core radius
    beta -- powerlaw slope
    r_start -- starting point for integration

    This equation comes from Moretti et al. 2005. Their eq. 2.

    It's been updated with our beta.

    '''

    rc2 = rc * rc

    beta = -3.0 * beta + 0.5

    area1 = np.pi * rc2 / (1 - beta) * ((1 + r_start**2 / rc2)**(1 - beta) - 1)
    area2 = np.pi * rc2 / (1 - beta) * ((1 + r**2 / rc2)**(1 - beta) - 1)

    return area2 - area1

def inv_beta_model(So, rc, beta, bkg):
    ''' Solves for where (radius) the beta model is equal to the background

    Make sure you keep track of the units going into all this. You wanna make
    sure the radius comes out as something that makes sense.

    '''

    a = -3 * beta + 0.5

    b = (bkg / So)**(1 / a) - 1

    c = b * rc**2

    return np.sqrt(c)

def chi2(model, y, y_err):
    '''Chi error. We are going to square this to make it the chi2 error.'''
    return np.sum(((model - y) / y_err)**2)

def like(theta, x, y, yerr):
    So, rc, beta, bg = theta
    model = beta_model(So, x, rc, beta) + bg
    #return -0.5 * np.sum(np.log(2 * np.pi * yerr) + (y - model)**2 / yerr)
    return -chi2(model, y, yerr)

def prior(theta):
    So, rc, beta, bg = theta
    if So < 0:
        return -np.inf
    elif rc < 0 or rc > 10:
        return -np.inf
    elif beta < 0 or beta > 3:
        return -np.inf
    elif bg < 0 or bg > 1:
        return -np.inf
    else:
        return 0.0

def prob(theta, x, y, yerr):
    lp = prior(theta)
    if not np.isfinite(lp):
        return -np.inf
    return lp + like(theta, x, y, yerr)


import numpy as np

class LinearRegression:

    def __init__(self, iterations, learning_rate):
        self.iterations = iterations
        self.learning_rate = learning_rate

    def fit(self, X, Y):

        # m = training no. , n = features no.
        self.m , self.n = X.shape

        # Parameters Initialization
        self.W = np.zeros(self.n)

        # Bias
        self.b = 0
        self.X = X
        self.Y = Y

        for i in range(self.iterations):
            self.update_weights()
        return self

    def update_weights(self):
        #  Gradient Descent
        Y_pred = self.predict(self.X)

        #  Calculate Gradients
        dW = - ( 2 * (self.X.T).dot(self.Y - Y_pred) ) / self.m
        db = - ( 2 * np.sum(self.Y - Y_pred) ) / self.m

        #  Update W
        self.W = self.W - self.learning_rate * dW
        self.b = self.b - self.learning_rate * db

        return self

    def predict(self, X):
        return X.dot(self.W) + self.b


import numpy as np
import pandas as pd

def gini(actual, predicted):
    """
    Calculates Gini

    :param actual: array-like binary target
    :param predicted: array-like predicted values
    :return: float Gini value
    """
    a = np.asarray(np.c_[actual, predicted, np.arange(len(actual))], dtype=np.float)
    a = a[np.lexsort((a[:, 2], -1 * a[:, 1]))]
    total_losses = a[:, 0].sum()
    gini_sum = a[:, 0].cumsum().sum() / total_losses
    gini_sum -= (len(actual) + 1) / 2.
    return gini_sum / len(actual)

def gini_normalized(actual, predicted):
    """
    Calculates Normalized Gini

    :param actual: array-like binary target
    :param predicted: array-like predicted values
    :return: float normalized gini value in 0-1 range
    """
    return gini(actual, predicted) / gini(actual, actual)

def iv_table(x, y):
    """
    Calculates WoE-IV table

    :param x: array-like feature
    :param y: array-like binary target
    :return: Pandas DataFrame of WoE-IV statistics
    """
    table = pd.crosstab(x, y, normalize='columns') \
        .assign(woe=lambda dfx: np.log(dfx[1] / dfx[0])) \
        .assign(iv=lambda dfx: np.sum(dfx['woe'] * (dfx[1] - dfx[0])))
    return table