import pycuda.autoinit
import pycuda.driver as drv
from pycuda import gpuarray
from pycuda.compiler import SourceModule
import numpy as np

# https://docs.nvidia.com/cuda/cuda-math-api/index.html

MonteCarloKernelTemplate = '''
#include <curand_kernel.h>

#define ULL  unsigned long long
#define _R(z)   ( 1.0f / (z) )
#define _P2(z)   ( (z) * (z) )

// p stands for "precision" (single or double)
__device__ inline %(p)s  f(%(p)s x)
{
     %(p)s y;

     %(math_function)s;

    return y;
}

extern "C" {
__global__ void monte_carlo(int iters, %(p)s lo, %(p)s hi, %(p)s * ys_out)
{
    curandState cr_state;

	int tid = blockIdx.x * blockDim.x + threadIdx.x;

    int num_threads = blockDim.x * gridDim.x;

    %(p)s t_width = (hi - lo) / ( %(p)s ) num_threads;

    %(p)s density = ( ( %(p)s ) iters ) / t_width;

    %(p)s t_lo = t_width*tid + lo;
    %(p)s t_hi = t_lo + t_width;

	curand_init( (ULL)  clock() + (ULL) tid, (ULL) 0, (ULL) 0, &cr_state);

     %(p)s y, y_sum = 0.0f;

     %(p)s rand_val, x;
     for (int i=0; i < iters; i++)
     {
         rand_val = curand_uniform%(p_curand)s(&cr_state);

         x = t_lo + t_width * rand_val;

         y_sum += f(x);
     }

     y = y_sum / density;

     ys_out[tid] = y;
}

}
'''

class MonteCarloIntegrator:

    def __init__(self, math_function='y = sin(x)', precision='d', lo=0, hi=np.pi, samples_per_thread=10**5, num_blocks=100):

        self.math_function = math_function

        if precision in [None, 's', 'S', 'single', np.float32]:
            self.precision = 'float'
            self.numpy_precision = np.float32
            self.p_curand = ''
        elif precision in ['d','D', 'double', np.float64]:
            self.precision = 'double'
            self.numpy_precision = np.float64
            self.p_curand = '_double'
        else:
            raise Exception('precision is invalid datatype!')

        if (hi - lo <= 0):
            raise Exception('hi - lo <= 0!')
        else:
            self.hi = hi
            self.lo = lo

        MonteCarloDict = {'p' : self.precision, 'p_curand' : self.p_curand, 'math_function' : self.math_function}

        self.MonteCarloCode = MonteCarloKernelTemplate % MonteCarloDict

        self.ker = SourceModule(no_extern_c=True , options=['-w'], source=self.MonteCarloCode)

        self.f = self.ker.get_function('monte_carlo')

        self.num_blocks = num_blocks

        self.samples_per_thread = samples_per_thread

    def definite_integral(self, lo=None, hi=None, samples_per_thread=None, num_blocks=None):

        if lo is None or hi is None:
            lo = self.lo
            hi = self.hi

        if samples_per_thread is None:
            samples_per_thread = self.samples_per_thread

        if num_blocks is None:
            num_blocks = self.num_blocks
            grid = (num_blocks,1,1)
        else:
            grid = (num_blocks,1,1)

        block = (32,1,1)

        num_threads = 32*num_blocks

        self.ys = gpuarray.empty((num_threads,) , dtype=self.numpy_precision)

        self.f(np.int32(samples_per_thread), self.numpy_precision(lo), self.numpy_precision(hi), self.ys, block=block, grid=grid)

        self.nintegral = np.sum(self.ys.get() )

        return np.sum(self.nintegral)

if __name__ == '__main__':

    integral_tests = [('y =log(x)*_P2(sin(x))', 11.733 , 18.472, 8.9999), ('y = _R( 1 + sinh(2*x)*_P2(log(x)) )', .9, 4, .584977), ('y = (cosh(x)*sin(x))/ sqrt( pow(x,3) + _P2(sin(x)))', 1.85, 4.81,  -3.34553) ]

    for f, lo, hi, expected in integral_tests:
        mci = MonteCarloIntegrator(math_function=f, precision='d', lo=lo, hi=hi)
        print('The Monte Carlo numerical integration of the function\n \t f: x -> %s \n \t from x = %s to x = %s is : %s ' % (f, lo, hi, mci.definite_integral()))
        print('where the expected value is : %s\n' % expected)


# <nbformat>2</nbformat>

# <markdowncell>

# # Eigenvalue distribution of Gaussian orthogonal random matrices

# <markdowncell>

# The eigenvalues of random matrices obey certain statistical laws. Here we construct random matrices
# from the Gaussian Orthogonal Ensemble (GOE), find their eigenvalues and then investigate the nearest
# neighbor eigenvalue distribution $\rho(s)$.

# <codecell>

from rmtkernel import ensemble_diffs, normalize_diffs, GOE
import numpy as np
from IPython.parallel import Client

# <markdowncell>

# ## Wigner's nearest neighbor eigenvalue distribution

# <markdowncell>

# The Wigner distribution gives the theoretical result for the nearest neighbor eigenvalue distribution
# for the GOE:
#
# $$\rho(s) = \frac{\pi s}{2} \exp(-\pi s^2/4)$$

# <codecell>

def wigner_dist(s):
    """Returns (s, rho(s)) for the Wigner GOE distribution."""
    return (np.pi*s/2.0) * np.exp(-np.pi*s**2/4.)

# <codecell>

def generate_wigner_data():
    s = np.linspace(0.0,4.0,400)
    rhos = wigner_dist(s)
    return s, rhos

# <codecell>

s, rhos = generate_wigner_data()

# <codecell>

plot(s, rhos)
xlabel('Normalized level spacing s')
ylabel('Probability $\rho(s)$')

# <markdowncell>

# ## Serial calculation of nearest neighbor eigenvalue distribution

# <markdowncell>

# In this section we numerically construct and diagonalize a large number of GOE random matrices
# and compute the nerest neighbor eigenvalue distribution. This comptation is done on a single core.

# <codecell>

def serial_diffs(num, N):
    """Compute the nearest neighbor distribution for num NxX matrices."""
    diffs = ensemble_diffs(num, N)
    normalized_diffs = normalize_diffs(diffs)
    return normalized_diffs

# <codecell>

serial_nmats = 1000
serial_matsize = 50

# <codecell>

%timeit -r1 -n1 serial_diffs(serial_nmats, serial_matsize)

# <codecell>

serial_diffs = serial_diffs(serial_nmats, serial_matsize)

# <markdowncell>

# The numerical computation agrees with the predictions of Wigner, but it would be nice to get more
# statistics. For that we will do a parallel computation.

# <codecell>

hist_data = hist(serial_diffs, bins=30, normed=True)
plot(s, rhos)
xlabel('Normalized level spacing s')
ylabel('Probability $P(s)$')

# <markdowncell>

# ## Parallel calculation of nearest neighbor eigenvalue distribution

# <markdowncell>

# Here we perform a parallel computation, where each process constructs and diagonalizes a subset of
# the overall set of random matrices.

# <codecell>

def parallel_diffs(rc, num, N):
    nengines = len(rc.targets)
    num_per_engine = num/nengines
    print "Running with", num_per_engine, "per engine."
    ar = rc.apply_async(ensemble_diffs, num_per_engine, N)
    diffs = np.array(ar.get()).flatten()
    normalized_diffs = normalize_diffs(diffs)
    return normalized_diffs

# <codecell>

client = Client()
view = client[:]
view.run('rmtkernel.py')
view.block = False

# <codecell>

parallel_nmats = 40*serial_nmats
parallel_matsize = 50

# <codecell>

%timeit -r1 -n1 parallel_diffs(view, parallel_nmats, parallel_matsize)

# <codecell>

pdiffs = parallel_diffs(view, parallel_nmats, parallel_matsize)

# <markdowncell>

# Again, the agreement with the Wigner distribution is excellent, but now we have better
# statistics.

# <codecell>

hist_data = hist(pdiffs, bins=30, normed=True)
plot(s, rhos)
xlabel('Normalized level spacing s')
ylabel('Probability $P(s)$')


from math import inf, isnan
import numpy as np

def xy_to_complex(xy):
    return xy[0] + 1j*xy[1]

def complex_to_xy(comp):
    return [np.real(comp), np.imag(comp)]

def complex_plane(width=1000, height=1000, centered=True):
    x, x_step = np.linspace(0, 1, num=width, endpoint=False, retstep=True)
    y, y_step = np.linspace(0, 1, num=height, endpoint=False, retstep=True)
    if centered:
        x += 0 if isnan(x_step) else (x_step / 2)
        y += 0 if isnan(y_step) else (y_step / 2)
    return x[np.newaxis,:] + 1j*y[:,np.newaxis]

def distance_plane(complex_plane, point):
    return np.abs(complex_plane - point)

def waveform_plane(distance_plane, cycle_count=inf):
    source_phase = np.pi/2 - (cycle_count % 1) * 2*np.pi if cycle_count != inf else 0
    phase_plane = distance_plane * 2*np.pi + source_phase
    wave = np.exp(1j*phase_plane)
    np.putmask(wave, distance_plane > cycle_count, 0)
    return wave

def source_waveform_plane(complex_plane, source, wavelength=1, cycle_count=inf, spreading=True, min_spreading_distance=1e-14):
    distance = distance_plane(complex_plane, source)
    source_waveform_plane = waveform_plane(distance / wavelength, cycle_count)
    if spreading: source_waveform_plane /= np.fmax(distance, min_spreading_distance)
    return source_waveform_plane


import numpy as np

def gaussian( stdx, stdy, A, x, y ):

    return A * np.exp( -1. * ( x*x/(2*stdx*stdx) + y*y/(2*stdy*stdy) ))

A = 1.
stdx = .1
stdy = .1
nx = 5
ny = 5
shiftx = -2
shifty = -2
x = np.linspace(-.4, .4, nx)
y = np.linspace(-.4, .4, ny)

xv, yv = np.meshgrid(x,y, sparse=False, indexing='ij')

f = open('handoff_scalar.dat','w')
f.write('gaussian variable\n')
f.write('0.2 0.2\n')
f.write(np.str(nx*ny)+'\n')

for i in range(nx):
    for j in range(ny):

        g = gaussian(stdx, stdy, A, xv[i][j], yv[i][j])
        to_write = '0 '+np.str(i+shiftx)+' '+np.str(j+shifty)+' '+np.str(g)+'\n'
        f.write(to_write)

f.close()

f = open('handoff_velocity.dat','w')
f.write('gaussian_variable\n')
f.write('0.2 0.2\n')
f.write(np.str(nx*ny)+'\n')

for i in range(nx):
    for j in range(ny):

        g = gaussian(stdx, stdy, A, xv[i][j], yv[i][j])
        to_write = '0 '+np.str(i+shiftx)+' '+np.str(j+shifty)+' '+np.str(g)+' 1. 1.\n'
        f.write(to_write)

f.close()


import numpy as np
x = np.array([6,8,12,14,18])
y = np.array([7,9,13,15,19])

def gradiant_descent(x,y):
    m = c = 0
    loop = 10
    alpha = 0.3
    n = len(x)
    for i in range(loop):
        yp = m*x + c
        costfunc = (1/n)*sum(val*val for val in (y-yp))
        dm = -(2/n)*sum(x*(y-yp))
        dn = -(2/n)*sum(y-yp)
        m = m - alpha*dm
        c = c - alpha*dn
        print(f"m:{m} c:{c} y:{yp} cost: {costfunc}")

gradiant_descent(x,y)


#Python3 Steven
#common proability distrubutions
#import math
import numpy as np
from scipy import integrate
from scipy.special import gamma,beta,factorial
#from sklearn.preprocessing import Normalizer
#https://en.wikipedia.org/wiki/Gamma_function
#https://en.wikipedia.org/wiki/Beta_function
#https://en.wikipedia.org/wiki/Factorial
from permutationAndCom import combinat #,permut

#---------------------------Discrete distribution------------------------------#
def Discrete_uniform_distribution(x, N=5):#https://en.wikipedia.org/wiki/Discrete_uniform_distribution
    return np.zeros(len(x)) + 1/N

def Binomial_distribution(N, p=1): #https://en.wikipedia.org/wiki/Binomial_distribution
    assert N>=1
    assert 0<=p<=1
    def Binomial(k):
        return combinat(N,k)*np.power(p,k)*np.power(1-p,N-k)

    return list(map(Binomial, range(N)))

def Geometric_distribution(N, p):#https://en.wikipedia.org/wiki/Geometric_distribution
    def Geometric(k):
        return np.power(1-p,k)*p

    return list(map(Geometric, range(N)))

def Hypergeometric_distribution(N, K, n): #https://en.wikipedia.org/wiki/Hypergeometric_distribution
    def Hypergeometric(k):
        return combinat(K,k)*combinat(N-K,n-k)/combinat(N,n)

    return list(map(Hypergeometric, range(n)))

def Poisson_distribution(N=20, lam=1): #https://en.wikipedia.org/wiki/Poisson_distribution
    def Poisson(k):
        return np.power(lam,k)*np.power(np.e, -1*lam)/factorial(k)

    return list(map(Poisson, range(N)))

def ZipfsLaw(N=10, s=1):#https://en.wikipedia.org/wiki/Zipf%27s_law
    v = np.sum(1/np.power([i+1 for i in range(N)],s))
    def ZipfLaw(k):
        return (1/np.power(k,s))/v

    return list(map(ZipfLaw, [i+1 for i in range(N)]))

def Beta_binomial_distribution(N=10, alpha=0.2, bta=0.25): #https://en.wikipedia.org/wiki/Beta-binomial_distribution
    def Beta_binomial(k):
        return combinat(N,k)*beta(k+alpha, N-k+bta)/beta(alpha,bta)

    return list(map(Beta_binomial, range(N)))

def Logarithmic_distribution(N, p=0.33): #https://en.wikipedia.org/wiki/Logarithmic_distribution
    def Logarithmic(k):
        return -1*np.power(p,k)/(np.log(1-p)*k)

    return list(map(Logarithmic, [i+1 for i in range(N)]))

def Conway_Maxwell_Poisson_distribution(N=20, lam=1, v=1.5): # https://en.wikipedia.org/wiki/Conway%E2%80%93Maxwell%E2%80%93Poisson_distribution
    def Conway(k):
        return np.power(lam,k)/np.power(factorial(k),v)

    max_num = 10000
    z = np.sum(list(map(Conway, range(max_num))))
    res = list(map(Conway, range(N)))
    return res/z

def Modified_Bessel_functions(x, alpha=0): #https://en.wikipedia.org/wiki/Bessel_function#Modified_Bessel_functions
    def Bessel(k):
        return 1/(factorial(k)*gamma(k+alpha+1))*np.power(0.2*x,2*k+alpha)

    max_num = 10000
    return np.sum(list(map(Bessel, range(max_num))))

def Skellam_distribution(N=8, u1=1, u2=1): #https://en.wikipedia.org/wiki/Skellam_distribution
    def Skellam(k):
        return np.exp(-u1-u2)*np.power(u1/u2, 0.5*k)*Modified_Bessel_functions(x=2*np.sqrt(u1*u2),alpha=k)
    return list(map(Skellam, [i-N/2 for i in range(N+1)]))

def Yule_Simon_distribution(N=20, ru=0.25): #https://en.wikipedia.org/wiki/Yule%E2%80%93Simon_distribution
    def Yule_Simon(k):
        return ru*beta(k, ru+1)
    return list(map(Yule_Simon, range(1, N+1)))

def Riemann_zeta_function(s=2):#https://en.wikipedia.org/wiki/Riemann_zeta_function
    def Riemann_zeta(k):
        return 1/np.power(k,s)

    max_num = 100
    return np.sum(list(map(Riemann_zeta, range(1, max_num+1))))

def Zeta_distribution(N=16, s=2): #https://en.wikipedia.org/wiki/Zeta_distribution
    def Zeta(k):
        return 1/(Riemann_zeta_function(s=s)*np.power(k,s))
    return list(map(Zeta, range(1, N+1)))

#---------------------------Continuous distribution------------------------------#
def Uniform_distribution(N, a=1, b=3): #https://en.wikipedia.org/wiki/Uniform_distribution_(continuous)
    return np.zeros(N) + 1/(b-a)

def Error_function(x): #https://en.wikipedia.org/wiki/Error_function
    def erf(t):
        return (2/np.sqrt(np.pi))*np.exp(-t**2)
    return integrate.quad(erf, 0, x)[0]

def NormalDistribution(x, delta=1, u=0): #https://en.wikipedia.org/wiki/Normal_distribution
    return (1/(delta*np.sqrt(2*np.pi)))*np.exp(-0.5*((x-u)/delta)**2)

def Cauchy_pdf(x, x0=0, scaler=1):#https://en.wikipedia.org/wiki/Cauchy_distribution
    return (1/(np.pi*scaler))*(scaler**2/((x-x0)**2+scaler**2))

def Generalized_logistic_distribution(x, alpha=1):#https://en.wikipedia.org/wiki/Generalized_logistic_distribution
    return alpha*np.exp(-1*x)/np.power(1+np.exp(-1*x),alpha+1)

def Gumbel_distribution(x, u=0, belta=1):#https://en.wikipedia.org/wiki/Gumbel_distribution
    z = (x-u)/belta
    return np.exp(-1*(z + np.exp(-1*z)))/belta

def Laplace_distribution(x, u=0, b=1):#https://en.wikipedia.org/wiki/Laplace_distribution
    return np.exp(-1*np.abs(x-u)/b)

def Logistic_distribution(x, u=0, s=1):#https://en.wikipedia.org/wiki/Logistic_distribution
    z = np.exp(-1*(x-u)/s)
    return z/(s*(1+z)**2)

def Gamma_distribution(x, k=1, theta=2): #https://en.wikipedia.org/wiki/Gamma_distribution
    return np.power(x,k-1)*np.exp(-1*x/theta)/(gamma(k)*np.power(theta,k))

def StudentT_distribution(x, v=1):#https://en.wikipedia.org/wiki/Student%27s_t-distribution
    z = np.power(1+x**2/v, -0.5*(v+1))
    return z*gamma((v+1)/2)/(gamma(v/2)*np.sqrt(np.pi*v))
    #return z/(beta(0.5, v/2)*np.sqrt(v))

def Log_normal_distribution(x, delta=1, u=0):#https://en.wikipedia.org/wiki/Log-normal_distribution
    z = -1*(np.log(x)-u)**2/(2*delta**2)
    return (1/(x*delta*np.sqrt(2*np.pi)))* np.exp(z)

def Weibull_distribution(x, lamda=1, k=1): #https://en.wikipedia.org/wiki/Weibull_distribution
    y = np.zeros((len(x),))
    l = np.where(x < 0)
    if len(l) != 0:
        y[l[0]]=0

    l = np.where(x >= 0)
    if len(l) != 0:
        y[l[0]] = (k/lamda)*np.power(x[l[0]]/lamda,k-1)*np.exp(-1*np.power(x[l[0]]/lamda,k))
    return y

def Pareto_distribution(x, alpha=1, Xm=1):#https://en.wikipedia.org/wiki/Pareto_distribution
    y = np.zeros((len(x),))
    l = np.where(x < Xm)
    if len(l) != 0:
        y[l[0]]=0

    l = np.where(x >= Xm)
    if len(l) != 0:
        y[l[0]] = alpha*np.power(Xm,alpha)/np.power(x[l[0]],alpha+1)
    return y

def Rayleigh_distribution(x, delta=0.5):#https://en.wikipedia.org/wiki/Rayleigh_distribution
    y = np.zeros((len(x),))
    l = np.where(x >= 0)
    if len(l) != 0:
        y[l[0]] = (x/delta**2)*np.exp(-1*x**2/(2*delta**2))
    return y

def Beta_distribution(x, alpha=0.5, bta=0.5): #https://en.wikipedia.org/wiki/Beta_distribution
    z = gamma(alpha)*gamma(bta)/gamma(alpha+bta)
    return np.power(x, alpha-1)*np.power(1-x, bta-1)/z

def Chi_distribution(x, k=1): #https://en.wikipedia.org/wiki/Chi_distribution
    return (2/ (np.power(2,k/2)* gamma(k/2)))*np.power(x,k-1)*np.exp(-0.5*x**2)

def Erlang_distribution(x, k=1, u=2.0): #https://en.wikipedia.org/wiki/Erlang_distribution
    return np.power(x,k-1)*np.exp(-1*x/u)/(np.power(u,k)*factorial(k-1))

def Exponential_distribution(x, lam=1): #https://en.wikipedia.org/wiki/Exponential_distribution
    return lam*np.exp(-1*lam*x)

def Von_Mises_distribution(x, u=0, k=0): #https://en.wikipedia.org/wiki/Von_Mises_distribution
    return np.exp(k*np.cos(x-u))/(2*np.pi*Modified_Bessel_functions(alpha=0,x=k))

def Boltzmann_distribution(x, a=1): #https://en.wikipedia.org/wiki/Maxwell%E2%80%93Boltzmann_distribution
    return x**2*np.sqrt(2/np.pi)*np.exp(-0.5*x**2/a**2)/a**3

def logit(x):#https://en.wikipedia.org/wiki/Logit
    return np.log(x/(1-x))

def Logit_normal_distribution(x, deta=0.2, u=0): #https://en.wikipedia.org/wiki/Logit-normal_distribution
    z = np.exp(-1*((logit(x)-u)**2)/(2*deta**2))
    return z/(deta*np.sqrt(2*np.pi)*x*(1-x))

def Irwin_Hall_distribution(x,n=1): #https://en.wikipedia.org/wiki/Irwin%E2%80%93Hall_distribution
    def Irwin_Hall(x):
        def f(k):
            return np.power(-1,k)*combinat(n,k)*np.power(x-k,n-1)
        #print('x_=',np.floor(x))
        NN = int(np.floor(x))
        res = np.array(list(map(f, range(NN+1))))
        res = np.sum(res)*(1/factorial(n-1))
        return res

    return list(map(Irwin_Hall, list(x)))

def Bates_distribution(x, n=1, a=0, b=1): #https://en.wikipedia.org/wiki/Bates_distribution
    def sgn(x, k):
        if x < k:
            return -1
        elif x == k:
            return 0
        return 1

    def Bates(x):
        def f(k):
            return np.power(-1, k)*combinat(n,k)*np.power((x-a)/(b-a)-k/n,n-1)*sgn((x-a)/(b-a),k/n)
        return np.sum(list(map(f, range(n))))

    return list(map(Bates, list(x)))

def Kumaraswamy_distribution(x, a=0.5, b=0.5): #https://en.wikipedia.org/wiki/Kumaraswamy_distribution
    return a*b*np.power(x,a-1)*np.power(1-np.power(x,a),b-1)

def PERT_distribution(x, a=0, b=10, c=100): #https://en.wikipedia.org/wiki/PERT_distribution
    alpha = 1+4*(b-a)/(c-a)
    bta = 1+4*(c-b)/(c-a)
    return np.power(x-a, alpha-1)*np.power(c-x, bta-1)/(beta(alpha, bta)*np.power(c-a, alpha+bta-1))

def Reciprocal_distribution(x, a=1, b=4): #https://en.wikipedia.org/wiki/Reciprocal_distribution
    return 1/x*np.log(b/a)

def Triangular_distribution(x, a=1, c=3, b=4): #https://en.wikipedia.org/wiki/Triangular_distribution
    assert(a < c < b)
    y = np.zeros((len(x),))
    l = np.where(x > b)
    if len(l) != 0:
        y[l[0]]=0

    l = np.where(x <= b)
    if len(l) != 0:
        y[l[0]]= 2*(b-x[l[0]])/((b-a)*(b-c))
    l = np.where(x <= c)
    if len(l) != 0:
        y[l[0]]= 2*(x[l[0]]-a)/((b-a)*(c-a))
    l = np.where(x < a )
    if len(l) != 0:
        y[l[0]]= 0
    return y

def Trapezoidal_distribution(x, a=-3, b=-2, c=-1, d=0): #https://en.wikipedia.org/wiki/Trapezoidal_distribution
    assert(a<b<c<d)
    y = np.zeros((len(x),))
    l = np.where(x <= d)
    if len(l) != 0:
        y[l[0]]=2*(d-x[l[0]])/((d+c-a-b)*(d-c))

    l = np.where(x <= c)
    if len(l) != 0:
        y[l[0]]=2/(d+c-a-b)

    l = np.where(x < b)
    if len(l) != 0:
        y[l[0]]=2*(x[l[0]]-a)/((d+c-a-b)*(b-a))
    return y

def PIFuc(x):
    return 0.5*(1+Error_function(x/np.sqrt(2)))

def Truncated_normal_distribution(x, a=-10, b=10, u=-8, deta=2): #https://en.wikipedia.org/wiki/Truncated_normal_distribution
    return (1/deta)*NormalDistribution((x-u)/deta)/(PIFuc((b-u)/deta)-PIFuc((a-u)/deta))

def Wigner_semicircle_distribution(x, r=0.25): #https://en.wikipedia.org/wiki/Wigner_semicircle_distribution
    return 2*np.sqrt(r**2-x**2)/(np.pi*r**2)

def Dagum_distribution(x, b=1, p=1, a=0.5): #https://en.wikipedia.org/wiki/Dagum_distribution
    z1=(a*p/x)*np.power(x/b, a*p)
    z2 = np.power(np.power(x/b, a)+1, p+1)
    return z1/z2

def F_distribution(x,d1=1,d2=1): #https://en.wikipedia.org/wiki/F-distribution
    z = np.power(d1*x,d1)*np.power(d2,d2)/np.power(x*d1+d2,d1+d2)
    return np.sqrt(z)/(x*beta(0.5*d1,0.5*d2))

def Folded_normal_distribution(x, u=1, deta=1): #https://en.wikipedia.org/wiki/Folded_normal_distribution
    z=1/(deta*np.sqrt(2*np.pi))
    z1 = z*np.exp(-0.5*(x-u)**2/deta**2)
    z2 = z*np.exp(-0.5*(x+u)**2/deta**2)

    y = z1+z2
    l = np.where(x < 0)
    if len(l) != 0:
        y[l[0]]=0
    return y

def Frechet_distribution(x, alpha=1, s=1, m=0): #https://en.wikipedia.org/wiki/Fr%C3%A9chet_distribution
    z=(x-m)/s
    z1=np.power(z, -1-alpha)
    z2 = np.exp(-1*np.power(z, -1*alpha))
    return (alpha/s)*z1*z2

def Gompertz_distribution(x, eta=0.1, b=1): #https://en.wikipedia.org/wiki/Gompertz_distribution
    z=eta+b*x-eta*np.exp(b*x)
    return b*eta*np.exp(z)

def Half_normal_distribution(x, delta=1): #https://en.wikipedia.org/wiki/Half-normal_distribution
    return np.sqrt(2)/(delta*np.sqrt(np.pi))*np.exp(-0.5*x**2/delta**2)

def Nakagami_distribution(x, m=0.5, w=1): #https://en.wikipedia.org/wiki/Nakagami_distribution
    z = gamma(m)*np.power(w,m)
    return (2*np.power(m,m)/z)*np.power(x,2*m-1)*np.exp(-1*m*x**2/w)

def Levy_distribution(x, u=0, c=0.5): #https://en.wikipedia.org/wiki/L%C3%A9vy_distribution
    return np.sqrt(0.5*c/np.pi)*np.exp(-0.5*c/(x-u))/np.power(x-u, 1.5)

def Shifted_Gompertz_distribution(x, b=0.4, ni=0.01): #https://en.wikipedia.org/wiki/Shifted_Gompertz_distribution
    z=np.exp(-1*b*x)
    return b*z*np.exp(-1*ni*z)*(1+ni*(1-z))

def JohnsonSU_distribution(x, gama=-2, deta=2, xi=1.1, lam=1.5): #https://en.wikipedia.org/wiki/Johnson%27s_SU-distribution
    z1=deta/(lam*np.sqrt(2*np.pi))
    z2=1/np.sqrt(1+((x-xi)/lam)**2)
    z3=np.exp(-0.2*(gama+deta*np.arcsinh((x-xi)/lam))**2)
    return z1*z2*z3

def Landau_distribution(x, u=0, c=1): #https://en.wikipedia.org/wiki/Landau_distribution
    def getFunctionSympy(t,x=x,u=u,c=c):
        #print('x,u,c=',x,u,c)
        y=(1/np.pi*c)*np.exp(-t)*np.cos(t*((x-u)/c)+ 2*t/np.pi*np.log(t/c))
        #y=np.exp(-t)
        return y

    return integrate.quad(getFunctionSympy, 0, np.inf)[0]

def Stable_distribution(x, beta=0, c=1, alpha=0.5): #https://en.wikipedia.org/wiki/Stable_distribution
    def sgnArray(x,k):
        return np.where(x>k, 1, 0) + np.where(x<k, -1, 0)

    z1 = np.power(np.abs(x),1+alpha)
    z2 = np.power(c,alpha)*(1+sgnArray(x,0)*beta)*np.sinh(0.5*np.pi*alpha)
    z3 = gamma(alpha+1)/np.pi
    res = z2*z3/z1
    # res = res.reshape(1,-1)
    # res = Normalizer().fit(res).transform(res)
    # res = res.reshape((len(x)))
    # print(res.shape,len(x))
    return res

def PIFuc2(x):
    #return 0.5*(1+Error_function(x/np.sqrt(2)))
    return np.array(list(map(PIFuc, list(x))))

def Skew_normal_distribution(x, alpha=-4): #https://en.wikipedia.org/wiki/Skew_normal_distribution
    return 2* NormalDistribution(x)*PIFuc2(alpha*x)

def Noncentral_t_distribution(x, v=1): #https://en.wikipedia.org/wiki/Noncentral_t-distribution
    return StudentT_distribution(x,v=v)


import math
import pandas as pd
import numpy as np
import scipy as sp
import matplotlib.pyplot as plt

#We will draw samples from phi(x)*I_{x>c}

#This is the _err_ function needed for the Gaussian distribution CDF
def err(x):
    err_x = 0
    for i in range(125):
        err_x += ((-1.0)**i) * (x**((2.0*i)+1))/(math.factorial(i)*((2*i)+1))
    err_x = (2.0/math.sqrt(math.pi))*err_x
    return err_x

#Approximation to Gaussian Distribution CDF
def Phi(x):
    return 0.5*(1.0+err(x/math.sqrt(2.0)))

#Inverve of Exponential CDF with parameter lambda_0
def inverse_exp(x,lambda_0):
    return -1.0*math.log(1.0-x)/float(lambda_0)

#Exponential density
def expo(x,lambda_0):
    return lambda_0*math.exp(-lambda_0*x)

#Gaussian density
def phi(x):
    return math.exp((-(x**2.0))/2.0)/math.sqrt(2.0*math.pi)

def truncated_gaussian_sampler(n,c):
	Phi_c = Phi(c)
	lambda_0 = (c + math.sqrt((c**2.0)+4.0))/2.0
	b = math.exp(((lambda_0**2.0) - (2.0*lambda_0*c))/2.0)/(math.sqrt(2.0*math.pi)*lambda_0*(1.0-Phi_c))
	const = ((1-Phi_c)*b)
	samples = []
	while len(samples)<n:
		x_inv = np.random.uniform()
		x = inverse_exp(x_inv, lambda_0)
		r = (phi(x+c)/(expo(x,lambda_0))*const)
		coin = np.random.uniform()
		if coin<r:
			samples.append(x+c)
	return np.array(samples)

n=1
c = 4
print(truncated_gaussian_sampler(n,c))


#!/usr/bin/env python3
import numpy as np
import matplotlib.pyplot as plt

y0 = np.arange(0, 11) ** 3

mean = [69, 0]
cov = [[15, 8], [8, 15]]
np.random.seed(5)
x1, y1 = np.random.multivariate_normal(mean, cov, 2000).T
y1 += 180

x2 = np.arange(0, 28651, 5730)
r2 = np.log(0.5)
t2 = 5730
y2 = np.exp((r2 / t2) * x2)

x3 = np.arange(0, 21000, 1000)
r3 = np.log(0.5)
t31 = 5730
t32 = 1600
y31 = np.exp((r3 / t31) * x3)
y32 = np.exp((r3 / t32) * x3)

np.random.seed(5)
student_grades = np.random.normal(68, 15, 50)

fig = plt.figure(tight_layout=True)
plt.suptitle('All in One')

fig.add_subplot(321)
plt.xlim(0, 10)
plt.plot(y0, 'r-')

fig.add_subplot(322)
plt.title("Men's Height vs Weight", fontsize="x-small")
plt.xlabel("Height (in)", fontsize="x-small")
plt.ylabel("Weight (lbs)", fontsize="x-small")
plt.plot(x1, y1, color="magenta", marker=".", linestyle="")

fig.add_subplot(323)
plt.title("Exponential Decay of C-14", fontsize="x-small")
plt.xlabel("Time (years)", fontsize="x-small")
plt.ylabel("Fraction Remaining", fontsize="x-small")
plt.xlim(0, max(x2))
plt.plot(x2, y2)
plt.yscale('log')

fig.add_subplot(324)
plt.title("Exponential Decay of C-14", fontsize="x-small")
plt.xlabel("Time (years)", fontsize="x-small")
plt.ylabel("Fraction Remaining", fontsize="x-small")
plt.ylim(bottom=0)
plt.xlim(0, max(x3))
plt.plot(x3, y31, label="C-14", linestyle="dashed", color="red")
plt.plot(x3, y32, label="Ra-226", color="green")
plt.legend(loc='upper right', fontsize='x-small')

fig.add_subplot(313)
plt.xlim(0, 100)
plt.ylim(0, 30)
plt.hist(student_grades, range=(0, 100), edgecolor='black')
plt.title("Project A", fontsize='x-small')
plt.xlabel("Grades", fontsize='x-small')
plt.ylabel("Number of Students", fontsize='x-small')

plt.show()


import numpy as np

def sigmoid(x):
  return 1 / (1 + np.exp(-x))

training_inputs = np.array([ [0,0,1], [1,1,1], [1,0,1], [0,1,1] ])
training_outputs = np.array([[0,1,1,0]]).T

np.random.seed(1)

synaptic_weights = 2 * np.random.random((3,1)) - 1

print ("Սկզբնարժեքավորման պատահական կշիռներ։")
print(synaptic_weights)
print()

# Ետ տարածման մեթոդ
for i in range(200000):
  input_layer = training_inputs
  outputs = sigmoid( np.dot( input_layer, synaptic_weights ))

  err = training_outputs - outputs
  adjustments = np.dot( input_layer.T, err * ( outputs * ( 1-outputs )) )

  synaptic_weights += adjustments

print( "Կշիռները սովորեցնելուց հետո ։" )
print(synaptic_weights)
print()

print( "Արդյունքը սովորեցնելուց հետո ։" )
print(outputs)

new_inputs = np.array([1, 1, 0])
output = sigmoid( np.dot( new_inputs, synaptic_weights ))
print()

print("Նոր իրադարձություն ։")
print(new_inputs, " ==> ", output)


# Custom Functions
import numpy as np
import pandas as pd

def ATransformMatrix (theta): #A
    ATransform = np.array([[np.cos(theta), -np.sin(theta)],
                            [np.sin(theta), np.cos(theta)]])
    return ATransform

def ATransformMatrixTHETA (theta): #A_theta
    ATransformTHETA = np.array([[-np.sin(theta), -np.cos(theta)],
                                [np.cos(theta), -np.sin(theta)]])
    return ATransformTHETA

def prettyMatVect(matVect):
    prettyMatVect = pd.DataFrame(matVect, columns =
                                ['R1x', 'R1y', 'theta1',
                                'R2x','R2y','theta2',
                                'R3x','R3y','theta3',
                                'R4x','R4y','theta4'])
    return prettyMatVect

def calcGenCoor(qi, u_bar_2O, u_bar_2A, u_bar_3A, u_bar_3B, u_bar_4B):
    # To calculate Point of Interest positions
    # GLOBAL COORDINATES
    R1 = np.array([qi[0], qi[1]], dtype = float)
    R2 = np.array([qi[3], qi[4]], dtype = float)

    # Point O at link 1
    r1O = R1
    # Point O at link 2
    r2O = R2 + np.matmul(ATransformMatrix(float(qi[5])), u_bar_2O)

    # Point A at link 2
    r2A = R2 + np.matmul(ATransformMatrix(float(qi[5])), u_bar_2A)
    # Point A at link 3
    R3 = np.array([qi[6], qi[7]], dtype = float)
    r3A = R3 + np.matmul(ATransformMatrix(float(qi[8])), u_bar_3A)

    # Point B at link 3
    r3B = R3 + np.matmul(ATransformMatrix(float(qi[8])),u_bar_3B)
    # Point B at link 4
    R4 = np.array([qi[9], qi[10]], dtype = float)
    r4B = R4 + np.matmul(ATransformMatrix(float(qi[11])),u_bar_4B)

    return r1O, r2O, r2A, r3A, r3B, r4B

def constraintEquation(constraintVector, qi, r1O, r2O, r2A,
                    r3A, r3B, r4B, sliderHeight, theta2Initial,
                    omega2, timeNow):
    # Ground constraint
    for i in range(3):
        constraintVector[i] = qi[i]

    # Pin joint O
    constraintPinO = r2O # r1O -
    for i in range(np.size(constraintPinO)):
    # Equation 4-5
        constraintVector[i+3] = constraintPinO[i]

    # Pin joint A
    constraintPinA = r2A - r3A
    for i in range(np.size(constraintPinA)):
    # Equation 6-7
        constraintVector[i+5] = constraintPinA[i]

    # Pin joint B
    constraintPinB = r3B - r4B
    for i in range(np.size(constraintPinB)):
    # Equation 8-9
        constraintVector[i+7] = constraintPinB[i]

    # Equation 10 - 12
    constraintVector[9] = qi[10]-sliderHeight
    constraintVector[10] = qi[11] - 0
    constraintVector[11] = qi[5] - theta2Initial - omega2*timeNow

    return constraintVector

def jacobianMatrix(qi, u_bar_2O, u_bar_2A, u_bar_3A, u_bar_3B, u_bar_4B):
    n = np.size(qi)
    jacobianMatrixCq = np.zeros((n,n), dtype = float)
    identity3x3 = np.identity(3)
    identity2x2 = np.identity(2)

    # row 1-3
    jacobianMatrixCq[0:3,0:3] = identity3x3

    # row 4-5
    Cq45 = np.matmul(ATransformMatrixTHETA(float(qi[5])), u_bar_2O)
    jacobianMatrixCq[3:5,3:5] = identity2x2
    jacobianMatrixCq[3:5,5:6] = Cq45

    # row 6-7
    Cq67_link2 = np.matmul(ATransformMatrixTHETA(float(qi[5])), u_bar_2A)
    Cq67_link3 = np.matmul(ATransformMatrixTHETA(float(qi[8])), u_bar_3A)
    jacobianMatrixCq[5:7,3:5] = identity2x2
    jacobianMatrixCq[5:7,5:6] = Cq67_link2
    jacobianMatrixCq[5:7,6:8] = -identity2x2
    jacobianMatrixCq[5:7,8:9] = -Cq67_link3

    # row 8-9
    Cq89_link3 = np.matmul(ATransformMatrixTHETA(float(qi[8])), u_bar_3B)
    Cq89_link4 = np.matmul(ATransformMatrixTHETA(float(qi[11])), u_bar_4B)
    jacobianMatrixCq[7:9,6:8] = identity2x2
    jacobianMatrixCq[7:9,8:9] = Cq89_link3
    jacobianMatrixCq[7:9,9:11] = -identity2x2
    jacobianMatrixCq[7:9,11:12] = -Cq89_link4

    # row 10-11
    jacobianMatrixCq[9:11,10:12] = identity2x2

    # row 12
    jacobianMatrixCq[11][5] = 1

    return jacobianMatrixCq

def constrEqDot(constraintVectorVel, omega):

    constraintVectorVel[11] = -omega

    return constraintVectorVel

def positionAnalysis(constraintVector, jacobianMatrix, qi):
    inverse_jacobian = np.linalg.inv(jacobianMatrix)
    delta_qi = - np.matmul(inverse_jacobian, constraintVector)
    delta_qi_norm = np.linalg.norm(delta_qi)
    qi = qi + delta_qi

    return qi, delta_qi_norm

def velocityAnalysis(constraintVectorVel, jacobianMatrix, qiDot):
    inverse_jacobian = np.linalg.inv(jacobianMatrix)
    qiDot= - np.matmul(inverse_jacobian, constraintVectorVel)

    return qiDot

def accelerationAnalysis(jacobianMatrix, qiDotDot, qiDot, qi,
                        u_bar_2O, u_bar_2A, u_bar_3A, u_bar_3B, u_bar_4B):
    # Qd line 3-4
    Qd34 = np.square(float(qiDot[5]))*np.matmul(ATransformMatrix(float(qi[5])), u_bar_2O)

    # Qd line 5-6
    Qd56a = np.square(float(qiDot[5]))*np.matmul(ATransformMatrix(float(qi[5])), u_bar_2A)
    Qd56b = np.square(float(qiDot[8]))*np.matmul(ATransformMatrix(float(qi[8])), u_bar_3A)
    Qd56 = Qd56a-Qd56b

    # Qd line 7-8
    Qd78a = np.square(float(qi[8]))*np.matmul(ATransformMatrix(float(qi[8])), u_bar_3B)
    Qd78b = np.square(float(qi[11]))*np.matmul(ATransformMatrix(float(qi[11])), u_bar_4B)
    Qd78 = Qd78a-Qd78b

    Qd = np.array([[0],[0],[0], Qd34[0], Qd34[1],
                    Qd56[0], Qd56[1], Qd78[0], Qd78[1],
                    [0], [0], [0]])

    inverse_jacobian = np.linalg.inv(jacobianMatrix)
    qiDotDot= np.matmul(inverse_jacobian, Qd)

    return qiDotDot


import numpy as np
import math
import scipy.integrate as integrate
from __future__ import division

def jzs_corbf(r,n):
    # This function computes the Bayes Factor to evaluate evidence for or against a linear relationship
    # H0: no linear relationship
    # H1: linear relationship exists
    # For interpretation of the result see below

    # Usage
    # r = correlation_coefficient from sample
    # n = n_samples (e.g. datapoints)
    # BF10 = jzs_corbf(r,n)

    # Interpretation:
    #       >   100   Decisive evidence for H1
    # 30    -   100   Very Strong evidence for H1
    # 10    -   30    Strong evidence for H1
    # 3     -   10    Substantial evidence for H1
    # 1     -   3     Anecdotal evidence for H1
    # 1               No evidence
    # 1/3   -   1     Anecdotal evidence for H0
    # 1/10  -  1/3    Substantial evidence for H0
    # 1/30  -  1/10   Strong evidence for H0
    # 1/100 -  1/30   Very Strong evidence for H0
    #       <  1/100  Decisive evidence for H0

    # As described in Wetzels & Wagemakers 2012
    # Psychon Bull Rev. 2012 Dec; 19(6): 1057–1064.
    # doi:  10.3758/s13423-012-0295-x

    g1 = lambda g: (1 + g)**((n - 2)/2) * (1+(1-r**2)*g)**(-(n-1)/2) * g**(-3/2) * np.exp(-n/(2*g))
    BF10 = (np.sqrt(n/2)/math.gamma(1/2))* integrate.quad(g1, 0,np.inf)[0]
    return BF10

def eval_null(SSE1, SSE0, k1,k0, n)
    # Model Comparisons using SSE/SST:
    # To identify posterior probability of null
    # It is often useful to consider the delta BIC10 quantity using BF01
    # Note that BF01 != BF10

    # Usage
    # SSE1: sum of squared errors for H1 (alternative)
    # SSE0: sum of squared errors for H0 (null)
    # k1: # parameters for alternative model
    # k0: # parameters for null model
    # n:number of subjects

    # BF01
    #       >   100   Decisive evidence for H1
    # 30    -   100   Very Strong evidence for H1
    # 10    -   30    Strong evidence for H1
    # 3     -   10    Substantial evidence for H1
    # 1-3      Anecdotal evidence for H1

    # As described in Wetzels & Wagemakers 2012
    # Psychon Bull Rev. 2012 Dec; 19(6): 1057–1064.
    # doi:  10.3758/s13423-012-0295-x

    deltaBIC10 = n*np.log(SSE1/SSE0) + (k1 - k0)*np.log(n)
    BF01 = np.exp(deltaBIC10/2)
    return BF01


import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler

def sigmoid(z) :

    return 1 / (1 + np.exp(-z))

def initialize_with_zeros(dim) :

    w = np.zeros((dim, 1))
    b = 0

    return w, b

def propagate(w, b, X, Y) :

    # No. of training examples
    m = X.shape[1]

    # Forward propagation
    A = sigmoid(np.dot(w.T, X) + b)     # Activation function
    cost = (-1 / m) * np.sum(Y * np.log(A) + (1 - Y) * np.log(1 - A))   # Cost function

    # Backward propagation
    dw = (1 / m) * np.dot(X, (A - Y).T)    # dJ/dw
    db = (1 / m) * np.sum(A - Y)    # dJ/db

    cost = np.squeeze(cost)

    grads = {'dw' : dw, 'db' : db}

    return grads, cost

def optimize(w, b, X, Y, num_iterations, learning_rate, print_cost = False) :

    costs = []

    for i in range(num_iterations) :

        # Cost and gradient calculation
        grads, cost = propagate(w, b, X, Y)

        dw = grads['dw']
        db = grads['db']

        # Gradient descent
        w = w - learning_rate * dw
        b = b - learning_rate * db

        if i % 100 == 0 :
            costs.append(cost)

        if print_cost and i % 100 == 0 :
            print(f"Cost after iteration {i} : {cost}")

    params = {'w' : w, 'b' : b}
    grads = {'dw' : dw, 'db' : db}

    return params, grads, costs

def predict(w, b, X) :

    m = X.shape[1]
    Y_prediction = np.zeros((1, m))
    w = w.reshape((X.shape[0], 1))

    A = sigmoid(np.dot(w.T, X) + b)

    for i in range(A.shape[1]) :

        # Binary classification
        Y_prediction[0, i] = 0 if A[0, i] <= 0.5 else 1

    return Y_prediction

def model(X_train, Y_train, X_test, Y_test, num_iterations = 2000, learning_rate = 0.5, print_cost = False) :

    w, b = initialize_with_zeros(X_train.shape[0])

    parameters, grads, costs = optimize(w, b, X_train, Y_train, num_iterations, learning_rate, print_cost)

    w = parameters['w']
    b = parameters['b']

    Y_prediction_test = predict(w, b, X_test)
    Y_prediction_train = predict(w, b, X_train)

    print(f'\nTraining accuracy : {100 - np.mean(np.abs(Y_prediction_train - Y_train)) * 100}')
    print(f'Testing accuracy : {100 - np.mean(np.abs(Y_prediction_test - Y_test)) * 100}')

    d = {'costs' : costs,
         'Y_prediction_test' : Y_prediction_test,
         'T_prediction_train' : Y_prediction_train,
         'w' : w,
         'b' : b,
         'grads' : grads,
         'learning_rate' : learning_rate,
         'num_iterations' : num_iterations}

    return d

def binary_sex(sex) :
    return 1 if sex == 'male' else 0

def ternary_embarked(s) :
    if s == 'S' :
        return 0
    if s == 'C' :
        return 1
    if s == 'Q' :
        return 2

    return -1

def preprocess_data(file_obj) :

    df = file_obj.copy()

    df.drop(columns = ['Name', 'Ticket', 'Cabin'], inplace = True)

    df['Age'].fillna(value = df['Age'].median(), inplace = True)
    df['Embarked'].fillna(value = 'N', inplace = True)

    df['Sex'] = df['Sex'].apply(binary_sex)
    df['Embarked'] = df['Embarked'].apply(ternary_embarked)

    dataset = df.to_numpy()

    return dataset

def load_dataset() :

    file_obj = pd.read_csv('titanic_x_y_train.csv')

    training_data = preprocess_data(file_obj)

    X = training_data[ : , : -1]
    Y = training_data[ : , -1 ]

    # Normalizing data
    X = StandardScaler().fit_transform(X)

    X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size = 0.2)

    return X_train.T, Y_train.reshape((1, Y_train.shape[0])), X_test.T, Y_test.reshape((1, Y_test.shape[0]))

# Main
X_train, Y_train, X_test, Y_test = load_dataset()

learning_rates = [0.0001, 0.001, 0.01, 0.1]
models = {}

for lr in learning_rates :

    print(f'learning rate is : {lr}')

    models[str(lr)] = model(X_train, Y_train, X_test, Y_test, num_iterations = 2000, learning_rate = lr, print_cost = False)

    print('\n' + "-------------------------------------------------------" + '\n')

# Plot learning curve (with costs)
for lr in learning_rates :

    d = models[str(lr)]

    costs = np.squeeze(d['costs'])
    plt.plot(costs, label = str(d['learning_rate']))

plt.ylabel('cost')
plt.xlabel('iterations (per hundred)')

legend = plt.legend(loc='upper right', shadow = True)

plt.show()


import numpy as np
import math

# Fixed for the time being...
ADV_WAVE_FREQ = 1  # This scalar defines the speed of the wave
IMAGE_DIMENSION = 64
NUM_FRAMES = 100
dt = 0.01  # Constant now (Don't change this)

# This vector defines the direction of the travelling wave.
# The magnitude of this vector defines the "length" of the wave
ADV_WAVE_VECTOR = np.array([1, 0])

def create_adv_diff_wave(
    image_dimension=IMAGE_DIMENSION,
    num_frames=NUM_FRAMES,
    adv_wave_vector=ADV_WAVE_VECTOR,
    adv_wave_freq=ADV_WAVE_FREQ,
):

    # Degrees between wave direction and x-axis
    theta = math.atan2(adv_wave_vector[1], adv_wave_vector[0]) * 180 / math.pi

    # Data to generate
    data = np.zeros((num_frames, image_dimension, image_dimension))

    # Deltas, num ticks in example
    dx = 2 * np.pi / (image_dimension - 1)
    dy = dx
    ticks = np.linspace(0, dt * num_frames, num_frames)

    # Create frames
    for k in range(num_frames):
        t0 = ticks[k]
        for i in range(image_dimension):
            for j in range(image_dimension):
                x = dx * j
                y = (image_dimension - i) * dy
                r = np.array([x, y])
                data[k, i, j] = math.sin(
                    np.dot(adv_wave_vector, r) - adv_wave_freq * t0
                )

    return data, theta


#!/usr/bin/env python
# coding: utf-8

# # Contaminant Concentration

#
# ## Task 1
# A saline solution with a concentration of 370mg/l is itroduced to a 2-m-longsand column in which the pores are initially filled with distilled water.If the solution drains through the column at an average linear velocity of 0.79 m/day and the dynamic dispersivity of the sand columnis 15 cm, what would the conceration of the effluent be at 1.8 days after the flow begins?
#
# source: "Applied Hydrogeology" by C.W. Fetter <br>
#

# In[1]:

import math
import scipy.special as sc

# In[2]:

# A.inputs
C0 = 370  #mg/l initial concentration
L = 2 #m length of the sand column
Vx = 0.79  #m/d linear velocity of groundwater flow
Disp = 0.15 #m dispersivity of sand column
t = 1.8  #days after the flow begins
print("A. Given information: ")
print("Initial concentration = {} mg/l\nlength of the sand column = {} m\nLinear velocity of groundwater flow = {} m/d\nDispersivity = {} m\nTime={} d".format(C0, L, Vx, Disp, t),"\n")

# B.Calculations of :
DL=Disp*Vx # longitudinal dispersivity coefficient

X=(L-(Vx*t))/(2*((DL*t)**0.5))#  Error functions
Y=(L+(Vx*t))/(2*((DL*t)**0.5))#  Error functions
C=(C0/2)*(sc.erfc(X)+(math.exp((Vx*L)/DL)*sc.erfc(Y)))# Concentration after t days

#  Defining a concentration function and conditional statemen based on error functions
print("B. Calculations: ")

if X>3 and Y<3:
    sc.erfc(X)==0, print("Error function of X [erfc(X)] is negligible or zero\n")
    C=(C0/2)*(math.exp((Vx*L)/DL)*sc.erfc(Y))
    rsults=C
else :
    (math.exp((Vx*L)/DL)*sc.erfc(Y))==0, print("Error function of Y [erfc(Y)] is negligble or zero\n")
    C=(C0/2)*(math.erfc(X))
    results= C

# C.output
print("C. Output: ")
print("Concentration after 1.8 days is: ", C, "mg/l")

# In[ ]:


import scipy as sp
data = sp.genfromtxt("data/web_traffic.tsv", delimiter="\t")

# 先頭10件を表示
print(data[:10])

# 1列目を1次元配列で取り出す(経過時間？)
x = data[:, 0]

# 2列目を1次元配列で取り出す(アクセス数)
y = data[:, 1]

# 不正なデータが入っている要素を除去(アクセス数が不正な値(NAN)の行を除去)

invalid_data = ~sp.isnan(y)
x = x[invalid_data]
y = y[invalid_data]

print("x[-1] = %d" % x[-1])

# フォント
import matplotlib
matplotlib.rcParams['font.family'] = 'IPAexGothic'

# 近似関数(1次)を得る
fp1, residuals, rank, sv, rcond = sp.polyfit(x, y, 1, full=True)
print("Model parameters: %s" % fp1)
print("residuals: %s" % residuals)

# 近似関数からモデル関数を作る
f1 = sp.poly1d(fp1)

# モデル関数(f1)と実測値(x,y)から誤差を得る
def error(f,x,y):
    return sp.sum((f(x) - y) ** 2)

print(error(f1, x, y )) # 317389767.339778

# 近似関数(2次)を得る
fp2, residuals, rank, sv, rcond = sp.polyfit(x, y, 2, full=True)
print("Model parameters: %s" % fp2) # Model parameters: [ 1.05322215e-02 -5.26545650e+00  1.97476082e+03]
print("residuals: %s" % residuals) # residuals: [1.79983508e+08]

# 近似関数からモデル関数を作る
f2 = sp.poly1d(fp2)
print(error(f2, x, y )) # 179983507.8781792 (f1と比べて半分ぐらいに減った！)

# 表示
import matplotlib.pyplot as plt
plt.scatter(x,y)
plt.title("先月のWebトラフィック量")
plt.xlabel("時間")
plt.ylabel("件/時")
plt.xticks([w*7*24 for w in range(10)], ['week %i'% w for w in range(10)]) # 1週間(7日*24時間)毎に区切る。0周目～9周目まで。
plt.autoscale(tight=True) # 実際は4周目分のデータしかないので、表示を0周目～4周目までを収まるようにスケール
plt.grid()

# # モデル関数f1を使い、線を引く
fx = sp.linspace(0, x[-1], 1000) # 間隔が等しい数列を1000個作る。始点:0,終点:x[-1]=743,数:1000個 => 約0.743づつ増加する1000個の数列
# # print(sp.linspace(0, x[-1], 1000)) # 数列を確認

# 変化がある3.5週目から分けて２つの直線で近似させる
inflection = int(3.5 * 7 * 24)
xa = x[:inflection]
ya = y[:inflection]
xb = x[inflection:]
yb = y[inflection:]

fa = sp.poly1d(sp.polyfit(xa, ya, 1))
fb = sp.poly1d(sp.polyfit(xb, yb, 1))

fa_error = error(fa, xa, ya)
fb_error = error(fb, xb, yb)
print("fa_error = %s" % fa_error )
print("fb_error = %s" % fb_error )
print("fa_error+fb_error = %s" % (fa_error + fb_error) )

# 3.5週目以降のデータのうち、
# 30%をテストデータとして使う

# 再現性を出すためのseed固定
sp.random.seed(314159263)

frac = 0.30
split_idx = int(frac * len(xb))
shuffled = sp.random.permutation(list(range(len(xb))))
test = sorted(shuffled[:split_idx])
train = sorted(shuffled[split_idx:])

fbt1 = sp.poly1d(sp.polyfit(xb[train], yb[train], 1))
fbt2 = sp.poly1d(sp.polyfit(xb[train], yb[train], 2))
fbt3 = sp.poly1d(sp.polyfit(xb[train], yb[train], 3))
fbt10 = sp.poly1d(sp.polyfit(xb[train], yb[train], 10))
fbt100 = sp.poly1d(sp.polyfit(xb[train], yb[train], 100))

# 学習データで誤差を見る
print("(train)fbt1_error  = %s" % error(fbt1, xb[train], yb[train]) ) # 15846249.623390418
print("(train)fbt2_error  = %s" % error(fbt2, xb[train], yb[train]) ) # 14048047.808891317
print("(train)fbt3_error  = %s" % error(fbt3, xb[train], yb[train]) ) # 14014812.664947439
print("(train)fbt10_error = %s" % error(fbt10, xb[train], yb[train]) )# 13092597.979928792
print("(train)fbt100_error = %s" % error(fbt100, xb[train], yb[train]) )# 12652070.062779315 <= 最も誤差が小さい、が、学習データに最適化されている状態なので、評価としては不適切。

# 未知データで誤差を見る
print("(test) fbt1_error  = %s" % error(fbt1, xb[test], yb[test]) ) # 6387914.410987866
print("(test) fbt2_error  = %s" % error(fbt2, xb[test], yb[test]) ) # 5841997.167708391 <= 最も誤差が小さい。乱数を変えても大体これが良い結果に。
print("(test) fbt3_error  = %s" % error(fbt3, xb[test], yb[test]) ) # 5949556.660118614
print("(test) fbt10_error = %s" % error(fbt10, xb[test], yb[test]) )# 6525801.591164470
print("(test)fbt100_error = %s" % error(fbt100, xb[test], yb[test]))# 6677906.164012078

### 描画 ###

# 未来を含めて描画(10週目まで描画してみる。)
fx = sp.linspace(0, 7*24*10, 1000)

# # f1
# plt.plot(fx, f1(fx), linewidth=2) # 直線を描く(xに対するy(f1(x))の線を引く)
# plt.legend(["d=%i" % f1.order], loc="upper left") # 凡例

# # f2
# plt.plot(fx, f2(fx), linewidth=2) # 直線を描く(xに対するy(f1(x))の線を引く)
# plt.legend(["d=%i" % f2.order], loc="upper left") # 凡例

# # 変化がある3.5週目から分けて２つの直線を引く(fa,fb)
# plt.plot(fx, fa(fx), linewidth=2)
# plt.plot(fx, fb(fx), linewidth=2)
# plt.legend(["d=%i" % fa.order, "d=%i" % fb.order], loc="upper left") # 凡例

# 変化がある3.5週目から70%を学習に使って作ったモデルで線を引く
plt.plot(fx, fbt1(fx), linewidth=2)
plt.plot(fx, fbt2(fx), linewidth=2)
plt.plot(fx, fbt3(fx), linewidth=2)
plt.plot(fx, fbt10(fx), linewidth=2)
plt.plot(fx, fbt100(fx), linewidth=2)

plt.legend(["d=%i" % fbt1.order, "d=%i" % fbt2.order, "d=%i" % fbt3.order, "d=%i" % fbt10.order, "d=%i" % fbt100.order], loc="upper left") # 凡例

plt.show()


"""
Gaussian Transformation
- Some machine learning models like linear and logistic regression assume that the variables are normally distributed.
        Often, variables are not normally distributed, but, transforming the variables to map their distribution to
        a Gaussian distribution may, and often does, boost the performance of the machine learning algorithm.

- If a variable is not normally distributed, it is often possible to find a mathematical transformation to
        normalise its distribution.

How can we transform variables so that they follow a normal distribution?
The most commonly used methods to transform variables are:
- Logarithmic transformation - np.log(X)
- Reciprocal transformation - 1 / X
- Square root transformation - X**(1/2)
- Exponential transformation (more general, you can use any exponent)
- Box-Cox transformation
- Yeo-Johnson transformation

You can find the formulas for Box-Cox and Yeo-Johnson here https://scikit-learn.org/stable/modules/preprocessing.html#mapping-to-a-gaussian-distribution

Briefly,
- the Box-Cox transformation is an adaptation of the exponential transformation, scanning through various
        exponents, and it already represents the untransformed variable, as well as the log transformed,
        reciprocal, square and cube root transformed, as the lambda varies across the range of -5 to 5
        (see formula, to understand this better).
        So by doing Box-Cox transformation, in a way, we are evaluating all the other transformations and
        choosing the best one. Box-Cox can only be applied to positive variables.
- Yeo-Johnson is a modification of the Box-Cox transformation so that it can be applied as well to non-positive
        variables

To read more about the Box-Cox transformation visit any of these links:
- http://www.statisticshowto.com/box-cox-transformation/
- http://www.itl.nist.gov/div898/handbook/eda/section3/eda336.htm
- http://onlinestatbook.com/2/transformations/box-cox.html

To read more about the Yeo-Johnson transformation follow this link: https://www.stat.umn.edu/arc/yjpower.pdf

In this example:
- We will see how to implement variable transformations using NumPy and the House Prices dataset.
 """
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import scipy.stats as stats

# load the data
data = pd.read_csv('dataset/house-prices-advanced-regression-techniques/train.csv')

data.head()
"""
   Id  MSSubClass MSZoning  LotFrontage  LotArea Street Alley LotShape LandContour Utilities  ... PoolArea PoolQC Fence MiscFeature MiscVal MoSold YrSold  SaleType  SaleCondition  SalePrice
0   1          60       RL         65.0     8450   Pave   NaN      Reg         Lvl    AllPub  ...        0    NaN   NaN         NaN       0      2   2008        WD         Normal     208500
1   2          20       RL         80.0     9600   Pave   NaN      Reg         Lvl    AllPub  ...        0    NaN   NaN         NaN       0      5   2007        WD         Normal     181500
2   3          60       RL         68.0    11250   Pave   NaN      IR1         Lvl    AllPub  ...        0    NaN   NaN         NaN       0      9   2008        WD         Normal     223500
3   4          70       RL         60.0     9550   Pave   NaN      IR1         Lvl    AllPub  ...        0    NaN   NaN         NaN       0      2   2006        WD        Abnorml     140000
4   5          60       RL         84.0    14260   Pave   NaN      IR1         Lvl    AllPub  ...        0    NaN   NaN         NaN       0     12   2008        WD         Normal     250000 """

# let's explore the distribution of the numerical variables
data.hist(figsize=(20,20))
plt.show()

"""
Plots to assess normality
- To visualise the distribution of the variables, we plot a histogram and a Q-Q plot.
- In the Q-Q pLots, if the variable is normally distributed, the values of the variable should fall in a 45 degree
        line when plotted against the theoretical quantiles.
 """

# plot the histograms to have a quick look at the variable distribution
# histogram and Q-Q plots

def diagnostic_plots(df, variable):

    # function to plot a histogram and a Q-Q plot
    # side by side, for a certain variable

    plt.figure(figsize=(15,6))
    plt.subplot(1, 2, 1)
    df[variable].hist(bins=30)

    plt.subplot(1, 2, 2)
    stats.probplot(df[variable], dist="norm", plot=plt)

    plt.show()

# Original distribution
diagnostic_plots(data, 'GrLivArea')

""" We can see in the plots that the variable is not normally distributed.
The values depart from the red line towards the ends of the distribution and we can see in the histogram that
 it is skewed to the right.
 """

# Logarithmic transformation
### Logarithmic transformation

data['GrLivArea_log'] = np.log(data['GrLivArea'])

diagnostic_plots(data, 'GrLivArea_log')

# The logarithmic transformation returned a more Gaussian for this variable.

# Reciprocal transformation
data['GrLivArea_reciprocal'] = 1 / (data['GrLivArea'])

# np.reciprocal(data['GrLivArea'])

diagnostic_plots(data, 'GrLivArea_reciprocal')

# The reciprocal transformation was not useful to transform this variable.

# Square root transformation
data['GrLivArea_sqr'] = data['GrLivArea']**(1/2)

# np.power(data['GrLivArea'], 1/2), np.sqrt(data['GrLivArea'])

diagnostic_plots(data, 'GrLivArea_sqr')

# The square root transformation offers a good alternative to normalise this variable.

# Exponential transformation
data['GrLivArea_exp'] = data['GrLivArea']**(1/1.5) # you can vary the exponent as needed

# np.power(data['GrLivArea'], any exponent we want)

diagnostic_plots(data, 'GrLivArea_exp')

# The exponential transformation did not work so nicely for this variable.

# Box-Cox transformation
"""
The Box-Cox transformation is defined as:
- T(Y)=(Y exp(λ)−1)/λ if λ!=0, or log(Y) otherwise.
        - where Y is the response variable and λ is the transformation parameter.
            λ varies from -5 to 5.
            In the transformation, all values of λ are considered and the optimal value for a given variable
            is selected.

- Briefly, for each λ (the transformation tests several λs), the correlation coefficient of the Probability Plot
        (Q-Q plot below, correlation between ordered values and theoretical quantiles) is calculated
        (this optimisation equation actually varies with the implementation).
- The value of λ corresponding to the maximum correlation on the plot is then the optimal choice for λ.

In python, we can evaluate and obtain the best λ with the stats.boxcox function from the package scipy.
Let's have a look. """

data['GrLivArea_boxcox'], param = stats.boxcox(data['GrLivArea'])

print('Optimal λ: ', param)
# Optimal λ:  0.006304886038020439

diagnostic_plots(data, 'GrLivArea_boxcox')

"""
Yeo-Johnson transformation
- Yeo-Johnson is the same as Box-Cox for the positive values of the variable, but it has different equations
        for the negative values of the variable as described here

- Again, the function searches over a bunch of λ and selects the one that returns the best fit to a normal distribution.
 """

# to avoid a NumPy error
data['GrLivArea'] = data['GrLivArea'].astype('float')
data['GrLivArea_yeojohnson'], param = stats.yeojohnson(data['GrLivArea'])

print('Optimal λ: ', param)
# Optimal λ:  0.005539641278705955

diagnostic_plots(data, 'GrLivArea_yeojohnson')

"""
- We can see that after the transformation, the quantiles are somewhat more aligned over the 45 degree
        line with the theorical quantiles of the Gaussian distribution. """


__all__ = ['SigmoidFunction']

import numpy as np

class SigmoidFunction:
    def __call__(self, inputs):
        # fix overflow
        inputs = np.clip(inputs, -500, 500)
        return 1. / (1. + np.exp(-inputs))

    def derivative(self, inputs, outputs=None):
        if outputs is None:
            outputs = self(inputs)
        return outputs * (1 - outputs)

    def __repr__(self):
        return "SigmoidLayer()"


# -*- coding: utf-8 -*-

# Author: Daniel Yang <daniel.yj.yang@gmail.com>
#
# License: BSD 3 clause

from sklearn.tree import DecisionTreeClassifier
from sklearn.tree import DecisionTreeRegressor

import numpy as np
import pandas as pd

from ..base import classifier

# Reduction in uncertainty = gain in information
#def Information_Gain(y, X):
#   IG(y, X) = Entropy(y) - Entropy(y | X)

# “purity” means how homogenized a group is.

#from numpy import ma # masked array

# A good read: https://towardsdatascience.com/entropy-how-decision-trees-make-decisions-2946b9c18c8#:~:text=Entropy%20is%20a%20measure%20of,general%20is%20to%20reduce%20uncertainty.&text=This%20is%20called%20Information%20Gain,gained%20about%20Y%20from%20X.

def Entropy(splitted_sample=[]):
    """
    For example: [194, 106]
    Entropy is a measure of disorder/uncertainty (= low purity, a lack of dominant class)
    Entropy vs. Gini impurity: both of them involve p_j * p_j, but Entropy takes the form of S = k_b*ln(Ω)
    """

    import math

    denominator = sum(splitted_sample)
    if denominator == 0:
        return 0
    Entropy_index = 0
    for numerator_i in range(len(splitted_sample)):
        p_i = splitted_sample[numerator_i]/denominator
        if p_i != 0:
            Entropy_index -= p_i * math.log2(p_i)
        #Entropy_index -= p_i * ma.array(ma.log2(p_i)).filled(0) # handle log2(0) warning
    return Entropy_index

def Gini_impurity(splitted_sample=[]):
    """
    For example: [194, 106]
    note: summation( p_j * (1 - p_j) ) = 1 - summation( p_j^2 )
    """
    denominator = sum(splitted_sample)
    if denominator == 0:
        return 0
    Gini_index = 0
    for numerator_i in range(len(splitted_sample)):
        p_i = splitted_sample[numerator_i]/denominator
        # p_j * (1 - p_j) is the likelihood of misclassifying a new instance
        Gini_index += p_i * (1 - p_i)
    return Gini_index

def impurity_measure_with_sample_weight(y_class0_value=0, y_class1_value=1, y=[], sample_weight=None, impurity_func=Entropy):
    """
    For example: y=[0,1,1,0,0,1], sample_weight=[0.1, 0.1, 0.2, 0.1, 0.1, 0.1]
    """

    y_class0_sample_weighted_sum = 0
    y_class1_sample_weighted_sum = 0

    length_y = len(y)
    if length_y == 0:
        return 0

    if sample_weight is None:
        sample_weight = np.ones(shape=(length_y,))

    #print(f"{sample_weight}")

    if length_y != len(sample_weight):
        raise ValueError("unequal inputs of y[] and sample_weight[]")

    y_class0_sample_weighted_sum = np.sum(sample_weight[y == y_class0_value])
    y_class1_sample_weighted_sum = np.sum(sample_weight[y == y_class1_value])

    for i in range(length_y):
        if y[i] not in [y_class0_value, y_class1_value]:
            raise ValueError(f"y[{i}]=[{y[i]}] must be either y_class0_value=[{y_class0_value}] or y_class1_value=[{y_class1_value}]")

    return impurity_func(splitted_sample=[y_class0_sample_weighted_sum, y_class1_sample_weighted_sum])

def Impurity_plot():

    import matplotlib.pyplot as plt

    n = 200
    X = np.arange(0,n/2+1,1)
    Y = X[::-1]

    def plot(impurity_index, X, index_name="", title="", y_ref=None):
        plt.figure(figsize=(5, 5))
        plt.plot(X, impurity_index)
        plt.axhline(y=y_ref, color='r', linestyle='--')
        plt.title(f"{title}")
        plt.xlabel(f'n splitted as y=0 (total #sample = {n})')
        plt.ylabel(f"{index_name}")
        plt.ylim([0, 1.1])
        plt.show()

    measure = np.empty_like(X)
    for i in range(len(X)):
        measure[i] = Gini_impurity([X[i], Y[i]])
    plot(measure, X, index_name="Gini Impurity", title="Gini Impurity Plot (highest = 0.5 for two classes)", y_ref=0.5)

    measure = np.empty_like(X)
    for i in range(len(X)):
        measure[i] = Entropy([X[i], Y[i]])
    plot(measure, X, index_name="Entropy",       title="Entropy Plot (highest = 1.0 for two classes)",       y_ref=1.0)

def demo_metrics():
    """
    These generally measure the homogeneity of the target variable within the subsets.
    """
    Impurity_plot()

class decision_tree_classifier_node(object):
    def __init__(self, curr_depth=None, curr_impurity=None, curr_sample_size=None, curr_y_distribution={}, best_split_feature_i=None, best_x_cutoff_value=None, y_class0_value=0, y_class1_value=1):
        self.curr_depth = curr_depth
        self.curr_impurity = curr_impurity
        self.curr_sample_size = curr_sample_size
        self.curr_y_distribution = curr_y_distribution

        y_class0_n = curr_y_distribution.get(y_class0_value)
        y_class1_n = curr_y_distribution.get(y_class1_value)
        y_classes_count = len(curr_y_distribution)

        if y_classes_count == 0:
            self.y_dominant_class = None
            self.y_class1_prob = None

        if y_classes_count == 1:
            self.y_dominant_class = list(curr_y_distribution.keys())[0]
            if self.y_dominant_class == y_class0_value:
                self.y_class1_prob = 0
            elif self.y_dominant_class == y_class1_value:
                self.y_class1_prob = 1
            else:
                raise ValueError(f"unexpected y_dominant_class (should be either {y_class0_value} or {y_class1_value})")

        if y_classes_count == 2:
            # must be {y_class0_value: sample_size_associated_with_y=y_class0_value, y_class1_value: sample_size_associated_with_y=y_class1_value}
            self.y_class1_prob = y_class1_n / (y_class1_n + y_class0_n)
            if self.y_class1_prob >= 0.50:
                self.y_dominant_class = y_class1_value
            else:
                self.y_dominant_class = y_class0_value

        if y_classes_count > 2:
            raise ValueError('more than 2 classes in y detected')

        self.best_split_feature_i = best_split_feature_i
        self.best_x_cutoff_value = best_x_cutoff_value
        self.left = None
        self.right = None

    def to_dict(self):
        return {'curr_depth': self.curr_depth, 'curr_impurity': f"{self.curr_impurity:.3f}" if self.curr_impurity is not None else None, 'curr_sample_size': self.curr_sample_size, 'curr_y_distribution': self.curr_y_distribution, 'curr_dominant_y_class': self.y_dominant_class, 'curr_y_class1_prob': f"{self.y_class1_prob:.3f}" if self.y_class1_prob is not None else None, 'best_split_feature_i': self.best_split_feature_i if self.best_x_cutoff_value is not None else None, 'best_x_cutoff_value': f"{self.best_x_cutoff_value:.3f}" if self.best_x_cutoff_value is not None else None}

class decision_tree_classifier_from_scratch(classifier):
    """
    Shortcoming of DT:
    (1) biggest information gain locally, but globally; that is, there is always a better tree in the overall picture
    (2) we may be asking too trivial questions at the greater depths (e.g., > 5 or 6-depth)
    """

    def __init__(self, max_depth = 10, impurity_measure='entropy', features_indices_actually_used='all', annotation=None, verbose=False):
        """
        "features_indices_actually_used": limits the analysis on only these feature indices if not 'all'
            for example, if there are 30 features, then "features_indices_actually_used" = [2, 15] means that only the 3th and 16th features will be used for analysis
        """
        super().__init__()
        self.features_indices_actually_used = features_indices_actually_used  # limits the analysis on only these feature indices
        self.max_depth = max_depth
        self.verbose=verbose
        self.root_node = decision_tree_classifier_node()
        if impurity_measure not in ['entropy', 'gini_impurity']:
            raise ValueError('invalid impurity_measure value')
        self.impurity_measure = impurity_measure
        if self.impurity_measure == 'entropy':
            self.impurity_func = Entropy
        elif self.impurity_measure == 'gini_impurity':
            self.impurity_func = Gini_impurity
        self.annotation = annotation
        # default values
        self.y_class0_value = 0
        self.y_class1_value = 1

    def find_best_split_in_one_specific_feature(self, x, y_true, sample_weight=None):

        if type(x) in [pd.DataFrame, pd.Series]:
            x = x.to_numpy()

        if type(y_true) in [pd.DataFrame, pd.Series]:
            y_true = y_true.to_numpy()

        if sample_weight is None:
            sample_weight = np.ones(shape=(len(y_true),))

        best_impurity = float('Inf')

        from sortedcontainers import SortedSet

        y_classes_values = list(SortedSet(y_true))
        if len(y_classes_values) != 2:
            raise ValueError("y must be binary")

        from collections import Counter

        #y_counts = Counter(y_true)
        #before_split_impurity = self.impurity_func([ y_counts[y_classes_values[0]], y_counts[y_classes_values[1]] ])
        before_split_impurity = impurity_measure_with_sample_weight(y_class0_value=self.y_class0_value, y_class1_value=self.y_class1_value, y=y_true, sample_weight=sample_weight, impurity_func=self.impurity_func)

        x_values_array = list(SortedSet(x))
        x_values_array_length = len(x_values_array)

        for value_i, this_x_value in enumerate(x_values_array):  # iterating through each value in this feature

            if value_i < (x_values_array_length-1):
                this_x_cutoff_value = (this_x_value + x_values_array[value_i+1]) / 2
            else:
                this_x_cutoff_value = this_x_value

            # let's say we split y on this value of x
            y_pred_left_node_list = y_true[x <= this_x_cutoff_value]
            y_pred_left_node_counts = Counter(y_pred_left_node_list)
            left_node_y_true_array  = [ y_pred_left_node_counts[y_classes_values[0]],  y_pred_left_node_counts[ y_classes_values[1]] ]
            #left_node_impurity  = self.impurity_func(left_node_y_true_array)
            left_node_impurity = impurity_measure_with_sample_weight(y_class0_value=self.y_class0_value, y_class1_value=self.y_class1_value, y=y_pred_left_node_list, sample_weight=sample_weight[x <= this_x_cutoff_value], impurity_func=self.impurity_func)
            left_node_n = sum(y_pred_left_node_counts.values())

            y_pred_right_node_list = y_true[x > this_x_cutoff_value]
            y_pred_right_node_counts = Counter(y_pred_right_node_list)
            right_node_y_true_array = [ y_pred_right_node_counts[y_classes_values[0]], y_pred_right_node_counts[y_classes_values[1]] ]
            #right_node_impurity = self.impurity_func(right_node_y_true_array)
            right_node_impurity = impurity_measure_with_sample_weight(y_class0_value=y_classes_values[0], y_class1_value=y_classes_values[1], y=y_pred_right_node_list, sample_weight=sample_weight[x > this_x_cutoff_value], impurity_func=self.impurity_func)
            right_node_n = sum(y_pred_right_node_counts.values())

            this_y_true_split_array = [left_node_y_true_array, right_node_y_true_array]

            total_n = left_node_n + right_node_n
            if total_n != len(y_true):
                raise ValueError("internal inconsistency")

            this_weighted_impurity = (left_node_impurity * left_node_n / total_n) + (right_node_impurity * right_node_n / total_n)
            this_information_gain = before_split_impurity - this_weighted_impurity

            if self.verbose:
                print(f"#{value_i:3d}: x_cutoff_value = {this_x_cutoff_value: .3f}, impurity = {this_weighted_impurity:.3f}, information_gain = {this_information_gain:.3f}, split_y_true_array = {this_y_true_split_array}")

            if this_weighted_impurity < best_impurity:
                best_impurity = this_weighted_impurity
                best_information_gain = this_information_gain
                best_x_cutoff_value = this_x_cutoff_value
                best_y_true_split_array = this_y_true_split_array

            if best_impurity == 0: # a perfect split was found
                break

        return best_x_cutoff_value, best_impurity, best_information_gain, best_y_true_split_array

    def find_best_split_across_all_features(self, X, y_true, sample_weight=None):

        if type(X) in [pd.DataFrame, pd.Series]:
            X = X.to_numpy()

        if type(y_true) in [pd.DataFrame, pd.Series]:
            y_true = y_true.to_numpy()

        if sample_weight is None:
            sample_weight = np.ones(shape=(len(y_true),))

        best_impurity = float('Inf')

        n_samples = X.shape[0]
        n_features = X.shape[1]

        if self.features_indices_actually_used == 'all':
            features_indices_actually_used = range(n_features)
        else:
            features_indices_actually_used = self.features_indices_actually_used

        for this_feature_i in features_indices_actually_used:
            x_cutoff_value, impurity, information_gain, y_true_split_array = self.find_best_split_in_one_specific_feature(x=X[:,this_feature_i], y_true=y_true, sample_weight=sample_weight)
            if self.verbose:
                print(f"feature # {this_feature_i: 2d}, x_cutoff_value = {x_cutoff_value: .3f}, impurity = {impurity:.3f}, information_gain = {information_gain:.3f}, y_true_split_array = {y_true_split_array}")
            if impurity == 0:  # a perfect split was found
                best_split_feature_i = this_feature_i
                best_x_cutoff_value, best_impurity, best_information_gain, best_y_true_split_array = x_cutoff_value, impurity, information_gain, y_true_split_array
                break
            elif impurity < best_impurity:
                best_split_feature_i = this_feature_i
                best_x_cutoff_value, best_impurity, best_information_gain, best_y_true_split_array = x_cutoff_value, impurity, information_gain, y_true_split_array

        return best_split_feature_i, best_x_cutoff_value, best_impurity, best_information_gain, best_y_true_split_array

    def fit(self, X, y, depth=0, sample_weight=None):

        y_true = y

        # init
        if depth == 0:
            from sortedcontainers import SortedSet
            y_classes_values = list(SortedSet(y_true))
            if len(y_classes_values) != 2:
                raise ValueError("y must be binary")
            else:
                self.y_class0_value = y_classes_values[0]
                self.y_class1_value = y_classes_values[1]

        if sample_weight is None:
            sample_weight = np.ones(shape=(len(y_true),))

        from collections import Counter

        if self.verbose:
            print(f"depth={depth}")

        if type(X) in [pd.DataFrame, pd.Series]:
            X = X.to_numpy()

        if type(y_true) in [pd.DataFrame, pd.Series]:
            y_true = y_true.to_numpy()

        curr_sample_size = len(y_true)
        if curr_sample_size == 0:  # no data left to be split
            return None

        from sortedcontainers import SortedDict
        curr_y_distribution = SortedDict(Counter(y_true))

        #curr_impurity = self.impurity_func(list(Counter(y_true).values()))
        curr_impurity = impurity_measure_with_sample_weight(y_class0_value=self.y_class0_value, y_class1_value=self.y_class1_value, y=y_true, sample_weight=sample_weight, impurity_func=self.impurity_func)

        if curr_impurity == 0 or depth >= self.max_depth: # curr_impurity = 0 means already perfect, no need to split
            curr_node = decision_tree_classifier_node(curr_depth = depth, curr_impurity = curr_impurity, curr_sample_size = curr_sample_size, curr_y_distribution = curr_y_distribution, best_split_feature_i = None, best_x_cutoff_value = None, y_class0_value=self.y_class0_value, y_class1_value=self.y_class1_value)
            return curr_node

        best_split_feature_i, best_x_cutoff_value, best_impurity, best_information_gain, best_y_true_split_array = self.find_best_split_across_all_features(X=X, y_true=y_true, sample_weight=sample_weight)
        left_rows = X[:, best_split_feature_i] <= best_x_cutoff_value
        right_rows = X[:, best_split_feature_i] > best_x_cutoff_value
        X_left, X_right = X[left_rows], X[right_rows]
        y_true_left, y_true_right = y_true[left_rows], y_true[right_rows]
        sample_weight_left, sample_weight_right = sample_weight[left_rows], sample_weight[right_rows]
        curr_node = decision_tree_classifier_node(curr_depth = depth, curr_impurity = curr_impurity, curr_sample_size = curr_sample_size, curr_y_distribution = curr_y_distribution, best_split_feature_i = best_split_feature_i, best_x_cutoff_value = best_x_cutoff_value, y_class0_value=self.y_class0_value, y_class1_value=self.y_class1_value)
        # adding left and right children nodes into the node dict
        curr_node.left  = self.fit( X=X_left,  y=y_true_left,  sample_weight=sample_weight_left,  depth=depth+1)
        curr_node.right = self.fit( X=X_right, y=y_true_right, sample_weight=sample_weight_right, depth=depth+1)
        #print(parent_node)

        if depth == 0:
            self.root_node = curr_node
            return self # return the fitted estimator
        else:
            return curr_node

    def _order(self, curr_node, type="Inorder"):
        # Recursive travesal
        if curr_node:
            if type == "Inorder":
                return_dict = {}
                return_dict['left'] = self._order(curr_node.left, type=type)
                return_dict['curr'] = curr_node.to_dict()
                return_dict['right'] = self._order(curr_node.right, type=type)
                return return_dict
            if type == "Preorder":
                return_dict = {}
                return_dict['curr'] = curr_node.to_dict()
                return_dict['left'] = self._order(curr_node.left, type=type)
                return_dict['right'] = self._order(curr_node.right, type=type)
                return return_dict
            if type == "Postorder":
                return_dict = {}
                return_dict['left'] = self._order(curr_node.left, type=type)
                return_dict['right'] = self._order(curr_node.right, type=type)
                return_dict['curr'] = curr_node.to_dict()
                return return_dict

    def order(self, type="Inorder"):
        return self._order(curr_node=self.root_node, type=type)

    def _predict(self, one_X_row, proba=False):
        curr_node = self.root_node
        while curr_node.best_x_cutoff_value: # not a leaf node yet
            if one_X_row[ curr_node.best_split_feature_i ] <= curr_node.best_x_cutoff_value:
                curr_node = curr_node.left  # but what if curr_node.left = None?
            else:
                curr_node = curr_node.right # but what if curr_node.right = None?
        # arriving at a leaf node now
        if proba:
            return np.array([1-curr_node.y_class1_prob, curr_node.y_class1_prob])
        else:
            return np.array([curr_node.y_dominant_class])

    def predict(self, X, proba=False):
        if type(X) in [pd.DataFrame, pd.Series]:
            X = X.to_numpy()

        n_rows = X.shape[0]
        if proba:
            prediction = np.zeros(shape=(n_rows, 2))
        else:
            prediction = np.zeros(shape=(n_rows,))

        for this_row_i in range(n_rows):
            prediction[this_row_i] = self._predict(X[this_row_i,:], proba=proba)
        return prediction

    def predict_proba(self, X):
        return self.predict(X, proba=True)

    def score(self, X_test, y_test):
        if type(X_test) in [pd.DataFrame, pd.Series]:
            X_test = X_test.to_numpy()
        if type(y_test) in [pd.DataFrame, pd.Series]:
            y_test = y_test.to_numpy()
        from sklearn.metrics import confusion_matrix
        cm = confusion_matrix(y_test, self.predict(X_test))
        accuracy = np.trace(cm) / np.sum(cm)
        return accuracy

def decision_tree_classifier(*args, **kwargs):
    """
    """
    return DecisionTreeClassifier(*args, **kwargs)

#######################################################################################################################################

class decision_tree_regressor_node(object):
    def __init__(self, X, y, subset_sample_indices, min_samples_leaf=5, max_depth=None, curr_depth=0):
        """
        min_samples_leaf: The minimum number of samples required to be at a leaf node.
        """
        self.X = X
        self.y = y
        self.subset_sample_indices = subset_sample_indices
        self.min_samples_leaf = min_samples_leaf
        self.max_depth = max_depth
        self.curr_depth = curr_depth
        self.n_samples = len(subset_sample_indices)
        self.n_features = X.shape[1]
        self.predicted_value = np.mean(y[subset_sample_indices]) # the decision (prediction) is based on the value the node holds.
        # best_after_split_purity_score, the lower the better, meaning more homogeneous elements within left and right nodes
        self.best_after_split_purity_score = float('inf') # the score indicates how effect the split was. It will be the weighted sum of the variance in the left and right nodes. leaf nodes do not have a score, thus set to infinity.
        if self.max_depth is None:
            self.find_best_feature_to_split()
        elif self.curr_depth < self.max_depth:
            self.find_best_feature_to_split()

    def find_best_feature_to_split(self):
        for feature_index in range(self.n_features):
            self.check_if_better_split_feature_and_its_value(feature_index)
        if self.is_leaf_node:
            return
        best_split_feature_x_series = self.best_split_feature_x_series
        best_split_left_node_series_indices  = np.nonzero(best_split_feature_x_series <= self.best_split_feature_value)[0]
        best_split_right_node_series_indices = np.nonzero(best_split_feature_x_series >  self.best_split_feature_value)[0]
        # recursively find all the left and right children nodes
        self.left  = decision_tree_regressor_node(X=self.X, y=self.y, subset_sample_indices=self.subset_sample_indices[best_split_left_node_series_indices],  min_samples_leaf=self.min_samples_leaf, max_depth=self.max_depth, curr_depth=self.curr_depth+1)
        self.right = decision_tree_regressor_node(X=self.X, y=self.y, subset_sample_indices=self.subset_sample_indices[best_split_right_node_series_indices], min_samples_leaf=self.min_samples_leaf, max_depth=self.max_depth, curr_depth=self.curr_depth+1)

    def check_if_better_split_feature_and_its_value(self, feature_index):
        """
        try to see if this feature and a certain value of the feature is a better split;
        criterion: a lower "weighted averages of the standard deviations", which is equivalent to minimizing RMSE.
        """
        specific_x_series = self.X[self.subset_sample_indices, feature_index]

        for this_row in range(self.n_samples):
            specific_x_value_in_this_row = specific_x_series[this_row]
            left_node_series_indices  = specific_x_series <= specific_x_value_in_this_row # left_node  holds those in specific_x_series smaller than (or equal to) specific_x_value_in_this_row
            right_node_series_indices = specific_x_series >  specific_x_value_in_this_row # right_node holds those in specific_x_series larger than                specific_x_value_in_this_row
            n_samples_in_left_node  = left_node_series_indices.sum() # a shorthand way to count
            n_samples_in_right_node = right_node_series_indices.sum()
            # the leaf nodes must have at least n = "min_samples_left" samples
            if (n_samples_in_right_node < self.min_samples_leaf) or (n_samples_in_left_node < self.min_samples_leaf):
                continue

            # this_row is now a candidate as it satisfies the condition that the leaf nodes must have at least n = "min_samples_left" samples
            curr_after_split_purity_score = self.find_after_split_purity_score(left_node_series_indices, right_node_series_indices)
            if curr_after_split_purity_score < self.best_after_split_purity_score:
                # a better split has been found
                self.best_split_feature_index = feature_index
                self.best_after_split_purity_score = curr_after_split_purity_score
                self.best_split_feature_value = specific_x_value_in_this_row

    def find_after_split_purity_score(self, left_node_indices, right_node_indices):
        """
        the weighted sum of the variance of y within left and right nodes
        """
        y_subset = self.y[self.subset_sample_indices]
        left_node_variance  = y_subset[left_node_indices].var()
        right_node_variance = y_subset[right_node_indices].var()
        n_samples_in_left_node  = left_node_indices.sum()
        n_samples_in_right_node = right_node_indices.sum()
        return (left_node_variance * n_samples_in_left_node) + (right_node_variance * n_samples_in_right_node)

    @property
    def best_split_feature_x_series(self):
        return self.X[self.subset_sample_indices, self.best_split_feature_index]

    @property
    def is_leaf_node(self):
        return self.best_after_split_purity_score == float('inf')

    def predict(self, X_test):
        return np.array([self.predict_row(X_test_specific_row) for X_test_specific_row in X_test])

    def predict_row(self, X_test_specific_row):
        if self.is_leaf_node:
            return self.predicted_value
        child_node = self.left if X_test_specific_row[self.best_split_feature_index] <= self.best_split_feature_value else self.right
        return child_node.predict_row(X_test_specific_row)

class decision_tree_regressor_from_scratch(object):

    def __init__(self, min_samples_leaf=5, max_depth=None):
        """
        min_samples_leaf: The minimum number of samples required to be at a leaf node.
        """
        self.X_train = None
        self.y_train = None
        self.DT_root_node = None
        self.min_samples_leaf = min_samples_leaf
        self.max_depth = max_depth

    def fit(self, X, y):
        self.X_train = X
        self.y_train = y
        if type(self.X_train) in [pd.DataFrame, pd.Series]:
            self.X_train = self.X_train.to_numpy()
        if type(self.y_train) in [pd.DataFrame, pd.Series]:
            self.y_train = self.y_train.to_numpy()
        n_samples = len(self.y_train)
        self.DT_root_node = decision_tree_regressor_node(X=self.X_train, y=self.y_train, subset_sample_indices=np.arange(n_samples), min_samples_leaf=self.min_samples_leaf, max_depth=self.max_depth)
        return self

    def predict(self, X):
        X_test = X
        if type(X_test) in [pd.DataFrame, pd.Series]:
            X_test = X_test.to_numpy()
        return self.DT_root_node.predict(X_test)

def decision_tree_regressor(*args, **kwargs):
    """
    """
    return DecisionTreeRegressor(*args, **kwargs)

#######################################################################################################################################

def _demo(dataset="Social_Network_Ads", classifier_func="decision_tree"): # DT: decision_tree
    """
    classifier_func: "decision_tree" or "DT", "GBM", "AdaBoost", "bagging"
    """
    from ..datasets import public_dataset
    from ..ensemble import bagging_classifier, random_forest_classifier, adaptive_boosting_classifier, gradient_boosting_classifier

    if dataset == "iris":
        data = public_dataset(name="iris")
        y_classes = ['setosa', 'versicolor', 'virginica']
        X = data[['sepal length (cm)', 'sepal width (cm)', 'petal length (cm)', 'petal width (cm)']]
        y = data['target']

    if dataset == "Social_Network_Ads":
        data = public_dataset(name="Social_Network_Ads")
        print(f"{data.head()}\n")
        del data['User ID']
        # Recode the data: Gender as Male
        mapper = {'Male': 1, 'Female': 0}
        data['Male'] = data['Gender'].map(mapper)
        # pairplot
        import seaborn as sns
        sns.pairplot(data, hue="Purchased", markers=["o", "s"])
        import matplotlib.pyplot as plt
        plt.show()
        # X and y
        X = data[['Age', 'EstimatedSalary']] # dropping Male to simplify the analysis
        y = data['Purchased']
        y_classes = ['not_purchased (y=0)', 'purchased (y=1)']

    if dataset == "bank_note_authentication":
        data = public_dataset(name="bank_note_authentication")
        y_classes = ['genuine (y=0)', 'forged (y=1)']
        X = data[['variance', 'skewness', 'curtosis', 'entropy']]
        y = data['class']
        import seaborn as sns
        sns.pairplot(data, hue="class", markers=["o", "s"])

    from sklearn.model_selection import train_test_split, GridSearchCV
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.25, random_state=123)

    # Feature scaling
    from sklearn.preprocessing import StandardScaler
    # scaler = StandardScaler()
    # X_train = scaler.fit_transform(X_train) # fit to the data and then transform it
    # X_test = scaler.transform(X_test) # 	uses a previously computed mean and std to scale the data

    # create pipeline
    from sklearn.pipeline import Pipeline

    ########################################################################################################################
    if classifier_func in ["decision_tree", "DT"]:
        pipeline = Pipeline(steps=[('scaler', StandardScaler(with_mean=True, with_std=True)),
                                   ('classifier', decision_tree_classifier(max_depth=1, random_state=123)),  # default criterion = 'gini'
                                   ])

        # pipeline parameters to tune
        hyperparameters={
            'scaler__with_mean': [True],
            'scaler__with_std': [True],
            'classifier__criterion': ("gini", "entropy",),
            'classifier__max_depth': range(1, 15),
        }

        model_name = "Decision Tree"

    ########################################################################################################################
    if classifier_func == "bagging":
        pipeline = Pipeline(steps=[('scaler', StandardScaler(with_mean=True, with_std=True)),
                                   ('classifier', bagging_classifier(random_state=123)),
                                   ])

        # pipeline parameters to tune
        hyperparameters = {
            'scaler__with_mean': [True],
            'scaler__with_std': [True],
        }

        model_name = "Bagging"

    ########################################################################################################################
    if classifier_func in ["random_forest"]:
        pipeline = Pipeline(steps=[('scaler', StandardScaler(with_mean=True, with_std=True)),
                                   ('classifier', random_forest_classifier(max_depth=1, random_state=123)),  # default criterion = 'gini'
                                   ])

        # pipeline parameters to tune
        hyperparameters={
            'scaler__with_mean': [True],
            'scaler__with_std': [True],
            'classifier__criterion': ("gini", "entropy",),
            'classifier__max_depth': range(1, 10),
        }

        model_name = "Random Forest"

    ########################################################################################################################
    if classifier_func == "AdaBoost":
        pipeline = Pipeline(steps=[('scaler', StandardScaler(with_mean=True, with_std=True)),
                                   ('classifier', adaptive_boosting_classifier(random_state=123)),
                                   ])

        # pipeline parameters to tune
        hyperparameters = {
            'scaler__with_mean': [True],
            'scaler__with_std': [True],
        }

        model_name = "Adaptive Boosting"

    ########################################################################################################################
    if classifier_func == "GBM":
        pipeline = Pipeline(steps=[('scaler', StandardScaler(with_mean=True, with_std=True)),
                                   ('classifier', gradient_boosting_classifier(max_depth=1, random_state=123)),
                                   ])

        # pipeline parameters to tune
        hyperparameters = {
            'scaler__with_mean': [True],
            'scaler__with_std': [True],
            'classifier__criterion': ("friedman_mse", "mse", "mae"),
            'classifier__max_depth': range(1, 10),
        }

        model_name = "Gradient Boosting"

    ########################################################################################################################

    grid=GridSearchCV(
        pipeline,
        hyperparameters,  # parameters to tune via cross validation
        refit=True,       # fit using all data, on the best detected classifier
        n_jobs=-1,
        scoring='accuracy',
        cv=5,
    )

    # Training and predicting
    # classifier = DecisionTreeClassifier(max_depth=3) # default criterion = 'gini'
    # classifier = classifier.fit(X_train,y_train)
    classifier_grid=grid.fit(X_train, y_train)

    ########################################################################################################################
    if classifier_func in ["decision_tree", "DT", "random_forest", "GBM"]:
        criterion=classifier_grid.best_params_['classifier__criterion']
        max_depth=classifier_grid.best_params_['classifier__max_depth']
        print(
            f"Using a grid search and a {model_name} classifier, the best hyperparameters were found as following:\n"
            f"Step1: scaler: StandardScaler(with_mean={repr(classifier_grid.best_params_['scaler__with_mean'])}, with_std={repr(classifier_grid.best_params_['scaler__with_std'])});\n"
            f"Step2: classifier: {classifier_func}(criterion={repr(criterion)}, max_depth={repr(max_depth)}).\n")
        model_desc = f"{model_name} (criterion={repr(criterion)}, max_depth={repr(max_depth)})"

    if classifier_func in ["AdaBoost", "bagging"]:
        print(
            f"Using a grid search and a {model_name} classifier, the best hyperparameters were found as following:\n"
            f"Step1: scaler: StandardScaler(with_mean={repr(classifier_grid.best_params_['scaler__with_mean'])}, with_std={repr(classifier_grid.best_params_['scaler__with_std'])});\n"
            f"Step2: classifier: {classifier_func}().\n")
        model_desc = f"{model_name}"

    ########################################################################################################################

    y_pred=classifier_grid.predict(X_test)
    y_pred_score=classifier_grid.predict_proba(X_test)

    # Model evaluation
    from ..model_evaluation import plot_confusion_matrix, plot_ROC_and_PR_curves, visualize_classifier_decision_boundary_with_two_features
    plot_confusion_matrix(y_true=y_test, y_pred=y_pred, y_classes=y_classes)

    if dataset in ['Social_Network_Ads', 'bank_note_authentication']:
        plot_ROC_and_PR_curves(fitted_model=classifier_grid, X=X_test,
                            y_true=y_test, y_pred_score=y_pred_score[:, 1], y_pos_label=1, model_name=f"{model_name}")

    if dataset in ['Social_Network_Ads',]:
        visualize_classifier_decision_boundary_with_two_features(
            classifier_grid, X_train.to_numpy(), y_train.to_numpy(), y_classes, title=f"{model_desc} / training set", X1_lab='Age', X2_lab='Estimated Salary')
        visualize_classifier_decision_boundary_with_two_features(
            classifier_grid, X_test.to_numpy(),  y_test.to_numpy(),  y_classes, title=f"{model_desc} / testing set",  X1_lab='Age', X2_lab='Estimated Salary')

    # Plotting the tree
    if classifier_func in ["decision_tree", "DT"]:

        if dataset == 'iris':
            feature_cols = list(X.columns)
            #class_names = ['0', '1', '2']
            class_names = ['0=setosa', '1=versicolor', '2=virginica']

        if dataset == 'bank_note_authentication':
            feature_cols = list(X.columns)
            #class_names = ['0', '1']
            class_names = ['0=genuine', '1=forged']

        if dataset == 'Social_Network_Ads':
            feature_cols=['Age', 'EstimatedSalary']
            #class_names = ['0', '1']
            class_names = ['0=not purchased', '1=purchased']

        # Approach 1
        from dtreeviz.trees import dtreeviz
        viz = dtreeviz(classifier_grid.best_estimator_.steps[1][1], StandardScaler().fit_transform(X_train), y_train, target_name="target", feature_names=feature_cols, class_names=class_names)
        # print(type(viz)) # <class 'dtreeviz.trees.DTreeViz'>
        viz.view()

        # Approach 2
        from sklearn.tree import export_graphviz
        import io
        import pydotplus
        # from IPython.display import Image
        dot_data=io.StringIO()

        export_graphviz(classifier_grid.best_estimator_.steps[1][1], out_file=dot_data,
                        filled=True, rounded=True,
                        special_characters=True, feature_names=feature_cols, class_names=class_names)

        graph=pydotplus.graph_from_dot_data(dot_data.getvalue())
        image=graph.create_png()
        from PIL import Image
        Image.open(io.BytesIO(image)).show()

def demo(dataset="Social_Network_Ads", classifier_func="decision_tree"):
    """
    This function provides a demo of selected functions in this module.

    Required arguments:
        - dataset:         A string. Possible values: "Social_Network_Ads", "iris", "bank_note_authentication"
        - classifier_func: A string. Possible values: "decision_tree" or "DT", "random_forest", "bagging", "AdaBoost", "GBM"
    """

    available_datasets = ("Social_Network_Ads","iris","bank_note_authentication")
    available_classifier_functions = ("decision_tree", "DT", "random_forest", "bagging", "AdaBoost", "GBM",)

    if dataset in available_datasets and classifier_func in available_classifier_functions:
        return _demo(dataset = dataset, classifier_func = classifier_func)
    else:
        raise TypeError(f"either dataset [{dataset}] or classifier function [{classifier_func}] is not defined")

def demo_from_scratch(question_type="classification"):
    """
    "classification" or "regression"
    """
    if question_type not in ['classification', 'regression',]:
        raise ValueError('question_type should be either classification or regression')

    if question_type == "regression":
        from ..datasets import public_dataset
        df = public_dataset(name="boston")
        print(f"{df.head()}\n")
        feature_names = df.columns.drop('MEDV')
        X, y = df[feature_names], df['MEDV']

        for model in [decision_tree_regressor_from_scratch(min_samples_leaf=5), decision_tree_regressor(min_samples_leaf=5)]:
            print(f"------ model: {repr(model)} ------")
            model.fit(X, y)
            y_pred = model.predict(X)
            from ..model_evaluation import evaluate_continuous_prediction
            RMSE, R_squared = evaluate_continuous_prediction(y, y_pred)
            print(f"R_squared = {R_squared:.3f}, RMSE = {RMSE:.3f}")

    if question_type == "classification":
        impurity_measure = 'entropy'  # impurity_measure: 'entropy' or 'gini_impurity'
        max_depth = 2

        from ..datasets import public_dataset
        data = public_dataset('Social_Network_Ads')
        X = data[['Age', 'EstimatedSalary']]
        y = data['Purchased']
        y_classes = ['not_purchased (y=0)', 'purchased (y=1)']
        from sklearn.model_selection import train_test_split
        X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.25, random_state=123)

        DT_model = decision_tree_classifier_from_scratch(impurity_measure=impurity_measure, max_depth = max_depth)
        from sklearn.preprocessing import scale
        print(DT_model.find_best_split_in_one_specific_feature(x=scale(X_train['Age']), y_true=y_train))
        print(DT_model.find_best_split_across_all_features(X=scale(X_train), y_true=y_train))

        DT_model.fit(X_train, y_train)
        print(DT_model.order(type="Preorder"))
        print(f"\nAccuracy in predicting the target in the testing set: {DT_model.score(X_test, y_test)}")

        #print(DT_model.predict(X_train))
        #print(DT_model.predict_proba(X_train))
        from ..model_evaluation import plot_confusion_matrix, plot_ROC_and_PR_curves
        y_pred_score = DT_model.predict_proba(X_test)
        plot_confusion_matrix(y_true=y_test, y_pred=DT_model.predict(X_test), y_classes=y_classes)
        plot_ROC_and_PR_curves(fitted_model=DT_model, X=X_test, y_true=y_test, y_pred_score=y_pred_score[:,1], model_name = 'DT from scratch')

#
# References
#
# decision tree:
#
# 1. Gini impurity:
# https://towardsdatascience.com/gini-impurity-measure-dbd3878ead33
# http://www.learnbymarketing.com/481/decision-tree-flavors-gini-info-gain/
# https://towardsdatascience.com/gini-index-vs-information-entropy-7a7e4fed3fcb
# https://medium.com/analytics-steps/understanding-the-gini-index-and-information-gain-in-decision-trees-ab4720518ba8
#
# 2. Entropy:
# https://www.geeksforgeeks.org/gini-impurity-and-entropy-in-decision-tree-ml/
#
# 3. Gini vs. Entropy
# https://datascience.stackexchange.com/questions/10228/when-should-i-use-gini-impurity-as-opposed-to-information-gain-entropy
#
# 4. DT from scratch in Python:
# https://medium.com/@penggongting/implementing-decision-tree-from-scratch-in-python-c732e7c69aea
# https://towardsdatascience.com/decision-tree-from-scratch-in-python-46e99dfea775
# https://machinelearningmastery.com/implement-decision-tree-algorithm-scratch-python/
#
# 5. Decision Tree Regressor in Python
# https://levelup.gitconnected.com/building-a-decision-tree-from-scratch-in-python-machine-learning-from-scratch-part-ii-6e2e56265b19
#

# References
#
# Ensemble methods:
#
# 1. Methods evolution and comparison
# https://scikit-learn.org/stable/modules/ensemble.html#bagging-meta-estimator
# https://towardsdatascience.com/decision-tree-ensembles-bagging-and-boosting-266a8ba60fd9


import numpy as np
import matplotlib.pyplot as plt
import matplotlib.animation as animation
from pylab import rcParams
rcParams['figure.figsize'] = 16, 9

fig, ax = plt.subplots(1,1)
plt.axis('equal')

ax.set_xlim([0, 2*np.pi])
ax.set_ylim([-2.5, 2.5])

# sine function plot
x = np.linspace(0, 2*np.pi, 100)
y = np.sin(x)

# x axis path
ax.plot(3*x - 3, 0*y, linewidth=1, color='#dddddd')
# y axis path
ax.plot(0*x, 2.5*y, linewidth=1, color='#dddddd')

# unit circle path
ax.plot(np.cos(x), np.sin(x), linewidth=1)
# sine path
ax.plot(x + 1, np.sin(x), linewidth=1)

# ------ anim -------
sineLine, = ax.plot([], [], linewidth=4)
sineDot, = ax.plot([], [], 'o', color='#ff0000')

circleLine, = ax.plot([], [],linewidth=4)
circleDot, = ax.plot([], [], 'o', color='black')

def sineAnim(i):
    # sine anim
    sineLine.set_data(x[:i] + 1,y[:i])
    sineDot.set_data(x[i] + 1, y[i])
    # circle anim
    circleLine.set_data(np.cos(x[:i]), np.sin(x[:i]))
    circleDot.set_data(np.cos(x[i]), np.sin(x[i]))

anim = animation.FuncAnimation(fig, sineAnim, frames=len(x), interval=50)
# -------------

anim.save('sine-py-effect.mp4', writer='ffmpeg')

# plt.show()
# HTML(anim.to_html5_video())


import numpy as np
from scipy.stats.distributions import norm

"""
    Simulate from a Multivariate Normal Distribution

    Produces one or more samples from the specified multivariate normal distribution.

    B. D. Ripley (1987) Stochastic Simulation. Wiley. Page 98.

    Borrowed from R: mvrnorm {MASS}

    TODO: numba
"""
def mvrnorm(n, mu, Sigma, tol = 1e-6):
    p = len(mu)
    # TODO: add exception: (p, p) == Sigma.shape
    eigValues, eigVectors = np.linalg.eig(Sigma)
    # TODO: check positive definite: eigen values > tol * abs(eigV[0])
    X = np.array(norm.rvs(size = n * p)).reshape(n, p)
    eigValues[eigValues < 0] = 0
    eigValues = np.sqrt(eigValues)
    X = mu + np.dot(np.dot(eigVectors, np.diag(eigValues)), X.T).T
    return X