from numpy.random import randn
import numpy as np
import scipy as sp
from matrix_util import *

def Ginibre(M, N, COMPLEX=False):
    if COMPLEX==True:
        out=(randn(M,N) + 1j*randn(M,N) )/ sp.sqrt(2*N)
    else:
        out=randn(M,N)/ sp.sqrt(N)
    return np.matrix(out)

def haar_unitary(M, COMPLEX=False):
    G = Ginibre(M,M,COMPLEX)
    U,D,V = np.linalg.svd(G)
    return U

def random_from_diag(M,N,zero_dim=0, min_singular=0,COMPLEX=False):
    assert zero_dim <= min(M,N)
    array  = np.random.uniform( min_singular, 1, N)
    for i in range(zero_dim):
        array[i] = 0
    D = rectangular_diag(array, M,N)
    U = haar_unitary(M, COMPLEX)
    V = haar_unitary(N, COMPLEX)
    return U @ D @ V

def signal_plus_noise_symm(p_dim , dim, param_mat, sigma=1, COMPLEX=False):
    out = np.zeros([2*p_dim, 2*p_dim])
    C = Ginibre(p_dim , dim , COMPLEX)
    for i in range(p_dim):
        for j in range(dim):
            out[p_dim + i][j] = C[i,j]
            out[j][p_dim +i] = np.conj(C[i,j])

    return sigma*out

def signal_plus_noise(param_mat,sigma=1,COMPLEX=False, RAW=False):
    p_dim = param_mat.shape[0]
    dim = param_mat.shape[1]
    assert np.allclose(param_mat.shape, [p_dim,dim])
    X = Ginibre(p_dim, dim,COMPLEX)
    P = param_mat + sigma*X
    if RAW:
        return P
    else:
        if COMPLEX:
            Y = P.H.dot(P)
        else:
            Y = P.T.dot(P)

        return Y


#Nabin Chapagain
#1001551151

#knn_classify(<training_file>, <test_file>, <k>)

# Importing all needed libraries
import numpy as np
import math
import sys
import random
from scipy import stats
from scipy.spatial import distance
import statistics as s
from statistics import mean, median, mode, stdev

fname = sys.argv[1]
fname1 = sys.argv[2]
k_nearest = int(sys.argv[3])

mat_train = np.loadtxt(fname)
mat_train_prime = mat_train[:,0:-1]
mat_test= np.loadtxt(fname1)
mat_test_prime = mat_test[:,0:-1]

no_train_mat_row = len(mat_train)
no_train_mat_col = len(mat_train[0])

mean_arr = np.zeros(no_train_mat_col-1)
std_arr = np.zeros(no_train_mat_col-1)
std_in_between = 0

#getting mean and standard deviation for each attribute before normalizing

for i in range (0,no_train_mat_col-1):
    mean_arr[i] = mean(mat_train[:,i])
    std_in_between = stdev(mat_train[:,i])
    if (std_in_between == 0):
        std_in_between = 1
    std_arr[i] = std_in_between

#next step, normalizing the values

for j in range(0,no_train_mat_col-1):
    mat_train[:,j] = (mat_train[:,j] - mean_arr[j])/std_arr[j]
    # F(v) = (v - mean)/std

for j in range(0,no_train_mat_col-1):
    mat_test[:,j] = (mat_test[:,j] - mean_arr[j])/std_arr[j]
    # F(v) = (v - mean)/std

#here, we have normalized training matrix and test matrix

classification_accuracy = 0

for i in range(0,len(mat_test)):
    E_dist = np.zeros(no_train_mat_row)
    for k in range(len(E_dist)):
        E_dist[k]= distance.euclidean(mat_test_prime[i],mat_train_prime[k])

    accuracy = 0
    true_class =  mat_test[i][-1]

    k_nearest_points = E_dist.argsort()[:k_nearest]
    #This is the index of k_nearest_points number of lowest numbers in the list E_dist

    predicted_array = mat_train[k_nearest_points,-1]

    accuracy = 0
    mode_class = s.multimode(predicted_array)
    mode_class.sort()
    #print(mode_class,"\n")
    if(len(mode_class) == 1):
        predicted_class = mode_class[0]

        if(predicted_class == true_class):
            accuracy = 1
    else:
        #now to deal with ties
        for m in range(0,len(mode_class)):
            predicted_class = mode_class[m]
            if(true_class == predicted_class):
                accuracy = len(np.where(true_class == mode_class))/len(mode_class)
                break

    object_id = i+1
    print('ID=%5d, predicted=%3d, true=%3d, accuracy=%4.2f'%( object_id, predicted_class, true_class, accuracy))
    classification_accuracy+=accuracy
print('classification accuracy=%6.4f\n'% (classification_accuracy/len(mat_test)))


from sklearn.base import BaseEstimator, TransformerMixin
from mlpouch import BaseRecommender
import logging
import numpy as np
import pandas as pd

class FunkSVD(TransformerMixin, BaseEstimator, BaseRecommender):

    def __init__(self, latent_features = 4, learning_rate = 0.0001, iters = 100):
        self.latent_features = latent_features
        self.learning_rate = learning_rate
        self.iters = iters

    def fit(self, X, y = None):

        X_matrix = self.fit_transform(X, y)

        X_transformed = pd.DataFrame(X_matrix)
        X_transformed[self.U_index.name] = self.U_index.array
        X_transformed = X_transformed.set_index(self.U_index.name)
        X_transformed.columns = self.VT_index

        self.X_transformed = X_transformed

        return self

    def fit_transform(self, X, y = None):

        self.U_index = X.index
        self.VT_index = X.columns

        # Convert to matrix
        X = np.matrix(X)

        # Set up useful values to be used through the rest of the function
        n_U = X.shape[0]
        n_VT = X.shape[1]
        with_values = np.count_nonzero(~np.isnan(X))

        # initialize the user and movie matrices with random values
        U_matrix = np.random.rand(n_U, self.latent_features)
        VT_matrix = np.random.rand(self.latent_features, n_VT)

        # initialize sse at 0 for first iteration
        sse_accum = 0

        # keep track of iteration and MSE
        logging.debug("Optimization Statistics")
        logging.debug("Iterations | Mean Squared Error ")

        # for each iteration
        for iteration in range(self.iters):

            # update our sse
            old_sse = sse_accum
            sse_accum = 0

            # For each user-movie pair
            for i in range(n_U):
                for j in range(n_VT):

                    # if the rating exists
                    if X[i, j] > 0:

                        # compute the error as the actual minus the dot product of the user and movie latent features
                        diff = X[i, j] - np.dot(U_matrix[i, :], VT_matrix[:, j])

                        # Keep track of the sum of squared errors for the matrix
                        sse_accum += diff**2

                        # update the values in each matrix in the direction of the gradient
                        for k in range(self.latent_features):
                            U_matrix[i, k] += self.learning_rate * (2 * diff * VT_matrix[k, j])
                            VT_matrix[k, j] += self.learning_rate * (2 * diff * U_matrix[i, k])

            self.sse_score = sse_accum / with_values

            # print results
            logging.debug(f"{iteration + 1} \t\t {self.sse_score}")

        self.U_matrix = U_matrix
        self.VT_matrix = VT_matrix

        X_matrix = np.dot(self.U_matrix, self.VT_matrix)

        return X_matrix

    def recommend(self, U, rec_num = 5):
        recommends = []

        for U_item in U:
            recommends.append([ U_item, self._recommend(U_item, rec_num) ])

        return recommends

    def _recommend(self, U_item, rec_num = 5):
        return self.X_transformed.loc[U_item] \
            .sort_values(ascending = False) \
            .head(rec_num) \
            .index.tolist()


#!/usr/bin/env python3
# -*- coding: utf-8 -*-

import numpy as np
import matplotlib.pyplot as plt
np.set_printoptions(threshold=np.nan)
np.set_printoptions(precision=5)
from scipy import integrate
from scipy import io

#%%
def f(t,u):
    return np.array([-1000*u[0] + u[1], -u[1]/10])

#<startRK>
def r_k(f,u0,N,T):
    k = T/N
    u = np.zeros((len(u0),N+1))
    u[:,0] = u0
    for n in range(N):
        tn = k*n
        Y1 = u[:,n]
        Y2 = u[:,n] + k/2*f(tn,Y1)
        Y3 = u[:,n] + k/2*f(tn+k/2,Y2)
        Y4 = u[:,n] + k*f(tn+k/2,Y3)
        u[:,n+1] = u[:,n] + k/6*( f(tn,Y1) + 2*f(tn+k/2,Y2) + 2*f(tn+k/2,Y3) + f(tn+k,Y4))
    return u
#<endRK>

#%%
T = 1
u0 = [1,2]
error_rk = []
Ns = np.array([51,375,500,1200])
for N in Ns:
    t = np.linspace(0,T,N+1)
    u_true = np.array([9979/9999*np.exp(-1000*t)+20/9999*np.exp(-t/10),2*np.exp(-t/10)])
    u_rk = r_k(f,u0,N,T)

    error_rk.append(np.linalg.norm(u_true-u_rk,np.inf))

    plt.figure()
    ax = plt.gca()
    ax.plot(t,u_true[0],color='.7')
    ax.plot(t,u_rk[0],color='.1',marker='o',ms=3,linestyle='None')

    ax.plot(t,u_true[1],color='.7')
    ax.plot(t,u_rk[1],color='.1',marker='s',ms=3,linestyle='None')

#        ax.set_xscale('log')
    ax.set_yscale('log')
    plt.savefig('img/5/N'+str(N)+'.pdf')

#%%
sol = integrate.solve_ivp(f,[0,1],[1,2],method='RK23')

t_23 = sol.t
u_23 = sol.y
u_true = np.array([9979/9999*np.exp(-1000*t_23)+20/9999*np.exp(-t_23/10),2*np.exp(-t_23/10)])
error_23 = np.linalg.norm(u_true-u_23,np.inf)

ax = plt.gca()
ax.plot(t_23,u_true[0],color='.5')
ax.plot(t_23,u_23[0],color='.1',marker='o',ms=3,linestyle='None')

ax.plot(t_23,u_true[1],color='.5')
ax.plot(t_23,u_23[1],color='.1',marker='s',ms=3,linestyle='None')

#        ax.set_xscale('log')
ax.set_yscale('log')

plt.savefig('img/5/RK23.pdf')

plt.figure()
ax = plt.gca()
ax.plot(np.linspace(0,1,len(t_23)),t_23,'k')
plt.savefig('img/5/RK23_t.pdf')

#%%

tu = io.loadmat('tu.mat')
t_23s = tu['t'][:,0]
u_23s = tu['u'].T

u_true = np.array([9979/9999*np.exp(-1000*t_23s)+20/9999*np.exp(-t_23s/10),2*np.exp(-t_23s/10)])
error_23s = np.linalg.norm(u_true-u_23s,np.inf)

ax = plt.gca()
ax.plot(t_23s,u_true[0],color='.5')
ax.plot(t_23s,u_23s[0],color='.1',marker='o',ms=3,linestyle='None')

ax.plot(t_23s,u_true[1],color='.5')
ax.plot(t_23s,u_23s[1],color='.1',marker='s',ms=3,linestyle='None')

#        ax.set_xscale('log')
ax.set_yscale('log')

plt.savefig('img/5/ode23s.pdf')

plt.figure()
ax = plt.gca()
ax.plot(np.linspace(0,1,len(t_23s)),t_23s,'k')
plt.savefig('img/5/ode23s_t.pdf')


from sage.all import *

def bsgs(g, y, p):
    """
    Reference:

    To solve DLP: y = g^x % p and get the value of x.
    We use the property that x = i*m + j, where m = ceil(sqrt(n))

    :parameters:
        g : int/long
                Generator of the group
        y : int/long
                Result of g**x % p
        p : int/long
                Group over which DLP is generated. Commonly p is a prime number

    :variables:
        m : int/long
                Upper limit of baby steps / giant steps
        x_poss : int/long
                Values calculated in each giant step
        c : int/long
                Giant Step pre-computation: c = g^(-m) % p
        i, j : int/long
                Giant Step, Baby Step variables
        lookup_table: dictionary
                Dictionary storing all the values computed in the baby step
    """
    mod_size = len(bin(p-1)[2:])

    print "[+] Using BSGS algorithm to solve DLP"
    print "[+] Modulus size: " + str(mod_size) + ". Warning! BSGS not space efficient\n"

    m = ceil(sqrt(p-1))
    # Baby Step
    lookup_table = {pow(g, j, p): j for j in range(m)}
    # Giant Step pre-computation
    c = pow(g, m*(p-2), p)
    # Giant Steps
    for i in range(m):
        temp = (y*pow(c, i, p)) % p
        if temp in lookup_table:
            # x found
            return i*m + lookup_table[temp]
    return None

if __name__ == "__main__":
    try:
        assert pow(2, bsgs(2, 4178319614, 6971096459), 6971096459) == 4178319614
        assert pow(3, bsgs(3, 362073897, 2500000001), 2500000001) == 362073897
    except:
        print "[+] Function inconsistent and incorrect, check the implementation"


from math import pi, radians, degrees, sin, cos, atan, sqrt, sinh, cosh, asinh
import numpy as np
from scipy.integrate import quadrature
from scipy.optimize import newton
import matplotlib.pyplot as plt

# Parameters of projectile (Modelled after a baseball)
V_0     = 44.7                 # Initial velocity (m/s)
g       = 9.81                 # Acceleration due to gravity (m/s^2)
psi     = 75                   # Launch angle (deg.)
c       = 0.5                  # Drag coefficient (spherical projectile)
r       = 0.0366               # Radius of projectile (m)
m       = 0.145                # Mass of projectile (kg)
rho_air = 1.29                 # Air density (kg/m^3)
a       = pi * r**2.0          # Cross-sectional area of projectile (m^2)
psi     = radians(psi)         # Convert to radians

# Display of parameters
print('Parameters:')
print('Launch angle (deg.)                      : {:.3f}'.format(degrees(psi)))
print('Launch speed (m/s)                       : {:.3f}'.format(V_0))
print('Drag coefficient - Spherical projectile  : {:.3f}'.format(c))
print('Radius of spherical projectile (m)       : {:.3f}'.format(r))
print('Mass of projectile (kg)                  : {:.3f}'.format(m))
print('Air density (kg/m^3)                     : {:.3f}'.format(rho_air))
print('Cross-sectional area of projectile       : {:.3f}'.format(a))

# Initial position and launch velocity
x_0 = 0.0
u_0 = V_0 * cos(psi)
y_0 = 0.0
v_0 = V_0 * sin(psi)

# Constants and function definitions for solution
mu = 0.5 * c * rho_air * a / m
Q_0 = asinh(v_0 / u_0)
A   = g / (mu * u_0**2.0) + (Q_0 + 0.5 * sinh(2.0 * Q_0))

def lam(Q):
    return A - (Q + 0.5 * sinh(2.0 * Q))

def u_s(Q):
    return sqrt(g / mu) / sqrt(lam(Q))

def v_s(Q):
    return sqrt(g / mu) * sinh(Q) / sqrt(lam(Q))

def f_t(Q):
    return cosh(Q) / sqrt(lam(Q))

def f_x(Q):
    return cosh(Q) / lam(Q)

def f_y(Q):
    return sinh(2.0 * Q) / lam(Q)

def t_s(Q):
    return - quadrature(f_t, Q_0, Q, vec_func=False)[0] / sqrt(g * mu)

def x_s(Q):
    return x_0 - quadrature(f_x, Q_0, Q, vec_func=False)[0] / mu

def y_s(Q):
    return y_0 - quadrature(f_y, Q_0, Q, vec_func=False)[0] / (2.0 * mu)

def y_s_p(Q):
    return -(1.0 / (2.0 * mu)) * sinh(2.0 * Q) / lam(Q)

# Time of flight
Q_T_est = asinh(-v_0 / u_0)      # Initial estimate for Newton's method
Q_T = newton(y_s, Q_T_est, y_s_p)
T = t_s(Q_T)
print('\nResults:')
print('Time of flight (s)                       : {:.3f}'.format(T))

# Horizontal range
R = x_s(Q_T)
print('Horizontal range (m)                     : {:.3f}'.format(R))

# Maximum height
H = y_s(0.0)
print('Maximum height (m)                       : {:.3f}'.format(H))

# Vectorize scalar functions
t_vec = np.vectorize(t_s)
x_vec = np.vectorize(x_s)
y_vec = np.vectorize(y_s)
u_vec = np.vectorize(u_s)
v_vec = np.vectorize(v_s)

# Array for variable 'Q'
N = 101
psi_T = degrees(atan(sinh(Q_T)))
Q = np.arcsinh(np.tan(np.radians(np.linspace(degrees(psi), psi_T, N))))

# Arrays for projectile path variables
t = t_vec(Q)
x = x_vec(Q)
y = y_vec(Q)
u = u_vec(Q)
v = v_vec(Q)

# Plot of trajectory
fig, ax = plt.subplots()
line, = ax.plot(x, y, 'r-', label='Numerical')
ax.set_title(r'Projectile path')
ax.set_aspect('equal')
ax.grid(b=True)
ax.legend()
ax.set_xlabel('x (m)')
ax.set_ylabel('y (m)')
plt.show()

# Plot of velocity components
fig, ax = plt.subplots()
line, = ax.plot(t, u, 'b-', label='u')
ax.set_title(r'Horizontal velocity component')
ax.grid(b=True)
ax.legend()
ax.set_xlabel('t (s)')
ax.set_ylabel('u (m/s)')
plt.show()

fig, ax = plt.subplots()
line, = ax.plot(t, v, 'b-', label='v')
ax.set_title(r'Vertical velocity component')
ax.grid(b=True)
ax.legend()
ax.set_xlabel('t (s)')
ax.set_ylabel('v (m/s)')
plt.show()


import numpy as np
m = np.matrix([[1,-2,3],[0,4,5],[7,8,-9]])
print( m )
#transpose
print( m.T )
#inverse
print( m.I )
#vector
v = np.matrix([[2],[3],[4]])
print( v )

print( m * v )

import numpy.linalg

#determinant
print(numpy.linalg.det( m ))

#Eigenvalues
print( numpy.linalg.eigvals( m ) )

#solve x for mx=v
x = numpy.linalg.solve( m,v)
print( x )
print( m * x )

chapter3/3_10Performing Matrix and Linear Algebra Calculations/3_10.py


from math import sqrt
import stats_batch as sb
import numpy as np
from pytest import approx
from scipy.stats import ttest_ind_from_stats
from scipy.stats import ttest_ind

def test_batch_mean_var_t_test():
    n = 10_000
    a = np.random.normal(size=n)
    b = np.random.normal(size=n)

    # First batch
    # a -----------
    batch_1_a = a[:100]
    a_current = sb.mean_var_batch(batch_1_a)
    # b -----------
    batch_1_b = b[:100]
    b_current = sb.mean_var_batch(batch_1_b)

    # Second batch
    # a -----------
    batch_2_a = a[100:n]
    a_current.update(batch_2_a)
    assert a_current.mean == approx(np.mean(a))
    assert a_current.var  == approx(np.var(a), rel = 1e-3)

    # b -----------
    batch_2_b = b[100:n]
    b_current.update(batch_2_b)
    assert b_current.mean == approx(np.mean(b))
    assert b_current.var == approx(np.var(b), rel = 1e-3)

    # t-test
    batch_t = a_current.ttest_ind(b_current)
    list_t = ttest_ind(a, b)
    assert batch_t[0] == approx(list_t[0])
    assert batch_t[1] == approx(list_t[1])


from math import sqrt, asin, acos, sin, cos, pi, atan2

from scipy.optimize import fsolve

R = 6.378138 * 10 ** 8
f = 0.003353

def wgs2xyz(lat, long):
    global R
    global f

    latRad = lat * pi / 180
    longRad = long * pi / 180

    theta = fsolve(lambda thetaIt: latRad - thetaIt - f * sin(2 * thetaIt), latRad)
    theta = theta[0]

    Re = R * (1 + (1 / (1 - f) ** 2 - 1) * sin(theta)) ** (-1 / 2)

    x = Re * cos(theta) * cos(longRad)
    y = Re * cos(theta) * sin(longRad)
    z = Re * sin(theta)

    return x, y, z

def xyz2wgs(x, y, z):
    global R
    global f

    Re = sqrt(x ** 2 + y ** 2 + z ** 2)

    theta = asin(z / Re)
    lat = f * sin(2 * theta) + theta
    long = acos(round(x / (Re * cos(theta)), 10))

    if y < 0:
        long = -long

    return lat * 180 / pi, long * 180 / pi

def flat2lla(flat, llo, psiRef, hRef):
    global R
    global f

    xp = flat[0]
    yp = flat[1]
    zp = flat[2]

    muRef = llo[0]
    lRef = llo[1]

    muRef = muRef * pi / 180
    lRef = lRef * pi / 180
    psiRef = psiRef * pi / 180

    dN = xp * cos(psiRef) - yp * psiRef
    dE = xp * sin(psiRef) + yp * cos(psiRef)

    Rn = R / sqrt(1 - (2 * f - f ** 2) * sin(muRef) ** 2)
    Rm = Rn * (1 - (2 * f - f ** 2)) / (1 - (2 * f - f ** 2) * sin(muRef) ** 2)

    dMu = atan2(1, Rm) * dN
    dL = atan2(1, Rn * cos(muRef)) * dE

    mu = muRef + dMu
    L = lRef + dL
    h = -zp - hRef

    return mu * 180 / pi, L * 180 / pi, h

def lla2flat(lla, llo, psio, href):
    global R
    global f
    '''
    from:https://gist.github.com/skakri/ca94fd73bf7483c1fffec500c92cacf8
    lla  -- array of geodetic coordinates
            (latitude, longitude, and altitude),
            in [degrees, degrees, meters].
            Latitude and longitude values can be any value.
            However, latitude values of +90 and -90 may return
            unexpected values because of singularity at the poles.
    llo  -- Reference location, in degrees, of latitude and
            longitude, for the origin of the estimation and
            the origin of the flat Earth coordinate system.
    psio -- Angular direction of flat Earth x-axis
            (degrees clockwise from north), which is the angle
            in degrees used for converting flat Earth x and y
            coordinates to the North and East coordinates.
    href -- Reference height from the surface of the Earth to
            the flat Earth frame with regard to the flat Earth
            frame, in meters.
    usage: print(lla2flat((0.1, 44.95, 1000.0), (0.0, 45.0), 5.0, -100.0))
    '''

    Lat_p = lla[0] * pi / 180.0  # from degrees to radians
    Lon_p = lla[1] * pi / 180.0  # from degrees to radians
    Alt_p = lla[2]  # meters

    # Reference location (lat, lon), from degrees to radians
    Lat_o = llo[0] * pi / 180.0
    Lon_o = llo[1] * pi / 180.0

    psio = psio * pi / 180.0  # from degrees to radians

    dLat = Lat_p - Lat_o
    dLon = Lon_p - Lon_o

    ff = (2.0 * f) - (f ** 2)  # Can be precomputed

    sinLat = sin(Lat_o)

    # Radius of curvature in the prime vertical
    Rn = R / sqrt(1 - (ff * (sinLat ** 2)))

    # Radius of curvature in the meridian
    Rm = Rn * ((1 - ff) / (1 - (ff * (sinLat ** 2))))

    dNorth = (dLat) / atan2(1, Rm)
    dEast = (dLon) / atan2(1, (Rn * cos(Lat_o)))

    # Rotate matrice clockwise
    Xp = (dNorth * cos(psio)) + (dEast * sin(psio))
    Yp = (-dNorth * sin(psio)) + (dEast * cos(psio))
    Zp = -Alt_p - href

    return Xp, -Yp, Zp

if __name__ == '__main__':
    lat = -37
    long = 101.7210

    x, y, z = wgs2xyz(lat, long)
    lat2, long2 = xyz2wgs(x, y, z)

    print('x , y , z = ' + str(x) + ', ' + str(y) + ', ' + str(z))
    print('latOrig = ' + str(lat))
    print('lat2 = ' + str(lat2))

    print('LongOrig = ' + str(long))
    print('Long2 = ' + str(long2))

    # for latI in range(-90, 90):
    #     for longJ in range(-179, 180):
    #         x, y, z = wgs2xyz(latI, longJ)
    #         lat2, long2 = xyz2wgsV2(x, y, z)
    #         print('i = ' + str(latI) + ', j = ' + str(longJ))
    #         if (latI - lat2)>1 or latI-lat2<-1 or longJ-long2>1 or longJ-long2<-1:
    #             print('-----------------------')
    #             print('latOrig = ' + str(latI))
    #             print('lat = ' + str(lat2))
    #             print('LongOrig = ' + str(longJ))
    #             print('Long = ' + str(long2))
    #
    print(flat2lla([x, y, z], [0, 10], 0, -1000))


#==================================================================================================
import math
import numpy as np

#==================================================================================================
def romberg(f, a, b, eps = 1e-10, nmax = 20, nmin = 2):

    r = np.zeros((nmax,nmax))

    h = b-a
    n = 1
    r[0,0] = 0.5 * h * (f(a) + f(b))

    for i in range(1,nmax):
        n *= 2
        h = h / 2.0

        # Create array of x-values (spaced between existing points)
        xa = a + (2.0 * np.arange(1, n/2 +1) - 1.0) * h
        # Create array of function values from x-coords
        fa = f(xa)
        # Sum up
        t = fa.sum()

        # Set up first entry at this level (equivalent to trapezoidal)
        r[i,0] = 0.5 * r[i-1,0] + t*h

        # Extrapolate along row
        for j in range(1,i+1):
            r[i,j] = r[i,j-1] + 1.0 / (4**j - 1) * (r[i,j-1] - r[i-1,j-1])

        # Test relative tolerance if we've done the minimum number of
        # iterations
        if(i > nmin and np.abs(1.0 - r[i-1,j-1] / r[i,j]) < eps):
            return r[i,j]

    # If max number of iterations reached return the final
    # extrapolation
    return r[nmax-1,nmax-1]

#==================================================================================================


import numpy as np
import pandas as pd
from numpy.random import RandomState
from sklearn.cluster import KMeans as kmeans

def sse_l2(x, y):
    return np.sum((x - y) ** 2)

def wss(X, labels, centroids):
    """
    Calculate the WSS or cohesion, which is the within SSE
    of distances from each sample to its centroid
    """
    k = centroids.shape[0]
    wss = 0
    for i in range(k):
        cluster = X[labels == i]
        centroid = centroids[i]
        wss += sse_l2(cluster, centroid)

    return wss

def bss(X, labels, centroids):
    """
    Calculate the BSS or separation, which is the weighted SSE
    between each cluster centroid and the global centroid
    """
    k = centroids.shape[0]
    global_mean = X.mean(axis=0)
    cluster_sizes = np.array([len(X[labels == i]) for i in range(k)])

    dists = np.sum((centroids - global_mean) ** 2, axis=1)
    bss = np.sum(dists * cluster_sizes)
    return bss

def _sanitize(X):
    return X.values if isinstance(X, pd.DataFrame) else X

class KMeans:

    def __init__(self, k, tol=1e-7, max_iter=100, n_init=10, random_state=None):
        self.k = k
        self.tol = tol
        self.max_iter = max_iter
        self.n_init = n_init
        self.centroids_ = None

        if random_state is not None:
            random_state = np.int64(random_state)
        self.random_state = RandomState(seed=random_state)

    def fit(self, X):
        """
        Run the algorithm n_init times and keep
        the one with best cohesion
        """
        results = []
        for _ in range(self.n_init):
            res = self._fit(X)
            results.append(res)

        # evaluate results, keep the best
        best_result = self.eval_clusterings(results)
        (self.labels_,
         self.centroids_,
         self.n_iter_,
         self.cohesion_,
         self.separation_) = best_result
        return self

    def _fit(self, X):
        """
        Run main k-means algorithm to classify the data into
        k clusters
        """
        X = _sanitize(X)

        if self.centroids_ is None:
            # select k initial centroids randomly
            centroids_i = set()

            while len(centroids_i) != self.k:
                # ensure we generate a set of k unique centroids
                centroids_i = {self.random_state.randint(X.shape[0]) for _ in range(self.k)}

            centroids = X[list(centroids_i)]
        else:
            centroids = self.centroids_

        curr_iter = 0
        variance_diff = self.tol * 2
        variances = np.zeros(self.k)

        while curr_iter < self.max_iter and np.any(variance_diff > self.tol):
            # assign each point the closest centroid
            labels = self._assign(X, centroids)
            self._fix_empty(X, labels, centroids)

            # calculate new centroids
            for i in range(self.k):
                cluster = X[labels == i]
                new_centroid = np.mean(cluster, axis=0)
                centroids[i] = new_centroid

            # calculate difference in variance from previous clustering
            variances_old = variances
            variances = np.empty_like(variances_old)
            for i in range(self.k):
                cluster = X[labels == i]
                var = np.var(cluster, axis=0)
                variances[i] = np.linalg.norm(var, 2)

            variance_diff = variances - variances_old

            curr_iter += 1

        cohesion = wss(X, labels, centroids)
        separation = bss(X, labels, centroids)

        return labels, centroids, curr_iter, cohesion, separation

    def fit_predict(self, X):
        return self.fit(X).labels_

    def predict(self, X):
        if self.centroids_ is None:
            raise ValueError('This model is not fitted yet.')

        X = _sanitize(X)
        labels = self._assign(X, self.centroids_)
        return labels

    def _assign(self, X, centroids):
        """
        Assign each sample to the cluster of the closest centroid
        """
        # calculate distances to centroids
        centroid_dists = np.ndarray((X.shape[0], self.k))
        for i in range(self.k):
            # here einsum = sum squared factors
            diff = X - centroids[i]
            centroid_dists[:, i] = np.sqrt(np.einsum('ij,ij->i', diff, diff))

        return np.argmin(centroid_dists, axis=1)

    def _fix_empty(self, X, labels, centroids):
        """
        Fill/fix empty clusters if there are any by taking a random
        sample from the cluster with highest SSE and using it as a new
        centroid for the empty cluster, while ensuring that centroid
        was not already in use.
        """
        counts = np.array([np.count_nonzero(labels == i) for i in range(self.k)])

        # while there is an empty cluster
        while np.any(counts == 0):
            # index of empty cluster
            empty_i = np.argmin(counts)

            # compute SSE
            cluster_sses = []
            for i in range(self.k):
                if i != empty_i:
                    cluster = X[labels == i]
                    sse = sse_l2(cluster, cluster.mean())
                    cluster_sses.append(sse)
                else:
                    cluster_sses.append(0)

            # choose as new centroid a random point
            # from the cluster with highest SSE
            highest_i = np.argmax(cluster_sses)

            random_i = self.random_state.randint(counts[highest_i])
            new_centroid = X[labels == highest_i][random_i]

            # if it is already used as centroid, look for another one
            already_used = np.any([np.array_equal(new_centroid, c) for c in centroids])
            while already_used:
                random_i = self.random_state.randint(counts[highest_i])
                new_centroid = X[labels == highest_i][random_i]
                already_used = np.any([np.array_equal(new_centroid, c) for c in centroids])

            # reassign centroid
            labels[np.where(labels == highest_i)[0][random_i]] = empty_i
            centroids[empty_i] = new_centroid

            counts = np.array([np.count_nonzero(labels == i) for i in range(self.k)])

    def eval_clusterings(self, clusterings):
        """
        Custom criterion to evaluate clusterings and select the best
        """
        sse = np.array([[c[3], c[4]] for c in clusterings])

        intra_min = np.percentile(sse[:, 0], 10)
        intra_max = np.percentile(sse[:, 0], 90)
        intra_slope = 1 / (intra_min - intra_max)
        intra_b = -intra_max*intra_slope

        inter_min = np.percentile(sse[:, 1], 10)
        inter_max = np.percentile(sse[:, 1], 90)
        inter_slope = 1 / (inter_max - inter_min)
        inter_b = -inter_min*inter_slope

        intra_score = np.vectorize(lambda x: 0.75*(intra_slope*x + intra_b) if x > intra_min else 0.75*1)
        inter_score = np.vectorize(lambda x: 0.25*(inter_slope*x + inter_b) if x < inter_max else 0.25*1)

        sse = intra_score(sse[:, 0].clip(min=0)) + inter_score(sse[:, 1].clip(min=0))
        best_clustering = clusterings[np.argmax(sse)]

        return best_clustering

class BisectingKMeans:

    def __init__(self, k, tol=1e-7, max_iter=100, n_init=10,
                 criterion='heterogeneity', random_state=None):
        self.k = k
        self.tol = tol
        self.max_iter = max_iter
        self.n_init = n_init
        self.random_state = random_state

        if criterion == 'heterogeneity':
            self.eval_func = lambda x: sse_l2(x, x.mean())
        elif criterion == 'size':
            self.eval_func = len
        else:
            raise ValueError('Unknown criterion {}. Possible values'
                        ': \'heterogeneity\', \'size\''.format(criterion))

    def fit(self, X):
        X = _sanitize(X)

        labels = np.zeros(X.shape[0], dtype=np.int)
        clusters = [(self.eval_func(X), 0)]
        for i in range(1, self.k):
            target_label = self._pick(clusters)
            cluster_mask = labels == target_label

            bicluster = KMeans(2, self.tol, self.max_iter, self.n_init,
                self.random_state).fit_predict(X[cluster_mask])

            # mark one half of the new partition with a new label
            # we can keep the old label for the other half
            new_cluster_mask = np.where(cluster_mask)[0][bicluster == 0]
            labels[new_cluster_mask] = i

            self._insert(target_label, i, clusters, labels, X)

        centroids = np.array([X[labels == i].mean(axis=0) for i in range(self.k)])

        self.labels_ = labels
        self.cohesion_ = wss(X, labels, centroids)
        self.separation_ = bss(X, labels, centroids)
        return self

    def fit_predict(self, X):
        return self.fit(X).labels_

    def _insert(self, label_a, label_b, clusters, labels, X):
        # compute score according to criterion
        cluster_a = X[labels == label_a]
        cluster_b = X[labels == label_b]
        score_a = self.eval_func(cluster_a)
        score_b = self.eval_func(cluster_b)

        # add to list
        clusters.append((score_a, label_a))
        clusters.append((score_b, label_b))

    def _pick(self, clusters):
        # pick and remove the cluster with the highest value
        i = np.argmax([c[0] for c in clusters])
        return clusters.pop(i)[1]

class KMeanspp(KMeans):

    def _fit(self, X):
        if self.centroids_ is None:
            # run improved init only when not fitted already
            X = _sanitize(X)

            # select first random centroid
            centroids = np.ndarray((self.k, X.shape[1]))
            rand_i = self.random_state.randint(X.shape[0])
            centroids[0] = X[rand_i]

            dists = np.ndarray((X.shape[0], self.k))
            for i in range(1, self.k):
                # compute squared distances to all centroids
                for j in range(i):
                    diff = X - centroids[j]
                    dists[:, j] = np.einsum('ij,ij->i', diff, diff)

                # pick as centroid the point with largest minimum distance
                # to any centroid
                min_dists = np.min(dists[:, :i], axis=1)
                farthest_point_i = np.argmax(min_dists)
                centroids[i] = X[farthest_point_i]

            self.centroids_ = centroids
            res = super()._fit(X)
            self.centroids_ = None  # reset attribute since it is set by self.fit
            return res

        else:
            return super()._fit(X)

class FuzzyCMeans:

    def __init__(self, c, m=2, eps=1e-7, max_iter=100, n_init=10, random_state=None):
        self.c = c
        self.m = m
        self.eps = eps
        self.max_iter = max_iter
        self.n_init = n_init
        self.V_ = None

        if random_state is not None:
            random_state = np.int64(random_state)
        self.random_state = RandomState(seed=random_state)

    def fit(self, X):
        """
        Run the algorithm n_iter times and keep
        the one with best cohesion
        """
        results = []
        for _ in range(self.n_init):
            res = self._fit(X)
            results.append(res)

        # evaluate results, keep the best
        best_i = np.argmin([res[4] for res in results])
        (self.labels_,
         self.U_,
         self.V_,
         self.n_iter_,
         self.perfindex_) = results[best_i]
        return self

    def _fit(self, X):
        """
        Run main fuzzy c-means algorithm to classify the data into
        c labels or clusters
        """
        X = _sanitize(X)

        if self.V_ is None:
            # select c initial centroids randomly
            centroids_i = set()
            while len(centroids_i) != self.c:
                # ensure we generate a set of c unique centroids
                centroids_i = {self.random_state.randint(X.shape[0]) for _ in range(self.c)}

            V = X[list(centroids_i)] * 1.001    # avoid divisions by 0
        else:
            V = self.V_

        curr_iter = 0
        V_diff = self.eps * 2

        while curr_iter < self.max_iter and np.any(V_diff > self.eps):
            # compute U
            U = self._compute_U(X, V)

            # compute V
            V_old = V
            V = self._compute_V(X, U)

            V_diff = np.linalg.norm(V - V_old, 2, axis=1)
            curr_iter += 1

        labels = np.argmax(U, axis=0)
        perfindex = self._performance_index(X, U, V)

        return labels, U, V, curr_iter, perfindex

    def fit_predict(self, X):
        return self.fit(X).labels_

    def predict(self, X):
        if self.V_ is None:
            raise ValueError('This model is not fitted yet.')

        X = _sanitize(X)

        V = self.V_
        U = self._compute_U(X, V)
        return np.argmax(U, axis=0)

    def _compute_U(self, X, V):
        # precompute distances to centroids
        dists = np.empty((self.c, X.shape[0]))
        for i in range(self.c):
            diff = X - V[i]
            dists[i, :] = np.sqrt(np.einsum('ij,ij->i', diff, diff))

        # init with 1s, it will always be at least 1
        # since we always divide the distance to one cluster
        # by itelf for every cluster
        U = np.ones_like(dists)
        exp = 2 / (self.m - 1)

        # main formula broadcasted
        for i in range(self.c):
            for j in range(self.c):
                if i != j:  # thats why we init with 1s
                    U[i] += (dists[i] / dists[j]) ** exp

        return U ** -1.0

    def _compute_V(self, X, U):
        V = np.empty((self.c, X.shape[1]))
        Um = U ** self.m

        # main formula broadcasted
        for i in range(self.c):
            Um_col = Um[i].reshape(-1, 1)
            V[i] = np.sum(Um_col * X, axis=0) / np.sum(Um_col)

        return V

    def _performance_index(self, X, U, V):
        # precompute errors
        inner_errors = np.empty((self.c, X.shape[0]))
        for i in range(self.c):
            # here einsum = sum of squared factors
            diff = X - V[i]
            inner_errors[i, :] = np.einsum('ij,ij->i', diff, diff)

        global_mean = X.mean(axis=0)
        diff = V - global_mean
        outer_errors = np.einsum('ij,ij->i', diff, diff).reshape(-1, 1)

        # main formula
        perfindex = np.sum(U * (inner_errors - outer_errors))
        return perfindex


import pymc3 as pm
import numpy as np
import pandas as pd
from theano import shared
import scipy.stats as stats
import matplotlib.pyplot as plt
import arviz as az

az.style.use('arviz-darkgrid')

## Simple linear regression
np.random.seed(1)
N = 100
alpha_real = 2.5
beta_real = 0.9
eps_real = np.random.normal(0, 0.5, size=N)

x = np.random.normal(10, 1, N)
y_real = alpha_real + beta_real * x
y = y_real + eps_real

# we can center the data
#x = x - x.mean()
# or standardize the data
#x = (x - x.mean())/x.std()
#y = (y - y.mean())/y.std()

# In[4]:

_, ax = plt.subplots(1, 2, figsize=(8, 4))
ax[0].plot(x, y, 'C0.')
ax[0].set_xlabel('x')
ax[0].set_ylabel('y', rotation=0)
ax[0].plot(x, y_real, 'k')
az.plot_kde(y, ax=ax[1])
ax[1].set_xlabel('y')
plt.tight_layout()
plt.savefig('B11197_03_02.png', dpi=300)

# In[5]:

with pm.Model() as model_g:
    α = pm.Normal('α', mu=0, sd=10)
    β = pm.Normal('β', mu=0, sd=1)
    ϵ = pm.HalfCauchy('ϵ', 5)

    μ = pm.Deterministic('μ', α + β * x)
    y_pred = pm.Normal('y_pred', mu=μ, sd=ϵ, observed=y)

    trace_g = pm.sample(2000, tune=1000)

# In[6]:

az.plot_trace(trace_g, var_names=['α', 'β', 'ϵ'])
plt.savefig('B11197_03_03.png', dpi=300)

# ### Modyfing the data before running the models

# In[7]:

az.plot_pair(trace_g, var_names=['α', 'β'], plot_kwargs={'alpha': 0.1})
plt.savefig('B11197_03_04.png', dpi=300)

# ### interpreting the posterior

# In[8]:

plt.plot(x, y, 'C0.')

alpha_m = trace_g['α'].mean()
beta_m = trace_g['β'].mean()

draws = range(0, len(trace_g['α']), 10)
plt.plot(x, trace_g['α'][draws] + trace_g['β'][draws]
         * x[:, np.newaxis], c='gray', alpha=0.5)

plt.plot(x, alpha_m + beta_m * x, c='k',
         label=f'y = {alpha_m:.2f} + {beta_m:.2f} * x')

plt.xlabel('x')
plt.ylabel('y', rotation=0)
plt.legend()
plt.savefig('B11197_03_05.png', dpi=300)

# In[9]:

plt.plot(x, alpha_m + beta_m * x, c='k',
         label=f'y = {alpha_m:.2f} + {beta_m:.2f} * x')

sig = az.plot_hpd(x, trace_g['μ'], credible_interval=0.98, color='k')

plt.xlabel('x')
plt.ylabel('y', rotation=0)
plt.legend()
plt.savefig('B11197_03_06.png', dpi=300)

# In[10]:

ppc = pm.sample_posterior_predictive(trace_g, samples=2000, model=model_g)

# In[11]:

plt.plot(x, y, 'b.')
plt.plot(x, alpha_m + beta_m * x, c='k',
         label=f'y = {alpha_m:.2f} + {beta_m:.2f} * x')

az.plot_hpd(x, ppc['y_pred'], credible_interval=0.5, color='gray')
az.plot_hpd(x, ppc['y_pred'], color='gray')

plt.xlabel('x')
plt.ylabel('y', rotation=0)
plt.savefig('B11197_03_07.png', dpi=300)

# In[12]:

az.r2_score(y, ppc['y_pred'])

# ### The multivariate normal distribution
# Actually the bivariate

# In[13]:

sigma_x1 = 1
sigmas_x2 = [1, 2]
rhos = [-0.90, -0.5, 0, 0.5, 0.90]

k, l = np.mgrid[-5:5:.1, -5:5:.1]
pos = np.empty(k.shape + (2,))
pos[:, :, 0] = k
pos[:, :, 1] = l

f, ax = plt.subplots(len(sigmas_x2), len(rhos),
                     sharex=True, sharey=True, figsize=(12, 6),
                     constrained_layout=True)
for i in range(2):
    for j in range(5):
        sigma_x2 = sigmas_x2[i]
        rho = rhos[j]
        cov = [[sigma_x1**2, sigma_x1*sigma_x2*rho],
               [sigma_x1*sigma_x2*rho, sigma_x2**2]]
        rv = stats.multivariate_normal([0, 0], cov)
        ax[i, j].contour(k, l, rv.pdf(pos))
        ax[i, j].set_xlim(-8, 8)
        ax[i, j].set_ylim(-8, 8)
        ax[i, j].set_yticks([-5, 0, 5])
        ax[i, j].plot(0, 0,
                      label=f'$\\sigma_{{x2}}$ = {sigma_x2:3.2f}\n$\\rho$ = {rho:3.2f}', alpha=0)
        ax[i, j].legend()
f.text(0.5, -0.05, 'x_1', ha='center', fontsize=18)
f.text(-0.05, 0.5, 'x_2', va='center', fontsize=18, rotation=0)
plt.savefig('B11197_03_08.png', dpi=300)

# In[14]:

data = np.stack((x, y)).T
with pm.Model() as pearson_model:

    μ = pm.Normal('μ', mu=data.mean(0), sd=10, shape=2)

    σ_1 = pm.HalfNormal('σ_1', 10)
    σ_2 = pm.HalfNormal('σ_2', 10)
    ρ = pm.Uniform('ρ', -1., 1.)
    r2 = pm.Deterministic('r2', ρ**2)

    cov = pm.math.stack(([σ_1**2, σ_1*σ_2*ρ],
                         [σ_1*σ_2*ρ, σ_2**2]))

    y_pred = pm.MvNormal('y_pred', mu=μ, cov=cov, observed=data)

    trace_p = pm.sample(1000)

# In[15]:

az.plot_trace(trace_p, var_names=['r2'])
plt.savefig('B11197_03_9.png', dpi=300)

# In[16]:

az.summary(trace_p, var_names=['r2'])

# ## Robust linear regression

# In[17]:

ans = pd.read_csv('../data/anscombe.csv')
x_3 = ans[ans.group == 'III']['x'].values
y_3 = ans[ans.group == 'III']['y'].values
x_3 = x_3 - x_3.mean()

# In[18]:

_, ax = plt.subplots(1, 2, figsize=(10, 5))
beta_c, alpha_c = stats.linregress(x_3, y_3)[:2]
ax[0].plot(x_3, (alpha_c + beta_c * x_3), 'k',
           label=f'y ={alpha_c:.2f} + {beta_c:.2f} * x')
ax[0].plot(x_3, y_3, 'C0o')
ax[0].set_xlabel('x')
ax[0].set_ylabel('y', rotation=0)
ax[0].legend(loc=0)
az.plot_kde(y_3, ax=ax[1], rug=True)
ax[1].set_xlabel('y')
ax[1].set_yticks([])
plt.tight_layout()
plt.savefig('B11197_03_10.png', dpi=300)

# In[19]:

with pm.Model() as model_t:
    α = pm.Normal('α', mu=y_3.mean(), sd=1)
    β = pm.Normal('β', mu=0, sd=1)
    ϵ = pm.HalfNormal('ϵ', 5)
    ν_ = pm.Exponential('ν_', 1/29)
    ν = pm.Deterministic('ν', ν_ + 1)

    y_pred = pm.StudentT('y_pred', mu=α + β * x_3,
                         sd=ϵ, nu=ν, observed=y_3)

    trace_t = pm.sample(2000)

# In[20]:

varnames = ['α', 'β', 'ϵ', 'ν']
az.plot_trace(trace_t, var_names=varnames);

# In[21]:

beta_c, alpha_c = stats.linregress(x_3, y_3)[:2]

plt.plot(x_3, (alpha_c + beta_c * x_3), 'k', label='non-robust', alpha=0.5)
plt.plot(x_3, y_3, 'C0o')
alpha_m = trace_t['α'].mean()
beta_m = trace_t['β'].mean()
plt.plot(x_3, alpha_m + beta_m * x_3, c='k', label='robust')

plt.xlabel('x')
plt.ylabel('y', rotation=0)
plt.legend(loc=2)
plt.tight_layout()

plt.savefig('B11197_03_11.png', dpi=300)

# In[22]:

az.summary(trace_t, var_names=varnames)

# In[23]:

ppc = pm.sample_posterior_predictive(trace_t, samples=200, model=model_t, random_seed=2)

# In[24]:

data_ppc = az.from_pymc3(trace=trace_t, posterior_predictive=ppc)
ax = az.plot_ppc(data_ppc, figsize=(12, 6), mean=True)
plt.xlim(0, 12)
plt.savefig('B11197_03_12.png', dpi=300)

# ## Hierarchical linear regression

# In[25]:
N = 20
M = 8
idx = np.repeat(range(M-1), N)
idx = np.append(idx, 7)
np.random.seed(314)

alpha_real = np.random.normal(2.5, 0.5, size=M)
beta_real = np.random.beta(6, 1, size=M)
eps_real = np.random.normal(0, 0.5, size=len(idx))

y_m = np.zeros(len(idx))
x_m = np.random.normal(10, 1, len(idx))
y_m = alpha_real[idx] + beta_real[idx] * x_m + eps_real

_, ax = plt.subplots(2, 4, figsize=(10, 5), sharex=True, sharey=True)
ax = np.ravel(ax)
j, k = 0, N
for i in range(M):
    ax[i].scatter(x_m[j:k], y_m[j:k])
    ax[i].set_xlabel(f'x_{i}')
    ax[i].set_ylabel(f'y_{i}', rotation=0, labelpad=15)
    ax[i].set_xlim(6, 15)
    ax[i].set_ylim(7, 17)
    j += N
    k += N
plt.tight_layout()
plt.savefig('B11197_03_13.png', dpi=300)

# In[26]:

x_centered = x_m - x_m.mean()

# In[27]:

with pm.Model() as unpooled_model:
    α_tmp = pm.Normal('α_tmp', mu=0, sd=10, shape=M)
    β = pm.Normal('β', mu=0, sd=10, shape=M)
    ϵ = pm.HalfCauchy('ϵ', 5)
    ν = pm.Exponential('ν', 1/30)

    y_pred = pm.StudentT('y_pred', mu=α_tmp[idx] + β[idx] * x_centered,
                         sd=ϵ, nu=ν, observed=y_m)

    α = pm.Deterministic('α', α_tmp - β * x_m.mean())

    trace_up = pm.sample(2000)

# In[28]:

az.plot_forest(trace_up, var_names=['α', 'β'], combined=True)
plt.savefig('B11197_03_14.png', dpi=300)

#
#  <img src='B11197_03_15.png' width="700">

# In[29]:

with pm.Model() as hierarchical_model:
    # hyper-priors
    α_μ_tmp = pm.Normal('α_μ_tmp', mu=0, sd=10)
    α_σ_tmp = pm.HalfNormal('α_σ_tmp', 10)
    β_μ = pm.Normal('β_μ', mu=0, sd=10)
    β_σ = pm.HalfNormal('β_σ', sd=10)

    # priors
    α_tmp = pm.Normal('α_tmp', mu=α_μ_tmp, sd=α_σ_tmp, shape=M)
    β = pm.Normal('β', mu=β_μ, sd=β_σ, shape=M)
    ϵ = pm.HalfCauchy('ϵ', 5)
    ν = pm.Exponential('ν', 1/30)

    y_pred = pm.StudentT('y_pred', mu=α_tmp[idx] + β[idx] * x_centered,
                         sd=ϵ, nu=ν, observed=y_m)

    α = pm.Deterministic('α', α_tmp - β * x_m.mean())
    α_μ = pm.Deterministic('α_μ', α_μ_tmp - β_μ * x_m.mean())
    α_σ = pm.Deterministic('α_sd', α_σ_tmp - β_μ * x_m.mean())

    trace_hm = pm.sample(1000)

# In[30]:

az.plot_forest(trace_hm, var_names=['α', 'β'], combined=True)
plt.savefig('B11197_03_16.png', dpi=300)

# In[31]:

_, ax = plt.subplots(2, 4, figsize=(10, 5), sharex=True, sharey=True,
                     constrained_layout=True)
ax = np.ravel(ax)
j, k = 0, N
x_range = np.linspace(x_m.min(), x_m.max(), 10)
for i in range(M):
    ax[i].scatter(x_m[j:k], y_m[j:k])
    ax[i].set_xlabel(f'x_{i}')
    ax[i].set_ylabel(f'y_{i}', labelpad=17, rotation=0)
    alpha_m = trace_hm['α'][:, i].mean()
    beta_m = trace_hm['β'][:, i].mean()
    ax[i].plot(x_range, alpha_m + beta_m * x_range, c='k',
               label=f'y = {alpha_m:.2f} + {beta_m:.2f} * x')
    plt.xlim(x_m.min()-1, x_m.max()+1)
    plt.ylim(y_m.min()-1, y_m.max()+1)
    j += N
    k += N
plt.savefig('B11197_03_17.png', dpi=300)

# ## Polynomial regression

# In[32]:

x_2 = ans[ans.group == 'II']['x'].values
y_2 = ans[ans.group == 'II']['y'].values
x_2 = x_2 - x_2.mean()

plt.scatter(x_2, y_2)
plt.xlabel('x')
plt.ylabel('y', rotation=0)
plt.savefig('B11197_03_18.png', dpi=300)

# In[33]:

with pm.Model() as model_poly:
    α = pm.Normal('α', mu=y_2.mean(), sd=1)
    β1 = pm.Normal('β1', mu=0, sd=1)
    β2 = pm.Normal('β2', mu=0, sd=1)
    ϵ = pm.HalfCauchy('ϵ', 5)

    mu = α + β1 * x_2 + β2 * x_2**2

    y_pred = pm.Normal('y_pred', mu=mu, sd=ϵ, observed=y_2)

    trace_poly = pm.sample(2000)

# In[34]:

#az.plot_trace(trace_poly);

# In[35]:

#az.summary(trace_poly)

# In[36]:

x_p = np.linspace(-6, 6)
y_p = trace_poly['α'].mean() + trace_poly['β1'].mean() *     x_p + trace_poly['β2'].mean() * x_p**2
plt.scatter(x_2, y_2)
plt.xlabel('x')
plt.ylabel('y', rotation=0)
plt.plot(x_p, y_p, c='C1')
plt.savefig('B11197_03_19.png', dpi=300)

# ## Multiple Linear regression

# In[37]:

np.random.seed(314)
N = 100
alpha_real = 2.5
beta_real = [0.9, 1.5]
eps_real = np.random.normal(0, 0.5, size=N)

X = np.array([np.random.normal(i, j, N) for i, j in zip([10, 2], [1, 1.5])]).T
X_mean = X.mean(axis=0, keepdims=True)
X_centered = X - X_mean
y = alpha_real + np.dot(X, beta_real) + eps_real

# In[38]:

def scatter_plot(x, y):
    plt.figure(figsize=(10, 10))
    for idx, x_i in enumerate(x.T):
        plt.subplot(2, 2, idx+1)
        plt.scatter(x_i, y)
        plt.xlabel(f'x_{idx+1}')
        plt.ylabel(f'y', rotation=0)

    plt.subplot(2, 2, idx+2)
    plt.scatter(x[:, 0], x[:, 1])
    plt.xlabel(f'x_{idx}')
    plt.ylabel(f'x_{idx+1}', rotation=0)

scatter_plot(X_centered, y)
plt.savefig('B11197_03_20.png', dpi=300)

# In[39]:

with pm.Model() as model_mlr:
    α_tmp = pm.Normal('α_tmp', mu=0, sd=10)
    β = pm.Normal('β', mu=0, sd=1, shape=2)
    ϵ = pm.HalfCauchy('ϵ', 5)

    μ = α_tmp + pm.math.dot(X_centered, β)

    α = pm.Deterministic('α', α_tmp - pm.math.dot(X_mean, β))

    y_pred = pm.Normal('y_pred', mu=μ, sd=ϵ, observed=y)

    trace_mlr = pm.sample(2000)

# In[40]:

varnames = ['α', 'β', 'ϵ']
#az.plot_trace(trace_mlr, var_names=varnames);

# In[41]:

az.summary(trace_mlr, var_names=varnames)

# ## Confounding variables and redundant variables

# In[42]:

np.random.seed(42)
N = 100
x_1 = np.random.normal(size=N)
x_2 = x_1 + np.random.normal(size=N, scale=1)
#x_2 = x_1 + np.random.normal(size=N, scale=0.01)
y = x_1 + np.random.normal(size=N)
X = np.vstack((x_1, x_2)).T

# In[43]:

scatter_plot(X, y)
plt.savefig('B11197_03_21.png', dpi=300)

# In[44]:

with pm.Model() as m_x1x2:
    α = pm.Normal('α', mu=0, sd=10)
    β1 = pm.Normal('β1', mu=0, sd=10)
    β2 = pm.Normal('β2', mu=0, sd=10)
    ϵ = pm.HalfCauchy('ϵ', 5)

    μ = α + β1 * X[:, 0] + β2 * X[:, 1]

    y_pred = pm.Normal('y_pred', mu=μ, sd=ϵ, observed=y)

    trace_x1x2 = pm.sample(2000)

with pm.Model() as m_x1:
    α = pm.Normal('α', mu=0, sd=10)
    β1 = pm.Normal('β1', mu=0, sd=10)
    ϵ = pm.HalfCauchy('ϵ', 5)

    μ = α + β1 * X[:, 0]

    y_pred = pm.Normal('y_pred', mu=μ, sd=ϵ, observed=y)

    trace_x1 = pm.sample(2000)

with pm.Model() as m_x2:
    α = pm.Normal('α', mu=0, sd=10)
    β2 = pm.Normal('β2', mu=0, sd=10)
    ϵ = pm.HalfCauchy('ϵ', 5)

    μ = α + β2 * X[:, 1]

    y_pred = pm.Normal('y_pred', mu=μ, sd=ϵ, observed=y)

    trace_x2 = pm.sample(2000)

# In[45]:

az.plot_forest([trace_x1x2, trace_x1, trace_x2],
               model_names=['m_x1x2', 'm_x1', 'm_x2'],
               var_names=['β1', 'β2'],
               combined=False, colors='cycle', figsize=(8, 3))
plt.savefig('B11197_03_22.png', dpi=300)

# In[46]:

# just repeating the code from a couple of cells before, but with a lower value of `scale`.
np.random.seed(42)
N = 100
x_1 = np.random.normal(size=N)
x_2 = x_1 + np.random.normal(size=N, scale=0.01)
y = x_1 + np.random.normal(size=N)
X = np.vstack((x_1, x_2)).T

# In[47]:

scatter_plot(X, y)
plt.savefig('B11197_03_23.png', dpi=300)

# In[48]:

with pm.Model() as model_red:
    α = pm.Normal('α', mu=0, sd=10)
    β = pm.Normal('β', mu=0, sd=10, shape=2)
    ϵ = pm.HalfCauchy('ϵ', 5)

    μ = α + pm.math.dot(X, β)

    y_pred = pm.Normal('y_pred', mu=μ, sd=ϵ, observed=y)

    trace_red = pm.sample(2000)

# In[49]:

az.plot_forest(trace_red, var_names=['β'], combined=True, figsize=(8, 2))
plt.savefig('B11197_03_24.png', dpi=300)

# In[50]:

az.plot_pair(trace_red, var_names=['β'])
plt.savefig('B11197_03_25.png', dpi=300)

# ## Masking effect variables

# In[51]:

np.random.seed(42)
N = 126
r = 0.8
x_1 = np.random.normal(size=N)
x_2 = np.random.normal(x_1, scale=(1 - r ** 2) ** 0.5)
y = np.random.normal(x_1 - x_2)
X = np.vstack((x_1, x_2)).T

# In[52]:

scatter_plot(X, y)
plt.savefig('B11197_03_26.png', dpi=300, figsize=(5.5, 5.5))

# In[53]:

with pm.Model() as m_x1x2:
    α = pm.Normal('α', mu=0, sd=10)
    β1 = pm.Normal('β1', mu=0, sd=10)
    β2 = pm.Normal('β2', mu=0, sd=10)
    ϵ = pm.HalfCauchy('ϵ', 5)

    μ = α + β1 * X[:, 0] + β2 * X[:, 1]

    y_pred = pm.Normal('y_pred', mu=μ, sd=ϵ, observed=y)

    trace_x1x2 = pm.sample(1000)

with pm.Model() as m_x1:
    α = pm.Normal('α', mu=0, sd=10)
    β1 = pm.Normal('β1', mu=0, sd=10)
    ϵ = pm.HalfCauchy('ϵ', 5)

    μ = α + β1 * X[:, 0]

    y_pred = pm.Normal('y_pred', mu=μ, sd=ϵ, observed=y)

    trace_x1 = pm.sample(1000)

with pm.Model() as m_x2:
    α = pm.Normal('α', mu=0, sd=10)
    β2 = pm.Normal('β2', mu=0, sd=10)
    ϵ = pm.HalfCauchy('ϵ', 5)

    μ = α + β2 * X[:, 1]

    y_pred = pm.Normal('y_pred', mu=μ, sd=ϵ, observed=y)

    trace_x2 = pm.sample(1000)

# In[54]:

az.plot_forest([trace_x1x2, trace_x1, trace_x2],
               model_names=['m_x1x2', 'm_x1', 'm_x2'],
               var_names=['β1', 'β2'],
               combined=True, colors='cycle', figsize=(8, 3))
plt.savefig('B11197_03_27.png', dpi=300, bbox_inches='tight')

# ## Variable variance

# In[55]:

data = pd.read_csv('../data/babies.csv')
data.plot.scatter('Month', 'Lenght')
plt.savefig('B11197_03_28.png', dpi=300)

# In[56]:

with pm.Model() as model_vv:
    α = pm.Normal('α', sd=10)
    β = pm.Normal('β', sd=10)
    γ = pm.HalfNormal('γ', sd=10)
    δ = pm.HalfNormal('δ', sd=10)

    x_shared = shared(data.Month.values * 1.)

    μ = pm.Deterministic('μ', α + β * x_shared**0.5)
    ϵ = pm.Deterministic('ϵ', γ + δ * x_shared)

    y_pred = pm.Normal('y_pred', mu=μ, sd=ϵ, observed=data.Lenght)

    trace_vv = pm.sample(1000, tune=1000)

# In[57]:

plt.plot(data.Month, data.Lenght, 'C0.', alpha=0.1)

μ_m = trace_vv['μ'].mean(0)
ϵ_m = trace_vv['ϵ'].mean(0)

plt.plot(data.Month, μ_m, c='k')
plt.fill_between(data.Month, μ_m + 1 * ϵ_m, μ_m -
                 1 * ϵ_m, alpha=0.6, color='C1')
plt.fill_between(data.Month, μ_m + 2 * ϵ_m, μ_m -
                 2 * ϵ_m, alpha=0.4, color='C1')

plt.xlabel('x')
plt.ylabel('y', rotation=0)
plt.savefig('B11197_03_29.png', dpi=300)

# In[58]:

x_shared.set_value([0.5])
ppc = pm.sample_posterior_predictive(trace_vv, 2000, model=model_vv)
y_ppc = ppc['y_pred'][:, 0]

# In[59]:

ref = 53
density, l, u = az._fast_kde(y_ppc)
x_ = np.linspace(l, u, 200)
plt.plot(x_, density)
percentile = int(sum(y_ppc <= ref) / len(y_ppc) * 100)
plt.fill_between(x_[x_ < ref], density[x_ < ref],
                 label='percentile = {:2d}'.format(percentile))
plt.xlabel('length')
plt.yticks([])
plt.legend()
plt.savefig('B11197_03_30.png', dpi=300)

# In[ ]:

x_4 = ans[ans.group == 'IV']['x'].values
y_4 = ans[ans.group == 'IV']['y'].values

with pm.Model() as model_t2:
    α = pm.Normal('α', mu=0, sd=100)
    β = pm.Normal('β', mu=0, sd=1)
    ϵ = pm.HalfCauchy('ϵ', 5)
    ν = pm.Exponential('ν', 1/30)
    #ν = pm.Gamma('ν', mu=20, sd=15)
    #ν = pm.Gamma('ν', 2, 0.1)

    y_pred = pm.StudentT('y_pred', mu=α + β * x_4, sd=ϵ, nu=ν, observed=y_4)
    trace_t2 = pm.sample(2000)

import numpy,import scipy,from theano,import pymc3


#!/usr/bin/env python
# coding: utf-8

# DO NOT EDIT
# Autogenerated from the notebook copula.ipynb.
# Edit the notebook and then sync the output with this file.
#
# flake8: noqa
# DO NOT EDIT

# # Copula - Multivariate joint distribution

import matplotlib.pyplot as plt
import numpy as np
import seaborn as sns
from scipy import stats

sns.set_style("darkgrid")
sns.mpl.rc("figure", figsize=(8, 8))

# When modeling a system, there are often cases where multiple parameters
# are involved. Each of these parameters could be described with a given
# Probability Density Function (PDF). If would like to be able to generate a
# new set of parameter values, we need to be able to sample from these
# distributions-also called marginals. There are mainly two cases: *(i)*
# PDFs are independent; *(ii)* there is a dependency. One way to model the
# dependency it to use a **copula**.

# ## Sampling from a copula
#
# Let's use a bi-variate example and assume first that we have a prior and
# know how to model the dependence between our 2 variables.
#
# In this case, we are using the Gumbel copula and fix its hyperparameter
# `theta=2`. We can visualize it's 2-dimensional PDF.

from statsmodels.distributions.copula.api import (CopulaDistribution,
                                                  GumbelCopula,
                                                  IndependenceCopula)

copula = GumbelCopula(theta=2)
_ = copula.plot_pdf()  # returns a matplotlib figure

# And we can sample the PDF.

sample = copula.rvs(10000)
h = sns.jointplot(x=sample[:, 0], y=sample[:, 1], kind="hex")
_ = h.set_axis_labels("X1", "X2", fontsize=16)

# Let's come back to our 2 variables for a second. In this case we
# consider them to be gamma and normally distributed. If they would be
# independent from each other, we could sample from each PDF individually.
# Here we use a convenient class to do the same operation.
#
# ### Reproducibility
#
# Generating reproducible random values from copulas required explicitly
# setting the `seed` argument.
# `seed` accepts either an initialized NumPy `Generator` or `RandomState`,
# or any argument acceptable
# to `np.random.default_rng`, e.g., an integer or a sequence of integers.
# This example uses an
# integer.
#
# The singleton `RandomState` that is directly exposed in the `np.random`
# distributions is
# not used, and setting `np.random.seed` has no effect on the values
# generated.

marginals = [stats.gamma(2), stats.norm]
joint_dist = CopulaDistribution(copula=IndependenceCopula(),
                                marginals=marginals)
sample = joint_dist.rvs(512, random_state=20210801)
h = sns.jointplot(x=sample[:, 0], y=sample[:, 1], kind="scatter")
_ = h.set_axis_labels("X1", "X2", fontsize=16)

# Now, above we have expressed the dependency between our variables using
# a copula, we can use this copula to sample a new set of observation with
# the same convenient class.

joint_dist = CopulaDistribution(copula, marginals)
# Use an initialized Generator object
rng = np.random.default_rng([2, 0, 2, 1, 0, 8, 0, 1])
sample = joint_dist.rvs(512, random_state=rng)
h = sns.jointplot(x=sample[:, 0], y=sample[:, 1], kind="scatter")
_ = h.set_axis_labels("X1", "X2", fontsize=16)

# There are two things to note here. *(i)* as in the independent case, the
# marginals are correctly showing a gamma and normal distribution; *(ii)*
# the dependence is visible between the two variables.

# ## Estimating copula parameters
#
# Now, imagine we already have experimental data and we know that there is
# a dependency that can be expressed using a Gumbel copula. But we don't
# know what is the hyperparameter value for our copula. In this case, we can
# estimate the value.
#
# We are going to use the sample we just generated as we already know the
# value of the hyperparameter we should get: `theta=2`.

copula = GumbelCopula()
theta = copula.fit_corr_param(sample)
print(theta)

# We can see that the estimated hyperparameter value is close to the value
# set previously.

import numpy,from scipy,from statsmodels


import numpy as np
import matplotlib.pyplot as plt
import numpy as np
import matplotlib.pyplot as plt
import matplotlib
from sklearn.linear_model import LinearRegression
from sklearn.linear_model import SGDRegressor
from sklearn.preprocessing import PolynomialFeatures
from sklearn.preprocessing import StandardScaler
from sklearn.pipeline import Pipeline
from sklearn.metrics import mean_squared_error
from sklearn.model_selection import train_test_split
from sklearn.pipeline import Pipeline
from sklearn.linear_model import Ridge
from sklearn.linear_model import Lasso
from sklearn.linear_model import ElasticNet
from sklearn.base import clone

def plot_gradient_descent(theta, eta, theta_path=None):
    m = len(X_b)
    plt.plot(X, y, "b.")
    n_iterations = 1000
    for iteration in range(n_iterations):
        if iteration < 10:
            y_predict = X_new_b.dot(theta)
            style = "b-" if iteration > 0 else "r--"
            plt.plot(X_new, y_predict, style)
        gradients = 2/m * X_b.T.dot(X_b.dot(theta) - y)
        theta = theta - eta * gradients
        if theta_path is not None:
            theta_path.append(theta)
    plt.xlabel("$x_1$", fontsize=18)
    plt.axis([0, 2, 0, 15])
    plt.title(r"$\eta = {}$".format(eta), fontsize=16)

def learning_schedule(t):
    return t0 / (t + t1)

def plot_learning_curves(model, X, y):
    X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.2, random_state=10)
    train_errors, val_errors = [], []
    for m in range(1, len(X_train)):
        model.fit(X_train[:m], y_train[:m])
        y_train_predict = model.predict(X_train[:m])
        y_val_predict = model.predict(X_val)
        train_errors.append(mean_squared_error(y_train[:m], y_train_predict))
        val_errors.append(mean_squared_error(y_val, y_val_predict))

    plt.plot(np.sqrt(train_errors), "r-+", linewidth=2, label="train")
    plt.plot(np.sqrt(val_errors), "b-", linewidth=3, label="validation")
    plt.legend(loc="upper right", fontsize=14)
    plt.xlabel("train set size", fontsize=14)
    plt.ylabel("RMSE", fontsize=14)

def plot_model(model_class, polynomial, alphas, **model_kargs):
    for alpha, style in zip(alphas, ("b-", "g--", "r:")):
        model = model_class(alpha, **model_kargs) if alpha > 0 else LinearRegression()
        if polynomial:
            model = Pipeline([
                    ("poly_features", PolynomialFeatures(degree=10, include_bias=False)),
                    ("std_scaler", StandardScaler()),
                    ("regul_reg", model),
                ])
        model.fit(X, y)
        y_new_regul = model.predict(X_new)
        lw = 2 if alpha > 0 else 1
        plt.plot(X_new, y_new_regul, style, linewidth=lw, label=r"$\alpha = {}$".format(alpha))
    plt.plot(X, y, "b.", linewidth=3)
    plt.legend(loc="upper left", fontsize=15)
    plt.xlabel("$x_1$", fontsize=18)
    plt.axis([0, 3, 0, 4])

if __name__=="__main__":

    np.random.seed(42)
    X = 2 * np.random.rand(100,1)
    y = 4+ 3 * X + np.random.randn(100, 1)

    # plt.plot(x,y,"b.")
    # plt.axis([0,2,0,15])
    # plt.show()

    X_b = np.c_[np.ones((100,1)), X]
    theta_best = np.linalg.inv(X_b.T.dot(X_b)).dot(X_b.T).dot(y)
    print(theta_best)

    X_new = np.array([[0],[2]])
    X_new_b = np.c_[np.ones((2,1)), X_new]
    y_predict = X_new_b.dot(theta_best)
    print(y_predict)

    # plt.plot(X_new,y_predict,"r-")
    # plt.plot(x,y,"b.")
    # plt.axis([0,2,0,15])
    # plt.show()

    from sklearn.linear_model import LinearRegression

    lin_reg = LinearRegression()
    lin_reg.fit(X,y)
    '''
    coef_ : 가중치
    intercept_ : 편향
    '''
    print(lin_reg.intercept_, lin_reg.coef_)
    print(lin_reg.predict(X_new))

    # 경사하강법은 스케일링을 하고 해야한다.
    learning_rate = 0.1
    n_iterations = 1000
    m= 100

    theta = np.random.randn(2,1)

    for iteration in range(n_iterations):
        gradients = 2/m * X_b.T.dot(X_b.dot(theta) - y)
        theta = theta - learning_rate * gradients

    print(theta)

    # 배치 경사 하강법

    np.random.seed(42)
    theta = np.random.randn(2,1)
    theta_path_bgd = []

    # plt.figure(figsize=(10,4))
    # plt.subplot(131); plot_gradient_descent(theta, eta=0.02)
    # plt.subplot(131); plot_gradient_descent(theta, eta=0.1, theta_path=theta_path_bgd)
    # plt.subplot(131); plot_gradient_descent(theta, eta=0.5)
    # plt.show()

    # 확률적 경사 하강법
    theta_path_sgd = []
    m = len(X_b)
    np.random.seed(42)

    n_epochs = 50
    t0,t1 = 5, 50

    theta = np.random.randn(2,1)
    for epoch in range(n_epochs) :
        for i in range(m) :
            if epoch == 0 and i < 20:
                y_predict = X_new_b.dot(theta)
                style = "b-" if i > 0 else "r--"
                # plt.plot(X_new, y_predict, style)
            random_index = np.random.randint(m)
            xi = X_b[random_index:random_index+1]
            yi = y[random_index:random_index+1]
            gradients = 2 * xi.T.dot(xi.dot(theta) - yi)
            learning_rate = learning_schedule(epoch * m +i)
            theta = theta - learning_rate * gradients
            theta_path_sgd.append(theta)

    print(theta)

    # plt.plot(X,y,"b.")
    # plt.axis([0,2,0,15])
    # plt.show()
    # 사이킷런을 사용할 때
    from sklearn.linear_model import SGDRegressor
    sgd_reg = SGDRegressor(max_iter=50, penalty=None, eta0=0.1, random_state=42)
    sgd_reg.fit(X, y.ravel())
    print(sgd_reg.intercept_, sgd_reg.coef_)

    # 미니배치 경사하강법
    theta_path_mgd = []

    n_iterations = 50
    minibatch_size = 20

    np.random.seed(42)
    theta = np.random.randn(2, 1) # 무작위 최기화

    t0, t1 = 200, 1000

    t = 0
    for epoch in range(n_iterations):
        shuffled_indices = np.random.permutation(m)
        X_b_shuffled = X_b[shuffled_indices]
        y_shuffled = y[shuffled_indices]

        for i in range(0, m, minibatch_size):
            t += 1
            xi = X_b_shuffled[i:i+minibatch_size]
            yi = y_shuffled[i:i+minibatch_size]
            gradients = 2 / minibatch_size * xi.T.dot(xi.dot(theta) - yi)
            eta = learning_schedule(t)
            theta = theta - eta * gradients
            theta_path_mgd.append(theta)

    print(theta)

    theta_path_bgd = np.array(theta_path_bgd)
    theta_path_sgd = np.array(theta_path_sgd)
    theta_path_mgd = np.array(theta_path_mgd)

    # plt.figure(figsize=(7, 4))
    #
    # plt.plot(theta_path_sgd[:, 0], theta_path_sgd[:, 1], "r-s", linewidth=1, label="SGD")
    # plt.plot(theta_path_mgd[:, 0], theta_path_mgd[:, 1], "g-+", linewidth=2, label="MGD")
    # plt.plot(theta_path_bgd[:, 0], theta_path_bgd[:, 1], "b-o", linewidth=3, label="BGD")
    #
    # plt.legend(loc="upper left", fontsize=16)
    # plt.xlabel(r"$\theta_0$", fontsize=20)
    # plt.ylabel(r"$\theta_1$    ", fontsize=20, rotation=0)
    # plt.axis([2.5, 4.5, 2.3, 3.9])
    # plt.show()

    # 다항 회귀
    np.random.seed(42)
    m = 100
    X = 6 * np.random.rand(m,1) -3
    y = 0.5 * X**2 + X + 2 + np.random.randn(m,1)

    # plt.plot(X,y,"b.")
    # plt.axis([-3,3,0,10])
    # plt.show()

    from sklearn.preprocessing import PolynomialFeatures
    poly_features = PolynomialFeatures(degree=2, include_bias=False)
    X_poly = poly_features.fit_transform(X)
    print(X[0])
    print(X_poly[0])

    lin_reg = LinearRegression()
    lin_reg.fit(X_poly, y)

    print(lin_reg.intercept_, lin_reg.coef_)
    print(poly_features.get_feature_names())

    X_new = np.linspace(-3,3,100).reshape(100,1)
    X_new_poly = poly_features.transform(X_new)
    y_new = lin_reg.predict(X_new_poly)

    # plt.plot(X,y,"b.")
    # plt.plot(X_new, y_new, "r-", linewidth=2, label="Predict")
    # plt.legend(loc="upper left", fontsize=14)
    # plt.axis([-3,3,0,10])
    # plt.show()

    # 학습 곡선
    # lin_reg = LinearRegression()
    # plot_learning_curves(lin_reg, X, y)
    # plt.axis([0,80,0,3])
    # plt.show()
    #
    from sklearn.pipeline import Pipeline

    polynomial_regression = Pipeline([
        ("poly_features", PolynomialFeatures(degree=10, include_bias=False)),
        ("lin_reg", LinearRegression()),
    ])

    plot_learning_curves(polynomial_regression, X, y)
    # plt.axis([0,80,0,3])
    # plt.show()

    # 규제가 있는 선형 모델
    np.random.seed(42)
    m = 20
    X = 3 * np.random.rand(m, 1)
    y = 1 + 0.5*X + np.random.randn(m, 1) / 1.5
    X_new = np.linspace(0,3,100).reshape(100, 1)

    from sklearn.linear_model import Ridge

    ridge_reg = Ridge(alpha=1, solver="cholesky", random_state=42)
    ridge_reg.fit(X, y)
    print(ridge_reg.predict([[1.5]]))

    sgd_reg = SGDRegressor(max_iter=5, penalty="l2", random_state=42)
    sgd_reg.fit(X, y.ravel())
    print(sgd_reg.predict([[1.5]]))

    from sklearn.linear_model import Lasso
    lasso_reg = Lasso(alpha=0.1)
    lasso_reg.fit(X, y)
    print(lasso_reg.predict([[1.5]]))

    from sklearn.linear_model import ElasticNet
    elastic_net = ElasticNet(alpha=0.1, l1_ratio= 0.5, random_state=42)
    elastic_net.fit(X,y)
    print(elastic_net.predict([[1.5]]))

    # 조기 종료
    np.random.seed(42)
    m = 100
    X = 6 * np.random.rand(m, 1) - 3
    y = 2 + X + 0.5 * X**2 + np.random.randn(m, 1)

    X_train, X_val, y_train, y_val = train_test_split(X[:50], y[:50].ravel(), test_size=0.5, random_state=10)

    poly_scaler = Pipeline([
        ("poly_features", PolynomialFeatures(degree=90, include_bias=False)),
        ("std_scaler", StandardScaler()),
    ])

    X_train_poly_scaled = poly_scaler.fit_transform(X_train)
    X_val_poly_scaled = poly_scaler.transform(X_val)

    from sklearn.base import clone
    sgd_reg = SGDRegressor(max_iter=1, warm_start=True, penalty=None,
                           learning_rate="constant", eta0=0.0005, random_state=42)

    minimum_val_error = float("inf")
    best_epoch = None
    best_model = None

    for epoch in range(1000):
        sgd_reg.fit(X_train_poly_scaled, y_train)
        y_val_predict = sgd_reg.predict(X_train_poly_scaled)
        val_error = mean_squared_error(y_val, y_val_predict)
        if val_error < minimum_val_error:
            minimum_val_error = val_error
            best_epoch = epoch
            best_model = clone(sgd_reg)

    print(best_epoch, best_model)


#%%
import numpy as np
import matplotlib.pyplot as plt
from sklearn.metrics import mean_squared_error
# from nirom_dd_orig import plot_starshape

save_bf = False

snapshots_matrix_1 = np.load('snapshots/snapshots_pod_0709.npy')
snapshots_matrix_2 = np.load('snapshots/snapshots_pod.npy')
snapshots_matrix = np.concatenate((snapshots_matrix_1, snapshots_matrix_2), axis = 1)

print("Number of snapshots = ",snapshots_matrix.shape[1])

xlength = 3.0
ylength = 3.0
grid_width = [0.48,0.48]
# spacing inside small grid
# %%
snapshots_matrix_vel = snapshots_matrix[:nx*ny*2,:]
snapshots_matrix_b = snapshots_matrix[nx*ny*2:nx*ny*3,:]
# %%
nx = int(grid_width[0]*100)
ny = nx
nz = 1
ddx = np.array((0.01,0.01))

nloc = 3
nScalar = 1 #because I calculate u and v separately
nDim = 2 # dimension of problem (no need to interpolate in the third dimension)

#%%
# cumulative_tol = 0.95
# nPOD = [nTime] # len(nPOD) = nFields
# nPOD = [-2]
nPOD = [-1]
nPOD = [100] # 100 50 10
bases = []
singular_values = []

#snapshots matrix done!
nrows, ncols = snapshots_matrix.shape
if nrows > ncols:
    SSmatrix = np.dot(snapshots_matrix.T, snapshots_matrix)
else:
    SSmatrix = np.dot(snapshots_matrix, snapshots_matrix.T)
    print('WARNING - CHECK HOW THE BASIS FUNCTIONS ARE CALCULATED WITH THIS METHOD')

# print('SSmatrix', SSmatrix.shape)
eigvalues, v = np.linalg.eigh(SSmatrix)
# print('eigenvalues', eigvalues)
# print('normalized eigenvector ([:j]) v = ', v)
eigvalues =  eigvalues[::-1]
# get rid of small negative eigenvalues (there shouldn't be any as the eigenvalues of a real, symmetric
# matrix are non-negative, but sometimes very small negative values do appear)
eigvalues[eigvalues<0] = 0
s_values = np.sqrt(eigvalues)
# print('s values', s_values[0:20])

singular_values.append(s_values)

cumulative_info = np.zeros(len(eigvalues))
for j in range(len(eigvalues)):
    if j==0:
        cumulative_info[j] = eigvalues[j]
    else:
        cumulative_info[j] = cumulative_info[j-1] + eigvalues[j]

cumulative_info = cumulative_info / cumulative_info[-1]
nAll = len(eigvalues)

# Apply POD
#if nPOD = -1, use cumulative tolerance
#if nPOD = -2 use all coefficients (or set nPOD = nTime)
#if nPOD > 0 use nPOD coefficients as defined by the user

if nPOD[0] == -1:
    # SVD truncation - percentage of information captured or number
    # cumulative_tol = nirom_options.compression.cumulative_tol[iField]
    nPOD_iField = sum(cumulative_info <= cumulative_tol) #tolerance
    nPOD[0] = nPOD_iField
elif nPOD[0] == -2:
    nPOD_iField = nAll
    nPOD[0] = nPOD_iField
else:
    nPOD_iField = nPOD[0]

print("retaining", nPOD_iField, "basis functions of a possible", len(eigvalues))

basis_functions = np.zeros((nx*ny*nz*(nDim+1),nPOD_iField)) # nDim should be nScalar?
for j in reversed(range(nAll-nPOD_iField,nAll)):
    Av = np.dot(snapshots_matrix,v[:,j])
    basis_functions[:,nAll-j-1] = Av/np.linalg.norm(Av)
# %%
plt.plot(range(len(singular_values[0])), np.log(singular_values[0]), 'k.-')
plt.xlabel("Log of singular values")
plt.ylabel("Index of basis function")
plt.grid(':')
# plt.savefig("singular_plot.png")
plt.show()
# %%
plt.plot(range(len(cumulative_info)), cumulative_info, 'g.-')
plt.xlabel("Log of singular values")
plt.ylabel("Index of basis function")
plt.grid(':')
# plt.savefig("cumulative_info.png")
plt.show()
#%%
"""
Compress and reconstruct the snapshots and graph the MSE reconstruction error
R- basis_functions
alpha - pod coeff
phi - reconstruction
"""
alphas = np.zeros((snapshots_matrix.shape[1], nPOD_iField)) #array that stores compressions
for i in range(snapshots_matrix.shape[1]):
    alphas[i] = basis_functions.T @ snapshots_matrix[:,i]

#reconstruct
reconstructions = np.zeros(snapshots_matrix.shape)
mse_errors = []
for i in range(snapshots_matrix.shape[1]):
    reconstructions[:,i] = basis_functions @ alphas[i]
    mse_errors.append(mean_squared_error(snapshots_matrix[:,i], reconstructions[:,i]))
#%%
# with 278 basis functions, np.mean(mse_errors) = 0.0024700645895766376

#save basis functions
if save_bf:
    np.save("basis_functions.npy", basis_functions)

# %%
"""
Import the starshape grid and compress that information
"""
starshape = np.load("starshape_pod.npy")

alphas_s = np.zeros((starshape.shape[1], nPOD_iField)) #array that stores compressions
for i in range(starshape.shape[1]):
    alphas_s[i] = basis_functions.T @ starshape[:,i]

# reconstruct
reconstructions_s = np.zeros(starshape.shape)
mse_errors_s = []
for i in range(starshape.shape[1]):
    reconstructions_s[:,i] = basis_functions @ alphas_s[i]
    mse_errors_s.append(mean_squared_error(starshape[:,i], reconstructions_s[:,i]))

# Export the compressions
np.save("ss_pod_coeffs.npy",alphas_s)

# %%
def create_basis_func(snapshot_matrix, nPOD):
    nrows, ncols = snapshot_matrix.shape
    # if nrows > ncols:
    #     SSmatrix = np.dot(snapshot_matrix.T, snapshot_matrix)
    # else:
    #     SSmatrix = np.dot(snapshot_matrix, snapshot_matrix.T)
    #     print('WARNING - CHECK HOW THE BASIS FUNCTIONS ARE CALCULATED WITH THIS METHOD')

    SSmatrix = np.dot(snapshot_matrix.T, snapshot_matrix)

    # print('SSmatrix', SSmatrix.shape)
    eigvalues, v = np.linalg.eigh(SSmatrix)
    # print('eigenvalues', eigvalues)
    # print('normalized eigenvector ([:j]) v = ', v)
    eigvalues =  eigvalues[::-1]
    # get rid of small negative eigenvalues (there shouldn't be any as the eigenvalues of a real, symmetric
    # matrix are non-negative, but sometimes very small negative values do appear)
    eigvalues[eigvalues<0] = 0
    # s_values = np.sqrt(eigvalues)
    # singular_values.append(s_values)

    cumulative_info = np.zeros(len(eigvalues))
    for j in range(len(eigvalues)):
        if j==0:
            cumulative_info[j] = eigvalues[j]
        else:
            cumulative_info[j] = cumulative_info[j-1] + eigvalues[j]

    cumulative_info = cumulative_info / cumulative_info[-1]
    nAll = len(eigvalues)

    # Apply POD
    #if nPOD = -1, use cumulative tolerance
    #if nPOD = -2 use all coefficients (or set nPOD = nTime)
    #if nPOD > 0 use nPOD coefficients as defined by the user

    # if nPOD[0] == -1:
    #     # SVD truncation - percentage of information captured or number
    #     # cumulative_tol = nirom_options.compression.cumulative_tol[iField]
    #     nPOD_iField = sum(cumulative_info <= cumulative_tol) #tolerance
    #     nPOD[0] = nPOD_iField
    # elif nPOD[0] == -2:
    #     nPOD_iField = nAll
    #     nPOD[0] = nPOD_iField
    # else:
    #     nPOD_iField = nPOD[0]

    print("retaining", nPOD, "basis functions of a possible", len(eigvalues))

    basis = np.zeros((snapshot_matrix.shape[0],nPOD)) # nDim should be nScalar?
    for j in reversed(range(nAll-nPOD,nAll)):
        Av = np.dot(snapshot_matrix,v[:,j])
        basis[:,nAll-j-1] = Av/np.linalg.norm(Av)

    return basis
#%%
def extract_POD(basis_func, starshape_sm):
    nPOD = basis_func.shape[1]
    alpha_s = np.zeros((starshape_sm.shape[1], nPOD)) #array that stores compressions
    for i in range(starshape.shape[1]):
        alpha_s[i] = basis_func.T @ starshape_sm[:,i]

    return alpha_s
# %%
basis_vel = create_basis_func(snapshots_matrix_vel, 20)
basis_buildings = create_basis_func(snapshots_matrix_b, 10)
starshape_vel = starshape[:nx*ny*2,:]
starshape_b = starshape[nx*ny*2:nx*ny*3,:]
# %%
ss_pod_vel = extract_POD(basis_vel, starshape_vel)
ss_pod_buildings = extract_POD(basis_buildings, starshape_b)

np.save("ss_pod_vel_3000.npy", ss_pod_vel)
np.save("ss_pod_b_3000.npy", ss_pod_buildings)

np.save("basis_b_3000.npy", basis_buildings)
np.save("basis_vel_3000.npy", basis_vel)


## Visualize Arakawa C-grid numbering
import numpy as np
import matplotlib.pyplot as plt

nx = 3
ny = 3

Lx = nx
Ly = ny

dx = Lx/nx
dy = Ly/ny

# grid vectors for T-points
x_T = np.arange(dx/2.,Lx,dx)
y_T = np.arange(dy/2.,Ly,dy)

# grid vectors for u-points
x_u = x_T[:-1] + dx/2.
y_u = y_T

#grid vectors for v-points
x_v = x_T
y_v = y_T[:-1] + dy/2.

# grid vectors for q-points
x_q = np.arange(0,Lx+dx/2.,dx)
y_q = np.arange(0,Ly+dy/2.,dy)

## meshgrid

xx_T,yy_T = np.meshgrid(x_T,y_T)
xx_u,yy_u = np.meshgrid(x_u,y_u)
xx_v,yy_v = np.meshgrid(x_v,y_v)
xx_q,yy_q = np.meshgrid(x_q,y_q)

xx_T = xx_T.flatten()
xx_u = xx_u.flatten()
xx_v = xx_v.flatten()
xx_q = xx_q.flatten()

yy_T = yy_T.flatten()
yy_u = yy_u.flatten()
yy_v = yy_v.flatten()
yy_q = yy_q.flatten()

fig,ax = plt.subplots(1,1)

ax.plot(xx_T,yy_T,'r.',ms=10,mec='k')
ax.plot(xx_u,yy_u,'g.',ms=10,mec='k')
ax.plot(xx_v,yy_v,'b.',ms=10,mec='k')
ax.plot(xx_q,yy_q,'k.',ms=10)

ax.plot(x_v,[0]*nx,'b.',ms=10,alpha=.6)
ax.plot(x_v,[Ly]*nx,'b.',ms=10,alpha=.6)
ax.plot([0]*ny,y_u,'g.',ms=10,alpha=.6)
ax.plot([Lx]*ny,y_u,'g.',ms=10,alpha=.6)

plt.grid()

for xx,yy,vv in zip([xx_T,xx_u,xx_v,xx_q],[yy_T,yy_u,yy_v,yy_q],['T','u','v','q']):
    for i in range(len(xx)):
        ax.text(xx[i]+0.03,yy[i]+0.03,r'$'+vv+'_{%i}$' %(i+1),fontsize=15)

ax.set_xlabel(r'$x$')
ax.set_ylabel(r'$y$')

ax.set_xticks(range(Lx+1))
ax.set_yticks(range(Ly+1))

ax.set_xlim(-0.1,nx+0.1)
ax.set_ylim(-0.1,ny+0.1)

ax.set_frame_on(False)

dxstring = [(r'$%i\Delta x$' % i) if i > 1 else r'$\Delta x$' for i in range(1,nx)]
dystring = [(r'$%i\Delta y$' % i) if i > 1 else r'$\Delta y$' for i in range(1,ny)]

ax.set_xticklabels([r'$0$']+dxstring+[r'$L_x$'])
ax.set_yticklabels([r'$0$']+dystring+[r'$L_y$'])
plt.tight_layout()
plt.show()


### Start initialization
import sys
import numpy as np
import matplotlib.pyplot as plt
### End initialization
### Start Alt2: Streamlines with streamplot ###
w = 3
Y, X = np.mgrid[-w:w:100j, -w:w:100j]
R = np.sqrt(X**2+Y**2)
U = -Y / R**2 # Note that -Y/R = -sin(phi)
V = X / R**2  # Note that X/R = cos(phi)

fig = plt.figure()
ax = fig.add_subplot(111)

strm = ax.streamplot(X, Y, U, V, linewidth=2)

ax.set_title(r'Streamplot, vortex field ' +\
        r'$\vec{F} = \frac{J}{2\pi\rho}\hat\varphi$')
ax.set_xlabel('$x$')
ax.set_ylabel('$y$')

ax.set_aspect('equal')

plt.savefig('streamlines-vortex.png')
### End Alt2

### Start Alt1: Streamlines as 2d curves ###
def curve(rho0,x):
    """Fieldline for vortex streamlines in Cartesian coordinates.
    Assume z=0 => Fieldline in xy-plane.
    We also avoid the singularity at x=y=0."""
    if max(abs(x)) > rho0:
        print('Fieldline for rho_0=%3.1f is only defined for |x| <= %3.1f' %(rho0,rho0))
        return None
    return np.sqrt(rho0**2 - x**2)

fig = plt.figure()
ax = fig.add_subplot(111)

# Grid of x points
nx = 64

# Streamline constant
for rho0 in np.linspace(1,3,5):

    # Grid of x points
    x = np.linspace(-rho0,rho0, nx)

    yp = curve(rho0,x)
    ym = -curve(rho0,x)

    # Plot the streamlines with an appropriate colormap and arrow style
    ax.plot(x, yp, 'b-', x, ym, 'b-', linewidth=2)
    ax.annotate("", xy=(max(x), 0.1), xytext=(max(x), 0), arrowprops=dict(arrowstyle="->",color='b'))

ax.set_title(r'Streamplot, vortex field $\vec{F} = \frac{J}{2\pi\rho}\hat\varphi$')
ax.set_xlabel('$x$')
ax.set_ylabel('$y$')

ax.set_aspect('equal')
### End Alt1

#plt.show()
plt.savefig('streamlines-vortex-curves.png')


%matplotlib inline
import tqdm
import numpy as np
import matplotlib.pyplot as plt
from IPython import display # ?
display.set_matplotlib_formats('svg') # ?

import torch
from torch import Tensor
from torchvision import transforms, datasets

data_transform = transforms.Compose([transforms.Grayscale(), transforms.ToTensor(), transforms.Normalize(mean=[0], std=[1])])

mnist_train = datasets.MNIST(root='./data', train=True, download=True, transform=data_transform)
mnist_test = datasets.MNIST(root='./data', train=False, download=True, transform=data_transform)

xcount = torch.ones((10, 28, 28), dtype=torch.float32)
ycount = torch.ones((10), dtype=torch.float32)
##* Note: by inititalizing as ones, this conduct Laplace smoothing also.

for data, label in mnist_train:
    y = int(label)
    ycount[y] += 1
    xcount[y,:,:] += data.squeeze() # data.shape=(1,28,28); xcount[y,:,:].shape=(28,28)

py = ycount / ycount.sum()
px = xcount / ycount.reshape((10, 1, 1)) # ycount.shape=(10), xcount.shape=(10,28,28), non-broadcastable
##* Note this py, px are also the MLE.

## hotmap count for each class
fig,ax = plt.subplots(1,10,figsize=(10, 10))
for i in range(10):
    ax[i].imshow(xcount[i,:,:].numpy(), cmap='hot')
    ax[i].axes.get_xaxis().set_visible(False)
    ax[i].axes.get_yaxis().set_visible(False)
plt.show()
print('Class probabilities', py)

## give prediction/recognition (as discrimnant task)
data, label = mnist_test[0]
# recall that we classify y as to the class maximize the posterior prob.
# so we need to compute:
# 1) conditional liklihood. p(x|y). log-lik is computed to avoid overflow.
# 2) p(y): this is governed by the py we trained.
logpx = torch.log(px)
logpxneg = torch.log(1-px)
logpy = torch.log(py)

condloglik = data * logpx + (1-data) * logpxneg
condloglik = condloglik.sum((1,2))
bayespost = condloglik + logpy # log[p(x|y)p(y)] i.e. log[p(y|x)]
bayespost -= torch.max(bayespost) # Normalizing to prevent overflow or underflow
post = torch.exp(bayespost)

post /= post.sum()
#* normalizing the post to make it a distribution.
#* note that: \sum_y {p(x|y)p(y)}=p(x); so this gives posterior distribution
torch.argmax(post) # map prediction
# wrap this as function
del(logpx, logpy, logpxneg, condloglik, bayespost, post)

def bayespost(data, px, py):
    logpx = torch.log(px)
    logpxneg = torch.log(1-px)
    logpy = torch.log(py)

    logpost = (data*logpx + (1-data)*logpxneg).sum((1,2)) + logpy
    logpost -= logpost.max()
    post = logpost.exp()

    post /= post.sum()
    return post

post = bayespost(data, px, py)
post.argmax().item()

# now lets try 10 images for testing
fig, ax = plt.subplots(2, 10, figsize=(10, 3))
cnt = 0
for data, label in mnist_test:
    post = bayespost(data, px, py)
    # pic
    ax[1, cnt].bar(range(10), post) # draw the posterior
    ax[1, cnt].axes.get_yaxis().set_visible(False)
    ax[0, cnt].imshow(data.reshape((28,28)), cmap='hot')
    ax[0, cnt].axes.get_xaxis().set_visible(False)
    ax[0, cnt].axes.get_yaxis().set_visible(False)
    cnt += 1

# now calculate the testing error
cnt = 0
err = 0
for data, label in mnist_test:
    cnt += 1
    post = bayespost(data,px,py)

    if (post[int(label)] < post.max()): err += 1

print(f'Naive Bayes has error rate = {err/cnt}')


# Principal component analysis is a method that rotates the dataset in a way such that
# the rotated features are statistically uncorrelated. This rotation is often followed by
# selecting only a subset of the new features, according to how important they are for
# explaining the data.

import matplotlib.pyplot as plt

import mglearn

import numpy as np

from sklearn.datasets import load_breast_cancer
from sklearn.preprocessing import StandardScaler
from sklearn.decomposition import PCA

cancer = load_breast_cancer()

def visualize_cancer_dataset_histograms():
    # One of the most common applications of PCA is visualizing high-dimensional datasets.
    fig, axes = plt.subplots(15, 2, figsize=(10, 20))
    malignant = cancer.data[cancer.target == 0]
    benign = cancer.data[cancer.target == 1]

    ax = axes.ravel()

    # Create a histogram for each of the features, counting how often a data point appears with a feature in a certain range (called a bin).
    for i in range(30):
        _, bins = np.histogram(cancer.data[:, i], bins=50)
        ax[i].hist(malignant[:, i], bins=bins, alpha=.5)
        ax[i].hist(benign[:, i], bins=bins, alpha=.5)
        ax[i].set_title(cancer.feature_names[i])
        ax[i].set_yticks(())
    ax[0].set_xlabel("Feature magnitude")
    ax[0].set_ylabel("Frequency")
    ax[0].legend(["malignant", "benign"], loc="best")
    plt.show()

# these plots dont show us anything about the interactions between variables and how these relate to the classes.
# visualize_cancer_dataset_histograms()

def visualize_cancer_dataset_pca():
    # Before applying PCA, scale the data so that each feature has unit variance.
    scaler = StandardScaler()
    scaler.fit(cancer.data)
    X_scaled = scaler.transform(cancer.data)

    pca = PCA(n_components=2)  # keep the first 2 principal components of the data
    pca.fit(X_scaled)

    X_pca = pca.transform(X_scaled)  # transform data onto the first two principal components.

    print("Original shape: {}".format(str(X_scaled.shape)))  # (569, 30)
    print("Reduced shape: {}".format(str(X_pca.shape)))  # (569, 2)

    print("PCA component shape: {}".format(pca.components_.shape))  # (2, 30)

    # plot first vs second principal component
    plt.figure(figsize=(8, 8))
    mglearn.discrete_scatter(X_pca[:, 0], X_pca[:, 1], cancer.target)
    plt.legend(cancer.target_names, loc="best")
    plt.gca().set_aspect("equal")
    plt.xlabel("First principal component")
    plt.ylabel("Second principal component")
    plt.show()

    # Visualize PCA coefficients
    plt.matshow(pca.components_, cmap='viridis')
    plt.yticks([0, 1], ["First component", "Second component"])
    plt.colorbar()
    plt.xticks(range(len(cancer.feature_names)), cancer.feature_names, rotation=60, ha='left')
    plt.xlabel("Feature")
    plt.ylabel("Principal components")
    plt.show()

visualize_cancer_dataset_pca()


from sympy import symbols, rf, Symbol, factorial, Factorial, ff, nan, oo,\
        Factorial2, factorial2

def test_rf_eval_apply():
    x, y = symbols('x,y')

    assert rf(nan, y) == nan

    assert rf(x, y) == rf(x, y)

    assert rf(oo, 0) == 1
    assert rf(-oo, 0) == 1

    assert rf(oo, 6) == oo
    assert rf(-oo, 7) == -oo

    assert rf(oo, -6) == oo
    assert rf(-oo, -7) == oo

    assert rf(x, 0) == 1
    assert rf(x, 1) == x
    assert rf(x, 2) == x*(x+1)
    assert rf(x, 3) == x*(x+1)*(x+2)
    assert rf(x, 5) == x*(x+1)*(x+2)*(x+3)*(x+4)

    assert rf(x, -1) == 1/(x-1)
    assert rf(x, -2) == 1/((x-1)*(x-2))
    assert rf(x, -3) == 1/((x-1)*(x-2)*(x-3))

    assert rf(1, 100) == factorial(100)

def test_ff_eval_apply():
    x, y = symbols('x,y')

    assert ff(nan, y) == nan

    assert ff(x, y) == ff(x, y)

    assert ff(oo, 0) == 1
    assert ff(-oo, 0) == 1

    assert ff(oo, 6) == oo
    assert ff(-oo, 7) == -oo

    assert ff(oo, -6) == oo
    assert ff(-oo, -7) == oo

    assert ff(x, 0) == 1
    assert ff(x, 1) == x
    assert ff(x, 2) == x*(x-1)
    assert ff(x, 3) == x*(x-1)*(x-2)
    assert ff(x, 5) == x*(x-1)*(x-2)*(x-3)*(x-4)

    assert ff(x, -1) == 1/(x+1)
    assert ff(x, -2) == 1/((x+1)*(x+2))
    assert ff(x, -3) == 1/((x+1)*(x+2)*(x+3))

    assert ff(100, 100) == factorial(100)

def test_factorials():
    n = Symbol('n', integer=True)
    assert factorial(-2) == 0
    assert factorial(0) == 1
    assert factorial(7) == 5040
    assert factorial(n).func == Factorial
    assert factorial(2*n).func == Factorial

def test_factorial2():
    n = Symbol('n', integer=True)
    assert factorial2(-1) == 1
    assert factorial2(0) == 1
    assert factorial2(7) == 105
    assert factorial2(8) == 384
    assert factorial2(n).func == Factorial2


# ！/usr/bin/python
# -*- coding: utf-8 -*-
# Created by PJZero on 17-4-22

import numpy as np

def gen_line_data(sample_num=100):
    """
    y = 3*x1 + 4*x2
    :return:
    """
    x1 = np.linspace(0, 9, sample_num)
    x2 = np.linspace(4, 13, sample_num)
    x = np.concatenate(([x1], [x2]), axis=0).T
    y = np.dot(x, np.array([3, 4]).T)  # y 列向量
    return x, y

def line_regression_mat(samples, labels):
    """
    线性回归：具体请参照西瓜书
    :param samples:
    :param labels:
    :return:
    """
    assert isinstance(samples, np.ndarray)
    assert isinstance(labels, np.ndarray)
    return np.dot(np.linalg.inv(np.dot(samples.T, samples)),
                  np.dot(samples.T, labels))

if __name__ == '__main__':
    x, y = gen_line_data()
    weight = line_regression_mat(x, y)
    print(weight)  # [ 3.  4.]


import pandas as pd
import numpy as np
from scipy import stats
from numpy.linalg import inv
import matplotlib.pylab as plt
import os

os.chdir('e:/MIT4/6.439/pset1')
# 1.2
df_gamma = pd.read_csv('data/gamma-ray.csv')
df_gamma.head()
lam = np.sum(df_gamma['count'])/np.sum(df_gamma['seconds'])
lam

df_gamma['count'] = pd.to_numeric(df_gamma['count'], downcast='signed')
df_gamma['count'].dtype

df_gamma['x!'] = df_gamma['count'].apply(np.math.factorial)

# 1.4
df_golub = pd.read_csv('data/golub_data/golub.csv', index_col=0)
df_cl = pd.read_csv('data/golub_data/golub_cl.csv' ,index_col=0)
df_names = pd.read_csv('data/golub_data/golub_gnames.csv', index_col=0)

df_golub.columns = list(range(1,39))
# 3051 genes of 18 patients
df_golub
df_cl
df_cl
(1-df_cl)['x'].sum()
# How many genes are associated with the different tumor types (meaning that their expression level differs between the two tumor types) using (i) the uncorrected p-values, (ii)the Holm-Bonferroni correction, and (iii) the Benjamini-Hochberg correction?

# split the data into ALL and AML
col_ALL = (df_cl.loc[df_golub.columns] == 0).transpose().values.tolist()[0]
col_AML = (df_cl.loc[df_golub.columns] == 1).transpose().values.tolist()[0]
df_ALL = df_golub[df_golub.columns[col_ALL]]
df_AML = df_golub[df_golub.columns[col_AML]]
df_ALL
# hypothesis testing
from scipy.stats import ttest_ind

# test each gene
t_stat, p_value = [],[]
for i in range(df_golub.shape[0]):
    # t, p =  ttest_ind( df_ALL.iloc[i], df_AML.iloc[i])
    t, p =  ttest_ind( df_ALL.iloc[i], df_AML.iloc[i], equal_var=False )
    t_stat.append(t)
    p_value.append(p)

df_welch_ttest = pd.DataFrame(index = range(1, df_golub.shape[0]+1))
df_welch_ttest['t_stat'] = t_stat
df_welch_ttest['p_value'] = p_value

df_welch_ttest
df_welch_ttest['significant_uncorrected'] = df_welch_ttest['p_value']<0.05
pvals = df_welch_ttest['p_value']

# bonferroni correction
def holm_bonferroni(pvals, alpha=0.05):
    m, pvals = len(pvals), np.asarray(pvals)
    ind = np.argsort(pvals)
    test = [p > alpha/(m+1-k) for k, p in enumerate(pvals[ind])]
    significant = np.zeros(np.shape(pvals), dtype='bool')
    significant[ind[0:m-np.sum(test)]] = True
    return significant

# Benjamini-Hochberg procedure
def BH(pvals, q=0.05):
    m = len(pvals)
    significant = np.zeros(m, dtype='bool')
    sort_ind = np.argsort(pvals).astype(int)+1 # sort the p-values

    for i in range(1,m+1): #i = the individual p-value’s rank
         if pvals[sort_ind[i]] < (i)*q/m:
            significant[sort_ind[i]-1] = True # record the significant index
    return significant

significant_pvals = holm_bonferroni(pvals, alpha=0.05)
df_welch_ttest['significant_pvals'] = significant_pvals
significant_pvals_BH = BH(pvals, q=0.05)
df_welch_ttest['significant_pvals_BH'] = significant_pvals_BH
df_welch_ttest

df_welch_ttest.sum()

# 1.6
syn_x = pd.read_csv('data/syn_X.csv',header=None)
syn_y = pd.read_csv('data/syn_Y.csv',header=None)

syn_x.columns = ['x1','x2']
syn_x['x0'] = 1

syn_x = syn_x[['x0','x1','x2']]
X = syn_x.values
Y = syn_y.values
X.shape
#beta = (X'X)^-1X'Y
a = np.matmul(inv(np.matmul(np.transpose(X),X)),np.transpose(X))

beta = np.matmul(a,Y)

beta

def gradientDescent(X, Y, beta_0, alpha, t):
    m, n = X.shape # m is number of cases, n is the number of variables
    cost = pd.DataFrame(np.zeros([t,2]))
    cost.columns = ['step','cost']
    beta = beta_0

    for i in range(t):
        # vectorized gradient: X'*(Y-X*beta)
        res = Y- np.matmul(X, beta)
        beta = beta + 2 * alpha * (1/m) * np.matmul(np.transpose(X), res)
        # calculate the cost base on current beta
        cost['step'][i] = i
        cost['cost'][i] = calCost(X, Y, beta)

    cost.plot(kind = 'scatter', x = 'step',y = 'cost')
    return beta, cost

def calCost(X, Y, beta):
    m, n = X.shape
    # vectorized cost: (X*beta - Y)'(X*beta - Y)
    residual = Y- np.matmul(X, beta)
    return (1/(2*m))*np.matmul(np.transpose(residual), residual)

beta_0 = np.matrix('0.5; 0.5; 0.5')
alpha = 0.1
t = 50
beta, cost = gradientDescent(X, Y, beta_0,alpha, t )

beta_0 = np.matrix('0.5; 0.5; 0.5')
alpha = 0.01
t = 50
beta, cost = gradientDescent(X, Y, beta_0,alpha, t )

beta_0 = np.matrix('0.5; 0.5; 0.5')
alpha = 0.05
t = 50
beta, cost = gradientDescent(X, Y, beta_0,alpha, t )

beta_0 = np.matrix('0.5; 0.5; 0.5')
alpha = 0.8
t = 50
beta, cost = gradientDescent(X, Y, beta_0,alpha, t )

beta_0 = np.random.rand(3,1)
alpha = 0.05
t = 100
beta, cost = gradientDescent(X, Y, beta_0,alpha, t )

df_mort_0 =  pd.read_csv('data/mortality.csv')
df_mort = df_mort_0

# check the scatterplot

df_mort.head()
# check the correlation matrix
import seaborn as sns

corr = df_mort.iloc[:,2:].corr()
corrplot = sns.heatmap(corr,xticklabels=corr.columns,yticklabels=corr.columns, linewidths=.01,cmap="YlGnBu")
fig = corrplot.get_figure()
fig.savefig('corrplot.png')

df_mort.columns
df_mort = df_mort.drop(['HC'], axis = 1) # drop the intercorrelated

sns.set(style="ticks")
pairplot = sns.pairplot(df_mort.iloc[:,1:], diag_kind="kde",markers="+",plot_kws=dict(s=50, edgecolor="b", linewidth=1),diag_kws=dict(shade=True))

pairplot.savefig('pairplot1.png')
# log-transformation
df_mort[['SO2','NOx','Pop']] = np.log(df_mort[['SO2','NOx','Pop']])
df_morthead()
# normalize the Data
data = df_mort.iloc[:,1:]
data = (data - data.mean())/(data.max() - data.min())
df_mort.iloc[:,1:] = data

df_mort.plot(kind = 'bar', x = 'City', y = 'Mortality',fontsize = 5)
plt.savefig('city.pdf')
df_mort[df_mort['Mortality'] == df_mort['Mortality'].max()]
df_mort[df_mort['Mortality'] == df_mort['Mortality'].min()]

# GD on raw data
Y_r = pd.DataFrame(df_mort_0['Mortality']).values
X_r = df_mort_0.iloc[:,2:].values
Y_r.shape
m, n = X_r.shape

beta_0 = np.random.rand(n,1)
beta_0

alpha = 0.00001
t = 1000
beta, cost = gradientDescent(X_r, Y_r, beta_0, alpha, t )
cost

Y= pd.DataFrame(df_mort['Mortality']).values
X = df_mort.iloc[:,2:].values
m, n = X.shape
beta_0 = np.random.rand(n,1)
alpha = 0.05
t = 2000
beta, cost = gradientDescent(X, Y, beta_0, alpha, t )
beta

#plot the residual
import scipy.stats as stats
import pylab

residuals = np.transpose(Y - np.matmul(X, beta))
res_list = sorted(residuals[0].tolist())
def q_q_plot(data):
    norm=np.random.normal(0,2,len(data))
    norm.sort()

    plt.plot(norm,data,"o")
    z = np.polyfit(norm,data, 1)
    p = np.poly1d(z)
    plt.plot(norm,p(norm),"k--", linewidth=2)
    plt.title("Normal Q-Q plot", size=20)
    plt.xlabel("Theoretical quantiles", size=18)
    plt.ylabel("Expreimental quantiles", size=18)
    plt.tick_params(labelsize=16)
    plt.savefig('qqplot.png')
    plt.show()

q_q_plot(res_list)

#

import numpy,from numpy,import scipy,from scipy


import unittest
import numpy as np

from data_structure_collection.matrix import product

class TestProduct(unittest.TestCase):
    def test_product(self):
        mat1 = [[2, 4], [3, 4]]
        mat2 = [[1, 2], [1, 3]]
        m1, m2, n1, n2 = 2, 2, 2, 2

        res = product(m1, m2, mat1, n1, n2, mat2)
        self.assertTrue(np.array_equal(np.array(res), np.dot(mat1, mat2)))


import numpy as np

class CrossEntropyLoss():
    """
    Numerically stable Cross Entropy Loss (Logistic Loss) function.

    Reference: https://web.stanford.edu/~jurafsky/slp3/5.pdf
    Page 7, Formula 5.12

    Parameters
    ----------
    prob: array_like
        Predicted probabilities.
    y: array_like
        Actual labels.
    epsilon: scalar
        If any of probabilities values are 0 or 1, loss will be undefined, so epsilon is added inside the logs.

    Returns
    -------
    loss: scalar
        Cross Entropy Loss value of given predicted probabilities and actual labels.
    """
    def __call__(self, prob, y, epsilon=1e-15):
        loss = ( -( y * np.log(prob + epsilon) + (1 - y) * np.log(1 - prob + epsilon) )).mean()
        return loss

class Accuracy():
    """
    Calculates mean accuracy of predictions on a given data.

    Parameters
    ----------
    y_pred: array_like
        Predicted labels.
    y_true: array_like
        Actual labels.

    Returns
    -------
    mean_accuracy: scalar
        Mean accuracy of predictions on a given data.
    """
    def __call__(self, y_pred, y_true):
        accuracy = (y_pred == y_true).mean()
        return accuracy

class MeanSquaredError():
    """
    Calculates mean squared error of predictions on a given data.

    Parameters
    ----------
    y_pred: array_like
        Predicted values.
    y_true: array_like
        Actual values.

    Returns
    -------
    error: scalar
        Mean squared error of predictions on a given data.
    """
    def __call__(self, y_pred, y_true):
        error = np.abs(y_pred - y_true).mean()
        return error

class RSquared():
    """
    Calculate R^2.

    Parameters
    ----------
    y_pred: array_like
        Predicted values.
    y_true: array_like
        Actual values.

    Returns
    -------
    r_squared: scalar
        Calculated R^2.
    """
    def __call__(self, y_true, y_pred):

        # Calculate Pearson product-moment correlation coefficient
        correlation_matrix = np.corrcoef(y_true, y_pred)
        correlation = correlation_matrix[0,1]

        # Calculate R^2
        r_squared = correlation**2

        return r_squared