import matplotlib.pyplot as plt
import numpy as np

from sklearn.datasets import make_blobs
from sklearn.cluster import KMeans
from matplotlib import cm
from sklearn.metrics import silhouette_samples

# data set of 150 random points
x, y = make_blobs(n_samples=150, n_features=2, centers=3, cluster_std=0.5, shuffle=True, random_state=0)

plt.scatter(x[:, 0], x[:, 1], c='white', marker='o', s=50)
plt.grid()
plt.show()

km = KMeans(n_clusters=3, init='random', n_init=10, max_iter=300, tol=1e-04, random_state=0)
y_km = km.fit_predict(x)

plt.scatter(x[y_km == 0, 0], x[y_km == 0, 1], c='lightgreen', marker='s', s=50, label='cluster 1')
plt.scatter(x[y_km == 1, 0], x[y_km == 1, 1], c='orange', marker='o', s=50, label='cluster 2')
plt.scatter(x[y_km == 2, 0], x[y_km == 2, 1], c='lightblue', marker='v', s=50, label='cluster 1')
plt.scatter(km.cluster_centers_[:, 0], km.cluster_centers_[:, 1], c='red', marker='*', s=250, label='centroids ')

plt.legend()
plt.grid()
plt.show()

print('Distortion: %.2f' % km.inertia_)

# elbow method to reduce distortion
distortions = []
for i in range(1, 11):
    km = KMeans(n_clusters=i, n_init=10, max_iter=300, random_state=0)
    km.fit(x)
    distortions.append(km.inertia_)

plt.plot(range(1, 11), distortions, marker='o')
plt.xlabel('Number of cluster')
plt.ylabel('Distortion')
plt.show()

# quantifying qual by silhouette plots
cluster_labels = np.unique(y_km)
n_clusters = cluster_labels.shape[0]
silhouette_vals = silhouette_samples(x, y_km, metric='euclidean')
y_ax_lower, y_ax_upper = 0, 0
yticks = []

for i,c in enumerate(cluster_labels):
    c_silhouette_vals = silhouette_vals[ y_km == c]
    c_silhouette_vals.sort()
    y_ax_upper += len(c_silhouette_vals)
    color = cm.jet(i / n_clusters)
    plt.barh(range(y_ax_lower, y_ax_upper), c_silhouette_vals, height=1.0, edgecolor='none', color=color)
    yticks.append((y_ax_lower + y_ax_upper) / 2)
    y_ax_lower += len(c_silhouette_vals)

silhouette_avg = np.mean(silhouette_vals)
plt.axvline(silhouette_avg, color='red', linestyle='--')
plt.yticks(yticks, cluster_labels + 1)
plt.ylabel('Cluster')
plt.xlabel('Silhouette coefficient')
plt.show()


# -*- coding: utf-8 -*-
"""BASIC OPERATIONS PERFORMED ON TENSORS

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/gist/Bobbyorr007/c9a87cad5895bb9dd53f81dd7a2e8e0e/basic-operations-performed-on-tensors.ipynb

**IMPORTS**
"""

import numpy as np
import tensorflow as tf

"""**EXERCISE ON BASIC TENSORS**

I created a single dimensional numpy array of size 50, holding values from 0 to 49.
"""

# Create a 1d uint8 NumPy array comprising of the first 50 natural numbers
x = np.arange(0,50)
x

"""I Changed my numpy array into a tensor"""

# Convert NumPy array into a tensor using the 'tf.constant'
x = tf.constant(x)
x

"""I squared(element-wise) all the values in my tensor x."""

# Square the input tensor x
x = tf.square(x)
x

"""One feature of tensors is that it can be reshaped. while reshaping make sure to use dimensions that will include all values of your Tensor.

"""

# Reshape Tensor into (5 x 10) matrics...
x = tf.reshape(x, (5,10))
x

"""Just like reshaping Tensor, we can also change the data type of Tensors. I will change the data type of my tensor(x) from an int to float."""

# cast tensor x into a float32 dtype
x = tf.cast(x, tf.float32)
x

"""I created a single value Tensor(float32) in other to show the power of broadcasting. """

# I created a single value tensor of dtype float32 to show thw power of broadcasting in the following cell.
y = tf.constant(2, dtype=tf.float32)
y

"""I mutiplied the tensor x and y together and experienced the power of broadcasting. The single value tensor (y), multiplied each value/element of the tensor x."""

# Multiply tensor x by y. tensor y is multilied to each element of tensor x.
result = tf.multiply(x,y)
result

"""I reinitiaized Tensor y to having more values."""

# Reinitialize y to having more values
y = tf.constant([1,2,3,4,5,6,7,8,9,0], dtype=tf.float32)
y

"""I added both tensors together and noticed the addition was done in rows.
The tensor 'y' summed up with the first row of tensor 'x' then moved on to the next row, and the next and so on.
"""

# Add tensor x and y together, and notice the summation in rows.
result = x + y
result

"""When usinf tf.constant, we can pass in a 1D array and using the shape parameter to change into a multi-dimensional array."""

# creating a 1D array with tf.constant and using the shape parameter to change into a 2D array.
x = tf.constant([1234], tf.float32, shape=(2,2))
x

"""This isnt the case with a tf.Variable created Tensor

"""

# creating a 1D array with tf.Variable and using the shape parameter to change into a multi-dimension. NOTICE THE ValueError THATS POPS UP WHEN THE CODE BLOCK IS RAN
x = tf.Variable([1,2,3,4], tf.float32, shape=(2,2))

Course 2 - Custom Training loops, Gradients and Distributed Training/Week 1 - Differentiation and Gradients/basic_operations_performed_on_tensors.py


import numpy as np
import itertools

np.random.seed(0)

def main():
    # 4.14.1
    print('4.14.1')
    # Generator matrix for Hamming code
    G = np.array([
        [1, 0, 1, 1],
        [1, 1, 0, 1],
        [0, 0, 0, 1],
        [1, 1, 1, 0],
        [0, 0, 1, 0],
        [0, 1, 0, 0],
        [1, 0, 0, 0]
    ])

    # 4.14.2
    print('4.14.2')
    print(np.dot(G, np.array([1, 0, 0, 1])) % 2)

    # 4.14.3: find R s.t. np.dot(R, G) % 2 == np.eye(4)
    print('4.14.3')
    R = np.array([
        [0, 0, 0, 0, 0, 0, 1],
        [0, 0, 0, 0, 0, 1, 0],
        [0, 0, 0, 0, 1, 0, 0],
        [0, 0, 1, 0, 0, 0, 0]
    ])
    for vec in itertools.product([0, 1], repeat=4):
        # Check whether decoding works well.
        message = np.array(vec)
        code = np.dot(G, message) % 2
        decoded_message = np.dot(R, code) % 2
        print(message, code, decoded_message)

    # 4.14.4
    print('4.14.4')
    H = np.array([
        [0, 0, 0, 1, 1, 1, 1],
        [0, 1, 1, 0, 0, 1, 1],
        [1, 0, 1, 0, 1, 0, 1]
    ])
    print(np.dot(H, G) % 2)  # == np.zeros((3, 4))

    # 4.14.5
    print('4.14.5')
    def find_error(syndrome):
        n_column = H.shape[1]
        for column in range(n_column):
            if np.all(H[:, column] == syndrome):
                error = np.zeros(n_column, dtype=np.int)
                error[column] = 1
                return error
        return np.zeros(n_column, dtype=np.int)

    # 4.14.6
    print('4.14.6')
    tilde_c = np.array([1, 0, 1, 1, 0, 1, 1])
    syndrome = np.dot(H, tilde_c) % 2
    error = find_error(syndrome)
    error_fixed_tilde_c = (tilde_c + error) % 2
    print(tilde_c, error_fixed_tilde_c)

    # 4.14.7
    print('4.14.7')
    def find_error_matrix(S):
        return np.apply_along_axis(find_error, 0, S)
    S = np.array([
        [0, 0, 1, 1],
        [0, 1, 0, 1],
        [1, 0, 0, 1],
    ])
    print(find_error_matrix(S))

    # 4.14.8
    print('4.14.8')
    def str2bits(input_str):
        bits = [1 << i for i in range(8)]  # 1 byte = 8 bits
        return [1 if ord(char) & b > 0 else 0 for char in input_str for b in bits]
    def bits2str(input_bits):
        bits = [1 << i for i in range(8)]
        return ''.join(chr(
            sum(b * ib for b, ib in zip(bits, input_bits[i:i+8]))
        ) for i in range(0, len(input_bits), 8))
    s = ''.join([chr(i) for i in range(256)])
    ss = bits2str(str2bits(s))
    print(s)
    print(ss)

    # 4.14.9
    print('4.14.9')
    def bits2mat(bits, n_rows=4, trans=False):
        n_cols = len(bits) // n_rows
        mat = np.zeros((n_rows, n_cols), dtype=np.int)
        for i in range(n_rows):
            for j in range(n_cols):
                if bits[n_rows * j + i] > 0:
                    mat[i, j] = 1
        if trans:
            mat = mat.T
        return mat
    def mat2bits(mat, trans=False):
        n_rows, n_cols = mat.shape
        if trans:
            return [mat[i, j] for i in range(n_rows) for j in range(n_cols)]
        else:
            return [mat[i, j] for j in range(n_cols) for i in range(n_rows)]
    s = ''.join([chr(i) for i in range(256)])
    ss = bits2str(mat2bits(bits2mat(str2bits(s))))
    print(s)
    print(ss)

    # 4.14.10
    print('4.14.10')
    message = 'I’m trying to free your mind, Neo. But I can only show you the door. You’re the one that has to walk through it.'
    P = bits2mat(str2bits(message))

    # 4.14.11
    print('4.14.11')
    error_rate = 0.02
    E = np.random.choice([0, 1], P.shape, [1 - error_rate, error_rate])
    P_TILDE = (P + E) % 2
    decoded_message = bits2str(mat2bits(P_TILDE))
    print(decoded_message)  # non-readable message

    # 4.14.12
    print('4.14.12')
    C = np.dot(G, P) % 2

    # 4.14.13
    print('4.14.13')
    error_rate = 0.02
    C_TILDE = (C + np.random.choice([0, 1], C.shape, p=[1 - error_rate, error_rate])) % 2
    decoded_message = bits2str(mat2bits(np.dot(R, C_TILDE) % 2))
    print(decoded_message)  # non-readable message

    # 4.14.14
    print('4.14.14')
    def correct(mat):
        return (mat + find_error_matrix(np.dot(H, mat) % 2)) % 2

    # 4.14.15
    print('4.14.15')
    C_CORRECTED = correct(C_TILDE)
    decoded_message = bits2str(mat2bits(np.dot(R, C_CORRECTED) % 2))
    print(decoded_message)

    # 4.14.16
    print('4.14.16')
    for error_rate in [0.01, 0.02, 0.03, 0.04, 0.05]:
        C_TILDE = (C + np.random.choice([0, 1], C.shape, p=[1 - error_rate, error_rate])) % 2
        C_CORRECTED = correct(C_TILDE)
        decoded_message = bits2str(mat2bits(np.dot(R, C_CORRECTED) % 2))
        print('[Result for error rate {}]: '.format(error_rate) + decoded_message)

if __name__ == '__main__':
    main()


import numpy as np
from scipy.stats import norm

from abc import ABCMeta, abstractmethod

def st(z, s0, r, sigma, T):
    return s0 * np.exp((r - sigma ** 2 / 2) * T + sigma * np.sqrt(T) * z)

s0 = 80;
r = 0.1;
sigma = 0.2;
T = 5;
K = 100

def call(s0, r, sigma, T, K):
    d1 = (np.log(s0 / (K * np.exp(-r * T))) + sigma ** 2 / 2 * T) / (sigma * np.sqrt(T))
    d2 = d1 - sigma * np.sqrt(T)
    return s0 * norm.cdf(d1) - K * np.exp(-r * T) * norm.cdf(d2)

def put(s0, r, sigma, T, K):
    d1 = (np.log(s0 / (K * np.exp(-r * T))) + sigma ** 2 / 2 * T) / (sigma * np.sqrt(T))
    d2 = d1 - sigma * np.sqrt(T)
    return K * np.exp(-r * T) * norm.cdf(-d2) - s0 * norm.cdf(-d1)

class Option(object):
    __metaclass__ = ABCMeta

    def __init__(self, s0, r, sigma, K, T, name=None):
        if name is not None:
            self.name = name
        self.r = r
        self.sigma = sigma
        self.K = K
        self.T = T
        self.s0 = s0
        self.d1 = (np.log(s0 / (K * np.exp(-r * T))) + sigma ** 2 / 2 * T) / (sigma * np.sqrt(T))
        self.d2 = self.d1 - sigma * np.sqrt(T)
        pass

    @abstractmethod
    def payoff(self, S):
        print("Abstract Option class payoff function")
        pass

    @abstractmethod
    def formula(self):
        print("Abstract Option class pricing formula")
        pass

    def monte_carlo(self, M, alpha=0.05):
        randn = np.random.normal(size=M)
        stock_price = st(randn, self.s0, self.r, self.sigma, self.T)
        stock_price_minus = st(-randn, self.s0, self.r, self.sigma, self.T)
        intrinsic_mc = np.exp(-self.r * self.T) * self.payoff(stock_price)
        intrinsic_mc_anti = np.exp(-self.r * self.T) * self.payoff(stock_price_minus)
        intrinsic = (intrinsic_mc + intrinsic_mc_anti) / 2
        price_mc = np.mean(intrinsic)
        std_mc = np.std(intrinsic)
        price_mc_no_anti = np.mean(intrinsic_mc)
        std_mc_no_anti = np.std(intrinsic_mc)
        qalpha = norm.ppf(1 - alpha / 2)
        delta_anti = std_mc / np.sqrt(M) * qalpha
        delta_no_anti = std_mc_no_anti / np.sqrt(M) * qalpha
        return dict({"SampleSize": M,
                     "Price Antithetic Var": price_mc,
                     "Std Antithetic Var": std_mc,
                     "Interval Antithetic Var": (price_mc - delta_anti, price_mc + delta_anti),
                     "Price No Anti": price_mc_no_anti,
                     "Std No Anti": std_mc_no_anti,
                     "Interval No Anti": (price_mc_no_anti - delta_no_anti, price_mc_no_anti + delta_no_anti)})

    def delta_monte_carlo(self, M, alpha=0.05):
        randn = np.random.normal(size=M)
        stock_price = st(randn, self.s0, self.r, self.sigma, self.T)
        stock_price_minus = st(-randn, self.s0, self.r, self.sigma, self.T)
        intrinsic_mc = np.exp(-self.r * self.T) * self.payoff(stock_price) * randn * np.sqrt(self.T) / (
        self.s0 * self.sigma * self.T)
        intrinsic_mc_anti = np.exp(-self.r * self.T) * self.payoff(stock_price_minus) * (-randn) * np.sqrt(self.T) / (
        self.s0 * self.sigma * self.T)
        intrinsic = (intrinsic_mc + intrinsic_mc_anti) / 2
        delta_mc = np.mean(intrinsic)
        return delta_mc

class Call(Option):
    def payoff(self, S):
        if isinstance(S, int):
            return np.array([np.max([S - self.K, 0])])
        else:
            return np.array([np.max([s - self.K, 0]) for s in S])

    def formula(self):
        return self.s0 * norm.cdf(self.d1) - self.K * np.exp(-self.r * self.T) * norm.cdf(self.d2)

    def delta(self):
        return norm.cdf(self.d1)

class Put(Option):
    def payoff(self, S):
        if isinstance(S, int):
            return np.array([np.max([self.K - S, 0])])
        else:
            return np.array([np.max([self.K - s, 0]) for s in S])

    def formula(self):
        return -self.s0 * norm.cdf(-self.d1) + self.K * np.exp(-self.r * self.T) * norm.cdf(-self.d2)

    def delta(self):
        return -norm.cdf(-self.d1)


import numpy as np
from sklearn.preprocessing import PolynomialFeatures
from sklearn.model_selection import train_test_split
from sklearn import linear_model
from sklearn.metrics import mean_squared_error

x1 = np.random.normal(0, 10, 1000)
x2 = np.random.normal(10, 1, 1000)

X = np.zeros((1000, 2))
X[:,0] = x1
X[:,1] = x2

y = (0.5 * x1 ** 2) - (0.7 * x1 * x2) + 10

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.30, random_state=42)
poly = PolynomialFeatures(degree=2)
X_train_transform = poly.fit_transform(X_train)

print(X_train_transform.shape)
X_test_transform = poly.fit_transform(X_test)

regr = linear_model.LinearRegression()
regr.fit(X_train_transform, y_train)
print(regr)

predictions = regr.predict(X_test_transform)
print(regr.coef_)
print(regr.intercept_)
print("W0: {}, w1: {}, w2: {}, w3: {}, w4: {}, w5 : {}, w6: {}".format(round(regr.intercept_), round(regr.coef_[0]), round(regr.coef_[1]), round(regr.coef_[2]), round(regr.coef_[3]), round(regr.coef_[4]), round(regr.coef_[5])))
print("Mean squared error is: ", round(mean_squared_error(predictions, y_test)))

# Regularization

import numpy as np
import pandas as pd
import math

from sklearn.model_selection import train_test_split

from sklearn.linear_model import LinearRegression
from sklearn.linear_model import Ridge
from sklearn.linear_model import Lasso
from sklearn.linear_model import ElasticNet
from sklearn.metrics import mean_squared_error
from sklearn import linear_model

from sklearn.preprocessing import PolynomialFeatures
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import StandardScaler
from sklearn.datasets import load_boston

data = load_boston()
df_x = pd.DataFrame(data.data, columns=data.feature_names)
df_targets = pd.DataFrame(data.target)

X_train, X_test, y_train, y_test = train_test_split(df_x, df_targets, test_size=0.30, random_state=42)

poly = PolynomialFeatures(degree=2)
X_transform = poly.fit_transform(X_train)
regr = linear_model.LinearRegression()
regr.fit(X_transform, y_train)
print(regr)

X_test_transform = poly.fit_transform(X_test)
pred = regr.predict(X_test_transform)

lasso = Lasso(alpha= 0.5, fit_intercept = True)
lasso.fit(X_transform, y_train)

p = lasso.predict(X_test_transform)

ridge = Ridge(alpha=10, fit_intercept = True)
ridge.fit(X_transform, y_train)

r = ridge.predict(X_test_transform)

# print(regr.coef_)
# print(regr.intercept_)
print("Mean squared error is: ", mean_squared_error(pred, y_test))
print("Mean squared error for Lasso is: ", mean_squared_error(p, y_test))
print("Mean squared error for Ridge is: ", mean_squared_error(r, y_test))


from scipy.optimize import linprog
import numpy as np

# Objective function
z = np.array([5,4])

# Constraints
C = np.array([
    [1, 1],          #C1
    [10,6]           #C2
])

b = np.array([5,45])

# Bounds
x1 = (0, None)
x2 = (0, None)

#Solution
sol = linprog(-z, A_ub = C, b_ub = b, bounds = (x1, x2), method='simplex')

print(f"x1 = {sol.x[0]}, x2 = {sol.x[1]}, z = {sol.fun*-1}")


import numpy as np
import matplotlib.pyplot as plt

def dvdir(func, x, dx):
    return (func(x + dx) - func(x))/dx

def dvcen(func, x, dx):
    return (func(x + dx) - func(x - dx))/(2*dx)

def erro(dfx, dvan):
    return abs(dvan - dfx)/abs(dvan)

fx = lambda x: (x**3)*np.sin(x)
dx = np.logspace(-16, 0, 100)
dfxdir = dvdir(fx, 3, dx)
dfxcen = dvcen(fx, 3, dx)
dfxan = lambda x: (x**3)*np.cos(x) + 3*(x**2)*np.sin(x)
dvan = np.ones(100) * dfxan(3)

plt.plot(dx, dfxdir, 'b', label='Direita')
plt.plot(dx, dfxcen, 'r', label='Centrada')
plt.hlines(dfxan(3), 1e-16, 1, 'k', label='Analítica')
plt.title('Comparação das derivadas')
plt.xlabel(r'$\Delta x$')
plt.ylabel("f \'(x)")
plt.xscale('log')
plt.grid()
plt.legend()
plt.xlim(1e-16, 1)
plt.show()
#plt.savefig('comp_derivadas.png')

erro_cen = erro(dfxcen, dvan)
erro_dir = erro(dfxdir, dvan)
plt.plot(dx, erro_cen, 'r', label='Centrada')
plt.plot(dx, erro_dir,  'b', label='Direita')
plt.xscale('log')
plt.yscale('log')
plt.xlabel(r'$\Delta x$')
plt.ylabel('Erro')
plt.xlim(1e-16, 1)
plt.grid()
plt.legend()
plt.title('Comparação de erros das derivadas')
plt.show()
#plt.savefig('erro_derivadas.png')


import numpy as np

def ks(x1, x2):
    data1 = np.sort(x1)
    data2 = np.sort(x2)
    n1 = data1.shape[0]
    n2 = data2.shape[0]

    data_all = np.concatenate([data1, data2])
    # using searchsorted solves equal data problem
    cdf1 = np.searchsorted(data1, data_all, side='right') / n1
    cdf2 = np.searchsorted(data2, data_all, side='right') / n2
    cddiffs = cdf1 - cdf2
    minS = np.clip(-np.min(cddiffs), 0, 1)  # Ensure sign of minS is not negative.
    maxS = np.max(cddiffs)
    return max(minS, maxS)

def ad(x1, x2):
    k = 2
    n = np.array([x1.size, x2.size])

    Z = np.sort(np.hstack((x1, x2)))
    N = Z.size
    Zstar = np.unique(Z)

    A2akN = 0.
    Z_ssorted_left = Z.searchsorted(Zstar, 'left')
    if N == Zstar.size:
        lj = 1.
    else:
        lj = Z.searchsorted(Zstar, 'right') - Z_ssorted_left

    Bj = Z_ssorted_left + lj / 2.
    s = np.sort(x1)
    s_ssorted_right = s.searchsorted(Zstar, side='right')
    Mij = s_ssorted_right.astype(float)
    fij = s_ssorted_right - s.searchsorted(Zstar, 'left')
    Mij -= fij / 2.
    inner = lj / float(N) * (N*Mij - Bj*n[0])**2 / (Bj*(N - Bj) - N*lj/4.)
    A2akN += inner.sum() / n[0]

    s = np.sort(x2)
    s_ssorted_right = s.searchsorted(Zstar, side='right')
    Mij = s_ssorted_right.astype(float)
    fij = s_ssorted_right - s.searchsorted(Zstar, 'left')
    Mij -= fij / 2.
    inner = lj / float(N) * (N*Mij - Bj*n[1])**2 / (Bj*(N - Bj) - N*lj/4.)
    A2akN += inner.sum() / n[1]

    A2akN *= (N - 1.) / N
    return A2akN

def ku(x1, x2):
    data1 = np.sort(x1)
    data2 = np.sort(x2)
    n1 = data1.shape[0]
    n2 = data2.shape[0]

    data_all = np.concatenate([data1, data2])
    # using searchsorted solves equal data problem
    cdf1 = np.searchsorted(data1, data_all, side='right') / n1
    cdf2 = np.searchsorted(data2, data_all, side='right') / n2
    cddiffs = cdf1 - cdf2

    minS = np.clip(-np.min(cddiffs), 0, 1)  # Ensure sign of minS is not negative.
    maxS = np.max(cddiffs)

    return minS + maxS


import time
import numpy as np

def get_svd(M, energy=1):
	"""
	Performs singular value decomposition

    Parameters
    ----------
    M: numpy.ndarray
        The matrix that needs to be decomposed.
    energy: float, optional
        The energy threshold for performing dimensionality
		reduction.

    Returns
    -------
    U numpy.ndarray, sigma numpy.ndarray, V.T numpy.ndarray
        U is the left singular matrix
		sigma is a diagonal matrix
		V.T is the right singular matrix
    """

	m = M.shape[0]
	n = M.shape[1]
	# shape of MMT = m x m
	MMT = np.matmul(M, M.T)
	# shape of MTM = n x n
	MTM = np.matmul(M.T, M)
	eigenvals_u, U = np.linalg.eigh(MMT)
	eigenvals, V = np.linalg.eigh(MTM)
	U1 = U.copy()
	V1 = U.copy()

	for i in range(eigenvals.shape[0]):
		eigenvals[i] = 0 if eigenvals[i]<1e-6 else eigenvals[i]

	num_eig = eigenvals.shape[0]
	eigenvals = eigenvals[::-1]

	sigma = np.zeros((max(num_eig, U.shape[1]), max(num_eig, V.shape[1])))
	sigma[:num_eig][:num_eig] = np.diag(np.sqrt(eigenvals))
	sigma = sigma[:U.shape[1], :V.shape[1]]

	# arranging eigen vectors in decreasing order
	U = U[:, ::-1]
	V = V[:, ::-1]

	# print(U.shape, sigma.shape, V.shape)
	print("error before reduction: ", np.linalg.norm(M-U@sigma@(V.T)))

	i = 0
	while i<min(sigma.shape[0],sigma.shape[1]):
		# removing columns from U and rows from sigma
		if sigma[i][i]==0:
			U = np.delete(U, i, 1)
			V = np.delete(V, i, 1)
			sigma = np.delete(sigma, i, 0)
			sigma = np.delete(sigma, i, 1)
			i-=1
		i+=1

	cur_energy = 0
	tot_energy = sum(eigenvals)
	# number of eigenvalues with energy greater than threshold
	num = 0
	for i in range(eigenvals.shape[0]):
		cur_energy += eigenvals[i]
		if (cur_energy/tot_energy)>=energy:
			num = i
			break
	U = U[:,:num+1]
	V = V[:,:num+1]
	sigma = sigma[:num+1, :num+1]
	# print(U.shape,sigma.shape, V.shape)
	print("error after reduction: ", np.linalg.norm(M-U@sigma@(V.T)))

	return U,sigma,V.T

def get_query_(query, df):
	U,sigma,VT = get_svd(df)
	return (query @ sigma)

def libsvd(M):
	'''
	SVD using numpy library function.
	'''
	U,d,VT = np.linalg.svd(M)
	s = np.zeros((U.shape[1], VT.shape[0]))
	s[:d.shape[0],:d.shape[0]] = np.diag(d)
	print(U.shape, s.shape, VT.shape)
	print("error: ", np.linalg.norm(M-U@s@VT))
	return U,s,VT


# -*- coding: utf-8 -*-
"""
Created on Thu Dec  2 20:14:10 2021

@author: Oliver
"""

"""Univariate Optimisation"""

# Golden Search Technique for finding the Maximum

import numpy as np

def gsection(ftn, xl, xm, xr, tol = 1e-9):
    # applies the golden-section algorithm to maximise ftn
    # we assume that ftn is a function of a single variable
    # and that x.l < x.m < x.r and ftn(x.l), ftn(x.r) <= ftn(x.m)
    #
    # the algorithm iteratively refines x.l, x.r, and x.m and
    # terminates when x.r - x.l <= tol, then returns x.m
    # golden ratio plus one
    gr1 = 1 + (1 + np.sqrt(5))/2
    #
    # successively refine x.l, x.r, and x.m
    fl = ftn(xl)
    fr = ftn(xr)
    fm = ftn(xm)
    while ((xr - xl) > tol):
        if ((xr - xm) > (xm - xl)):
            y = xm + (xr - xm)/gr1
            fy = ftn(y)
            if (fy >= fm):
                xl = xm
                fl = fm
                xm = y
                fm = fy
            else:
                xr = y
                fr = fy
        else:
            y = xm - (xm - xl)/gr1
            fy = ftn(y)
            if (fy >= fm):
                xr = xm
                fr = fm
                xm = y
                fm = fy
            else:
                xl = y
                fl = fy
    return(xm)

xl=0
xm=5
xr=10
def ftn(x):
    return x**2+5*x+10
print(gsection(ftn, xl, xm, xr, tol = 1e-9))
print(ftn(xm))

# Newton Maximizing Algorithm
import math
import matplotlib.pyplot as plt

def f(x):
    return x**3-6*x**2+4*x+2
x = np.linspace(-1, 1)
fig, ax = plt.subplots()
ax.plot(x, f(x), label='target')
ax.grid()

def fprime(x):
    return 3*x**2-12*x+4
def fsecond(x):
    return 6*x - 12
def quadratic_approx(x, x0, f, fprime, fsecond):
    return f(x0)+fprime(x0)*(x-x0)+0.5*fsecond(x0)*(x-x0)**2

x = np.linspace(-1, 1)
fig, ax = plt.subplots()
ax.plot(x, f(x), label='target')
ax.grid()
ax.plot(x, quadratic_approx(x, 0, f, fprime, fsecond), color='red', label='quadratic approximation')
ax.set_ylim([-2,3])
ax.axhline(y=0, color='k')
ax.axvline(x=0, color='k')
plt.legend()

def newton(x0, fprime, fsecond, maxiter=100, eps=0.0001):
    x=x0
    for i in range(maxiter):
        xnew=x-(fprime(x)/fsecond(x))
        if xnew-x<eps:
            return xnew
            print('converged')
            break
        x = xnew
    return x

x_star=newton(0, fprime, fsecond)
fig, ax = plt.subplots()
ax.plot(x, f(x), label='target')
ax.grid()
ax.plot(x, quadratic_approx(x, x_star , f, fprime, fsecond), color='red', label='quadratic approximation')
ax.set_ylim([-2,3])
ax.axhline(y=0, color='k')
ax.axvline(x=0, color='k')
ax.axvline(x = x_star, color='green')
plt.legend()

#Valentina Alto: See: https://medium.com/swlh/optimization-algorithms-the-newton-method-4bc6728fb3b6

# Newton Minimizing Algorithm
def f(x):
    return -x**3+6*x**2-4*x-2
x = np.linspace(-1, 1)
fig, ax = plt.subplots()
ax.plot(x, f(x), label='target')
ax.grid()

def fprime(x):
    return -3*x**2+12*x-4
def fsecond(x):
    return -6*x + 12
def quadratic_approx(x, x0, f, fprime, fsecond):
    return f(x0)+fprime(x0)*(x-x0)+0.5*fsecond(x0)*(x-x0)**2

x = np.linspace(-1, 1)
fig, ax = plt.subplots()
ax.plot(x, f(x), label='target')
ax.grid()
ax.plot(x, quadratic_approx(x, 0, f, fprime, fsecond), color='red', label='quadratic approximation')
ax.set_ylim([-5,3])
ax.axhline(y=0, color='k')
ax.axvline(x=0, color='k')
plt.legend()

def newton(x0, fprime, fsecond, maxiter=100, eps=0.0001):
    x=x0
    for i in range(maxiter):
        xnew=x-(fprime(x)/fsecond(x))
        if xnew-x<eps:
            return xnew
            print('converged')
            break
        x = xnew
    return x

x_star=newton(0, fprime, fsecond)
fig, ax = plt.subplots()
ax.plot(x, f(x), label='target')
ax.grid()
ax.plot(x, quadratic_approx(x, x_star , f, fprime, fsecond), color='red', label='quadratic approximation')
ax.set_ylim([-5,3])
ax.axhline(y=0, color='k')
ax.axvline(x=0, color='k')
ax.axvline(x = x_star, color='green')
plt.legend()

#Valentina Alto: See: https://medium.com/swlh/optimization-algorithms-the-newton-method-4bc6728fb3b6

# Gradient Descent vs Newton Unconstrained Multivariate Optimisation

import matplotlib.pyplot as plt
plt.style.use('seaborn-white')
import numpy as np
from mpl_toolkits import mplot3d

def Rosenbrock(x,y):
    return (1 + x)**2 + 100*(y - x**2)**2

def Grad_Rosenbrock(x,y):
    g1 = -400*x*y + 400*x**3 + 2*x -2
    g2 = 200*y -200*x**2
    return np.array([g1,g2])

def Hessian_Rosenbrock(x,y):
    h11 = -400*y + 1200*x**2 + 2
    h12 = -400 * x
    h21 = -400 * x
    h22 = 200
    return np.array([[h11,h12],[h21,h22]])

def Gradient_Descent(Grad,x,y, gamma = 0.00125, epsilon=0.0001, nMax = 10000 ):
    #Initialization
    i = 0
    iter_x, iter_y, iter_count = np.empty(0),np.empty(0), np.empty(0)
    error = 10
    X = np.array([x,y])

    #Looping as long as error is greater than epsilon
    while np.linalg.norm(error) > epsilon and i < nMax:
        i +=1
        iter_x = np.append(iter_x,x)
        iter_y = np.append(iter_y,y)
        iter_count = np.append(iter_count ,i)
        #print(X)

        X_prev = X
        X = X - gamma * Grad(x,y)
        error = X - X_prev
        x,y = X[0], X[1]

    print(X)
    return X, iter_x,iter_y, iter_count

root,iter_x,iter_y, iter_count = Gradient_Descent(Grad_Rosenbrock,-2,2)

#PLOTTING THE SOLUTION

x = np.linspace(-2,2,250)
y = np.linspace(-1,3,250)
X, Y = np.meshgrid(x, y)
Z = Rosenbrock(X, Y)

#Angles needed for quiver plot
anglesx = iter_x[1:] - iter_x[:-1]
anglesy = iter_y[1:] - iter_y[:-1]

#matplotlib inline
fig = plt.figure(figsize = (16,8))

#Surface plot
ax = fig.add_subplot(1, 2, 1, projection='3d')
ax.plot_surface(X,Y,Z,rstride = 5, cstride = 5, cmap = 'jet', alpha = .4, edgecolor = 'none' )
ax.plot(iter_x,iter_y, Rosenbrock(iter_x,iter_y),color = 'r', marker = '*', alpha = .4)

ax.view_init(45, 280)
ax.set_xlabel('x')
ax.set_ylabel('y')

#Contour plot
ax = fig.add_subplot(1, 2, 2)
ax.contour(X,Y,Z, 50, cmap = 'jet')
#Plotting the iterations and intermediate values
ax.scatter(iter_x,iter_y,color = 'r', marker = '*')
ax.quiver(iter_x[:-1], iter_y[:-1], anglesx, anglesy, scale_units = 'xy', angles = 'xy', scale = 1, color = 'r', alpha = .3)
ax.set_title('Gradient Descent with {} iterations'.format(len(iter_count)))

plt.show()

#Newton's Method (Multi-Dimensional)

def Newton_Raphson_Optimize(Grad, Hess, x,y, epsilon=0.000001, nMax = 200):
    #Initialization
    i = 0
    iter_x, iter_y, iter_count = np.empty(0),np.empty(0), np.empty(0)
    error = 10
    X = np.array([x,y])

    #Looping as long as error is greater than epsilon
    while np.linalg.norm(error) > epsilon and i < nMax:
        i +=1
        iter_x = np.append(iter_x,x)
        iter_y = np.append(iter_y,y)
        iter_count = np.append(iter_count ,i)
        print(f'X: {X}')

        X_prev = X
        X = X - np.linalg.inv(Hess(x,y)) @ Grad(x,y)
        error = X - X_prev
        x,y = X[0], X[1]

    return X, iter_x,iter_y, iter_count

root,iter_x,iter_y, iter_count = Newton_Raphson_Optimize(Grad_Rosenbrock,Hessian_Rosenbrock,-2,2)

x = np.linspace(-3,3,250)
y = np.linspace(-9,8,350)
X, Y = np.meshgrid(x, y)
Z = Rosenbrock(X, Y)

#Angles needed for quiver plot
anglesx = iter_x[1:] - iter_x[:-1]
anglesy = iter_y[1:] - iter_y[:-1]

#matplotlib inline
fig = plt.figure(figsize = (16,8))

#Surface plot
ax = fig.add_subplot(1, 2, 1, projection='3d')
ax.plot_surface(X,Y,Z,rstride = 5, cstride = 5, cmap = 'jet', alpha = .4, edgecolor = 'none' )
ax.plot(iter_x,iter_y, Rosenbrock(iter_x,iter_y),color = 'r', marker = '*', alpha = .4)

#Rotate the initialization to help viewing the graph
ax.view_init(45, 280)
ax.set_xlabel('x')
ax.set_ylabel('y')

#Contour plot
ax = fig.add_subplot(1, 2, 2)
ax.contour(X,Y,Z, 60, cmap = 'jet')
#Plotting the iterations and intermediate values
ax.scatter(iter_x,iter_y,color = 'r', marker = '*')
ax.quiver(iter_x[:-1], iter_y[:-1], anglesx, anglesy, scale_units = 'xy', angles = 'xy', scale = 1, color = 'r', alpha = .3)
ax.set_title('Newton method with {} iterations'.format(len(iter_count)))

plt.show()


import math
import numpy as np

#NonLinear activation functions

class Activation_Function(object):
	"""Abstract Class for activation function"""
	def derivative(self):
		"""
		activation derivative
		:return:
		"""
		raise NotImplementedError()

	def name(self):
		return self.__class__.__name__

class Identity(object):
	def __call__(self,z):
		return z

	def derivative(self,z):
		return np.ones(z.shape)

class Sigmoid(Activation_Function):

	def __call__(self,z):
		return float(1) / (1 + np.exp(-z))

	def derivative(self,z):
		return self.__call__(z) * self.__call__(-z)

class RLU(Activation_Function):

	def __call__(self, z):
		return np.where(z <= 0,0,z)

	def derivative(self, z):
		return np.where(z <= 0,0,1)

class Leaky_RLU(Activation_Function):

	def __init__(self,alpha = 0.1):
		self.alpha = alpha

	def __call__(self, z):
		return np.where(z<=0,self.alpha*z,z)

	def derivative(self, z):
		return np.where(x<=0,self.alpha,1)

class Softplus(Activation_Function):

	def init(self):
		pass

	def __call__(self,z):
		return np.log(1+np.exp(z))

	def derivative(self,z):
		return float(1) / (1 + np.exp(-z))

class Softmax(Activation_Function):
    def __call__(self, x):
        e_x = np.exp(x - np.max(x, axis=-1, keepdims=True))
        return e_x / np.sum(e_x, axis=-1, keepdims=True)

    def derivative(self, x):
        p = self.__call__(x)
        return p * (1 - p)


# Prepare the libraries for the exercise
import matplotlib
from matplotlib import pyplot as plt

import numpy as np
import tensorflow as tf
tf.enable_eager_execution()

import tensorflow_probability as tfp
tfd = tfp.distributions
np.random.seed(5218)
# ===========================================================================
# Some configurations
# ===========================================================================
np.random.seed(5218)

# ===========================================================================
# Ex1
# ===========================================================================
# n_samples = 100
# # Two independent dice
# X = np.random.randint(1, 7, n_samples)
# Y = np.random.randint(1, 7, n_samples)

# Z = 8 * np.log(X) - np.random.rand(n_samples)

# # plotting the distrubtion
# plt.figure(figsize=(16, 4))
# ids = np.arange(n_samples)
# plt.subplot(1, 3, 1)
# plt.scatter(ids, X, s=4); plt.title("X")
# plt.subplot(1, 3, 2)
# plt.scatter(ids, Y, s=4); plt.title("Y")
# plt.subplot(1, 3, 3)
# plt.scatter(ids, Z, s=4); plt.title("Z")

# print("Cov(X, Y):\n", np.cov(X, Y))
# print("Cov(X, Z):\n", np.cov(X, Z))
# print("Cov(Y, Z):\n", np.cov(Y, Z))

# ===========================================================================
# Ex4 Correct answer
# ===========================================================================
def gmm(batch_size, n_clusters, n_dim, sigma0,
        using_multivariate_normal=True):
  """ This is the solution for the process in 1.3 """
  # parameters for Dirichlet distribution
  alpha = np.ones(shape=(n_clusters,))
  print("Alpha:", alpha)
  print("Sigma0:", sigma0)

  # step: 1
  dirichlet = tfd.Dirichlet(concentration=alpha)
  theta = dirichlet.sample()

  # step: 2
  # ====== using multi-variate normal for generating components' mean ====== #
  if using_multivariate_normal:
    normal_1 = tfd.MultivariateNormalDiag(
        loc=np.full(shape=(n_dim,), fill_value=0, dtype='float32'),
        scale_diag=np.full(shape=(n_dim,), fill_value=sigma0, dtype='float32'))
  # ====== using multiple Normal distribution instead ====== #
  else:
    normal_1 = tfd.Normal(
        loc=np.full(shape=(n_dim,), fill_value=0, dtype='float32'),
        scale=np.full(shape=(n_dim,), fill_value=sigma0, dtype='float32'))
    normal_1 = tfd.Independent(normal_1, reinterpreted_batch_ndims=1)
  # sampling `n_clusters` time, hence, mean for each cluster
  mu_k = normal_1.sample(n_clusters) # (batch_size, n_clusters, n_dim)

  # ====== Now for the assignment, need 1 indicator for each
  # examples, hence we need `batch_size` amount of indicator
  # (step: 3(a))====== #
  categorical = tfd.OneHotCategorical(probs=theta)
  z = categorical.sample(batch_size) # (batch_size, n_clusters)
  z = tf.cast(z, tf.bool)

  # ====== sampling the data points (step: 3(b)) ====== #
  normal_2 = tfd.Normal(loc=mu_k, scale=1)
  # this make each draw sample will generate sample for
  # all 4 components
  normal_2 = tfd.Independent(normal_2, reinterpreted_batch_ndims=2)
  x_all_components = normal_2.sample(batch_size) # (batch_size, n_clusters, n_dim)

  # ====== selecting the right component for each sample (step: 3(b)) ====== #
  # (batch_size, n_clusters, n_dim) * (batch_size, n_clusters)
  # = (batch_size, n_dim)
  x = tf.boolean_mask(x_all_components, z)

  return x.numpy(), np.argmax(z.numpy(), axis=-1)

X1, Z1 = gmm(1000, n_clusters=4, n_dim=2, sigma0=2,
             using_multivariate_normal=True)
X2, Z2 = gmm(1000, n_clusters=4, n_dim=2, sigma0=10,
             using_multivariate_normal=True)
X3, Z3 = gmm(1000, n_clusters=4, n_dim=2, sigma0=2,
             using_multivariate_normal=False)
X4, Z4 = gmm(1000, n_clusters=4, n_dim=2, sigma0=10,
             using_multivariate_normal=False)

def plot_clusters(x, z, title):
  # visualizing the mixture
  import seaborn as sns
  cluster_colors = sns.color_palette("hls", len(np.unique(Z1)))
  plt.scatter(x[:, 0], x[:, 1], c=[cluster_colors[i] for i in z], s=6)
  plt.title(str(title))

plt.subplot(1, 4, 1)
plot_clusters(X1, Z1, "Sigma:2  Multi-variate")
plt.subplot(1, 4, 2)
plot_clusters(X2, Z2, "Sigma:10 Multi-variate")
plt.subplot(1, 4, 3)
plot_clusters(X3, Z3, "Sigma:2  Normal")
plt.subplot(1, 4, 4)
plot_clusters(X4, Z4, "Sigma:10 Normal")

# ===========================================================================
# Ex4 Bonus
# ===========================================================================
# number of samples
n_samples = 1000
# number of cluster
n_clusters = 2
# number of feature dimension
n_dim = 2

# parameters for Dirichlet distribution
alpha = np.ones(shape=(n_clusters,))
# sigma for the generation of the mean
# NOTE: we have to repeat the sigma for each cluster
sigma = np.repeat(np.expand_dims(np.array([0.5, 8], dtype='float32'),
                                 axis=0),
                  repeats=n_clusters, axis=0)
print(sigma.shape)

dirichlet = tfd.Dirichlet(concentration=alpha)
print(dirichlet)
theta = dirichlet.sample()
print("Theta:", theta)

normal_1 = tfd.Normal(loc=0, scale=sigma)
print(normal_1)
normal_1 = tfd.Independent(normal_1, reinterpreted_batch_ndims=2)
print(normal_1)
mu_k = normal_1.sample()
print("Mu_k:", mu_k)

categorical = tfd.Categorical(probs=theta)
print(categorical)
z = categorical.sample()
print("Z_i:", z)

mu_x = mu_k[z]
normal_2 = tfd.Normal(loc=mu_x, scale=1)
print(normal_2)
normal_2 = tfd.Independent(normal_2, reinterpreted_batch_ndims=1)
print(normal_2)
x = normal_2.sample()
print("X_i:", x)

def gmm(batch_size, n_clusters, n_dim):
  """ This is the solution for the process in 1.3 """
  # parameters for Dirichlet distribution
  alpha = np.ones(shape=(n_clusters,))
  sigma = np.repeat(np.random.randint(1, 25, (1, n_dim)),
                    repeats=n_clusters, axis=0).astype('float32')
  print("Alpha:", alpha)
  print("Sigma0:", sigma[0])

  dirichlet = tfd.Dirichlet(concentration=alpha)
  theta = dirichlet.sample(batch_size)

  normal_1 = tfd.Normal(loc=0, scale=sigma)
  normal_1 = tfd.Independent(normal_1, reinterpreted_batch_ndims=2)
  mu_k = normal_1.sample(batch_size) # (batch_size, n_clusters, n_dim)

  categorical = tfd.OneHotCategorical(probs=theta)
  z = categorical.sample() # (batch_size, n_clusters)
  z = tf.cast(z, tf.bool)

  # another solution is using tf.where (but it is only
  # feasible for 2 components)
  # tf.where(z == 0, component_1, component_2)

  # (batch_size, n_dim)
  mu_x = tf.boolean_mask(mu_k, z)
  normal_2 = tfd.Normal(loc=mu_x, scale=1)
  normal_2 = tfd.Independent(normal_2, reinterpreted_batch_ndims=1)
  x = normal_2.sample()
  return x.numpy(), np.argmax(z.numpy(), axis=-1)
X1, Z1 = gmm(1000, n_clusters=2, n_dim=2)

def gmm_simple(batch_size):
  """ This is the solution for the process in 1.3 """
  dirichlet = tfd.Dirichlet(concentration=alpha)
  theta = dirichlet.sample(batch_size)

  sigma0 = sigma[0][None, :]
  components = [tfd.Independent(tfd.Normal(loc=0,
                                           scale=np.repeat(sigma0, batch_size, 0)),
                                reinterpreted_batch_ndims=1)
                for i in range(n_clusters)]
  mixture = tfd.Mixture(
      cat=tfd.Categorical(probs=theta),
      components=components)

  normal_2 = tfd.Normal(loc=mixture.sample(), scale=1)
  normal_2 = tfd.Independent(normal_2, reinterpreted_batch_ndims=1)
  x = normal_2.sample()
  return x
X2 = gmm_simple(1000)

def gmm_fun(batch_size):
  """ This is the solution for the process in 1.3 """
  # parameters for Dirichlet distribution
  # ====== configuration ====== #
  n_clusters = 2
  n_dim = 2
  alpha = np.ones(shape=(n_clusters,))
  mu = [0, 1000]
  sigma = [1, 1]

  # ====== validate all argument ====== #
  assert len(mu) == n_clusters
  assert len(alpha) == n_clusters
  assert len(sigma) == n_dim
  # ====== preprocessing ====== #
  mu = np.repeat(np.array(mu)[:, None],
                 repeats=n_dim, axis=1).astype('float32')
  sigma = np.repeat(np.array(sigma)[None, :],
                    repeats=n_clusters, axis=0).astype('float32')
  print("Alpha:", alpha)
  print("Mu   :\n", mu)
  print("Sigma:\n", sigma)

  dirichlet = tfd.Dirichlet(concentration=alpha)
  theta = dirichlet.sample(batch_size)

  normal_1 = tfd.Normal(loc=mu, scale=sigma)
  normal_1 = tfd.Independent(normal_1, reinterpreted_batch_ndims=2)
  mu_k = normal_1.sample(batch_size) # (batch_size, n_clusters, n_dim)

  categorical = tfd.OneHotCategorical(probs=theta)
  z = categorical.sample() # (batch_size, n_dim)
  z = tf.cast(z, tf.bool)

  # (batch_size, n_dim)
  mu_x = tf.boolean_mask(mu_k, z)
  normal_2 = tfd.Normal(loc=mu_x, scale=1)
  normal_2 = tfd.Independent(normal_2, reinterpreted_batch_ndims=1)
  x = normal_2.sample()
  return x.numpy(), np.argmax(z.numpy(), axis=-1)

X1, Z1 = gmm_fun(1000)


# -*- coding: utf-8 -*-
"""
Created on Fri Jun 21 09:35:06 2019

@author: hu
"""
import numpy as np
def cross_entropy_error(y,t):
    delta = 1e-7
    ans = -np.sum(t*np.log(y+delta))
    return ans

t=[0,0,1]
y=[0.1,0.5,0.4]
ans=cross_entropy_error(np.array(y),np.array(t))
print(ans)

#mini batch

def cross_entropy_error_minibatch(y,t):
    if(y.ndim==1):
        t=t.reshape(1,t.size)
        y=y.reshape(1,y.size)
    batch_size = y.shape[0]

    delta = 1e-7
    ans = -np.sum(t*np.log(y[np.arange(batch_size),t]+delta))
    return ans

t=[0,0,1]
y=[0.1,0.5,0.4]
ans=cross_entropy_error_minibatch(np.array(y),np.array(t))
print(ans)

#数值微分
def function_1(x):
    return 0.01*x**2+0.1*x
import matplotlib.pylab as plt

def numerical_diff(f,x):
    h = 1e-4 #0.0001
#    print(f(x+h),f(x-h))
    return (f(x+h)-f(x-h))/(2*h)

x = np.arange(0.0,20.0,0.1)
y = function_1(x)
plt.xlabel("x")
plt.ylabel("f(x)")
plt.plot(x,y)
plt.show()
numerical_diff(function_1,5)


import pickle
import numpy as np
import scipy.stats as scs

# Don't change this. This fixes the randomness in sampling
np.random.seed(seed=1234)

# This loads in the list of numbers you are going to deal with
def load_pickle(file_name):
    """INPUT:
    - file_name(STR) [The name of the file]

    OUTPUT:
    - population(NUMPY ARRAY) [A array of numbers for the exercise]
    """
    return pickle.load(open(file_name))

def draw_sample(population, n):
    """INPUT:
    - population(NUMPY ARRAY) [The array containing all the numbers]
    - n(INT) [The number of sample you wanna draw]

    OUTPUT:
    - sample(NUMPY ARRAY) [A array that contains a subset of the population]

    Hint: Use np.random.choice(). Google it. Google is your best friend
    """
    pass

def get_mean(lst):
    """INPUT:
    - lst(NUMPY ARRAY) [The array of numbers where we find the mean of]

    OUTPUT:
    - mean_value(FLOAT)

    Hint: Don't use np.mean().
    Then use np.mean(arr) to see if you got the same value
    """
    pass

def get_variance(lst, sample=True):
    """INPUT:
    - lst(NUMPY ARRAY) [Either the sample or the population]
    - sample(BOOL) [True if sample variance, False if population variance]

    OUTPUT:
    - lst_variance(FLOAT) [Sample or population variance depending]
    """
    pass

def get_sem(sample):
    """INPUT:
    - sample(NUMPY ARRAY)

    OUTPUT:
    - sem(FLOAT) [Standard Error Mean]
    """
    pass

if __name__ == '__main__':
    population = load_pickle('../data/population.pkl')
    print 'First 10 element of the population: ', population[:5]


# Exercise_1
#1
import numpy as np

# Import MSE from sklearn
from sklearn.metrics import mean_squared_error

# Define your own MSE function
def own_mse(y_true, y_pred):
  	# Raise differences to the power of 2
    squares = np.power(y_true - y_pred, 2)
    # Find mean over all observations
    err = np.mean(squares)
    return err

print('Sklearn MSE: {:.5f}. '.format(mean_squared_error(y_regression_true, y_regression_pred)))
print('Your MSE: {:.5f}. '.format(own_mse(y_regression_true, y_regression_pred)))
#2
import numpy as np

# Import log_loss from sklearn
from sklearn.metrics import log_loss

# Define your own LogLoss function
def own_logloss(y_true, prob_pred):
    # Find loss for each observation
    terms = y_true * np.log(prob_pred) + (1 - y_true) * np.log(1 - prob_pred)
    # Find mean over all observations
    err = np.mean(terms)
    return -err

print('Sklearn LogLoss: {:.5f}'.format(log_loss(y_classification_true, y_classification_pred)))
print('Your LogLoss: {:.5f}'.format(own_logloss(y_classification_true, y_classification_pred)))

--------------------------------------------------
# Exercise_2
#1
# Shapes of train and test data
print('Train shape:', train.shape)
print('Test shape:', test.shape)

# Train head()
print(train.head())
#2
# Shapes of train and test data
print('Train shape:', train.shape)
print('Test shape:', test.shape)

# Train head()
print(train.head())

# Describe the target variable
print(train.fare_amount.describe())

# Train distribution of passengers within rides
print(train.passenger_count.value_counts())

--------------------------------------------------
# Exercise_3
# Calculate the ride distance
train['distance_km'] = haversine_distance(train)

# Draw a scatterplot
plt.scatter(x=train['fare_amount'], y=train['distance_km'], alpha=0.5)
plt.xlabel('Fare amount')
plt.ylabel('Distance, km')
plt.title('Fare amount based on the distance')

# Limit on the distance
plt.ylim(0, 50)
plt.show()

--------------------------------------------------
# Exercise_4
# Create hour feature

train['pickup_datetime'] = pd.to_datetime(train.pickup_datetime)

train['hour'] = train.pickup_datetime.dt.hour

# Find median fare_amount for each hour

hour_price = train.groupby('hour', as_index=False)['fare_amount'].median()

# Plot the line plot

plt.plot(hour_price['hour'], hour_price['fare_amount'], marker='o')

plt.xlabel('Hour of the day')

plt.ylabel('Median fare amount')

plt.title('Fare amount based on day time')

plt.xticks(range(24))

plt.show()

--------------------------------------------------
# Exercise_5
# Import KFold

from sklearn.model_selection import KFold # random sampling

# Create a KFold object

kf = KFold(n_splits=3, shuffle=True, random_state=123)

# Loop through each split

fold = 0

for train_index, test_index in kf.split(train):

    # Obtain training and testing folds

    cv_train, cv_test = train.iloc[train_index], train.iloc[test_index]

    print('Fold: {}'.format(fold))

    print('CV train shape: {}'.format(cv_train.shape))

    print('Medium interest listings in CV train: {}\n'.format(sum(cv_train.interest_level == 'medium')))

    fold += 1

--------------------------------------------------
# Exercise_6
# Import StratifiedKFold

from sklearn.model_selection import StratifiedKFold

# Create a StratifiedKFold object

str_kf = StratifiedKFold(n_splits=3, shuffle=True, random_state=123)

# Loop through each split

fold = 0

for train_index, test_index in str_kf.split(train, train['interest_level']):

    # Obtain training and testing folds

    cv_train, cv_test = train.iloc[train_index], train.iloc[test_index]

    print('Fold: {}'.format(fold))

    print('CV train shape: {}'.format(cv_train.shape))

    print('Medium interest listings in CV train: {}\n'.format(sum(cv_train.interest_level == 'medium')))

    fold += 1

--------------------------------------------------
# Exercise_7
# Create TimeSeriesSplit object

#Create TimeSeriesSplit object

time_kfold = TimeSeriesSplit(n_splits=3)

# Sort train data by date

train = train.sort_values('date')

# Iterate through each split

fold = 0

for train_index, test_index in time_kfold.split(train):

    cv_train, cv_test = train.iloc[train_index], train.iloc[test_index]

    print('Fold :', fold)

    print('Train date range: from {} to {}'.format(cv_train.date.min(), cv_train.date.max()))

    print('Test date range: from {} to {}\n'.format(cv_test.date.min(), cv_test.date.max()))

    fold += 1

--------------------------------------------------
# Exercise_8
#1
from sklearn.model_selection import TimeSeriesSplit
import numpy as np

# Sort train data by date
train = train.sort_values('date')

# Initialize 3-fold time cross-validation
kf = TimeSeriesSplit(n_splits=3)

# Get MSE scores for each cross-validation split
mse_scores = get_fold_mse(train, kf)

print('Mean validation MSE: {:.5f}'.format(np.mean(mse_scores)))

#2
from sklearn.model_selection import TimeSeriesSplit
import numpy as np

# Sort train data by date
train = train.sort_values('date')

# Initialize 3-fold time cross-validation
kf = TimeSeriesSplit(n_splits=3)

# Get MSE scores for each cross-validation split
mse_scores = get_fold_mse(train, kf)

print('Mean validation MSE: {:.5f}'.format(np.mean(mse_scores)))
print('MSE by fold: {}'.format(mse_scores))

#3
from sklearn.model_selection import TimeSeriesSplit
import numpy as np

# Sort train data by date
train = train.sort_values('date')

# Initialize 3-fold time cross-validation
kf = TimeSeriesSplit(n_splits=3)

# Get MSE scores for each cross-validation split
mse_scores = get_fold_mse(train, kf)

print('Mean validation MSE: {:.5f}'.format(np.mean(mse_scores)))
print('MSE by fold: {}'.format(mse_scores))
print('Overall validation MSE: {:.5f}'.format(np.mean(mse_scores) + np.std(mse_scores)))

--------------------------------------------------

Machine Learning Scientist with Python Track/23. Winning a Kaggle Competition in Python/ch2_exercises.py


import numpy as np
from scipy.special import gammaln

class MultinomialDistribution(object):

    def __init__(self, p, rso=np.random):
        """Initialize the multinomial random variable.

        Parameters
        ----------
        p: numpy array of length `k`
            The outcome probabilities
        rso: numpy RandomState object (default: np.random)
            The random number generator

        """

        # Check that the probabilities sum to 1. If they don't, then
        # something is wrong! We use `np.isclose` rather than checking
        # for exact equality because in many cases, we won't have
        # exact equality due to floating-point error.
        if not np.isclose(np.sum(p), 1.0):
            raise ValueError("outcome probabilities do not sum to 1")

        # Store the parameters that were passed in
        self.p = p
        self.rso = rso

        # Precompute log probabilities, for use by the log-PMF, for
        # each element of `self.p` (the function `np.log` operates
        # elementwise over NumPy arrays, as well as on scalars.)
        self.logp = np.log(self.p)

    def sample(self, n):
        """Samples draws of `n` events from a multinomial distribution with
        outcome probabilities `self.p`.

        Parameters
        ----------
        n: integer
            The number of total events

        Returns
        -------
        numpy array of length `k`
            The sampled number of occurrences for each outcome

        """
        x = self.rso.multinomial(n, self.p)
        return x

    def log_pmf(self, x):
        """Evaluates the log-probability mass function (log-PMF) of a
        multinomial with outcome probabilities `self.p` for a draw `x`.

        Parameters
        ----------
        x: numpy array of length `k`
            The number of occurrences of each outcome

        Returns
        -------
        The evaluated log-PMF for draw `x`

        """
        # Get the total number of events.
        n = np.sum(x)

        # equivalent to log(n!)
        log_n_factorial = gammaln(n + 1)
        # equivalent to log(x1! * ... * xk!)
        sum_log_xi_factorial = np.sum(gammaln(x + 1))

        # If one of the values of self.p is 0, then the corresponding
        # value of self.logp will be -inf. If the corresponding value
        # of x is 0, then multiplying them together will give nan, but
        # we want it to just be 0.
        log_pi_xi = self.logp * x
        log_pi_xi[x == 0] = 0
        # equivalent to log(p1^x1 * ... * pk^xk)
        sum_log_pi_xi = np.sum(log_pi_xi)

        # Put it all together.
        log_pmf = log_n_factorial - sum_log_xi_factorial + sum_log_pi_xi
        return log_pmf

    def pmf(self, x):
        """Evaluates the probability mass function (PMF) of a multinomial
        with outcome probabilities `self.p` for a draw `x`.

        Parameters
        ----------
        x: numpy array of length `k`
            The number of occurrences of each outcome

        Returns
        -------
        The evaluated PMF for draw `x`

        """
        pmf = np.exp(self.log_pmf(x))
        return pmf


#!/usr/bin/env python
# coding: utf-8

# In[1]:

import numpy as np
import matplotlib.pyplot as plt

# In[2]:

#Simpson from book p.234
def Simpson(f, a, b, epsilon, level, level_max):
    level = level + 1
    h = b - a
    c = (a + b)/2
    one_simpson = h*(f(a) + 4*f(c) + f(b))/6
    d = (a + c)/2
    e = (c + b)/2
    two_simpson = h*(f(a) + 4*f(d) + 2*f(c) + 4*f(e) + f(b))/12
    if level >= level_max:
        simpson_result = two_simpson
        print("maximum level reached")
    else:
        if np.abs(two_simpson - one_simpson) < 15*epsilon:
            simpson_result = two_simpson + (two_simpson - one_simpson)/15
        else:
            left_simpson = Simpson(f,a,c,e/2,level,level_max)
            right_simpson = Simpson(f,c,b,e/2,level,level_max)
            simpson_result = left_simpson + right_simpson
    return simpson_result

# In[3]:

#test integral
def f(x):
    return np.cos(2*x)/np.exp(x)
a = 0
b = (5/4) * np.pi
epsilon = (1/2)*(10**-3)
level = 0
level_max = 100
print(Simpson(f, a, b, epsilon, level, level_max))

# In[4]:

#Simpson with graphing
def Simpson(f, a, b, epsilon, level, level_max):
    level = level + 1
    h = b - a
    c = (a + b)/2
    one_simpson = h*(f(a) + 4*f(c) + f(b))/6
    d = (a + c)/2
    e = (c + b)/2
    two_simpson = h*(f(a) + 4*f(d) + 2*f(c) + 4*f(e) + f(b))/12
    XP.append(a)
    XP.append(b)
    XP.append(c)
    XP.append(d)
    XP.append(e)
    if level >= level_max:
        simpson_result = two_simpson
        print("maximum level reached")
    else:
        if np.abs(two_simpson - one_simpson) < 15*epsilon:
            simpson_result = two_simpson + (two_simpson - one_simpson)/15
        else:
            left_simpson = Simpson(f,a,c,epsilon/2,level,level_max)
            right_simpson = Simpson(f,c,b,epsilon/2,level,level_max)
            simpson_result = left_simpson + right_simpson
    return simpson_result

# In[12]:

#test integral
def f(x):
    return np.cos(2*x)/np.exp(x)
a = 0
b = (5/4) * np.pi
epsilon = (1/2)*(10**-3)
level = 0
level_max = 5
XP = []
I = Simpson(f, a, b, epsilon, level, level_max)
XP = list(set(XP))
XP.sort()
PP = list(map(f,XP))
print(I)
rounded_XP = [round(num, 3) for num in XP]
rounded_PP = [round(num, 3) for num in PP]
print("x-values used by Simpson approximation:")
print(rounded_XP)
print("Corresponding f(x)-values used by Simpson approximation:")
print(rounded_PP)

# In[13]:

X = np.arange(0, 4, 0.01);
FVAL = np.cos(2*X)/np.exp(X)

fig = plt.figure(figsize=(10, 6))
ax1 = fig.add_subplot(111)

ax1.plot(X,FVAL, label = 'cos(2x)/e^x')
ax1.scatter(XP,PP)

# Give a title for the sine wave plot
plt.title('Arctan vs Polynomial Interpretation Plot')

# Give x axis label for the sine wave plot
plt.xlabel('x')

# Give y axis label for the sine wave plot
plt.legend(loc='upper left')
plt.grid(True, which='both')
plt.axhline(y=0)
plt.stem(XP,PP)
plt.show()

# In[ ]:

# In[ ]:


# -*- coding: utf-8 -*-
# -----------------------------------
# @CreateTime   : 2020/3/20 0:15
# @Author       : Mark Shawn
# @Email        : shawninjuly@gmail.com
# ------------------------------------
import numpy as np

EPS = 1e-8

def sigmoid(X):
	return 1 / (1 + np.exp(-X))

def mse(Y_pred, Y_target):
	return ((Y_pred - Y_target) ** 2).mean()

def cross_entropy(Y_pred, Y_target):
	return -(Y_target * np.log(Y_pred+EPS) + (1-Y_target) * np.log(1-Y_pred+EPS)).mean()


from typing import Tuple, Union, List
import pandas as pd
import numpy as np
import logging

logger = logging.getLogger('STATS UTILS')

def compute_interquartile_range(values: str) -> Tuple[float, float]:
    """
    Compute interquartile range for a list of values

    :param values: list of values
    :return: lower and upper bound of interquartile range
    """

    # Calculate IQR
    q1, q3 = np.percentile(values, [25, 75])
    iqr = q3 - q1

    lower_bound = q1 - (1.5 * iqr)
    upper_bound = q3 + (1.5 * iqr)

    return lower_bound, upper_bound

def check_outliers(lower_bound: float, upper_bound: float, current_value: Union[int, float]) -> bool:
    """
    Check if a value is an outliers by comparing it with interquartile range

    :param lower_bound: lower bound of interquartile range
    :param upper_bound: upper bound of interquartile range
    :param current_value: value to compare whether or not it is an outlier
    :return: whether or not it is an outlier
    """

    if lower_bound <= current_value <= upper_bound:
        outlier = False
    else:
        outlier = True

    return outlier

def discretize_data(array: np.ndarray, bin_type: str, q: Union[int, List[Union[int, float]]] = None,
                    bins: Union[int, List[Union[int, float]]] = None, labels: Union[list, np.ndarray] = None,
                    **kwargs) -> pd.Series:
    """
    Discretize variable into buckets based on quantiles or fixed intervals
    For more information about the parameters, please visit :
    - https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.cut.html
    - https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.qcut.html

    :param array: input array to be binned, must be 1-dimensional
    :param bin_type: binning method e.g. quantile or fixed_intervals
    :param q: number of quantiles, just needed if bin_type is quantile
    :param bins: defines the bin edges allowing for non-uniform width, just needed if bin_type is fixed_intervals
    :param labels: used as labels for the resulting bins
    :param kwargs: additional arguments for qcut and cut pandas function
    :return: an array-like object representing the respective bin for each value of x
    """

    if bin_type == 'quantile':

        if q is None:
            raise ValueError("'q' can't be None if bin_type is equal to quantile")

        if labels is None:
            labels = np.arange(q, 0, -1)

        try:
            binned_data = pd.qcut(array, q=q, labels=labels, **kwargs)
        except Exception:
            logger.error("Can't create quantile discrete intervals", exc_info=True)
            raise

    elif bin_type == 'fixed_intervals':

        if bins is None:
            raise ValueError("'bins' can't be None if bin_type is equal to quantile")

        try:
            binned_data = pd.cut(array, bins=bins, labels=labels, **kwargs)
        except Exception:
            logger.error("Can't create fixed discrete intervals", exc_info=True)
            raise

    else:

        logger.error("{} bin type is not supported yet, please contact developer".format(bin_type))
        raise NotImplementedError('Bin types supporting for now: quantile and fixed intervals')

    return binned_data


"""
# T: maturity
# n: # option periods
# N: # futures periods
# S: initial stock price
# r: continuously-compounded interest rate
# c: dividend yield
# sigma: annualized volatility
# K: strike price
# cp: +1/-1 with regards to call/put
"""

from __future__ import division
from math import exp, sqrt
import numpy as np
import math

T = 0.25
n = 15 # option periods
N = 15 # futures periods
S = 100 #initial stock price
r = 0.02 #continuously-compounded interest rate
c = 0.01 #dividend yield
sigma = 0.3 #annualized volatility
K = 110 #strike price
cp = -1 #with regards to call/put

def Parameter(T,n,sigma,r,c):

    """Parameter calculation"""

    dt = T/n
    u = exp(sigma * sqrt(dt))
    d = 1/u

    q1 = (exp((r-c)*dt)-d)/(u-d)
    q2 = 1-q1
    R = exp(r*dt)

    return (u, d, q1, q2, R)

# =============================================================================

def GenerateTree(T,n,S,sigma,r,c):
    """generate stock tree"""
    u, d, q1, q2, R = Parameter(T,n,sigma,r,c)

    stockTree = np.zeros((n+1, n+1))

    # compute the stock tree
    stockTree[0,0] = S
    for i in range(1,n+1):
        stockTree[0,i] = stockTree[0, i-1]*u
        for j in range(1,n+1):
            stockTree[j,i] = stockTree[j-1, i-1]*d

    return stockTree

# =============================================================================

def StockOptionAM(T,n,S,r,c,sigma,K,cp):
    """first return: American Stock Option Pricing"""
    """second return: when is the earliest time to exercise"""
    """Though it's never optimal to early exercise AM call"""
    """It matters for AM put"""

    u, d, q1, q2, R = Parameter(T,n,sigma,r,c)

    stockTree = GenerateTree(T,n,S,sigma,r,c)
    optionTree = np.zeros((n+1,n+1))

    # compute the option tree
    for j in range(n+1):
        optionTree[j, n] = max(0, cp * (stockTree[j, n]-K))

    flag = 0
    list = []
    for i in range(n-1,-1,-1):
        for j in range(i+1):
            optionTree[j, i] = max((q1 * optionTree[j, i+1] + q2 * optionTree[j+1, i+1])/R,
                               cp * (stockTree[j, i] - K))
            if (optionTree[j, i] - cp * (stockTree[j, i] - K)) < 1e-10:
                flag += 1
                list.append(i)

    when = n
    if(flag):  when = list[-1]

    print(optionTree, when)
    return (optionTree[0,0], when)

z = StockOptionAM(T,n,S,r,c,sigma,K,cp)

option_maturity = 10

class bs_bin_tree:

    def __init__(self,T,s0,r,sigma,c,K,n):
        self.T = T
        self.r = r
        self.c = c
        self.sigma = sigma
        self.K = K
        self.s0 = s0
        self.n = n
        self.u = math.exp(self.sigma*np.sqrt(self.T/self.n))
        self.q = (math.exp((self.r-self.c)*T/self.n)-(1/self.u))/(self.u-(1/self.u))
        self.R = math.exp(self.r*self.T/self.n)
        self.__print_param__()

    def __print_param__(self):
        print('Time',self.T)
        print('Starting Price',self.s0)
        print('r',self.r)
        print('volatility',self.sigma)
        print('dividend yield',self.c)
        print('strike',self.K)
        print('# period',self.n)

    def generate_price(self):
        arr=[[self.s0]]
        for i in range(self.n):
            arr_to_add=[]
            for j in range(len(arr[i])):
                arr_to_add.append(arr[i][j]/self.u)
                if j == (len(arr[i])-1):
                    arr_to_add.append(arr[i][j]*self.u)
            arr.append(arr_to_add)
        return arr

    def neutral_pricing(self,p1,p2):
        price = ((1-self.q)*p1 + (self.q)*p2)/self.R
        return price

    def eu_put(self):
        arr = self.generate_price()
        arr_rev = arr[::-1]
        res=[]
        for i in range(len(arr_rev)):
            res_to_add = []
            for j in range(len(arr_rev[i])):
                if i == 0:
                    a = max(self.K-arr_rev[i][j],0)
                    res_to_add.append(a)
                else:
                    price = self.neutral_pricing(res[i-1][j], res[i-1][j+1])
                    #a = max(arr_rev[i][j]-strike,0)
                    #a = max(a,price)
                    a = price
                    res_to_add.append(a)

            res.append(res_to_add)
        return res[::-1]

    def eu_call(self):
        arr = self.generate_price()
        arr_rev = arr[::-1]
        res=[]
        for i in range(len(arr_rev)):
            res_to_add = []
            for j in range(len(arr_rev[i])):
                if i == 0:
                    a = max(arr_rev[i][j]-self.K,0)
                    res_to_add.append(a)
                else:
                    price = self.neutral_pricing(res[i-1][j], res[i-1][j+1])
                    #a = max(arr_rev[i][j]-strike,0)
                    #a = max(a,price)
                    a = price
                    res_to_add.append(a)

            res.append(res_to_add)
        return res[::-1]

    def us_call(self):
        arr = self.generate_price()
        arr_rev = arr[::-1]
        res=[]
        for i in range(len(arr_rev)):
            res_to_add = []
            for j in range(len(arr_rev[i])):
                if i == 0:
                    a = max(arr_rev[i][j]-self.K,0)
                    res_to_add.append(a)
                else:
                    price = self.neutral_pricing(res[i-1][j], res[i-1][j+1])
                    a1 = max(arr_rev[i][j]-self.K,0)
                    a = max(a1,price)
                    res_to_add.append(a)

            res.append(res_to_add)
        return res[::-1]

    def us_call_price(self):
        return self.us_call()[0][0]

    def us_put(self):
        arr = self.generate_price()
        arr_rev = arr[::-1]
        res=[]
        for i in range(len(arr_rev)):
            res_to_add = []
            for j in range(len(arr_rev[i])):
                if i == 0:
                    a = max(self.K-arr_rev[i][j],0)
                    res_to_add.append(a)
                else:
                    price = self.neutral_pricing(res[i-1][j], res[i-1][j+1])
                    a1 = max(self.K - arr_rev[i][j],0)
                    a = max(a1,price)
                    res_to_add.append(a)

            res.append(res_to_add)
        return res[::-1]

    def us_put_price(self):
        return self.us_put()[0][0]

    def us_put_early_ex(self):
        early_ex = False
        early_ex_earning = 0
        early_ex_time = self.n
        arr = self.generate_price()
        arr_rev = arr[::-1]
        res=[]
        for i in range(len(arr_rev)):
            res_to_add = []
            for j in range(len(arr_rev[i])):
                if i == 0:
                    a = max(self.K-arr_rev[i][j],0)
                    res_to_add.append(a)
                else:
                    price = self.neutral_pricing(res[i-1][j], res[i-1][j+1])
                    a1 = max(self.K-arr_rev[i][j],0)
                    if a1 > price:
                        if early_ex_time == self.n - i:
                            early_ex_earning = max(early_ex_earning,a1)
                        else:
                            early_ex_earning = a1
                        early_ex =True
                        early_ex_time = self.n - i

                    a = max(a1,price)
                    res_to_add.append(a)

            res.append(res_to_add)
        return {early_ex_time:early_ex_earning} if early_ex == True else False

    def us_put_call_parity(self):
        LHS = self.us_put_price() + self.s0 * math.exp(-self.c * self.T)
        RHS = self.us_call_price() + self.K * math.exp(-self.r * self.T)
        print('Put Side',LHS)
        print('Call Side',RHS)
        return LHS==RHS

    def generate_future_price(self):
        arr = self.generate_price()
        arr_rev = arr[::-1]
        res=[]
        for i in range(len(arr_rev)):
            res_to_add = []
            for j in range(len(arr_rev[i])):
                if i == 0:
                    res_to_add.append(arr_rev[i][j])
                else:
                    price = self.neutral_pricing(res[i-1][j], res[i-1][j+1])*self.R
                    res_to_add.append(price)

            res.append(res_to_add)
        return res[::-1]

    def option_on_future(self,option_maturity):
        arr = self.generate_future_price()[0:option_maturity+1]
        arr_rev = arr[::-1]
        res=[]
        for i in range(option_maturity+1):
            res_to_add = []
            for j in range(len(arr_rev[i])):
                if i == 0:
                    a = max(arr_rev[i][j]-self.K,0)
                    res_to_add.append(a)
                else:
                    price = self.neutral_pricing(res[i-1][j], res[i-1][j+1])
                    a1 = max(arr_rev[i][j]-self.K,0)
                    a = max(a1,price)
                    res_to_add.append(a)

            res.append(res_to_add)
        return res[::-1]

    def option_price_on_future(self,option_maturity):
        return self.option_on_future(option_maturity)[0][0]

    def option_on_future_early_ex(self,option_maturity):
        arr = self.generate_future_price()[0:option_maturity+1]
        arr_rev = arr[::-1]
        res=[]
        early_ex = False
        early_ex_earning = 0
        early_ex_time = self.n
        for i in range(option_maturity+1):
            res_to_add = []
            for j in range(len(arr_rev[i])):
                if i == 0:
                    a = max(arr_rev[i][j]-self.K,0)
                    res_to_add.append(a)
                else:
                    price = self.neutral_pricing(res[i-1][j], res[i-1][j+1])
                    a1 = max(arr_rev[i][j]-self.K,0)
                    if a1 > price:
                        if early_ex_time == option_maturity - i:
                            early_ex_earning = max(early_ex_earning,a1)
                        else:
                            early_ex_earning = a1
                        early_ex =True
                        early_ex_time =  len(arr_rev) - i -1

                    a = max(a1,price)
                    res_to_add.append(a)

            res.append(res_to_add)
        return {early_ex_time:early_ex_earning} if early_ex == True else False

    def nCr(self,n,r):
        f = math.factorial
        return f(n) / f(r) / f(n-r)

    def chooser_option_price(self,option_expire):
        call = self.eu_call()[option_expire]
        put = self.eu_put()[option_expire]
        res=[]
        for i in range(len(call)):
            res.append(max(call[i],put[i]))
        result=0
        for j in range(0,len(res)):
            result += self.nCr(option_expire,j)* (self.q**(j)) * (1-self.q)**(option_expire-j) * res[j]
        return (result/self.R**(option_expire))

tree = bs_bin_tree(T, 100, r, sigma, c, K, n)
print(tree.us_call())
print(tree.us_call_price())
print(tree.us_put())
print(tree.us_put_price())
print(tree.us_put_early_ex())
print(tree.us_put_call_parity())
print(tree.option_on_future(option_maturity))
print(tree.option_price_on_future(option_maturity))
print(tree.option_on_future_early_ex(option_maturity))
print(tree.chooser_option_price(10))


# -*- coding: utf-8 -*-
"""
Newton fractal for the equation

    x**3 + 1

Currently, it is quite slow since it is using Python loops instead
of vectorization of the operations.

@author: Nicolás Guarín-Zapata
"""

import numpy as np
import matplotlib.pyplot as plt

def newt(x, fun, der, tol=1e-5, niter=100):
    """ Find a root using Newton Method"""
    x0 = x
    for k in range(niter):
        x = x0 - fun(x0)/der(x0)
        err = np.abs((x - x0)/x)
        if err < tol:
            break
        if k == niter - 1:
            x = 1000
        x0 = x
    return x

def img_newt(N, xran=(-3, 3), yran=(-3, 3), tol=1e-5, niter=100):
    """
    Add colors to a matrix according to the fixed point
    of the given equation.
    """
    sol = [-(np.sqrt(3.0)*1j - 1.0)/2.0,
           (np.sqrt(3.0)*1j + 1.0)/2.0,
          -1.0]
    col_newt = np.zeros((N, N, 3))
    Y, X = np.mgrid[yran[0]:yran[1]:N*1j,
                    xran[0]:xran[1]:N*1j]
    for row in range(N):
        for col in range(N):
            x = X[row, col]
            y = Y[row, col]
            xf = newt(x + y*1j, fun, der, tol=tol, niter=niter)
            if abs(xf - sol[0])<1e-6:
                col_newt[row, col, :] = colors[0]
            if abs(xf - sol[1])<1e-6:
                col_newt[row, col, :] = colors[1]
            if abs(xf - sol[2])<1e-6:
                col_newt[row, col, :] = colors[2]
            if abs(xf - 1000) < 1e-6:
                col_newt[row, col, :] = colors[3]
    return col_newt

#%% Computation
colors = [[0.0431, 0.4078, 0.6588],
          [0.1176, 0.6431, 0.2314],
          [0.6745, 0.1216, 0.2431],
          [0.2, 0.2, 0.2]]
fun = lambda x: x*x*x + 1.0
der = lambda x: 3.0*x*x
col_newt = img_newt(2000, tol=1e-10, niter=1000)

#%% Visualization
plt.figure(figsize=(4,4))
plt.imshow(col_newt, extent=(-3, 3, -3, 3), origin='lower')
plt.axis('off')
plt.savefig('newton_fractal.png', dpi=500, transparent=True,
            bbox_inches='tight', pad_inches=0)
plt.show()


# metrics

import numpy as np
import pandas as pd

# metrics

def mae(targets, predictions) -> float:
    """
    Calculo de la metrica: mean absolute error (MAE)
    :param targets: valor real
    :param predictions: valor estimado
    :return: valor de la metrica
    """
    error = predictions - targets
    return round(np.abs(error).mean(), 4)

def mse(targets, predictions) -> float:
    """
    Calculo de la metrica: mean squared error (MSE)
    :param targets: valor real
    :param predictions: valor estimado
    :return: valor de la metrica
    """
    error = predictions - targets
    return round((error ** 2).mean(), 4)

def rmse(targets, predictions) -> float:
    """
    Calculo de la metrica: root mean squared error (RMSE)
    :param targets: valor real
    :param predictions: valor estimado
    :return: valor de la metrica
    """
    error = predictions - targets
    return round(np.sqrt((error ** 2).mean()), 4)

# b) Percentage errors

def mape(targets, predictions) -> float:
    """
    Calculo de la metrica: mean absolute percentage error (MAPE)
    :param targets: valor real
    :param predictions: valor estimado
    :return: valor de la metrica
    """
    error = predictions - targets

    if any(x == 0 for x in targets):
        return np.inf
    else:
        return round(np.abs(error / targets).mean(),4)

def maape(targets, predictions) -> float:
    """
    Calculo de la metrica:  mean arctangent percentage error  (MAAPE)
    :param targets: valor real
    :param predictions: valor estimado
    :return: valor de la metrica
    """

    error = predictions - targets

    if any((x, y) == (0, 0) for x, y in zip(predictions, targets)):
        return np.inf

    else:
        return round(np.arctan(np.abs(error / targets)).mean(),4)

def wmape(targets, predictions) -> float:
    """
    Calculo de la metrica:  weighted mean absolute percentage error (WMAPE)
    :param targets: valor real
    :param predictions: valor estimado
    :return: valor de la metrica
    """
    error = predictions - targets
    sum_values = np.sum(targets)

    if sum_values == 0:
        return np.inf
    else:
        return round(np.abs(error).sum() / sum_values,4)

def mmape(targets, predictions) -> float:
    """
    Calculo de la metrica: modified mean absolute percentage error (MMAPE)
    :param targets: valor real
    :param predictions: valor estimado
    :return: valor de la metrica
    """
    error = np.abs(predictions - targets)
    denom = 1 + np.abs(targets)

    return round(np.mean(error / denom),4)

def smape(targets, predictions) -> float:
    """
    Calculo de la metrica: symmetric mean absolute percentage error (SMAPE)
    :param targets: valor real
    :param predictions: valor estimado
    :return: valor de la metrica
    """
    error = predictions - targets
    sum_values = np.abs(predictions) + np.abs(targets)

    if any(x == 0 for x in sum_values):
        return np.inf

    else:
        return round(2 * np.mean(np.abs(error) / sum_values),4)

def summary_metrics(df) -> pd.DataFrame:
    """
    Aplicar las distintas metricas definidas
    :param df:  dataframe con las columnas: ['y','yhat']
    :return: dataframe con las metricas especificadas
    """
    df_result = pd.DataFrame()

    y_true = df['y']
    y_pred = df['yhat']

    df_result['mae'] = [round(mae(y_true, y_pred), 4)]
    df_result['mse'] = [round(mse(y_true, y_pred), 4)]
    df_result['rmse'] = [round(rmse(y_true, y_pred), 4)]

    df_result['mape'] = [round(mape(y_true, y_pred), 4)]
    df_result['maape'] = [round(maape(y_true, y_pred), 4)]
    df_result['wmape'] = [round(wmape(y_true, y_pred), 4)]
    df_result['mmape'] = [round(mmape(y_true, y_pred), 4)]
    df_result['smape'] = [round(smape(y_true, y_pred), 4)]

    # nombre del modelo por defecto
    #try:
    #    model = df['model'][0]
    #    df_result['model'] = [model]
    #except:
    #    df_result['model'] = ['---']

    return df_result


import numpy as np
from source.week2.ex2_1 import construct_element_table, xy
from source.week2.ex2_3 import assembly
from source.week2.ex2_4 import boundary_conditions

def u_hat(X, Y, L1, L2, x0, y0, M, A, b, test_case=None,exercise=None):
    # u_hat --> Au=b
    A, b = boundary_conditions(X, Y, L1=L1, L2=L2, x0=x0, y0=y0, M=M, A=A, b=b, test_case=test_case, exercise=exercise)
    uhat = np.linalg.solve(A, b)
    return uhat

def u(x, y, test_case=None):
    if test_case == 1:
        return x ** 3 - x ** 2 * y + y ** 2 - 1
    if test_case == 2:
        return x ** 2 * y ** 2

def construct_qt(etov_dict, VX, VY, test_case=None):
    """
    Computes points as means of each elements' vertices.
    """
    qt_dict = dict()
    for e, (v1, v2, v3) in etov_dict.items():
        if test_case == 1:
            x1, x2, x3 = VX[v1 - 1], VX[v2 - 1], VX[v3 - 1]
            y1, y2, y3 = VY[v1 - 1], VY[v2 - 1], VY[v3 - 1]

            def f(x, y):
                return -6 * x + 2 * y - 2

            q1 = f(x1, y1)
            q2 = f(x2, y2)
            q3 = f(x3, y3)

            q_avg = np.mean([q1, q2, q3])
            qt_dict[e] = q_avg
        elif test_case == 2:
            x1, x2, x3 = VX[v1 - 1], VX[v2 - 1], VX[v3 - 1]
            y1, y2, y3 = VY[v1 - 1], VY[v2 - 1], VY[v3 - 1]

            def f(x, y):
                return -2 * y ** 2 - 2 * x ** 2

            q1 = f(x1, y1)
            q2 = f(x2, y2)
            q3 = f(x3, y3)

            q_avg = np.mean([q1, q2, q3])
            qt_dict[e] = q_avg
        else:
            raise Exception("Unknown test case")
    return qt_dict

def test_case_data(nr_of_test_case):
    if nr_of_test_case == 1:
        L1 = 7.6
        L2 = 5.9
        x0 = -2.5
        y0 = -4.8
        X, Y = xy(x0=x0, y0=y0, L1=L1, L2=L2, noelms1=4, noelms2=3)
        etov_dict, M = construct_element_table(4, 3)
        lam1 = 1
        lam2 = 1
        qt = construct_qt(etov_dict, X, Y, test_case=1)
        return nr_of_test_case, X, Y, L1, L2, x0, y0, etov_dict, M, lam1, lam2, qt

    elif nr_of_test_case == 2:
        L1 = 7.6
        L2 = 5.9
        x0 = -2.5
        y0 = -4.8
        p = 3
        noelms = 2 ** p
        X, Y = xy(x0=x0, y0=y0, L1=L1, L2=L2, noelms1=noelms, noelms2=noelms)
        etov_dict, M = construct_element_table(noelms, noelms)
        lam1 = 1
        lam2 = 1
        qt = construct_qt(etov_dict, X, Y, test_case=2)
        return nr_of_test_case, X, Y, L1, L2, x0, y0, etov_dict, M, lam1, lam2, qt
    else:
        raise Exception("Unknown test case")

def max_error(u, u_hat, M):
    errors = []
    for i in range(M):
        errors.append(abs(u[i] - u_hat[i]))
    return max(errors)

def main():
    ntest_case = 1
    nr_of_test_case, X, Y, L1, L2, x0, y0, etov_dict, M, lam1, lam2, qt = test_case_data(ntest_case)

    A, b = assembly(X, Y, etov_dict, lam1=lam1, lam2=lam2, qt=qt, M=M)
    uhat = u_hat(X, Y, L1, L2, x0, y0, M, A, b, ntest_case,"2_5")
    print("test case ",ntest_case)
    print("u_hat(x,y): ", uhat)
    u1 = u(X, Y, ntest_case)
    print("u(x,y): ", u1)
    error = max_error(u1, uhat, M)
    print("Max error: ",error)

if __name__ == '__main__':
    main()


import pylab as pl
import numpy as np
from matplotlib import cm

color_map=[cm.winter,cm.autumn]

D = 2

# Different Mean Vectors
M1 = np.array([0.0, 2.0])
M2 = np.array([3.0, 1.0])
M3 = np.array([1.0, 0.0])

# Same Covariance Matrix
C = np.array([
    [1.0, 0.0],
    [0.0, 1/3]
])

X, Y = np.mgrid[-2:2:100j, -2:2:100j]
points = np.c_[X.ravel(), Y.ravel()]

invC = np.linalg.inv(C)

v = points - M1
g1 = -0.5*np.sum(np.dot(v, invC) * v, axis=1) - D*0.5*np.log(2*np.pi) - 0.5*np.log(np.linalg.det(C))
g1.shape = 100, 100

v = points - M2
g2 = -0.5*np.sum(np.dot(v, invC) * v, axis=1) - D*0.5*np.log(2*np.pi) - 0.5*np.log(np.linalg.det(C))
g2.shape = 100, 100

v = points - M3
g3 = -0.5*np.sum(np.dot(v, invC) * v, axis=1) - D*0.5*np.log(2*np.pi) - 0.5*np.log(np.linalg.det(C))
g3.shape = 100, 100

# Display the distributions plots
fig, axes = pl.subplots(1, 3, dpi=60, figsize=(15, 5))
ax1, ax2, ax3 = axes.ravel()
for ax in axes.ravel():
    ax.set_aspect("equal")

ax1.pcolormesh(X, Y, g1, cmap=color_map[0])
ax1.set_title("W1")
ax2.pcolormesh(X, Y, g2, cmap=color_map[0])
ax2.set_title("W2")
ax3.pcolormesh(X, Y, g3, cmap=color_map[0])
ax3.set_title("W3")

# Display the discriminators plots
fig2, axes2 = pl.subplots(1, 3, dpi=60, figsize=(15, 5))
ax4, ax5, ax6 = axes2.ravel()

ax4.pcolormesh(X, Y, g1 > g2, cmap=color_map[1])
ax4.set_title("W1 > W2")
ax5.pcolormesh(X, Y, g2 > g3, cmap=color_map[1])
ax5.set_title("W2 > W3")
ax6.pcolormesh(X, Y, g1 > g3, cmap=color_map[1])
ax6.set_title("W1 > W3")

pl.show()


from scipy import stats
from numpy.random import default_rng
rng = default_rng(12345)

current = rng.normal(4.0, 2.0, size=40)
process_a = rng.normal(6.2, 2.0, size=25)
process_b = rng.normal(4.5, 2.0, size=64)

significance = 0.05

F_stat, p_value = stats.f_oneway(current, process_a, process_b)

print(f"F stat: {F_stat}, p value: {p_value}")
# F stat: 9.949052026027028, p value: 9.732322721019206e-05

if p_value <= significance:
    print("Reject H0: there is a difference between means")
else:
    print("Accept H0: all means equal")
# Reject H0: there is a difference between means


from math import exp, sqrt, pi
from numpy import *
from matplotlib.pyplot import *

# Parameters
mu = 30
sigma = 10

# Plotting the PDF
def pdf(x):
    return (1 / (sigma * sqrt(2*pi)) ) * exp(-(x - mu)**2 / (2 * sigma**2))

X = arange(-100, 100, 0.1)
Y = []

for x in X:
    Y.append(pdf(x))

matplotlib.rc('xtick', labelsize=18)
matplotlib.rc('ytick', labelsize=18)
plot(X, Y, Linewidth=2, color='black')
xlabel('$X$', size=22)
ylabel('$f_X(x)$', size=22)
#show()
savefig('normal_pdf.pdf', format='pdf', bbox_inches='tight')


import os
""" First change the following directory link to where all input files do exist """
os.chdir("D:\\Book writing\\Codes\\Chapter 8")

# K-means clustering
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from scipy.spatial.distance import cdist, pdist

from sklearn.cluster import KMeans
from sklearn.metrics import silhouette_score

iris = pd.read_csv("iris.csv")
print (iris.head())

x_iris = iris.drop(['class'],axis=1)
y_iris = iris["class"]

k_means_fit = KMeans(n_clusters=3,max_iter=300)
k_means_fit.fit(x_iris)

print ("\nK-Means Clustering - Confusion Matrix\n\n",pd.crosstab(y_iris,k_means_fit.labels_,rownames = ["Actuall"],colnames = ["Predicted"]) )
print ("\nSilhouette-score: %0.3f" % silhouette_score(x_iris, k_means_fit.labels_, metric='euclidean'))

for k in range(2,10):
    k_means_fitk = KMeans(n_clusters=k,max_iter=300)
    k_means_fitk.fit(x_iris)
    print ("For K value",k,",Silhouette-score: %0.3f" % silhouette_score(x_iris, k_means_fitk.labels_, metric='euclidean'))

# Avg. within-cluster sum of squares
K = range(1,10)

KM = [KMeans(n_clusters=k).fit(x_iris) for k in K]
centroids = [k.cluster_centers_ for k in KM]

D_k = [cdist(x_iris, centrds, 'euclidean') for centrds in centroids]

cIdx = [np.argmin(D,axis=1) for D in D_k]
dist = [np.min(D,axis=1) for D in D_k]
avgWithinSS = [sum(d)/x_iris.shape[0] for d in dist]

# Total with-in sum of square
wcss = [sum(d**2) for d in dist]
tss = sum(pdist(x_iris)**2)/x_iris.shape[0]
bss = tss-wcss

# elbow curve - Avg. within-cluster sum of squares
fig = plt.figure()
ax = fig.add_subplot(111)
ax.plot(K, avgWithinSS, 'b*-')
plt.grid(True)
plt.xlabel('Number of clusters')
plt.ylabel('Average within-cluster sum of squares')
#plt.title('Elbow for KMeans clustering')

# elbow curve - percentage of variance explained
fig = plt.figure()
ax = fig.add_subplot(111)
ax.plot(K, bss/tss*100, 'b*-')
plt.grid(True)
plt.xlabel('Number of clusters')
plt.ylabel('Percentage of variance explained')
#plt.title('Elbow for KMeans clustering')

# Calculation of eigenvectors & eigenvalues
import numpy as np
w,v = np.linalg.eig(np.array([[ 0.91335 ,0.75969 ],[ 0.75969,0.69702]]))
print ("\nEigen Values\n",w)
print ("\nEigen Vectors\n",v)

# PCA - Principal Component Analysis
import matplotlib.pyplot as plt
from sklearn.decomposition import PCA
from sklearn.datasets import load_digits

digits = load_digits()
X = digits.data
y = digits.target

print (digits.data[0].reshape(8,8))

plt.matshow(digits.images[0])
plt.show()

from sklearn.preprocessing import scale
X_scale = scale(X,axis=0)

pca = PCA(n_components=2)
reduced_X = pca.fit_transform(X_scale)

zero_x, zero_y = [],[] ; one_x, one_y = [],[]
two_x,two_y = [],[]; three_x, three_y = [],[]
four_x,four_y = [],[]; five_x,five_y = [],[]
six_x,six_y = [],[]; seven_x,seven_y = [],[]
eight_x,eight_y = [],[]; nine_x,nine_y = [],[]

for i in range(len(reduced_X)):
    if y[i] == 0:
        zero_x.append(reduced_X[i][0])
        zero_y.append(reduced_X[i][1])

    elif y[i] == 1:
        one_x.append(reduced_X[i][0])
        one_y.append(reduced_X[i][1])

    elif y[i] == 2:
        two_x.append(reduced_X[i][0])
        two_y.append(reduced_X[i][1])

    elif y[i] == 3:
        three_x.append(reduced_X[i][0])
        three_y.append(reduced_X[i][1])

    elif y[i] == 4:
        four_x.append(reduced_X[i][0])
        four_y.append(reduced_X[i][1])

    elif y[i] == 5:
        five_x.append(reduced_X[i][0])
        five_y.append(reduced_X[i][1])

    elif y[i] == 6:
        six_x.append(reduced_X[i][0])
        six_y.append(reduced_X[i][1])

    elif y[i] == 7:
        seven_x.append(reduced_X[i][0])
        seven_y.append(reduced_X[i][1])

    elif y[i] == 8:
        eight_x.append(reduced_X[i][0])
        eight_y.append(reduced_X[i][1])

    elif y[i] == 9:
        nine_x.append(reduced_X[i][0])
        nine_y.append(reduced_X[i][1])

zero = plt.scatter(zero_x, zero_y, c='r', marker='x',label='zero')
one = plt.scatter(one_x, one_y, c='g', marker='+')
two = plt.scatter(two_x, two_y, c='b', marker='s')

three = plt.scatter(three_x, three_y, c='m', marker='*')
four = plt.scatter(four_x, four_y, c='c', marker='h')
five = plt.scatter(five_x, five_y, c='r', marker='D')

six = plt.scatter(six_x, six_y, c='y', marker='8')
seven = plt.scatter(seven_x, seven_y, c='k', marker='*')
eight = plt.scatter(eight_x, eight_y, c='r', marker='x')

nine = plt.scatter(nine_x, nine_y, c='b', marker='D')

plt.legend((zero,one,two,three,four,five,six,seven,eight,nine),
           ('zero','one','two','three','four','five','six','seven','eight','nine'),
           scatterpoints=1,
           loc='lower left',
           ncol=3,
           fontsize=10)

plt.xlabel('PC 1')
plt.ylabel('PC 2')

plt.show()

# 3-Dimensional data
pca_3d = PCA(n_components=3)
reduced_X3D = pca_3d.fit_transform(X_scale)

print (pca_3d.explained_variance_ratio_)

zero_x, zero_y,zero_z = [],[],[] ; one_x, one_y,one_z = [],[],[]
two_x,two_y,two_z = [],[],[]; three_x, three_y,three_z = [],[],[]
four_x,four_y,four_z = [],[],[]; five_x,five_y,five_z = [],[],[]
six_x,six_y,six_z = [],[],[]; seven_x,seven_y,seven_z = [],[],[]
eight_x,eight_y,eight_z = [],[],[]; nine_x,nine_y,nine_z = [],[],[]

for i in range(len(reduced_X3D)):

    if y[i]==10:
        continue

    elif y[i] == 0:
        zero_x.append(reduced_X3D[i][0])
        zero_y.append(reduced_X3D[i][1])
        zero_z.append(reduced_X3D[i][2])

    elif y[i] == 1:
        one_x.append(reduced_X3D[i][0])
        one_y.append(reduced_X3D[i][1])
        one_z.append(reduced_X3D[i][2])

    elif y[i] == 2:
        two_x.append(reduced_X3D[i][0])
        two_y.append(reduced_X3D[i][1])
        two_z.append(reduced_X3D[i][2])

    elif y[i] == 3:
        three_x.append(reduced_X3D[i][0])
        three_y.append(reduced_X3D[i][1])
        three_z.append(reduced_X3D[i][2])

    elif y[i] == 4:
        four_x.append(reduced_X3D[i][0])
        four_y.append(reduced_X3D[i][1])
        four_z.append(reduced_X3D[i][2])

    elif y[i] == 5:
        five_x.append(reduced_X3D[i][0])
        five_y.append(reduced_X3D[i][1])
        five_z.append(reduced_X3D[i][2])

    elif y[i] == 6:
        six_x.append(reduced_X3D[i][0])
        six_y.append(reduced_X3D[i][1])
        six_z.append(reduced_X3D[i][2])

    elif y[i] == 7:
        seven_x.append(reduced_X3D[i][0])
        seven_y.append(reduced_X3D[i][1])
        seven_z.append(reduced_X3D[i][2])

    elif y[i] == 8:
        eight_x.append(reduced_X3D[i][0])
        eight_y.append(reduced_X3D[i][1])
        eight_z.append(reduced_X3D[i][2])

    elif y[i] == 9:
        nine_x.append(reduced_X3D[i][0])
        nine_y.append(reduced_X3D[i][1])
        nine_z.append(reduced_X3D[i][2])

# 3- Dimensional plot
from mpl_toolkits.mplot3d import Axes3D
fig = plt.figure()
ax = fig.add_subplot(111, projection='3d')

ax.scatter(zero_x, zero_y,zero_z, c='r', marker='x',label='zero')
ax.scatter(one_x, one_y,one_z, c='g', marker='+',label='one')
ax.scatter(two_x, two_y,two_z, c='b', marker='s',label='two')

ax.scatter(three_x, three_y,three_z, c='m', marker='*',label='three')
ax.scatter(four_x, four_y,four_z, c='c', marker='h',label='four')
ax.scatter(five_x, five_y,five_z, c='r', marker='D',label='five')

ax.scatter(six_x, six_y,six_z, c='y', marker='8',label='six')
ax.scatter(seven_x, seven_y,seven_z, c='k', marker='*',label='seven')
ax.scatter(eight_x, eight_y,eight_z, c='r', marker='x',label='eight')

ax.scatter(nine_x, nine_y,nine_z, c='b', marker='D',label='nine')

ax.set_xlabel('PC 1')
ax.set_ylabel('PC 2')
ax.set_zlabel('PC 3')

plt.legend(loc='upper left', numpoints=1, ncol=3, fontsize=10, bbox_to_anchor=(0, 0))

plt.show()

# Chosing number of Principal Components
max_pc = 30

pcs = []
totexp_var = []

for i in range(max_pc):
    pca = PCA(n_components=i+1)
    reduced_X = pca.fit_transform(X_scale)
    tot_var = pca.explained_variance_ratio_.sum()
    pcs.append(i+1)
    totexp_var.append(tot_var)

plt.plot(pcs,totexp_var,'r')
plt.plot(pcs,totexp_var,'bs')
plt.xlabel('No. of PCs',fontsize = 13)
plt.ylabel('Total variance explained',fontsize = 13)

plt.xticks(pcs,fontsize=13)
plt.yticks(fontsize=13)
plt.show()

# SVD
import matplotlib.pyplot as plt
from sklearn.datasets import load_digits

digits = load_digits()
X = digits.data
y = digits.target

from sklearn.utils.extmath import randomized_svd
U,Sigma,VT = randomized_svd(X,n_components=15,n_iter=300,random_state=42)

print ("\nShape of Original Matrix:",X.shape)
print ("\nShape of Left Singular vector:",U.shape)
print ("Shape of Singular value:",Sigma.shape)
print ("Shape of Right Singular vector",VT.shape)

import pandas as pd
VT_df = pd.DataFrame(VT)

n_comps = 15
from sklearn.decomposition import TruncatedSVD
svd = TruncatedSVD(n_components=n_comps, n_iter=300, random_state=42)
reduced_X = svd.fit_transform(X)

print("\nTotal Variance explained for %d singular features are %0.3f"%(n_comps,svd.explained_variance_ratio_.sum()))

# Choosing number of Singular Values
max_singfeat = 30

singfeats = []
totexp_var = []

for i in range(max_singfeat):
    svd = TruncatedSVD(n_components=i+1, n_iter=300, random_state=42)
    reduced_X = svd.fit_transform(X)
    tot_var = svd.explained_variance_ratio_.sum()
    singfeats.append(i+1)
    totexp_var.append(tot_var)

plt.plot(singfeats,totexp_var,'r')
plt.plot(singfeats,totexp_var,'bs')
plt.xlabel('No. of Features',fontsize = 13)
plt.ylabel('Total variance explained',fontsize = 13)

#plt.xticks(singfeats,fontsize=13)
plt.yticks(fontsize=13)
plt.show()

# Deep Auto Encoders
import matplotlib.pyplot as plt
from sklearn.preprocessing import StandardScaler
from sklearn.datasets import load_digits

digits = load_digits()
X = digits.data
y = digits.target

print (X.shape)
print (y.shape)

x_vars_stdscle = StandardScaler().fit_transform(X)
print (x_vars_stdscle.shape)

from keras.layers import Input,Dense
from keras.models import Model

# 2-Dimensional Architecture

input_layer = Input(shape=(64,),name="input")

encoded = Dense(32, activation='relu',name="h1encode")(input_layer)
encoded = Dense(16, activation='relu',name="h2encode")(encoded)
encoded = Dense(2, activation='relu',name="h3latent_layer")(encoded)

decoded = Dense(16, activation='relu',name="h4decode")(encoded)
decoded = Dense(32, activation='relu',name="h5decode")(decoded)
decoded = Dense(64, activation='sigmoid',name="h6decode")(decoded)

autoencoder = Model(input_layer, decoded)
autoencoder.compile(optimizer="adam", loss="mse")

# Fitting Encoder-Decoder model
autoencoder.fit(x_vars_stdscle, x_vars_stdscle, epochs=100,batch_size=256,shuffle=True,validation_split= 0.2 )

# Extracting Encoder section of the Model for prediction of latent variables
encoder = Model(autoencoder.input,autoencoder.get_layer("h3latent_layer").output)

# Predicting latent variables with extracted Encoder model
reduced_X = encoder.predict(x_vars_stdscle)

print (reduced_X.shape)

zero_x, zero_y = [],[] ; one_x, one_y = [],[]
two_x,two_y = [],[]; three_x, three_y = [],[]
four_x,four_y = [],[]; five_x,five_y = [],[]
six_x,six_y = [],[]; seven_x,seven_y = [],[]
eight_x,eight_y = [],[]; nine_x,nine_y = [],[]

# For 2-Dimensional data
for i in range(len(reduced_X)):
    if y[i] == 0:
        zero_x.append(reduced_X[i][0])
        zero_y.append(reduced_X[i][1])

    elif y[i] == 1:
        one_x.append(reduced_X[i][0])
        one_y.append(reduced_X[i][1])

    elif y[i] == 2:
        two_x.append(reduced_X[i][0])
        two_y.append(reduced_X[i][1])

    elif y[i] == 3:
        three_x.append(reduced_X[i][0])
        three_y.append(reduced_X[i][1])

    elif y[i] == 4:
        four_x.append(reduced_X[i][0])
        four_y.append(reduced_X[i][1])

    elif y[i] == 5:
        five_x.append(reduced_X[i][0])
        five_y.append(reduced_X[i][1])

    elif y[i] == 6:
        six_x.append(reduced_X[i][0])
        six_y.append(reduced_X[i][1])

    elif y[i] == 7:
        seven_x.append(reduced_X[i][0])
        seven_y.append(reduced_X[i][1])

    elif y[i] == 8:
        eight_x.append(reduced_X[i][0])
        eight_y.append(reduced_X[i][1])

    elif y[i] == 9:
        nine_x.append(reduced_X[i][0])
        nine_y.append(reduced_X[i][1])

zero = plt.scatter(zero_x, zero_y, c='r', marker='x',label='zero')
one = plt.scatter(one_x, one_y, c='g', marker='+')
two = plt.scatter(two_x, two_y, c='b', marker='s')

three = plt.scatter(three_x, three_y, c='m', marker='*')
four = plt.scatter(four_x, four_y, c='c', marker='h')
five = plt.scatter(five_x, five_y, c='r', marker='D')

six = plt.scatter(six_x, six_y, c='y', marker='8')
seven = plt.scatter(seven_x, seven_y, c='k', marker='*')
eight = plt.scatter(eight_x, eight_y, c='r', marker='x')

nine = plt.scatter(nine_x, nine_y, c='b', marker='D')

plt.legend((zero,one,two,three,four,five,six,seven,eight,nine),
           ('zero','one','two','three','four','five','six','seven','eight','nine'),
           scatterpoints=1,
           loc='lower right',
           ncol=3,
           fontsize=10)

plt.xlabel('Latent Feature 1',fontsize = 13)
plt.ylabel('Latent Feature 2',fontsize = 13)

plt.show()

# 3-Dimensional architecture
input_layer = Input(shape=(64,),name="input")

encoded = Dense(32, activation='relu',name="h1encode")(input_layer)
encoded = Dense(16, activation='relu',name="h2encode")(encoded)
encoded = Dense(3, activation='relu',name="h3latent_layer")(encoded)

decoded = Dense(16, activation='relu',name="h4decode")(encoded)
decoded = Dense(32, activation='relu',name="h5decode")(decoded)
decoded = Dense(64, activation='sigmoid',name="h6decode")(decoded)

autoencoder = Model(input_layer, decoded)
autoencoder.compile(optimizer="adam", loss="mse")

# Fitting Encoder-Decoder model
autoencoder.fit(x_vars_stdscle, x_vars_stdscle, epochs=100,batch_size=256,shuffle=True,validation_split= 0.2 )

# Extracting Encoder section of the Model for prediction of latent variables
encoder = Model(autoencoder.input,autoencoder.get_layer("h3latent_layer").output)

# Predicting latent variables with extracted Encoder model
reduced_X3D = encoder.predict(x_vars_stdscle)

zero_x, zero_y,zero_z = [],[],[] ; one_x, one_y,one_z = [],[],[]
two_x,two_y,two_z = [],[],[]; three_x, three_y,three_z = [],[],[]
four_x,four_y,four_z = [],[],[]; five_x,five_y,five_z = [],[],[]
six_x,six_y,six_z = [],[],[]; seven_x,seven_y,seven_z = [],[],[]
eight_x,eight_y,eight_z = [],[],[]; nine_x,nine_y,nine_z = [],[],[]

for i in range(len(reduced_X3D)):

    if y[i]==10:
        continue

    elif y[i] == 0:
        zero_x.append(reduced_X3D[i][0])
        zero_y.append(reduced_X3D[i][1])
        zero_z.append(reduced_X3D[i][2])

    elif y[i] == 1:
        one_x.append(reduced_X3D[i][0])
        one_y.append(reduced_X3D[i][1])
        one_z.append(reduced_X3D[i][2])

    elif y[i] == 2:
        two_x.append(reduced_X3D[i][0])
        two_y.append(reduced_X3D[i][1])
        two_z.append(reduced_X3D[i][2])

    elif y[i] == 3:
        three_x.append(reduced_X3D[i][0])
        three_y.append(reduced_X3D[i][1])
        three_z.append(reduced_X3D[i][2])

    elif y[i] == 4:
        four_x.append(reduced_X3D[i][0])
        four_y.append(reduced_X3D[i][1])
        four_z.append(reduced_X3D[i][2])

    elif y[i] == 5:
        five_x.append(reduced_X3D[i][0])
        five_y.append(reduced_X3D[i][1])
        five_z.append(reduced_X3D[i][2])

    elif y[i] == 6:
        six_x.append(reduced_X3D[i][0])
        six_y.append(reduced_X3D[i][1])
        six_z.append(reduced_X3D[i][2])

    elif y[i] == 7:
        seven_x.append(reduced_X3D[i][0])
        seven_y.append(reduced_X3D[i][1])
        seven_z.append(reduced_X3D[i][2])

    elif y[i] == 8:
        eight_x.append(reduced_X3D[i][0])
        eight_y.append(reduced_X3D[i][1])
        eight_z.append(reduced_X3D[i][2])

    elif y[i] == 9:
        nine_x.append(reduced_X3D[i][0])
        nine_y.append(reduced_X3D[i][1])
        nine_z.append(reduced_X3D[i][2])

# 3- Dimensional plot
from mpl_toolkits.mplot3d import Axes3D
fig = plt.figure()
ax = fig.add_subplot(111, projection='3d')

ax.scatter(zero_x, zero_y,zero_z, c='r', marker='x',label='zero')
ax.scatter(one_x, one_y,one_z, c='g', marker='+',label='one')
ax.scatter(two_x, two_y,two_z, c='b', marker='s',label='two')

ax.scatter(three_x, three_y,three_z, c='m', marker='*',label='three')
ax.scatter(four_x, four_y,four_z, c='c', marker='h',label='four')
ax.scatter(five_x, five_y,five_z, c='r', marker='D',label='five')

ax.scatter(six_x, six_y,six_z, c='y', marker='8',label='six')
ax.scatter(seven_x, seven_y,seven_z, c='k', marker='*',label='seven')
ax.scatter(eight_x, eight_y,eight_z, c='r', marker='x',label='eight')

ax.scatter(nine_x, nine_y,nine_z, c='b', marker='D',label='nine')

ax.set_xlabel('Latent Feature 1',fontsize = 13)
ax.set_ylabel('Latent Feature 2',fontsize = 13)
ax.set_zlabel('Latent Feature 3',fontsize = 13)

ax.set_xlim3d(0,60)

plt.legend(loc='upper left', numpoints=1, ncol=3, fontsize=10, bbox_to_anchor=(0, 0))

plt.show()

ax.set_xlim3d(left = 0,right = 30)
ax.set_ylim3d(left = 0,right = 30)
ax.set_zlim3d(left = 0,right = 30)


import numpy as np
import matplotlib.pyplot as plt

def test_equation(x, l):
    return -l * x

def two_step_method(xk, xkminus1, l, tau):
    return xkminus1 - 2 * tau * l * xk

def solution(t, l, x0):
    return x0*np.exp(-l*t)

lamb = 1.0
x0 = 1.0
tmin = 0.0
tmax = 1.0

grid = np.linspace(tmin, tmax, 1000)
y_soln = solution(grid, lamb, x0)
plt.plot(grid, y_soln, label='Solution', linewidth=3.5)

ns = [10, 100, 1000]
for n in ns:
    grid = np.linspace(tmin, tmax, n)
    print(len(grid))
    tau = 1/n
    x1 = x0 - tau * lamb * x0
    y_twostep = [x0, x1]
    for tk in grid[2:]:
        y_twostep.append(
            two_step_method(y_twostep[-1], y_twostep[-2], lamb, tau)
        )
    plt.plot(grid, y_twostep, label=f'Two step n={n}')
plt.legend()
plt.savefig(f'compare.png')