# We are going to implement a linear regression with degree one.
# Simple equation would be y = mx + c
#
# Formula to find m,
#       mean(x).mean(y) - mean(xy)
#  m = ----------------------------
#       sqr(mean(x)) - mean(sqr(x))
#
# Formula to find c,
#  c = mean(y) - m.mean(x)

from statistics import mean
import numpy as np
import matplotlib.pyplot as plot
from matplotlib import style

x = [1,2,3,4,5,6,7,8]
y = [9,8,5,5,3,3,3,2]
x = np.array(x, dtype=float)
y = np.array(y, dtype=float)

style.use('fivethirtyeight')
plot.scatter(x, y)
# plot.show()
# plot.hold()

def best_fit(x, y):
    m = ((mean(x) * mean(y)) - mean(x * y)) / ((mean(x) ** 2) - mean(x ** 2))
    c = mean(y) - m * mean(x)
    return m, c

m, c = best_fit(x, y)
print(m,c)

regressionLine = [(m * x_ + c) for x_ in x]
# The above equation is equivalent to the following
# for x_ in x:
#     regressionLine.append(m*x_ + c)

predictX = 10
predictY = m * predictX + c

plot.scatter(predictX, predictY, edgecolors='g')

plot.plot(x, regressionLine)
plot.show()


# coding: utf-8

# # Regularization

# ## 1. Introduction
#
# ### 1.1 Libraries

# In[1]:

import numpy as np
import matplotlib.pyplot as plt
from sklearn import linear_model
from scipy.optimize import minimize
from numpy.random import permutation
from sympy import var, diff, exp, latex, factor, log, simplify
from IPython.display import display, Math, Latex
np.set_printoptions(precision=4)
get_ipython().magic('matplotlib inline')

# ### 1.2 Dataset
#
# The in-sample and out-of-sample data is retrieved from the following sources:
#
# ```bash
# curl -O http://work.caltech.edu/data/in.dta
# curl -O http://work.caltech.edu/data/out.dta
# ```
#
#

# ## 2. Non-linear Transforms
#
# ### 2.1 Hypothesis Set
#
# Consider the linear transform $z_i = \phi_i\left(\mathbf{x}\right)$ or $\mathbf{z} = \Phi\left(\mathbf{x}\right)$, with the following mapping:
#
# $$\mathbf{x} = \left(x_0, x_1, \dots, x_d\right) \rightarrow \mathbf{z} = \left(z_0, z_1, \dots, z_{\tilde d}\right)$$
#
# The final hypothesis, $\mathcal{X}$ space is:
#
# $$g\left(\mathbf{x}\right) = \mathbf{\tilde w^T} \Phi\left(\mathbf{x}\right)$$
#
# $$g\left(\mathbf{x}\right) = \left(w_0, w_1, w_2, w_3, w_4, w_5, w_6, w_7\right) \left(\begin{array}{c}1\\x_1\\x_2\\x_1^2\\x_2^2\\x_1 x_2\\|x_1 - x_2|\\|x_1 + x_2|\end{array}\right) = w_0 + w_1 x_1 + w_2 x_2 + \dots$$

# To implement the above hypothesis set on an arbitrary matrix $\mathbf{X}$,
# we augment the matrix $\mathbf{X}$ with a *ones column*, and
# non-linear features are added as new columns via the following two subroutines.

# In[2]:

def add_ones_column(X):
    n = X.shape[0]
    x0 = np.ones((n,1))
    return np.hstack((x0,X))

# In[3]:

def add_nonlinear_features(X,n):
    """
    Here 'n' can be 3 to 7, corresponding to one of the following ϕ_n:
    ϕ_0 = 1
    ϕ_1 = 1 + x_1
    ϕ_2 = 1 + x_1 + x_2
    ϕ_3 = 1 + x_1 + x_2 + x_1^2
    ϕ_4 = 1 + x_1 + x_2 + x_1^2 + x_2^2
    ϕ_5 = 1 + x_1 + x_2 + x_1^2 + x_2^2 + x_1 x_2
    ϕ_6 = 1 + x_1 + x_2 + x_1^2 + x_2^2 + x_1 x_2 + |x_1 - x_2|
    ϕ_7 = 1 + x_1 + x_2 + x_1^2 + x_2^2 + x_1 x_2 + |x_1 - x_2| + |x_1 + x_2|
    """
    m = X.shape[0]
    X = np.hstack((X,np.zeros((m,n-X.shape[1]+1))))
    if n >= 3: X[:,3] = X[:,1]**2
    if n >= 4: X[:,4] = X[:,2]**2
    if n >= 5: X[:,5] = X[:,1]*X[:,2]
    if n >= 6: X[:,6] = np.abs(X[:,1] - X[:,2])
    if n >= 7: X[:,7] = np.abs(X[:,1] + X[:,2])
    return(X)

# ### 2.2 Loading and Transforming the Data
#
# The data is loaded and transformed as follows:

# In[4]:

training = np.loadtxt("data/in.dta")
X_train = add_ones_column(training[:,:-1])
y_train = training[:,-1]
X_train = add_nonlinear_features(X_train,7)
n_train = y_train.shape[0]

testing  = np.loadtxt("data/out.dta")
X_test  = add_ones_column(testing[:,:-1])
y_test  = testing[:,-1]
X_test  = add_nonlinear_features(X_test,7)
n_test = y_test.shape[0]

n_features = X_train.shape[1]

# ### 2.2 Linear Regression (without regularization)
#
# The [ordinary least squares](https://en.wikipedia.org/wiki/Ordinary_least_squares) (OLS) estimator for the weights is:
#
# $$w = \left(\mathbf{X^T X}\right)^{-1}\mathbf{X^T y} = \mathbf{X^\dagger}y$$
#
# where $\mathbf{X^\dagger} = \left(\mathbf{X^T X}\right)^{-1}\mathbf{X^T}$ is the [Moore–Penrose pseudoinverse](https://en.wikipedia.org/wiki/Moore–Penrose_pseudoinverse).
#
# This is implemented via the following line of code in the subroutine below:
#
# `w_linreg = np.dot(np.linalg.pinv(X),y)`

# In[5]:

def train_without_regularization(X_train,y_train):
    w_no_reg = np.dot(np.linalg.pinv(X_train),y_train)
    return w_no_reg

def get_E_in_E_out(X_test,y_test,X_train,y_train,w,verbose=True):
    n_train = y_train.shape[0]
    n_test  = y_test.shape[0]
    y_train_eval = np.sign(np.dot(X_train,w)).astype(int)
    y_test_eval  = np.sign(np.dot(X_test,w)).astype(int)
    E_in = np.sum(y_train != y_train_eval) / n_train
    E_out = np.sum(y_test != y_test_eval) / n_test
    if verbose is True:
        print("E_in = {}".format(E_in))
        print("E_out = {}".format(E_out))
    return E_in,E_out

# The un-regularized in-sample and out-of-sample error can be calculated as follows (in the next section, the regularized in-sample and out-of-sample errors will be calculated):

# In[6]:

w_linreg = np.dot(np.linalg.pinv(X_train),y_train)
y_train_linreg = np.sign(np.dot(X_train,w_linreg))
y_test_linreg  = np.sign(np.dot(X_test,w_linreg))
E_in = np.sum(y_train != y_train_linreg) / n_train
E_out = np.sum(y_test != y_test_linreg) / n_test
print("E_in = {}".format(E_in))
print("E_out = {}".format(E_out))

# ### 2.3 Linear Regression (with Regularization)
#
# #### 2.3.1 Background
#
# In order to perform regularization, a "soft-order" constraint is employed for the weights:
#
# $$\sum\limits_{q=0}^Q w_q^2 \le C$$
#
# Subject to the above constraint, our aim is to minimize the following cost function on the in-sample points:
#
# $$E_{in}\left(\mathbf{w}\right) = \frac{1}{N} \sum\limits_{n=1}^N \left(\mathbf{w^T z_n - y_n}\right)^2 = \frac{1}{N}\left(\mathbf{Zw-y}\right)^T\left(\mathbf{Zw-y}\right)$$
#
# The minimum of the above function occurs when:
#
# $$\nabla E_{in}\left(\mathbf{w}\right) \propto -\mathbf{w_{reg}}$$
# $$\nabla E_{in}\left(\mathbf{w}\right) = -2\frac{\lambda}{N}\mathbf{w_{reg}}$$
#
# where in the last step, we have introduced $2\frac{\lambda}{N}$ as our constant of proportionality.

# Hence, we wish to solve:
#
# $$\nabla E_{in}\left(\mathbf{w_{reg}}\right) + 2\frac{\lambda}{N}\mathbf{w_{reg}} = 0$$
#
# Notice that solving the above equation corresponds to minimizing the following:
#
# $$E_{in}\left(\mathbf{w}\right) + \frac{\lambda}{N}\mathbf{w^T w}$$
#
# This new cost function can be viewed as an augmented error, which we can write as follows:
#
# $$E_{aug}\left(\mathbf{w}\right) = \frac{1}{N}\left(\mathbf{Zw-y}\right)^T \left(\mathbf{Zw-y}\right) + \frac{\lambda}{N}\mathbf{w^T w}$$
#
# Minimizing the above cost function is equivalent to minimizing $E_{in}\left(\mathbf{w}\right)$ subject to the constraint $\mathbf{w^T w} \le C$
#
# To determine the **regularized weights**, we minimize the augmented error as follows:
#
# $$\nabla E_{aug}\left(\mathbf{w}\right) = 0$$
#
# $$\mathbf{Z^T} \left(\mathbf{Zw-y}\right) + \lambda\mathbf{w} = 0$$
#
# $$\left(\mathbf{Z^T Z} + \lambda\mathbf{I}\right)\mathbf{w} - \mathbf{Z^T y} = 0$$
#
# $$\mathbf{w_{reg}} = \left(\mathbf{Z^T Z} + \lambda\mathbf{I}\right)^{-1}\mathbf{Z^T y}$$

# In[7]:

def train_with_regularization(lmbda,X_train,y_train,use_sklearn=False):
    n_samples = y_train.shape[0]
    if use_sklearn is True:
        reg = linear_model.Ridge(alpha = lmbda/n_samples, fit_intercept=True)
        reg.fit(X_train,y_train)
        w_reg = reg.coef_
        w_reg[0] = reg.intercept_
    else:
        n_features=X_train.shape[1]
        ZTy = np.dot(X_train.T,y_train)
        ZTZ = np.dot(X_train.T,X_train)
        w_reg = np.dot(np.linalg.inv(ZTZ + lmbda*np.identity(n_features)),ZTy)
    return w_reg

# In[8]:

def get_E_in_E_out(X_test,y_test,X_train,y_train,w):
    n_train = y_train.shape[0]
    n_test = y_test.shape[0]
    y_train_w = np.sign(np.dot(X_train,w))
    y_test_w  = np.sign(np.dot(X_test,w))
    E_in = np.sum(y_train != y_train_w) / n_train
    E_out = np.sum(y_test != y_test_w) / n_test
    return E_in, E_out

# #### 2.3.2 Checking the above subroutine.
#
# The data can be trained by either using the subroutine `train_without_regularization()` or setting $\lambda=0$ in the subroutine `train_with_regularization()`.  The weights obtained should be the same in either case, and this is easy to check with `numpy.allclose()`.

# In[9]:

w_no_reg = train_without_regularization(X_train,y_train)
w_reg = train_with_regularization(0.0,X_train,y_train,use_sklearn=False)
assert np.allclose(w_no_reg,w_reg)
E_in,E_out = get_E_in_E_out(X_test,y_test,X_train,y_train,w_no_reg)

# #### 2.3.3 Finding the best $\lambda$
#
# To find the best choice of $\lambda$, the values of $E_{in}$ and $E_{out}$ are plotted over a range of values of $\lambda$.

# In[10]:

def find_best_lambda(lmbd_arr,X_train,y_train,X_test,y_test,use_sklearn=False):
    E_in_arr = []
    E_out_arr = []
    w_arr = []
    for lmbda in lmbd_arr:
        w_reg = train_with_regularization(lmbda,X_train,y_train,use_sklearn=use_sklearn)
        E_in, E_out = get_E_in_E_out(X_test,y_test,X_train,y_train,w_reg)
        E_in_arr.append(E_in)
        E_out_arr.append(E_out)
        w_arr.append(w_reg)
    return np.array(E_in_arr), np.array(E_out_arr), np.array(w_arr)

# In[11]:

def plot_regularization(fig,plot_id,title,lmbd_arr,E_in_reg,E_out_reg):
    ax = fig.add_subplot(plot_id)
    ax.semilogx(lmbd_arr,E_in_reg,'x-',label="$E_{in}$")
    ax.semilogx(lmbd_arr,E_out_reg,'o-',label="$E_{out}$")
    ax.set_xlabel('$\lambda$')
    ax.grid(True)
    ax.legend(loc='best',frameon=False)
    ax.set_title(title)

# In[12]:

lmbd_arr1 = [0.0001, 0.0002, 0.0005, 0.001, 0.002, 0.005, 0.01, 0.02, 0.05, 0.1, 0.2, 0.5, 1.0]
fig = plt.figure(figsize=(12,5))
E_in_reg1, E_out_reg1, w_reg1 = find_best_lambda(lmbd_arr1,X_train,y_train,X_test,y_test,False)
plot_regularization(fig,121,"",lmbd_arr1,E_in_reg1,E_out_reg1)
#E_in_sk, E_out_sk, w_sk = find_best_lambda(lmbd_arr,X_train,y_train,X_test,y_test,True)
#plot_regularization(fig,122,"sklearn",lmbd_arr,E_in_sk,E_out_sk)
lmbd_arr2 = [1.0, 2.0, 5.0, 10.0, 20.0, 50.0, 100.0, 200.0, 500.0, 1000.0]
E_in_reg2, E_out_reg2, w_reg2 = find_best_lambda(lmbd_arr2,X_train,y_train,X_test,y_test,False)
plot_regularization(fig,122,"",lmbd_arr2,E_in_reg2,E_out_reg2)

# Here we can see that when regularization turned on ($\lambda \ne 0$):
#
# * $\lambda = 10^{-3}$ has an $E_{in}$ of about 0.03, and an $E_{out}$ of 0.08.
# * $\lambda = 10^{-1}$ has the best $E_{out}$ of about 0.06.
# * $\lambda = 10^{3}$ has an $E_{in}$ and $E_{out}$ of about 0.4.

# ### 2.4 Validation Sets
#
# The data may also be split into validation sets for training, and the new model evaluated on the final, held-out, test set.  In the following subroutine, the training set (`X_train_all,y_train_all`) is split one more time into a new training set (`X_train,y_train`) and a validation set (`X_valid,y_valid`).  By default, the first 25 points are designated as training points (`train_pts`).

# In[13]:

def training_validation_test_split(training,testing,train_pts=25,k=7,swap=False):
    X_train_all = add_ones_column(training[:,:-1])
    y_train_all = training[:,-1]
    X_train_all = add_nonlinear_features(X_train_all,k)

    X_test  = add_ones_column(testing[:,:-1])
    y_test  = testing[:,-1]
    X_test  = add_nonlinear_features(X_test,k)

    val_pts = X_train_all.shape[0]-train_pts
    assert val_pts == y_train_all.shape[0]-train_pts
    if swap is True:
        X_train = X_train_all[val_pts:,:]
        X_valid = X_train_all[:val_pts,:]
        y_train = y_train_all[val_pts:]
        y_valid = y_train_all[:val_pts]
    else:
        X_train = X_train_all[:train_pts,:]
        X_valid = X_train_all[train_pts:,:]
        y_train = y_train_all[:train_pts]
        y_valid = y_train_all[train_pts:]

    assert X_train.shape[0]==train_pts
    assert X_valid.shape[0]==val_pts
    assert y_train.shape[0]==train_pts
    assert y_valid.shape[0]==val_pts
    n_features = X_train.shape[1]
    return X_train_all,y_train_all,X_train,y_train,X_valid,y_valid,X_test,y_test

# In this example, the first 3,4,5,6, or 7 terms (corresponding to $\phi_3, \phi_4, \dots, \phi_7$) of the non-linear transform are used and the classification error of the validation set is evaluated.

# In[14]:

lmbd_arr = [0.0001, 0.0002, 0.0005, 0.001, 0.002, 0.005, 0.01, 0.02, 0.05,
            0.1, 0.2, 0.5, 1.0, 2.0, 5.0, 10.0, 20.0, 50.0, 100.0, 500.0, 1000.0]
k_arr = np.array([3, 4, 5, 6, 7])
fig = plt.figure(figsize=(12,10))
plot_ids = [ 321, 322, 323, 324, 325 ]
for i,k in enumerate(k_arr):
    X_train_all,y_train_all,X_train,y_train,X_valid,y_valid,X_test,y_test =         training_validation_test_split(training,testing,k=k)
    E_in_reg, E_val_reg, w_reg_arr = find_best_lambda(lmbd_arr,X_train,y_train,X_valid,y_valid,False)
    plot_regularization(fig,plot_ids[i],"$\phi_{}$".format(k),lmbd_arr,E_in_reg,E_val_reg)

# Here we can see that for small $\lambda$, $\phi_6$ has the lowest classification error on the validation set.

# ### 2.5 Plotting
#
# The in-sample and out-of-sample error from the non-linear transforms can be visualized via contour plots.  In this case, we will use $\lambda = 0.1$ which had the lowest out-of-sample error.

# In[15]:

def plot_data_nonlinear(fig,plot_id,X,y,w_arr,w_colors,titles):
    p = 2.0
    x1 = np.linspace(-p,p,100)
    x2 = np.linspace(-p,p,100)
    X1,X2 = np.meshgrid(x1,x2)
    X1_sq= X1**2
    X2_sq= X2**2
    X1X2 = X1*X2
    X1mX2= np.abs(X1-X2)
    X1pX2= np.abs(X1+X2)

    for i,w in enumerate(w_arr):
        G = w[0] + w[1]*X1 + w[2]*X2 + w[3]*X1_sq + w[4]*X2_sq +             w[5]*X1X2 + w[6]*X1mX2 + w[7]*X1pX2
        ax = fig.add_subplot(plot_id[i])
        ax.plot(X[y > 0,1],X[y > 0,2],'b+',label='Positive (+)')
        ax.plot(X[y < 0,1],X[y < 0,2],'r_',label='Negative (-)')
        cp0 = ax.contour(X1,X2,G,1,linewidth=4, levels=[0.0],
                         colors=w_colors[i])
        ax.clabel(cp0, inline=True, fontsize=14)
        #cp1 = ax.contour(X1,X2,Y,N=1,linewidth=4, levels=[-1.0, 1.0],
        #                 linestyles='dashed', colors=w_colors[i], alpha=0.3)
        cp1 = ax.contourf(X1,X2,G,1,linewidth=4, linestyles='dashed', alpha=0.8)
        ax.clabel(cp1, inline=True, fontsize=14)

        plt.colorbar(cp1)
        ax.set_title(titles[i])
        #ax.set_axis_off()  #ax.axis('off')
        #ax.axes.xaxis.set_ticks([])
        #ax.axes.yaxis.set_ticks([])
        ax.set_xlim(-p,p)
        ax.set_ylim(-p,p)
    fig.tight_layout()

# In[16]:

w_no_reg = train_with_regularization(0.0,X_train_all,y_train_all,use_sklearn=False)
w_reg    = train_with_regularization(0.1,X_train_all,y_train_all,use_sklearn=False)
E_in_no_reg,E_out_no_reg = get_E_in_E_out(X_test,y_test,X_train_all,y_train_all,w_no_reg)
E_in_reg,   E_out_reg    = get_E_in_E_out(X_test,y_test,X_train_all,y_train_all,w_reg)

# In[17]:

def string_Ein(E_in):
    return '$E_{in}=' + '{}'.format(np.round(E_in,5)) + '$'

def string_Eout(E_out):
    return '$E_{out}=' + '{}'.format(np.round(E_out,5)) + '$'

# In[18]:

w_arr = [ w_no_reg, w_reg ]
w_colors = ['red', 'blue' ]
title_train = ['Training Set: (a) $\mathbf{w}$, ' + string_Ein(E_in_no_reg),
               'Training Set: (b) $\mathbf{w_{reg}}$, ' + string_Ein(E_in_reg)]
title_test  = ['Testing Set: (a) $\mathbf{w}$, ' + string_Eout(E_out_no_reg),
               'Testing Set: (b) $\mathbf{w_{reg}}$, ' + string_Eout(E_out_reg)]
fig = plt.figure(figsize=(14,12))
plot_data_nonlinear(fig,[ 221, 222 ],X_train,y_train,w_arr,w_colors,title_train)
plot_data_nonlinear(fig,[ 223, 224 ],X_test, y_test, w_arr,w_colors,title_test)

# ## 3. Hypothesis sets
#
# Consider the following hypothesis set
#
# $$\mathcal{H}_Q = \left\{ h \; | \; h(x) = \mathbf{w^T z} = \sum\limits_{q=0}^Q w_q L_q(x) \right\}$$

# We thus have $\mathcal{H_4, H_3, H_2, H_1}$ as:
#
# $$\mathcal{H}_4 = \left\{ h \; | \; h(x) = \mathbf{w^T z} = \sum\limits_{q=0}^4 w_q L_q(x) \right\}$$
#
# $$\mathcal{H}_3 = \left\{ h \; | \; h(x) = \mathbf{w^T z} = \sum\limits_{q=0}^3 w_q L_q(x) \right\}$$
#
# $$\mathcal{H}_2 = \left\{ h \; | \; h(x) = \mathbf{w^T z} = \sum\limits_{q=0}^2 w_q L_q(x) \right\}$$
#
# $$\mathcal{H}_1 = \left\{ h \; | \; h(x) = \mathbf{w^T z} = \sum\limits_{q=0}^1 w_q L_q(x) \right\}$$

# We also have:
#
# $$\mathcal{H}\left(Q, C, Q_0\right) = \left\{ h \; | \; h(x) = \mathbf{w^T z} \in \mathcal{H}_Q; w_Q = C \text{ for } q \ge Q_0 \right\}$$
#
# For $Q = 10, C = 0, Q_0 = 3$:
# $$\mathcal{H}\left(10, C, 3\right) = \left\{ h \; | \; h(x) = \mathbf{w^T z} \in \mathcal{H}_{10}; w_Q = 0 \text{ for } q \ge 3 \right\} = \mathcal{H}_2$$
#
# For $Q = 10, C = 0, Q_0 = 4$:
# $$\mathcal{H}\left(10, C, 4\right) = \left\{ h \; | \; h(x) = \mathbf{w^T z} \in \mathcal{H}_{10}; w_Q = 0 \text{ for } q \ge 4 \right\} = \mathcal{H}_3$$
#
# Hence we have:
#
# $$\mathcal{H}\left(10, C, 3\right) \cap \mathcal{H}\left(10, C, 4\right) = \mathcal{H}_2 \cap \mathcal{H}_3 = \mathcal{H}_2$$

# ## 4. Neural Networks
#
# ### Forward Propagation
#
# $$x_j^{(l)} = \theta \left( s_j^{(l)} \right) = \theta \left( \sum\limits_{i=0}^{d^{(l-1)}} w_{ij}^{(l)} x_i^{(l-1)} \right)$$
#
# where
#
# $$\theta\left(s\right) = \tanh\left(s\right)$$
#
# ### Backpropagation
#
# $$\delta w_{ij}^{(l)} = -\eta x_i^{(l-1)} \delta_j^{(l)}$$
#
# where
#
# $$\delta_j^{(l-1)} = \left[1 - \left(x_i^{(l-1)}\right)^2\right] \sum_{j=1}^{d^{(l)}} w_{ij}^{(l)} \delta_j^{(l)}$$

# ### Gradient Descent
#
# $$\mathbf{w}\left(t+1\right) = \mathbf{w}\left(t\right) - \eta \nabla E_{in}\left(\mathbf{w}\left(t\right)\right) - 2\eta \frac{\lambda}{N} \mathbf{w}\left(t\right)$$
#
# $$\mathbf{w}\left(t+1\right) = \mathbf{w}\left(t\right)\left(1 - 2\eta \frac{\lambda}{N}\right) - \eta \nabla E_{in}\left(\mathbf{w}\left(t\right)\right)$$
#
# where
#
# $$\mathbf{w^T w} = \sum\limits_{l=1}^L \sum\limits_{i=0}^{d^{(l-1)}} \sum\limits_{j=1}^{d^{(l)}} \left( w_{ij}^{(l)}\right)^2$$
#

# Consider a neural network with 5 inputs, 3 nodes at the hidden layer, and a single node in the output layer, i.e. $d^{(0)} = 5, d^{(1)} = 3, d^{(2)} = 1$:
#
# Further reading: http://home.agh.edu.pl/~vlsi/AI/backp_t_en/backprop.html

# In[19]:

def calculate_weights(layer_nodes):
    """
    Calculate the number of weights required for a neural network,
    where layer_nodes contains the number of nodes for each layer,
    excluding bias units.
    """
    n = len(layer_nodes)
    total = 0
    for i in range(n-1):
        prev = layer_nodes[i]+1  # Add 1 to include bias unit
        curr = layer_nodes[i+1]
        total += prev*curr
        print("Layer {}: d+1={} -> {} = {}".format(i+1,prev,curr,prev*curr))
    return total

# For a neural network with 5 inputs, 3 nodes in the hidden layer, and 1 node in the output layer, the total number of weights is calculated as follows:

# In[20]:

calculate_weights(layer_nodes = [5, 3, 1])

# Suppose we wish to calculate the minimum and maximum number of weights of a neural network with the restriction that it has 10 input units, 36 hidden units, and one output unit.  The minimum number of weights occurs when the 36 hidden units is split into 18 hidden layers (each with 1 bias node + 1 regular node).  The number of weights is calculated as follows:

# In[21]:

calculate_weights(layer_nodes = [9] + ([1] * 18) + [1])

# To calculate the maximum number of weights, a neural network with a single hidden layer would have the following number of weights:

# In[22]:

calculate_weights(layer_nodes = [9, 35, 1])

# If we had two hidden layers:

# In[23]:

calculate_weights([9, 17, 17, 1])

# In[24]:

calculate_weights([9, 18, 16, 1])

# In[25]:

calculate_weights([9, 19, 15, 1])

# In[26]:

calculate_weights([9, 20, 14, 1])

# In[27]:

calculate_weights([9, 21, 13, 1])

# If we had 3 hidden layers:

# In[28]:

calculate_weights([9, 11, 11, 11, 1])

# If we had 4 hidden layers:

# In[29]:

calculate_weights([9, 8, 8, 8, 8, 1])

# Hence, the maximum number of weights corresponds to the neural network [9, 21, 13, 1], with 10 units in the input layer, 22 units in the first hidden layer, and 14 units in the second hidden layer, and 1 unit in the final output layer.  The total number of weights for this network is 510.

import numpy,from numpy,from scipy,from sympy


import numpy as np
import matplotlib.pyplot as plt
import math

# 특정 서브플롯 axis에 벡터 v를 시작점 start에서 출발하도록 그림
# 화살표의 폭, 머리의 크기, 화살표 색상을 지정하고 선 스타일은 ls로 지정 가능
def draw_vector(axis, v, start = (0,0),
                width = 1, headsize=0.3, color='k', ls='-'):

    # 인자에 따라 적절한 화살표를 그린다.
    s = np.array(start)
    if headsize > 0:
        headsize = np.linalg.norm(v)/15
    axis.arrow(start[0], start[1],
               v[0], v[1],
               linewidth = width,
               head_width=headsize, head_length=headsize,
               fc=color, ec=color,
               length_includes_head = True, ls = ls)

def draw_vector3d(axis, v, start = (0,0,0) , color='k', ls='-', arrow=True):
    s = np.array(start)
    if arrow == True:
        axis.quiver(s[0], s[1], s[2],
                    v[0], v[1], v[2], color=color, ls=ls)
    else:
        axis.plot([s[0], v[0]+s[0]],
                  [s[1], v[1]+s[1]],
                  [s[2], v[2]+s[2]], color=color, ls=ls)

def draw_line(axis, start, end, color='k', ls='-'):
    if len(start) == 2:
        axis.plot([start[0], end[0]],
                  [start[1], end[1]], color=color, ls=ls)
    elif len(start) == 3:
        axis.plot([start[0], end[0]],
                  [start[1], end[1]],
                  [start[2], end[2]], color=color, ls=ls)

def draw_vectors(vector_list, start = (0,0,0), color_list = [], figsize=(10,10), linked=False, axis=None) :
    if vector_list != None and len( vector_list) > 0:
        if len(vector_list) != len(color_list) :
            if len(color_list) > 0:
                 colors = [color_list[0] for _ in range(len(vector_list))]
            else:
                 colors = ['k' for _ in range(len(vector_list))]
        else:
             colors = color_list
    dim = len(vector_list[0])
    # figure and axis preparation

    vecsum = np.zeros_like(vector_list[0])
    for v in vector_list:
        vecsum += v
    maxvalue = max(abs(vecsum.min()), abs(vecsum.max())) * 1.1

    if dim == 2:
        s = np.array([start[0], start[1]], dtype=float)
        if axis == None:
            ax = createAxis2d(-maxvalue, maxvalue,  figsize = figsize)
            print(ax)
        else:
            ax = axis
        for v, c in zip(vector_list, colors):
            draw_vector(ax, v, color=c, start=s)
            if linked: s += v
        if linked:
            draw_vector(ax, s, color='k', ls='--', headsize=0.0)
        if axis == None:
            plt.show()
    if dim == 3:
        s = np.array([start[0], start[1], start[2]], dtype=float)
        if axis == None:
            ax = createAxis3d(-maxvalue, maxvalue,  figsize = figsize)
            print(ax)
        else:
            ax = axis
        for v, c in zip(vector_list, colors):
            draw_vector3d(ax, v, color=c, start=s)
            if linked: s += v
        if linked:
            draw_vector3d(ax, s, color='k', ls='--', arrow=False)
        if axis == None:
            plt.show()

def draw_matrix(mat, figsize=(10, 10), axis = None):
    cols = mat.shape[1]
    color = ['r', 'g', 'b' ]
    vectors = []
    colors = []
    for i in range(cols):
        vectors.append(mat[:, i])
        colors.append(color[i%3])

    dim = vectors[0].shape[0]

    flatten_numbers = np.array(vectors).flatten()
    minvalue = flatten_numbers.min()
    maxvalue = flatten_numbers.max()
    d = maxvalue - minvalue
    minvalue -= d*0.25
    maxvalue += d*0.25

    if dim == 2:
        if axis == None:
            ax = createAxis2d(-maxvalue, maxvalue, figsize = figsize)
        else:
            ax = axis
        u = vectors[0]
        v = vectors[1]
        for vec, c in zip(vectors, colors):
            draw_vector(ax, vec, color=c, start=(0,0))

        vectors = [-u,-v]
        for vec, c in zip(vectors, colors):
            draw_vector(ax, vec, start=u+v, color='gray', ls='--', headsize=0)
        if axis==None: plt.show()

    if dim == 3:
        if axis == None:
            ax = createAxis3d(-maxvalue, maxvalue,  figsize = figsize)
        else:
            ax = axis
        u = vectors[0]
        v = vectors[1]
        w = vectors[2]
        for vec, c in zip(vectors, colors):
            draw_vector3d(ax, vec, color=c, start=(0,0,0))
        colors = ['r', 'g', 'b']
        vectors = [-u, v, -w]
        for vec, c in zip(vectors, colors):
            draw_vector3d(ax, vec, start=u+w, color='gray', ls='--', arrow=False)
        vectors = [-u,-v, w]
        for vec, c in zip(vectors, colors):
            draw_vector3d(ax, vec, start=u+v, color='gray', ls='--', arrow=False)
        vectors = [u, -v,-w]
        for vec, c in zip(vectors, colors):
            draw_vector3d(ax, vec, start=v+w, color='gray', ls='--', arrow=False)
        if axis==None: plt.show()

def draw_points(mat, figsize=(10, 10), color='k', ls='-', axis=None):

    dim = mat.shape[0]

    flatten_numbers = np.array(mat).flatten()
    minvalue = flatten_numbers.min()
    maxvalue = flatten_numbers.max()
    d = maxvalue - minvalue
    minvalue -= d * 0.25
    maxvalue += d * 0.25

    if dim == 2:
        if axis == None:
            ax = createAxis2d(-maxvalue, maxvalue, figsize=figsize)
        else:
            ax = axis

        ax.plot(mat[0,:], mat[1,:], color=color, ls=ls)
        if axis == None: plt.show()

    if dim == 3:
        if axis == None:
            ax = createAxis3d(-maxvalue, maxvalue, figsize=figsize)
        else:
            ax = axis
        ax.plot(mat[0,:], mat[1,:], mat[2, :], color=color, ls=ls)
        if axis == None: plt.show()

def draw_line(a, b, c, axis, ls = '-', color=None): # draw line: ax + by + c = 0
    if math.fabs(b) > 1e-5 :
        y = - (c / b)
        dir = np.array([1, -a / b])
        if math.fabs(a) > 1e-5:
            x = - (c + b*y) / a
        else:
            x = 0.0
    else:
        y = 0.0
        x = - (c / a)
        dir = np.array([0, 1])

    p = np.array([x, y])
    p0 = p - dir * 10000
    p1 = p + dir* 10000
    axis.plot([p0[0], p1[0]], [p0[1], p1[1]], ls = ls, color=color)

def createAxis2d(minvalue, maxvalue, subplot=(1,1), figsize=(10,10)):

    if subplot == (1,1):
        fig, ax = plt.subplots(figsize=figsize)
        ax.set_xlim([minvalue, maxvalue])
        ax.set_ylim([minvalue, maxvalue])
        ax.set_xlabel('$x$')
        ax.set_ylabel('$y$')
        ax.grid(True)
        return ax
    elif subplot[0] > 1 and subplot[1] > 1:
        fig, ax = plt.subplots(subplot[0], subplot[1], figsize=figsize)  # figure 크기를 결정
        for row in range(subplot[0]):
            for col in range(subplot[1]):
                ax[row, col].set_xlim([minvalue, maxvalue])
                ax[row, col].set_ylim([minvalue, maxvalue])
                ax[row, col].set_xlabel('$x$')
                ax[row, col].set_ylabel('$y$')
                ax[row, col].grid(True)
        return ax
    else:
        dim = max(subplot[0], subplot[1])
        fig, ax = plt.subplots(subplot[0], subplot[1], figsize=figsize)  # figure 크기를 결정
        for d in range(dim):
            ax[d].set_xlim([minvalue, maxvalue])
            ax[d].set_ylim([minvalue, maxvalue])
            ax[d].set_xlabel('$x$')
            ax[d].set_ylabel('$y$')
            ax[d].grid(True)
        return ax

def createAxis3d(minvalue, maxvalue, subplot=(1,1), figsize=(10,10)):

    if subplot == (1,1):
        fig = plt.figure(figsize = figsize)   # figure 크기를 결정
        ax = fig.gca(projection='3d')
        ax.set_xlim([minvalue, maxvalue])
        ax.set_ylim([minvalue, maxvalue])
        ax.set_zlim([minvalue, maxvalue])
        ax.set_xlabel('$x$')
        ax.set_ylabel('$y$')
        ax.set_zlabel('$z$')
        ax.grid(True)
        return ax
    elif subplot[0] > 1 and subplot[1] > 1:
        fig, ax = plt.subplots(subplot[0], subplot[1], figsize = figsize)   # figure 크기를 결정
        i = 1
        for row in range(subplot[0]):
            for col in range(subplot[1]):
                ax[row, col] = fig.add_subplot(subplot[0], subplot[1], i, projection='3d')
                ax[row, col].set_xlim([minvalue, maxvalue])
                ax[row, col].set_ylim([minvalue, maxvalue])
                ax[row, col].set_zlim([minvalue, maxvalue])
                ax[row, col].set_xlabel('$x$')
                ax[row, col].set_ylabel('$y$')
                ax[row, col].set_zlabel('$z$')
                ax[row, col].grid(True)
                i+=1
        return ax
    else:
        dim = max(subplot[0], subplot[1])
        fig, ax = plt.subplots(subplot[0], subplot[1], figsize=figsize)  # figure 크기를 결정
        i = 1
        for d in range(dim):
            ax[d] = fig.add_subplot(subplot[0], subplot[1], i, projection='3d')
            ax[d].set_xlim([minvalue, maxvalue])
            ax[d].set_ylim([minvalue, maxvalue])
            ax[d].set_zlim([minvalue, maxvalue])
            ax[d].set_xlabel('$x$')
            ax[d].set_ylabel('$y$')
            ax[d].set_zlabel('$z$')
            ax[d].grid(True)
            i += 1
        return ax


"""
Understanding data error mathematics and analysis

This script contains...
Functions for calculating error and deviation between two or one data lists.
    All expressions use the standard Python library only, except rmse which also uses numpy.
Documentation attempting to illuminate the reasoning behind the math.
Standard Deviation can be calculated specifying form='rmse' and list2 == None

Written by Felicty Nielson, mostly for my own intituion,
but if you find this useful, then I am happy to oblige.
"""

def print_outlim(val1, val2):
    """ Helper for printing outliers
    """
    print("Outlier found:  |{} - {}| = {}".format(val1, val2, abs(val1 - val2)))
    print("Outlier has been excluded from calculations")

def zero_chk(list1):
    """ Some of these formulas don't handle 0's in certain places well (e.g. diving by 0 in MAPE).
        For this reason, we raise an exception if any are found.
    """
    for x in list1:
        if x == 0:
            print("0 found in list1. MAPE can not divide by 0.")

def float_chk(list1):
    """ In python 2 versions, at least one of the lists will need to be in floats (not ints)
        to ensure correct calculation.
    """
    for i, j in enumerate(list1):
        list1[i] = float(j)
    return list1

def mean_list(list1, N):
    """ Helper function calculates mean and returns list of N means
    """
    ave = 0
    for i in list1:
        ave += i
    ave /= N
    lista = [ave for x in range(0,N)]
    return lista

def errors(form, list1, list2=None, deviation=False, outlim=0):
    """ Function for calculating error and deviation between two or one data lists.
        Lists must be of same length and the elements of the first list corresponding
        to the elements of the second.
        Args:
          form  (string): Mathematical formula for calculating error.
                          Availiable formulas are either 'mae', 'mape', 'mse', or 'rmse'
          deviation  (bool): default False. If specifying only one list, deviation is automatic.
                             If specifying two lists, deviation of the second list is measured off of
                             the mean of the first.
          list1 (list of floats): The "true" values
          list2 (list of floats): The "observed" values. Default None.
                                  If None, the deviation of list1 is calculated.
          outlim (float): default 0 equivalent to None. If the difference between two elements
                          is greater than the outlier limit, print exception and exclude from
                          error calculation.
        Returns:
          err (int): The computed error or deviation.
    """
    import numpy as np

    list1 = float_chk(list1)

    outlim_count = 0
    error = 0
    N = len(list1)

    # Determine if calculating error or deviation. If calculating deviation, build mean list.
    if list2 != None and deviation == False:
        print("Calculating error...")
    else:
        print("Calculating deviation...")
        if list2 == None:
            list2 = mean_list(list1, N)
        elif deviation == True:
            list1 = mean_list(list1, N)

    for i in range(0, N):
        if form == 'mae':
            err = abs(list1[i]-list2[i])
            """ Mean Absolute Error (MAE)

            Sum(|y-y'|)/N

            The simpliest way to decide the error between measurements is to take the difference.
            Say you measure a mile and it was 4 inch off of a true mile. The 4 inch is your error.
            Whether this error is + or - often doesn't matter and can complicate analysis,
            so we take the absolute value.
            That is the error between two values. If we have two lists and we want a single value
            to represent the error of all data, we can average them
            (sum and divide by the total number of errors or data point comparisons).

            To calculate deviation, we modify the formula to subtract the mean of one list from the values
            of another list. This lets us know how far the observed values are deviating from the true average.
            Or we can subtract the values of one list from the mean of the same list. This measures how
            much the data deviates from its own average.

            Standard deviation uses RMSE (or rather, RMSD...). See the 'rmse' condition below.
            """

        elif form  == 'mape':
            zero_chk(list1)
            err = abs(list1[i]-list2[i])/list1[i]
            """ Mean Absolute Percent Error (MAPE)

            Sum(|y-y'|/y)/N*100

            We are looking at a percent on the original or true y value,
            rather than a value of the differences of the measurement (MAE),
            which can be meaningless to the user unless they understand the scale of their data.

            For example,
            say I measure the size of giant bacteria in cm I get
            y' = [1.59, 4.37, 10.95, 6.27, 8.04, 9.29, 3.48]
            And the true vales are
            y = [1.62, 4.10, 10.81, 6.28, 8.10, 9.50, 3.47]

            The MAE is 0.104285714285714

            Then I do the same thing but convert to mm first

            The MAE is 104.285714285714

            But if I use MAPE, I get the same value for both errors (1.828204390849)
            since the hard difference is taken as a percentage of the true measurement.
            Therefore if I use MAPE, I don't need to know what scale I'm working with,
            at least for this purpose.
            """

        elif form == 'mse' or form == 'rmse':
            err = (list1[i] - list2[i])**2
            """ Mean Squared Error (MSE)

            Sum((y - y')^2)/N

            Another way of removing the negative from errors is to square them.
            This will convert everything to positive values without having to take the
            absolute value.

            --------

            Root Mean Squared Error (RMSE)

            Sqrt(Sum((y - y')^2)/N)

            Specifying only one list will result in the Standard Deviation (SD).

            Warning: outliers will dramatically increase the error for MSE and RMSE.
            Suggest specifying an outlim (outlier limit) value.
            """

        else:
            print("form should be 'mae', 'mape', 'mse', or 'rmse'")

        if outlim != 0 and abs(list1[i]-list2[i]) > outlim:
            print_outlim(list1[i], list2[i])
            outlim_count += 1
        else:
            error += err

    error /= (N - outlim_count)
    if form == 'mape':
        error *= 100
    elif form == 'rmse':
        error = np.sqrt(error)

    return error


'''
Estimating an MA Model

You will estimate the MA(1) parameter, θ
θ
, of one of the simulated series that you generated in the earlier exercise. Since the parameters are known for a simulated series, it is a good way to understand the estimation routines before applying it to real data.

For simulated_data_1 with a true θ
θ
 of -0.9, you will print out the estimate of θ
θ
. In addition, you will also print out the entire output that is produced when you fit a time series, so you can get an idea of what other tests and summary statistics are available in statsmodels.

INSTRUCTIONS
100XP
Import the class ARMA in the module statsmodels.tsa.arima_model.
Create an instance of the ARMA class called mod using the simulated data simulated_data_1 and the order (p,q) of the model (in this case, for an MA(1)), is order=(0,1).
Fit the model mod using the method .fit() and save it in a results object called res.
Print out the entire summmary of results using the .summary() method.
Just print out an estimate of the constant and phi parameter using the .params attribute (no arguments).
'''
# Import the ARMA module from statsmodels
from statsmodels.tsa.arima_model import ARMA

# Fit an MA(1) model to the first simulated data
mod = ARMA(simulated_data_1, order=(0,1))
res = mod.fit()

# Print out summary information on the fit
print(res.summary())

# Print out the estimate for the constant and for theta
print("When the true theta=-0.9, the estimate of theta (and the consant) are:")
print(res.params)


import numpy
import matplotlib.pyplot as plt

def interpolateCatmulRomeSegment(P0, P1, P2, P3, nPoints=20):
    """
    P0, P1, P2, and P3 should be (x,y) point pairs that define the Catmull-Rom spline.
    nPoints is the number of points to include in this curve segment.
    """
    # Convert the points to numpy so that we can do array multiplication
    P0, P1, P2, P3 = map(numpy.array, [P0, P1, P2, P3])

    # Calculate t0 to t4
    t0 = 0
    t1 = calculateT(t0, P0, P1)
    t2 = calculateT(t1, P1, P2)
    t3 = calculateT(t2, P2, P3)

    # Only calculate points between P1 and P2
    t = numpy.linspace(t1,t2,nPoints)

    # Reshape so that we can multiply by the points P0 to P3
    # and get a point for each value of t.
    t = t.reshape(len(t),1)

    A1 = (t1-t)/(t1-t0)*P0 + (t-t0)/(t1-t0)*P1
    A2 = (t2-t)/(t2-t1)*P1 + (t-t1)/(t2-t1)*P2
    A3 = (t3-t)/(t3-t2)*P2 + (t-t2)/(t3-t2)*P3
    print
    B1 = (t2-t)/(t2-t0)*A1 + (t-t0)/(t2-t0)*A2
    B2 = (t3-t)/(t3-t1)*A2 + (t-t1)/(t3-t1)*A3

    curve  = (t2-t)/(t2-t1)*B1 + (t-t1)/(t2-t1)*B2
    return curve

def calculateT(ti, Pi, Pj, alpha=0.5):
    xi, yi = Pi
    xj, yj = Pj
    return ( ( (xj-xi)**2 + (yj-yi)**2 )**0.5 )**alpha + ti

def interpolateCatmulRomeSpline(points):
    """
    Calculate Catmull Rom for a chain of points and return the combined curve.
    """
    pointsLen = len(points)

    # The curve curve(C) will contain an array of (x,y) points.
    curve = []
    for i in range(pointsLen-3):
        segment = interpolateCatmulRomeSegment(points[i], points[i+1], points[i+2], points[i+3])
        curve.extend(segment)

    return curve

def main():
    # Define a set of points for curve to go through
    points = [[0,1.5],[2,2],[3,1],[4,0.5],[5,1],[6,2],[7,3]]

    # Calculate the Catmull-Rom splines through the points
    c = interpolateCatmulRomeSpline(points)

    # Convert the Catmull-Rom curve points into x and y arrays and plot
    x,y = zip(*c)
    plt.plot(x,y)

    # Plot the control points
    px, py = zip(*points)
    plt.plot(px,py,'or')

    plt.show()

if __name__ == "__main__": main()


#!/usr/bin/env python
# coding: utf-8

# ### Understanding Probability & Statistics…

# In[1]:

#Import Common Libraries
import numpy as np
from matplotlib import pyplot as plt
plt.style.use('seaborn')
import seaborn as sns

# ### Bernoulli Distribution

# In[2]:

#Bernoulli Distribution
from scipy.stats import bernoulli
p = 0.7
x = np.arange(bernoulli.ppf(0.01, p), bernoulli.ppf(0.99, p)) #Percent Point Function (inverse of cdf — percentiles)

print("Mean              : ", bernoulli.stats(p, moments='m'))
print("Variance          : ", bernoulli.stats(p, moments='v'))
print("Prob. Mass Func.  : ", bernoulli.pmf(x, p).item())
print("Cum. Density Func.: ", bernoulli.cdf(x, p).item())

fig = plt.figure(figsize=(20,10))
plt.subplot(221)
plt.plot(x, bernoulli.pmf(x, p), 'ro', ms=8, label='PMF=(1-p)')
plt.plot(1 - x, 1 - bernoulli.pmf(x, p), 'go', ms=8, label='PMF=p')
plt.vlines(x, 0, bernoulli.pmf(x, p), colors='r', lw=5, alpha=0.5)
plt.vlines(1 - x, 0, 1 - bernoulli.pmf(x, p), colors='g', lw=5, alpha=0.5)
plt.xlabel("Sample Space of Bernoulli Distribution", fontsize=14)
plt.ylabel("PMF", fontsize=14)
plt.title("Probability Distribution of Bernoulli(p=0.7) Distribution", fontsize=16)
plt.xticks(np.arange(0, 2, 1))
plt.yticks(np.arange(0, 1.1, 0.1))
plt.legend(loc='best', shadow=True)

plt.subplot(222)
plt.hlines(0, -2, 0, colors='b', lw=5, alpha=0.5, linestyle='dashed')
plt.plot(0, 0, 'bo', ms=8)
plt.plot(x, bernoulli.cdf(x, p), 'ro', ms=8, label='CDF')
plt.hlines(0.3, 0, 1, colors='r', lw=5, alpha=0.5, linestyle='dashed')
plt.plot(1, 0.3, 'bo', ms=8)
plt.plot(1 - x, 1, 'ro', ms=8)
plt.hlines(1, 1, 2, colors='r', lw=5, alpha=0.5, linestyle='dashed')
plt.vlines(0, 0, 0.3, colors='b', lw=5, alpha=0.5)
plt.vlines(1, 0.3, 1, colors='b', lw=5, alpha=0.5)
plt.xlabel("Observation", fontsize=14)
plt.ylabel("CDF", fontsize=14)
plt.title("Cumulative Density Function of Bernoulli(p=0.7) Distribution", fontsize=16)
plt.xticks(np.arange(-2, 3, 1))
plt.yticks(np.arange(0, 1.1, 0.1))
plt.legend(loc='upper left', shadow=True)

plt.show()

# ### Binomial Distribution

# In[3]:

#Binomial Distribution
from scipy.stats import binom
n, p = 10, 0.4
x = np.arange(binom.ppf(0.01, n, p), binom.ppf(0.99, n, p)) #Percent Point Function (inverse of cdf — percentiles)

print("Mean              : ", binom.stats(n, p, moments='m'))
print("Variance          : ", binom.stats(n, p, moments='v'))
print("Prob. Mass Func.  : ", binom.pmf(x, n, p))
print("Cum. Density Func.: ", binom.cdf(x, n, p))

CDF = binom.cdf(x, n, p)

fig = plt.figure(figsize=(20,10))
plt.subplot(221)
plt.plot(x, binom.pmf(x, n, p), 'go', ms=8, label='PMF')
plt.vlines(x, 0, binom.pmf(x, n, p), colors='g', lw=5, alpha=0.5)
plt.xlabel("Sample Space of Binomial Distribution", fontsize=14)
plt.ylabel("PMF", fontsize=14)
plt.title("Probability Distribution of Binomial(n=10,p=0.4) Distribution", fontsize=16)
plt.xticks(np.arange(0, 8, 1))
plt.yticks(np.arange(0, 0.5, 0.1))
plt.legend(loc='best', shadow=True)

plt.subplot(222)
plt.hlines(0, -2, 1, colors='b', lw=5, alpha=0.5, linestyle='dashed')
plt.vlines(1, 0, CDF[0], colors='b', lw=5, alpha=0.5)
plt.plot(1, 0, 'bo', ms=8)
for i in range(len(CDF) - 1):
    plt.plot(i + 2, CDF[i], 'bo', ms=8)
    plt.vlines(i + 2, CDF[i], CDF[i + 1], colors='b', lw=5, alpha=0.5)
    plt.plot(i + 1, CDF[i], 'ro', ms=8)
    plt.hlines(CDF[i], i + 1, i + 2, colors='r', lw=5, alpha=0.5, linestyle='dashed')

plt.plot(len(CDF), CDF[len(CDF) - 1], 'ro', ms=8, label='CDF')
plt.hlines(CDF[len(CDF) - 1], len(CDF), len(CDF) + 1, colors='r', lw=5, alpha=0.5, linestyle='dashed')

plt.hlines(1, 7, 8, colors='r', lw=5, alpha=0.5, linestyle='dashed')
plt.xlabel("Observation", fontsize=14)
plt.ylabel("CDF", fontsize=14)
plt.title("Cumulative Density Function of Binomial(n=10,p=0.4) Distribution", fontsize=16)
plt.xticks(np.arange(-2, 9, 1))
plt.yticks(np.arange(0, 1.1, 0.1))
plt.legend(loc='upper left', shadow=True)

plt.show()

# ### Geometric Distribution

# In[4]:

#Geometric Distribution
from scipy.stats import geom
p = 0.6
x = np.arange(geom.ppf(0.01, p), geom.ppf(0.99, p)) #Percent Point Function (inverse of cdf — percentiles)

print("Mean              : ", geom.stats(p, moments='m'))
print("Variance          : ", geom.stats(p, moments='v'))
print("Prob. Mass Func.  : ", geom.pmf(x, p))
print("Cum. Density Func.: ", geom.cdf(x, p))

CDF = geom.cdf(x, p)

fig = plt.figure(figsize=(20,10))
plt.subplot(221)
plt.plot(x, geom.pmf(x, p), 'go', ms=8, label='PMF')
plt.vlines(x, 0, geom.pmf(x, p), colors='g', lw=5, alpha=0.5)
plt.xlabel("Sample Space of Geometric Distribution", fontsize=14)
plt.ylabel("PMF", fontsize=14)
plt.title("Probability Distribution of Geometric(p=0.6) Distribution", fontsize=16)
plt.xticks(np.arange(0, 6, 1))
plt.yticks(np.arange(0, 0.8, 0.1))
plt.legend(loc='best', shadow=True)

plt.subplot(222)
plt.hlines(0, -2, 1, colors='b', lw=5, alpha=0.5, linestyle='dashed')
plt.vlines(1, 0, CDF[0], colors='b', lw=5, alpha=0.5)
plt.plot(1, 0, 'bo', ms=8)
for i in range(len(CDF) - 1):
    plt.plot(i + 2, CDF[i], 'bo', ms=8)
    plt.vlines(i + 2, CDF[i], CDF[i + 1], colors='b', lw=5, alpha=0.5)
    plt.plot(i + 1, CDF[i], 'ro', ms=8)
    plt.hlines(CDF[i], i + 1, i + 2, colors='r', lw=5, alpha=0.5, linestyle='dashed')

plt.plot(len(CDF), CDF[len(CDF) - 1], 'ro', ms=8, label='CDF')
plt.hlines(CDF[len(CDF) - 1], len(CDF), len(CDF) + 1, colors='r', lw=5, alpha=0.5, linestyle='dashed')

plt.hlines(1, 5, 6, colors='r', lw=5, alpha=0.5, linestyle='dashed')
plt.xlabel("Observation", fontsize=14)
plt.ylabel("CDF", fontsize=14)
plt.title("Cumulative Density Function of Geometric(p=0.6) Distribution", fontsize=16)
plt.xticks(np.arange(-2, 7, 1))
plt.yticks(np.arange(0, 1.1, 0.1))
plt.legend(loc='upper left', shadow=True)

plt.show()

# ### Poisson Distribution

# In[5]:

#Poisson Distribution
from scipy.stats import poisson

loc, mu = 0, 10 # Mu is basically Lambda
x = np.arange(poisson.ppf(0.01, mu, loc), poisson.ppf(0.99, mu, loc)) #Percent Point Function (inverse of cdf — percentiles)

print("Mean              : ", poisson.stats(mu, loc, moments='m'))
print("Variance          : ", poisson.stats(mu, loc, moments='v'))
print("Prob. Dens. Func. : ", poisson.pmf(x, mu, loc))
print("Cum. Density Func.: ", poisson.cdf(x, mu, loc))

CDF = poisson.cdf(x, mu, loc)

fig = plt.figure(figsize=(20,10))
plt.subplot(221)
plt.plot(x, poisson.pmf(x, mu, loc), 'go', ms=8, label='PMF')
plt.vlines(x, 0, poisson.pmf(x, mu, loc), colors='g', lw=5, alpha=0.5)
plt.xlabel("Sample Space of Poisson Distribution", fontsize=14)
plt.ylabel("PMF", fontsize=14)
plt.title("Probability Distribution of Poisson(λ=10) Distribution", fontsize=16)
plt.xticks(np.arange(0, 20, 1))
plt.yticks(np.arange(0, 0.25, 0.05))
plt.legend(loc='best', shadow=True)

plt.subplot(222)
plt.hlines(0, -1, 3, colors='b', lw=5, alpha=0.5, linestyle='dashed')
plt.vlines(3, 0, CDF[0], colors='b', lw=5, alpha=0.5)
plt.plot(3, 0, 'bo', ms=8)
for i in range(len(CDF) - 1):
    plt.plot(i + 4, CDF[i], 'bo', ms=8)
    plt.vlines(i + 4, CDF[i], CDF[i + 1], colors='b', lw=5, alpha=0.5)
    plt.plot(i + 3, CDF[i], 'ro', ms=8)
    plt.hlines(CDF[i], i + 3, i + 4, colors='r', lw=5, alpha=0.5, linestyle='dashed')

plt.plot(len(CDF) + 2, CDF[len(CDF) - 1], 'ro', ms=8, label='CDF')
plt.hlines(CDF[len(CDF) - 1], len(CDF) + 2, len(CDF) + 3, colors='r', lw=5, alpha=0.5, linestyle='dashed')

plt.hlines(1, 17, 18, colors='r', lw=5, alpha=0.5, linestyle='dashed')
plt.xlabel("Observation", fontsize=14)
plt.ylabel("CDF", fontsize=14)
plt.title("Cumulative Density Function of Poisson(λ=10) Distribution", fontsize=16)
plt.xticks(np.arange(-1, 20, 1))
plt.yticks(np.arange(0, 1.1, 0.1))
plt.legend(loc='upper left', shadow=True)

plt.show()

# ### Uniform (Discrete) Distribution

# In[6]:

#Uniform (Discrete) Distribution
from scipy.stats import randint

low, high = 1, 10
x = np.arange(randint.ppf(0.01, low, high), randint.ppf(0.99, low, high)) #Percent Point Function (inverse of cdf — percentiles)

print("Mean              : ", randint.stats(low, high, moments='m'))
print("Variance          : ", randint.stats(low, high, moments='v'))
print("Prob. Mass Func.  : ", randint.pmf(x, low, high))
print("Cum. Density Func.: ", randint.cdf(x, low, high))

CDF = randint.cdf(x, low, high)

fig = plt.figure(figsize=(20,10))
plt.subplot(221)
plt.plot(x, randint.pmf(x, low, high), 'go', ms=8, label='PMF')
plt.vlines(x, 0, randint.pmf(x, low, high), colors='g', lw=5, alpha=0.5)
plt.xlabel("Sample Space of Discrete Uniform Distribution", fontsize=14)
plt.ylabel("PMF", fontsize=14)
plt.title("Probability Distribution of Discrete Uniform(a=1,b=10) Distribution", fontsize=16)
plt.xticks(np.arange(0, 10, 1))
plt.yticks(np.arange(0, 0.3, 0.05))
plt.legend(loc='best', shadow=True)

plt.subplot(222)
plt.hlines(0, -1, 1, colors='b', lw=5, alpha=0.5, linestyle='dashed')
plt.vlines(1, 0, CDF[0], colors='b', lw=5, alpha=0.5)
plt.plot(1, 0, 'bo', ms=8)
for i in range(len(CDF) - 1):
    plt.plot(i + 2, CDF[i], 'bo', ms=8)
    plt.vlines(i + 2, CDF[i], CDF[i + 1], colors='b', lw=5, alpha=0.5)
    plt.plot(i + 1, CDF[i], 'ro', ms=8)
    plt.hlines(CDF[i], i + 1, i + 2, colors='r', lw=5, alpha=0.5, linestyle='dashed')

plt.plot(len(CDF), CDF[len(CDF) - 1], 'ro', ms=8, label='CDF')
plt.hlines(CDF[len(CDF) - 1], len(CDF), len(CDF) + 1, colors='r', lw=5, alpha=0.5, linestyle='dashed')

plt.plot(9, CDF[-1], 'bo', ms=8)
plt.vlines(9, CDF[-1], 1, colors='b', lw=5, alpha=0.5)

plt.hlines(1, 9, 10, colors='r', lw=5, alpha=0.5, linestyle='dashed')
plt.xlabel("Observation", fontsize=14)
plt.ylabel("CDF", fontsize=14)
plt.title("Cumulative Density Function of Discrete Uniform(a=1,b=10) Distribution", fontsize=16)
plt.xticks(np.arange(-1, 12, 1))
plt.yticks(np.arange(0, 1.1, 0.1))
plt.legend(loc='upper left', shadow=True)

plt.show()

# ### Exponential Distribution

# In[7]:

#Exponential Distribution
from scipy.stats import expon

loc, scale = 0, 0.67 # Scale is 1/Lambda
x = np.linspace(expon.ppf(0.01, loc, scale), expon.ppf(0.99, loc, scale), 25) #Percent Point Function (inverse of cdf — percentiles)

print("Mean              : ", expon.stats(loc, scale, moments='m'))
print("Variance          : ", expon.stats(loc, scale, moments='v'))
print("Prob. Dens. Func. : ", expon.pdf(x, loc, scale))
print("Cum. Density Func.: ", expon.cdf(x, loc, scale))

CDF = expon.cdf(x, loc, scale)

fig = plt.figure(figsize=(20,10))
plt.subplot(221)
plt.plot(x, expon.pdf(x, loc, scale), 'g', ms=8, label='PDF')
plt.xlabel("Sample Space of Exponential Distribution", fontsize=14)
plt.ylabel("PDF", fontsize=14)
plt.title("Probability Distribution of Exponential(λ=1.5) Distribution", fontsize=16)
plt.xticks(np.arange(0, 5, 1))
plt.yticks(np.arange(0, 1.7, 0.1))
plt.legend(loc='best', shadow=True)

plt.subplot(222)
plt.plot(x, expon.cdf(x, loc, scale), 'r', ms=8, label='CDF')
plt.xlabel("Observation", fontsize=14)
plt.ylabel("CDF", fontsize=14)
plt.title("Cumulative Density Function of Exponential(λ=1.5) Distribution", fontsize=16)
plt.xticks(np.arange(-1, 5, 1))
plt.yticks(np.arange(0, 1.1, 0.1))
plt.legend(loc='upper left', shadow=True)

plt.show()

# ### Uniform (Continuous) Distribution

# In[8]:

#Uniform (Continuous) Distribution
from scipy.stats import uniform

loc, scale = 1, 10
x = np.linspace(uniform.ppf(0.01, loc, scale), uniform.ppf(0.99, loc, scale), 100) #Percent Point Function (inverse of cdf — percentiles)

print("Mean              : ", uniform.stats(loc, scale, moments='m'))
print("Variance          : ", uniform.stats(loc, scale, moments='v'))
print("Prob. Dens. Func. : ", uniform.pdf(x, loc, scale))
print("Cum. Density Func.: ", uniform.cdf(x, loc, scale))

CDF = randint.cdf(x, loc, scale)

fig = plt.figure(figsize=(20,10))
plt.subplot(221)
plt.plot(x, uniform.pdf(x, loc, scale), 'g', ms=8, label='PDF')
plt.vlines(loc, 0, 0.1, colors='g', lw=5, alpha=0.5, linestyle='dashed')
plt.vlines(scale + 1, 0, 0.1, colors='g', lw=5, alpha=0.5, linestyle='dashed')
plt.xlabel("Sample Space of Continuous Uniform Distribution", fontsize=14)
plt.ylabel("PDF", fontsize=14)
plt.title("Probability Distribution of Continuous Uniform(a=1,b=10) Distribution", fontsize=16)
plt.xticks(np.arange(0, 12, 1))
plt.yticks(np.arange(0, 0.3, 0.05))
plt.legend(loc='best', shadow=True)

plt.subplot(222)
plt.hlines(0, -1, 1, colors='b', lw=5, alpha=0.5, linestyle='dashed')
plt.plot(loc, 0, 'bo', ms=8)
plt.plot(scale + 1, CDF[-1], 'bo', ms=8)
x_lab = [loc, scale + 1]
y_lab = [0, CDF[-1]]
plt.plot(x_lab, y_lab, color='red', label='CDF')
plt.hlines(1, 11, 12, colors='r', lw=5, alpha=0.5, linestyle='dashed')
plt.vlines(11, 0, CDF[-1], colors='r', lw=5, alpha=0.5, linestyle='dashed')
plt.xlabel("Observation", fontsize=14)
plt.ylabel("CDF", fontsize=14)
plt.title("Cumulative Density Function of Continuous Uniform(a=1,b=10) Distribution", fontsize=16)
plt.xticks(np.arange(-1, 14, 1))
plt.yticks(np.arange(0, 1.1, 0.1))
plt.legend(loc='upper left', shadow=True)

plt.show()

# ### Gaussian (Normal) Distribution

# In[9]:

#Gaussian (Normal) Distribution
from scipy.stats import norm

loc, scale = 1, 2 # Mean and Variance
x = np.linspace(norm.ppf(0.01, loc, scale), norm.ppf(0.99, loc, scale), 25) #Percent Point Function (inverse of cdf — percentiles)

print("Mean              : ", norm.stats(loc, scale, moments='m'))
print("Variance          : ", norm.stats(loc, scale, moments='v'))
print("Prob. Dens. Func. : ", norm.pdf(x, loc, scale))
print("Cum. Density Func.: ", norm.cdf(x, loc, scale))

CDF = norm.cdf(x, loc, scale)

fig = plt.figure(figsize=(20,10))
plt.subplot(221)
plt.plot(x, norm.pdf(x, loc, scale), 'g', ms=8, label='PDF')
plt.xlabel("Sample Space of Gaussian Distribution", fontsize=14)
plt.ylabel("PDF", fontsize=14)
plt.title("Probability Distribution of Gaussian(µ=1,σ=2) Distribution", fontsize=16)
plt.xticks(np.arange(-5, 7, 1))
plt.yticks(np.arange(0, 0.30, 0.05))
plt.legend(loc='best', shadow=True)

plt.subplot(222)
plt.plot(x, norm.cdf(x, loc, scale), 'r', ms=8, label='CDF')
plt.xlabel("Observation", fontsize=14)
plt.ylabel("CDF", fontsize=14)
plt.title("Cumulative Density Function of Gaussian(µ=1,σ=2) Distribution", fontsize=16)
plt.xticks(np.arange(-5, 7, 1))
plt.yticks(np.arange(0, 1.1, 0.1))
plt.legend(loc='upper left', shadow=True)

plt.show()

# ### Covariance and Correlation

# In[10]:

#Covariance and Correlation
from scipy.stats import pearsonr

np.random.seed(1)
data1 = 20 * np.random.randn(20000) + 100
data2 = data1 + 10 * np.random.randn(20000) - 50

print("data1: Mean=%.3f; Variance=%.3f" % (np.mean(data1), np.var(data1)))
print("data2: Mean=%.3f; Variance=%.3f" % (np.mean(data2), np.var(data2)))
print("Covariance between data1 and data2: \n", np.cov(data1, data2))

corr, p = pearsonr(data1, data2)
print("Pearson Correlation Coefficient is %.3f with p-value %.3f" % (corr, p))

plt.scatter(data1, data2)
plt.title("Correlation", fontsize=16)
plt.xlabel("data1", fontsize=14)
plt.ylabel("data2", fontsize=14)

plt.show()

# ### Law of Large Numbers

# In[11]:

#Generate Samples from Standard Normal Distribution and Multiply Them by 5 and Add 10;
#So the Resulting Distribution of 'data' is Also Gaussian

np.random.seed(1)
data = 5 * np.random.randn(10) + 10
print("Average when n=10 is    :", np.mean(data))

data = 5 * np.random.randn(1000) + 10
print("Average when n=1000 is  :", np.mean(data))

data = 5 * np.random.randn(10000) + 10
print("Average when n=10000 is :", np.mean(data))

data = 5 * np.random.randn(100000) + 10
print("Average when n=100000 is:", np.mean(data))

# ### Central Limit Theorem

# In[12]:

#Generate a Sample of Dice Rolls
np.random.seed(1)
T_n = []
mu = (1 + 2 + 3 + 4 + 5 + 6) / 6
n = 1000
for i in range(n):
    data = np.random.randint(low=1, high=7, size=1000) #Returns discrete uniform integers
    X_n  = np.sum(data)/1000 #Sample average
    X_c  = X_n - mu #Subtract true mean mu
    X_f  = X_c * np.sqrt(n) #Multiply by sqrt(n)
    T_n.append(X_f)

fig = plt.figure(figsize=(20,10))
plt.hist(T_n, bins=100)
plt.title("Histogram of Sample Means from Dice Roll Simulations", fontsize=16)
plt.xticks(np.arange(-5, 6, 1))

plt.show()

# ### Confidence Interval

# In[13]:

#Confidence Interval
from scipy.stats import norm

np.random.seed(1)
data = 5 * np.random.randn(100) + 50
q_a_2 = norm.ppf(0.90)

low  = np.mean(data) - (q_a_2 * np.std(data)) / np.sqrt(len(data))
high = np.mean(data) + (q_a_2 * np.std(data)) / np.sqrt(len(data))

print("90 percent Confidence Interval is: %.3f, %.3f" % (low, high))

# ### Student's t-Distribution

# In[14]:

#Student's t-Distribution
from scipy.stats import t

sample_space = np.arange(-5, 5, 0.001)
dof = len(sample_space) - 1 #Number of independent variables
pdf = t.pdf(sample_space, dof)
cdf = t.cdf(sample_space, dof)

fig = plt.figure(figsize=(20,10))
plt.subplot(221)
plt.plot(sample_space, pdf, 'g', ms=8, label='PDF')
plt.xlabel("Sample Space of t-Distribution", fontsize=14)
plt.ylabel("PDF", fontsize=14)
plt.title("Probability Distribution of t-Distribution", fontsize=16)
plt.xticks(np.arange(-5, 7, 1))
plt.yticks(np.arange(0, 0.50, 0.1))
plt.legend(loc='best', shadow=True)

plt.subplot(222)
plt.plot(sample_space, cdf, 'r', ms=8, label='CDF')
plt.xlabel("Observation", fontsize=14)
plt.ylabel("CDF", fontsize=14)
plt.title("Cumulative Density Function of t-Distribution", fontsize=16)
plt.xticks(np.arange(-5, 7, 1))
plt.yticks(np.arange(0, 1.1, 0.1))
plt.legend(loc='upper left', shadow=True)

plt.show()

# ### Chi-Squared Distribution

# In[15]:

#Chi-Squared Distribution
from scipy.stats import chi2

sample_space = np.arange(0, 50, 0.01)
dof = 20 #Number of independent variables
pdf = chi2.pdf(sample_space, dof)
cdf = chi2.cdf(sample_space, dof)

fig = plt.figure(figsize=(20,10))
plt.subplot(221)
plt.plot(sample_space, pdf, 'g', ms=8, label='PDF')
plt.xlabel("Sample Space of Chi Square Distribution", fontsize=14)
plt.ylabel("PDF", fontsize=14)
plt.title("Probability Distribution of Chi Square Distribution", fontsize=16)
plt.xticks(np.arange(0, 60, 10))
plt.yticks(np.arange(0, 0.07, 0.01))
plt.legend(loc='best', shadow=True)

plt.subplot(222)
plt.plot(sample_space, cdf, 'r', ms=8, label='CDF')
plt.xlabel("Observation", fontsize=14)
plt.ylabel("CDF", fontsize=14)
plt.title("Cumulative Density Function of Chi Square Distribution", fontsize=16)
plt.xticks(np.arange(0, 60, 10))
plt.yticks(np.arange(0, 1.1, 0.1))
plt.legend(loc='upper left', shadow=True)

plt.show()

# ### Student's t-Test

# In[16]:

#Student's t-Test
from scipy.stats import ttest_ind

np.random.seed(1)
#np.random.randn generates Standard Normal data
data1 = 20 * np.random.randn(200) + 50 # mean=50, standard-deviation=20
data2 = 10 * np.random.randn(200) + 51 # mean=51, standard-deviation=10

stat, p = ttest_ind(data1, data2)
print('Test Statistic=%.3f, p=%.3f' % (stat, p))

alpha = 0.05 #Our desired confidence interval is 0.95

if p > alpha:
    print('Same distributions (fail to reject H0)')
else:
    print('Different distributions (reject H0)')

# ### Paired Student's t-Test

# In[17]:

#Paired Student's t-Test
from scipy.stats import ttest_rel

np.random.seed(1)
#np.random.randn generates Standard Normal data
data1 = 20 * np.random.randn(200) + 50 # mean=50, standard-deviation=20
data2 = 10 * np.random.randn(200) + 51 # mean=51, standard-deviation=10

stat, p = ttest_rel(data1, data2)
print('Test Statistic=%.3f, p=%.3f' % (stat, p))

alpha = 0.05 #Our desired confidence interval is 0.95

if p > alpha:
    print('Same distributions (fail to reject H0)')
else:
    print('Different distributions (reject H0)')

# ### Analysis of Variance (ANOVA)

# In[18]:

#ANOVA
from scipy.stats import f_oneway

np.random.seed(1)

data1 = np.array([6, 8, 4, 5, 3, 4])
data2 = np.array([8, 12, 9, 11, 6, 8])
data3 = np.array([13, 9, 11, 8, 7, 12])

stat, p = f_oneway(data1, data2, data3)
print('Test Statistic=%.3f, p=%.3f' % (stat, p))

alpha = 0.05 #Our desired confidence interval is 0.95

if p > alpha:
    print('Same distributions (fail to reject H0)')
else:
    print('Different distributions (reject H0)')

# Native Method
print("==> Native Method To Show the Calculation Method <==")
# Mean within each group
mean_1 = np.mean(data1)
mean_2 = np.mean(data2)
mean_3 = np.mean(data3)
n = len(data1)

# Overall mean
mean_o = (mean_1 + mean_2 + mean_3) / 3

# Calculate the "between-group" sum of squared differences
S_B = n * (mean_1 - mean_o) ** 2 + n * (mean_2 - mean_o) ** 2 + n * (mean_3 - mean_o) ** 2
dof_B = 3 - 1

# Between-group mean square value
MS_B = S_B / dof_B

# "within-group" sum of squares
# Centering the data in each group
data1_C = data1 - mean_1
data2_C = data2 - mean_2
data3_C = data3 - mean_3

# Within-group sum of squares
S_W = (np.sum(data1_C ** 2) + np.sum(data2_C ** 2) + np.sum(data3_C ** 2)).item()

# Within-group degrees of freedom
dof_W = 3 * (n - 1)

# Within-group mean square value
MS_W = S_W / dof_W

# F-Ratio
F = MS_B / MS_W

print("F Statistic:", F)

# ### Chi-Squared Test

# In[19]:

#Chi-Squared Test

#We will use this table from Wikipedia
#-------------------------------------
#           	A	B	C	D	Total
#White collar	90	60	104	95	349
#Blue collar	30	50	51	20	151
#No collar  	30	40	45	35	150
#Total      	150	150	200	150	650

from scipy.stats import chi2, chi2_contingency
np.random.seed(1)
# contingency table
observed = np.array([[90, 60, 104, 95],[30, 50, 51, 20], [30, 40, 45, 35]], dtype=np.float64)
print("Observed Frequencies: \n", observed)
stat, p, dof, expected = chi2_contingency(observed)

print('dof=%d' % dof)
print("Expected Frequencies: \n", expected)
# interpret test-statistic
prob = 0.95
critical = chi2.ppf(prob, dof)
print('probability=%.3f, critical=%.3f, stat=%.3f' % (prob, critical, stat))
if abs(stat) >= critical:
    print('Dependent (reject H0)')
else:
    print('Independent (fail to reject H0)')
# interpret p-value
alpha = 1.0 - prob
print('significance=%.3f, p=%.3f' % (alpha, p))
if p <= alpha:
    print('Dependent (reject H0)')
else:
    print('Independent (fail to reject H0)')

# Native Method
print("==> Native Method <==")
for i in range(observed.shape[0]):
    for j in range(observed.shape[1]):
        expected[i, j] = (np.sum(observed, axis=1))[i] * (np.sum(observed, axis=0))[j] / np.sum(observed)

print("Observed: \n", observed)
print("Expected: \n", expected)

stat = ((observed - expected) ** 2) / expected

dof = (observed.shape[0] - 1) * (observed.shape[1] - 1)

print("Statistic, dof: ", (np.sum(stat)).item(), ",", dof)

# ### KDE Plot

# In[20]:

#KDE Plot
np.random.seed(1)
data = np.random.random(5000) + 50

fig = plt.figure(figsize=(20,10))
sns.kdeplot(data, shade=True, color="r", legend=True)
plt.title("Kernel Density Plot of the Data", fontsize=16)
plt.xticks(np.arange(49.5, 51.5, 0.2))
plt.yticks(np.arange(0, 1.2, 0.1))

plt.show()

# ### Q-Q Plot

# In[21]:

#Q-Q Plot
from statsmodels.graphics.gofplots import qqplot
np.random.seed(1)
data = np.random.random(5000) + 50

qqplot(data, line='s')

plt.show()

# #### Shapiro-Wilk Test|D'Agostino's K2 Test|Anderson-Darling Test

# In[22]:

#Shapiro-Wilk Test
from scipy.stats import shapiro, normaltest, anderson
np.random.seed(1)
data = np.random.random(5000) + 50
alpha = 0.05

print("\n ==> Shapiro-Wilk Test <==")
stat, p = shapiro(data)
print('Statistic=%.3f, p=%.3f' % (stat, p))

if p > alpha:
    print('Sample looks Gaussian (fail to reject H0)')
else:
    print('Sample does not look Gaussian (reject H0)')

#D'Agostino's K2 Test
print("\n ==> D'Agostino's K2 Test <==")
stat, p = normaltest(data)
print('Statistic=%.3f, p=%.3f' % (stat, p))

if p > alpha:
    print('Sample looks Gaussian (fail to reject H0)')
else:
    print('Sample does not look Gaussian (reject H0)')

#Anderson-Darling Test
print("\n ==> Anderson-Darling Test <==")
result = anderson(data, dist='norm')

print('Statistic, Critical Values: %.3f' % result.statistic, result.critical_values)
for i in range(len(result.critical_values)):
    sl, cv = result.significance_level[i], result.critical_values[i]
    if result.statistic < result.critical_values[i]:
        print('%.3f: %.3f, data looks normal (fail to reject H0)' % (sl, cv))
    else:
        print('%.3f: %.3f, data does not look normal (reject H0)' % (sl, cv))

import numpy,from scipy,from statsmodels


#!/usr/bin/env python
# vim: set fileencoding=utf-8
""" This file is a Python translation of the MATLAB file prony.m

 Python version by RDL 12 Jan 2012
 Copyright notice from prony.m:
 copyright 1996, by M.H. Hayes.  For use with the book
 "Statistical Digital Signal Processing and Modeling"
 (John Wiley & Sons, 1996).
"""

from __future__ import print_function,division
import sys
import numpy as np

from convm import convm

def prony(x, p, q):
    """Model a signal using Prony's method

    Usage: [b,a,err] = prony(x,p,q)

    The input sequence x is modeled as the unit sample response of
    a filter having a system function of the form
        H(z) = B(z)/A(z)
    The polynomials B(z) and A(z) are formed from the vectors
        b=[b(0), b(1), ... b(q)]
        a=[1   , a(1), ... a(p)]
    The input q defines the number of zeros in the model
    and p defines the number of poles. The modeling error is
    returned in err.

    This comes from Hayes, p. 149, 153, etc

    """
    x = x[:]
    N = len(x)
    if p+q >= len(x):
        print('ERROR: model order too large')
        print ("p q len(x) " + str(p) + " " + str(q) + " " + str(len(x)))
        sys.exit(1)

    # This formulation uses eq. 4.50, p. 153
    # Set up the convolution matrices
    X = convm(x, p+1)
    Xq = X[q:N+p-1, 0:p]
    xq1 = -X[q+1:N+p,0]

    # Solve for denominator coefficients
    if p>0:
        a = np.linalg.lstsq(Xq, xq1)[0]
        a = np.insert(a, 0, 1) # a(0) is 1
    else:
        # all-zero model
        a = np.array(1)

    # Solve for the model error
    err = np.dot(x[q+1:N].conj().T,X[q+1:N, 0:p+1])
    err = np.dot(err, a)

    # Solve for numerator coefficients
    if q>0:
        # (This is the same as for Pad?)
        b = np.dot(X[0:q+1,0:p+1], a)
    else:
        # all-pole model
        # b(0) is x(0), but a better solution is to match energy
        b = np.sqrt(err)

    return (b,a)

#function [a,b,err] = prony(x,p,q)
#x   = x(:);
#N   = length(x);
#if p+q>=length(x), error('Model order too large'), end
#X   = convm(x,p+1);
#Xq  = X(q+1:N+p-1,1:p);
#a   = [1;-Xq\X(q+2:N+p,1)];
#b   = X(1:q+1,1:p+1)*a;
#err = x(q+2:N)'*X(q+2:N,1:p+1)*a;

def main():
    """Test driver"""
    # From pp. 149-150
    x = np.ones(21)
    p = q = 1
    print('x: {}\np: {}\nq: {}'.format(x,p,q))
    b,a,err = prony(x, p, q)
    print('a: {}\nb: {}\nerr: {}'.format(a,b,err))

    # From pp. 152-153
    # Note that these results don't match the book, but they do match the
    # MATLAB version. So I'm either setting things up wrong or this is an
    # errata in the book.
    p = q = 5
    nd = 5
    n = np.arange(11)
    i = np.sinc((n-nd)/2)/2
    b,a,err = prony(i, p, q)
    print('a: {}\nb: {}\nerr: {}'.format(a,b,err))

if __name__ == '__main__':
    main()


import numpy as np
import matplotlib.pyplot as plt
from matplotlib.patches import Rectangle, Arc
from matplotlib.collections import PatchCollection
import seaborn as sns
sns.set()

from check import *
from methods import compute_fibonacci_sequence

# -----------------------------------------------------------------------------
# Golden spiral approximation
# -----------------------------------------------------------------------------
def plot_golden_spiral_approximation(n):
    # check value of the argument
    if n < 1:
        raise ValueError("Argument n should be larger than 1")
    # compute F_i with i from 0 to n
    sequence = compute_fibonacci_sequence(n)

    # initialize variables
    x_ref, y_ref = 0, 0
    x_min, x_max, y_min, y_max = 0, 0, 0, 0
    angle = 180
    center = (1, 1)

    # initialize figure
    fig, ax = plt.subplots(1, figsize=(16, 16))
    rectangles = []
    for i, side in enumerate(sequence):
        if i == 0:
            previous_side = side
            continue
        # add a square of side F_i
        rectangles.append(Rectangle([x_ref, y_ref], side, side))
        # annotate the square if not too small
        if i > n - 8:
            ax.annotate(side, xy=(x_ref + 0.45 * side, y_ref + 0.45 * side), fontsize=10)
        # add the corresponding arc of the spiral
        ax.add_patch(
            Arc(center, 2 * side, 2 * side,
                angle=angle, theta1=0, theta2=90,
                edgecolor="black", antialiased=True)
            )

        # update variables
        angle += 90
        angle %= 360
        if i % 4 == 1:
            if i == 1:
                x_max, y_max = 1, 1
            else:
                y_min -= side
            x_ref += side
            center = (x_ref, y_ref + side + previous_side)
        elif i % 4 == 2:
            x_max += side
            x_ref -= previous_side
            y_ref += side
            center = (x_ref, y_ref)
        elif i % 4 == 3:
            y_max += side
            x_ref -= (side + previous_side)
            y_ref -= previous_side
            center = (x_ref + side + previous_side, y_ref)
        else:
            x_min -= side
            y_ref -= (side + previous_side)
            center = (x_ref + side + previous_side, y_ref + side + previous_side)
        previous_side = side

    # color the squares depending on their size and add them to the figure
    col = PatchCollection(rectangles, alpha=.95, edgecolor="black")
    col.set(array=np.asarray(range(n + 1)), cmap="Blues")
    ax.add_collection(col)

    # customize the axes
    ax.set_aspect("equal", "box")
    ax.set_xticks([])
    ax.set_yticks([])
    ax.set_xlim(x_min, x_max)
    ax.set_ylim(y_min, y_max)
    plt.tight_layout()
    plt.savefig("plots/golden_spiral_approximation.png")
    plt.show()

# -----------------------------------------------------------------------------
# Main
# -----------------------------------------------------------------------------
if __name__ == "__main__":

    n = 16
    check_fibonacci_argument(n)

    plot_golden_spiral_approximation(n)


import numpy as np

from ruleex.tree.ruletree import RuleTree
from ruleex.tree.rule import Leaf

def entropy(l0, l1):
    """
    computes entropy impurity for two list of labels l0 and l1
    :param l0: list of labels
    :param l1: list of labels
    :return: entropy impurity
    """
    _, p0 = np.unique(l0, return_counts=True)
    p0 = p0 / np.sum(p0)
    _, p1 = np.unique(l1, return_counts=True)
    p1 = p1 / np.sum(p1)
    return - np.sum(p0 * np.log2(p0)) - np.sum(p1 * np.log2(p1))

def gini(l0, l1):
    """
    computes gini impurity for two list of labels l0 and l1
    :param l0: list of labels
    :param l1: list of labels
    :return: gini impurity
    """
    _, p0 = np.unique(l0, return_counts=True)
    p0 = p0 / np.sum(p0)
    _, p1 = np.unique(l1, return_counts=True)
    p1 = p1 / np.sum(p1)
    return np.sum(p0 * (1 - p0)) + np.sum(p1 * (1 - p1))

def get_leaf(labels, class_num):
    """
    creates tŕee.Leaf object, i.e., subclass of tree.Rule
    class_set will be set to maximal occurring class in labels
    :param labels: a list of labels
    :param class_num: the number of classes
    :return: tree.Leaf with maximal occurring class as class_set
    """
    values, counts = np.unique(labels, return_counts=True)
    c = np.zeros(class_num)
    c[values] = counts
    max_label = np.argmax(c)
    node = Leaf({max_label})
    node.class_hits = c
    return node

def build_from_rules(rule_list, data, labels,
                     min_split_fraction: float = 0.98,
                     num_of_classes=None,
                     impurity=gini,
                     max_depth=None):
    """
    Builds tree.RuleTree from available splits from rule_list using data and its labels
    :param rule_list: a list of rules (subclass of tree.Rule)
    :param data: a list of inputs of the samples
    :param labels: a list of classificaitons for the samples
    :param min_split_fraction: minimal split fraction
    :param num_of_classes: number of classes
    :param impurity: impurity measure (function which takes two labels and return impurity value)
    :param max_depth: maximal depth of built tree
    :return: built RuleTree
    """
    if not num_of_classes:
        num_of_classes = np.max(labels)+1

    def build_from_rules_rek(rule_list, x, l, depth):
        if max_depth is not None and depth == max_depth:
            return get_leaf(l, num_of_classes)
        best_impurity = np.inf
        best_node_index = None
        best_node_mask = None
        for i, node in enumerate(rule_list):
            mask = node.eval_all(x)
            if np.all(mask) or not np.any(mask):
                continue # node do not provide any split for given data
            if np.mean(mask) > min_split_fraction or np.mean(mask) < 1 - min_split_fraction:
                continue
            node_imp = impurity(l[mask], l[~mask])
            if node_imp < best_impurity:
                best_impurity = node_imp
                best_node_index = i
                best_node_mask = mask
        if best_node_index is None:
            return get_leaf(l, num_of_classes)
        node = rule_list[best_node_index].copy()
        next_rule_list = rule_list.copy()
        next_rule_list.pop(best_node_index)

        node.true_branch = build_from_rules_rek(next_rule_list, x[best_node_mask], l[best_node_mask], depth + 1)
        node.false_branch = build_from_rules_rek(next_rule_list, x[~best_node_mask], l[~best_node_mask], depth + 1)
        return node

    rt = RuleTree(num_of_classes, data.shape[1])
    rt.root = build_from_rules_rek(rule_list, data, labels, 0)
    return rt


import numpy as np

def sigmoid(n):
    return 1.0 / (1.0 + np.exp(-n))

def sigmoid_derivative(n):
    return sigmoid(n) * (1 - sigmoid(n))


import numpy as np
from scipy.optimize import minimize

def direction(theta, phi):
    '''Return the direction vector of a cylinder defined
    by the spherical coordinates theta and phi.
    '''
    return np.array([np.cos(phi) * np.sin(theta), np.sin(phi) * np.sin(theta),
                     np.cos(theta)])

def projection_matrix(w):
    '''Return the projection matrix  of a direction w.'''
    return np.identity(3) - np.dot(np.reshape(w, (3,1)), np.reshape(w, (1, 3)))

def skew_matrix(w):
    '''Return the skew matrix of a direction w.'''
    return np.array([[0, -w[2], w[1]],
                     [w[2], 0, -w[0]],
                     [-w[1], w[0], 0]])

def calc_A(Ys):
    '''Return the matrix A from a list of Y vectors.'''
    return sum(np.dot(np.reshape(Y, (3,1)), np.reshape(Y, (1, 3)))
            for Y in Ys)

def calc_A_hat(A, S):
    '''Return the A_hat matrix of A given the skew matrix S'''
    return np.dot(S, np.dot(A, np.transpose(S)))

def preprocess_data(Xs_raw):
    '''Translate the center of mass (COM) of the data to the origin.
    Return the prossed data and the shift of the COM'''
    n = len(Xs_raw)
    Xs_raw_mean = sum(X for X in Xs_raw) / n

    return [X - Xs_raw_mean for X in Xs_raw], Xs_raw_mean

def G(w, Xs):
    '''Calculate the G function given a cylinder direction w and a
    list of data points Xs to be fitted.'''
    n = len(Xs)
    P = projection_matrix(w)
    Ys = [np.dot(P, X) for X in Xs]
    A = calc_A(Ys)
    A_hat = calc_A_hat(A, skew_matrix(w))

    u = sum(np.dot(Y, Y) for Y in Ys) / n
    v = np.dot(A_hat, sum(np.dot(Y, Y) * Y for Y in Ys)) / np.trace(np.dot(A_hat, A))

    return sum((np.dot(Y, Y) - u - 2 * np.dot(Y, v)) ** 2 for Y in Ys)

def C(w, Xs):
    '''Calculate the cylinder center given the cylinder direction and
    a list of data points.
    '''
    n = len(Xs)
    P = projection_matrix(w)
    Ys = [np.dot(P, X) for X in Xs]
    A = calc_A(Ys)
    A_hat = calc_A_hat(A, skew_matrix(w))

    return np.dot(A_hat, sum(np.dot(Y, Y) * Y for Y in Ys)) / np.trace(np.dot(A_hat, A))

def r(w, Xs):
    '''Calculate the radius given the cylinder direction and a list
    of data points.
    '''
    n = len(Xs)
    P = projection_matrix(w)
    c = C(w, Xs)

    return np.sqrt(sum(np.dot(c - X, np.dot(P, c - X)) for X in Xs) / n)

def fit(data, guess_angles=None):
    '''Fit a list of data points to a cylinder surface. The algorithm implemented
    here is from David Eberly's paper "Fitting 3D Data with a Cylinder" from
    https://www.geometrictools.com/Documentation/CylinderFitting.pdf

    Arguments:
        data - A list of 3D data points to be fitted.
        guess_angles[0] - Guess of the theta angle of the axis direction
        guess_angles[1] - Guess of the phi angle of the axis direction

    Return:
        Direction of the cylinder axis
        A point on the cylinder axis
        Radius of the cylinder
        Fitting error (G function)
    '''
    Xs, t = preprocess_data(data)

    # Set the start points

    start_points = [(0, 0), (np.pi / 2, 0), (np.pi / 2, np.pi / 2)]
    if guess_angles:
        start_points = guess_angles

    # Fit the cylinder from different start points

    best_fit = None
    best_score = float('inf')

    for sp in start_points:
        fitted = minimize(lambda x : G(direction(x[0], x[1]), Xs),
                    sp, method='Powell', tol=1e-6)

        if fitted.fun < best_score:
            best_score = fitted.fun
            best_fit = fitted

    w = direction(best_fit.x[0], best_fit.x[1])

    return w, C(w, Xs) + t, r(w, Xs), best_fit.fun


import numpy as np
from numpy import linalg as La
import matplotlib.pyplot as plt
import time

n =225
K = np.sqrt(n)
k = int(K)

# Here I set the Matrix

a = np.zeros((n,n))

for i in range(n):
    for j in range(n):
        if i == j:
            a[i,j] = -4
        elif i == j-3 or i ==j+3:
            a[i,j] = 1
        elif ((i+1) % 3 != 0 and i == j-1) or ((i+1) % 3 != 1 and i == j+1): # (i+1) because in Python we start from 0
            a[i,j] = 1

print('The coefficient Matrix is:')
print(a)
print('\n')

b = np.zeros(n)

for i in range(k):
    if i < k-1: # (k-1) because in Python we start from 0
        b[i] = -50
    else:
        b[i] = -150

for i in range(k,n-k):
    if (i+1)%k != 0: # (i+1) because in Python we start from 0
        b[i] = 0
    else:
        b[i] = -100

for i in range(n-k,n):
    if i < n-1: # (k-1) because in Python we start from 0
        b[i] = -50
    else:
        b[i] = -150

print('The result Matrix is:')
print(b)
print('\n')


import numpy as np

def QRDecomposition(A):
	n = np.shape(A)[0] #pegando o tamanho das linhas de A
	m = np.shape(A)[1] #pegando o tamanho das colunas de A
	Q = np.zeros((n,m)) #declarando a matriz Q
	R = np.zeros((m,m)) #declarando a matriz R

	for j in range(0, m):
		A_column = A[:, j] #pegando as colunas da matriz A
		V = np.zeros(n) #declarando o vetor V
		V = A_column #V igual a coluna j de A
		for i in range (0, j):
			R[i,j] = Q[:,i].dot(A_column) #fazendo o calculo do R[i,j] = coluna i de Q * coluna j de A ( i != j)
			V -= (Q[:,i].dot(A_column))*Q[:,i] #fazendo o calculo de V
		R[j,j] = np.linalg.norm(V) # R[j,j] = norma da coluna j de A (i == j)
		Q[:,j] = V/np.linalg.norm(V) #normalizando V e atribuindo a coluna j de Q

	return Q, R

def FrancisMethod(A, error):

	m = np.inf #declarando m como infinito
	Autovetores = np.eye(np.shape(A)[0]) #declarando a matriz de autovetores como uma matriz indentidade
	while m > error:
		(Q, R) = QRDecomposition(A) #fazendo a decomposicao QR de A
		A = R.dot(Q) #fazendo A = R*Q
		Autovetores = Autovetores.dot(Q) #pegando os autovetores V = Q1 * Q2 * Q3 * ...
		m = -np.inf #declarando m como menos infinito
		#pegando o maior valor da matriz A fora da diagonal principal
		for i in range (0, np.shape(A)[0]):
			for j in range (0, np.shape(A)[1]):
				if (i != j) and (m < np.abs(A[i,j])):
					m = np.abs(A[i,j])

	Autovalores = np.diag(A) #pegando a diagonal principal da matriz A

	return Autovalores, Autovetores #retornando o resultado

'''
A = np.array([[2,0,1], [0,1,0], [1,0,1]], dtype = 'double')
(Aut, Vet) = FrancisMethod(A, 0.01)
print(Aut)
'''
A = np.array([[2,1,1],[1,3,5], [1,5,14]], dtype = 'double')
(Aut, Vet) = FrancisMethod(A, 0.000001)
print('{}\n\n{}'.format(Aut, Vet))


import numpy as np
import scipy.spatial.distance

def compute_laplacian(adjacency: np.ndarray, normalize: bool):
    """ Return:
        L (n x n ndarray): combinatorial or symmetric normalized Laplacian.
    """
    D = np.diag(np.sum(adjacency, 1)) # Degree matrix
    combinatorial = D - adjacency
    if normalize:
        D_norm = np.diag(np.clip(np.sum(adjacency, 1), 1, None)**(-1/2))
        return D_norm @ combinatorial @ D_norm
    else:
        return combinatorial

def spectral_decomposition(laplacian: np.ndarray):
    """ Return:
        lamb (np.array): eigenvalues of the Laplacian
        U (np.ndarray): corresponding eigenvectors.
    """
    return np.linalg.eigh(laplacian)

def compute_number_connected_components(lamb: np.array, threshold: float):
    """ lamb: array of eigenvalues of a Laplacian
        Return:
        n_components (int): number of connected components.
    """
    return np.count_nonzero(lamb <= threshold)


#%% Modules

from data import get_mnist

import matplotlib
import matplotlib.pyplot as plt
import numpy as np

#%% Train Three Layer (Input-Hidden-Output) Neural Network

# Nomenclature,
#
# w -> connection/link weights
# b -> bias values
# i -> input layer
# h -> hidden layer
# o -> output layer
# l -> labels

images, labels = get_mnist()  # 60000-by-784, 60000-by-1, respectively.

# Initialize input-to-hidden layer weights and set bias values to zero.
np.random.seed(0)
w_i_h = np.random.uniform( -0.5, 0.5, (20, 784) )  # 20-by-784
b_i_h = np.zeros((20, 1))  # 20-by-1

# Initialize hidden-to-output layer weights and set bias values to zero.
np.random.seed(0)
w_h_o = np.random.uniform( -0.5, 0.5, (10, 20) )  # 10-by-20
b_h_o = np.zeros((10, 1))  # 20-by-1

print( "\n\n" )

learn_rate = 0.01;  epochs = 50

nr_correct = 0;  accuracy = list()

for epoch in range(epochs):  # From 0 to (epochs - 1)

    for img, l in zip(images, labels):  # Group a row from images and labels

        img.shape += (1,)  # Make the second dimension of the 1-Tuple (i.e. 1) explicit.
        l.shape += (1,)  # Make the second dimension of the 1-Tuple (i.e. 1) explicit.

        # Forward Propagation -> Input Layer to Hidden Layer
        h_pre = b_i_h  +  w_i_h @ img
        h = 1 / (1 + np.exp(-h_pre))  # Activation

        # Forward Propagation -> Hidden Layer to Output Layer
        o_pre = b_h_o  +  w_h_o @ h
        o = 1 / (1 + np.exp(-o_pre))  # Activation

        # Compute Cost/Error/Performance Metric
        e = 1 / len(o) * np.sum( (o - l) ** 2, axis=0 )
        #
        # e -> 1-by-1, a scalar -> Average squared difference between predicted and expected output.
        # o, l -> 10-by-1#
        #
        nr_correct = nr_correct + int( np.argmax(o) == np.argmax(l) )

        # Backpropagation -> Output Layer to Hidden Layer -> The cost function derivative.
        delta_o = o - l  # 10-by-1
        #
        w_h_o = w_h_o  +  -1.0*learn_rate*delta_o @ np.transpose(h)  # 10-by-20 + (1-by-1 * 10-by-1 * 1-by-20) -> 10-by-20
        b_h_o = b_h_o  +  -1.0*learn_rate*delta_o  # 10-by-1 + (1-by-1 * 10-by-1) -> 10-by-1

        # Backpropagation -> Hidden Layer to Input Layer -> The activation function derivative.
        delta_h = np.transpose(w_h_o) @ delta_o * (h * (1 - h))  # 20-by-10 @ 10-by-1 * 20-by-1 -> 20-by-1 (@ -> matrix multiplication; * -> element-by-element multiplication)
        #
        w_i_h = w_i_h  +  -1.0*learn_rate*delta_h @ np.transpose(img)  # 20-by-784 + (1-by-1 * 20-by-1 @ 1-by-784) -> 20-by-784
        b_i_h = b_i_h  +  -1.0*learn_rate*delta_h  # 20-by-1 + (1-by-1 *20-by-1) -> 20-by-1

    # Show accuracy for this epoch.
    print( f"Epoch {epoch + 1} of {epochs} -> Accuracy: {round((nr_correct / images.shape[0]) * 100, 2)}% ({nr_correct}/{images.shape[0]})" )

    accuracy.append( nr_correct / images.shape[0] * 100 )
    nr_correct = 0  # Reset counter for next epoch.

#%% Plot Accuracy Versus Epoch

epochValues = list(range(epochs));

fig, ax = plt.subplots()
ax.plot( [x+1 for x in epochValues], accuracy)

ax.set(xlabel='Training Epoch', ylabel='Accuracy (%)', title='Accuracy Versus Training Epoch')
ax.grid()
fig.savefig('50 Epochs of Training MNIST February 14, 2021.png')
plt.show()

# https://matplotlib.org/3.1.1/gallery/lines_bars_and_markers/simple_plot.html

#%% Predict Selected Digit

# index = int(input("Enter a number (0 - 59999): "))
index = 50  # Digit 3

img = images[index]
plt.imshow( img.reshape(28, 28), cmap="Blues" )

img.shape += (1,)  # Make the second dimension of the 1-Tuple (i.e. 1) explicit.

# Forward Popagation -> Input Layer to Hidden Layer
h_pre = b_i_h  +  w_i_h @ img.reshape(784, 1)
#
# b_i_h -> Array of type float64, 20-by-1
# w_i_h -> Array of type float64, 20-by-784
# img -> Array of float32, 784-by-1
#
# 20-by-1 + 20-by-784 * 784-by-1 -> 20-by-1 + 20-by-1 -> 20-by-1
#
h = 1 / ( 1 + np.exp(-h_pre) )  # Apply activation function.

# Forward Propagation -> Hidden Layer to Output Layer
o_pre = b_h_o  +  w_h_o @ h
#
# b_h_o -> Array of type float64, 10-by-1
# w_h_o -> Array of type float64, 10-by-20
# h -> Array of type float64, 20-by-1
#
# 10-by-1 + 10-by-20 * 20-by-1 -> 10-by-1 + 10-by-1 -> 10-by-1
#
o = 1 / (1 + np.exp(-o_pre))  # Apply activation function.

plt.title( f"Predicted Digit: {o.argmax()}" ); plt.show()


import numpy as np
import matplotlib
import matplotlib.pyplot as plt
from scipy import stats

matplotlib.rc('font', size=32)
lw = 6
s = 200

def kernel_comparison():
    fig = plt.figure()

    loc = 0.0
    scale = 3.0

    x = np.linspace(-20, 20, 1000)
    plt.plot(x, stats.norm.pdf(x, loc=loc, scale=scale), lw=lw, color='dimgrey', label='Gaussian')
    plt.plot(x, stats.cauchy.pdf(x, loc=loc, scale=scale), lw=lw, color='black', label='Cauchy')
    plt.plot(x, stats.uniform.pdf(x, loc=loc-scale, scale=loc+2*scale), lw=lw, color='crimson', label='Uniform')

    plt.legend()
    plt.title('Kernel comparison')
    plt.xticks([loc], ['y'])
    plt.yticks([])
    plt.show()

def kernel_tuning():
    y = 5.0
    u = np.array([-1.0, 0.0, 2.0, 6.0, 8.0, 10.0])
    u_alpha = 6.0 + 1

    p = 0.95
    eps = abs(u_alpha - y) / stats.norm.ppf((1+p)/2)

    fig = plt.figure()

    x = np.linspace(-5, 15, 1000)
    plt.plot(x, stats.norm.pdf(x, loc=y, scale=eps), lw=lw, color='black')

    plt.vlines(y, ymin=0.0, ymax=stats.norm.pdf(y, loc=y, scale=eps), lw=lw, color='crimson')
    plt.vlines(u_alpha, ymin=0.0, ymax=stats.norm.pdf(u_alpha, loc=y, scale=eps), lw=lw, color='crimson')
    plt.scatter(u, np.zeros(u.size), s=s, color='dimgrey', zorder=1000)
    plt.scatter(u_alpha, 0, s=s, color='crimson', zorder=1000)
    plt.scatter(y, 0, s=s, color='crimson', zorder=1000)

    shaded_x = x[(x >= y) & (x <= u_alpha)]
    plt.fill_between(shaded_x, stats.norm.pdf(shaded_x, loc=y, scale=eps), color='black', alpha=0.4, linewidth=0)

    plt.xticks([y, u_alpha], ['$y_t$', r'$u_t^{[\alpha]}$'])
    plt.yticks([])
    plt.title('Kernel tuning')
    plt.show()

if __name__ == '__main__':
    #kernel_comparison()
    kernel_tuning()


import numpy as np
import matplotlib.pyplot as plt
#Solution of tridiagonal amtrix from 2.6 Eperson "An Introduction to Numerical Methods and Analysis"
#Of the form Ax=f
#l is lower diagonal, d is diagonal and u is the upper diagonal

#n=4
#l=np.array([np.NaN,1.0,1.0,1.0])
#d=np.array([4.0,4.0,4.0,4.0])
#u=np.array([1.0,1.0,1.0,np.NaN])
#f=np.array([6.0,12.0,18.0,19.0])
#x=np.array([0.0,0.0,0.0,0.0])
#
##Elimination stage
#for i in range(1,(n)):
#    d[i] = d[i] - u[i-1] *l[i] / d[i-1]
#    f[i] = f[i] - f[i-1] *l[i] / d[i-1]
#
##backsolve stage
#x[(n-1)]= f[(n-1)]/d[(n-1)]
#for i in range((n-2),-1,-1):
#    x[i] = (f[i] - u[i] * x[i+1])/d[i]
#
#print(x)

#Calculate true values and plot
x_true=np.linspace(0,1)
y_true=(x_true)-np.sin(np.pi*x_true)

plt.plot(x_true,y_true,'b',label='True')

#Edit for given function
def f(x):
    return (np.pi**2.0)*np.sin(np.pi*x)
#Setting up the problem
alpha=0
beta=1
for m in [3,9,19]:
    h=1/(m+1)
    u=(1/(h**2))*np.ones(m-1)
    d=(1/(h**2))*(-2.0)*np.ones(m)
    l=(1/(h**2))*np.ones(m-1)
    F=np.empty(m)
    U=np.empty(m)
    x=np.arange(0,(1+h),h)

    #Creating F
    F[0]=f(x[1]) - ( alpha/(h **2) )
    F[m-1]=f(x[m-1]) - ( beta/(h **2) )
    for i in range(1,m-1):
        F[i]=f(x[i+1])

    #Solving the tridiagonal problem
    #Elimination stage
    for i in range(1,(m)):
        d[i] = d[i] - u[i-1] *l[i-1] / d[i-1]
        F[i] = F[i] - F[i-1] *l[i-1] / d[i-1]

    #backsolve stage
    U[(m-1)]= F[(m-1)]/d[(m-1)]
    for i in range((m-2),-1,-1):
        U[i] = (F[i] - u[i] * U[i+1])/d[i]

    #Insert end values
    U=np.insert(U,0,alpha)
    U=np.insert(U,m+1,beta)
    #Print answer
    #print(U)
    x_approx=np.linspace(0,1,num=m+2)

    plt.plot(x_approx,U,'o-',label=f'{h}',linewidth=0.9)
plt.legend(loc=0)
plt.xlabel('x')
plt.ylabel('u(x)')
plt.show()


import numpy as np

from zero2ml.supervised_learning._base import BaseModel
from zero2ml.utils.activation_functions import Sigmoid
from zero2ml.utils.evaluation_metrics import CrossEntropyLoss

class LogisticRegression(BaseModel):
    """
    Logistic Regression classifier.

    Parameters
    ----------
    learning_rate: scalar
        Learning rate.

    Attributes
    ----------
    X: array_like
        Training features.
    y: array_like
        Training labels.
    samples_num: scalar
        Number of training samples.
    features_num: scalar
        Number of features.
    self.W: array_like
        Weights.
    self.b: array_like
        Intercept term.
    self.training_loss: list
        List containing training loss for each iteration of model training.
    """
    def __init__(self, learning_rate=0.01):

        # Specify model type
        self.model_type = "classifier"

        # Training data
        self.X = None

        # Training targets
        self.y = None

        # Number of training samples
        self.samples_num = None

        # Number of features
        self.features_num = None

        # Weights and intercept
        self.W = None
        self.b = None

        # Sigmoid function
        self.__sigmoid = Sigmoid()

        # Learning rate
        self.learning_rate = learning_rate

        # Loss and loss function
        self.training_loss = None
        self.__loss_function = CrossEntropyLoss()

    def fit(self, X, y, iterations=5000):
        """
        Fit the logistic regression model with given training feature inputs.

        Parameters
        ----------
        X: array_like
            Training features.
        y: array_like
            Training labels.
        iterations: scalar
            Number of iterations for gradient descent to converge.
        """
        # Save X and y into model object
        self.X = X.copy()
        self.y = y.copy()

        # Calculate number of samples and features
        self.samples_num, self.features_num = self.X.shape

        # Initiate weights and intercept
        self.W = np.zeros(self.features_num)
        self.b = 0

        # Initiate loss
        self.training_loss = []

        # Gradient descent
        for iteration in range(iterations):

            # Make prediction
            prob = self.__sigmoid( np.dot(self.X, self.W) + self.b)

            # Calculate loss and save to training loss list
            self.training_loss.append( self.__loss_function(prob, self.y) )

            # Calculate gradients
            dW = 1 / self.samples_num * np.dot(self.X.T, prob - self.y)
            db = 1 / self.samples_num * np.sum(prob - self.y)

            # Update weights and intercept
            self.W -= self.learning_rate * dW
            self.b -= self.learning_rate * db

    def predict(self, X):
        """
        Predict the label(s) with given feature inputs.

        Parameters
        ----------
        X: array_like
            Training features.

        Returns
        -------
        pred: array_like
            Label predictions. Could be 0 or 1.
        """
        # Calculate probabilities
        prob = self.__sigmoid( np.dot(X, self.W) + self.b )

        # Calculate actual predictions
        pred = (prob > 0.5).astype(int)

        return pred


import numpy as np
import matplotlib.pyplot as plt

omega = 2
P = 2*np.pi/omega
dt = P/20
T = 40*P
T = P
N_t = int(round(T/dt))
t = np.linspace(0, N_t*dt, N_t+1)
print('N_t:', N_t)

u = np.zeros(N_t+1)
v = np.zeros(N_t+1)

# Initial condition
X_0 = 2
u[0] = X_0
v[0] = 0

# Step equations forward in time
for n in range(N_t):
    v[n+1] = v[n] - dt*omega**2*u[n]
    u[n+1] = u[n] + dt*v[n+1]

# Plot the last four periods to illustrate the accuracy
# in long time simulations
N4l = int(round(4*P/dt))  # No of intervals to be plotted
fig = plt.figure()
l1, l2 = plt.plot(t[-N4l:], u[-N4l:], 'b-',
                  t[-N4l:], X_0*np.cos(omega*t)[-N4l:], 'r--')
fig.legend((l1, l2), ('numerical', 'exact'), 'upper left')
plt.xlabel('t')
plt.savefig('tmp.pdf'); plt.savefig('tmp.png')
plt.show()
print('{:.16f} {:.16f}'.format(u[-1], v[-1]))


import numpy as np
import math

'''
Different permutation methods using python
'''

# Using raw mathematics
def perm(n, r):
  num = denom = 1

  if n >= 1:
    for i in range (1,n+1):
      num=num*i

  if r < n:
    for i in range (1,n-r+1):
      denom=denom*i

  return num/denom

# Using factorial
def perm_fact(n, r):
  return math.factorial(n)/math.factorial(n-r)

# Using Stirling Approximation
def perm_stir(n,r):
  def stirling(n):
      return math.sqrt(2*math.pi*n)*(n/math.e)**n
  return stirling(n)/stirling(n-r)

# Using list
def perm_list(n, r):
  num_list = []
  denom_list = []

  if n >= 1:
    for i in range (1,n+1):
      num_list.append(i)

  if r < n:
    for i in range (1,n-r+1):
      denom_list.append(i)

  new_num = [x for x in num_list if x not in denom_list]

  return np.prod(new_num)

# Using list (shortened)
def perm_list_v2(n, r):
  num_list = []
  denom_list = []

  if n >= 1:
    for i in range (n-r+1,n+1):
      num_list.append(i)

  return np.prod(num_list)


import scipy as sp
import math

#Hypothesis Testing Variables
Null_Mean = 3
Sample_Mean = 3.38
Sample_Variance = 1.769
Sample_Standard_Deviation = 5.237
Sample_Size = 50

#Finds the T value for a specific significance level for the data
def FindTValue(x, u, s, n):
    t = (x-u)/(s / math.sqrt(n))
    return abs(t)

#T-Test using sample variance
print(FindTValue(Sample_Mean, Null_Mean, Sample_Variance, Sample_Size))

#T-Test using sample deviation
print(FindTValue(Sample_Mean, Null_Mean, Sample_Standard_Deviation, Sample_Size))


import numpy as np

def pei_fib(n, ta, tb, a, b):
    if n == 0:
        return tb
    if n & 1:
        return pei_fib(n / 2, a * ta + b * tb, b * ta + tb * (a - b), a * a + b * b, b * (2 * a - b))
    return pei_fib(n / 2, ta, tb, a * a + b * b, b * (2 * a - b))

# if number of bits is odd, we multiply a
def q_pow(a, b):
    t = 1
    while b > 0:
        if b & 1:
            t *= a
        a = a * a
        b /= 2
    return t

def yche_pow(a, b):
    if b == 0:
        return 1
    if b == 1:
        return a
    else:
        return yche_pow(a, b / 2) * yche_pow(a, (b + 1) / 2)

def yche_matrix_pow(a, b):
    if b == 1:
        return a
    else:
        return np.dot(yche_pow(a, b / 2), yche_pow(a, (b + 1) / 2))

def fib(n):
    primitive_matrix = np.matrix([[1, 1], [1, 0]])
    if n < 2:
        return n
    else:
        return yche_matrix_pow(primitive_matrix, n - 1).item(0, 0)

def naive_fib(n):
    if n < 2:
        return n
    else:
        return naive_fib(n - 1) + naive_fib(n - 2)

def fib_mat_pow(pow_num, idx):
    if pow_num < 2:
        return 1
    else:
        a = fib_mat_pow(pow_num / 2, 0)
        b = fib_mat_pow(pow_num / 2, 1)
        a_plus = fib_mat_pow((pow_num + 1) / 2, 0)
        b_plus = fib_mat_pow((pow_num + 1) / 2, 1)
        if idx < 1:
            return a * a_plus + b * b_plus
        else:
            return a * b_plus + b * (a_plus - b_plus)

def fast_fib(idx):
    if idx < 2:
        return idx
    else:
        return fib_mat_pow(idx - 1, 0)

print q_pow(2, 4)
print yche_pow(2, 4)
print yche_pow(2, 5)
print yche_pow(2, 0)

print fib(10)
print naive_fib(10)
print 'fib demo:'
# for i in range(12):
#     print fast_fib(i),
# print fast_fib(1000)pei_fib
#
# print fib(1000000)

# print pei_fib(1000000,1,1,1,1)
print yche_matrix_pow([[1,1],[1,0]],4)


import numpy as np
from scipy import stats

def kmeans(X, n, n_iter=100, centroids=None):
    """
    simple k-means implemenmtation

    Parameters:
    -----------
    X: array (n_samples, n_dim)
        the data
    n : int
        number of clusters
    n_iter : int
        number of iterations
    centroids : array (optional)

    Returns:
    --------

    centroids : array
    labels : array
    """

    def assign(centroids):
        dist = np.average(np.square(X[:, :, np.newaxis] - centroids.T), axis=1)
        return np.argmin(dist, axis=1)

    def get_centroids(labels):
        centroids = []
        for i in range(np.max(labels)+1):
            centroids.append(np.average(X[labels==i], axis=0))
        return np.stack(centroids)

    if centroids is None:
        # start with random seed
        centroids = np.random.rand(n, X.shape[1])
        # scale to roughly match the data
        centroids *= np.std(X, axis=0)
        centroids += np.mean(X, axis=0)

    # iterate
    for i in range(n_iter):
        labels = assign(centroids)
        centroids = get_centroids(labels)

    return centroids, labels

def gmm(X, n, n_iter=100, mus=None, covs=None):
    """
    simple GMM implemenmtation (2d right now)

    Parameters:
    -----------
    X: array (n_samples, n_dim)
        the data
    n : int
        number of gaussians
    n_iter : int
        number of iterations
    mus : array (optional)
    covs = array (optional)

    Returns:
    --------
    mus : array
        mean values
    covs : array
        covariances
    exps : array
        probabilities
    """

    def E(mus, covs):

        probs = []
        exps = []

        for i in range(n):
            probs.append(stats.multivariate_normal(mus[i], covs[i]).pdf(X))

        probs = np.array(probs)
        tot_prob = np.sum(probs, axis=0)

        for i in range(n):
            exps.append(probs[i] / tot_prob)

        return np.nan_to_num(exps)

    def M(exps):

        mus = []
        covs = []

        for i in range(n):
            mus.append(np.average(X, weights=exps[i], axis=0))
            covs.append(np.cov(X.T, aweights=exps[i]))

        return mus, covs

    if mus is None:
        std = np.std(X, axis=0)
        mus = np.random.rand(n, X.shape[1])
        # scale to roughly match the data
        mus *= std
        mus += np.mean(X, axis=0)
        # some random covariances
        covs = np.random.rand(n, X.shape[1], X.shape[1])
        covs = covs @ np.swapaxes(covs, 1, 2)

    for i in range(n_iter):
        exps = E(mus, covs)
        mus, covs = M(exps)

    return mus, covs, np.array(E(mus, covs)).T