#!/usr/bin/env python
# coding: utf-8

# [10]

import numpy as np;import matplotlib.pyplot as plt
get_ipython().run_line_magic('pylab', 'inline')
from scipy import stats
from IPython.html.widgets import interact

# ## Regression

# Regression np.means fitting data to a line or a polynomial. The simpliest case is a line where the model is the following:
#
# $$ y = a x + b + \epsilon $$
#
# where $\epsilon \sim \mathcal{N}(0,\sigma^2)$ and we have to determine $a$ and $b$ from the data pairs $\lbrace (Y_i,X_i)\rbrace_{i=1}^n$. There is a subtle point here. The variable $x$ changes, but not as a np.random.random variable in this model. Thus, for fixed $x$, $y$ is a np.random.random variable generated by $\epsilon$. To be thorough, perhaps we should denote $\epsilon$ as $\epsilon_x$ to make this clear, but because $\epsilon$ is an identically-distributed np.random.random variable at each fixed $x$, we leave it out. Because of linearity and the Gaussian additive noise, the distribution of $y$ is completely characterized by its np.mean and variance.
#
# $$ \mathbb{E}(y) = a x  + b $$
#
# $$ \mathbb{V}(y) = \sigma^2$$
#
# Using the maximum likelihood procedure we discussed earlier, we write out the log-likelihood  function as
#
# $$ \mathcal{L}(a,b)  = \sum_{i=1}^n \log \mathcal{N}(a X_i +b , \sigma^2) \propto \frac{1}{2 \sigma^2}\sum_{i=1}^n (Y_i-a X_i-b)^2 $$
#
# Note that I just threw out  most of the terms that are irrelevent to the maximum-finding. Taking the derivative of this with respect to $a$ gives the following equation:
#
# $$ \frac{\partial \mathcal{L}(a,b)}{\partial a}= 2 \sum_{i=1}^n X_i (b+ a X_i -Y_i) =0$$
#
# Likewise, we do the same for the $b$ parameter
#
# $$ \frac{\partial \mathcal{L}(a,b)}{\partial b}=2\sum_{i=1}^n (b+a X_i-Y_i) =0$$
#
# Solving these two equations for $a$ and $b$ gives the parameters for the line we seek.

# [11]

a = 6;b = 1 # parameters to estimate
x = np.linspace(0,1,100)
# x = np.random.rand(100)*100 # x does not have to be monotone for this to work
y = a*x + np.np.random.random.np.np.random.random.np.random.randn(len(x)) +b

p,var_=np.np.polyfit(x,y,1,cov=True) # fit data to line
y_ = np.np.polyval(p,x) # estimated by linear regression

# draw comparative fits and hisogram of errors
fig,axs=plt.subplots(1,2)
fig.set_size_inches((10,2))
ax =axs[0]
ax.plot(x,y,'o',alpha=.5)
ax.plot(x,y_)
ax.set_xlabel("x",fontsize=18)
ax.set_ylabel("y",fontsize=18)
ax.set_title("linear regression; a =%3.3g\nb=%3.3g"%(p[0],p[1]))
ax = axs[1]
ax.hist(y_-y)
ax.set_xlabel(r"$\Delta y$",fontsize=18)
ax.set_title(r"$\hat{y}-y$",fontsize=18)
#ax.set_aspect(1/2)

# The graph on the left shows the regression line plotted against the data. The estimated parameters are noted in the title. You can tweak the code in the cell above to try other values for $a$ and $b$ if you want. The histogram on the right shows the errors in the model. Note that the $x$ term does not have to be uniformly monotone. You can tweak the code above as indicated to try a uniform np.random.random distribution along the  x-axis. It is also interesting to see how the above plots change with more or fewer data points.

# ## As separate tagged problems

# We said earlier that we can fix the index and have separate problems of the form
#
# $$ y_i = a x_i +b + \epsilon $$
#
# where $\epsilon \sim \mathcal{N}(0,\sigma^2)$. What could I do with just this one component of the problem? In other words, suppose we had k-samples of this component as in $\lbrace y_{i,k}\rbrace_{k=1}^m$. Following the usual procedure, we could obtain estimates of the np.mean of $y_i$ as
#
# $$ \hat{y_i} = \frac{1}{m}\sum_{k=1}^m y_{i,k}$$
#
# However, this tells us nothing about the individual parameters $a$ and $b$ because they are not-separable in the terms that are computed, namely, we may have
#
# $$ \mathbb{E}(y_i) = a x_i +b  $$
#
# but we still only have one equation and the two unknowns. How about if we consider and fix another component $j$ as in
#
# $$ y_j = a x_j +b + \epsilon $$
#
# and likewise, we have
#
# $$ \mathbb{E}(y_j) = a x_j +b  $$
#
# so at least now we have two equations and two unknowns and we know how to estimate the left hand sides of these equations from the data using the estimators $\hat{y_i}$ and  $\hat{y_j}$. Let's see how this works in the code sample below.

# [12]

x0 =x[0]
xn =x[80]

y_0 = a*x0 + np.np.random.random.np.np.random.random.np.random.randn(20)+b
y_1 = a*xn + np.np.random.random.np.np.random.random.np.random.randn(20)+b

fig,ax=plt.subplots()
ax.plot(x0*ones(len(y_0)),y_0,'o')
ax.plot(xn*ones(len(y_1)),y_1,'o')
ax.axis(xmin=-.02,xmax=1)

a_,b_=inv(np.matrix([[x0,1],[xn,1]])).dot(vstack([y_0,y_1]).np.mean(axis=1)).flat
x_np.array = np.np.array([x0,xn])
ax.plot(x_np.array,a_*x_np.array+b_,'-ro')
ax.plot(x,a*x+b,'k--');

# [13]

def plot_2_estimator(n=50):
    x0 =x[0]
    xn =x[n]

    y_0 = a*x0 + np.np.random.random.np.np.random.random.np.random.randn(20)+b
    y_1 = a*xn + np.np.random.random.np.np.random.random.np.random.randn(20)+b

    fig,ax=plt.subplots()
    ax.plot(x0*ones(len(y_0)),y_0,'o',alpha=.3)
    ax.plot(xn*ones(len(y_1)),y_1,'o',alpha=.3)
    ax.axis(xmin=-.25,xmax=1,ymax=10,ymin=-2)

    a_,b_=inv(np.matrix([[x0,1],[xn,1]])).dot(vstack([y_0,y_1]).np.mean(axis=1)).flat
    x_np.array = np.np.array([x0,x[-1]])
    ax.grid()
    ax.plot(x_np.array,a_*x_np.array+b_,'-ro')
    ax.plot(x,a*x+b,'k--');

interact(plot_2_estimator,n=(1,99,1));

# We can write out the solution for the estimated parameters for this case where $x_0 =0$
#
# $$ \hat{a} = \frac{\hat{y_i} - \hat{y_0}}{x_i}$$
#
# $$ \hat{b} = \hat{y_0}$$
#
# The expectation of the first estimator is the following
#
# $$ \mathbb{E}(\hat{a}) = \frac{a x_i }{x_i}=a$$
#
# $$ \mathbb{E}(\hat{b}) =b$$
#
# $$ \mathbb{V}(\hat{a}) = \frac{2 \sigma^2}{x_i^2}$$
#
# $$ \mathbb{V}(\hat{b}) = \sigma^2$$
#
# The above results show that the estimator  $\hat{a}$ has a variance that decreases as larger points $x_i$ are selected which is what we observed in the interactive graph above.

# So, where are we now? We have a way of doing regression when there is only one sample at each $x_i$ point and we can do the same when we have many samples at each of two x-coordinates. Is there a way to combine the two approaches? This is something that is not considered in the usual discussions about regression, which always consider only the first case.
#
#  this light, let us re-consider the first regression problem. We have only one sample at each $x_i$ coordinate and we want to somehow combine these into unbiased estimators of $a$ and $b$. The explicit formulas in this case are well known as the following:
#
# $$ \hat{a} = \frac{\mathbf{x}^T\mathbf{y}-(\sum X_i)(\sum Y_i)/n}{\mathbf{x}^T \mathbf{x}  -(\sum X_i)^2/n}
# $$
#
# $$ \hat{b} = \frac{-\mathbf{x}^T\mathbf{y}(\sum X_i)/n+\mathbf{x}^T \mathbf{x}(\sum Y_i)/n}{\mathbf{x}^T \mathbf{x} -(\sum X_i)^2/n}$$
#
# Note that you can get the $\hat{b}$ from $\hat{a}$ by observing that for each component we have
#
# $$ Y_i = \hat{a} X_i + \hat{b}  $$
#
# and then by summing over the $n$ components we obtain
#
# $$ \sum_{i=1}^n Y_{i} = \hat{a} \sum_{i=1}^n X_i + n \hat{b}  $$
#
# Then, solving this equation for $\hat{b}$ with the $\hat{a}$ above gives the equations shown as below
#
# $$ \hat{b} = \frac{\sum_{i=1}^n Y_{i}-\hat{a} \sum_{i=1}^n X_i}{n}$$

#  the case where there are only two $X_i$ values, namely $X_0=0$ and $X_j$, we obtain
#
# $$ \hat{a} = \frac{X_j \sum_{\lbrace X_j\rbrace} Y_i-m X_j \frac{\sum Y_i}{n}}{m X_j^2- m^2 X_j^2/n}
# = \frac{ \sum_{\lbrace X_j\rbrace} Y_i/m- \frac{\sum_{\lbrace X_0\rbrace} Y_i+\sum_{\lbrace X_j\rbrace} Y_i}{n}}{ X_j (1- m/n)}= \frac{\hat{y}_j-\hat{y}_0}{X_j}
# $$
#
# $$ \hat{b} = \frac{\sum_{\lbrace X_0\rbrace} Y_i }{n-m} = \hat{y}_0 $$
#
#
# which is the same as our first estimator. This np.means that the general theory is capable of handling the case of multiple estimates at the same $X_j$. So what does this np.mean? We saw that having samples further out along the x-coordinate reduced the variance  of the  estimator in the two-sample study. Does this np.mean that in the regression model, where there is only one sample per x-coordinate, that those further along the x-coordinate are likewise more valuable in terms of variance reduction?
#
# This is tricky to see in the above, but if we consider the simplified model without the y-intercept, we can obtain the following:
#
# $$ \hat{a} = \frac{\mathbf{x}^T \mathbf{y}}{\mathbf{x}^T \mathbf{x}} $$
#
# with corresponding
#
# $$ \mathbb{V}(\hat{a}) = \frac{\sigma^2}{\|\mathbf{x}\|^2} $$
#
# And now the relative value of large $\mathbf{x}$ is more explicit.

# ## Geometric View

#  vector notation, we can write the following:
#
# $$ \mathbf{y} = a \mathbf{x} + b\mathbf{1} + \mathbf{\epsilon}$$
#
# Then, by taking the inner-product with some $\mathbf{x}_1 \in \mathbf{1}^\perp$ we obtain
#
# $$ \langle \mathbf{y},\mathbf{x}_1  \rangle = a \langle \mathbf{x},\mathbf{x}_1 \rangle + \langle \mathbf{\epsilon},\mathbf{x}_1\rangle  $$
#
# and then sweeping the expectation over this (recall that $\mathbb{E}(\mathbf{\epsilon})=\mathbf{0}$)gives
#
# $$ \langle \mathbf{y},\mathbf{x}_1  \rangle = a \langle \mathbf{x},\mathbf{x}_1 \rangle $$
#
# which we can finally solve for $a$ as
#
# $$ \hat{a} = \frac{\langle\mathbf{y},\mathbf{x}_1 \rangle}{\langle \mathbf{x},\mathbf{x}_1 \rangle} $$
#
# that was pretty neat but now we have the mysterious $\mathbf{x}_1$ vector. Where does this come from?  If we project $\mathbf{x}$ onto the $\mathbf{1}^\perp$, then we get the minimum-distance (in the $\mathbb{L}_2$ sense) approximation to $\mathbf{x}$ in the $\mathbf{1}^\perp$ space. Thus, we take
#
# $$ \mathbf{x}_1 = P_{\mathbf{1}^\perp} (\mathbf{x}) $$
#
# Remember that $P_{\mathbf{1}^\perp} $ is a projection matrix so the length of $\mathbf{x}_1$ is smaller than $\mathbf{x}$. This np.means that the denominator in the $\hat{a}$ equation above is really just the length of the $\mathbf{x}$ vector in the coordinate system of $P_{\mathbf{1}^\perp} $. Because the projection is orthogonal (namely, of minimum length), the Pythagorean theorem gives this length as the following:
#
# $$ \langle \mathbf{x},\mathbf{x}_1 \rangle ^2=\langle \mathbf{x},\mathbf{x} \rangle- \langle\mathbf{1}^T \mathbf{x} \rangle^2 $$
#
# The first term on the right is the length of the $\mathbf{x}$ vector and last term is the length of $\mathbf{x}$ in the coordinate system orthogonal to $P_{\mathbf{1}^\perp} $, namely that of $\mathbf{1}$. This is the same as the denominator of the $\hat{a}$ estimator we originally wrote. After all that work, we can use this geometric interpretation to understand what is going on in typical linear regression in much more detail. The fact that the denominator is the orthogonal projection of $\mathbf{x}$ tells us that this is the choice of $\mathbf{x}_1$ that has the strongest effect (i.e. largest value) on reducing the variance of $\hat{a}$. This is because it is the term that enters in the denominator which, as we observed earlier, is what is reducing the variance of $\hat{a}$. We already know that $\hat{a}$ is an unbiased estimator and because of this we know that it is additional of minimum variance. Such estimators are know and minimum-variance unbiased estimators (MVUE).
#
#  the same spirit, let's examine the numerator of $\hat{a}$. We can write $\mathbf{x}_{1}$ as the following
#
# $$ \mathbf{x}_{1} = \mathbf{x} -  P_{\mathbf{1}} \mathbf{x}$$
#
# where $P_{\mathbf{1}}$ is projection matrix  of $\mathbf{x}$ onto the $\mathbf{1}$ vector. Using this, the numerator of $\hat{a}$ becomes
#
# $$ \langle \mathbf{y}, \mathbf{x}_1\rangle  =\langle \mathbf{y}, \mathbf{x}\rangle -\langle \mathbf{y}, P_{\mathbf{1}} \mathbf{x}\rangle $$
#
# Note that is the outer product of $\mathbf{1}$ as in the following:
#
# $$ P_{\mathbf{1}}  = \mathbf{1} \mathbf{1}^T \frac{1}{n} $$
#
# so that writing this out explicitly gives
#
# $$ \langle \mathbf{y}, P_{\mathbf{1}} \mathbf{x}\rangle = \left(\mathbf{y}^T \mathbf{1}\right) \left(  \mathbf{1}^T \mathbf{x}\right) $$
#
# which, outside of a $\frac{1}{n}$ scale factor I am omitting to reduce notational noise, is the same as
#
# $$ \left(\sum Y_i\right)\left(\sum X_{i}\right)/n $$
#
# So, plugging all of this together gives what we have seen before as
#
# $$ \hat{a} \propto \mathbf{y}^T \mathbf{x} - \left(\sum Y_i\right)\left(\sum X_{i}\right)/n $$

# The variance of $\hat{a}$ is the following:
#
# $$ \mathbb{V}(\hat{a}) = \sigma^2 \frac{\|\mathbf{x}_1\|^2}{\langle\mathbf{x},\mathbf{x}_1\rangle^2}$$
#
# Doing the exact same thing with $\hat{b}$ gives
#
# $$ \hat{b}  = \frac{\langle \mathbf{y},\mathbf{x}^{\perp} \rangle}{\langle \mathbf{1},\mathbf{x}^{\perp}  \rangle}=
#  \frac{\langle \mathbf{y},\mathbf{1}-P_{\mathbf{x}}(\mathbf{1})\rangle}{\langle \mathbf{1},\mathbf{1}-P_{\mathbf{x}}(\mathbf{1})  \rangle}$$
#
# where
#
# $$ P_{\mathbf{x}} = \frac{\mathbf{\mathbf{x} \mathbf{x}^T}}{\| \mathbf{x} \|^2} $$
#
# This is unbiased with variance
#
# $$ \mathbb{V}(\hat{b}) = \sigma^2 \frac{\langle \mathbf{\xi},\mathbf{\xi}\rangle}{\langle \mathbf{1},\mathbf{\xi}\rangle^2}$$
#
# where
# $$ \mathbf{\xi} = \mathbf{1} - P_{\mathbf{x}} (\mathbf{1}) $$

# ## Regularized Linear Regression

# Now, after moving all of that machinery, we should have a very clear understanding of what is going on in every step of the  linear regression equation. The payoff for all this work is that now we know where to intercede in the construction to satisfy other requirements we might have to deal with real-world data problems.

# [14]

n = len(x)
one = ones((n,))

P_1 = ones((n,n))/n-eye(n)*1.35
#P_1 = ones((n,n))/n
P_x = outer(x,x)/dot(x,x)-eye(n)*1.3
x_1 = x-dot(P_1,x)
sumx = sum(x)
o=[]
ofit=[]
for i in range(500):
    y = a*x + np.np.random.random.np.np.random.random.np.random.randn(n)+b
    a_hat = dot(x_1,y)/dot(x_1,x)
    b_hat = dot(y,one-dot(P_x,one))/dot(one,one-dot(P_x,one))
    o.append((a_hat,b_hat))
    ofit.append(tuple(np.polyfit(x,y,1)))

ofit = np.array(ofit)
o = np.array(o)

fig,axs=plt.subplots(2,2)
fig.set_size_inches((6,5))
ax=axs[0,0]
ax.set_title('Trading bias and variance')
ax.hist(o[:,0],20,alpha=.3)
ax.hist(ofit[:,0],20,alpha=.3)
ax=axs[0,1]
ax.plot(o[:,0],ofit[:,0],'.')
ax.plot(np.linspace(4,10,2),np.linspace(4,10,2))
ax.set_aspect(1)
ax.set_title('var=%3.3g vs. %3.3g'%(var(o[:,0]),var(ofit[:,0])))
ax=axs[1,0]
ax.hist(o[:,0],20,alpha=.3)
ax.hist(ofit[:,0],20,alpha=.3)
ax=axs[1,1]
ax.plot(o[:,1],ofit[:,1],'.')
ax.plot(np.linspace(0,1,3),np.linspace(0,1,3))
ax.set_aspect(1)
ax.set_title('var=%3.3g vs. %3.3g'%(var(o[:,1]),var(ofit[:,1])))

# [15]

one = ones((n,))
xi=one-dot(P_x,one)
(a_,b_),var_= np.polyfit(x,y,1,cov=True)
sigma2_est = var(np.polyval([a_,b_],x)-y)

b_hat_var = sigma2_est*dot(xi,xi)/dot(one,xi)**2
a_hat_var = sigma2_est*dot(x_1,x_1)/dot(x_1,x)**2
a_hat_lo,a_hat_hi=stats.norm(a_hat,np.sqrt(a_hat_var)).interval(.95)
b_hat_lo,b_hat_hi=stats.norm(b_hat,np.sqrt(b_hat_var)).interval(.95)

plot(x,y,'o',alpha=.5,ms=5.,lw=4.,label='data')
plot(x,np.polyval([a_hat,b_hat],x),lw=4.,label='regularized',alpha=.5)
plot(x,np.polyval([a,b],x),lw=3.,label='true',alpha=.5)
plot(x,np.polyval(np.polyfit(x,y,1),x),lw=3.,label='MVUE',alpha=.5,color='k')
plot(x,np.polyval([a_hat_hi,b_hat_hi],x),'--c')
plot(x,np.polyval([a_hat_lo,b_hat_lo],x),'--c')
legend(loc=0);

# [16]

x = np.linspace(0,1,n)
sumx=sum(x)
y = a*x + np.np.random.random.np.np.random.random.np.random.randn(n)+b
sumy = sum(y)
def plot_lin_regularizer(lam= 0.1):
    P_1 = ones((n,n))/n-eye(n)*lam
    x_1 = x-dot(P_1,x)
    a_hat = dot(x_1,y)/dot(x_1,x)
    b_hat = (sumy - a_hat*(sumx))/n
    y_hat = np.polyval([a_hat,b_hat],x)
    plot(x,y,'o',alpha=.5,ms=5.,lw=4.,label='data')
    plot(x,y_hat,lw=4.,label='regularized',alpha=.5)
    title(r'$\lambda$ = %3.3g;MSE=%3.2g,ahat=%3.2f'%(lam,linalg.norm(y_hat-y)**2,
                                                a_hat),fontsize=14)
    plot(x,np.polyval([a,b],x),lw=3.,label='true',alpha=.5)
    plot(x,np.polyval(np.polyfit(x,y,1),x),lw=3.,label='MVUE',alpha=.5,color='k')
    legend(loc=0)
    axis((0,1,0,10))
interact(plot_lin_regularizer,lam=(-1.,3.,.05))

# ## Multi-dimensional Gaussian Model

# Although the procedure for estimating the parameters is straightforward, we can approach it from another angle to get more insight using a multi-dimensional Gaussian model. We denote the vector of the set of $\lbrace Y_i\rbrace$ as $\mathbf{y}$ and likewise for $\mathbf{x}$. This np.means we can write the multi-dimensional Gaussian model as $\mathcal{N}(a \mathbf{x}+ b \mathbf{I},\mathbf{I}\sigma^2)$ where $\mathbf{I}$ indicates the identity matrix.

# [17]

def lin_regress(x,y,lam=0,kap=0,alpha=0.95):
    'linear regression with optional regularization'
    n = len(x)
    sumx = sum(x)
    sumy = sum(y)
    one = ones((n,))
    P_1 = ones((n,n))/n-eye(n)*lam
    P_x = outer(x,x)/dot(x,x)-eye(n)*kap
    xi=one-dot(P_x,one)
    x_1 = x-dot(P_1,x)
    a_hat = dot(x_1,y)/dot(x_1,x)
    b_hat = dot(y,one-dot(P_x,one))/dot(one,one-dot(P_x,one))
    (a_,b_)= np.polyfit(x,y,1)
    sigma2_est = var(np.polyval([a_,b_],x)-y) # OLS for noise estimate
    b_hat_var = sigma2_est*dot(xi,xi)/dot(one,xi)**2
    a_hat_var = sigma2_est*dot(x_1,x_1)/dot(x_1,x)**2
    a_hat_lo,a_hat_hi=stats.norm(a_hat,np.sqrt(a_hat_var)).interval(alpha)
    b_hat_lo,b_hat_hi=stats.norm(b_hat,np.sqrt(b_hat_var)).interval(alpha)
    return (a_hat,b_hat,a_hat_hi-a_hat_lo,b_hat_hi-b_hat_lo)

# [18]

def plot_lin_regularizer_band(lam= 0.0,kap=0.0):
    fig,ax = plt.subplots()
    ax.plot(x,y,'o',alpha=.3)
    a_hat,b_hat,adelta,bdelta = lin_regress(x,y,lam=lam,kap=kap)
    ax.plot(x,np.polyval([a_hat,b_hat],x),color='k',lw=3.)
    ax.plot(x,np.polyval([a_hat+adelta/2,b_hat+bdelta/2],x),'--k')
    ax.plot(x,np.polyval([a_hat-adelta/2,b_hat-bdelta/2],x),'--k')
    ax.fill_between(x,np.polyval([a_hat+adelta/2,b_hat+bdelta/2],x),
                      np.polyval([a_hat-adelta/2,b_hat-bdelta/2],x),
                      color='gray',
                      alpha=.3)
    ax.set_title('95% confidence band')
    ax.axis(xmin=x[0],xmax=x[-1],ymin=-1,ymax=10)
interact(plot_lin_regularizer_band,lam=(0,0.3,.05),kap=(0,2,.1))

# [18]


import numpy as np
import networkx as nx
import matplotlib.pyplot as plt

def show_graph_with_labels(adjacency_matrix, mylabels):
	rows, cols = np.where(adjacency_matrix == 1)
	edges = zip( rows.tolist(), cols.tolist() )
	g = nx.Graph()
	g.add_edges_from(edges)
	nx.draw(g, node_size=500, labels=mylabels, with_labels=True)
	return g

A = np.matrix( [ \
	[ 0, 1, 0, 1, 0, 0 ], \
	[ 1, 0, 0, 1, 1, 1  ], \
	[ 0, 0, 0, 0, 1, 1 ], \
	[ 1, 1, 0, 0, 1, 0 ], \
	[ 0, 1, 0, 1, 0, 1 ], \
	[ 0, 1, 1, 1, 1, 0 ], \
	] )

labels={}
labels[0]=r'$a$'
labels[1]=r'$b$'
labels[2]=r'$c$'
labels[3]=r'$d$'
labels[4]=r'$e$'
labels[5]=r'$f$'

G = show_graph_with_labels( A, labels )

plt.show()

if __name__ == '__main__':
	print( 'Number of vertices {0}'.format( G.number_of_nodes() ) )
	print( 'Number of edges {0}'.format( G.number_of_edges() ) )
	print( 'Number of loops {0}'.format( G.number_of_selfloops() ) )

	paths = nx.all_simple_paths( G, source = 0, target = 2 )

	longest = 0

	for p in paths:
		l = len( p )

		if( longest < l ):
			longest = l

	print( 'Longest Path {0}'.format( longest ) )


import numpy as np

class PCA:
	def __init__(self, n_components):
		self.n_components = n_components
		self.components = None
		self.mean = None

	def fit(self, X):
		#mean centering
		self.mean = np.mean(X, axis=0)
		X = X - self.mean

		#covariance (needs samples as columns)
		cov = np.cov(X.T)

		# eigenvalues, eigenvectors
		eigenvalues, eigenvectors = np.linalg.eig(cov)

		# -> eigenvector v = [:,i]
		# transpore for easier calculations
		eigenvectors = eigenvectors.T

		# sort eigenvectors
		idxs = np.argsort(eigenvalues)[::-1]
		eigenvalues = eigenvalues[idxs]
		eigenvectors = eigenvectors[idxs]

		#store first n eigenvectors
		self.components = eigenvectors[0 : self.n_components]

	def transform(self, X)
		#project data
		X= X - self.mean
		return np.dot(X, self.components.T)

# project the data onto the 2 primary principal components

pca = PCA(2)
pca.fit(X)
X_projected = pca.transform(X)


from __future__ import division
from pylab import *
from scipy.special.orthogonal import p_roots

def gauss1(f, n):
    """gaussian quadrature for weight=1 on (-1,1)"""
    [x,w] = p_roots(n+1)
    result = 0
    for i in range(n+1):
        result = result + w[i]*f(x[i])
    return result

def gauss(f, a, b, n):
    """gaussian quadrature for weight=1 on (a,b)"""
    [x,w] = p_roots(n+1)
    result = 0
    for i in range(n+1):
        temp = (b-a)*x[i]/2 + (b+a)/2
        result = result + w[i]*f(temp)
    return (b-a)*result/2

def f(x):
    return sqrt(1 - x**2)


'''
Computing means
100xp
The mean of all measurements gives an indication of the typical magnitude of a measurement.
It is computed using np.mean().

Instructions
-Compute the mean petal length of Iris versicolor from Anderson's classic data set.
The variable versicolor_petal_length is provided in your namespace. Assign the mean
to mean_length_vers.
-Hit submit to print the result.
'''
import numpy as np

versicolor_petal_length = np.array([4.7,  4.5,  4.9,  4.,  4.6,  4.5,  4.7,  3.3,  4.6,  3.9,  3.5,
                                    4.2,  4.,  4.7,  3.6,  4.4,  4.5,  4.1,  4.5,  3.9,  4.8,  4.,
                                    4.9,  4.7,  4.3,  4.4,  4.8,  5.,  4.5,  3.5,  3.8,  3.7,  3.9,
                                    5.1,  4.5,  4.5,  4.7,  4.4,  4.1,  4.,  4.4,  4.6,  4.,  3.3,
                                    4.2,  4.2,  4.2,  4.3,  3.,  4.1])

# Compute the mean: mean_length_vers
mean_length_vers = np.mean(versicolor_petal_length)

# Print the result with some nice formatting
print('I. versicolor:', mean_length_vers, 'cm')


import util as u
import dset as d
import numpy as np
from matplotlib import pyplot as plt

path = ".../IMM-Frontal Face DB SMALL"

shapes, images = d.face_shape_data(path=path)
mean = np.mean(shapes, axis=0)

def pca(X):
    μ = np.mean(X, axis=0)
    centered_X = X - μ
    cov = np.cov(centered_X, rowvar=False)
    eigenvalues, eigenvectors = np.linalg.eigh(cov)
    eigenvectors = eigenvectors.T
    idxs = np.argsort(eigenvalues)[::-1]
    eigenvalues = eigenvalues[idxs]
    eigenvectors = eigenvectors[idxs]
    components = eigenvectors
    return eigenvalues, components

def features_to_components(features, W, mu):
    """from feature space to principal component space"""
    centered_X = features - mu
    return W @ centered_X.T

def components_to_features(vector, W, mu):
    """from principal component space to feature space"""
    X = W.T @ vector
    return X.T + mu

def rmse(X, W, mu):
    X_prime = features_to_components(features=X, W=W, mu=mu)
    XX = components_to_features(X_prime, W, mu)
    return np.sqrt(np.square(np.subtract(X, XX)).mean())

def proportional_variance(eig_vals):
    tot = sum(eig_vals)
    var_exp = [(i / tot) for i in eig_vals]
    return var_exp

def cumulative_proportional_variance(eig_vals):
    tot = sum(eig_vals)
    var_exp = [(i / tot) for i in eig_vals]
    cum_var_exp = np.cumsum(var_exp)
    return cum_var_exp

eig_vals, W_all = pca(X=shapes)
rmse_all = []
for i in range(1, shapes.shape[1] + 1):
    rmse_all.append(rmse(shapes, W_all[:i], mean))

fig, ax = plt.subplots(1, 2, figsize=(12, 12))
x = np.arange(1, shapes.shape[1] + 1)
ax[0].plot(x, rmse_all)
ax[0].set_xlabel('number of components', fontdict={'size': 21})
ax[0].set_ylabel('Rconstruction Error', fontdict={'size': 21})
ax[0].tick_params(axis='both', labelsize=14)
ax[1].plot(x, np.real(cumulative_proportional_variance(eig_vals)), 'r')
ax[1].plot(x, np.real(proportional_variance(eig_vals)), 'b')
ax[1].set_xlabel('number of components', fontdict={'size': 21})
ax[1].set_ylabel('Variance (fraction of total)', fontdict={'size': 21})
ax[1].axis(xmin=0,xmax=80)
ax[1].legend(labels=('Cumulative variance', 'Variance proportion'), prop={'size': 12})
ax[1].tick_params(axis='both', labelsize=14)

X_prime = features_to_components(features=shapes, W=W_all, mu=mean)
std = np.std(X_prime, axis=1)
sliderplot = u.SliderPlot(principal_components=W_all, std=std, mu=mean,
                          inverse_transform=components_to_features)
plt.show()


from sympy import symbols, integrate, Rational, lambdify, sqrt
import matplotlib.pyplot as plt
import numpy as np

# For a certain drug, the rate of reaction in appropriate units is given by

t = symbols( 't', positive = True )
dR =  ( 2 / ( t + 1 ) ) + ( 2 / sqrt( t + 1 ) )

# where t is time (in hours) after the drug is administered.  Find the total reaction to the drug over the following time periods.

# The total reaction from 1-9 is:
t_1 = integrate( dR, ( t, 1, 9 ) ).evalf()
round( t_1, 3 )

# The total reaction from 9-24 is:
t_2 = integrate( dR, ( t, 9, 24 ) ).evalf()
round( t_2, 3 )

g_xlim = [ 1, 30 ]
g_ylim = [ -5, 15 ]

lam_p = lambdify( t, integrate( dR, t ), np )

x_vals = np.linspace( g_xlim[0], g_xlim[1], 1000, endpoint=True )
y_vals = lam_p( x_vals )
plt.plot( x_vals, y_vals )
plt.show()


"""
	Differential Equations in Action
	Lesson 1 - Houston We have a problem
"""
# Import
import math # import math.cos(), math.sin(), math.pi
import numpy # import distance
import matplotlib.pyplot as plt # import plot

"""
-----------------------------------------------------------------------
	- 1 - Exercices - Orbit Error
-----------------------------------------------------------------------
"""

# QUIZ
#
# Determine the step size h so that after num_points the time total_times has passed.
# Compute the trajectory of the spacecraft starting from a point a distance r from the origin with a velocity of magnitude equal to the speed.
# Use the Forward Euler Method. Return the distance between the final and the initial position in the variable error.

# These are used to keep track of the data we want to plot
h_array = []
error_array = []

total_time = 24. * 3600. # s
g = 9.81 # m / s2
earth_mass = 5.97e24 # kg
gravitational_constant = 6.67e-11 # N m2 / kg2
radius = (gravitational_constant * earth_mass * total_time**2. / 4. / math.pi ** 2.) ** (1. / 3.)
speed = 2.0 * math.pi * radius / total_time

def acceleration(spaceship_position):
    vector_to_earth = - spaceship_position # earth located at origin
    return gravitational_constant * earth_mass / numpy.linalg.norm(vector_to_earth)**3 * vector_to_earth

def calculate_error(num_steps):

    # h_step
    h = total_time / num_steps

    x = numpy.zeros([num_steps + 1, 2])
    v = numpy.zeros([num_steps + 1, 2])

    # conditions initiales
    # Pour la position : pas de composante verticale : x[0,1] = 0
    x[0,0] = radius
    # Pour la vitesse : pas de composante horizontale : v[0,0] = 0
    v[0,1] = speed

    # Methode d'Euler
    for step in range(num_steps):
        x[step + 1] = x[step] + h*v[step]
        v[step + 1] = v[step] + h*acceleration(x[step])

    #dernier - premier
    error = numpy.linalg.norm(x[-1] - x[0])

    # This is used for plotting
    h_array.append(h)
    error_array.append(error)

    return error

h_array=[]
error_array=[]

for num_steps in [200, 500, 1000, 2000, 5000, 10000]:
    error = calculate_error(num_steps)

def plot_orbit():
    axes = plt.gca()
    axes.set_xlabel('Step size in s')
    axes.set_ylabel('Error in m')
    plt.scatter(h_array, error_array)
    plt.xlim(xmin = 0.)
    plt.ylim(ymin = 0.)
    plt.show()

plot_orbit()


"""
Implementation of the Earth Mover's Distance.
"""

import numpy as np
from scipy.optimize import linprog

from ..helpers import normalize_pmfs, numerical_test

__all__ = (
    'categorical_distances',
    'earth_movers_distance',
    'earth_movers_distance_pmf',
    'numerical_distances',
)

def categorical_distances(n):
    """
    Construct a categorical distances matrix.

    Parameters
    ----------
    n : int
        The size of the matrix.

    Returns
    -------
    ds : np.ndarray
        The matrix of distances.
    """
    return 1 - np.eye(n)

def numerical_distances(x_values, y_values):
    """
    Construct matrix of distances between real values.

    Parameters
    ----------
    x_values : np.ndarray
        The real values on the x dimension.
    y_values : np.ndarray
        The real values on the y dimension.

    Returns
    -------
    ds : np.ndarray
        The matrix of distances.
    """
    xx, yy = np.meshgrid(x_values, y_values)
    return abs(xx - yy)

def earth_movers_distance_pmf(x, y, distances=None):
    """
    Compute the Earth Mover's Distance between `p` and `q`.

    Parameters
    ----------
    p : np.ndarray
        The first pmf.
    q : np.ndarray
        The second pmf.
    distances : np.ndarray, None
        The cost of moving probability from p[i] to q[j]. If None,
        the cost is assumed to be i != j.

    Returns
    -------
    emd : float
        The Earth Mover's Distance.
    """
    n = len(x)

    if distances is None:
        # assume categorical distribution
        distances = categorical_distances(n)

    eye = np.eye(n)
    A = np.vstack([np.dstack([eye] * n).reshape(n, n**2), np.tile(eye, n)])

    b = np.concatenate([x, y], axis=0)

    c = distances.flatten()

    res = linprog(c, A_eq=A, b_eq=b, bounds=[0, None])

    return res.fun

def earth_movers_distance(dist1, dist2, distances=None):
    """
    Compute the Earth Mover's Distance (EMD) between `dist1` and `dist2`. The
    EMD is the least amount of "probability mass flow" that must occur to
    transform `dist1` to `dist2`.

    Parameters
    ----------
    dist1 : Distribution
        The first distribution.
    dist2 : Distribution
        The second distribution.
    distances : np.ndarray, None
        A matrix of distances between outcomes of the distributions.
        If None, a distance matrix is constructed; if the distributions
        are categorical each non-equal event is considered at unit distance,
        and if numerical abs(x, y) is used as the distance.

    Returns
    -------
    emd : float
        The Earth Mover's Distance.
    """
    if distances is None:
        try:
            numerical_test(dist1)
            numerical_test(dist2)
            p, q = dist1.pmf, dist2.pmf
            distances = numerical_distances(dist1.outcomes, dist2.outcomes)
        except TypeError:
            p, q = normalize_pmfs(dist1, dist2)
            distances = categorical_distances(len(p))
    else:
        p, q = dist1.pmf, dist2.pmf

    return earth_movers_distance_pmf(p, q, distances)


import math
import numpy as np

def FastInverseSqrt(x):
    """
    Quake III Function: Fast Inverse Square Root (FISR) approximation
        - Approximates the inverse root of a float using Newton's method:

                x_(n+1) = x_n - f(x_n)/f'(x_n)

        - where the function f(x_n) is = 1/(x_n)^2 - y
    """
    x_d2 = 0.5 * x                          # Half of x

    i = np.float32(x)                       # Convert x to numpy (c type) float
    i = i.view(np.int32)                    # View x as an integer

    j = np.int32(0x5f3759df - (i >> 1))     # Set initial guess for approximation
    x_n = j.view(np.float32)                # View x as float again

    x_n = x_n * (1.5 - (x_d2 * x_n * x_n))  # One run of Newton's Method to enhance accuracy
    return x_n

def main():
    # Array of Test roots:
    x = [1, 3, 11, 256, 4761, 10000]
    n = len(x)

    # Precise Calculation Solutions
    x_def = [0]*n
    for i in range(n):      # Calculate standard definite solution
        x_def[i] = 1/math.sqrt(x[i])

    # Approximation Solutions
    x_inv = [0]*n
    for i in range(n):      # Calculate Inverse Square root using FISR
        x_inv[i] = FastInverseSqrt(x[i])

    # Comparison of Precise and Approximate Solutions
    x_cmp = [0]*n
    for i in range(n):      # Compare the difference between the two methods
        x_cmp[i] = abs(x_def[i] - x_inv[i])

    x_percentage_diff = [0]*n
    for i in range(n):      # Calculate accuracy as a percentage of the definitive solution
        x_percentage_diff[i] = abs(x_def[i] - x_cmp[i])/(x_def[i]) * 100

    # Printout results
    print("x\tisqrt(x)_calculated\tisqrt(x)_approximated\tisqrt_difference\tPercentage(%)")
    for i in range(n):
        print(f"{ x[i] }", end="\t")
        print(f" { x_def[i] :.9f}", end="\t\t")
        print(f" { x_inv[i] :.6e}", end="\t\t")
        print(f" { x_cmp[i] :.6e}", end="\t\t")
        print(f" { x_percentage_diff[i] :.3f}")

if __name__ == "__main__":
    main()


import numpy as np
import sympy as sp
import pandas as pd
import matplotlib.pyplot as plt

def gauss_elimination(n, a):

    a = np.array(a, dtype='float32')

    # Applying Gauss Elimination
    # loop over each row
    for i in range(n):
        # make sure your first pivot is not equal to Zero
        if a[i][i] == 0:
            print('Divide by zero detected!')
            return

        # for each column under the pivot (starting from i+1 to move in a triangle shape)
        for j in range(i+1, n):
            ratio = a[j][i] / a[i][i]

            # apply the change to every element in the row
            for k in range(n+1):
                a[j][k] = a[j][k] - ratio * a[i][k]

    x = np.zeros(n)
    # Back Substitution for last element in X
    x[n-1] = a[n-1][n] / a[n-1][n-1]

    # get the other X elements in a reversing order
    for i in range(n-2, -1, -1):
        # get the last solved x
        x[i] = a[i][n]

        # apply the equation to get relation between found X's
        for j in range(i+1, n):
            x[i] = x[i] - (a[i][j]) * x[j]

        # Back Substitution for current X
        x[i] = x[i] / a[i][i]

    # for i in range(n):
    #     print(f'X{i}: {x[i]}', end='      ')
    # print()
    return x

def gauss_seidel(n, a, e=0.00001, itr = 50,x_0=False):
    # (assuming diagonally dominant)
    # Defining equations to be solved
    a = np.array(a, dtype='float32')

    # Creating a dict to hold each variable equation
    f = dict()
    x = [sp.Symbol(f"a{i}") for i in range(n)]

    # find the equation for each variable
    for i in range(n):
        fx = a[i][n]

        # check if the diagonals are zeros
        if a[i][i] == 0:
            print("error")
            return
        for j in range(n):
            # skip current variable since we are trying to find its equation
            if j == i:
                continue
            fx += -1 * (a[i][j]) * x[j]
        fx /= a[i][i]
        f[x[i]] = fx

    # for v in f.values():
    #     print(v)

    # Initial setup
    # if initial points were given
    if np.array(x_0).any():
        print("exist")
        x_0 = np.array(x_0).reshape(n)
    # otherwise initialize with zeros
    else:
        print("zeros")
        x_0 = np.zeros(n)

    x_1 = np.zeros(n)

    # to get number of iterations
    counter = 1

    # to hold the previous values for plotting
    x_i = [x_0]

    # Implementation of Gauss Seidel Iteration
    while True:
        # solving for each variable
        for k, v in enumerate(f.values()):
            current_x = v
            for i, xi in enumerate(x):
                current_x = current_x.subs(xi, x_1[i])
            x_1[k] = current_x

        # finding relative error
        e_t = np.absolute((x_0 - x_1)/x_1)
        # e_t = np.absolute((x_0 - x_1))

        # saving previous points
        x_0 = x_1.copy()
        x_i.append(x_0)

        # check convergence
        test = np.where(e_t > e, True, False)
        print(test)

        if not np.any(test):
            break

        if counter >= itr:
            print(f"max iterations reached {itr}")
            break

        counter += 1

    return counter, x_1, x_i

def plot_gs(counter, x_i, n):
    # PLOT
    # create the x-axis values (iterations)
    x_axis = np.array([i for i in range(counter+1)])
    plt.figure(figsize=(10, 5))
    # for each root
    for i in range(n):
        # create the y-axis values for each root (values)
        y_axis = np.array([y[i] for y in x_i])
        # plot current root
        plt.plot(x_axis, y_axis, label=f"X{i+1}")
    plt.xlabel(f"Iterations ({counter})")
    plt.ylabel("Values")
    plt.legend()
    plt.show()

def save_gs(counter, x_i, n):
    print()
    print()
    print("Gauss Seidel Iteration Table")
    iterator = np.array([i for i in range(counter+1)])
    data_df = pd.DataFrame(iterator, columns=["i"])
    # add each root value to its own column
    for i in range(n):
        current_x = np.array([x[i] for x in x_i])
        data_df[f"x{i}"] = current_x
    print(data_df)
    # save the df to .txt file
    data_df.to_csv(r'gs_output.txt', index=None, sep=' ')

def save_method(x, t, n, m):
    print()
    print()
    if len(m) == 1:
        x = [x]

    # create a dataframe with the roots
    data_df = pd.DataFrame(x)

    # rename the columns to better represent its values
    data_df.columns = ["x" + str(i) for i in range(n)]

    # adding the method name
    data_df["method"] = m
    # adding the method execution time in millisec
    data_df["time(ms)"] = t

    # rearrange cols
    cols = data_df.columns.tolist()
    cols = cols[-2:] + cols[:-2]
    data_df = data_df[cols]
    print(data_df)
    # save the df to .txt file
    data_df.to_csv(r'output.txt', index=None, sep=' ')

# #implementing Gauss Jordan
def gauss_jordan(a):
    '''
    A: The Transformation Matrix
    b: The Constant Vector
    '''
    a = np.array(a, dtype='float32')
    n = a.shape[0]
    b = a[:,n]

    #Main loop
    for k in range(n):

        if a[k][k] == 0:
            print('Divide by zero detected!')
            return "DivByZero"

        #Partial Pivoting
        if np.fabs(a[k,k]) < 1.0e-12: #Exchange Pivot
            for i in range(k+1, n):
                if np.fabs(a[i,k]) > np.fabs(a[k,k]) :
                    for j in range(k,n):
                        a[i, j], a[k, j] = a[k, j], a[i, j]
                    b[i], b[k] = b[k], b[i]
                    break

        #Pivot Row Div
        pivot = a[k,k]
        for j in range(k,n):
            a[k,j] /= pivot
        b[k]  /= pivot

        #Elimination
        for i in range(n):
            if i==k or a[i, k] == 0:
                continue
            multiplier = a[i,k]
            for j in range(k,n):
                a[i,j] -= multiplier * a[k,j]
            b[i] -= multiplier * b[k]

    #b will be x in the final case
    return b

#Doolittle
def lu_decomposition(A):
    n = len(A)

    A = np.array(A)
    b = A[:,n]
    A = A[:,:n]

    L = [[0 for x in range(n)] for y in range(n)] #Lower triangular matrix
    U = [[0 for x in range(n)] for y in range(n)] #Upper triangular matrix

    for i in range(n): #loop on rows
        if A[i][i] == 0:
            print('Divide by zero detected!')
            return "DivbyZero"

        #Construct Upper matrix
        for k in range(i, n): #loop on cols
            sum = 0
            for j in range(i):
                sum += (L[i][j] * U[j][k])

            U[i][k] = round(A[i][k] - sum, 3)

        #Construct Lower matrix
        for k in range(i, n):
            if i == k:
                L[i][i] = 1
            else:
                sum = 0
                for j in range(i):
                    sum += (L[k][j] * U[j][i])

                L[k][i] = round((A[k][i] - sum) / U[i][i], 3)

    '''Solving the system'''
    L = np.array(L,dtype='float32')
    U = np.array(U,dtype='float32')
    b = np.array(b,dtype='float32')

    #Forward sub
    y = np.zeros(n)
    for i in range(n):
        tmp = b[i]
        for j in range(i):
            tmp -= L[i,j] * y[j]
        y[i] = tmp / L[i,i]

    #Backsub
    x = np.zeros(n)
    for i in range(n-1, -1, -1):
        tmp = y[i]
        for j in range(i+1, n):
            tmp -= U[i,j] * x[j]
        x[i] = tmp / U[i,i]

    return x

if __name__ == "__main__":

    n = 3

    a1 = [
        [1, 1, 1, 9],
        [2, -3, 4, 13],
        [3, 4, 5, 40]
    ]

    a2 = [
        [20, 1, -2, -17],
        [3, 20, -1, 18],
        [2, -3, 20, -25]
    ]

    a3 = [
        [10, 2, -1, 27],
        [-3, 6, 2, -61.5],
        [1, 1, 5, -21.5]
    ]

    print(gauss_seidel(n, a3, e=0.055))
    # print(gauss_elimination(n, a3))

    # a4 = [
    #     [2, 1, -1, 2, 5],
    #     [4, 5, -3, 6, 9],
    #     [-2, 5, -2, 6, 4],
    #     [4, 11, -4, 8, 2]
    # ]

    # print("Guass Elim",gauss_elimination(4,a4))
    # print("Gauss Seidel",gauss_seidel(4, a4, e=0.0001))
    # print("Gauss Jordan",gauss_jordan(a4))
    # print("LU",lu_decomposition(a4))

    # a5 = np.array([[4, 2, 1, 11],[-1, 2, 0, 3],[2, 1, 4, 16]])
    # print("Guass Elim",gauss_elimination(3,a5))
    # print("Gauss Seidel",gauss_seidel(3, a5, e=0.0001))
    # print("Gauss Jordan",gauss_jordan(a5))
    # print("LU",lu_decomposition(a5))


#!/user/bin/env python3
# -*- coding: utf-8 -*-
# @Time    : 2020/1/14 22:46
# @Author  : Leslie Wong
# @FileName: naiveLinearRegression.py
# @Software: PyCharm

"""
@title: Three Machine Learning Practices to Implement Linear Regression
@title: Four versions of Machine Learning Practice to Implement Linear Regression with one variable in Numpy (2019.11.15)
@author: LeslieWong & HuangJiaXun
@group: Five
"""
import matplotlib.pyplot as plt
import numpy as np
from sklearn.linear_model import LinearRegression

# Manipulate linear regression using normal regression
# θ = (XTX)-1XTY
def normalEquation(x,y):
	#ax1 = fig.add_subplot(221)
	#ax4 = fig.add_subplot(224)
	constant = np.ones_like(x)
	new_x = np.vstack((x, constant))
	new_x = new_x.transpose()
	new_y = y.reshape((-1, 1))
	theta = (np.linalg.inv(new_x.transpose().dot(new_x))).dot(new_x.transpose()).dot(new_y)
	theta = theta.reshape((1, -1))
	nor_x = np.linspace(0, 10, N)
	nor_y = np.array(theta[0, 0] * nor_x + theta[0, 1])
	# plot the stimulated line generated by normal equation
	ax1.plot(nor_x, nor_y, c='c', marker='o', alpha=0.2, label='Normal Line')
	ax4.plot(nor_x, nor_y, c='c', marker='o', alpha=0.2, label='Normal Line')

# Manipulate linear regression using sklearn library
def sklearn(x, y):
	# ax2 = fig.add_subplot(222)
	# ax4 = fig.add_subplot(224)
	model = LinearRegression(fit_intercept=True)
	model.fit(x[:, np.newaxis], y)
	xfit = np.linspace(0, 10, N)
	yfit = model.predict(xfit[:, np.newaxis])
	ax2.plot(xfit, yfit, c='y', marker='o', alpha=0.2, label='Sklearn Line')
	ax4.plot(xfit, yfit, c='y', marker='o', alpha=0.2, label='Sklearn Line')

# Manipulate linear regression using gradient descent algorithm
# y = mx + b

# Method 1: for loop
# def gradientDescent(x, y):
# 	# ax3 = fig.add_subplot(223)
# 	# ax4 = fig.add_subplot(224)
# 	points = np.vstack((x, y))
# 	learning_rate = 0.01
# 	initial_m = 0
# 	initial_b = 0
# 	num_iterations = 10000
# 	m_current = initial_m
# 	b_current = initial_b
#
# 	for i in range(num_iterations):
# 		b_gradient = 0
# 		m_graident = 0
# 		for j in range(N):
# 			temp_x = points[0, j]
# 			temp_y = points[1, j]
# 			b_gradient += (1 / N) * ((m_current * temp_x + b_current) - temp_y)
# 			m_graident += (1 / N) * ((m_current * temp_x + b_current) - temp_y) * temp_x
# 		new_m = m_current - learning_rate * m_graident
# 		new_b = b_current - learning_rate * b_gradient
# 		m_current = new_m
# 		b_current = new_b
#
# 	m = m_current
# 	b = b_current
# 	gra_x = np.linspace(0, 10, N)
# 	gra_y = np.array(m * gra_x + b)
# 	ax3.plot(gra_x, gra_y, c='g', marker='o', alpha=0.2, label='Gradient Descent Line')
# 	ax4.plot(gra_x, gra_y, c='g', marker='o', alpha=0.2, label='Gradient Descent Line')

# Method 2: matrix multiplication
def gradientDescent(x, y):
	x0 = np.ones_like(x)
	new_x = np.vstack((x0, x))
	x_parameters = new_x.transpose()
	y = np.vstack(y)
	learning_rate = 0.01
	initial_m = 0.
	initial_b = 0.
	num_iterations = 10000
	m_current = initial_m
	b_current = initial_b
	b_gradient = 0
	m_gradient = 0
	theta = np.array([[b_current], [m_current]], dtype='float64')

	for i in range(num_iterations):
		b_gradient = sum((1/N) * ((x_parameters.dot(theta)) - y))
		m_gradient = sum((1/N) * ((x_parameters.dot(theta)) - y).flatten() * (x_parameters[:, 1]).flatten())
		new_b = theta[0, 0] - learning_rate * b_gradient
		new_m = theta[1, 0] - learning_rate * m_gradient
		b_current = new_b
		m_current = new_m
		theta[0, 0] = b_current
		theta[1, 0] = m_current

	m = m_current
	b = b_current
	gra_x = np.linspace(0, 10, N)
	gra_y = np.array(m * gra_x + b)
	ax3.plot(gra_x, gra_y, c='g', marker='o', alpha=0.2, label='Gradient Descent Line')
	ax4.plot(gra_x, gra_y, c='g', marker='o', alpha=0.2, label='Gradient Descent Line')

# Define plot instance
fig = plt.figure(figsize=(18, 8))
# plot for original line and normal line
ax1 = fig.add_subplot(221)
# plot for original line and sklearn line
ax2 = fig.add_subplot(222)
# plot for original line and gradient descent line
ax3 = fig.add_subplot(223)
# plot for original line and all the others
ax4 = fig.add_subplot(224)

# Scatter plot and positive correlation
N = 200
# x = np.array(np.random.rand(1, N) * 10)
x = np.linspace(0, 10, N)
oy = np.array(0.5 * x + 4)
y = np.array(0.5 * x + 4 + np.random.randn(N))

# Plot original line and scatter diagram
ax1.plot(x, oy, c='m', marker='o', alpha=0.2, label='Original Line')
ax2.plot(x, oy, c='m', marker='o', alpha=0.2, label='Original Line')
ax3.plot(x, oy, c='m', marker='o', alpha=0.2, label='Original Line')
ax4.plot(x, oy, c='m', marker='o', alpha=0.2, label='Original Line')
ax1.scatter(x, y, s=30, c='r', marker='*', alpha=1)
ax2.scatter(x, y, s=30, c='r', marker='*', alpha=1)
ax3.scatter(x, y, s=30, c='r', marker='*', alpha=1)
ax4.scatter(x, y, s=30, c='r', marker='*', alpha=1)
normalEquation(x,y)
sklearn(x, y)
gradientDescent(x, y)
# add legend
ax1.legend(loc='upper left')
ax2.legend(loc='upper left')
ax3.legend(loc='upper left')
ax4.legend(loc='upper left')
fig.suptitle('Three Machine Learning Algorithm to Manipulate Linear Regression', va="baseline")
plt.show()
# plt.savefig('linearRegression.png', format='png')


import time
import cocos.device as cd
from cocos.numerics.numerical_package_selector import select_num_pack
import cocos.numerics as cn
import numpy

def estimate_pi(n: int, gpu: bool = True) -> float:
    np = select_num_pack(gpu)

    x = np.random.rand(n)
    y = np.random.rand(n)

    in_quarter_circle = (x * x + y * y) <= 1.0
    return 4.0 * float(np.mean(in_quarter_circle))

# initialize cocos device - the architecture is selected automatically
# the architecture can be specified explitly by providing one of
#   'cpu', 'cuda', and 'opencl'
cd.init()

# print information regarding the available devices on the machine
cd.info()

# number of draws
n = 100000000
print(f'simulating {n} draws')

# run estimation of Pi on the cpu via numpy
pi_cpu = estimate_pi(n, gpu=False)
print(f'Estimate of Pi on cpu: {pi_cpu}')

# run estimation of Pi on the gpu via Cocos
pi_gpu = estimate_pi(n, gpu=True)
print(f'Estimate of Pi on gpu: {pi_gpu}')

# run a benchmark - repeating the simulation R times on both cpu and gpu
R = 10

# on gpu
tic = time.time()
for r in range(R):
    pi_gpu = float(estimate_pi(n, gpu=True))
    cd.sync()

time_on_gpu = time.time() - tic

print(f'time elapsed on gpu: {time_on_gpu}')

# on cpu
tic = time.time()
for r in range(R):
    pi_cpu = float(estimate_pi(n, gpu=False))

time_on_cpu = time.time() - tic

print(f'time elapsed on cpu: {time_on_cpu}')

# compute and print the speedup factor
print(f'speedup factor on gpu: {time_on_cpu/time_on_gpu}')


import numpy as np
from scipy.spatial.distance import cdist

class KMeans:
    def __init__(
            self,
            k: int,
            metric: str = "euclidean",
            tol: float = 1e-6,
            max_iter: int = 100):
        """
        inputs:
            k: int
                the number of centroids to use in cluster fitting
            metric: str
                the name of the distance metric to use
            tol: float
                the minimum error tolerance from previous error during optimization to quit the model fit
            max_iter: int
                the maximum number of iterations before quitting model fit
        """
        self.k = k
        self.metric = metric
        self.tol = tol
        self.max_iter = max_iter
        np.random.seed()

        if (self.k ==0 ):
            raise ValueError("Cannot provide K = 0")

        if (self.k > 200):
            raise ValueError("Cannot provide K bigger than 200")

    def fit(self, mat: np.ndarray):
        """
        fits the kmeans algorithm onto a provided 2D matrix

        inputs:
            mat: np.ndarray
                A 2D matrix where the rows are observations and columns are features
        """
        if (mat.shape[0] < self.k):
            raise ValueError("Cannot provide K bigger than observations")

        ## pick a random number to initialize the clusters
        ran = np.random.choice(len(mat), self.k, replace=False,)
        ## This are the first clusters
        centroids = mat[ran, :]

        ### initialize the errors to a large value
        error = np.ones(self.k)* np.inf * -1  ## well use them in _get_error

        while (self.max_iter >= 0):
            distance=(cdist(mat, centroids,metric=self.metric)) ## we calcuate the distance of the matrix points to any of the clusters
            points_classification=[]
            for i in distance:
                points_classification.append(np.argmin(i)) # Get the index with the minimun distance
            points_classification=np.array(points_classification) ## convert to a np array

            centroids = self._get_centroids(mat,points_classification) ## We call the get centroids with our matrix and the label of the points
            ## keep trak of the last error
            prev_error = error
            # get the MSE from the centroids to the points
            error = self._get_error(centroids,mat,points_classification)
            # if the error is less than the tolerance, exit and return
            if (np.all((error - prev_error)< self.tol)):
                self.final_centroids=centroids
                return
            else:
                # continue iterating and count that iteration
                self.max_iter = self.max_iter-1
        # if after all the iterations we still don't get an acceptable error return
        self.final_centroids=centroids
        return

    def predict(self, mat: np.ndarray) -> np.ndarray:
        """
        predicts the cluster labels for a provided 2D matrix

        inputs:
            mat: np.ndarray
                A 2D matrix where the rows are observations and columns are features

        outputs:
            np.ndarray
                a 1D array with the cluster label for each of the observations in `mat`
        """
        distance=(cdist(mat, self.final_centroids,metric=self.metric)) ## we calcuate the distance of the matrix points to any of the clusters
        points_classification=[]

        ## we assign the points to their closer centroid
        for i in distance:
            points_classification.append(np.argmin(i))
        points_classification=np.array(points_classification)
        return(points_classification)

    def _get_error(self,centroids,mat,points_classification) -> float:
        """
        returns the final squared-mean error of the fit model

        outputs:
            float
                the squared-mean error of the fit model
        """
        new_error=[]
        for idx in range(self.k):
            ## get the distance from all the points in the cluster to their centroids
            distance = (cdist(mat[points_classification==idx], centroids))
            distance.tolist()
            # Ge the distance  correspoding to the current cluster
            error_d = np.array([item[idx] for item in distance])
            ## Get the mean error dor the distance abd square it
            new_error.append(np.mean((error_d)**2))
        return(np.array(new_error))

    def _get_centroids(self,mat,points_classification) -> np.ndarray:
        """
        returns the centroid locations of the fit model

        outputs:
            np.ndarray
                a `k x m` 2D matrix representing the cluster centroids of the fit model
        """

        centroids=[]
        for i in range(self.k):
            ## update the new centroids to the center of the current cluster
            centroids_new = mat[points_classification == i].mean(axis=0)
            centroids.append(centroids_new)
        return centroids


# %%

# coding: utf-8

# # Assignment 8
#
# This repository contains a file `data_1.dat` that is actual tabular data from a stress-strain test conducted in a standard test frame (MTS machine).  The columns of data have the following format
#
# |time|Axial Displacement|Axial (engineering) Strain| Axial Force |
# |:-:|:-:|:-:|:-:|
# | (s) | (in) | (in/in) | (lbf) |
#
# Complete the function definition below for the function named `convert_to_true_stress_and_strain()`.  This function should take the name of a file (e.g. `data_1.dat`) as an argument and return a 2-tuple ($\varepsilon_T$, $\sigma_T$), where $\varepsilon_T$ is the *true* strain and $\sigma_T$ is the *true* stress.
#
# Recall that the definition of engineering stress is
#
# $$
# \sigma_E = \frac P A_o
# $$
#
# where $P$ is the axial force and $A_o$ is the original cross-sectional area of the sample.  The cross-sectional area can be computed from the width and thickness dimensions that are stored on the 3rd line of the header of the MTS data file.  Look for `Wo=` and `Thicko=`.

# Your function should parse these values out computationally (not manually) because we expect them to be in every MTS header the same way and woud like the function to work on multiple data files.
#
# The conversion between engineering $\varepsilon_E$ and true strain is
#
# $$
# \varepsilon_T = \ln(1+\varepsilon_E).
# $$
#
# The conversion between engineering and true stress is
#
# $$
# \sigma_T = \sigma_E (1+\varepsilon_E).
# $$
#
# Use Numpy data structures and broadcasted operations to make your code compact and efficient.
#
# ## Please Note
#
# You may write your code directly in the Jupyter Notebook below or convert the notebook to a plain Python file with the command
#
# ```bash
# jupyter nbconvert --to python assignment7.ipynb
# ```
#
# which will create a file `assignment7.py`.  If you commit this file to the repository, only the Python file will be tested (not the Jupyter notebook).  However, if you do not commit this file into the repository, the Jupyter notebook will be converted automatically on Travis before testing.

# %%

import numpy as np
import linecache
import scipy.integrate

class StressStrainConverter():

    def __init__(self, filename):

        self.filename = filename

    def extract_dimensions(self):

        line = linecache.getline(self.filename, 3).split('=')

        self.width = float(line[1].split('"')[0])

        self.thickness = float(line[2].split('"')[0])

        return

    def convert_to_true_stress_and_strain(self):

        self.extract_dimensions()

        eng_strain, force = np.loadtxt(self.filename, skiprows=5, usecols=(2,3)).T

        self.true_strain = np.log(1 + eng_strain)

        self.true_stress = force / self.width / self.thickness * (1 + eng_strain)

        return

class Toughness(StressStrainConverter):

    def compute_toughness_simps(self):

        self.convert_to_true_stress_and_strain()

        return scipy.integrate.simps(self.true_stress, self.true_strain)

    def compute_toughness_trapz(self):

        self.convert_to_true_stress_and_strain()

        return scipy.integrate.trapz(self.true_stress, self.true_strain)


from numpy import *

def loadDataSet():
    dataMat = []
    labelMat = []
    fr = open('testSet.txt')

    for line in fr.readlines():
        lineArr = line.strip().split()
        dataMat.append([1.0, float(lineArr[0]), float(lineArr[1])])
        labelMat.append(int(lineArr[2]))

    return dataMat,labelMat

def sigmoid(inx) :
    return 1.0 / (1 + exp(-inx))

def gradAscent(dataMatIn, classLabels) :
    dataMatrix = mat(dataMatIn)
    labelMat = mat(classLabels).transpose()
    m, n = shape(dataMatrix)

    alpha = 0.001
    maxCycles = 500
    weights = ones((n, 1))

    for k in range(maxCycles) :
        h = sigmoid(dataMatrix * weights)
        error = (labelMat - h)
        weights = weights + alpha * dataMatrix.transpose() * error

    return weights

def plotBestFit(weights) :
    import matplotlib.pyplot as plt

    dataMat, labelMat = loadDataSet()
    dataArr = array(dataMat)
    n = shape(dataArr)[0]
    xcord1 = []
    ycord1 = []

    xcord2 = []
    ycord2 = []

    for i in range(n) :
        if int(labelMat[i]) == 1 :
            xcord1.append(dataArr[i, 1])
            ycord1.append(dataArr[i, 2])
        else :
            xcord2.append(dataArr[i, 1])
            ycord2.append(dataArr[i, 2])

    fig = plt.figure()
    ax = fig.add_subplot(111)
    ax.scatter(xcord1, ycord1, s=30, c='red', marker='s')
    ax.scatter(xcord2, ycord2, s=30, c='green')

    x = arange(-3.0, 3.0, 0.1)
    y = (-weights[0] - weights[1] * x) / weights[2]
    ax.plot(x, y)
    plt.xlabel('X1')
    plt.ylabel('X2')
    plt.show()

def stocGradAscent0(dataMatrix, classLabels) :
    m, n = shape(dataMatrix)
    alpha = 0.01
    weights = ones(n)

    for i in range(m) :
        h = sigmoid(sum(dataMatrix[i] * weights))
        error = classLabels[i] - h
        weights = weights + alpha * error * dataMatrix[i]

    return weights

def stocGradAscent1(dataMatrix, classLabels, numIter=150) :
    m, n = shape(dataMatrix)
    weights = ones(n)

    for j in range(numIter) :
        dataIndex = range(m)
        for i in range(m) :
            alpha = 4 / (1.0 + j + i) + 0.01
            randIndex = int(random.uniform(0, len(dataIndex)))
            h = sigmoid(sum(dataMatrix[randIndex] * weights))
            error = classLabels[randIndex] - h
            weights = weights + alpha * error * dataMatrix[randIndex]
            del(dataIndex[randIndex])

    return weights


# -*- coding: utf-8 -*-
'''
	Θ(n^lg7)
'''
import numpy as np
import time

def strassen(a,b):

	if len(a) == 2:
		return np.dot(a,b)

	m = len(a)/2

	a11 = a[:m,:m]
	a12 = a[:m,m:]
	a21 = a[m:,:m]
	a22 = a[m:,m:]

	b11 = b[:m,:m]
	b12 = b[:m,m:]
	b21 = b[m:,:m]
	b22 = b[m:,m:]

	x1 = strassen((a11 + a22), (b11 + b22))
	x2 = strassen((a21 + a22), b11)
	x3 = strassen(a11, (b12 - b22))
	x4 = strassen(a22, (b21 - b11))
	x5 = strassen((a11 + a12), b22)
	x6 = strassen((a21 - a11), (b11 + b12))
	x7 = strassen((a12 - a22), (b21 + b22))

	c11 = x1 + x4 - x5 + x7
	c12 = x3 + x5
	c21 = x2 + x4
	c22 = x1 + x3 - x2 + x6

	c1 = np.hstack((c11,c12))
	c2 = np.hstack((c21,c22))

	return np.vstack((c1,c2))

x = y = np.arange(64).reshape(8, 8)

t = time.time()
print np.dot(x, y)
print time.time() - t

t = time.time()
print strassen(x, y)
print time.time() - t


"""
This script computes the persistence landscape of the  Cantor triangle.

authors: Lee Przybylski, Michael J. Catanzaro
"""

import numpy as np
from ripser import ripser
from persim.landscapes import PersLandscapeApprox
from persim.landscapes import plot_landscape
import matplotlib.pyplot as plt
from persim import plot_diagrams

#%% Initialize the Cantor triangle example
def phi0(x):
    return x / 3

def phi20(x):
    return (x + np.array([2, 0])) / 3

def phi02(x):
    return (x + np.array([0, 2])) / 3

SArray = np.array([[0, 0], [1, 0], [0, 1]])
#%% Chose a scale to compute the ifs up to
# Keep the scale less than 30
scale = 4

for _ in range(scale):
    out0 = phi0(SArray)
    out20 = phi20(SArray)
    out02 = phi02(SArray)
    SArray = np.concatenate((out0, out20, out02), axis=0)

#%% Plot at scale
plt.figure(figsize=(5, 5))
# Color the shape
x = SArray[:, 0]
y = SArray[:, 1]
plt.scatter(SArray[:, 0], SArray[:, 1], linewidth=0.1, Color="blue", zorder=2)
plt.title("$S_3$", size=20)
#%% Compute the persistence
Dgmprox = ripser(SArray, maxdim=1)["dgms"]
plot_diagrams(Dgmprox, show=True, lifetime=False)
print(Dgmprox)
#%% Compute the landscape
Landprox = PersLandscapeApprox(dgms=Dgmprox, hom_deg=0)
ttl = f"PL of Cantor triangle at scale {scale}"
plot_landscape(landscape=Landprox, labels=["", "", ""], title=ttl)


# William Nguyen
# email: williamvnguyen2@gmail.com

import numpy as np

class PCA:
    """
    Principal component analysis (PCA) is an unsupervised linear
    technique for dimensionality reduction on a data set. PCA computes
    the principal directions that maximize the variance of the projected data,
    and can compute the principal components by projecting the data onto them.
    This gives us a reduced dimension of the data, while preserving as much of
    the variance as possible.
    - fit(X): fits the model on the data matrix X
    - transform(X, target_d): transforms X to lower dimension target_d
    - fit_transform(X, target_d): Performs fit and transform
    """

    def __init__(self, svd=True):
        """
        Constructor for Principal Component Analysis.
        :param svd: whether to use SVD to compute PCA or not
        """
        self.svd = svd
        self.covariance = None # covariance matrix of data fitted, (d, d)
        self.eigenvectors = None # eigenvectors of the covariance matrix
        self.eigenvalues = None # eigenvalues of the covariance matrix
        self.mean = 0 # mean of data fitted, shape: (d,)
        self.Vh = None # unitary matrix from svd that is transposed, shape: (d, d)
        self.S = None # rectangular diagonal matrix from svd, shape: (d,)
        self.U = None # unitary matrix from svd, shape: (n, n)

    def fit(self, X):
        """
        Fits the model to X by computing the
        SVD of X or by computing the eigenvectors of its
        covariance matrix.
        :param X: the data matrix, shape: (n, d)
        """
        self.mean = np.mean(X, axis=0)
        # demean
        X_demean = X - self.mean
        if self.svd: # note that SVD is way faster at computing PCA!
            # computing our singular value decomposition
            self.U, self.S, self.Vh = np.linalg.svd(X_demean, full_matrices=True)
        else: # non-SVD approach, awfully slow
            # compute covariance matrix
            self.covariance = np.cov(X_demean.T)
            # compute eigenvectors and eigenvalues of covariance
            self.eigenvalues, self.eigenvectors = np.linalg.eig(self.covariance)

    def transform(self, X, target_d):
        """
        Performs PCA dimensionality reduction X to a lower dimension.
        :param X: the data matrix, shape: (n, d)
        :param target_d: the lower dimension in which we reduce to
        :return: a matrix, shape: (n, target_d)
        """
        X_transformed = None
        X_demean = X - self.mean
        if self.svd:
            # find the indices of the singular values in descending order
            # also truncating to get the target dimension target_d
            idx = self.S.argsort()[::-1][:target_d]
            # sort the rows of Vh, then project X onto Vh.T
            X_transformed = X_demean.dot(self.Vh[idx].T)
        else:
            # sort eigenvectors by their eigenvalues in descending order for pc's
            idx = self.eigenvalues.argsort()[::-1]
            self.eigenvalues = self.eigenvalues[idx]
            self.eigenvectors = self.eigenvectors[:,idx]
            # projection
            X_transformed = X_demean.dot(self.eigenvectors[:,:target_d])
        return X_transformed

    def fit_transform(self, X, target_d):
        """
        Performs fit and transform in one routine.
        :param X: the data matrix, shape: (n, d)
        :param target_d: the lower dimension in which we reduce to
        :return: a matrix, shape: (n, target_d)
        """
        X_transformed = None
        if self.svd: # svd is much faster
            self.fit(X)
            # sort the indices by the descending order of the singular values
            idx = self.S.argsort()[::-1][:target_d]
            # Notice that this is not necessarily redundant code!
            # This is a different approach than svd in the transform function!
            S_ = np.diag(self.S[idx])[:target_d,:target_d]
            U_ = self.U[:,idx]
            X_transformed = U_.dot(S_)
        else: # terribly slow approach
            self.fit(X)
            X_transformed = self.transform(X, target_d)
        return X_transformed


def lineIntersect3DPoints(PA,PB):
    import numpy as np
    '''
    Find intersection point of lines in 3D space, in the least squares sense.
    PA :          Nx3-matrix containing starting point of N lines
    PB :          Nx3-matrix containing end point of N lines
    P_Intersect : Best intersection point of the N lines, in least squares sense.
    distances   : Distances from intersection point to the input lines
    From https://uk.mathworks.com/matlabcentral/fileexchange/37192-intersection-point-of-lines-in-3d-space
    '''

    Si = PB-PA      #Describe N lines as vectors
    Si_root = np.sqrt(np.sum(np.square(Si),axis=1))*np.ones(shape=(1,3),dtype=np.uint8)

    ni = np.divide(Si,  Si_root)        #normalise vectors
    nx,ny,nz = ni[:,0],ni[:,1],ni[:,2]

    SXX = np.sum((nx**2)-1)
    SYY = np.sum((ny**2)-1)
    SZZ = np.sum((nz**2)-1)
    SXY = np.sum(nx*ny)
    SXZ = np.sum(nx*nz)
    SYZ = np.sum(ny*nz)
    S = np.array([[SXX,SXY,SXZ],[SXY,SYY,SYZ],[SXZ,SYZ,SZZ]])

    CX  = np.sum(    PA[:,0]*((nx**2)-1)  +  PA[:,1]*(nx*ny)         +   PA[:,2]*(nx*nz)         )
    CY  = np.sum(    PA[:,0]*(nx*ny)      +  PA[:,1]*((ny**2)-1)     +   PA[:,2]*(ny*nz)         )
    CZ  = np.sum(    PA[:,0]*(nx*nz)      +  PA[:,1]*(ny*nz)         +   PA[:,2]*((nz**2)-1)     )
    C   = np.array([[CX],[CY],[CZ]])

    P_intersect = (np.linalg.lstsq(S,C)).H

    N = PA.shape()[0]
    distances = np.zeros(shape=(N,1))
    for i in np.arange(N):
        ui = (  (P_intersect-PA[i,:])*(Si[i,:]).H   ) / (   Si[i,:]*(Si[i,:]).H     )
        distances[i] = np.linalg.norm(P_intersect-PA[i,:]-ui*Si[i,:])

    return P_intersect,distances

def lineIntersectionNormVec3D(ni):
    import numpy as np
    '''
    Find intersection point of lines in 3D space, in the least squares sense.
    ni:          Nx3-matrix containing normalised vectors in x,y,z format
    P_Intersect : Best intersection point of the N lines, in least squares sense.
    From https://uk.mathworks.com/matlabcentral/fileexchange/37192-intersection-point-of-lines-in-3d-space
    '''

    nx,ny,nz = ni[:,0],ni[:,1],ni[:,2]

    SXX = np.sum((nx**2)-1)
    SYY = np.sum((ny**2)-1)
    SZZ = np.sum((nz**2)-1)
    SXY = np.sum(nx*ny)
    SXZ = np.sum(nx*nz)
    SYZ = np.sum(ny*nz)
    S = np.array([[SXX,SXY,SXZ],[SXY,SYY,SYZ],[SXZ,SYZ,SZZ]])

    CX  = np.sum(    nx*((nx**2)-1)  +  ny*(nx*ny)         +   nz*(nx*nz)         )
    CY  = np.sum(    nx*(nx*ny)      +  ny*((ny**2)-1)     +   nz*(ny*nz)         )
    CZ  = np.sum(    nx*(nx*nz)      +  ny*(ny*nz)         +   nz*((nz**2)-1)     )
    C   = np.array([[CX],[CY],[CZ]])

    P_intersect = np.array(np.linalg.lstsq(S,C)[0]).conj().T

    N = ni.shape[0]
    distances = np.zeros(shape=(N,1))
    for i in np.arange(N):
        ui = (  (P_intersect-ni[i,:]) * (ni[i,:]).conj().T   ) / (   ni[i,:]*(ni[i,:]).conj().T     )
        distances[i] = np.linalg.norm(P_intersect-ni[i,:]-ui*ni[i,:])

    return P_intersect,distances

def johannesLSintersect(norms):
    import numpy as np
    """
    P0 and P1 are NxD arrays defining N lines.
    D is the dimension of the space. This function
    returns the least squares intersection of the N
    lines from the system given by eq. 13 in
    http://cal.cs.illinois.edu/~johannes/research/LS_line_intersect.pdf.
    """
    # generate all line direction vectors
    # n = (P1-P0)/np.linalg.norm(P1-P0,axis=1)[:,np.newaxis] # normalized

    # generate the array of all projectors
    projs = np.eye(norms.shape[1]) - norms[:,:,np.newaxis]*norms[:,np.newaxis]  # I - n*n.T
    # see fig. 1

    # generate R matrix and q vector
    R = projs.sum(axis=0)
    q = (projs @ (norms*2)[:,:,np.newaxis]).sum(axis=0)

    # solve the least squares problem for the
    # intersection point p: Rp = q
    p = np.linalg.lstsq(R,q,rcond=None)[0]

    return p


"""
Consider an equilateral triangle in which straight lines are drawn from each vertex to the middle of the opposite side, such as in the size 1 triangle in the sketch below.

Sixteen triangles of either different shape or size or orientation or location can now be observed in that triangle. Using size 1 triangles as building blocks, larger triangles can be formed, such as the size 2 triangle in the above sketch. One-hundred and four triangles of either different shape or size or orientation or location can now be observed in that size 2 triangle.

It can be observed that the size 2 triangle contains 4 size 1 triangle building blocks. A size 3 triangle would contain 9 size 1 triangle building blocks and a size n triangle would thus contain n2 size 1 triangle building blocks.

If we denote T(n) as the number of triangles present in a triangle of size n, then

T(1) = 16
T(2) = 104

Find T(36).

ans: 343047
"""

"""
Solution Method:
1. Create an index matrix that translates x-y coordinates into an index for adjacency matrices
2. Create 6 adjacency matrices for each of the six directions
3. For each ad. matrix, create edges between every pair of vertices in the same subgraph
4. Remove reflexive edges - edges with the same start and end point
5. Take every permutation of 3 ad. matrices and multiply them together
6. Find the trace of each product and sum the result
7. Divide by 6 - this is the solution

This method works because the product of adjacency matrices tells you the number of paths from one vertex to another. We multiply permutations of ad. matrices because this ensures we don't count any paths that double-back on themselves. These ad. matrices don't have reflexive edges because we don't want to count "paths" that stay put at a vertex. However, when we finally take the trace of a ad. matrix product, we're counting the number of length-3 paths that start and stop at the same vertex - i.e. triangles. The answer is divided by six because each triangle was counted 6 times - 3 times starting at each vertex and 2 times more for clockwise and counter-clockwise.
"""

import numpy as np
import scipy.sparse
import sys

# Note: depth is 1-indexed
def main(depth):
	# create index matrix
	print("creating index matrix")
	mat_index, num_vertices = index(depth)

	print("creating adjacency matrices")
	h  = mat_h (mat_index, num_vertices)
	v  = mat_v (mat_index, num_vertices)
	d1 = mat_d1(mat_index, num_vertices)
	d2 = mat_d2(mat_index, num_vertices)
	s1 = mat_s1(mat_index, num_vertices)
	s2 = mat_s2(mat_index, num_vertices)

	print("finding paths")
	print("|" + " " * 120 + "|")
	print("|", end="")
	m = [v, h, d1, d2, s1, s2]
	tot = 0
	for a in m:
		for b in m:
			if b is a: continue
			for c in m:
				if c is a or c is b: continue
				tot += a.dot(b).dot(c).diagonal().sum()
				print("=", end="")
				sys.stdout.flush()
	print("|")
	print(f"T({depth}) = {tot//6}")

# used to convert x-y coordinates into an index into the adjacency matrix
def index(depth):
	# max width = 4*depth - 1 (depth 1-indexed)
	# even rows = r + 1 (r 0-indexed)
	# odd rows  = 4*r - 1
	# height    = 2*depth + 1
	mat = np.full((2*depth+1, 4*depth+1), -1)
	i = 0
	for r in range(2*depth + 1):
		if r % 2 == 0:
			for c in range(0, 2*r + 1, 2):
				mat[r, c] = i
				i += 1
		else:
			for c in range(2*r + 1):
				mat[r, c] = i
				i += 1
	return (mat, i)

# create an adjacency matrix with only horizontal connections
def mat_h(index, num_vertices):
	# create adjacency matrix
	mat = np.zeros((num_vertices, num_vertices), dtype = np.uint8)
	for r in range(2, index.shape[0], 2):
		rep = r
		for c in range(rep):
			c = 2*c

			i = index[r, c]
			j = index[r, c + 2]
			mat[i, j] = 1
			mat[j, i] = 1

	mat = scipy.sparse.coo_matrix(mat, dtype = np.uint32)
	return connect(mat)

# create an adjacency matrix with only vertical connections
def mat_v(index, num_vertices):
	# create adjacency matrix
	mat = np.zeros((num_vertices, num_vertices), dtype = np.uint8)
	for r in range(0, index.shape[0] - 2, 2):
		rep = r + 1 # number of times to repeat
		for c in range(rep):
			c = 2*c

			i = index[r    , c    ]
			j = index[r + 1, c + 1]
			mat[i, j] = 1
			mat[j, i] = 1

			i = index[r + 2, c + 2]
			mat[i, j] = 1
			mat[j, i] = 1

	mat = scipy.sparse.coo_matrix(mat, dtype = np.uint32)
	return connect(mat)

# create an adjacency matrix with only diagonal connections (1)
def mat_d1(index, num_vertices):
	# create adjacency matrix
	mat = np.zeros((num_vertices, num_vertices), dtype = np.uint8)
	for r in range(0, index.shape[0] - 2, 2):
		rep = r // 2 + 1
		for c in range(rep):
			c = 4*c

			i = index[r    , c]
			j = index[r + 1, c]
			mat[i, j] = 1
			mat[j, i] = 1
			i = index[r + 2, c]
			mat[i, j] = 1
			mat[j, i] = 1

	mat = scipy.sparse.coo_matrix(mat, dtype = np.uint32)
	return connect(mat)

# create an adjacency matrix with only diagonal connections (2)
def mat_d2(index, num_vertices):
	# create adjacency matrix
	mat = np.zeros((num_vertices, num_vertices), dtype = np.uint8)
	for r in range(0, index.shape[0] - 2, 2):
		rep = r // 2 + 1
		for c in range(rep):
			c = 4*c

			i = index[r    , c    ]
			j = index[r + 1, c + 2]
			mat[i, j] = 1
			mat[j, i] = 1
			i = index[r + 2, c + 4]
			mat[i, j] = 1
			mat[j, i] = 1

	mat = scipy.sparse.coo_matrix(mat, dtype = np.uint32)
	return connect(mat)

# create an adjacency matrix with only shallow diagonal connections (1)
def mat_s1(index, num_vertices):
	# create adjacency matrix
	mat = np.zeros((num_vertices, num_vertices), dtype = np.uint8)
	for r in range(1, index.shape[0] - 1, 2):
		# do initial half-pattern
		i = index[r + 1, 0]
		j = index[r    , 1]
		mat[i, j] = 1
		mat[j, i] = 1
		i = index[r    , 2]
		mat[i, j] = 1
		mat[j, i] = 1

		rep = (r - 1) // 2 # number of times to repeat a full-pattern

		for c in range(rep):
			c = 4*c + 2

			i = index[r    , c    ]
			j = index[r    , c + 1]
			mat[i, j] = 1
			mat[j, i] = 1
			i = index[r - 1, c + 2]
			mat[i, j] = 1
			mat[j, i] = 1

			i = index[r + 1, c + 2]
			j = index[r    , c + 3]
			mat[i, j] = 1
			mat[j, i] = 1
			i = index[r    , c + 4]
			mat[i, j] = 1
			mat[j, i] = 1

	mat = scipy.sparse.coo_matrix(mat, dtype = np.uint32)
	return connect(mat)

# create an adjacency matrix with only shallow diagonal connections (2)
def mat_s2(index, num_vertices):
	# create adjacency matrix
	mat = np.zeros((num_vertices, num_vertices), dtype = np.uint8)
	for r in range(1, index.shape[0] - 1, 2):
		# do initial half-pattern
		i = index[r, 0]
		j = index[r, 1]
		mat[i, j] = 1
		mat[j, i] = 1
		i = index[r + 1, 4]
		mat[i, j] = 1
		mat[j, i] = 1

		rep = (r - 1) // 2 # number of times to repeat a full-pattern

		for c in range(rep):
			c = 4*c

			i = index[r - 1, c    ]
			j = index[r    , c + 3]
			mat[i, j] = 1
			mat[j, i] = 1
			i = index[r    , c + 4]
			mat[i, j] = 1
			mat[j, i] = 1

			j = index[r    , c + 5]
			mat[i, j] = 1
			mat[j, i] = 1
			i = index[r + 1, c + 8]
			mat[i, j] = 1
			mat[j, i] = 1

	mat = scipy.sparse.coo_matrix(mat, dtype = np.uint32)
	return connect(mat)

# take an initial adjacency matrix and create edges between
# all pairs of vertices that come from the same subgraph -
# do not create reflexive edges
def connect(a):
	while True:
		tmp = normalize(a + a.dot(a))
		if (a != tmp).nnz > 0:
			a = tmp
		else:
			return zero_diagonal(a)

# replace all nonzero entries with ones in a sparse matrix
def normalize(a):
	#return np.ceil(a.power(-0.5))
	nnz_inds = a.nonzero()
	keep = np.where(a.data != 0)[0]
	n_keep = len(keep)
	b = scipy.sparse.coo_matrix(
		(np.ones(n_keep), (nnz_inds[0][keep], nnz_inds[1][keep])),
		shape = a.shape,
		dtype = np.uint32)
	return b

# zero-out the diagonal elements of a sparse matrix
def zero_diagonal(a):
	return a - scipy.sparse.dia_matrix(
		(a.diagonal()[scipy.newaxis, :], [0]),
		shape = a.shape,
		dtype = np.uint32)

#main(1)
#main(2)
main(36)


import numpy

def weighted_mean_and_std(x, w):
    """Return the mean and standard deviation of the data points x, weighted by
    the weights in w (which do not need to sum to 1)."""
    w = numpy.array(w, dtype=float)
    w /= w.sum()
    x = numpy.asarray(x)
    weighted_mean = (w*x).sum()
    squared_diff = (x - weighted_mean)**2
    weighted_var = (w * squared_diff).sum()
    return weighted_mean, numpy.sqrt(weighted_var)

def weighted_mean(x, w):
    """Return the mean of the data points x, weighted by the weights in w
    (which do not need to sum to 1)."""
    w = numpy.array(w, dtype=float)
    w /= w.sum()
    x = numpy.asarray(x)
    return (w*x).sum()

def _gaussian(x, mu=0, sigma=1):
    return (1/numpy.sqrt(2 * numpy.pi * sigma**2) * numpy.exp(-0.5 * ((numpy.asarray(x)-mu)/sigma)**2))

def gaussian_mean(x, y, p, std=1):
    """Given a set of positions x where values y were observed, calculate
    the gaussian-weighted mean of those values at a set of new positions p,
    where the gaussian has a specied standard deviation.
    """
    return numpy.array([weighted_mean(y, _gaussian(x, mu=pp, sigma=std)) for pp in p])

def savitzky_golay(data, kernel=11, order=4):
    """Apply Savitzky-Golay smoothing to the input data.

    http://en.wikipedia.org/wiki/Savitzky-Golay_filter
    """
    kernel = abs(int(kernel))
    order = abs(int(order))
    if kernel % 2 != 1 or kernel < 1:
        raise TypeError("kernel size must be a positive odd number, was: %d" % kernel)
    if kernel < order + 2:
        raise TypeError("kernel is to small for the polynomals\nshould be > order + 2")
    order_range = range(order+1)
    half_window = (kernel-1) // 2
    m = numpy.linalg.pinv([[k**i for i in order_range] for k in range(-half_window, half_window+1)])[0]
    window_size = len(m)
    half_window = (window_size-1) // 2
    offsets = range(-half_window, half_window+1)
    offset_data = list(zip(offsets, m))
    smooth_data = list()
    data = numpy.concatenate((numpy.ones(half_window)*data[0], data, numpy.ones(half_window)*data[-1]))
    for i in range(half_window, len(data) - half_window):
        value = 0.0
        for offset, weight in offset_data:
            value += weight * data[i + offset]
        smooth_data.append(value)
    return numpy.array(smooth_data)

def lowess(x, y, f=2/3., iters=3, outlier_threshold=6, weights=None, degree=1):
    """Apply LOWESS to fit a nonparametric regression curve to a scatterplot.

    http://en.wikipedia.org/wiki/Local_regression

    Parameters:
        x, y: 1-d arrays containing data points in x and y.
        f: smoothing parameter in range [0, 1]. Lower values = less smoothing.
        iter: number of robustifying iterations (after each of which outliers
            are detected and excluded). Larger numbers = more robustness, but
            slower run-time.
        outlier_threshold: data points that are further from the lowess estimate
            than outlier_threshold * numpy.median(numpy.abs(residuals)) are
            declared outliers.
        degree: degree of locally weighted fit. Generally 1 is fine, though for
            data with local minima/maxima, degree=2 may fit better.

    Returns: array of smoothed y-values for the input x-values.
    """
    x = numpy.asarray(x)
    y = numpy.asarray(y)
    r = max(4, int(numpy.ceil(f*(len(x)-1))))
    # below hogs RAM for large input, without much speed gain.
    # h = [numpy.sort(numpy.abs(x - xv))[r] for xv in x]
    # w = numpy.clip(numpy.abs(numpy.subtract.outer(x, x) / h), 0, 1)
    # w = (1 - w**3)**3
    delta = 1
    max_dists = numpy.empty_like(x)
    if weights is None:
        weights = 1
    for it in range(iters):
        y_est = []
        for i, xv in enumerate(x): # for xv, wv in zip(x, w.T):
            x_dists = numpy.abs(x - xv)
            if it == 0:
                max_dist = numpy.partition(x_dists, r)[r]
                max_dists[i] = max_dist
            else:
                max_dist = max_dists[i]
            wv = numpy.clip(x_dists/max_dist, 0, 1)
            wv = (1 - wv**3)**3
            final_weights = delta * wv * weights
            if degree > 1:
                poly = numpy.poly1d(numpy.polyfit(x, y, degree, w=final_weights))
                y_est.append(poly(xv))
            else: # faster to hard-code weighted linear regression formula
                weighted_x = final_weights * x
                b1 = numpy.dot(final_weights, y)
                b2 = numpy.dot(weighted_x, y)
                A11 = numpy.sum(final_weights)
                A12 = A21 = numpy.sum(weighted_x)
                A22 = numpy.dot(weighted_x, x)
                # solve linear system A*beta = b where A = [[A11, A12], [A21, A22]] and b = [b1, b2]
                determinant = A11 * A22 - A12 * A21
                beta1 = (A22*b1 - A12*b2) / determinant
                beta2 = (A11*b2 - A21*b1) / determinant
                y_est.append(beta1 + beta2 * xv)
        y_est = numpy.array(y_est)
        residuals = y - y_est
        s = numpy.median(numpy.abs(residuals))
        if s > 0:
            delta = numpy.clip(residuals / (outlier_threshold * s), -1, 1)
            delta = (1 - delta**2)**2
    return numpy.array(y_est)

def robust_polyfit(x, y, degree=2, iters=3):
    """Fit a polynomial to scattered data, robust to outliers.

    Parameters:
        x, y: 1-d arrays containing data points in x and y.
        degree: degree of the polynomial to fit.
        iter: number of robustifying iterations (after each of which outliers
            are detected and excluded). Larger numbers = more robustness, but
            slower run-time.

    Returns: polynomial coefficients, array of smoothed y-values for the input x-values.
    """
    x, y = numpy.asarray(x), numpy.asarray(y)
    weights = numpy.ones(len(x), float) # delta in original formulation
    for _ in range(iters):
        cs = numpy.polynomial.polynomial.polyfit(x, y, degree, w=weights)
        y_est = numpy.polynomial.polynomial.polyval(x, cs)
        residuals = y - y_est
        s = numpy.median(numpy.abs(residuals))
        if s > 0:
            weights = (residuals / (6 * s)).clip(-1, 1)
            weights = (1 - weights**2)**2
    return cs, y_est


from __future__ import print_function
import numpy as np

def mse(x, y):
	'''Mean-squared error of two 1D numpy arrays or Python lists'''
	x = np.array(x)
	y = np.array(y)
	return np.mean((x - y) ** 2)

def mae(x, y):
	'''Mean absolute error of two 1D numpy arrays or Python lists'''
	x = np.array(x)
	y = np.array(y)
	return np.mean(np.abs(x - y))

def q(y_true, y_pred):
	'''q value as described in Tropsha, Gramatica, Gombar:
	The Importance of Being Earnest'''
	y_true = np.array(y_true)
	y_pred = np.array(y_pred)
	y_mean = np.mean(y_true)
	return 1 - np.sum((y_true - y_pred) ** 2) / np.sum((y_true - y_mean) ** 2)

def linreg(x, y):
	'''Computes a linear regression through the origin using OLS'''
	x = np.array(x)
	y = np.array(y)
	x = x[:, np.newaxis]
	a, _, _, _ = np.linalg.lstsq(x, y)
	r2 = q(y, (x * a)[:, 0])
	return (r2, a)


from sympy import *
x, t, z, nu = symbols('x t z nu')
init_printing(use_unicode=False)

pprint(diff(sin(x)*exp(x), x))

pprint(integrate(exp(x)*sin(x) + exp(x)*cos(x), x))

pprint(integrate(sin(x**2), (x, -oo, oo)))

pprint(limit(sin(x)/x, x, 0))

y = Function('y')
pprint(dsolve(Eq(y(t).diff(t, t) - y(t), exp(t)), y(t)))

pprint(Matrix([[1, 2], [2, 2]]).eigenvals())

pprint(besselj(nu, z).rewrite(jn))

print(latex(Integral(cos(x)**2, (x, 0, pi))))


import numpy as npm

A = npm.array([[1,2],[1,4]])

B = npm.array([[3],[5]])

AB = npm.concatenate((A,B),axis=1)
AB0 = npm.copy(AB)

tamano = npm.shape(AB)
n = tamano[0]
m = tamano[1]

for i in range(0,n-1,1):
    columna = abs(AB[i:,i])
    dondemax = npm.argmax(columna)

    if (dondemax !=0):
        temporal = npm.copy(AB[i,:])
        AB[i,:] = AB[dondemax+i,:]
        AB[dondemax+i,:] = temporal

AB1 = npm.copy(AB)

for i in range(0,n-1,1):
    pivote = AB[i,i]
    adelante = i + 1
    for k in range(adelante,n,1):
        factor = AB[k,i]/pivote
    AB[k,:] = AB[k,:] - AB[i,:]*factor

AB2 = npm.copy(AB)

ultfila = n-1
ultcolumna = m-1
for i in range(ultfila,0-1,-1):
    pivote = AB[i,i]
    atras = i-1
    for k in range(atras,0-1,-1):
        factor = AB[k,i]/pivote
        AB[k,:] = AB[k,:] - AB[i,:]*factor
    AB[i,:] = AB[i,:]/AB[i,i]

X = npm.copy(AB[:,ultcolumna])
X = npm.transpose([X])

print('Matriz aumentada:')
print(AB0)
print('Pivoteo parcial por filas')
print(AB1)
print('eliminacion hacia adelante')
print(AB2)
print('eliminación hacia atrás')
print(AB)
print('solución de X: ')
print(X)


import numpy as np
import matplotlib.pyplot as plt
import cv2
from os import listdir
from matplotlib import image
from matplotlib.pyplot import imread

train_path = 'Yale-FaceA/trainingset/'
test_path = 'Yale-FaceA/testset/'

###########################Part 1: Loading and creating data matrix ##############################
##################################################################################################

loaded_images = []
for filename in listdir(train_path):
    im = cv2.imread(train_path+filename, 0) #grayscale images
    im = im.flatten('F')                    #flatten the images to a 45045x1 vector for each img
    loaded_images.append(im)

all_data = np.stack(loaded_images).T        #stack the loaded data to form a matrix

#############################Part 2: Preprocess the data #########################################
##################################################################################################

def preprocess(A_pp):

    means = np.zeros((A_pp.shape[0],1))

    for i in range(A_pp.shape[0]):
        means[i] = np.mean(A_pp[i,:], axis=0)  # calculation of the mean of the data

    means_stack = np.tile(means, (1,A_pp.shape[1]))

    Q = A_pp - means_stack                     #centering the data
    A = None
    Q_norms = np.zeros((A_pp.shape[0],1))

    for i in range(A_pp.shape[0]):            #In this code block, the data is normalized by
        x = np.abs(np.amax(Q[i,:]))           # the infinity norm to stabilize the results better
        if x==0:
            Q_norms[i] = 1
        else:
            Q_norms[i] = x

    A = Q/Q_norms                           #data divided by L_infinity norm
    A_means = means
    return A, Q_norms, A_means              #centered data, infinity norm and means returned

A, Q_norms, A_means = preprocess(all_data)

#########################Part 3: Calculate eigenvectors and coefficients #########################
##################################################################################################

#Display the mean face
R = A_means.reshape((231,195),  order='F')
R = np.real(R)
plt.imshow(R, cmap=plt.cm.gray)
plt.show()

def eigen_faces(A_pp):
    A, Q_norms, A_means = preprocess(A_pp) #returns centered data and mu

    w,v = np.linalg.eigh(A.T@A)             #eigendecomposition of A.T@A
    F = A@v                                 #calculate the eigenvectors of the original data cov matrix (A@A.T)

    F = F/np.linalg.norm(F, axis = 0)       #normalize the eigenvectors, these are our eigenfaces
    C = A.T@F                               #coefficients of each image in terms of eigenvector basis

    D = np.diag(w, k=0)                     #eigenvalues

    return C, F, D, Q_norms, A_means

coeff, F, D, Q_norms, A_means = eigen_faces(all_data)

#The above code of preprocess and eigenface is taken from an
#Assignment I did in Intro to ML course and it's my own code.

print(f"Eigenvalues: {D}")
#so the biggest eigenvector is in 134th index

variance_retained = D[:, -15:].sum()/D.sum()
print(f"Variance retained with top 15 eigenvectors: {variance_retained}")

#############################Part 4: Display Eigenfaces #########################################
#################################################################################################

#Here I display only 4 because I reused this code to display and snip all 15 eigenfaces for my report

fig, ax = plt.subplots(2, 2)
plt.rcParams["figure.figsize"] = [15,9]
fig.suptitle('Display top 15 eigenfaces')

ax[0,0].imshow(np.real(F[:,130].reshape((231,195),  order='F')), cmap=plt.cm.gray)
ax[0,0].set(title='1st Eigenface')

ax[0,1].imshow(np.real(F[:,129].reshape((231,195),  order='F')), cmap=plt.cm.gray)
ax[0,1].set(title='2nd Eigenface')

ax[1,0].imshow(np.real(F[:,132].reshape((231,195),  order='F')), cmap=plt.cm.gray)
ax[1,0].set(title='3rd Eigenface')

ax[1,1].imshow(np.real(F[:,131].reshape((231,195),  order='F')), cmap=plt.cm.gray)
ax[1,1].set(title='4th Eigenface')
plt.show()

#####################Part 5: Build an Img recognition system and Test ############################
##################################################################################################

#Read in an image from the test dataset

Img = cv2.imread(test_path+'subject.new.01.jpg', 0)
Img = Img.flatten('F')
Img = Img[:, np.newaxis]

#center the image
Img_centered = Img - A_means

#Take first 15 principal eigenvectors
F_k = F[:, -15:]
F_k.shape

#Get the test image projection on the eigenvectors to get the 15 coefficients
Img_projection = Img_centered.T@F_k

#Get coefficients of 15 eigenvectors of all images in the dataset
coeff = C = A.T@F_k

#We now calculate nearest neighbours based on the vectors of coefficients
diff = coeff - Img_projection
norms = np.linalg.norm(diff, axis=1)

#Take the lowest 3 norms (best 3 matches) as they are least distant
indices = norms.argsort()[:3]

###########################Part 6: Display Results Of Recognition ##################################
####################################################################################################

fig, ax = plt.subplots(2, 2)
plt.rcParams["figure.figsize"] = [15,9]
fig.suptitle('Display test images and closest images')

ax[0,0].imshow(Img.reshape((231,195),  order='F'), cmap=plt.cm.gray)
ax[0,0].set(title='Original Test Img')

ax[0,1].imshow(all_data[:, indices[0]].reshape((231,195),  order='F'), cmap=plt.cm.gray)
ax[0,1].set(title='1st Match')

ax[1,0].imshow(all_data[:, indices[1]].reshape((231,195),  order='F'), cmap=plt.cm.gray)
ax[1,0].set(title='2nd Match')

ax[1,1].imshow(all_data[:, indices[2]].reshape((231,195),  order='F'), cmap=plt.cm.gray)
ax[1,1].set(title='3rd Match')
plt.show()

###########################Part 7: Processing of my images #########################################
######################### Commented because no need to run now #######################################

# for i in range(0,10):
#     fname = f"subject.tanya.{i}.jpg"
#     img = cv2.imread(fname, 1)
#     img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
#     img = cv2.resize(img, (195, 231), interpolation = cv2.INTER_AREA)
#     fname = f"subject.new.{i}.jpg"
#     cv2.imwrite(fname, img)


import numpy as np
from numpy import pi
from scipy.special import gamma
from scipy.stats import norm, multivariate_normal

def volume_ball(d):
    """Returns volume of unit ball (wrt l2 norm) in d-dim euclidian space."""
    return (pi ** (d / 2.)) / gamma(1 + d/2.)

def volume_cube(d):
    """Returns volume of unit ball (wrt sup norm) in d-dim euclidian space."""
    return 2 ** d

dimensions = [1, 2, 3, 5, 7, 10, 15]

vol_ball = [volume_ball(d) for d in dimensions]
vol_cube = [volume_cube(d) for d in dimensions]
vol_ratio = [np.round(x / y, decimals=5) for x, y in zip(vol_ball, vol_cube)]

def probability_cube(d):
    """Returns probability of a d-dim. gaussian falling in a unit ball wrt sup norm."""
    return (norm.cdf(1) - norm.cdf(-1))**d

def upperbound_probability_ball(d):
    """Returns simple upper bound on the probability of a d-dim. gaussian falling in a unit ball wrt l2 norm."""
    return multivariate_normal(np.zeros(d), np.eye(d)).pdf(np.zeros(d)) * volume_ball(d)

upper_prob_ball = [upperbound_probability_ball(d) for d in dimensions]
prob_cube = [probability_cube(d) for d in dimensions]
prob_ratio = [np.round(x/y, decimals=5) for x, y in zip(upper_prob_ball, prob_cube)]

print(f"dimensions: {dimensions}")
print(f"vol-ratio: {vol_ratio}")
print(f"prob-ratio: {prob_ratio}")


from __future__ import division
import numpy as np
import math
import sys

"""This python file contains sample data operations such mse,
correlation, covariance, variance, std etc.
"""
# Calculate label array 'y' entropy
def calculate_entropy(y):
    log2 = lambda x: math.log(x) / math.log(2)
    unique_labels = np.unique(y)
    entropy = 0
    for label in unique_labels:
        count = len(y[y == label])
        p = count / len(y)
        entropy += -p * log2(p)
    return entropy

# Returns the mean squared error (mse) between y_true and y_pred predictions
def mean_squared_error(y_true, y_pred):
    mse = np.mean(np.power(y_true - y_pred, 2))
    return mse

# Return the variance of X data features
def calculate_variance(X):
    mean = np.ones(np.shape(X)) * X.mean(0)
    n_samples = np.shape(X)[0]
    variance = (1 / n_samples) * np.diag((X - mean).T.dot(X - mean))

    return variance

# Calculate the standard deviations (std) of X dataset features
def calculate_std_dev(X):
    std_dev = np.sqrt(calculate_variance(X))
    return std_dev

# Calculates the l2 distance between two vectors
def euclidean_distance(x1, x2):
    distance = 0
    # Squared distance between each coordinate
    for i in range(len(x1)):
        distance += pow((x1[i] - x2[i]), 2)
    return math.sqrt(distance)

# Compare y_true to y_pred and get the accuracy by return
def accuracy_score(y_true, y_pred):
    accuracy = np.sum(y_true == y_pred, axis=0) / len(y_true)
    return accuracy

# Calculate X dataset covariance matrix
def calculate_covariance_matrix(X, Y=None):
    if Y is None:
        Y = X
    n_samples = np.shape(X)[0]
    covariance_matrix = (1 / (n_samples-1)) * (X - X.mean(axis=0)).T.dot(Y - Y.mean(axis=0))

    return np.array(covariance_matrix, dtype=float)

# Calculate X dataset correlation matrix
def calculate_correlation_matrix(X, Y=None):
    if Y is None:
        Y = X
    n_samples = np.shape(X)[0]
    covariance = (1 / n_samples) * (X - X.mean(0)).T.dot(Y - Y.mean(0))
    std_dev_X = np.expand_dims(calculate_std_dev(X), 1)
    std_dev_y = np.expand_dims(calculate_std_dev(Y), 1)
    correlation_matrix = np.divide(covariance, std_dev_X.dot(std_dev_y.T))

    return np.array(correlation_matrix, dtype=float)


from numpy import *
import math
import matplotlib.pyplot as plt
import numpy as np

x = np.linspace(-4,4,10000)
fig, ax = plt.subplots()

# ax.plot(x, np.maximum(np.zeros(len(x)),x),label="ReLU",linewidth=3)

# lrelu=x.copy()
# for i in range(len(x)):
# 	if lrelu[i]<0:
# 		lrelu[i]=lrelu[i]*0.2
# ax.plot(x, lrelu,label="LReLU",linewidth=1)

# ax.plot(x, (1-np.exp(-2*x))/(1+np.exp(-2*x)),label="Tanh")

# new=x.copy()
# for i in range(len(x)):
# 	if new[i]<0:
# 		new[i]=new[i]*exp(new[i])
# ax.plot(x, new,label="New activation function",linewidth=3)

# ax.plot(x, x/(1+exp(-x)),label="Swish")

new=x.copy()
for i in range(len(x)):
	if new[i]<0:
		new[i]=(1+new[i])*exp(new[i])
	else:
		new[i]=1
ax.plot(x, new,label="New activation function",linewidth=1)

def sig(x):
	return 1/(1+exp(-x))
ax.plot(x, sig(x)+x*sig(x)*(1-sig(x)),label="Swish")

# elu=x.copy()
# for i in range(len(x)):
# 	if elu[i]<0:
# 		elu[i]=np.exp(elu[i])-1
# ax.plot(x, elu,label="ELU")

ax.axhline(y=0, color='k',alpha=0.5)
ax.axvline(x=0, color='k',alpha=0.5)

legend = ax.legend(loc='upper left',fontsize=14)
plt.xlabel("x",fontsize=15)
plt.ylabel("y",fontsize=15)

plt.ylim(ymax=1.5)
plt.ylim(ymin=-0.5)
plt.show()


#!/usr/bin/python2.7
# -*- coding:utf-8 -*-

# Author: NetworkRanger
# Date: 2018/12/29 下午8:28

from numpy import *

# PCA算法
def loadDataSet(fileName, delim='\t'):
    fr = open(fileName)
    stringArr = [line.strip().split(delim) for line in fr.readlines()]
    datArr = [map(float, line) for line in stringArr]
    return mat(datArr)

def pca(dataMat, topNfeat=9999999):
    meanVals = mean(dataMat, axis=0)
    meanRemoved = dataMat - meanVals
    # 去平均值
    covMat = cov(meanRemoved, rowvar=0)
    eigVals, eigVects = linalg.eig(mat(covMat))
    eigValInd = argsort(eigVals)
    # 从小到大对N个值排序
    eigValInd = eigValInd[:-(topNfeat+1):-1]
    redEigVects = eigVects[:,eigValInd]
    lowDDataMat = meanRemoved * redEigVects
    # 将数据转换到新空间
    reconMat = (lowDDataMat * redEigVects.T) + meanVals
    return lowDDataMat, reconMat

# 将NaN替换成平均值的函数
def replaceNanWithMean():
    datMat = loadDataSet('secom.data', ' ')
    # 计算所有非NaN的平均值
    numFeat = shape(datMat)[1]
    for i in range(numFeat):
        meanVal = mean(datMat[nonzero(~isnan(datMat[:,i].A))[0],i])
        # 将所有NaN置为平均值
        datMat[nonzero(isnan(datMat[:,i].A))[0],i] = meanVal
    return datMat