"""Algorithms for integration lecture."""
import chaospy as cp
import numpy as np

def quadrature_newton_trapezoid_one(f, a, b, n):
    """Return quadrature newton trapezoid example."""
    xvals = np.linspace(a, b, n + 1)
    fvals = np.tile(np.nan, n + 1)
    h = xvals[1] - xvals[0]

    weights = np.tile(h, n + 1)
    weights[0] = weights[-1] = 0.5 * h

    for i, xval in enumerate(xvals):
        fvals[i] = f(xval)

    return np.sum(weights * fvals)

def quadrature_newton_simpson_one(f, a, b, n):
    """Return quadrature newton simpson example."""
    if n % 2 == 0:
        raise Warning("n must be an odd integer. Increasing by 1")
        n += 1

    xvals = np.linspace(a, b, n)
    fvals = np.tile(np.nan, n)

    h = xvals[1] - xvals[0]

    weights = np.tile(np.nan, n)
    weights[0::2] = 2 * h / 3
    weights[1::2] = 4 * h / 3
    weights[0] = weights[-1] = h / 3

    for i, xval in enumerate(xvals):
        fvals[i] = f(xval)

    return np.sum(weights * fvals)

def quadrature_gauss_legendre_one(f, a, b, n):
    """Return quadrature gauss legendre example."""
    xvals, weights = np.polynomial.legendre.leggauss(n)
    xval_trans = (b - a) * (xvals + 1.0) / 2.0 + a

    fvals = np.tile(np.nan, n)
    for i, xval in enumerate(xval_trans):
        fvals[i] = ((b - a) / 2.0) * f(xval)

    return np.sum(weights * fvals)

def quadrature_gauss_legendre_two(f, a=-1, b=1, n=10):
    """Return quadrature gauss legendre example."""
    n_dim = int(np.sqrt(n))

    xvals, weight_uni = np.polynomial.legendre.leggauss(n_dim)
    xvals_transformed = (b - a) * (xvals + 1.0) / 2.0 + a

    weights = np.tile(np.nan, n_dim ** 2)
    fvals = np.tile(np.nan, n_dim ** 2)

    counter = 0
    for i, x in enumerate(xvals_transformed):
        for j, y in enumerate(xvals_transformed):
            weights[counter] = weight_uni[i] * weight_uni[j]
            fvals[counter] = f([x, y])
            counter += 1

    return ((b - a) / 2) ** 2 * np.sum(weights * np.array(fvals))

def monte_carlo_naive_one(f, a=0, b=1, n=10, seed=123):
    """Return naive monte carlo example."""
    np.random.seed(seed)
    xvals = np.random.uniform(size=n)
    fvals = np.tile(np.nan, n)
    weights = np.tile(1 / n, n)

    scale = b - a
    for i, xval in enumerate(xvals):
        fvals[i] = f(a + xval * (b - a))

    return scale * np.sum(weights * fvals)

def monte_carlo_naive_two_dimensions(f, a=0, b=1, n=10, seed=128):
    """Return naive monte carlo example (two-dimensional).

    Restricted to same integration domain for both variables.
    """
    np.random.seed(seed)
    xvals = np.random.uniform(low=a, high=b, size=2 * n).reshape(n, 2)
    volume = (b - a) ** 2

    fvals = np.tile(np.nan, n)
    weights = np.tile(1 / n, n)

    for i, xval in enumerate(xvals):
        fvals[i] = f(xval)

    return volume * np.sum(weights * fvals)

def monte_carlo_quasi_two_dimensions(f, a=0, b=1, n=10, rule="random"):
    """Return Monte Carlo example (two-dimensional).

    Corresponds to naive Monthe Carlo for `rule='random'`. Restricted to same
    integration domain for both variables.
    """
    distribution = cp.J(cp.Uniform(a, b), cp.Uniform(a, b))
    samples = distribution.sample(n, rule=rule).T
    volume = (b - a) ** 2

    fvals = np.tile(np.nan, n)
    weights = np.tile(1 / n, n)

    for i, xval in enumerate(samples):
        fvals[i] = f(xval)

    return volume * np.sum(weights * fvals)


import pandas as pd
import numpy as np
import matplotlib.pyplot as plt

ratings = pd.read_csv("ratings.csv")
print (ratings.head())

movies = pd.read_csv("movies.csv")
print (movies.head())

#Combining movie ratings & movie names
ratings = pd.merge(ratings[['userId','movieId','rating']],movies[['movieId','title']],
                   how='left',left_on ='movieId' ,right_on = 'movieId')

rp = ratings.pivot_table(columns = ['movieId'],index = ['userId'],values = 'rating')
rp = rp.fillna(0)

# Converting pandas dataframe to numpy for faster execution in loops etc.
rp_mat = rp.as_matrix()

from scipy.spatial.distance import cosine

#The cosine of the angle between them is about 0.822.
a= np.asarray( [2, 1, 0, 2, 0, 1, 1, 1])
b = np.asarray( [2, 1, 1, 1, 1, 0, 1, 1])

print("\n\n")
print ("Cosine similarity between A and B is",round(1-cosine(a,b),4))

m, n = rp.shape

# User similarity matrix
mat_users = np.zeros((m, m))
for i in range(m):
    for j in range(m):
        if i != j:
            mat_users[i][j] = (1- cosine(rp_mat[i,:], rp_mat[j,:]))
        else:
            mat_users[i][j] = 0.

pd_users = pd.DataFrame(mat_users,index =rp.index ,columns= rp.index )

# Finding similar users
def topn_simusers(uid = 16,n=5):
    users = pd_users.loc[uid,:].sort_values(ascending = False)
    topn_users = users.iloc[:n,]
    topn_users = topn_users.rename('score')
    print ("Similar users as user:",uid)
    return pd.DataFrame(topn_users)

print (topn_simusers(uid=17,n=10))

# Finding most rated movies of a user
def topn_movieratings(uid = 355,n_ratings=10):
    uid_ratings = ratings.loc[ratings['userId']==uid]
    uid_ratings = uid_ratings.sort_values(by='rating',ascending = [False])
    print ("Top",n_ratings ,"movie ratings of user:",uid)
    return uid_ratings.iloc[:n_ratings,]

print (topn_movieratings(uid=596,n_ratings=10))

# Movie similarity matrix
import time
start_time = time.time()
mat_movies = np.zeros((n, n))

for i in range(n):
    for j in range(n):
        if i!=j:
            mat_movies[i,j] = (1- cosine(rp_mat[:,i], rp_mat[:,j]))
        else:
            mat_movies[i,j] = 0.
print("--- %s seconds ---" % (time.time() - start_time))

pd_movies = pd.DataFrame(mat_movies,index =rp.columns ,columns= rp.columns )

#pd_movies.to_csv('pd_movies.csv',sep=',')
pd_movies = pd.read_csv("pd_movies.csv",index_col='movieId')

# Finding similar movies
def topn_simovies(mid = 588,n=15):
    mid_ratings = pd_movies.loc[mid,:].sort_values(ascending = False)
    topn_movies = pd.DataFrame(mid_ratings.iloc[:n,])
    topn_movies['index1'] = topn_movies.index
    topn_movies['index1'] = topn_movies['index1'].astype('int64')
    topn_movies = pd.merge(topn_movies,movies[['movieId','title']],how = 'left',left_on ='index1' ,right_on = 'movieId')
    print ("Movies similar to movie id:",mid,",",movies['title'][movies['movieId']==mid].to_string(index=False),",are")
    del topn_movies['index1']
    return topn_movies

print (topn_simovies(mid=589,n=15))

#Collaborative filtering

import os
""" First change the following directory link to where all input files do exist """
os.chdir("D:\\Book writing\\Codes\\Chapter 7\\ml-latest-small\\ml-latest-small")

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt

ratings = pd.read_csv("ratings.csv")
print (ratings.head())

movies = pd.read_csv("movies.csv")
print (movies.head())

rp = ratings.pivot_table(columns = ['movieId'],index = ['userId'],values = 'rating')
rp = rp.fillna(0)

A = rp.values

print ("\nShape of Original Sparse Matrix",A.shape)

W = A>0.5
W[W==True]=1
W[W==False]=0
W = W.astype(np.float64,copy=False)

W_pred = A<0.5
W_pred[W_pred==True]=1
W_pred[W_pred==False]=0
W_pred = W_pred.astype(np.float64,copy=False)
np.fill_diagonal(W_pred,val=0)

# Parameters
m,n = A.shape

n_iterations = 200
n_factors = 100
lmbda = 0.1

X = 5 * np.random.rand(m,n_factors)
Y = 5* np.random.rand(n_factors,n)

def get_error(A, X, Y, W):
    return np.sqrt(np.sum((W * (A - np.dot(X, Y)))**2)/np.sum(W))

errors = []
for itr in range(n_iterations):
    X = np.linalg.solve(np.dot(Y,Y.T)+ lmbda * np.eye(n_factors),np.dot(Y,A.T)).T
    Y = np.linalg.solve(np.dot(X.T,X)+ lmbda * np.eye(n_factors),np.dot(X.T,A))

    if itr%10 == 0:
        print(itr," iterations completed","RMSError value is:",get_error(A,X,Y,W))

    errors.append(get_error(A,X,Y,W))

A_hat = np.dot(X,Y)
print ("RMSError of rated movies: ",get_error(A,X,Y,W))

plt.plot(errors);
plt.ylim([0, 3.5]);
plt.xlabel("Number of Iterations");plt.ylabel("RMSE")
#plt.title("No.of Iterations vs. RMSE")
plt.show()

def print_recommovies(uid=315,n_movies=15,pred_mat = A_hat,wpred_mat = W_pred ):
    pred_recos = pred_mat*wpred_mat
    pd_predrecos = pd.DataFrame(pred_recos,index =rp.index ,columns= rp.columns )
    pred_ratings = pd_predrecos.loc[uid,:].sort_values(ascending = False)
    pred_topratings = pred_ratings[:n_movies,]
    pred_topratings = pred_topratings.rename('pred_ratings')
    pred_topratings = pd.DataFrame(pred_topratings)
    pred_topratings['index1'] = pred_topratings.index
    pred_topratings['index1'] = pred_topratings['index1'].astype('int64')
    pred_topratings = pd.merge(pred_topratings,movies[['movieId','title']],how = 'left',left_on ='index1' ,right_on = 'movieId')
    del pred_topratings['index1']
    print ("\nTop",n_movies,"movies predicted for the user:",uid," based on collaborative filtering\n")
    return pred_topratings

predmtrx = print_recommovies(uid=355,n_movies=10,pred_mat=A_hat,wpred_mat=W_pred)
print (predmtrx)

# Grid Search on Collaborative Filtering
def get_error(A, X, Y, W):
    return np.sqrt(np.sum((W * (A - np.dot(X, Y)))**2)/np.sum(W))

niters = [20,50,100,200]
factors = [30,50,70,100]
lambdas = [0.001,0.01,0.05,0.1]

init_error = float("inf")

print("\n\nGrid Search results of ALS Matrix Factorization:\n")
for niter in niters:
    for facts in factors:
        for lmbd in lambdas:

            X = 5 * np.random.rand(m,facts)
            Y = 5* np.random.rand(facts,n)

            for itr in range(niter):
                X = np.linalg.solve(np.dot(Y,Y.T)+ lmbd * np.eye(facts),np.dot(Y,A.T)).T
                Y = np.linalg.solve(np.dot(X.T,X)+ lmbd * np.eye(facts),np.dot(X.T,A))

            error = get_error(A,X,Y,W)

            if error<init_error:
                print ("No.of iters",niter,"No.of Factors",facts,"Lambda",lmbd,"RMSE",error)
                init_error = error


""" Autor V M Krushnarao Kotteda (Murali)
date Oct 31 2020 """

import numpy as np
import numpy.linalg as nl

def Jacobi_expand(A, b, n):
    n = len(b)
    X = np.zeros(n)
    Y = np.zeros(n)
    D = np.zeros(n)
    X1 = np.zeros([n, 1])
    b1 = np.zeros([n, 1])
    for i in range(0, n):
        b1[i][0] = b[i]
    k = 0
    while (True):
        for i in range(0, n):
            tmp = 0
            for j in range(0, n):
                if j != i:
                    tmp += A[i][j] * X[j]
            Y[i] = (b[i] - tmp) / A[i][i]
        for i in range(0, n):
            X1[i][0] = Y[i]
        D = np.dot(A, X1)
        D = D - b1
        k = k + 1
        if k >= 100:
            break
        if  (nl.norm(D) / nl.norm(b) < 1e-6) & (nl.norm(D) / nl.norm(b) > -(1e-6)):
            break
        X = Y.copy()
    return Y

def jacobi(A, b, x_init, epsilon=1e-6, max_iterations=100):
    D = np.diag(np.diag(A))
    LU = A - D
    x = x_init
    for i in range(max_iterations):
        D_inv = np.diag(1 / np.diag(D))
        x_new = np.dot(D_inv, b - np.dot(LU, x))
        residual = np.linalg.norm(x_new - x)
        print('i, Residual %s' %[i,residual])
        if np.linalg.norm(x_new - x) < epsilon:
            return x_new
        x = x_new
    return x

def GaussSeidel(A, b, n):
    X = np.zeros(n)
    Y = np.zeros(n)
    D = np.zeros(n)
    X1 = np.zeros([n, 1])
    b1 = np.zeros([n, 1])
    for i in range(0, n):
        b1[i][0] = b[i]
    k = 0
    while (True):
        for i in range(0, n):
            tmp = 0
            for j in range(0, n):
                if j < i:
                    tmp += A[i][j] * Y[j]
                if j > i:
                    tmp += A[i][j] * X[j]
            Y[i] = (b[i] - tmp) / A[i][i]
        for i in range(0, n):
            X1[i][0] = Y[i]
        D = np.dot(A, X1)
        D = D - b1
        k = k + 1
        if k >= 50:
            break
        if  (nl.norm(D) / nl.norm(b) < 1e-6) & (nl.norm(D) / nl.norm(b) > -(1e-6)):
            break
        X = Y.copy()
    return Y

def sor_solver(A, b, omega, initial_guess, convergence_criteria):
  """
  This is an implementation of the pseduo-code provided in the Wikipedia article.
  Inputs:
    A: nxn numpy matrix
    b: n dimensional numpy vector
    omega: relaxation factor
    initial_guess: An initial solution guess for the solver to start with
  Returns:
    phi: solution vector of dimension n
  """
  phi = initial_guess[:]
  residual = np.linalg.norm(np.matmul(A, phi) - b) #Initial residual
  ic = 0
  while residual > convergence_criteria:
    ic = ic + 1
    for i in range(A.shape[0]):
      sigma = 0
      for j in range(A.shape[1]):
        if j != i:
          sigma += A[i][j] * phi[j]
      phi[i] = (1 - omega) * phi[i] + (omega / A[i][i]) * (b[i] - sigma)
    residual = np.linalg.norm(np.matmul(A, phi) - b)
#    print('Residual: {0:10.6g}'.format(residual))
    print('ic, Residual %s' %[ic,residual])
  return phi

def SOR(A, b, n, w):
    n = len(b)
    X = np.zeros(n)
    Y = np.zeros(n)
    D = np.zeros(n)
    X1 = np.zeros([n, 1])
    b1 = np.zeros([n, 1])
    for i in range(0, n):
        b1[i][0] = b[i]
    k = 0
    while (True):
        for i in range(0, n):
            tmp1 = 0
            tmp2 = 0
            for j in range(0, n):
                if j < i:
                    tmp1 += A[i][j] * Y[j]
                if j > i:
                    tmp2 += A[i][j] * X[j]
            Y[i] = X[i] + (w * (b[i] - tmp1 - tmp2)) / A[i][i]
        for i in range(0, n):
            X1[i][0] = Y[i]
        D = np.dot(A, X1)
        D = D - b1
        k = k + 1
        if k >= 50:
            break
        if  (nl.norm(D) / nl.norm(b) < 1e-6) & (nl.norm(D) / nl.norm(b) > -(1e-6)):
            break
        X = Y.copy()

#An example case that mirrors the one in the Wikipedia article
residual_convergence = 1e-8
omega = 0.5 #Relaxation factor

#A = np.array([ [4, -1, -6, 0], [-5, -4, 10, 8], [0,9,4,-2], [1,0,-7,5]])
#b = np.array([2,21,-12,-6])
A = np.array([    [5, 2, 1, 1],     [2, 6, 2, 1],     [1, 2, 7, 1],     [1, 1, 2, 8] ])
#A = np.array([    [3, 2, 1, 1],     [2, 4, 2, 1],     [1, 2, 3, 1],     [1, 1, 2, 3] ])
b = np.array([29, 31, 26, 19])

#A = np.zeros([100, 100]);
#for i in range(0, 100):
#    A[i][i] = 2
#    if i == 0:
#        A[i][i + 1] = -1
#    elif i == 99:
#        A[i][i - 1] = -1
#    else:
#        A[i][i - 1] = A[i][i + 1] = -1

initial_guess = np.ones(4)
# 4x1  + 2x2 + x3 + x4 = 29
# 2x1  + 5x2 + 2x3 + x4 = 31
# x1   + 2x2 + 4x3 + x4 = 26
# x1   + x2  + 2x3 + 4x4 = 19
#n = 500
x = sor_solver(A, b, omega, initial_guess, 1e-6)
print("SOR")
print(x)
x = jacobi(A, b, initial_guess)
residual = np.linalg.norm(np.matmul(A, x) - b)
print('Residual: {0:10.6g}'.format(residual))
print(x)
print(np.matmul(A, x) - b)
#w = 0.5
#x=sor(A,b,n,w)
#residual = np.linalg.norm(np.matmul(A, x) - b)
#print('Residual: {0:10.6g}'.format(residual))
#print(x)

n=len(b)
result1 = Jacobi_expand(A, b, n)
print("Jacobi")
print(result1)
result2 = GaussSeidel(A, b, n)
print("Gauss seidel")
print(result2)
#result3 = SOR(A, b, n, 0.5)
#print("SOR")
#print(result3)


# Referenec Book: Deep Learning with Python

import numpy as np

# 2.3.1 Element-wise operations
x = np.array([[1, 2, 3],
              [0, -1, 3]])
y = np.array([[-1, 2, 0],
              [0, -1, 3]])

z = x + y
z = np.maximum(z, 0)
print(z)

# 2.3.2 Broadcasting
'''
When possible, and if there’s no ambiguity, the smaller tensor will be broadcasted to
match the shape of the larger tensor. Broadcasting consists of two steps:
    1. Axes (called broadcast axes) are added to the smaller tensor to match the ndim of
       the larger tensor.
    2. The smaller tensor is repeated alongside these new axes to match the full shape
       of the larger tensor.
'''
x = np.random.random((1,3,4,3))
y = np.random.random((4,3))
z = np.maximum(x, y)
print(z)
'''
[[[[0.61973766 0.66481166 0.81795199]
   [0.86333686 0.14431782 0.4936027 ]
   [0.65448209 0.85841484 0.9671061 ]
   [0.92107294 0.92064691 0.96485797]]

  [[0.36866181 0.33668766 0.81795199]
   [0.86333686 0.10121043 0.65585511]
   [0.53269415 0.5442037  0.33534223]
   [0.42725368 0.35350347 0.96485797]]

  [[0.67249464 0.4645158  0.81795199]
   [0.86333686 0.30171043 0.73432242]
   [0.53269415 0.40951789 0.42569725]
   [0.4497047  0.63681454 0.96485797]]]]
'''

# 2.3.3 Tensor dot
x = np.array([[1, 2, 3],
              [0, -1, 3]])
y = np.array([[-1, 2, 0],
              [0, -1, 3]])

# z = np.dot(x, y)
# print(z)
# ValueError: shapes (2,3) and (2,3) not aligned: 3 (dim 1) != 2 (dim 0)

# 2.3.4 Tensor reshaping

y = y.reshape(3,2)
z = np.dot(x, y)
print(z)
'''
[[-4 11]
 [-3  9]]
'''
x = np.array([[0., 1.],
              [2., 3.],
              [4., 5.]])

print(x.shape)
x = x.reshape((6,1))
print(x)
'''
[[0.]
 [1.]
 [2.]
 [3.]
 [4.]
 [5.]]
'''
print(x.shape)  # (6, 1)

# 2.3.5 Geometric interpretation of tensor operations
'''
 In general, elementary geometric operations such as affine transformations, rotations,
 scaling, and so on can be expressed as tensor operations. For instance, a rotation of a
 2D vector by an angle theta can be achieved via a dot product with a 2 × 2 matrix
 R = [u, v] , where u and v are both vectors of the plane: u = [cos(theta),
 sin(theta)] and v = [-sin(theta), cos(theta)] .
'''

# 2.3.6 A geometric interpretation of deep learning
'''
 You just learned that neural networks consist entirely of chains of tensor operations and
 that all of these tensor operations are just geometric transformations of the input data.
 It follows that you can interpret a neural network as a very complex geometric transfor-
 mation in a high-dimensional space, implemented via a long series of simple steps.
    In 3D , the following mental image may prove useful. Imagine two sheets of colored
 paper: one red and one blue. Put one on top of the other. Now crumple them
 together into a small ball. That crumpled paper ball is your input data, and each sheet
 of paper is a class of data in a classification problem. What a neural network (or any
 other machine-learning model) is meant to do is figure out a transformation of the
 paper ball that would uncrumple it, so as to make the two classes cleanly separable
 again. With deep learning, this would be implemented as a series of simple transfor-
 mations of the 3D space, such as those you could apply on the paper ball with your fin-
 gers, one movement at a time.

    Uncrumpling paper balls is what machine learning is about: finding neat representa-
 tions for complex, highly folded data manifolds. At this point, you should have a
 pretty good intuition as to why deep learning excels at this: it takes the approach of
 incrementally decomposing a complicated geometric transformation into a long
 chain of elementary ones, which is pretty much the strategy a human would follow to
 uncrumple a paper ball. Each layer in a deep network applies a transformation that
 disentangles the data a little—and a deep stack of layers makes tractable an extremely
 complicated disentanglement process.
'''


# -*- coding: utf-8 -*-
"""
Created on Fri Jun  5 11:56:32 2020
Some functions to do linear regression
@author: Ji-Ping Dai
"""
import numpy as np
import matplotlib.pyplot as plt
plt.rc('text',usetex=True)
plt.rc('font',family='Times New Roman')

def plotdata(X,y,ax):
    '''PLOTDATA Plots the data points x and y into a new figure'''
    plt.xticks(fontsize=20)
    plt.yticks(fontsize=20)
    ax.set_ylabel('Profit in \$10,000',size=20)
    ax.set_xlabel('Population of City in 10,000s',size=20)
    ax.plot(X, y, 'rx',label='Training data')
    return

def costfunc(x,y,theta):
    '''Compute cost for linear regression'''
    m = len(y)
    J = 1/2/m*np.sum((x@theta-y)**2)
    return J

def gradientDescent(x,y,theta,iterations,alpha):
    '''Performs gradient descent to learn theta'''
    m = len(y)
    cost = np.zeros(iterations)
    for i in range(iterations):
        theta = theta-1/m*alpha*x.T@(x@theta-y)
        cost[i] = costfunc(x,y,theta)
    return theta, cost

def featureNormalize(X):
    '''Normalizes the features in X '''
    mu = np.mean(X,axis=0)
    sigma = np.std(X,axis=0)
    X = (X-mu)/sigma
    return X, mu, sigma


import math

import numpy as np
import matplotlib.pyplot as plt

def create_plot():
    "Funcție ajutătoare pentru crearea graficului."
    # Creez un subplot
    fig, ax = plt.subplots(1, dpi=200)

    # Setez axele de coordonate
    ax.spines['bottom'].set_position('zero')
    ax.spines['top'].set_color('none')
    ax.spines['right'].set_color('none')

    return ax

###
print("Exercițiul 1")

# Pentru a aproxima valoarea lui radical din 8,
# căutăm soluția pozitivă a ecuației x^2 - 8 = 0
f = lambda x: x ** 2 - 8

# Metoda bisecției, dezvoltată la primul laborator
def bisectie(f, a, b, epsilon=1e-8):
    """Găsește rădăcina funcției `f` pe intervalul `[a, b]`
    cu precizia `epsilon`.
    """

    # Calculăm valorile în capătul stâng
    f_a = f(a)

    # Prima estimare, mijlocul intervalului inițial
    x_num = (a + b) / 2

    # Numărul necesar de iterații
    num_iterations = math.floor(math.log2((b - a) / epsilon) - 1) + 1

    # Aplicăm algoritmul
    for step in range(num_iterations):
        value = f(x_num)

        # Am găsit fix valoarea căutată, ieșim
        if value == 0:
            break
        elif f_a * value < 0:
            b = x_num
        else:
            a = x_num

        # Luăm mijlocul noului interval
        x_num = (a + b) / 2

    return x_num

# Căutăm rădăcina pozitivă
root = bisectie(f, 0, 8)

print("Radical din 8 este", root)
print()

# Afișez funcția folosită pentru calcularea radicalului
xs = np.linspace(0, 8, int(1e5))

ax = create_plot()
ax.set_title("Găsirea rădăcinii pătrate")

ax.plot(xs, f(xs), label="x^2 - 8")
ax.scatter(root, 0, c='red')

ax.legend()
plt.show()

###
print("Exercițiul 2")

# Definesc cele două părți ale egalității
f = lambda x: np.exp(x - 2)
g = lambda x: np.cos(np.exp(x - 2)) + 1

h = lambda x: f(x) - g(x)

# Găsim soluția ecuației h(x) = 0, care va fi și soluția ecuației inițiale
# Intervalul [1, 3] a fost ales analizând graficul
root = bisectie(h, 1, 3)
print("Funcțiile se intersectează în", root)

# Afișăm grafic rezultatul
xs = np.linspace(0, 5, int(1e5))

ax = create_plot()
ax.set_title("Rezolvarea unei ecuații")

ax.plot(xs, f(xs), label="e^(x - 2)")
ax.plot(xs, g(xs), label="cos(e^(x - 2)) + 1")
ax.scatter(root, f(root), c='red')

ax.legend()
plt.show()

print()

###
print("Exercițiul 3")

def pozitie_falsa(f, a, b, epsilon=1e-5):
    """Găsește soluția ecuației `f(x) = 0` folosind metoda poziției false
    în intervalul [a, b], cu precizie `epsilon`.
    """

    f_a = f(a)
    f_b = f(b)

    prev = (a * f_b - b * f_a)/(f_b - f_a)
    f_prev = f(prev)

    # Setez o valoare lui new, pentru cazul când f_prev == 0
    # și ies imediat din buclă.
    new = prev

    num_iterations = 0
    while True:
        # Dacă am nimerit soluția exactă, mă opresc
        if f_prev == 0:
            break
        elif f_a * f_prev < 0:
            # Intervalul bun e cel din stânga
            b = prev
            f_b = f(b)
        else:
            # Intervalul bun e cel din dreapta
            a = prev
            f_a = f(a)

        # Calculez noul punct de intersecție
        new = (a * f_b - b * f_a)/(f_b - f_a)
        f_new = f(new)

        if np.abs(new - prev) / np.abs(prev) < epsilon:
            break

        prev, f_prev = new, f_new
        num_iterations += 1

    return new, num_iterations

f = lambda x: (x ** 3) - 19 * x + 30

# Calculez rădăcinile
# Intervalele au fost alese pe baza graficului
r1, n1 = pozitie_falsa(f, -6, -4)
print("Am găsit soluția", r1, "în", n1, "iterații")
r2, n2 = pozitie_falsa(f, 1, 2.5)
print("Am găsit soluția", r2, "în", n2, "iterații")
r3, n3 = pozitie_falsa(f, 2.5, 4)
print("Am găsit soluția", r3, "în", n3, "iterații")

xs = np.linspace(-5, 5, int(1e5))

ax = create_plot()
ax.set_title("Metoda poziției false")

# Afișez funcția
ax.plot(xs, f(xs), label='x^3 - 19x + 30')
# Afișez rădăcinile
ax.scatter([r1, r2, r3], [0, 0, 0], c='red')

ax.legend()
plt.show()

print()

###
print("Exercițiul 4")

def secanta(f, a, b, x0, x1, epsilon=1e-5):
    """Găsește rădăcina lui f pe intervalul [a, b],
    plecând de la punctele x0 și x1.
    """
    num_iterations = 0
    # Ne oprim când eroarea relativă scade sub epsilon
    while np.abs(x1 - x0) / np.abs(x0) >= epsilon:
        # Calculăm următorul punct folosind secanta
        x_new = (x0 * f(x1) - x1 * f(x0)) / (f(x1) - f(x0))

        if x_new < a or x_new > b:
            raise Exception("Valorile alese pentru x0 și x1 nu converg")

        x0, x1 = x1, x_new
        num_iterations += 1

    return x1, num_iterations

f = lambda x: (x ** 3) - 7 * x + 6

a, b = -3, 3

r1, n1 = secanta(f, a, b, -3, -2.5)
print("Am găsit soluția", r1, "în", n1, "iterații")
r2, n2 = secanta(f, a, b, 0.5, 1.5)
print("Am găsit soluția", r2, "în", n2, "iterații")
r3, n3 = secanta(f, a, b, 1.5, 2.5)
print("Am găsit soluția", r3, "în", n3, "iterații")

xs = np.linspace(a, b, int(1e5))

ax = create_plot()
ax.set_title("Metoda secantei")

# Afișez funcția
ax.plot(xs, f(xs), label='x^3 - 7x + 6')
# Afișez rădăcinile
ax.scatter([r1, r2, r3], [0, 0, 0], c='red')

ax.legend()
plt.show()


#！/usr/bin/python
#-*- coding: <encoding name> -*-
import numpy as np
import pandas as pd
import os
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D

class Burgers:
	'''
	A Burgers 1-demension partial differential equation numerical solver with:
		- 3 inputs: viscosity ,time and iteration times N
		- viscosity = 0, 0.01, 0.05
		- time = 0.5, 1.2, 2.0
	Initial and boundary conditions are:
		- u(x,0) = -0.5*x, x belongs to [-1, 1]
	    - u(-1, t) = 0.5, u(1, t) = -0.5
	'''

	def __init__(self, visco, time, N):
		# Give the viscosity, time to stop and gird spacing.
	    self.visco = visco
	    self.time = time
	    self.N = N

	def discretization(self):
		# Discretization of the calculation zone.
		dx = 2.0 / self.N
		dt = 0.0001
		return (dx, dt)

	def macCormack(self, u):
		'''
		u is a numpy array with initial condition.
			- Predictive step: us;
			- Patial of test step: up
			- Intermediate step: unew
			- Storage of all the time step: U
		'''
		dx, dt = self.discretization()
		nt = int(self.time / dt)

		us = np.zeros(self.N + 1)
		up = np.zeros(self.N + 1)
		unew = np.zeros(self.N + 1)
		U = np.zeros((nt + 1, self.N + 1))
		U[0] = u

		for i in range(1, nt + 1):
			# Predictive step
			us[1: -1] = u[1: -1] + (self.visco * dt / (dx) ** 2) * (u[2:] - 2.0 * u[1:-1] + u[0:-2]) \
			- dt / (2.0 * dx) * (u[2:] ** 2 - u[1:-1] ** 2)
			us[0] = 0.5
			us[-1] = -0.5

			# Correction step and combine step
			up[1: -1] = (self.visco / (dx) ** 2) * (us[2:] - 2 * us[1: -1] + us[0: -2]) \
			- 1.0 / (2.0 * dx) * (us[1:-1] ** 2 - us[0: -2] ** 2)
			unew[1: -1] = 0.5 * (u[1: -1] + us[1: -1] + dt * up[1: -1])
			u[1: -1] = unew[1: -1]

			U[i] = u

		return U

if __name__ == '__main__':
	N = 200
	seper = os.sep 		# Use this seperator to satisfy both Windows and Linux usage

	# Read input file of time and viscosity
	time_visco = pd.read_csv('input.txt', sep = ',', usecols=['time', 'mu'])
	time = max(time_visco['time'])
	visco = time_visco['mu']

	# Compute Burgers equation at different viscosity
	for i in range(0, len(visco)):
		u = np.zeros(N + 1)
		x = np.zeros(N + 1)

		for j in range(0, N + 1):
			u[j] = -0.5 * (-1 + j * 2 / N)
			x[j] = -1 + j * 2 / N

		burgers = Burgers(visco[i], time, N)
		uall = burgers.macCormack(u)

		# Save 2-D data to files
		output = pd.DataFrame({'x': x, 't='+str(time_visco['time'][0]): uall[int(time_visco['time'][0] * 10000)], \
			't='+str(time_visco['time'][1]): uall[int(time_visco['time'][1] * 10000)], \
			't='+str(time_visco['time'][2]): uall[int(time_visco['time'][2] * 10000)]})
		output.to_csv('2D' + seper + 'mu=' + str(visco[i]) + '.dat', sep='\t', index=False)

		'''
			Plot 3-D Burgers results, this cost about 16 s, so I comment code below.
			If you want to generate 3-D images by yourself, please uncomment the code below,
			and change some value of the code, to see if there are some differences between
			my output 3-D images
		'''

		'''
		fig = plt.figure()
		ax = fig.add_subplot(111, projection='3d')
		y = np.array([[i * 0.0001] for i in range(0, len(uall))])
		ax.plot_surface(x, y, uall, cmap='viridis')
		ax.set_xlabel('x')
		ax.set_ylabel('t')
		ax.set_zlabel('u')
		ax.set_title(r'$\mu = $' + str(visco[i]))
		plt.savefig('3D' + seper + 'mu=' + str(visco[i]) + '.png', bbox_inches='tight')
		'''


#!/usr/bin/env python
# coding: utf-8

# In[1]:

import numpy as np
import pandas as pd
import math
from sklearn.model_selection import KFold
from scipy.stats import poisson
import matplotlib.pyplot as plt
from sklearn.metrics import pairwise_distances

# # Problem-2 Classification
#
# #### 1- Naive Bayes,  2- KNN

# In[2]:

X = pd.read_csv("Bayes_classifier\\X.csv",header=None).values
y = pd.read_csv("Bayes_classifier\\y.csv",header=None).values
kf = KFold(n_splits=10,shuffle=True,random_state=0)

# In[3]:

def compute_confusion_matrix(y_test,y_prediction):
    TP = 0
    FP = 0
    TN = 0
    FN = 0
    for i in range(len(y_test)):
        if(y_prediction[i] == y_test[i]):
            if(y_prediction[i] == 0):
                TN+=1
            else:
                TP+=1
        else:
            if(y_prediction[i] == 0):
                FN+=1
            else:
                FP+=1
    return [TP,FP,FN,TN]

# ####  Naive Bayes

# In[4]:

TP = []
FP = []
TN = []
FN = []
lambda0=[]
lambda1=[]
for train_index,test_index in kf.split(X):
    X_train,X_test = X[train_index],X[test_index]
    y_train,y_test = y[train_index],y[test_index]

    py0 = len(np.where(y_train == 0)[0])/len(X_train)
    py1 = len(np.where(y_train == 1)[0])/len(X_train)

    lambda_0_values = []
    lambda_1_values = []

    for j in range(len(X_train[0])):
        lambda0_numerator = 1
        lambda0_denominator = 1
        lambda1_numerator = 1
        lambda1_denominator = 1

        for i in range(len(X_train)):
            lambda1_numerator+=(y_train[i] * X_train[i][j])
            lambda1_denominator+=(y_train[i])

            lambda0_numerator+=((1-y_train[i]) * X_train[i][j])
            lambda0_denominator+=(1-y_train[i])
        lambda_0_values.append((lambda0_numerator/lambda0_denominator)[0])
        lambda_1_values.append((lambda1_numerator/lambda1_denominator)[0])

    lambda0.append(lambda_0_values)
    lambda1.append(lambda_1_values)
    y_predictions = []
    for i in range(len(X_test)):

        y_prob = []
        product_0 = 1
        product_1 = 1

        for j in range(len(X_test[0])):
            poisson_prob_0 = poisson.pmf(X_test[i][j],lambda_0_values[j])
            poisson_prob_1 = poisson.pmf(X_test[i][j],lambda_1_values[j])

            product_0*=poisson_prob_0
            product_1*=poisson_prob_1

        y_prob.append(product_0*py0)
        y_prob.append(product_1*py1)

        y_predictions.append(np.argmax(y_prob))

    results = compute_confusion_matrix(y_test,y_predictions)

    TP.append(results[0])
    FP.append(results[1])
    FN.append(results[2])
    TN.append(results[3])

# In[5]:

confusion_matrix = [[sum(TP),sum(FP)],[sum(FN),sum(TN)]]
print("The Confusion Matrix is:")

print(confusion_matrix[0])
print(confusion_matrix[1])

print("The Accuracy is:", ((confusion_matrix[0][0] + confusion_matrix[1][1]) /
                           (confusion_matrix[0][0] + confusion_matrix[0][1] +confusion_matrix[1][0] + confusion_matrix[1][1])))

# In[6]:

avg_poisson_parameter_0 = np.mean(lambda0, axis=0)
avg_poisson_parameter_1 = np.mean(lambda1, axis=0)

# In[23]:

fig, ax = plt.subplots(nrows=2, ncols=1, figsize=(15, 10))
ax[0].stem(range(1,55),avg_poisson_parameter_0,"blue", markerfmt="bo")
ax[0].set_title("Stem plot of Poisson Parameters (Class 0: Not Spam)")
ax[0].set_xlabel("feature number")
ax[0].set_ylabel("value of lambda")
ax[0].set_xticks(np.arange(1,55, step=1))

ax[1].stem(range(1,55),avg_poisson_parameter_1,"red", markerfmt="ro")
ax[1].set_title("Stem plot of Poisson Parameters (Class 1: Spam)")
ax[1].set_xlabel("feature number")
ax[1].set_ylabel("value of lambda")
ax[1].set_xticks(np.arange(1,55, step=1))
plt.show()

# In[21]:

plt.figure(figsize=(15, 5))
plt.stem(range(1,55), avg_poisson_parameter_0, "blue", markerfmt="bo", label="0 Class: Not Spam")
plt.stem(range(1,55), avg_poisson_parameter_1, "red", markerfmt="ro", label="1 Class: Spam")
plt.title("Stem plot of Poisson Parameters (seperated by classes)")
plt.xlabel("Feature number")
plt.ylabel("Value of Lambda")
plt.xticks(np.arange(1,55, step=1))
plt.legend()
plt.show()

# #### KNN

# In[140]:

TP = []
FP = []
TN = []
FN = []
all_predictions=[]
all_labels = []
for train_index,test_index in kf.split(X):
    X_train,X_test = X[train_index],X[test_index]
    y_train,y_test = y[train_index],y[test_index]
    all_labels.append([i[0] for i in y_test])
    tp = np.zeros(shape = (20))
    fp = np.zeros(shape = (20))
    fn = np.zeros(shape = (20))
    tn = np.zeros(shape = (20))
    mean = np.mean(X_train,axis=0)
    std = np.std(X_train,axis=0)
    X_train = (X_train - mean)/std
    X_test = (X_test - mean)/std
    predictions=[]
    for row in range(len(X_test)):
        distances=[]
        #computing the distance matrix
        for j in X_train:
            distances.append(np.sum(np.abs(X_test[row]-j)))
#       Finding 20 Nearest neighbours of the test sample.
#       list_neighbours = [i[0] for i in sorted(enumerate(distances), key=lambda x:x[1])][:20]
        list_neighbours = np.argsort(distances)[:20]
        pred = []
        for k in range(1,21):
            count_1 = len(np.where(y_train[list_neighbours[:k]] == 1)[0])
            count_0 = len(np.where(y_train[list_neighbours[:k]] == 0)[0])
            # picking the nearest neighbour's label in the case of tie.
            if(count_1==count_0):
                temp_prediction = y_train[list_neighbours[0]][0]
                pred.append(temp_prediction)
                if(temp_prediction == y_test[row][0]):
                    if(temp_prediction == 1):
                        tp[k-1]+=1
                    else:
                        tn[k-1]+=1
                else:
                    if(temp_prediction == 1):
                        fp[k-1]+=1
                    else:
                        fn[k-1]+=1
            elif(count_1>count_0):
                temp_prediction = 1
                pred.append(temp_prediction)
                if(temp_prediction == y_test[row][0]):
                    tp[k-1]+=1
                else:
                    fp[k-1]+=1
            else:
                temp_prediction = 0
                pred.append(temp_prediction)
                if(temp_prediction == y_test[row][0]):
                    tn[k-1]+=1
                else:
                    fn[k-1]+=1
        predictions.append(pred)
    all_predictions.append(predictions)
    TP.append(list(tp))
    FP.append(list(fp))
    TN.append(list(tn))
    FN.append(list(fn))

# In[142]:

TP_list = (list(np.sum(TP,axis=0)))
FP_list = (list(np.sum(FP,axis=0)))
FN_list = (list(np.sum(FN,axis=0)))
TN_list = (list(np.sum(TN,axis=0)))
prediction_accuracies_cumulative = []
for i in range(20):
    prediction_accuracies_cumulative.append((TP_list[i] + TN_list[i])/
                                            (TP_list[i] + TN_list[i] + FP_list[i] + FN_list[i]))

# In[143]:

plt.plot(range(1,21),prediction_accuracies_cumulative)
plt.scatter(range(1,21),prediction_accuracies_cumulative)
plt.title("Accuracy v/s K")
plt.xlabel("K")
plt.ylabel("Accuracy")
plt.xticks(np.arange(1,21, step=1))
plt.show()

# # Problem-3 : Gaussian Process

# In[69]:

X_train = pd.read_csv("Gaussian_process\\X_train.csv",header=None).values
y_train = pd.read_csv("Gaussian_process\\y_train.csv",header=None).values
X_test = pd.read_csv("Gaussian_process\\X_test.csv",header=None).values
y_test = pd.read_csv("Gaussian_process\\y_test.csv",header=None).values

# In[70]:

def RMSE(y_predicted,y_test):
    y_predicted = np.array(y_predicted)
    y_test = np.array(y_test)
    return np.sqrt(np.sum((y_predicted - y_test)**2)/len(y_test))

# In[71]:

class Gaussian_process:
    def __init__(self,X_train,y_train, sigma2, b):
        self.sigma2 = sigma2
        self.b = b
        self.X_train = X_train
        self.y_train = y_train

    def Kernel(self,xi,xj):
        return math.exp((-1/self.b) * np.sum((xi -xj)**2))

    def fit(self):
        K = []
        for i in self.X_train:
            temp = []
            for j in self.X_train:
                temp.append(self.Kernel(i,j))
            K.append(temp)
        self.K = np.asarray(K)

    def predict(self,x0):
        temp = []
        for j in self.X_train:
            temp.append(self.Kernel(x0,j))
        KxD = np.asarray(temp)
        sigma2I = self.sigma2 * np.identity(self.K.shape[0])
        mean = (np.matmul(np.matmul(KxD,np.linalg.inv(sigma2I + self.K)), self.y_train))[0]
        variance = self.sigma2 + self.Kernel(x0,x0) + np.matmul(np.matmul(KxD,np.linalg.inv(sigma2I + self.K)), np.transpose(KxD))

        return [mean,variance]

# In[51]:

B = [5,7,9,11,13,15]
Sigma2 = [0.1,0.2,0.3,0.4,0.5,0.6,0.7,0.8,0.9,1]

# In[52]:

RMSES = []
y_test = [i[0] for i in y_test]
for b in B:
    temp = []
    for sigma2 in Sigma2:
        y_predicted = []
        process = Gaussian_process(X_train,y_train,sigma2 = sigma2, b = b)
        process.fit()
        for i in X_test:
            y_predicted.append(process.predict(i)[0])
        temp.append(RMSE(y_predicted,y_test))
    RMSES.append(temp)

# In[53]:

rmse_values = pd.DataFrame(RMSES, columns = Sigma2, index = B)

# In[55]:

rmse_values

# In[72]:

X_carweight_train = X_train[:,3]
X_carweight_test = X_test[:,3]

# In[73]:

b2 = 5
sigma22 = 2

# In[74]:

y_predicted_train = []
process = Gaussian_process(X_carweight_train,y_train,sigma2 = sigma22, b = b2)
process.fit()
for i in X_carweight_train:
    y_predicted_train.append(process.predict(i)[0])

# In[75]:

y_train = [i[0] for i in y_train]
X_carweight_train = list(X_carweight_train)
y_preds = [i for i,_ in sorted(zip(y_predicted_train,X_carweight_train), key=lambda pair: pair[1])]

# In[79]:

plt.figure(figsize=(5,5))
plt.scatter(X_carweight_train,y_train, label = "True values of y")
plt.plot(sorted(X_carweight_train),y_preds,color="red",linewidth=4.0,label = "Predicted values of y")
plt.xlabel("x[4] : Car Weight")
plt.ylabel("y : Miles Per Gallon")
plt.title("Miles Per Gallon v/s Car weight")
plt.legend()
plt.show()

# In[ ]:

Bayes Classifier and Gaussian Process/hw2_pds2136.py


import numpy as np
import matplotlib.pyplot as plt
from autologging import logged, traced

from fullwavepy.generic.parse import kw
from fullwavepy.numeric.const import *
from fullwavepy.numeric.generic import *

@traced
@logged
def derivative(y, **kwargs):
  """
  Calculate a derivative.

  Parameters
  ----------
  y : array
    A vector of numbers.
  **kwargs : keyword arguments (optional)
    Current capabilities:
    - n : int
      Order of derivative.
      Default: 1.
    - acc : int
      Order of numerical accuracy.
      Default: 2.
    - dx : float
      Grid-cell size.
      Default: 1.

  Returns
  -------
  deriv : array
    Derivative of y. It has
    the same length as y.

  Notes
  -----
  We might as well use the scipy.misc.derivative or
  numpy.gradient but an own implementation
  is always more instructive.

  Benchmarked against numpy.gradient. Same results.

  Amplitude is the same as analytical derivative,
  no normalization needed - don't forget about
  differentiation rules that are applied implicitly,
  e.g.: d(sin(wx))/dx = w cos(wx)

  """
  n = kw('n', 1, kwargs) # ORDER OF DERIVATIVE
  acc = kw('acc', 2, kwargs) # ORDER OF NUMERICAL ACCURACY
  dx = kw('dx', 1, kwargs)

  deriv = np.zeros(len(y))

  if n == 1:
    if acc == 2:
      # INTERIOR
      deriv[1:-1] = [(-0.5*y[i-1] + 0.5*y[i+1]) / dx for i in range(1, len(y)-1)]

      # LEFT EDGE - FORWARD FD
      deriv[0] = ((-3/2.)*y[0] + 2*y[1] - 1/2.*y[2]) / dx

      # RIGHT EDGE - BACKWARD FD (NOTE: OPPOSITE SIGNS)
      deriv[-1] = (3/2.*y[-1] - 2*y[-2] + 1/2.*y[-3]) / dx

    else:
      raise NotImplementedError('Order of accuracy: %s' % acc)

  else:
    raise NotImplementedError('Order of derivative: %s' % n)

  return deriv

# -------------------------------------------------------------------------------


import numpy as np
from numba import jit, njit
from copy import copy
from enum import Enum, unique, auto
import 靴紐

@unique
class Order(Enum):
    CLOCKWISE = auto()
    COUNTERCLOCKWISE = auto()

@jit
def ensure(xys, order=Order.COUNTERCLOCKWISE):
    """
    args
        xys, ndarray
            of shape (n_pts, 2)
        order, CLOCKWISE/COUNTERCLOCKWISE

    return
        ordered_xys, ndarray

    Note. This function only works for convex polygones in 2D space.
    """
    if order not in set(Order):
        raise TypeError("order arg should be either Order.CLOCKWISE or Order.COUNTERCLOCKWISE")

    # We construct a list of indices that corresponds to clockwise order
    n = len(xys)
    clockwise_indices = [0]
    remaining_indices = list(range(1, n))
    while len(remaining_indices) > 1:
        best_index = remaining_indices[0]
        u = xys[best_index, :] - xys[clockwise_indices[-1], :]
        for index in range(1, len(remaining_indices)):
            v = xys[remaining_indices[index], :] - xys[clockwise_indices[-1], :]
            A = np.array([u, v])
            determinant = np.linalg.det(A)
            if determinant > 0:
                best_index = remaining_indices[index]
                u = v
            elif determinant == 0:
                raise ValueError("det(A) == 0 => two edges of the polygone colinear")
        clockwise_indices.append(best_index)
        remaining_indices.remove(best_index)
    clockwise_indices.append(remaining_indices[0])
    if order == Order.CLOCKWISE:
        ordered_xys = xys[clockwise_indices]
    else:
        ordered_xys = xys[[clockwise_indices[i%n] for i in range(0, -n, -1)]]
    return ordered_xys

@jit
def area(xys):
    xs = xys[:, 0]
    ys = xys[:, 1]
    n = len(xys)
    # the following line cannot be jit-ed, but the one following it can,
    # no jit here does not give good performance, if not worsen.
    #A = sum(ys[k]*(xs[(k-1)%n] - xs[(k+1)%n])for k in range(n)) * 0.5
    A = sum([ys[k]*(xs[(k-1)%n] - xs[(k+1)%n])for k in range(n)]) * 0.5
    return A

@jit
def area_vectorize(xys):
    xs = xys[:, 0]
    ys = xys[:, 1]
    n = len(xys)
    shifted_left = xs[[(i-1) % n for i in range(n)]]
    shifted_right = xs[[(i+1) % n for i in range(n)]]
    A = np.dot(ys, shifted_left - shifted_right).astype(np.float32) * 0.5
    return A

@njit
def area_stackoverflow(xys):
    xs = xys[:, 0]
    ys = xys[:, 1]
    S1 = np.sum(xs * np.roll(ys, -1))
    S2 = np.sum(ys * np.roll(xs, -1))
    A = .5 * np.abs(S1 - S2)
    return A


# Example of an HMM with Gaussian emission in 2D
# For a matlab version, see https://github.com/probml/pmtk3/blob/master/demos/hmmLillypadDemo.m

# Author: Gerardo Durán-Martín (@gerdm)

import numpy as np
import matplotlib.pyplot as plt
from scipy.stats import multivariate_normal
import pyprobml_utils as pml

def sample_hmm(n_samples, initial_probs, mu_collection, cov_collection):
    """
    Sample from a 2-dimensional HMM

    Parameters
    ----------
    n_samples: int
        Number of samples to be generated
    initial_probs: numpy.ndarray(n_states, )
        Initial probabilities for each state
    mu_collection: np.array(n_states, dim)
        Collection of means for each state
    cov_collection: np.array(n_states, dim, dim)
        Collection of covariances for each state

    Returns
    -------
    * numpy.ndarray(n_samples, d)
        Observations
    * numpy.ndarray(n_samples, )
        Latent states of the system
    """
    samples_obs = np.zeros((n_samples, 2))
    samples_state = np.zeros((n_samples,), dtype=int)
    zi = np.random.choice(z, p=initial_probs)
    N = multivariate_normal(mu_collection[zi], cov_collection[zi])
    samples_obs[0] = N.rvs()
    samples_state[0] = zi

    for i in range(1, n_samples):
        zi = np.random.choice(z, p=A[zi])
        N = multivariate_normal(mu_collection[zi], cov_collection[zi])
        samples_obs[i] = N.rvs()
        samples_state[i] = zi
    return samples_obs, samples_state

def plot_2dhmm(samples_obs, samples_state, colors, ax, xmin, xmax, ymin, ymax):
    """
    Plot the trajectory of a 2-dimensional HMM

    Parameters
    ----------
    samples_obs: numpy.ndarray(n_samples, 2)
        Observations
    samples_state: numpy.ndarray(n_samples, )
        Latent state of the system
    colors: list(int)
        List of colors for each latent state

    Returns
    -------
    * matplotlib.axes
    * colour of each latent state
    """
    color_sample = [colors[i] for i in samples_state]
    Xgrid = np.mgrid[xmin:xmax:0.01, ymin:ymax:0.01]

    for k, (mu, S, color) in enumerate(zip(mu_collection, cov_collection, colors)):
        N = multivariate_normal(mean=mu, cov=S)
        Z = np.apply_along_axis(N.pdf, 0, Xgrid)
        ax.contour(*Xgrid, Z, levels=[1], colors=color, linewidths=3)
        ax.text(*(mu + 0.13), f"$k$={k + 1}", fontsize=13, horizontalalignment="right")
    ax.plot(*samples_obs.T, c="black", alpha=0.3, zorder=1)
    ax.scatter(*samples_obs.T, c=color_sample, s=30, zorder=2, alpha=0.8)
    return ax, color_sample

if __name__ == "__main__":
    plt.rcParams["axes.spines.right"] = False
    plt.rcParams["axes.spines.top"] = False

    initial_probs = np.array([0.3, 0.2, 0.5])

    # transition matrix
    A = np.array([
        [0.3, 0.4, 0.3],
        [0.1, 0.6, 0.3],
        [0.2, 0.3, 0.5]
    ])

    S1 = np.array([
        [1.1, 0],
        [0, 0.3]
    ])

    S2 = np.array([
        [0.3, -0.5],
        [-0.5, 1.3]
    ])

    S3 = np.array([
        [0.8, 0.4],
        [0.4, 0.5]
    ])

    cov_collection = np.array([S1, S2, S3]) / 60
    mu_collection = np.array([
        [0.3, 0.3],
        [0.8, 0.5],
        [0.3, 0.8]
    ])

    np.random.seed(314)
    n_samples = 50
    z = np.array([0, 1, 2])
    colors = ["tab:green", "tab:blue", "tab:red"]

    xmin, xmax = 0, 1
    ymin, ymax = 0, 1.2
    samples_obs, samples_state = sample_hmm(n_samples, initial_probs, mu_collection, cov_collection)

    fig, ax = plt.subplots()
    _, color_sample = plot_2dhmm(samples_obs, samples_state, colors, ax, xmin, xmax, ymin, ymax)
    pml.savefig("hmm_lillypad_2d.pdf")

    fig, ax = plt.subplots()
    ax.step(range(n_samples), samples_state, where="post", c="black", linewidth=1, alpha=0.3)
    ax.scatter(range(n_samples), samples_state, c=color_sample, zorder=3)
    pml.savefig("hmm_lillypad_step.pdf")
    plt.show()


import numpy as np

alist = [1,2,3,4,5]
narray = np.array([1,2,3,4])

print(alist)
print(narray)

print(type(alist))
print(type(narray))

print(narray + narray)
print(alist + alist)

print(narray * 3)
print(alist * 4)

npmatrix1 = np.array([narray,narray,narray])
npmatrix2 = np.array([alist,alist,alist])
npmatrix3 = np.array([narray,[1,1,1,1],narray])

print(npmatrix1)
print(npmatrix2)
print(npmatrix3)

okmatrix = np.array([[1,2],[3,4]])
print(okmatrix)
print(okmatrix * 2)

print('bad matrix :')

badmatrix = np.array([[1,2],[3,4],[5,6,7]])
print(badmatrix)
print(badmatrix * 2)

result = okmatrix * 2 + 1
print('result',result)

result1 = okmatrix + okmatrix
print('result 1 :',result1)

result2 = okmatrix - okmatrix
print('result 2 :',result2)

result = okmatrix * okmatrix
print(result)

matrix3x2 = np.array([[1,2],[3,4],[5,6]])
print('original matrix :')
print(matrix3x2)
print('Transposed matrix :')
print(matrix3x2.T)

nparray = np.array([1,2,3,4])
print('Original array')
print(nparray)
print('Transposed array')
print(nparray.T)

nparray = np.array([[1,2,3,4]])
print('Original array')
print(nparray)
print('Transposed array')
print(nparray.T)

nparray1 = np.array([1,2,3,4])
norm1 = np.linalg.norm(nparray1)

nparray2 = np.array([[1,2],[3,4]])
norm2 = np.linalg.norm(nparray2)

print(norm1)
print(norm2)

nparray2 = np.array([[1, 1], [2, 2], [3, 3]])

normByCols = np.linalg.norm(nparray2, axis=0)
normByRows = np.linalg.norm(nparray2, axis=1)

print(normByCols)
print(normByRows)

nparray1 = np.array([0, 1, 2, 3])
nparray2 = np.array([4, 5, 6, 7])

flavor1 = np.dot(nparray1, nparray2)
print("flavor1 : ",flavor1)

flavor2 = np.sum(nparray1 * nparray2)
print("flavor2 :",flavor2)

flavor3 = nparray1 @ nparray2
print("flavor3 :",flavor3)

flavor4 = 0
for a, b in zip(nparray1, nparray2):
    flavor4 += a * b

print("flavor4 : ",flavor4)

norm1 = np.dot(np.array([1,2]),np.array([3,4]))
norm2 = np.dot([1,2],[3,4])

print(norm1, '=' , norm2)

nparray2 = np.array([[1,-1],[2,-2],[3,-3]])

sumByCols = np.sum(nparray2, axis=0)
sumByRows = np.sum(nparray2, axis=1)

print('Sum By Column : ')
print(sumByCols)
print('Sum by Rows : ')
print(sumByRows)

# nparray2 = np.array([[1,-1],[2,-2],[3,-3]])
mean = np.mean(nparray2)
meanByCols = np.mean(nparray2,axis=0)
meanByRows = np.mean(nparray2,axis=1)

print('mean By Column : ')
print(meanByCols)
print('mean by Rows : ')
print(meanByRows)

nparray2 = np.array([[1,1],[2,2],[3,3]])

nparrayCentered = nparray2 - np.mean(nparray2,axis=0)

print('Original metrix')
print(nparray2)
print('Centered by columns matrix')
print(nparrayCentered)
print('New mean by column')
print(nparrayCentered.mean(axis=0))

nparray2 = np.array([[1,3],[2,4],[3,5]])

nparrayCentered = nparray2.T - np.mean(nparray2, axis=1)
nparrayCentered = nparrayCentered.T

print('Original metrix')
print(nparray2)
print('Centered by columns matrix')
print(nparrayCentered)

print('New mean by column')
print(nparrayCentered.mean(axis=1))

#Exercise
NumPyarray1 = np.array([[1,2],[3,4],[5,6]])
NumPyarray2 = np.array([[1,2,3],[3,4,5]])


"""
Sierpinski3d fractal
"""
# Credits: K3D authors at
# https://github.com/K3D-tools/K3D-jupyter/tree/master/examples
#https://simple.wikipedia.org/wiki/Sierpinski_triangle
import numpy as np

N = 30000
x = np.zeros(N)
y = np.zeros(N)
z = np.zeros(N)

x1 = np.empty_like(x)
y1 = np.empty_like(y)
z1 = np.empty_like(z)

# Sierpinski triangle iterative functions
def f1(x, y, z, x1, y1, z1, c):
    x1[c] = 0.5*x[c]
    y1[c] = 0.5*y[c]
    z1[c] = 0.5*z[c]

def f2(x, y, z, x1, y1, z1, c):
    x1[c] = 0.5*x[c] + 1 / 2.0
    y1[c] = 0.5*y[c]
    z1[c] = 0.5*z[c]

def f3(x, y, z, x1, y1, z1, c):
    x1[c] = 0.5*x[c] + 1 / 4.0
    y1[c] = 0.5*y[c] + np.sqrt(3) / 4
    z1[c] = 0.5*z[c]

def f4(x, y, z, x1, y1, z1, c):
    x1[c] = 0.5*x[c] + 1 / 4.0
    y1[c] = 0.5*y[c] + 1.0 / 4
    z1[c] = 0.5*z[c] + np.sqrt(3) / 4

functions = [f1, f2, f3, f4]
probabilities = [1 / 4.0] * 4
assert len(functions) == len(probabilities)

X, Y, Z = x, y, z
for i in range(20):
    # pick indices for each function to be applied
    r = np.random.choice(len(probabilities), size=N, p=probabilities)
    for i, f in enumerate(functions):
        f(x, y, z, x1, y1, z1, r == i)
    x, x1 = x1, x
    y, y1 = y1, y
    z, z1 = z1, z
    if i > 0:
        X, Y, Z = np.hstack([X,x]), np.hstack([Y,y]), np.hstack([Z,z])

# how much memory are we using, how many points there are
print("used mem, Npts=", 3 * X.nbytes // 1024 ** 2, "MB", X.shape[0])

from vtkplotter import Points

Points([X, Y, Z], c="tomato").show(axes=1, bg='white', zoom=2.0)


import numpy as np

import jax.numpy as jnp
from jax import ops

import numpyro
import numpyro.distributions as dist

def linreg_model(X, y):

    ndims = X.shape[1]
    a = numpyro.sample("a", dist.Normal(0, 0.5))

    beta = numpyro.sample("beta", dist.Normal(0, 0.5).expand([ndims]))
    sigma_y = numpyro.sample("sigma_y", dist.Exponential(1))

    # LKJ is the distribution to model correlation matrices.
    rho = numpyro.sample("rho", dist.LKJ(ndims, 2))  # correlation matrix
    sigma_x = numpyro.sample("sigma_x", dist.Exponential(1).expand([ndims]))
    covariance_x = jnp.outer(sigma_x, sigma_x) * rho  # covariance matrix
    mu_x = numpyro.sample("mu_x", dist.Normal(0, 0.5).expand([ndims]))

    numpyro.sample("X", dist.MultivariateNormal(mu_x, covariance_x), obs=X)

    mu_y = a + X @ beta

    numpyro.sample("y", dist.Normal(mu_y, sigma_y), obs=y)

def linreg_imputation_model(X, y):

    ndims = X.shape[1]
    a = numpyro.sample("a", dist.Normal(0, 0.5))

    beta = numpyro.sample("beta", dist.Normal(0, 0.5).expand([ndims]))
    sigma_y = numpyro.sample("sigma_y", dist.Exponential(1))

    # X_impute contains imputed data for each feature as a list
    # X_merged is the observed data filled with imputed values at missing points.
    X_impute = [None] * ndims
    X_merged = [None] * ndims

    for i in range(ndims):  # for every feature
        no_of_missed = int(np.isnan(X[:, i]).sum())

        if no_of_missed != 0:
            # each nan value is associated with a imputed variable of std normal prior.
            X_impute[i] = numpyro.sample(
                "X_impute_{}".format(i), dist.Normal(0, 1).expand([no_of_missed]).mask(False))

            # merging the observed data with the imputed values.
            missed_idx = np.nonzero(np.isnan(X[:, i]))[0]
            X_merged[i] = ops.index_update(X[:, i], missed_idx, X_impute[i])

        # if there are no missing values, its just the observed data.
        else:
            X_merged[i] = X[:, i]

    merged_X = jnp.stack(X_merged).T

    # LKJ is the distribution to model correlation matrices.
    rho = numpyro.sample("rho", dist.LKJ(ndims, 2))  # correlation matrix
    sigma_x = numpyro.sample("sigma_x", dist.Exponential(1).expand([ndims]))
    covariance_x = jnp.outer(sigma_x, sigma_x) * rho  # covariance matrix
    mu_x = numpyro.sample("mu_x", dist.Normal(0, 0.5).expand([ndims]))

    numpyro.sample("X_merged", dist.MultivariateNormal(mu_x, covariance_x), obs=merged_X)

    mu_y = a + merged_X @ beta

    numpyro.sample("y", dist.Normal(mu_y, sigma_y), obs=y)


#!/usr/bin/env python3
import numpy as np

def test1():
    print("\ntest1")
    x = np.array([1,2,3,4,5])
    print(x)

    print("np.sum(x)=",np.sum(x)) # sum
    print("np.mean(x)=",np.mean(x)) # average
    print("np.var(x)=",np.var(x)) # dispertion
    print("np.std(x)=",np.std(x)) # standard diviation

    x = np.arange(5)
    print("\nx=np.arange(5)")
    print("x=",x)

    x = np.arange(1, 3, 0.2)
    print("\nx=np.arange(1,3,0.2)")
    print("x=",x)
    print("x.dtype=", x.dtype)

    x = np.array([1,2,3], dtype=float)
    print("\nx = np.array([1,2,3], dtype=float)")
    print("x=",x)

def test2():
    print("\ntest2")
    a = np.array([[1,2,3],[4,5,6]])
    print("a=",a)
    a = np.arange(15.).reshape(3,5)
    print("a=",a)
    print("a.shape=", a.shape)
    print("a.ndim=", a.ndim)
    print("a.size=", a.size)
    b = a.reshape(5, -1)
    print("b = a.reshape(5, -1)=", b)
    print("b.ravel()=", b.ravel())
    print("b[:, np.newaxis]=", b[:,np.newaxis])
    print("b[:, None]=", b[:, None])

def test3():
    print("\ntest3")
    print("\nnp.zeros((2,3))=",np.zeros((2,3)))
    print("\nnp.ones((3,3))=",np.ones((3,3)))
    print("\nnp.ones((2,2))*3=", np.ones((2,2))*3)
    print("\nnp.empty((5,5))=",np.empty((5,5)))
    print("\nnp.linspace(0,1,10)=", np.linspace(0,1,10))

def test4():
    print("\ntest4")
    a = np.array([[1,1,1],[1,1,1]])#2*3
    print("\na=", a)
    b = np.array([[2,2,2,2],[2,2,2,2]])#2*4
    print("\nb=", b)
    print("\nnumpy.c_[a,b]=",np.c_[a,b])
    a = np.array([[1,1,1],[1,1,1]])#2*3
    print("\na=", a)
    b = np.array([[2,2,2],[2,2,2],[2,2,2]])#3*3
    print("\nb=", b)
    print("\nnumpy.r_[a,b]=",np.r_[a,b])

def test5():
    print("\ntest5")
    a = np.arange(3.,8.)
    print("\na=",a)
    print("\nnp.exp(a)=",np.exp(a))
    print("\nnp.log(a)=",np.log(a))
    print("\nnp.sqrt(a)=",np.sqrt(a))
    print("\na+3=",a+3)
    print("\na*3=",a*3)
    print("\na>4=",a>4)
    print("\na[a>4]=",a[a>4])

    print("\nnp.arange(9.).reshape(3,3)=", np.arange(9.).reshape(3,3))

def test6():
    print("\ntest6")
    a = np.arange(4)
    b = np.arange(3,7)
    print("\na=",a)
    print("\nb=",b)
    print("\na+b=",a+b)
    print("\na*b=", a*b)
    print("\na@b=", a@b) # python3.5 latter
    print("\nnp.dot(a, b)=", np.dot(a, b))

if __name__=='__main__':
    test1()
    test2()
    test3()
    test4()
    test5()
    test6()


import logging
import cv2
import numpy as np
import copy

logging.basicConfig(level=logging.DEBUG, format='%(asctime)-15s %(levelname)s \t%(message)s')

def main():
    logging.info("compute_homography.py main()")
    image_filepath = "./images/checkerboard.png"
    features_mm_to_pixels_dict =  {(0, 0): (192, 801, ),
                                  (214.2, 0): (937, 652),
                                  (214.2, 178.5): (798, 67),
                                  (0, 178.5): (101, 197)}
    A = np.zeros((2 * len(features_mm_to_pixels_dict), 6), dtype=float)
    b = np.zeros((2 * len(features_mm_to_pixels_dict), 1), dtype=float)
    index = 0
    for XY, xy in features_mm_to_pixels_dict.items():
        X = XY[0]
        Y = XY[1]
        x = xy[0]
        y = xy[1]
        A[2 * index, 0] = x
        A[2 * index, 1] = y
        A[2 * index, 2] = 1
        A[2 * index + 1, 3] = x
        A[2 * index + 1, 4] = y
        A[2 * index + 1, 5] = 1
        b[2 * index, 0] = X
        b[2 * index + 1, 0] = Y
        index += 1
    # A @ x = b
    x, residuals, rank, singular_values = np.linalg.lstsq(A, b, rcond=None)

    pixels_to_mm_transformation_mtx = np.array([[x[0, 0], x[1, 0], x[2, 0]], [x[3, 0], x[4, 0], x[5, 0]], [0, 0, 1]])
    logging.debug("main(): pixels_to_mm_transformation_mtx = \n{}".format(pixels_to_mm_transformation_mtx))

    test_xy_1 = (399, 500, 1)
    test_XY_1 = pixels_to_mm_transformation_mtx @ test_xy_1
    logging.info("main(): test_xy_1 = {}; test_XY_1 = {}".format(test_xy_1, test_XY_1))

    mm_to_pixels_transformation_mtx = np.linalg.inv(pixels_to_mm_transformation_mtx)
    test_XY_2 = (107.1, 107.1, 1)
    test_xy_2 = mm_to_pixels_transformation_mtx @ test_XY_2
    image = cv2.imread(image_filepath)
    annotated_img = copy.deepcopy(image)
    draw_crosshair(annotated_img, (round(test_xy_2[0]), round(test_xy_2[1])), 20, (0, 255, 0))
    cv2.imwrite("./outputs/annotated.png", annotated_img)

def draw_crosshair(image, center, width, color):
    cv2.line(image, (center[0] - width//2, center[1]), (center[0] + width//2, center[1]), color)
    cv2.line(image, (center[0], center[1] - width//2), (center[0], center[1] + width//2), color)

if __name__ == '__main__':
    main()


from math import sqrt, ceil

from scipy.misc import comb

def count_monomials_leq(n, d):
  '''Number of monomials in n variables of degree less than or equal to d'''
  return int(comb(n+d, d))

def count_monomials_eq(n, d):
  '''Number of monomials in n variables of degree equal to d'''
  return int(comb(n+d-1, d))

def grlex_to_index(grlex):
  '''Returns the grlex ordering number for a given multi-index. Can be expensive for large degrees'''
  total_degree = sum(grlex)
  n = len(grlex)

  index = count_monomials_leq(n, total_degree - 1)
  remaining_degree = total_degree
  for i in range(n):
    for k in range(grlex[i]):
      index += count_monomials_eq(n-1-i, remaining_degree - k)
    remaining_degree -= grlex[i]
  return index

def index_to_grlex(index, n):
  '''Returns the multi-index of length n for a given grlex ordering number'''
  if n < 1:
    raise Exception('number of variables must be positive')
  if index < 0:
    raise Exception('invalid index')

  grlex = [0 for i in range(n)]

  # find sum of grlex
  total_degree = 0
  while count_monomials_leq(n, total_degree) <= index:
    total_degree += 1

  # index among all multiindices with sum total_degree
  rel_idx_search = index - count_monomials_leq(n, total_degree - 1)

  rem_deg = total_degree # keep track of sum of remaining indices
  rel_idx = 0 # keep track of number of multiindices to the left
  for i in range(n-1):
    for k in range(total_degree+1):
      new_rel_idx = rel_idx + count_monomials_eq(n-(i+1), rem_deg-k)
      if new_rel_idx > rel_idx_search:
        grlex[i] = k
        rem_deg -= k
        break
      rel_idx = new_rel_idx

  grlex[-1] = rem_deg

  return tuple(grlex)

def grlex_key(midx):
  return (sum(midx),) + tuple(midx)

def multi_grlex_iter(midx, groups, degrees):
  '''
  Create an iterator that produces ordered exponents, starting
  with the multiindex 'midx', such that total degree of the variables
  in groups[i] does not exceed degrees[i].

  The ordering is based on grlex groupwise, but does not result
  in an overall grlex ordering.

  Example: The iterator grlex_iter( (0,0), [ [0], [1] ], [2], [1] )
       produces the sequence
    (0,0) (1,0) (2,0) (0,1) (1,1) (2,1)
  '''

  # make sure 'groups' is a valid partition
  assert(set([dim for group in groups for dim in group]) == set(range(len(midx))))
  assert(len(groups) == len(degrees))

  # starting multiindices
  start_midx = [ tuple([ midx[dim] for dim in group ]) for group in groups ]

  iterators = [grlex_iter(m0, deg) for m0, deg in zip(start_midx, degrees) ]
  mons = [next(iterator) for iterator in iterators]

  ret = list(midx)
  while True:

    yield tuple(ret)

    for ind in range(len(degrees)):
      # find first that is not at max
      try:
        mons[ind] = next(iterators[ind])
        break
      except StopIteration:

        if ind == len(degrees) - 1:
          raise StopIteration

        # was at max, reset it
        iterators[ind] = grlex_iter(start_midx[ind], degrees[ind])
        mons[ind] = next(iterators[ind])

    # Fill out return tuple
    for group_nr, group in enumerate(groups):
      for pos_nr, pos in enumerate(group):
        ret[pos] = mons[group_nr][pos_nr]

def grlex_iter(midx, deg = None):
  '''
  Create an iterator that produces ordered grlex exponents, starting
  with the multiindex 'midx'. The iterator stops when the total degree
  reaches 'deg'+1. (default: never stop)

  Example: The iterator grlex_iter( (0,2) ) produces the sequence
    (0,2) (1,2) (2,0) (0,3) (1,2) (2,1) (3,0) ...
  '''

  if len(midx) == 0:
    return  # nothing to do here

  midx = list(midx)
  assert(min(midx) >= 0)

  if max(midx) == 0:
    right_ptr = 0
  else:
    # find right-most non-zero element
    for i in range(len(midx)-1, -1, -1):
      if midx[i] != 0:
        right_ptr = i
        break

  while deg is None or sum(midx) < deg + 1:
    yield tuple(midx)
    if right_ptr == 0:
      midx = [0] * (len(midx) - 1) + [sum(midx) + 1]
      right_ptr = len(midx) - 1
    else:
      at_right_ptr = midx[right_ptr]
      midx[right_ptr] = 0
      midx[right_ptr-1] += 1
      midx[-1] = at_right_ptr - 1
      right_ptr = len(midx) - 1 if at_right_ptr != 1 else right_ptr - 1

def vec_to_grlex(L, dim):
  '''
  Given vec(A) for a symmetric matrix A, such that len(vec(A)) = L,
  compute the mapping  vec(A) |--> x^T A x
  to the grlex dim-dimensional monomial basis on multiindex - coefficient form.

  Example: [ a b c ] represents the matrix [ a b ; b c], and the corresponding 1d
  polynomial is a + 2b x + c x^2. Thus the mapping can be represented as
  ( 1., 2., 1. ), [ (0,), (1,), (2,) ]
  '''
  n = (sqrt(1 + 8 * L) - 1)/2  # side of matrix

  if not ceil(n) == n:
    raise TypeError('wrong size for vec(A) for A symmetric')

  j = 0
  i = 0
  i_moniter = grlex_iter( (0,) * dim )
  j_moniter = grlex_iter( (0,) * dim )
  i_mon = next(i_moniter)
  j_mon = next(j_moniter)

  exp = []
  coef = []

  for k in range(L):
    # loop over all elements
    exp += [tuple( [ii+jj for ii,jj in zip(i_mon, j_mon) ] )]
    coef += [1. if i == j else 2.]

    if j == n-1:
      i += 1
      j = i
      i_mon = next(i_moniter)
      j_moniter = grlex_iter( i_mon )
      j_mon = next(j_moniter)

    else:
      j_mon = next(j_moniter)
      j += 1
  return coef, exp


from scipy.optimize import fmin
from functools import partial

# A generic function that we want to find the minimum of
def my_func(x):
    """A quadratic function
    :param x the input x value
    :returns f(x)"""
    return (x-2)**2 - 3

# A fancier quadratic-y type function (but in 2D)
def elliptic_paraboloid(x, y, x0, y0, a, b):
    """ x,y -> f(x) in the shape of a 'bowl'
    :param x - x value in the plane
    :param y - y value in the plane
    :param x0 - amount to shift the bottom of the bowl by in x
    :param y0 - amount to shift the bottom of the bowl by in y
    :param a - scale the bowl's side in x
    :param b - scale the bowl's side in y
    :returns - f(x,y) the bowl's height over the point x,y
    """
    return (x-x0)**2 / a**2 + (y-y0)**2 / b**2

if __name__ == '__main__':
    # Use fmin to find the minimum of my_func
    # The _ says ignore that returned value
    # Note that x_at_min is a list - in this case, a list of dimension 1
    #    The full_output=True prints out the optimization result to the console
    x_at_min, f_at_x_min, _, _, _ = fmin(my_func, 0.2, maxfun=100, full_output=True)
    print("Minimum of f is {0}, happens at x={1}".format(f_at_x_min, x_at_min[0]))
    print("Checking fmin result {0} against func eval {1}".format(f_at_x_min, my_func(x_at_min[0])))

    # This does the same thing
    #   See https://docs.scipy.org/doc/scipy/reference/generated/scipy.optimize.fmin.html
    #   for what the remaining values are
    f_min_ret_vals = fmin(my_func, 0.2, maxfun=100, full_output=True)
    x_at_min_2 = f_min_ret_vals[0][0]
    fx_at_min_2 = f_min_ret_vals[1]
    print("Minimum of f is {0}, happens at x={1}".format(fx_at_min_2, x_at_min_2))

    # Now going to do the 2 dimensional version of a parabaloid
    # Because the function definition has all these extra parameters that
    #  control the shape of the function (x0, etc), we have do what's called
    #  "binding" the variables to values, in this case x0 = 3, y0 = 2, a = 4, b=16
    # partial is a functools method to do this, which has the advantage over a lambda
    #  function of creating a python function that is more "efficient" because it
    #  "freezes" the constant parameters
    # partial takes the function elliptic_paraboloid and "binds" all of the named
    #   parameter arguments
    my_paraboloid = partial(elliptic_paraboloid, x0=3, y0=2, a=4, b=16)
    print("My parabaloid at 3,4: {0}, check {1}".format(my_paraboloid(3, 4), elliptic_paraboloid(3, 4, 3, 2, 4, 16)))

    # Almost done - fmin takes a function that takes in one argument (a list of dimension d)
    #  and outputs one number. Our my_paraboloid function currently takes in x,y, not [x,y].
    # Lambda functions fix that: In this case, it says make a new (temporary, unnamed function)
    #  that takes in one parameter (array) and calls my_paraboloid with array[0] and array[1]
    result = fmin(lambda array: my_paraboloid(array[0], array[1]), [0, 0], maxfun=200, full_output=True)
    print(result)
    print("Function minimum is {0}, found at {1}".format(result[1], result[0]))

    # You shouldn't really do this, because lambda functions are suppose to be unnamed,
    #  but this will show you what the lambda did
    my_func_2D = lambda array: my_paraboloid(array[0], array[1])
    # Notice that my_func_2D takes a list of two numbers, my_parabaloid takes 2 numbers
    print("Call my_parabaloid with 3, 4: {0} {1}".format(my_func_2D([3, 4]), my_paraboloid(3, 4)))


import sympy

def bisection(func, min_guess, max_guess, err_tolerance):
    """
    Find the root of a function using bisection method. (Bracketing method)
    arguments:
    func: f(x)
    min_guess: minimum x as guess
    max_guess: maximum x as guess
    err_tolerance: value where f(root) must be less than err_tolerance
    """

    x_new = 0.5 * (min_guess + max_guess)
    y_mid = func(x_new)
    print("x_new: ", x_new)
    if abs(y_mid) < err_tolerance:
        return x_new
    else:
        min_guess = x_new if y_mid <= 0 else min_guess
        max_guess = x_new if y_mid > 0 else max_guess
        return bisection(func, min_guess, max_guess, err_tolerance)

def regula_falsi(func, min_guess, max_guess, err_tolerance):
    """
    Find the root of a function using false position method. (Bracketing method)
    arguments:
    func: f(x)
    min_guess: minimum x as guess
    max_guess: maximum x as guess
    err_tolerance: value where f(root) must be less than err_tolerance
    """
    y_min = func(min_guess)
    y_max = func(max_guess)
    x_new = (0.5 * y_max * min_guess - y_min * max_guess)/(0.5 * y_max - y_min)
    y_new = func(x_new)
    if abs(y_new) <= err_tolerance:
        return x_new
    else:
        min_guess = x_new if y_new <= 0 else min_guess
        max_guess = x_new if y_new > 0 else max_guess
        return regula_falsi(func, min_guess, max_guess, err_tolerance)

def secant(func, min_guess, max_guess, err_tolerance, max_try=10000):
    """
    Find the root of a function using secant method. (Newton method)
    arguments:
    func: f(x)
    min_guess: minimum x as guess
    max_guess: maximum x as guess
    err_tolerance: value where f(root) must be less than err_tolerance
    """
    y_min = func(min_guess)
    y_max = func(max_guess)
    x_new = max_guess - (y_max * (min_guess - max_guess)/(y_min - y_max))
    y_new = func(x_new)
    print("x_new: ", x_new)
    if abs(y_new) <= err_tolerance:
        return x_new
    elif max_try == 0:
        return ValueError('Unable to find root after try')
    else:
        min_guess = x_new if y_new <= 0 else min_guess
        max_guess = x_new if y_new > 0 else max_guess
        return secant(func, min_guess, max_guess, err_tolerance, max_try - 1)

def newton_rhapson(func, min_guess, max_guess, err_tolerance):
    x = sympy.symbols('x')
    derivatives = sympy.derivatives(func, x)
    pass

def dekker():
    pass

def brent(func, min_guess, max_guess, err_tolerance):
    pass

def j(funct):
    return funct

def f(x):
    return ((x+3) ** 3) - 3

# find root
# root = bisection(f, -5, 5, 0.001)
# print('root for bisection is: ', root)

root = regula_falsi(f, -5, 5, 0.0001)
print('root for regula falsi is: ', root)

# root = secant(f, -2, 0, 0.0001)
# print('root for secant is: ', root)

al = j(f)
print(al(root))


def test_conditionNumber():
    import numpy as np
    from numa import LinearSystem
    tol = 1e-14

    A = np.array([[7, -1, 0], [4, 6, -3],
                  [-2, 6, 1]], dtype=float)
    true_value = np.linalg.cond(A, p=2)
    calc_value = LinearSystem.condition(A, p=2)
    assert np.allclose(
        calc_value, true_value, atol=tol), "LinearSystem.condition does not agree with known solution."

def test_Gauss():
    import numpy as np
    from numa import LinearSystem
    tol = 1e-14

    A = np.array([[7, -1, 0], [4, 6, -3],
                  [-2, 6, 1]], dtype=float)
    b = np.array([5, 7, 13], dtype=float)

    true_solution = np.array([1, 2, 3], dtype=float)
    x_calculated = LinearSystem.solveGauss(A, b)

    assert np.allclose(
        x_calculated, true_solution, atol=tol), "LinearSystem.solveGauss does not agree with known solution"

def test_Gauss_wrongDimensions():
    import numpy as np
    from numa import LinearSystem, utils
    import pytest

    A = np.array([[7, -1, 0], [4, 6, -3], [0, 2, 1],
                  [-2, 6, 1]], dtype=float)
    b = np.array([5, 7, 13], dtype=float)

    with pytest.raises(utils.DimensionError):
        x_calculated = LinearSystem.solveGauss(A, b)

    A = np.array([[7, -1, 0], [4, 6, -3],
                  [-2, 6, 1]], dtype=float)
    b = np.array([5, 7, 13, 3], dtype=float)
    with pytest.raises(utils.DimensionError):
        x_calculated = LinearSystem.solveGauss(A, b)
    return None

def test_Gauss_singularMatrix():
    import numpy as np
    from numa import LinearSystem, utils
    import pytest

    A = np.array([[0, 2, 1], [1, 1, 1], [4, 0, 2]], dtype=float)
    b = np.array([1, 1, 1], dtype=float)

    with pytest.raises(utils.SingularityError):
        x = LinearSystem.solveGauss(A, b)

def test_LU():
    import numpy as np
    from numa import LinearSystem
    tol = 1e-14

    A = np.array([[2, -1, -2], [-4, 6, 3], [-4, -2, 8]], dtype=float)
    L, U = LinearSystem.LU(A)
    A_reconstructed = np.matmul(L, U)
    assert np.allclose(
        A_reconstructed, A, atol=tol), "LinearSystem.LU does not agree with known solution"

def test_solveLU():
    import numpy as np
    from numa import LinearSystem
    tol = 1e-14

    A = np.array([[7, -1, 0], [4, 6, -3],
                  [-2, 6, 1]], dtype=float)
    b = np.array([5, 7, 13], dtype=float)
    true_solution = np.array([1, 2, 3], dtype=float)

    x_calculated = LinearSystem.solveLU(A, b)
    assert np.allclose(
        x_calculated, true_solution, atol=tol), "LinearSystem.solveLU does not agree with known solution"

def test_LU_singularMatrix():
    import numpy as np
    from numa import LinearSystem, utils
    import pytest

    A = np.array([[0, 2, 1], [1, 1, 1], [4, 0, 2]], dtype=float)
    b = np.array([1, 1, 1], dtype=float)

    with pytest.raises(utils.SingularityError):
        x = LinearSystem.solveLU(A, b)


"""
comput analytical response of a simply-supported (SSSS) plate under uniform load
By: Mehdi Paak
"""

import numpy as np

def w(x,y,a,b,h,nu,E,P0):
	"""
	deflextion of the plate at x,y
	units are SI meter

	:param x:
	:param y:
	:param a:
	:param b:
	:param h:
	:param nu:
	:param E:
	:param P0:
	:return:  deflection
	"""
	# 10 is a reasonable value for the analytical solution
	num_mode = 10

	D=E*h**3/(12.0*(1-nu**2))
	alph = 16.0*P0/(D*np.pi**6)
	N=[2*i+1 for i in range(num_mode)]
	M = N
	w=0.0

	for m in M:
		for n in N:
			beta = m*n*((m/a)**2+(n/b)**2)**2
			w = w +  (1.0/beta) * np.sin(m*np.pi*x/a) * np.sin(n*np.pi*y/b)
	w = w * alph
	return w

def wmax_nondim():
	"""
	returns nondimensional maximum deflection which
	is Wmax/(p0*a**4/D)
	D=E*h**3/(12(1-nu**2))

	:return:
	"""

	# 10 is a reasonable value for the analytical solution
	num_mode = 10
	alph = 16.0/np.pi**6

	N=[2*i+1 for i in range(num_mode)]
	M=N
	wmax = 0.0

	for m in M:
		for n in N:
			beta = m*n*(m**2+n**2)**2
			wmax = wmax +  (1.0/beta) * (-1.0)**((m+n-2)/2)
	return wmax*alph


import numpy as np
import h5py
# data file type h5py
import time
import copy
import matplotlib.pyplot as plt
from cfg import loadConfig

def load_mnist(filename):
    """load MNIST data"""
    MNIST_data = h5py.File(filename, 'r')
    x_train = np.float32(MNIST_data['x_train'][:])
    y_train = np.int32(np.array(MNIST_data['y_train'][:,0]))
    x_test = np.float32(MNIST_data['x_test'][:])
    y_test = np.int32(np.array(MNIST_data['y_test'][:,0]))
    MNIST_data.close()
    return x_train,y_train,x_test,y_test

def initialize(num_inputs,num_classes):
    """initialize the parameters"""
    # num_inputs = 28*28 = 784
    # num_classes = 10
    w = np.random.randn(num_classes, num_inputs) / np.sqrt(num_classes*num_inputs) # (10*784)
    b = np.random.randn(num_classes, 1) / np.sqrt(num_classes) # (10*1)

    param = {
        'w' : w, # (10*784)
        'b' : b  # (10*1)
    }
    return param

def softmax(z):
    """implement the softmax functions
    input: numpy ndarray
    output: numpy ndarray
    """
    exp_list = np.exp(z)
    result = 1/sum(exp_list) * exp_list
    result = result.reshape((len(z),1))
    assert (result.shape == (len(z),1))
    return result

def neg_log_loss(pred, label):
    """implement the negative log loss"""
    loss = -np.log(pred[int(label)])
    return loss

def mini_batch_gradient(param, x_batch, y_batch):
    """implement the function to compute the mini batch gradient
    input: param -- parameters dictionary (w, b)
           x_batch -- a batch of x (size, 784)
           y_batch -- a batch of y (size,)
    output: dw, db, batch_loss
    """
    batch_size = x_batch.shape[0]
    w_grad_list = []
    b_grad_list = []
    batch_loss = 0
    for i in range(batch_size):
        x,y = x_batch[i],y_batch[i]
        x = x.reshape((784,1)) # x: (784,1)
        E = np.zeros((10,1)) #(10*1)
        E[y][0] = 1
        pred = softmax(np.matmul(param['w'], x)+param['b']) #(10*1)

        loss = neg_log_loss(pred, y)
        batch_loss += loss

        w_grad = E - pred
        w_grad = - np.matmul(w_grad, x.reshape((1,784)))
        w_grad_list.append(w_grad)

        b_grad = -(E - pred)
        b_grad_list.append(b_grad)

    dw = sum(w_grad_list)/batch_size
    db = sum(b_grad_list)/batch_size
    return dw, db, batch_loss

def eval(param, x_data, y_data):
    """ implement the evaluation function
    input: param -- parameters dictionary (w, b)
           x_data -- x_train or x_test (size, 784)
           y_data -- y_train or y_test (size,)
    output: loss and accuracy
    """
    # w: (10*784), x: (10000*784), y:(10000,)
    loss_list = []
    w = param['w'].transpose()
    dist = np.array([np.squeeze(softmax(np.matmul(x_data[i], w))) for i in range(len(y_data))])

    result = np.argmax(dist,axis=1)
    accuracy = sum(result == y_data)/float(len(y_data))

    loss_list = [neg_log_loss(dist[i],y_data[i]) for i in range(len(y_data))]
    loss = sum(loss_list)
    return loss, accuracy

def train(param, hyp , x_train, y_train, x_test, y_test,cfg_idx):
    """ implement the train function
    input: param -- parameters dictionary (w, b)
           hyp -- hyperparameters dictionary
           x_train -- (60000, 784)
           y_train -- (60000,)
           x_test -- x_test (10000, 784)
           y_test -- y_test (10000,)
    output: test_loss_list, test_accu_list
    """
    num_epoches = hyp['num_epoches']
    batch_size = hyp['batch_size']
    learning_rate = hyp['learning_rate']
    mu = hyp['mu']
    test_loss_list, test_accu_list = [],[]
    if bool(hyp['momentum']) == True:
        w_velocity = np.zeros(param['w'].shape)
        b_velocity = np.zeros(param['b'].shape)

    for epoch in range(num_epoches):

        # select the random sequence of training set
        rand_indices = np.random.choice(x_train.shape[0],x_train.shape[0],replace=False)
        num_batch = int(x_train.shape[0]/batch_size)
        batch_loss100 = 0

        if bool(hyp['learning_decay']) == True:
            try:
                if test_accu_list[-1] - test_accu_list[-2] < 0.001:
                    learning_rate *= hyp['decay_factor']
            except:
                pass

            message = 'learning rate: %.8f' % learning_rate
            print(message)
            logging.info(message)

        # for each batch of train data
        for batch in range(num_batch):
            index = rand_indices[batch_size*batch:batch_size*(batch+1)]
            x_batch = x_train[index]
            y_batch = y_train[index]

            # calculate the gradient w.r.t w and b
            dw, db, batch_loss = mini_batch_gradient(param, x_batch, y_batch)
            batch_loss100 += batch_loss
            # update the parameters with the learning rate
            if bool(hyp['momentum']) == True:
                w_velocity = mu * w_velocity + learning_rate * dw
                b_velocity = mu * b_velocity + learning_rate * db
                param['w'] -= w_velocity
                param['b'] -= b_velocity
            else:
                param['w'] -= learning_rate * dw
                param['b'] -= learning_rate * db
            if batch % 100 == 0:
                message = 'Epoch %d, Batch %d, Loss %.2f' % (epoch+1, batch, batch_loss)
                print(message)
                # logging.info(message)

                batch_loss100 = 0
        train_loss, train_accu = eval(param,x_train,y_train)
        test_loss, test_accu = eval(param,x_test,y_test)
        test_loss_list.append(test_loss)
        test_accu_list.append(test_accu)

        message = 'Epoch %d, Train Loss %.2f, Train Accu %.4f, Test Loss %.2f, Test Accu %.4f' % (epoch+1, train_loss, train_accu, test_loss, test_accu)
        print(message)
        logging.info(message)
    return test_loss_list, test_accu_list

def plot(loss_list, accu_list, cfg_idx):
    """store the plots"""
    # epoch_list = list(range(len(loss_list)))
    plt.plot(loss_list)
    plt.ylabel('Loss Function')
    plt.xlabel('Epoch')
    plt.xticks(rotation=60)
    plt.title('Loss Function ~ Epoch')
    plt.savefig('assets/loss_{}.png'.format(cfg_idx))
    plt.show()

    plt.plot(accu_list)
    plt.ylabel('Test Accuracy')
    plt.xlabel('Epoch')
    plt.xticks(rotation=60)
    plt.title('Test Accuracy ~ Epoch')
    plt.savefig('assets/accr_{}.png'.format(cfg_idx))
    plt.show()

def main(args):
    cfg_idx = args.config
    cfg_name = 'config_{}.json'.format(cfg_idx)
    hyperpara = loadConfig(cfg_name)

    # loading MNIST data
    x_train,y_train,x_test,y_test = load_mnist('MNISTdata.hdf5')

    # setting the random seed
    np.random.seed(1024)

    # initialize the parameters
    num_inputs = x_train.shape[1]
    num_classes = len(set(y_train))
    param = initialize(num_inputs,num_classes)

    # train the model
    loss_list, accu_list = train(param,hyperpara,x_train,y_train,x_test,y_test, cfg_idx)

    # plot the loss and accuracy
    plot(loss_list, accu_list, cfg_idx)

if __name__ == "__main__":
    import argparse
    parser = argparse.ArgumentParser()
    parser.add_argument("-c", "--config", type=str,
                        default="sample", help="Config of hyperparameters")
    args = parser.parse_args()

    import logging
    logging.basicConfig(filename="./logs/{}.log".format(args.config), filemode="w", format="%(message)s", level=logging.DEBUG)

    main(args)


import numpy as np
from copy import deepcopy
from sklearn.cluster import KMeans

class KMeansClustering:
    def __init__(self, num_culsters) -> None:
        self.num_culsters = num_culsters
        self.labels_ = None
        self.cluster_centers_ = None

    def fit(self, data):
        # Number of training data.
        self.num_observations = data.shape[0]
        # Number of features in the data.
        self.num_features = data.shape[1]

        # Generate random centers for initialization.
        mean = np.mean(data, axis = 0)
        std = np.std(data, axis = 0)
        centers = np.random.randn(self.num_culsters, self.num_features) * std + mean

        # Setting up shape for containers.
        centers_old = np.zeros(centers.shape)
        centers_new = deepcopy(centers)
        clusters = np.zeros(self.num_observations)
        distances = np.zeros((self.num_observations, self.num_culsters))

        error = np.linalg.norm(centers_new - centers_old)

        # When, after an update, the estimate of that center stays the same, exit loop.
        while error != 0:
            # Measure the distance to every center.
            for i in range(self.num_culsters):
                distances[:,i] = np.linalg.norm(data - centers[i], axis=1)
            # Assign all training data to closest center.
            clusters = np.argmin(distances, axis = 1)

            centers_old = deepcopy(centers_new)
            # Calculate mean for every cluster and update the center.
            for i in range(self.num_culsters):
                centers_new[i] = np.mean(data[clusters == i], axis=0)
            error = np.linalg.norm(centers_new - centers_old)

        self.cluster_centers_ =  centers_new
        self.labels_ = clusters

if __name__ == "__main__":
    regions = ["Munster", "Bielefeld", "Duisburg", "Bonn", "Rhein-Main", "Dusseldorf"]

    data = np.array([
        [1524.8, 265.4, 272, 79, 24, 223.5],
        [1596.9, 323.6, 285, 87, 23, 333.9],
        [2299.7, 610.3, 241, 45, 9, 632.1],
        [864.1, 303.9, 220, 53, 11, 484.7],
        [2669.9, 645.5, 202, 61, 15, 438.6],
        [2985.2, 571.2, 226, 45, 16, 1103.9]
    ])

    names = ["Sklean KMeans", "Own implementaion of KMeans"]
    models = [
        KMeans(n_clusters=2, random_state=42),
        KMeansClustering(num_culsters=2)
    ]

    for name, model in zip(names, models):
        print(f"Model: {name}")
        model.fit(data)

        for region, label in zip(regions, model.labels_):
            print(f"Region: {region.ljust(15)}   Label: {label}")

        print("\n")


import numpy as np

# function for damped pseudo-inverse
def damped_pseudoinverse(jac, l = 0.01):
    m, n = jac.shape
    if n >= m:
        return jac.T @ np.linalg.inv(jac @ jac.T + l*l*np.eye(m))
    return np.linalg.inv(jac.T @ jac + l*l*np.eye(n)) @ jac.T

# function for skew symmetric
def skew_symmetric(v):
    mat = np.zeros((3, 3))
    mat[0, 1] = -v[2]
    mat[1, 0] = v[2]
    mat[0, 2] = v[1]
    mat[2, 0] = -v[1]
    mat[1, 2] = -v[0]
    mat[2, 1] = v[0]

    return mat

# function for Adjoint
def AdT(tf):
    R = tf.rotation()
    T = tf.translation()
    tr = np.zeros((6, 6))
    tr[0:3, 0:3] = R
    tr[3:6, 0:3] = skew_symmetric(T) @ R
    tr[3:6, 3:6] = R

    return tr

# angle wrap to [-pi,pi)
def angle_wrap(theta):
    while theta < -np.pi:
        theta += 2 * np.pi
    while theta > np.pi:
        theta -= 2 * np.pi
    return theta

# angle wrap multi
def angle_wrap_multi(theta):
    if isinstance(theta, list):
        th = theta
        for i in range(len(th)):
            th[i] = angle_wrap(th[i])
        return th
    elif type(theta) is np.ndarray:
        th = theta
        for i in range(theta.shape[0]):
            th[i] = angle_wrap(th[i])
        return th
    return angle_wrap(theta)


## 3. Bias-variance tradeoff ##

from sklearn.linear_model import LinearRegression
from sklearn.metrics import mean_squared_error
import numpy as np
import matplotlib.pyplot as plt

def train_and_test(cols):
    lr = LinearRegression()
    lr.fit(filtered_cars[cols], filtered_cars['mpg'])
    predictions = lr.predict(filtered_cars[cols])
    # Compute MSE and Variance
    mse = mean_squared_error(filtered_cars['mpg'], predictions)

    variance = np.var(predictions)
    return(mse, variance)

cyl_mse, cyl_var = train_and_test(['cylinders'])
print(cyl_mse, cyl_var)

weight_mse, weight_var = train_and_test(['weight'])
print(weight_mse, weight_var)

## 4. Multivariate models ##

# Our implementation for train_and_test, takes in a list of strings.
def train_and_test(cols):
    # Split into features & target.
    features = filtered_cars[cols]
    target = filtered_cars["mpg"]
    # Fit model.
    lr = LinearRegression()
    lr.fit(features, target)
    # Make predictions on training set.
    predictions = lr.predict(features)
    # Compute MSE and Variance.
    mse = mean_squared_error(filtered_cars["mpg"], predictions)
    variance = np.var(predictions)
    return(mse, variance)

one_mse, one_var = train_and_test(["cylinders"])

two_mse, two_var = train_and_test(['cylinders','displacement'])

three_mse, three_var = train_and_test(['cylinders','displacement','horsepower'])

four_mse, four_var = train_and_test(['cylinders','displacement','horsepower','weight'])

five_mse, five_var = train_and_test(['cylinders','displacement','horsepower','weight','acceleration'])

six_mse, six_var = train_and_test(['cylinders','displacement','horsepower','weight','acceleration','model year'])

seven_mse, seven_var = train_and_test(['cylinders','displacement','horsepower','weight','acceleration','model year','origin'])

print('The MSE and Variance of Two features are: '+ str(two_mse)+' and '+str(two_var) )
print('The MSE and Variance of Three features are: '+ str(three_mse)+' and '+str(three_var) )
print('The MSE and Variance of four features are: '+ str(four_mse)+' and '+str(four_var) )
print('The MSE and Variance of five features are: '+ str(five_mse)+' and '+str(five_var) )
print('The MSE and Variance of six features are: '+ str(six_mse)+' and '+str(six_var) )
print('The MSE and Variance of seven features are: '+ str(seven_mse)+' and '+str(seven_var) )

## 5. Cross validation ##

from sklearn.model_selection import KFold
from sklearn.metrics import mean_squared_error
import numpy as np

def train_and_cross_val(cols):
    features = filtered_cars[cols]
    target = filtered_cars['mpg']

    variance_values = []
    mse_values = []

    #KFold instance
    kf = KFold(n_splits = 10, shuffle = True, random_state = 3)
     # Iterate through over each fold.
    for train_index, test_index in kf.split(features):
        # Training and test sets.
        X_train, X_test = features.iloc[train_index], features.iloc[test_index]
        y_train, y_test = target.iloc[train_index], target.iloc[test_index]

        # Fit the model and make predictions.
        lr = LinearRegression()
        lr.fit(X_train, y_train)
        predictions = lr.predict(X_test)

        # Calculate mse and variance values for this fold.
        mse = mean_squared_error(y_test, predictions)
        var = np.var(predictions)

        # Append to arrays to do calculate overall average mse and variance values.
        variance_values.append(var)
        mse_values.append(mse)

    # Compute average mse and variance values.
    avg_mse = np.mean(mse_values)
    avg_var = np.mean(variance_values)
    return(avg_mse, avg_var)

two_mse, two_var = train_and_cross_val(["cylinders", "displacement"])
three_mse, three_var = train_and_cross_val(["cylinders", "displacement", "horsepower"])
four_mse, four_var = train_and_cross_val(["cylinders", "displacement", "horsepower", "weight"])
five_mse, five_var = train_and_cross_val(["cylinders", "displacement", "horsepower", "weight", "acceleration"])
six_mse, six_var = train_and_cross_val(["cylinders", "displacement", "horsepower", "weight", "acceleration", "model year"])
seven_mse, seven_var = train_and_cross_val(["cylinders", "displacement", "horsepower", "weight", "acceleration","model year", "origin"])

print('The MSE and Variance of Two features are: '+ str(two_mse)+' and '+str(two_var) )
print('The MSE and Variance of Three features are: '+ str(three_mse)+' and '+str(three_var) )
print('The MSE and Variance of four features are: '+ str(four_mse)+' and '+str(four_var) )
print('The MSE and Variance of five features are: '+ str(five_mse)+' and '+str(five_var) )
print('The MSE and Variance of six features are: '+ str(six_mse)+' and '+str(six_var) )
print('The MSE and Variance of seven features are: '+ str(seven_mse)+' and '+str(seven_var) )

## 6. Plotting cross-validation error vs. cross-validation variance ##

# We've hidden the `train_and_cross_val` function to save space but you can still call the function!
import matplotlib.pyplot as plt

two_mse, two_var = train_and_cross_val(["cylinders", "displacement"])
three_mse, three_var = train_and_cross_val(["cylinders", "displacement", "horsepower"])
four_mse, four_var = train_and_cross_val(["cylinders", "displacement", "horsepower", "weight"])
five_mse, five_var = train_and_cross_val(["cylinders", "displacement", "horsepower", "weight", "acceleration"])
six_mse, six_var = train_and_cross_val(["cylinders", "displacement", "horsepower", "weight", "acceleration", "model year"])
seven_mse, seven_var = train_and_cross_val(["cylinders", "displacement", "horsepower", "weight", "acceleration","model year", "origin"])

plt.scatter([2,3,4,5,6,7], [two_mse, three_mse, four_mse, five_mse, six_mse, seven_mse], c='red')
plt.scatter([2,3,4,5,6,7], [two_var, three_var, four_var, five_var, six_var, seven_var], c='blue')
plt.show()

6. Machine Learning Introduction/machine-learning-intermediate/Overfitting-132.py


'''To integrate any user defined function f, or any other function in NumPy, call:
int_simpson(function, lower_limit, upper_limit, num_of_intervals_of_integration)
'''
import numpy as np
def int_simpson(f,a,b,N):
	w=(b-a)/N
	area=(w/3)*(f(a)+f(b))
	i=a
	for i in range(1,N):
		if i%2==0:
			area+=(w/3)*2*f(a+i*w)
		else:
			area+=(w/3)*4*f(a+i*w)
	return area

#print(int_simpson(np.sin,0,(np.pi)/2,100))


import numpy as np

def computeNumericalGradient(J, theta):
  """ Compute numgrad = computeNumericalGradient(J, theta)

  theta: a matrix of parameters
  J: a function that outputs a real-number and the gradient.
  Calling y = J(theta)[0] will return the function value at theta.
  """

  # Initialize numgrad with zeros
  numgrad = np.zeros(theta.shape)

  ## ---------- YOUR CODE HERE --------------------------------------
  # Instructions:
  # Implement numerical gradient checking, and return the result in numgrad.
  # You should write code so that numgrad[i][j] is (the numerical approximation to) the
  # partial derivative of J with respect to theta[i][j], evaluated at theta.
  # I.e., numgrad[i][j] should be the (approximately) partial derivative of J with
  # respect to theta[i][j].
  #
  # Hint: You will probably want to compute the elements of numgrad one at a time.

  # Set Epsilon
  epsilon = 0.0001

  # Outer for loop to check across the x-axis
  for i in range(theta.shape[0]):
    # Inner for loop to check across the y-axis
    for j in range(theta.shape[1]):
      # Copy current theta value to min
      theta_min = theta.copy()
      # Subtract min point by epsilon and store
      theta_min[i,j] = theta_min[i,j] - epsilon
      # Not sure
      cost_min, dW, db = J(theta_min)
      # Copy current theta for max
      theta_max = theta.copy()
      # Add max point by epsilon and store
      theta_max[i,j] = theta_max[i,j] + epsilon
      # ?
      cost_max, dW, db = J(theta_max)

      # Final Result for gradient k
      numgrad[i][j] = (cost_max - cost_min) / (2 * epsilon)

  ## ---------------------------------------------------------------

  return numgrad


import numpy as np
import math

# Write a function that takes as input a list of numbers, and returns
# the list of values given by the softmax function.
def softmax(L):
    softmax_values = []
    denominator = 0
    for i in L:
        denominator += math.exp(i)
    for i in L:
        softmax_values.append(math.exp(i)/denominator)
    return softmax_values

"""
def softmax(L):
    expL = np.exp(L)
    sumExpL = sum(expL)
    result = []
    for i in expL:
        result.append(i*1.0/sumExpL)
    return result

"""

if __name__ == "__main__":
    L = [-1.0, 2.0, 0.5, 1.2]
    probs = softmax(L)
    print(probs)