import numpy as np

def check_invertible_array(X: np.array) -> bool:
    """
    Check if the input array is invertible based on its rank
    Parameters
    ----------
    X:  np.array with (n_samples, n_features) shape

    Returns
    -------
    boolean, True if X is invertible, False if NOT.
    """
    assert len(X.shape) == 2  # Has to be 2-D array
    assert np.divide(*X.shape) == 1  # Has to be a square array/matrix

    n_rank_raw, _ = X.shape
    n_rank_real = np.linalg.matrix_rank(X)

    return True if int(n_rank_raw) == int(n_rank_real) else False

def feature_norm(X: np.array, axis: int = 0) -> np.array:
    """
    Feature normalization.
    X_ = (X - mean(X)) / stdev(X)

    Parameters
    ----------
    X:  np.array with (n_samples, n_features) shape
    axis: int, which axis to perform the normalization too (features' axis).

    Returns
    -------
    X_new:  np.array with (n_samples, n_features) shape.
            The normalized X input based on its feature!
    """
    X_new = (X - np.mean(X, axis=axis)) / np.std(X, axis=axis)
    return X_new

def normal_eq(X: np.array, y: np.array) -> np.array:
    """
    Calculating normal equation to solve linear regression.

    Parameters
    ----------
    X: np.array with (n_samples, n_features) shape
    y: np.array with (n_samples,) shape

    Returns
    -------
    theta:  np.array with (n_features,).
            The trained parameters for the linear regression.
    """
    A = np.dot(X.T, X)
    theta = np.dot(np.linalg.pinv(A), np.dot(X.T, y))  # inv(A) dot X.T dot y
    return theta


### LINEAR ALGEBRA ###
import numpy as np
# from numpy.linalg import norm, det
# help(np.linalg)

# M = np.array([
#     [1, 2],
#     [3, 4]
# ])
# v = np.array([1, -1])

# # print(v.T) # Calcular transpuesta no funciona al ser vector (matriz 1D)

# u = np.dot(M, v)

# # Para hacer comparaciones entre arrays de punto flotante se puede usar:
# # np.allclose: comprueba si todos los elementos de los arrays son iguales dentro de una tolerancia,
# # np.isclose: compara elemento a elemento y devuelve un array de valores True y False
# # np.allclose(u, v)
# # np.isclose(0.0, 1e-8, atol=1e-10)

# # u = M @ v # Hace operaciones entre matrices
# # print(u)

# mat = np.array([[1, 5, 8, 5],
#                 [0, 6, 4, 2],
#                 [9, 3, 1, 6]])

# vec1 = np.array([5, 6, 2])

# print(vec1 @ mat)

## Exercise 1 ##
# Hallar el producto de estas dos matrices y su determinante:

# A = np.array([
#     [1, 0, 0],
#     [2, 1, 1],
#     [-1, 0, 1]
# ])
# B = np.array([
#     [2, 3, -1],
#     [0, -2, 1],
#     [0, 0, 3]
# ])

# C = A @ B
# determ = det(C)

## Exercise 2 ##
# 2- Resolver el siguiente sistema:

# $$ \begin{pmatrix} 2 & 0 & 0 \\ -1 & 1 & 0 \\ 3 & 2 & -1 \end{pmatrix} \begin{pmatrix} 1 & 1 & 1 \\ 0 & 1 & 2 \\ 0 & 0 & 1 \end{pmatrix} \begin{pmatrix} x \\ y \\ z \end{pmatrix} = \begin{pmatrix} -1 \\ 3 \\ 0 \end{pmatrix} $$
# M = (np.array([[2, 0, 0],
#                         [-1, 1, 0],
#                         [3, 2, -1]])
#      @
#         np.array([[1, 1, 1],
#                         [0, 1, 2],
#                         [0, 0, 1]]))
# res = np.array([-1, 3, 0])

# x = np.linalg.solve(M,res)

# print(np.allclose(M @ x, res))

## Exercise 3 ##
# 3- Hallar la inversa de la matriz 𝐻 y comprobar que 𝐻𝐻−1=𝐼 (recuerda la función np.eye)
# A = np.arange(1, 37).reshape(6,6)
# A[1, 1::2] = 0
# A[3, ::2] = 1
# A[4, :] += 30
# B = (2 ** np.arange(36)).reshape((6,6))
# H = A + B
# print(H)

# print(np.linalg.det(H))

# Hinv = np.linalg.inv(H)

# np.isclose(np.dot(Hinv, H), np.eye(6))

# np.set_printoptions(precision=3)
# print(np.dot(Hinv, H))

#¡No funciona! Y no solo eso sino que los resultados varían de un ordenador a otro.

## Exercise 4 ##
# print(np.linalg.cond(H))
# La matriz está mal condicionada

## Exercise 5 ## # Autovalores y autovectores #
A = np.array([
    [1, 0, 0],
    [2, 1, 1],
    [-1, 0, 1]
])

eigen = np.linalg.eig(A)


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Thu Jul  8 15:58:48 2021

@author: alessandro
"""

import math
import numpy as np
import matplotlib.pyplot as plt
import sympy as sym
from sympy.utilities.lambdify import lambdify
from scipy.optimize import fsolve

def f(x): return 2 * math.cos(x) + x * (7 - x) - math.tan((3 / 2) * x)

zero1 = fsolve(f, -1)
zero2 = fsolve(f, 0)
zero3 = fsolve(f, 1)

zeri = np.array([zero1, zero2, zero3])

x = sym.symbols('x')
gx = sym.tan((3 / 2) * x) - 2 * sym.cos(x) - x * (6 - x)
dgx = sym.diff(gx, x, 1)

g = lambdify(x, gx, np)
dg = lambdify(x, dgx, np)

xx = np.linspace(-1, 1, 100)
y = g(xx)

plt.ylim((-3, 3))
plt.plot(zeri, g(zeri), 'o')
plt.plot(zeri, dg(zeri), 'o')
plt.plot(xx, xx, label="y = x")
plt.plot(xx, y, label="g(x)")
plt.plot(xx, dg(xx), label="g'(x)")
plt.legend()
plt.show()

# --------

gx = (sym.tan((3 / 2) * x) - 2 * sym.cos(x)) / (7 - x)
dgx = sym.diff(gx, x, 1)

g = lambdify(x, gx, np)
dg = lambdify(x, dgx, np)

xx = np.linspace(-1, 1, 100)
y = g(xx)

plt.ylim((-3, 3))
plt.plot(zeri, g(zeri), 'o')
plt.plot(zeri, dg(zeri), 'o')
plt.plot(xx, xx, label="y = x")
plt.plot(xx, y, label="g(x)")
plt.plot(xx, dg(xx), label="g'(x)")
plt.legend()
plt.show()

# c)
def puntofisso(g, x0, tol, nmax):
    x = g(x0)
    it, xk = 1, [x]
    while it < nmax and abs(x - x0) >= tol * abs(x):
        x0 = x
        x = g(x)
        xk.append(x)
        it += 1
    return x, it, xk

# d)
x, it, xk = puntofisso(g, 0, 1e-7, 500)
print(x)
plt.plot(np.arange(1, it + 1), xk)

# e)

def stima_ordine(xk):
    k = len(xk) - 4
    n = np.log(np.abs(xk[k + 3] - xk[k + 2]) / np.abs(xk[k + 2] - xk[k + 1]))
    d = np.log(np.abs(xk[k + 2] - xk[k + 1]) / np.abs(xk[k + 1] - xk[k]))
    return n / d

ordine = stima_ordine(xk)
print(ordine)

import numpy,from scipy,import sympy,from sympy


# author: Krzysztof Sopyła (krzysztofsopyla@gmail.com)
# Twitter: ksopyla
# Blog: http://ksopyla.com

# If you want to use this material in your own trainning please let me know.
import numpy as np

# We will add the vector v to each row of the matrix x,
# storing the result in the matrix y
x = np.array([[1,2,3], [4,5,6], [7,8,9], [10, 11, 12]])
v = np.array([1, 0, 1])
y = np.empty_like(x)   # Create an empty matrix with the same shape as x

# Add the vector v to each row of the matrix x with an explicit loop
for i in range(4):
    y[i, :] = x[i, :] + v

# Now y is the following
# [[ 2  2  4]
#  [ 5  5  7]
#  [ 8  8 10]
#  [11 11 13]]
print y

# We will add the vector v to each row of the matrix x,
# storing the result in the matrix y
x = np.array([[1,2,3], [4,5,6], [7,8,9], [10, 11, 12]])
v = np.array([1, 0, 1])
vv = np.tile(v, (4, 1))  # Stack 4 copies of v on top of each other
print vv                 # Prints "[[1 0 1]
                         #          [1 0 1]
                         #          [1 0 1]
                         #          [1 0 1]]"
y = x + vv  # Add x and vv elementwise
print y  # Prints "[[ 2  2  4
         #          [ 5  5  7]
         #          [ 8  8 10]
         #          [11 11 13]]"

# We will add the vector v to each row of the matrix x,
# storing the result in the matrix y
x = np.array([[1,2,3], [4,5,6], [7,8,9], [10, 11, 12]])
v = np.array([1, 0, 1])
y = x + v  # Add v to each row of x using broadcasting
print y  # Prints "[[ 2  2  4]
         #          [ 5  5  7]
         #          [ 8  8 10]
         #          [11 11 13]]"

# Compute outer product of vectors
v = np.array([1,2,3])  # v has shape (3,)
w = np.array([4,5])    # w has shape (2,)
# To compute an outer product, we first reshape v to be a column
# vector of shape (3, 1); we can then broadcast it against w to yield
# an output of shape (3, 2), which is the outer product of v and w:
# [[ 4  5]
#  [ 8 10]
#  [12 15]]
print np.reshape(v, (3, 1)) * w

# Add a vector to each row of a matrix
x = np.array([[1,2,3], [4,5,6]])
# x has shape (2, 3) and v has shape (3,) so they broadcast to (2, 3),
# giving the following matrix:
# [[2 4 6]
#  [5 7 9]]
print x + v

# Add a vector to each column of a matrix
# x has shape (2, 3) and w has shape (2,).
# If we transpose x then it has shape (3, 2) and can be broadcast
# against w to yield a result of shape (3, 2); transposing this result
# yields the final result of shape (2, 3) which is the matrix x with
# the vector w added to each column. Gives the following matrix:
# [[ 5  6  7]
#  [ 9 10 11]]
print (x.T + w).T
# Another solution is to reshape w to be a row vector of shape (2, 1);
# we can then broadcast it directly against x to produce the same
# output.
print x + np.reshape(w, (2, 1))

# Multiply a matrix by a constant:
# x has shape (2, 3). Numpy treats scalars as arrays of shape ();
# these can be broadcast together to shape (2, 3), producing the
# following array:
# [[ 2  4  6]
#  [ 8 10 12]]
print x * 2


#
import math
import numpy as np
import matplotlib.pyplot as plt

class Chp003C003(object):
    def __init__(self):
        self.name = ''

    def run(self):
        print('一元高斯分布图像')
        mu = 3.0
        sigma = 0.5
        x = np.linspace(-1.0, 6.0, 100)
        y = self.gaussian(x, mu, sigma)
        plt.rcParams['font.sans-serif'] = ['SimHei']
        plt.rcParams['axes.unicode_minus'] = False
        plt.title('一元高斯分布图像')
        plt.plot(x, y, '-b')
        plt.show()

    def gaussian(self, x, mu, sigma):
        v1 = 1 / (math.sqrt(2 * math.pi) * sigma)
        v2 = (x-mu)*(x-mu) / (2 * sigma * sigma)
        v3 = np.exp(-v2)
        return v1 * v3

if '__main__' == __name__:
    app = Chp003C003()
    app.run()


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Sat Nov 25 16:35:07 2017

@author: ratnadeepb
@License: MIT
"""

'''
Let c[0, 1] be the space of all continuous real valued functions in the interval
[0, 1].

Let f(x) = x + 1
and g(x) = 2 + x^2

Inner Product is defined as the definite integral of f(x) * g(x) in the interval
0 to 1.
'''

import sympy
import numpy as np
import sys

# Function dot product
def fn_dot(f, g, upper, lower):
    try:
        return sympy.N(sympy.integrate(f*g, (x, lower, upper)))
    except:
        sys.exit("Problem not formatted correctly")

# Function norm
def fn_norm(f, upper, lower):
    return np.sqrt(float(fn_dot(f, f, upper, lower)))

# Angle between two functions
def fn_angle(f, g, upper, lower, op="radians"):
    if op not in ["radians", "degrees"]:
        sys.exit("At this time we only handle radians and degrees")
    return np.arccos(float(fn_dot(f, g, upper, lower) / (fn_norm(f, upper,
                           lower) * fn_norm(g, upper, lower))))

if __name__ == "__main__":
    upper = 1
    lower = 0

    x = sympy.symbols('x')

    f = x + 1
    g = x**2 + 2

    print("The inner product of the functions is: ", fn_dot(f, g, upper, lower))
    print("The norm of f is: ", fn_norm(f, upper, lower))
    print("The angle between the functions is: ", fn_angle(f, g, upper, lower))


## Worksheet 2

import numpy as np
import scipy.linalg as la

#### Answer Coding Question 1

def LU_decomposition(A):
    """Perform LU decomposition using the Doolittle factorisation."""

    L = np.zeros_like(A)
    U = np.zeros_like(A)
    N = np.size(A, 0)

    for k in range(N):
        L[k, k] = 1
        U[k, k] = (A[k, k] - np.dot(L[k, :k], U[:k, k])) / L[k, k]
        for j in range(k+1, N):
            U[k, j] = (A[k, j] - np.dot(L[k, :k], U[:k, j])) / L[k, k]
        for i in range(k+1, N):
            L[i, k] = (A[i, k] - np.dot(L[i, :k], U[:k, k])) / U[k, k]

    return L, U

#### Answer Coding Question 2

def ThomasAlgorithm(a, b, c, f):
    """Implement the Thomas algorithm to solve A x = f.
    The vectors a, b, c are the sub-diagonal, diagonal and super-diagonal
    vectors of the original matrix A."""

    # Make copies of the input
    aa = a.copy()
    bb = b.copy()
    cc = c.copy()

    x = np.zeros_like(f)
    d = np.zeros_like(f)
    d[:] = f[:]
    N = len(f)
    for k in range(1, N):
        m = aa[k-1] / bb[k-1]
        bb[k] -= m * cc[k-1]
        d[k] -= m * d[k-1]
    x[-1] = d[-1] / bb[-1]
    for k in range(N-2, -1, -1):
        x[k] = (d[k] - cc[k] * x[k+1]) / bb[k]

    return x

#### Answer Coding Question 3

def Jacobi(A, b, tolerance = 1.e-10, MaxSteps = 100):
    """Solve the linear system A x = b using Jacobi's method,
    starting from the trivial initial guess."""

    x = np.zeros_like(b)

    Anorm = A.copy()
    bnorm = b.copy()
    n = len(b)

    for i in range(n):
        bnorm[i] /= A[i, i]
        Anorm[i, :] /= A[i, i]

    # Compute the split
    N = np.eye(n)
    P = N - Anorm
    AL = la.tril(P)
    AU = la.triu(P)

    # Compute the convergence matrix and check its spectral radius
    M = np.dot(la.inv(N), P)
    eigenvalues, eigenvectors = la.eig(M)
    rho = np.amax(np.absolute(eigenvalues))
    if (rho > 1):
        print("Jacobi will not converge as the"\
            " largest eigenvalue of the convergence matrix is {}".format(rho))

    for j in range(MaxSteps):
        x_old = x.copy()
        x = bnorm + np.dot(AL + AU, x)
        if (la.norm(x - x_old) < tolerance):
            print "Jacobi converged in ", j, " iterations."
            break

    return x

#### Answer Coding Question 4

def GaussSeidel(A, b, tolerance = 1.e-10, MaxSteps = 100):
    """Solve the linear system A x = b using the Gauss-Seidel method,
    starting from the trivial initial guess."""

    x = np.zeros_like(b)

    Anorm = A.copy()
    bnorm = b.copy()
    n = len(b)

    for i in range(n):
        bnorm[i] /= A[i, i]
        Anorm[i, :] /= A[i, i]

    # Compute the split
    D = np.eye(n)
    AL = la.tril(D - Anorm)
    AU = la.triu(D - Anorm)
    N = np.eye(n) - AL
    P = AU

    # Compute the convergence matrix and check its spectral radius
    M = np.dot(la.inv(N), P)
    eigenvalues, eigenvectors = la.eig(M)
    rho = np.amax(np.absolute(eigenvalues))
    if (rho > 1):
        print("Gauss-Seidel will not converge as the"\
              " largest eigenvalue of the convergence matrix is {}".format(rho))

    for j in range(MaxSteps):
        x_old = x.copy()
        for i in range(n):
            x[i] = bnorm[i] + np.dot(AL[i, :], x) + np.dot(AU[i, :], x_old)
        if (la.norm(x - x_old) < tolerance):
            print("Gauss-Seidel converged in {} iterations.".format(j))
            break

    return x

#### Answer Coding Question 5

def chord(f, m, x0, tolerance = 1e-10, MaxSteps = 100):
    """Implement the chord method to find the root of the equation f(x) = 0,
    starting from the initial guess x^{(0)} = x0."""

    x = np.zeros(MaxSteps)
    x[0] = x0

    # Set up the map g
    g = lambda x: x - m * f(x)

    for i in range(1, MaxSteps):
        x[i] = g(x[i-1])
        if (np.absolute(f(x[i])) < tolerance):
            break
    return x[:i+1]


import random
import matplotlib.pyplot as plt
import numpy as np

def graph(x):
    return 5 * (x ** 4) - 8.7 * (x ** 3) + 33 * (x ** 2) + 21 * (x) + 10.8

xValues = np.linspace(0, 5, 10000)
yValues = np.array(list(map(graph, xValues)))

plt.figure()
plt.plot(xValues, yValues, '-b')
plt.xlabel("x")
plt.ylabel("y")
plt.title("Graph of y = 5(x^4) - 8.7(x^3) + 33(x^2) + 21(x) + 10.8")
plt.grid()
plt.show()

##################################################################################

xLimits = [0, 5]
yLimits = [0, graph(5)]
xRange = xLimits[1] - xLimits[0]
yRange = yLimits[1] - yLimits[0]

pointsUnderGraph = []
pointsOverGraph = []

def calcArea(iterations: int, appendToArray: bool) -> float:
    countUnder = 0

    for i in range(iterations):
        point = [random.uniform(xLimits[0], xLimits[1]), random.uniform(yLimits[0], yLimits[1])]
        isUnderGraph = point[1] < graph(point[0])
        countUnder += 1 if isUnderGraph else 0

        if appendToArray:
            if isUnderGraph:
                pointsUnderGraph.append(point)
            else:
                pointsOverGraph.append(point)

    return (countUnder / iterations) * (xRange * yRange)

##################################################################################

area = calcArea(100000, True)
analytical = 3457.125
print("Area after 100,000 iterations: %f" % area)
print("Area achieved analytically:    %f" % analytical)
print("Percentage error:              %f" % np.abs(100 * (analytical - area) / analytical))

np_underGraph = np.array(pointsUnderGraph)
np_overGraph = np.array(pointsOverGraph)

plt.figure()
plt.scatter(np_underGraph[:, 0], np_underGraph[:, 1], c='b', label="Under Graph")
plt.scatter(np_overGraph[:, 0], np_overGraph[:, 1], c='r', label="Over Graph")
plt.title("Plot of Random Points Under and Over the Graph")
plt.legend()
plt.grid()
plt.show()

##################################################################################

N_iters = range(10000, 1000001, 10000)
areaValues = []
print()
for N_iter in N_iters:
    areaValues.append(calcArea(N_iter, False))
    print("Area after %d iterations: %f" % (N_iter, areaValues[-1]))

averageArea = np.mean(areaValues)
print()
print("Average area:                %f" % averageArea)
print("Area achieved analytically:  %f" % analytical)
print("Percentage error:            %f" % np.abs(100 * (analytical - averageArea) / analytical))

plt.figure()
plt.plot(N_iters, areaValues, '-bx')
plt.plot([N_iters[0], N_iters[-1]], [analytical, analytical], '-r')
plt.xlabel("Number of iterations")
plt.ylabel("Area")
plt.title("Plot of Area versus No. of Iterations")
plt.grid()
# plt.show()

##################################################################################

errorValues = list(map(lambda a: 100 * np.abs(a - analytical) / analytical, areaValues))

plt.figure()
plt.plot(N_iters, errorValues, '-bx')
plt.plot([N_iters[0], N_iters[-1]], [1, 1], '-r')
plt.xlabel("Number of iterations")
plt.ylabel("Percentage Error")
plt.title("Plot of Percentage Error versus No. of Iterations")
plt.grid()
plt.show()

minimumIter = 0
for i in reversed(range(len(errorValues))):  # traverse in reverse
    if errorValues[i] > 1:
        minimumIter = N_iters[i+1]  # at i+1 was last for which error was < 1%
        break
print()
print("Minimum number of iterations: %d" % minimumIter)


""" Code for the EKF Refactored """

import sympy
from sympy import atan, pi, tan
from sympy import symbols, Matrix
from math import sqrt, tan, cos, sin, atan2
import matplotlib.pyplot as plt
import numpy as np
from numpy.random import randn
from filterpy.kalman import ExtendedKalmanFilter as EKF
from numpy import array, sqrt

class RobotEKF(EKF):
    def __init__(self, dt, wheelbase, std_vel, std_steer):
        EKF.__init__(self, 3, 2, 2)
        self.dt = dt
        self.wheelbase = wheelbase
        self.std_vel = std_vel
        self.std_steer = std_steer

        a, x, y, v, w, theta, time = symbols(
            'a, x, y, v, w, theta, t')
        d = v*time
        beta = (d/w)*sympy.tan(a)
        r = w/sympy.tan(a)

        self.fxu = Matrix(
            [[x-r*sympy.sin(theta)+r*sympy.sin(theta+beta)],
             [y+r*sympy.cos(theta)-r*sympy.cos(theta+beta)],
             [theta+beta]])

        self.F_j = self.fxu.jacobian(Matrix([x, y, theta]))
        self.V_j = self.fxu.jacobian(Matrix([v, a]))

        # save dictionary and it's variables for later use
        self.subs = {x: 0, y: 0, v:0, a:0,
                     time:dt, w:wheelbase, theta:0}
        self.x_x, self.x_y, = x, y
        self.v, self.a, self.theta = v, a, theta

    def predict(self, u):
        self.x = self.move(self.x, u, self.dt)

        self.subs[self.theta] = self.x[2, 0]
        self.subs[self.v] = u[0]
        self.subs[self.a] = u[1]

        F = array(self.F_j.evalf(subs=self.subs)).astype(float)
        V = array(self.V_j.evalf(subs=self.subs)).astype(float)

        # covariance of motion noise in control space
        M = array([[self.std_vel*u[0]**2, 0],
                   [0, self.std_steer**2]])

        self.P = np.dot(F, self.P).dot(F.T) + np.dot(V, M).dot(V.T)

    def move(self, x, u, dt):
        hdg = x[2, 0]
        vel = u[0]
        steering_angle = u[1]
        dist = vel * dt

        if abs(steering_angle) > 0.001: # is robot turning?
            beta = (dist / self.wheelbase) * tan(steering_angle)
            r = self.wheelbase / tan(steering_angle) # radius

            dx = np.array([[-r*sin(hdg) + r*sin(hdg + beta)],
                           [r*cos(hdg) - r*cos(hdg + beta)],
                           [beta]])
        else: # moving in straight line
            dx = np.array([[dist*cos(hdg)],
                           [dist*sin(hdg)],
                           [0]])
        return x + dx

   ##MIT license

import numpy,from numpy,import sympy,from sympy


# you should fill in the functions in this file,
# do NOT change the name, input and output of these functions

import numpy as np
import time
import matplotlib.pyplot as plt

# first function to fill, compute distance matrix using loops
def compute_distance_naive(X):
    N = X.shape[0]      # num of rows
    D = X[0].shape[0]   # num of cols

    M = np.zeros([N,N])
    for i in range(N):
        for j in range(N):
            xi = X[i,:]
            xj = X[j,:]
            dist = np.linalg.norm(xi-xj)
            M[i,j] = dist

    return M

# second function to fill, compute distance matrix without loops
def compute_distance_smart(X):
    N = X.shape[0]  # num of rows
    D = X[0].shape[0]  # num of cols

    # use X to create M
    M = np.zeros([N, N])

    sum = np.sum(np.multiply(X,X), axis=1)

    x2 = (sum * np.ones([N,1]))
    y2 = np.transpose(x2)
    xy = np.dot(X, X.T)

    M = np.sqrt(abs(x2 - 2 * xy + y2))
    return M

# third function to fill, compute correlation matrix using loops
def compute_correlation_naive(X):
    N = X.shape[0]  # num of rows
    D = X[0].shape[0]  # num of cols

    # use X to create M
    M = np.zeros([D, D])

    for i in range(D):
        for j in range(D):
            xi = X[:, i]
            xj = X[:, j]

            mui = np.sum(xi).astype(float) / N
            muj = np.sum(xj).astype(float) / N
            xni = xi - mui
            xnj = xj - muj
            sij = (np.dot(xni, xnj)).astype(float) / (N - 1)

            sigmai = np.sqrt(np.dot(xni, xni).astype(float) / (N - 1))
            sigmaj = np.sqrt(np.dot(xnj, xnj).astype(float) / (N - 1))
            sigma = sigmai * sigmaj

            cmatrix = sij.astype(float) / sigma
            M[i, j] = cmatrix

    return M

# fourth function to fill, compute correlation matrix without loops
def compute_correlation_smart(X):
    N = X.shape[0]  # num of rows
    D = X[0].shape[0]  # num of cols

    # use X to create M
    M = np.zeros([D, D])

    vectormu = (np.sum(X, axis=0)).astype(float) / N
    matrixmu = vectormu * np.ones([N, 1])
    x = X - matrixmu
    xT = np.transpose(x)
    covarience = (np.dot(xT, x)).astype(float) / (N - 1)
    x2 = np.multiply(x, x)
    varience = (np.sum(x2, axis=0)).astype(float) / (N - 1)
    sigma = np.sqrt(varience)
    denmatrix = np.outer(sigma, sigma)

    with np.errstate(divide='ignore', invalid='ignore'):
        c = np.power(denmatrix,-1)
        c[c == np.inf] = 0
        c = np.nan_to_num(c)
        M = np.multiply(covarience, c)

    return M

def main():
    print 'starting comparing distance computation .....'
    np.random.seed(100)
    params = range(10,141,10)   # different param setting
    nparams = len(params)       # number of different parameters

    perf_dist_loop = np.zeros([10,nparams])  # 10 trials = 10 rows, each parameter is a column
    perf_dist_cool = np.zeros([10,nparams])
    perf_corr_loop = np.zeros([10,nparams])  # 10 trials = 10 rows, each parameter is a column
    perf_corr_cool = np.zeros([10,nparams])

    counter = 0

    for ncols in params:
        nrows = ncols * 10

        print "matrix dimensions: ", nrows, ncols

        for i in range(10):
            X = np.random.rand(nrows, ncols)   # random matrix

            # compute distance matrices
            st = time.time()
            dist_loop = compute_distance_naive(X)
            et = time.time()
            perf_dist_loop[i,counter] = et - st              # time difference

            st = time.time()
            dist_cool = compute_distance_smart(X)
            et = time.time()
            perf_dist_cool[i,counter] = et - st

            assert np.allclose(dist_loop, dist_cool, atol=1e-06) # check if the two computed matrices are identical all the time

            # compute correlation matrices
            st = time.time()
            corr_loop = compute_correlation_naive(X)
            et = time.time()
            perf_corr_loop[i,counter] = et - st              # time difference

            st = time.time()
            corr_cool = compute_correlation_smart(X)
            et = time.time()
            perf_corr_cool[i,counter] = et - st

            assert np.allclose(corr_loop, corr_cool, atol=1e-06) # check if the two computed matrices are identical all the time

        counter = counter + 1

    mean_dist_loop = np.mean(perf_dist_loop, axis = 0)    # mean time for each parameter setting (over 10 trials)
    mean_dist_cool = np.mean(perf_dist_cool, axis = 0)
    std_dist_loop = np.std(perf_dist_loop, axis = 0)      # standard deviation
    std_dist_cool = np.std(perf_dist_cool, axis = 0)

    plt.figure(1)
    plt.errorbar(params, mean_dist_loop[0:nparams], yerr=std_dist_loop[0:nparams], color='red',label = 'Loop Solution for Distance Comp')
    plt.errorbar(params, mean_dist_cool[0:nparams], yerr=std_dist_cool[0:nparams], color='blue', label = 'Matrix Solution for Distance Comp')
    plt.xlabel('Number of Cols of the Matrix')
    plt.ylabel('Running Time (Seconds)')
    plt.title('Comparing Distance Computation Methods')
    plt.legend()
    plt.savefig('CompareDistanceCompFig.pdf')
    # plt.show()    # uncomment this if you want to see it right way
    print "result is written to CompareDistanceCompFig.pdf"

    mean_corr_loop = np.mean(perf_corr_loop, axis = 0)    # mean time for each parameter setting (over 10 trials)
    mean_corr_cool = np.mean(perf_corr_cool, axis = 0)
    std_corr_loop = np.std(perf_corr_loop, axis = 0)      # standard deviation
    std_corr_cool = np.std(perf_corr_cool, axis = 0)

    plt.figure(2)
    plt.errorbar(params, mean_corr_loop[0:nparams], yerr=std_corr_loop[0:nparams], color='red',label = 'Loop Solution for Correlation Comp')
    plt.errorbar(params, mean_corr_cool[0:nparams], yerr=std_corr_cool[0:nparams], color='blue', label = 'Matrix Solution for Correlation Comp')
    plt.xlabel('Number of Cols of the Matrix')
    plt.ylabel('Running Time (Seconds)')
    plt.title('Comparing Correlation Computation Methods')
    plt.legend()
    plt.savefig('CompareCorrelationCompFig.pdf')
    # plt.show()    # uncomment this if you want to see it right way
    print "result is written to CompareCorrelationCompFig.pdf"

if __name__ == "__main__": main()


# shows how linear regression analysis can be applied to 1-dimensional data
#
# notes for this course can be found at:
# https://deeplearningcourses.com/c/data-science-linear-regression-in-python
# https://www.udemy.com/data-science-linear-regression-in-python

from __future__ import print_function, division
from builtins import range
# Note: you may need to update your version of future
# sudo pip install -U future

import numpy as np
import matplotlib.pyplot as plt

# load the data
X = []
Y = []
for line in open('data_1d.csv'):
    x, y = line.split(',')
    X.append(float(x))
    Y.append(float(y))

# let's turn X and Y into numpy arrays since that will be useful later
X = np.array(X)
Y = np.array(Y)

# let's plot the data to see what it looks like
plt.scatter(X, Y)
plt.show()

# apply the equations we learned to calculate a and b

# denominator is common
# note: this could be more efficient if
#       we only computed the sums and means once
denominator = X.dot(X) - X.mean() * X.sum()
a = ( X.dot(Y) - Y.mean()*X.sum() ) / denominator
b = ( Y.mean() * X.dot(X) - X.mean() * X.dot(Y) ) / denominator

# let's calculate the predicted Y
Yhat = a*X + b

# let's plot everything together to make sure it worked
plt.scatter(X, Y)
plt.plot(X, Yhat)
plt.show()

# determine how good the model is by computing the r-squared
d1 = Y - Yhat
d2 = Y - Y.mean()
r2 = 1 - d1.dot(d1) / d2.dot(d2)
print("the r-squared is:", r2)


# Linear Regression

## Model

given a dataset $\left \{ (x^{(1)},y^{(1)}),...,(x^{(n)},y^{(n)}) \right \} $, where $x^{(i)} \in \mathbb{R}^{d},\ y^{(i)} \in \mathbb{R}$.

linear regression is trying to model $y^{(i)}$ from $x^{(i)}$ by a linear model:

$$h_{\theta}(x^{(i)}) = \theta_{0} + \theta_{1}x_{1}^{(i)} + \theta_{2}x_{2}^{(i)} + ... + \theta_{d}x_{d}^{(i)}\approx y^{(i)}$$

for the sake of simplicity, we set $x_{0}^{(i)}=1,\ x^{(i)} = (x_{0}^{(i)},...,x_{d}^{(i)}),\ \theta=(\theta_{0},...,\theta_{d})$, then:

$$h_{\theta}(x^{(i)}) = \sum_{j=0}^{d}\theta_{j}x_{j}^{(i)} = \theta^{T}x^{(i)}$$

## Loss Function

given $\hat{y}=(h_{\theta}^{(1)},...,h_{\theta}^{(n)}),\ y=(y^{(1)},...,y^{(n)})$.

we want to approximate $y$ by $\hat{y}$, or equivalent to say, we want to minimize the distance between $\hat{y}$ and $y$.

using euclidean distance, we derive the loss function for linear regression:

$$J(\theta) = \frac{1}{2}\sum_{i=1}^{n}(h_{\theta}x^{(i)} - y^{(i)})^2$$

## Gradient Descent

after defining model and loss function, our goal now is to find the model that minimize the loss function:

$$\hat{\theta} = \underset{\theta}{argmin}\ J(\theta)$$

to minimize the loss function, we can set

$$\nabla J(\theta)=0$$

but to find the analytic solution of this equation is usually impossible.

alternatively, we can init $\theta$ randomly, then iteratively move $\theta$ towards the direction that makes $J(\theta)$ smaller.

remmenber the oposite direction of gradient is the fastest direction that makes functions smaller, we derive gradient descent:

$$\theta := \theta - \alpha\nabla{J(\theta )}$$

$\alpha > 0$ is called the learning rate

appendix: proof of opposite gradient as fastest descent

for all $l\in \mathbb{R}^{d}, \left \| l \right \| =1$

$$\lim_{x \to 0} \frac{J(\theta + x l) - J(\theta ) }{x}=l \cdot \nabla{J(\theta )} >= -\left \| l \right \|\left \|J(\theta)  \right \| = -\left \|  J(\theta )\right \|   $$

equality obtained only if $l$ is in the opposite direction of $\nabla{J(\theta )}$

import numpy as np

X = 2 * np.random.rand(100, 1)
y = 4 + 3 * X + np.random.randn(100, 1)
X_new = np.array([[0], [2]])

"""LinearRegression actually uses SVD to get pseudoinverse, based on scipy.linalg.lstsq()"""
from sklearn.linear_model import LinearRegression

lin_reg = LinearRegression()
lin_reg.fit(X, y)
lin_reg.intercept_, lin_reg.coef_

y_predict = lin_reg.predict(X_new)
y_predict

import matplotlib.pyplot as plt

plt.plot(X_new, y_predict, "r-", linewidth=2, label="Predictions")
plt.plot(X, y, "b.")
plt.xlabel("$x_1$", fontsize=18)
plt.ylabel("$y$", rotation=0, fontsize=18)
plt.legend(loc="upper left", fontsize=14)
plt.axis([0, 2, 0, 15])
plt.show()

### stochastic gradient descent

in practice, to update $\theta$ by gradient descent(concretely batch gradient descent), at each step, we need to calculate all sample's gradient, too slow.

to fix this problem, we can only use one sample's gradient at a time, choose that sample randomly:

$$\theta := \theta - \alpha{l(h_{\theta}(x^{(i)}), y^{(i)})}$$

from sklearn.linear_model import SGDRegressor

sgd_reg = SGDRegressor(max_iter=1000, tol=1e-3, penalty=None, eta0=0.1)
sgd_reg.fit(X, y.ravel())

sgd_reg.intercept_, sgd_reg.coef_

### mini-batch gradient descent

sgd too random, batch gradient descent too slow, we can use some(not one, not all) samples at a time:

$$\theta := \theta - \alpha\sum_{i \in batch}{l(h_{\theta}(x^{(i)}), y^{(i)})}$$

## update rule of linear regression

for linear regression, we have:

$$
\begin{equation}
\begin{split}
\frac{\partial }{\partial \theta_{j}}J(\theta ) &=  \frac{\partial }{\partial \theta_{j}}\frac{1}{2}\sum_{i=1}^{n}(h_{\theta }(x^{(i)}) - y^{(i)})^2  \\
&=\sum_{i=1}^{n}(h_{\theta }(x^{(i)}) - y^{(i)})\cdot{}\frac{\partial }{\partial \theta_{j}}(h_{\theta }(x^{(i)}) - y^{(i)})\\
& =\sum_{i=1}^{n}(h_{\theta }(x^{(i)}) - y^{(i)})x_{j}^{(i)}
\end{split}
\end{equation}
$$

so we have the update rule for linear regression:

$$\theta_{j}: =\theta_{j} - \alpha\sum_{i=1}^{n} (h_{\theta }(x^{(i)}) - y^{(i)})x_{j}^{(i)} $$

combine all dimensions, we have:

$$\theta: =\theta - \alpha\sum_{i=1}^{n} (h_{\theta }(x^{(i)}) - y^{(i)})\cdot x^{(i)} $$

### matrix form

define $X = [(x^{(1)})^{T},...,(x^{(n)})^{T}]$, then we can write $J(\theta)$ in matrix form:

$$
\begin{equation}
\begin{split}
J(\theta) &= \frac{1}{2}\sum_{i=1}^{n}(h_{\theta}x^{(i)} - y^{(i)})^2 \\
&= \frac{1}{2}\sum_{i=1}^{n}(\theta^{T}x^{(i)} - y^{(i)})^2 \\
&= \frac{1}{2}(X\theta - y)^{T}(X\theta - y)
\end{split}
\end{equation}
$$

we then have:

$$
\begin{equation}
\begin{split}
\nabla{J(\theta )} &= \nabla\frac{1}{2}(X\theta - y)^{T}(X\theta - y) \\
&= \frac{1}{2}\nabla(\theta^{T}X^{T}X\theta - y^{T}(X\theta) - (X\theta)^{T}y) \\
&= \frac{1}{2}\nabla(\theta^{T}X^{T}X\theta - 2(X^{T}y)^{T}\theta) \\
&= \frac{1}{2}(2X^{T}X\theta - 2(X^{T}y)) \\
&= X^{T}X\theta - X^{T}y
\end{split}
\end{equation}
$$

here we use

1.$a^{T}b=b^{T}a$, obvious.

2.$\nabla{a^{T}x}=a$, obvious.

3.$\nabla{x^{T}Ax} = (A + A^{T})x$, proof:

$$
\begin{equation}
\begin{split}
\frac{\partial}{\partial x_{i}}x^{T}Ax &=
\frac{\partial}{\partial x_{i}}{\sum_{j=1}^{n}}{\sum_{k=1}^{n}}a_{jk}x_{j}x_{k} \\
&= \frac{\partial}{\partial x_{i}}(\sum_{j\ne{i}}a_{ji}x_{j}x_{i} + \sum_{k\ne{i}}a_{ik}x_{i}x_{k} + a_{ii}x_{i}^{2}) \\
&= \sum_{j\ne{i}}a_{ji}x_{j} + \sum_{k\ne{i}}a_{ik}x_{k} + 2a_{ii}x_{ii} \\
&= \sum_{j=1}^{n}a_{ji}x_{j} + \sum_{k=1}^{n}a_{ik}x_{k} \\
&= (A^{T}x)_{i} + (Ax)_{i}
\end{split}
\end{equation}
$$

finally, we get the matrix form of the update rule:

$$\theta: =\theta - \alpha X^{T}(X\theta-\mathbf{y} ) $$

## analytic solution

from above, we have:

$$\nabla{J(\theta )} = X^{T}X\theta - X^{T}y$$

so the equation of zero gradient change to:

$$X^{T}X\theta - X^{T}y = 0$$

if $X^{T}X$ is invertible:

$$\theta = (X^{T}X)^{-1}X^{T}y$$

if $X^{T}X$ is not invertible, the equation also have solution. proof:

on one hand $X\theta = 0 \Rightarrow X^{T}X\theta=0$

on the other hand $X^{T}X\theta=0 \Rightarrow \theta^{T}X^{T}X\theta=0 \Rightarrow (X\theta)^{T}X\theta=0 \Rightarrow X\theta=0$

so $X\theta=0 \Leftrightarrow X^{T}X\theta=0$, that is to say $null(X^{T}X)=null(X)$

we easily derive from the above that $rank(X^{T}X) = rank(X) = rank(X^{T})$

but $range(X^{T}X) \subseteq  range(X^{T})$, so we must have $range(X^{T}X)=range(X^{T})$

X_b = np.c_[np.ones((100, 1)), X]
theta_best = np.linalg.inv(X_b.T.dot(X_b)).dot(X_b.T).dot(y)
theta_best

## geometric interpretation of the linear regression

consider the linear space $S = span\left \{  columns\ of\ X \right \}$, linear combination of $S$ could be write as $X\theta$, then:

$X\theta$ is the projection of $y$ on $S \Leftrightarrow$ $X\theta - y$ orthogonal with $S \Leftrightarrow$ orthogonal with $columns\ of\ X \Leftrightarrow X^{T}(X\theta - y)=0$

so linear regression could be interpret as finding the projection of $y$ on $S$.

## probabilistic interpretation

assume targets and inputs are related via:

$$y^{(i)} = \theta^{T}x^{(i)} + \epsilon^{(i)}$$

where $\epsilon^{(i)}$ is the error term, assume that $\epsilon^{(i)}$ are distributed IID according to Gaussian with mean 0 and variance $\sigma^{2}$, i.e the density of $\epsilon^{(i)}$ is given by:

$$p(\epsilon^{(i)}) = \frac{1}{\sqrt{2\pi}\sigma}exp\left (-\frac{(\epsilon^{(i)})^{2}}{2\sigma^{2}}\right )$$

this implies:

$$p(y^{(i)}|x^{(i)}; \theta) = \frac{1}{\sqrt{2\pi}\sigma}exp\left ( -\frac{(y^{(i)} - \theta^{T}x^{(i)})^{2}}{2\sigma^{2}}\right)$$

we should denote that $\theta$ is not a random variable.

given $X$ and $\theta$, what is the probability of $y$? we call it the likelihood function:

$$L(\theta) = L(\theta; X,y) = p(y|X; \theta)$$

for the above assumptions:

$$
\begin{equation}
\begin{split}
L(\theta) &= \prod_{i=1}^{n}p(y^{(i)}|x^{(i)}; \theta) \\
&= \prod_{i=1}^{n}\frac{1}{\sqrt{2\pi}\sigma}exp\left ( -\frac{(y^{(i)} - \theta^{T}x^{(i)})^{2}}{2\sigma^{2}}\right)
\end{split}
\end{equation}
$$

we should choose $\theta$ so as to make the data as high probability as possible, i.e we should choose $\theta$ to maxmize $L(\theta)$, this is called maximum likelihood. one step further:

$$maximize\ L(\theta) \Leftrightarrow maxmize\ log(L(\theta))$$

so we maximize the log likelihood, this is simpler.

$$
\begin{equation}
\begin{split}
l(\theta) &= log(L(\theta)) \\
&= log\prod_{i=1}^{n}\frac{1}{\sqrt{2\pi}\sigma}exp\left ( -\frac{(y^{(i)} - \theta^{T}x^{(i)})^{2}}{2\sigma^{2}}\right) \\
&= \sum_{i=1}^{n}log\frac{1}{\sqrt{2\pi}\sigma}exp\left ( -\frac{(y^{(i)} - \theta^{T}x^{(i)})^{2}}{2\sigma^{2}}\right) \\
&= n\ log\frac{1}{\sqrt{2\pi}\sigma} - \frac{1}{2\sigma^{2}}\sum_{i=1}^{n}(y^{(i)} - \theta^{T}x^{(i)})^{2}
\end{split}
\end{equation}
$$

hence, maximizing $l(\theta)$ gives the same answer as minimizing

$$\frac{1}{2}\sum_{i=1}^{n}(y^{(i)} - \theta^{T}x^{(i)})^{2} = J(\theta)$$

so linear regression $\Leftrightarrow $ maximum likelihood given Gaussian error

## regularization

to lower variance $\Rightarrow $ to limit model's complexity $\Rightarrow $ to prevent the absolute value of parameters to be too large $\Rightarrow $ we add punishment term concerning the absolute value of parameters on $J(\theta)$

choose $\lambda{\left \| \theta   \right \|}_{2}^{2}$ as the punishment term:

$$J(\theta) := J(\theta) + \lambda{\left \| \theta   \right \|}_{2}^{2}$$

$\lambda$ is the regularization hyperparameter, linear regression with $l_{2}$ loss is ridge regression.

otherwise, if we choose $\lambda{\left \| \theta   \right \|}_{1}$ as the punishment term:

$$J(\theta) := J(\theta) + \lambda{\left \| \theta   \right \|}_{1}$$

that is called lasso regression.

## probabilistic interpretation of regularization

as before, we assume:

$$p(y^{(i)}|x^{(i)}; \theta) = \frac{1}{\sqrt{2\pi}\sigma}exp\left ( -\frac{(y^{(i)} - \theta^{T}x^{(i)})^{2}}{2\sigma^{2}}\right)$$

in bayes perspective, we can also assume $\theta \sim N(0, \sigma_{0})$, that is:

$$p(\theta) = \frac{1}{\sqrt{2\pi}\sigma_{0}}exp\left ( -\frac{\left \| \theta   \right \|_{2}^{2}}{2\sigma_{0}^{2}}\right)$$

by bayes rule:

$$P(\theta|y)=\frac{P(y|\theta)P(\theta)}{P(y)}$$

thus we have maximum a posteriori estimation(MAP):

$$
\begin{equation}
\begin{split}
\hat{\theta} =&\underset{\theta}{argmax}P(\theta|y)\\
=&\underset{\theta}{argmax}P(y|\theta)P(\theta) \\
=&\underset{\theta}{argmax}\ logP(y|\theta)P(\theta) \\
=&\underset{\theta}{argmax}\ log\ exp\left ( \sum_{i=1}^{n}-\frac{(y^{(i)} - \theta^{T}x^{(i)})^{2}}{2\sigma^{2}}\right)exp\left ( -\frac{\left \| \theta   \right \|_{2}^{2}}{2\sigma_{0}^{2}}\right)\\
=&\underset{\theta}{argmin}\left(\sum_{i=1}^{n}\frac{(y^{(i)} - \theta^{T}x^{(i)})^{2}}{2\sigma^{2}} + \frac{\left \| \theta   \right \|_{2}^{2}}{2\sigma_{0}^{2}}\right)\\
=&\underset{\theta}{argmin}\left(\sum_{i=1}^{n}(y^{(i)} - \theta^{T}x^{(i)})^{2} + \frac{\sigma^{2}}{\sigma_{0}^{2}}\left \| \theta   \right \|_{2}^{2}\right)
\end{split}
\end{equation}
$$

that is exactly minimize:

$$J(\theta) = \frac{1}{2}\sum_{i=1}^{n}(y^{(i)} - \theta^{T}x^{(i)})^2 + \lambda{\left \| \theta   \right \|}^{2}$$

with $\lambda=\frac{\sigma^{2}}{2\sigma_{0}^{2}}$

in one word:

1. linear regression $\Leftrightarrow $ maximum likelihood estimation(MLE) where noise is guassian.
2. ridge regression $\Leftrightarrow $ maximum a posteriori estimation(MAP) where noise and prior are guassian.

from sklearn.linear_model import Ridge

ridge_reg = Ridge(alpha=1, solver="cholesky", random_state=42)
ridge_reg.fit(X, y)
ridge_reg.predict([[1.5]])

"""set SGDRegressor penalty=l2"""
sgd_reg = SGDRegressor(penalty="l2", max_iter=1000, tol=1e-3, random_state=42)
sgd_reg.fit(X, y.ravel())
sgd_reg.predict([[1.5]])

## Lasso

"""use Lasso or set SGDRegressor penalty=l1"""
from sklearn.linear_model import Lasso

lasso_reg = Lasso(alpha=0.1)
lasso_reg.fit(X, y)
lasso_reg.predict([[1.5]])

"""combination of ridge and lasso"""
from sklearn.linear_model import ElasticNet

elastic_net = ElasticNet(alpha=0.1, l1_ratio=0.5)
elastic_net.fit(X, y)
elastic_net.predict([[1.5]])

## Polynomial Regression

m = 100
X = 6 * np.random.rand(m, 1) - 3
y = 0.5 * X**2 + X + 2 + np.random.randn(m, 1)

plt.plot(X, y, "b.")
plt.xlabel("$x_1$", fontsize=18)
plt.ylabel("$y$", rotation=0, fontsize=18)
plt.axis([-3, 3, 0, 10])
plt.show()

from sklearn.preprocessing import PolynomialFeatures

poly_features = PolynomialFeatures(degree=2, include_bias=False)
X_poly = poly_features.fit_transform(X)

lin_reg = LinearRegression()
lin_reg.fit(X_poly, y)
lin_reg.intercept_, lin_reg.coef_

X_new=np.linspace(-3, 3, 100).reshape(100, 1)
X_new_poly = poly_features.transform(X_new)
y_new = lin_reg.predict(X_new_poly)

plt.plot(X, y, "b.")
plt.plot(X_new, y_new, "r-", linewidth=2, label="Predictions")
plt.xlabel("$x_1$", fontsize=18)
plt.ylabel("$y$", rotation=0, fontsize=18)
plt.legend(loc="upper left", fontsize=14)
plt.axis([-3, 3, 0, 10])
plt.show()

"""using pipeline"""
from sklearn.preprocessing import StandardScaler
from sklearn.pipeline import Pipeline

def polynomial_regression(degree, include_bias=True):
    return Pipeline([
        ("poly_features", PolynomialFeatures(degree=degree, include_bias=False)),
        ("std_scaler", StandardScaler()),
        ("lin_reg", LinearRegression()),
    ])

poly_reg = polynomial_regression(degree=2)
poly_reg.fit(X, y)

X_new = np.linspace(-3, 3, 100).reshape(100, 1)
y_new = poly_reg.predict(X_new)

plt.plot(X, y, "b.")
plt.plot(X_new, y_new, "r-", linewidth=2, label="Predictions")
plt.xlabel("$x_1$", fontsize=18)
plt.ylabel("$y$", rotation=0, fontsize=18)
plt.legend(loc="upper left", fontsize=14)
plt.axis([-3, 3, 0, 10])
plt.show()


import math
import numpy as np

def f(x, y): #objective function
    return 2*x + y**2 + math.exp(-x-y+1)

def delta(x, y): #delta = -1 * gradient(f(x_(n)))
    return np.array([math.exp(-x-y+1) - 2, math.exp(-x-y+1) - 2*y])

def backtracking(x_n, c, t):
    global counter
    counter += 1
    x_n_new = np.round(x_n + t*delta(x_n[0], x_n[1]), 7) #calculate x_(n + 1)
    temp = round((-delta(x_n[0], x_n[1])[0])*((x_n_new-x_n)[0]) + (-delta(x_n[0], x_n[1])[1])*((x_n_new-x_n)[1]), 7)
    if round(f(x_n_new[0], x_n_new[1]), 7) <= round(f(x_n[0], x_n[1]) + c*temp, 7): #condition is true, algorithm stops
        print('Best approximation of x_n is:', x_n_new)
        return x_n_new
    else: #condition is false, algorithm is repeated with x_(n + 1) and t = t/2
        print('Current x_n is:', x_n_new)
        backtracking(x_n_new, c, t/2)

counter = 0
x_n = np.array([3, 3]) #inital point
c = 0.8
t = 1
print('Initial x_n is:', x_n)
backtracking(x_n, c, t)
print('\nThe steepest descent backtracking algorithm ran', counter, 'times.')
print('Total number of steps:', counter*3)


from numpy import cos, cosh, sin, sinh
import numpy as np
import matplotlib.pyplot as plt
'''
TASK 3
'''
#Newton's method
def newton(f,df,x_0,eps):
    xn = x_0

    while True:
        fxn = f(xn)
        if abs(fxn) < eps:
            return xn
        dfxn = df(xn)
        if dfxn == 0:
            print('Zero derivative -> no solution')
            return None
        xn = xn - fxn/dfxn

#math functions of f & df/dx
def f(r):
    return cos(r)*cosh(r) + 1

def df(r):
    return -sin(r)*cosh(r) + cos(r)*sinh(r)

#input parameters for newton's method
eps = 0.001

r1 = newton(f, df, 1.5, eps)          #r1 = 1.87527632324985
r2 = newton(f, df, 4.25, eps)         #r2 = 4.69409122046058

print("r_1: {:.3f}".format(r1))
print("r_2: {:.3f}".format(r2))

def phi_1(x):
    return sin(r1*x) + sinh(r1*x) + ((cos(r1)+cosh(r1))/(sin(r1)+sinh(r1)))*(cos(r1*x)-cosh(r1*x))

def phi_2(x):
    return sin(r2*x) + sinh(r2*x) + ((cos(r2)+cosh(r2))/(sin(r2)+sinh(r2)))*(cos(r2*x)-cosh(r2*x))

x = np.arange(-1,2,0.02)    #250 data points
p1 = []                     #y values for phi_1
p2 = []                     #y values for phi_2

for point in x:
    p1.append(phi_1(float(point)))
    p2.append(phi_2(float(point)))

p1 = np.array(p1)
p2 = np.array(p2)

#plot phi_1
plt.plot(x, p1)
plt.title("Plot of phi_1")
plt.grid(color='k', linestyle='--', linewidth=0.5)
plt.show()

#plot phi_2
plt.plot(x, p2)
plt.title("Plot of phi_2")
plt.grid(color='k', linestyle='--', linewidth=0.5)
plt.ylim(-6, 4)
#max point of phi_2 (for clarity)
# xmax = round(x[np.argmax(p2)], 2)
# ymax = round(p2.max(), 6)
# def annot_max(x,y, ax=None):
#     text= "Max point: x={:.2f}, y={:.6f}".format(xmax, ymax)
#     if not ax:
#         ax=plt.gca()
#     bbox_props = dict(boxstyle="square,pad=0.3", fc="w", ec="k", lw=0.72)
#     arrowprops=dict(arrowstyle="->",connectionstyle="angle,angleA=0,angleB=60")
#     kw = dict(xycoords='data',textcoords="axes fraction",
#               arrowprops=arrowprops, bbox=bbox_props, ha="right", va="top")
#     ax.annotate(text, xy=(xmax, ymax), xytext=(0.94,0.96), **kw)
# annot_max(x,p2)
plt.show()


import math
import numpy as np

def basic_sigmoid(x):
    """
    Compute sigmoid of x.

    Arguments:
    x -- A scalar

    Return:
    s -- sigmoid(x)
    """

    s = 1 /(1+math.exp(-x))

    return s

number = 1
print("sigmoid of "+str(number)+" is "+ str(basic_sigmoid(number)))

x = [1, 2, 3]
# basic_sigmoid(x)
x = np.array([1, 2, 3])
print(np.exp(x))

# example of vector operation
y= np.array([1, 2, 3])
print (y + 3)

def vectorSigmoid(x):
    """
    Compute the sigmoid of x

    Arguments:
    x -- A scalar or numpy array of any size

    Return:
    s -- sigmoid(x)
    """

    ### START CODE HERE ### (≈ 1 line of code)
    s = 1/(1+np.exp(-x))
    ### END CODE HERE ###
    return s

x = np.array([1, 2, 3])
print(vectorSigmoid(x))

def sigmoid_derivative(x):
    """
    Compute the gradient (also called the slope or derivative) of the sigmoid function with respect to its input x.
    You can store the output of the sigmoid function into variables and then use it to calculate the gradient.

    Arguments:
    x -- A scalar or numpy array

    Return:
    ds -- Your computed gradient.
    """

    ### START CODE HERE ### (≈ 2 lines of code)
    s = vectorSigmoid(x)
    ds = s*(1-s)
    ### END CODE HERE ###
    print("sigmoid is ",s)
    print("sigmoid gradiet is",ds)
    return ds

sigmoid_derivative(x)

def image2vector(image):
    """
    Argument:
    image -- a numpy array of shape (length, height, depth)

    Returns:
    v -- a vector of shape (length*height*depth, 1)
    """

    ### START CODE HERE ### (≈ 1 line of code)
    v = image.reshape(image.shape[0]*image.shape[1]*image.shape[2], 1)
    ### END CODE HERE ###

    return v

# This is a 3 by 3 by 2 array, typically images will be (num_px_x, num_px_y,3) where 3 represents the RGB values
image = np.array(
                [
                  [
                    [ 0.67826139,  0.29380381],
                    [ 0.90714982,  0.52835647],
                    [ 0.4215251 ,  0.45017551]
                  ],

                  [
                    [ 0.92814219,  0.96677647],
                    [ 0.85304703,  0.52351845],
                    [ 0.19981397,  0.27417313]
                  ],

                  [
                    [ 0.60659855,  0.00533165],
                    [ 0.10820313,  0.49978937],
                    [ 0.34144279,  0.94630077]
                  ]
                ]
)

print ("image2vector(image) = " + str(image2vector(image)))
def normalizeRows(x):
    """
    Implement a function that normalizes each row of the matrix x (to have unit length).

    Argument:
    x -- A numpy matrix of shape (n, m)

    Returns:
    x -- The normalized (by row) numpy matrix. You are allowed to modify x.
    """

    ### START CODE HERE ### (≈ 2 lines of code)
    # Compute x_norm as the norm 2 of x. Use np.linalg.norm(..., ord = 2, axis = ..., keepdims = True)
    x_norm = np.linalg.norm(x, ord = 2, axis = 1, keepdims = True)

    # Divide x by its norm.
    print("x is \n",x,)
    x = x/x_norm
    ### END CODE HERE ###
    print("x_norm is \n",x_norm)
    print(x.shape, x_norm.shape)

    return x

randomArray = np.array(
                      [
                        [0, 3, 4],
                        [1, 6, 4]
                      ]
)

print("normalizeRows(randomArray) = " + str(normalizeRows(randomArray)))

def softmax(x):
    """Calculates the softmax for each row of the input x.

    Your code should work for a row vector and also for matrices of shape (m,n).

    Argument:
    x -- A numpy matrix of shape (m,n)

    Returns:
    s -- A numpy matrix equal to the softmax of x, of shape (m,n)
    """

    ### START CODE HERE ### (≈ 3 lines of code)
    # Apply exp() element-wise to x. Use np.exp(...).
    print("x is \n ",x)
    x_exp = np.exp(x)

    # Create a vector x_sum that sums each row of x_exp. Use np.sum(..., axis = 1, keepdims = True).
    print("x_exp \n ", x_exp)
    x_sum = np.sum(x_exp, axis=1, keepdims=True)
    print("x_sum is \n ",x_sum)
    # Compute softmax(x) by dividing x_exp by x_sum. It should automatically use numpy broadcasting.
    s = x_exp/x_sum

    ### END CODE HERE ###

    return s
x = np.array([
    [9, 2, 5, 0, 0],
    [7, 5, 0, 0 ,0]])
print("softmax(x) = " + str(softmax(x)))


from numpy import array, zeros, sqrt

def decomposition(A):
    N = len(A)
    L = zeros((N,N))    # L = [[0.0] * n for i in xrange(n)]
    U = zeros((N,N))    # U = [[0.0 for x in range(n)] for y in range(n)]

    for j in range(N):                                      # j = 0 to N-1

        # Upper Triangular
        U[j,j] = 1                                          # i = j, Diagonal 1

        for i in range(j):                                  # i = 0 to j-1

            if j == 0: break                                # No value for Ui0

            U[i,j] = (A[i,j] - sum(L[i,:i]*U[:i,j]))/L[i,i] # k = 0 to i-1

            # sum(U[k][j] * L[i][k] for k in xrange(i))

        # Lower Triangular
        for i in range(j, N):                                   # i = j to N-1

            L[i,j] = A[i,j] - sum(L[i,:j]*U[:j,j])              # k = 0 to j-1

            # sum(U[k][j] * L[i][k] for k in xrange(j))

    print('LU Decomposition:')
    print('[L] =\n', L, sep='')
    print('[U] =\n', U, sep='', end='\n\n')
    return L, U

def solveLU(A, B):
    L, U = decomposition(A)
    N = len(L)

    X = zeros(N)
    Y = zeros(N)

    # Forward Substitution
    for i in range(N):                                      # i = 0 to N-1
        Y[i] = (B[i] - sum(L[i,:i] * Y[:i])) / L[i,i]       # j = 0 to i-1

        # sumj = 0
        # for j in range(i):
        #     sumj += L[i,j] * Y[j]
        # Y[i] = (B[i] - sumj) / L[i,i]

    # Backward Substitution
    for i in range(N-1, -1, -1):                            # i = N-1 to 0
        X[i] = (Y[i] - sum(U[i,i+1:] * X[i+1:]))            # j = i+1 to N-1

        # sumj = 0
        # for j in range(i+1, N):
        #     sumj += U[i,j] * X[j]
        # X[i] = (Y[i] - sumj)

    return X

# System of Equations

A = array([[3,  -.1, -.2],
           [.1,  7,  -.3],
           [.3, -.2, 10]], float)
B = array([10.3, 33.6, 60.2], float)

N = len(A)

X = solveLU(A, B)

print("The Solution of the System:")
for i in range(N):
    print('X[', i+1, '] = ', round(X[i], 6), sep='')

'''
Decomposition: [A] = [L][U]   # [L][U] ≠ [U][L]    Not commutative

│a11 a12 a13 ... a1n│   │L11  0   0  ...  0 ││U11 U12 U13 ... U1n│
│a21 a22 a23 ... a2n│   │L21 L22  0  ...  0 ││ 0  U22 U23 ... U2n│
│a31 a32 a33 ... a3n│ = │L31 L32 L33 ...  0 ││ 0   0  U33 ... U3n│
│... ... ... ... ...│   │... ... ... ... ...││... ... ... ... ...│
│an1 an2 an3 ... ann│   │Ln1 Ln2 Ln3 ... Lnn││ 0   0   0  ... Unn│

For Doolittle: Lii = 1  All elements on main diagonal of [L] are 1

│a11 a12 a13 ... a1n│   │L11  0   0  ...  0 ││ 1  U12 U13 ... U1n│
│a21 a22 a23 ... a2n│   │L21 L22  0  ...  0 ││ 0   1  U23 ... U2n│
│a31 a32 a33 ... a3n│ = │L31 L32 L33 ...  0 ││ 0   0   1  ... U3n│
│... ... ... ... ...│   │... ... ... ... ...││... ... ... ... ...│
│an1 an2 an3 ... ann│   │Ln1 Ln2 Ln3 ... Lnn││ 0   0   0  ...  1 │

│a11 a12 a13 ... a1n│
│a21 a22 a23 ... a2n│
│a31 a32 a33 ... a3n│ =
│... ... ... ... ...│
│an1 an2 an3 ... ann│

│L11 L11U12     L11U13            ... L11U1n                      │
│L21 L21U12+L22 L21U13+L22U23     ... L21U1n+L22U2n               │
│L31 L31U12+L32 L31U13+L32U23+L33 ... L31U1n+L32U2n+L33U3n        │
│...      ...             ...     ...             ...             │
│Ln1 Ln1U12+Ln2 Ln1U13+Ln2U23+Ln3 ... Ln1U1n+Ln2U2n+Ln3U3n+...+Lnn│

 Li1 Li1U1j+Li2 Li1U1j+Li2U2j+Li3 ... Li1U1j+Li2U2j+Li3U3j+...+Lnj

 where i = 1 to n, j = 1 to n, k = 1 to j

L11 = a11   U12 = a12 / L11   U13 =  a13 / L11          ...
L21 = a21   L22 = a22-L21U12  U23 = (a23-L21U13)/L22    ...
L31 = a31   L32 = a32-L31U12  L33 =  a33-L31U13-L32U23  ...
Ln1 = an1   Ln2 = an2-Ln1U12  Ln3 =  an3-Ln1U13-Ln2U23  ...

Lij = aij   Lij = aij-Li1U1j  Lij =  aij-Li1U1j-Li2U2j
            Uij = aij / Lii   Uij = (aij-Li1U1j)/Lii

Ujj = 1                       j = 1 to n
Uij = (aij - ∑ Lik*Ukj)/Lii   i = 1 to j-1, k = 1 to i-1
Lij =  aij - ∑ Lik*Ukj        i = j to n,   k = 1 to j-1

No value for Ui1 is implimented by skipping Upper Triangular loop for j == 1
For U1j there is no ∑ and is implimented by k = 1 to 1-1, not entering k loop
For Li1 there is no ∑ and is implimented by k = 1 to 1-1, not entering k loop

Substitution: [A]{X}={B}    => [L][U]{X}={B}    => [U]{X}={y} and [L]{y}={B}

Forward Substitution: [L]{y}={B}

│L11  0   0  ...  0 ││y1│   │b1│
│L21 L22  0  ...  0 ││y2│   │b2│
│L31 L32 L33 ...  0 ││y3│ = │b3│
│... ... ... ... ...││……│   │……│
│Ln1 Ln2 Ln3 ... Lnn││yn│   │bn│

y1 =  b1 / L11
y2 = (b2 - L21y1) / L22
y3 = (b3 - L31y1 - L32y2) / L33
yn = (bn - Ln1y1 - Ln2y2 - ... - L[n,n-1]y[n-1]) / Lnn

yi = (bi - ∑ Lij*yj) / Lii , i = 1 to n, j = 1 to i-1

For y1 there is no ∑ and is implimented by j = 1 to 1-1, not entering j loop

Backward Substitution: [U]{X}={y}

│U11 U12 U13 ... U1n││x1│   │y1│
│ 0  U22 U23 ... U2n││x2│   │y2│
│ 0   0  U33 ... U3n││x3│ = │y3│
│... ... ... ... ...││……│   │……│
│ 0   0   0  ... Unn││xn│   │yn│

xn =  yn
x3 = (y3                   - U34*x4)
x2 = (y2          - U23*x3 - U24*x4)
x1 = (y1 - U12*x2 - U13*x3 - U14*x4)
xi = (yi - Uij*xj - Uij*xj - Uij*xj)

xi = (yi - ∑ Uij*xj)       , i = n to 1, j = i+1 to n

For xn there is no ∑ and is implimented by j = n+1 to n, not entering j loop

'''

5. Systems of Linear Equations/5. Crout's (Uii=1) Triangularization (Factorization or LU Decomposition) Method.py


#! /usr/bin/env python
# -*- coding: utf-8 -*-
"""Helper file containing activation functions
"""

import numpy as np

def sigmoid(x):
    """Description: Calculates the sigmoid for each value in the the input array
    Params:
        x: Array for which sigmoid is to be calculated

    Returns:
        ndarray: Sigmoid of the input
    """
    return 1.0 / (1.0 + np.exp(-x))

def delta_sigmoid(x):
    """Description: Calculates the sigmoid derivative for the input array
    Params:
        x: Array for which sigmoid derivative is to be calculated

    Returns:
        ndarray: Sigmoid derivative of the input
    """
    return sigmoid(x) * (1 - sigmoid(x))

def softmax(x):
    """Description: Calculates softmax for each set of scores in the input array

    Params:
        x: Array for which softmax is to be calculated
            (axis_0 is the feature dimension, axis_1 is the n_samples dim)

    Returns:
        ndarray: Softmax of the input
    """
    e_x = np.exp(x - np.max(x, axis=0))
    return e_x / e_x.sum(axis=0)

def relu(x):
    """Description: Calculates ReLU for each value in the input array

    Params:
        x: Array for which ReLU is to be calculated

    Returns:
        ndarray: ReLU of the input
    """
    return np.maximum(x, 0)

def delta_relu(x):
    """
	Description: Calculates the ReLU derivative for the input array

	Params:
			x: Array for which ReLU derivative is to be calculated

	Returns:
			ndarray: ReLU derivative of the input
	"""
    return np.greater(x, 0).astype(np.float32)

def linear(x):
    """
	Description: Calculates the linear activation for the input array

	Params:
			x: Array for which linear activation is to be calculated

	Returns:
			ndarray: Linear activation of the input
	"""
    return x

def delta_linear(x):
    """
	Description: Calculates the linear activation derivative for for the input array

	Params:
			x: Array for which linear activation derivative is to be calculated

	Returns:
			ndarray: Linear activation derivative of the input
	"""
    return np.ones(x.shape).astype(np.float32)

def activation_function(x, type="linear"):
    """
	Description: Helper function for calculating activation of the input

	Params:
			out: Array for which activation is to be calculated
			type: Type of the activation function
			(can be linear, sigmoid, relu, softmax)

	Returns:
			ndarray: Activation of the input
	"""
    if (type == "linear"):
        return linear(x)
    elif type == "sigmoid":
        return sigmoid(x)
    elif type == "relu":
        return relu(x)
    elif type == "softmax":
        return softmax(x)
    else:
        raise ValueError('Invalid activation type entered')

def activation_derivative(x, name="linear"):
    """Description: Helper function for calculating activation derivative of the input

	Params:
			out: Array for which activation derivative is to be calculated
			name: Type of the activation derivative function
			(can be linear, sigmoid, relu, softmax)

	Returns:
			ndarray: Activation derivative of the input
	"""
    if (name == "linear"):
        return delta_linear(x)
    elif (name == "sigmoid"):
        return delta_sigmoid(x)
    elif (name == "relu"):
        return delta_relu(x)
    else:
        raise ValueError('Invalid activation type entered')


import math
import random
from matplotlib import pyplot as plt
import numpy as np

class utils:
    def power_mod(self, x, y, p):
        res = 1
        x = x % p
        while y > 0:
            if y & 1:
                res = (res*x) % p
            y = y>>1
            x = (x*x) % p
        return res

    def miller_test(self, d, n) -> bool:
        for a in range(2, int(2*math.log(n)**2)):
            x = self.power_mod(a, d, n)
            if x == 1 or x == n-1:
                continue
            c = False
            while d != n-1:
                x = (x * x) % n
                d *= 2
                if x == n-1:
                    c = True
                    break
            if c:
                continue
            return False
        return True

    def is_prime(self, n) -> bool:
        if n <= 1 or n == 4:
            return False
        if n <= 3:
            return True
        d = n - 1
        while d % 2 == 0:
            d >>= 1
        return self.miller_test(d, n)

    def primes2n(n):
        """ Returns  a list of primes < n """
        sieve = [True] * n
        for i in range(3,int(n**0.5)+1,2):
            if sieve[i]:
                sieve[i*i::2*i]=[False]*((n-i*i-1)//(2*i)+1)
        return [2] + [i for i in range(3,n,2) if sieve[i]]

    def nprimes(n):
        return len(utils.primes2n(n))

    def nthprime(n):
        num = 1
        primes = [2]
        if n == 1:
            return 2
        while n > 1 :
            num += 2
            limit = int(math.sqrt(num))+1
            for i in primes:
                if i > limit:
                    primes.append(num)
                    n -= 1
                    break
                if num % i == 0 :
                    break

            if i == primes[-1]:
                primes.append(num)
                n -= 1
        return num

    def prime_factors(n):
        i = 2
        factors = []
        if n % i == 0:
            factors.append(2)
        while n % i == 0:
            n //= i
        i += 1
        while i <= n:
            if n % i == 0:
                factors.append(i)
                n = n // i
                i = 1
            i += 2
        return factors

    def nfactors(n):
        return len(utils.prime_factors(n))

    def nthroot(num, n):
        u, s = num, num+1
        while u < s:
            s = u
            t = (n - 1) * s + num // pow(s, n-1)
            u = t // n
        return s
        if float(s) == num ** (1/n):
            return s
        else:
            return -1

    def plot_graph(self, title, x, y, xlabel, ylabel):
        plt.plot(x,y)
        plt.xlabel(xlabel)
        plt.ylabel(ylabel)
        plt.title(title)
        plt.show()


import numpy as np
import matplotlib.pyplot as plt
plt.rcParams['figure.figsize'] = [16, 12]
plt.rcParams.update({'font.size' : 18})

# creating a simple signal with 2 frequencies
dt = 0.001
t = np.arange(0, 1, dt)
f = np.sin(2*np.pi*50*t) + np.sin(2*np.pi*120*t)        # sum of 2 frequencies
f_clean = f                                             # clean 2 tone signal
f = f + 2.5*np.random.randn(len(t))                     # random noise addition

plt.plot(t, f, color='c', LineWidth='1.5', label='Noisy')
plt.plot(t, f_clean, color='k', Linewidth='2', label='Clean')
plt.xlim(t[0], t[-1])
plt.legend()
plt.show()

""" Denoise the noisy signal and try to obtain the clean one """

n = len(t)
fhat = np.fft.fft(f, n)                             # compute fft = (signal, data points)
                                                    # complex valued(magnitude&phase) fourier coefficients vector
PSD = fhat * np.conj(fhat) / n                      # Power spectrum (power per frequency) Density
freq =(1/(dt*n)) * np.arange(n)                     # create x-axis of frequencies
L = np.arange(1, np.floor(n/2), dtype='int')        # only plot the first half of

fig, axs = plt.subplots(2, 1)

plt.sca(axs[0])
plt.plot(t, f, color='c', LineWidth=1.5, label='Noisy')
plt.plot(t, f_clean, color='k', LineWidth=2, label='Clean')
plt.xlim(t[0], t[-1])
plt.legend()

plt.sca(axs[1])
plt.plot(freq[L], PSD[L], color='c', LineWidth =2, label='Noisy')
plt.xlim(freq[L[0]], freq[L[-1]])
plt.ylabel('Power Spectrum')
plt.xlabel('Hertz')
plt.legend()

plt.show()

""" Even though the signal is noisy; The Power Spectrum has 2 super-clean peaks(1st at 50Hz. and 2nd at 120Hz.)
    that means most of the power in noisy signal is in 50Hz and 120Hz
    and then there is a bunch of noise in noise floor contributing to the jitter on the data(signal)

    We can filter out; any fourier coefficient that is smaller than 100 -- just zero it out
    any fourier coefficient larger than 100 -- keep it
    do Inverse Fourier Transform -- reconstruct denoised signal """
# Use PSD to filter out noise
indices = PSD > 100                 # frequences larger than 100 --- large vector with a lot of 0s two entries of 1
PSDclean = PSD * indices            # Zero out all others
fhat = indices * fhat               # Zero out small fourier coefficients in Y
ffilt = np.fft.ifft(fhat)

# Plots
fig, axs = plt.subplots(3, 1)

plt.sca(axs[0])
plt.plot(t, f, color='c', LineWidth=1.5, label='Noisy')
plt.plot(t, f_clean, color='k', LineWidth='2', label='clean')
plt.xlim(t[0], t[-1])
plt.legend()

plt.sca(axs[1])
plt.plot(t, ffilt, color='k', LineWidth=2, label='Filtered')
plt.xlim(t[0], t[-1])
plt.legend()

plt.sca(axs[2])
plt.plot(freq[L], PSD[L], color='c', LineWidth=2, label='Noisy')
plt.plot(freq[L], PSDclean[L], color='k', LineWidth=1.5, label='Filtered')
plt.xlim(freq[L[0]], freq[L[-1]])
plt.legend()

plt.show()


"""
Sum square difference
Problem 6
The sum of the squares of the first ten natural numbers is,

12 + 22 + ... + 102 = 385
The square of the sum of the first ten natural numbers is,

(1 + 2 + ... + 10)2 = 552 = 3025
Hence the difference between the sum of the squares of the first ten natural numbers and the square of the sum is 3025 − 385 = 2640.

Find the difference between the sum of the squares of the first one hundred natural numbers and the square of the sum.
"""

import numpy as np

def sum_of_squares(range_start, range_end):
    return sum(i**2 for i in range(range_start, range_end))

def square_of_sum(range_start, range_end):
    return sum(range(range_start, range_end)) ** 2

def compute_difference(range_start, range_end):
    a = sum_of_squares(range_start, range_end)
    b = square_of_sum(range_start, range_end)

    return np.abs(a - b)

print(compute_difference(1, 101))

"""
25164150
"""


import Utils as u
from numpy import ones, append, array
from pylab import inv, dot

def normal_equation(x,y):
    """
    Description:
        Computes the parameters based the training examples and values of the target variables using
        the closed form formula theta = (inverse(X.transpose()*X)*X.transpose())*Y where X.transpose()
        computes the transpose of matrix X and * implies matrix multiplication.
    Parameters:
        x - an array of feature vectors
        y - target variables corresponding to the feature vectors
    Returns:
        theta - an array of parameters corresponding to the feature vectors.
    """

    z = inv(dot(x.transpose(), x))
    theta = dot(dot(z, x.transpose()), y)
    return theta

def main():
    """
    Description:
        Driver function for the script. Gets the training file from the user, parses it and computes
        the parameters using the normal_equation function.
    Parameters:
        None
    Returns:
        None
    """

    training_file = raw_input("Enter the filename in which the training data is present:")
    n = int(raw_input("Enter the number of features:"))
    m = int(raw_input("Enter the number of training examples:"))
    (x,y) = u.parse_csv(training_file,m,n)
    x = append(ones([m,1]),x,1)
    theta = normal_equation(x,y)
    print "Parameters Learned from the training set:\n",theta
    print "Enter the new values of feature vector x for which the target value should be predicted"
    x_new = []
    for i in xrange(n):
        x_new.append(float(raw_input()))
    feature_vector = array(x_new)
    feature_vector = append(ones([1,1]),feature_vector)
    print "Predicted value of target variable y corresponding to Linear Regression algorithm =",u.predict(feature_vector,theta)

# Execute main() only when this script is executed from the command line
print __name__
if __name__ == "__main__":
    main()


"""
Multi-Parameter Bayesian Inference Using Markov Chain Monte Carlo (MCMC)
Sampling and the Metropolis-Hastings Algorithm.

Copyright (c) 2021 Gabriele Gilardi

Features
--------
- Code has been written and tested in Python 3.8.5.
- Likelihood (pdf) can be defined as an arbitrary function with any number
  of independent parameters.
- Prior functions are defined using a list of list, and can be any pdf from
  function "prior_dist" in file "Metropolis.py" (other priors can be easily
  added).
- Jumps in the Metropolis-Hastings algorithm are proposed using a normal
  distribution of the parameters.
- Function "random_number" in file "Metropolis.py" can be used to generate
  random numbers from any arbitrary pdf.
- Results can be verified using the pymc3 library.
- Usage: python test.py <example>.

Main Parameters
---------------
example = Random, Coin, Normal, Coin_upd
    Name of the example to run.
likelihood
    Name of the likelihood function.
par
    Array with the parameters of the likelihood function.
n_data >=1
    Number of data to be sampled from the likelihood function.
data
    Array with the data sampled from the likelihood function.
a0, b0
    Support interval for the likelihood function.
priors
    List with the priors. Each prior is assigned to one of the likelihood
    parameter following the same order as in <par>.
samples > 0
    Number of jumps to perform in the Metropolis-Hastings algorithm.
par_init
    Initial value for the parameters in the Metropolis-Hastings algorithm.
width_prop > 0
    Standard deviation of the normal distribution used to search the neighboroud
    of a parameter. A good value should give about 50% of accepted jumps.
i0 >= 0
    Index specifying the burn-in/warm-up amount.
posterior
    Array containing the jumps (accepted or rejected) of all parameters.
jumps
    Number of jumps actually accepted.

Examples
--------
There are four examples: Coin, Normal, Coin_upd, and Random (see the code for
parameters and results).

- Coin: one parameter (theta), Bernoulli distribution as likelihood, beta
        distribution as prior, admit an analytical solution.

- Normal: two parameters (mean and standard deviation), normal distribution
          as likelihood, normal distribution as prior for the mean, gamma
          distribution as prior for the standard deviation, solution also
          checked with pymc3.

- Coin_upd: one parameter (theta), Bernoulli distribution as likelihood,
            uniform distribution as initial prior, previous posterior as
            successive prior.

- Random: generation of random numbers from a generic pdf.

References
----------
- Metropolis-Hastings algorithm @
  https://en.wikipedia.org/wiki/Metropolis-Hastings_algorithm
- Markov chain Monte Carlo @
  https://en.wikipedia.org/wiki/Markov_chain_Monte_Carlo
- Halls-Moore, 2016, "Bayesian statistics", chapter II in "Advanced Algorithmic
  Trading", @ https://www.quantstart.com/advanced-algorithmic-trading-ebook/
- Probabilistic programming in Python using pymc3 @ https://docs.pymc.io/
"""
if __name__ == '__main__':

    import sys
    import warnings

    import numpy as np
    from scipy import stats
    import matplotlib.pyplot as plt
    import seaborn as sns

    from Metropolis import metropolis, random_number

    # To avoid the warning about log(0) when one of the probability is zero
    warnings.filterwarnings("ignore")

    # Read example to run
    if len(sys.argv) != 2:
        print("Usage: python test.py <example>")
        sys.exit(1)
    example = sys.argv[1]

    # Seed the random generator
    np.random.seed(123)

    # Coin flip example:
    # - one parameter (the probability tail comes up)
    # - Bernoulli distribution as likelihood
    # - beta distribution as prior
    # - admit an analytical solution
    if (example == 'Coin'):

        def pdf_coin(x, par):
            """
            Bernoulli distribution.
            """
            theta = par[0]
            pdf = np.where(x, theta, 1.0-theta)

            return pdf

        # Generate data (0 = tail, 1 = head)
        likelihood = pdf_coin
        n_data = 50                 # Number of coin flip
        par = [0.63]                # Probability tail comes up
        data = (np.random.uniform(0, 1, size=n_data) > par[0])

        # Priors (coefficients are just arbitrary values used as example)
        alpha, beta = 4.0, 12.0
        priors = []
        priors.append(['beta', alpha, beta])

        # Solve
        samples = 10000
        par_init = [0.5]
        width_prop = 0.1
        i0 = int(np.floor(0.2 * samples))       # Burn-in period
        posterior, jumps = metropolis(data, likelihood, priors, samples=samples,
                                      par_init=par_init, width_prop=width_prop)

        # Results:
        # - accepted jumps = 52.9%
        # - <theta> mean and std = 0.289, 0.057
        print("\nAccepted jumps = {0:.1f}%".format(100 * jumps / samples))
        print("<theta> mean and std = {0:.3f}, {1:.3f}"
              .format(posterior[i0:, 0].mean(), posterior[i0:, 0].std()))

        # Analytical solution
        # - Bernoulli likelihood with beta prior results in a beta posterior
        # - posterior (alpha, beta) are converted to equivalent (mu, sigma)
        #   for comparison purpose
        # - Results: <theta> mean and std (anal.) = 0.288, 0.055
        # - Ref.: https://en.wikipedia.org/wiki/Conjugate_prior
        n_heads = data.sum()
        alpha_post = alpha + n_heads
        beta_post = beta + n_data - n_heads
        a = alpha_post + beta_post
        mu_post = alpha_post / a
        sigma_post = np.sqrt(alpha_post * beta_post / (a + 1.0)) / a
        print("\n<theta> mean and std (anal.) = {0:.3f}, {1:.3f}"
              .format(mu_post, sigma_post))

        # Plot posteriors (numerical, analytical, and histogram)
        plt.subplot(121)
        sns.kdeplot(posterior[i0:, 0], label='num.', c='b')
        plt.hist(posterior[i0:, 0], 30, histtype="step", density=True,
                 color='r', label='hist.')
        xx = np.linspace(0.0, 0.6, 1000)
        yy = stats.beta(alpha_post, beta_post).pdf(xx)
        plt.plot(xx, yy, label='anal.', c='g')
        plt.xlabel('x')
        plt.xticks(np.linspace(0.0, 0.6, num=7))
        plt.xlim(0.0, 0.6)
        plt.ylabel('$\Theta$')
        plt.yticks(np.linspace(0, 8, num=9))
        plt.ylim(0, 8)
        plt.grid(b=True)
        plt.legend()

        # Plot all <theta> values (both accepted and rejected)
        plt.subplot(122)
        plt.plot(posterior[:, 0], c='b')
        plt.xlabel('sample')
        plt.xticks(np.linspace(0, samples, num=5))
        plt.xlim(0, samples)
        plt.ylabel('$\Theta$')
        plt.yticks(np.linspace(0.1, 0.5, num=5))
        plt.ylim(0.1, 0.5)
        plt.axhline(posterior[i0:, 0].mean(), color='r')
        plt.grid(b=True)

        plt.show()

    # Example with normally distributed likelihood:
    # - two parameters (mean and standard deviation)
    # - Normal distribution as likelihood
    # - Normal distribution as prior for the mean <mu >and gamma distribution
    #   as prior for the standard deviation <sigma>
    # - solution checked with pymc3
    elif (example == 'Normal'):

        def pdf_normal(x, par):
            """
            Normal distribution.
            """
            mu = par[0]
            sigma = par[1]
            y = (x - mu) / sigma
            pdf = np.exp(-y * y / 2.0) / (sigma * np.sqrt(2.0 * np.pi))
            return pdf

        # Generate data
        likelihood = pdf_normal
        n_data = 20
        par = [-1.3, 1.0]               # Mean and standard deviation
        a0, b0 = -10.0, +10.0           # Support
        data = random_number(likelihood, par, a0, b0, size=n_data)

        # Priors (coefficients are just arbitrary values used as example)
        priors = []
        priors.append(['norm', 2.0, 1.0])               # Mean
        priors.append(['gamma', 6.0, 1.0])              # Standard deviation

        # Solve
        samples = 20000
        par_init = [2.0, 5.0]
        width_prop = 0.20
        i0 = int(np.floor(0.2 * samples))       # Burn-in period
        posterior, jumps = metropolis(data, likelihood, priors, samples=samples,
                                      par_init=par_init, width_prop=width_prop)

        # Results:
        # - accepted jumps = 52.8%
        # - <mu> mean and std = -1.182, 0.230
        # - <sigma> mean and std = 0.968, 0.198
        print("\nAccepted jumps = {0:.1f}%".format(100 * jumps / samples))
        print("<mu> mean and std = {0:.3f}, {1:.3f}"
              .format(posterior[i0:, 0].mean(), posterior[i0:, 0].std()))
        print("<sigma> mean and std = {0:.3f}, {1:.3f}"
              .format(posterior[i0:, 1].mean(), posterior[i0:, 1].std()))

        # Solve using pymc3 (set to false if pymc3 not installed)
        use_pymc3 = True
        if (use_pymc3):

            print("\n===== Solving using pymc3 =====")

            import pymc3 as pm

            with pm.Model():

                # Priors
                mu = pm.Normal('mu', 2.0, 1.0)
                sigma = pm.Gamma('sigma', 6.0, 1.0)

                # Best starting point
                start = pm.find_MAP()

                # Likelihood
                returns = pm.Normal('returns', mu=mu, sd=sigma, observed=data)

                # Algorithm
                step = pm.Metropolis()

                # Solve
                trace = pm.sample(samples, step, return_inferencedata=False)

            # Results:
            # - start point =  {'mu': array(2.0), 'sigma': array(5.0)}
            # - <mu> mean and std = -1.188, 0.224
            # - <sigma> mean and std = 0.972, 0.202
            print("Start point = ", start)
            print("<mu> mean and std = {0:.3f}, {1:.3f}"
                  .format(trace[i0:]['mu'].mean(), trace[i0:]['mu'].std()))
            print("<sigma> mean and std = {0:.3f}, {1:.3f}"
                  .format(trace[i0:]['sigma'].mean(), trace[i0:]['sigma'].std()))

        # Plot <mu> posteriors (numerical, analytical, and histogram)
        plt.subplot(221)
        sns.kdeplot(posterior[i0:, 0], label='num.', c='b')
        plt.hist(posterior[i0:, 0], 50, histtype="step", density=True,
                 color='r', label='hist.')
        if (use_pymc3):
            sns.kdeplot(trace[i0:]['mu'], label='pymc3', c='g')
        plt.xlabel('x')
        plt.xticks(np.linspace(-2.50, 0, num=6))
        plt.xlim(-2.50, 0)
        plt.ylabel('$\mu$')
        plt.yticks(np.linspace(0, 2.5, num=6))
        plt.ylim(0, 2.5)
        plt.grid(b=True)
        plt.legend()

        # Plot <sigma> posteriors (numerical, analytical, and histogram)
        plt.subplot(222)
        sns.kdeplot(posterior[i0:, 1], label='num.', c='b')
        plt.hist(posterior[i0:, 1], 50, histtype="step", density=True,
                 color='r', label='hist.')
        if (use_pymc3):
            sns.kdeplot(trace[i0:]['sigma'], label='pymc3', c='g')
        plt.xlabel('x')
        plt.xticks(np.linspace(0.0, 2, num=5))
        plt.xlim(0.0, 2)
        plt.ylabel('$\sigma$')
        plt.yticks(np.linspace(0, 2.5, num=6))
        plt.ylim(0, 2.5)
        plt.grid(b=True)
        plt.legend()

        # Plot all <mu> values (both accepted and rejected)
        plt.subplot(223)
        plt.plot(posterior[:, 0], c='b')
        plt.xlabel('sample')
        plt.xticks(np.linspace(0, samples, num=5))
        plt.xlim(0, samples)
        plt.ylabel('$\mu$')
        plt.yticks(np.linspace(-2, 0, num=5))
        plt.ylim(-2, 0)
        plt.axhline(posterior[i0:, 0].mean(), color='r')
        plt.grid(b=True)

        # Plot all <sigma> values (both accepted and rejected)
        plt.subplot(224)
        plt.plot(posterior[:, 1], c='b')
        plt.xlabel('sample')
        plt.xticks(np.linspace(0, samples, num=5))
        plt.xlim(0, samples)
        plt.ylabel('$\sigma$')
        plt.yticks(np.linspace(0.0, 2.50, num=6))
        plt.ylim(0.0, 2.50)
        plt.axhline(posterior[i0:, 1].mean(), color='r')
        plt.grid(b=True)

        plt.show()

    # Coin flip example with updates:
    # - one parameter (the probability tail comes up)
    # - Bernoulli distribution as likelihood
    # - uniform distribution as initial prior
    # - previous posterior as successive prior
    elif (example == 'Coin_upd'):

        def pdf_coin_upd(x, par):
            """
            Bernoulli distribution.
            """
            theta = par[0]
            pdf = np.where(x, theta, 1.0-theta)

            return pdf

        # Parameters
        likelihood = pdf_coin_upd
        n_data = 50                     # Number of coin flip
        par = [0.63]                    # Probability tail comes up
        samples = 5000
        i0 = int(np.floor(0.2 * samples))       # Burn-in period
        mu = 0.5
        width_prop = 0.1

        # Initial prior is uniform
        XX = np.linspace(0.0, 1.0, 100)
        YY = np.ones(len(XX))
        plt.plot(XX, YY, label='init')

        # The mean of the posterior should tend to the probability that head
        # comes up, i.e. 0.37, while its standard deviation should become
        # smaller and smaller.
        #
        # Results (after 15 steps):
        # - head freq. = 36.4%
        # - accepted jumps = 24.0%
        # - <theta> mean and std = 0.362, 0.020
        tot_heads = 0
        tot_data = 0
        n_steps = 15
        for step in range(n_steps):

            # Generate data (0 = tail, 1 = head)
            data = (np.random.uniform(0, 1, size=n_data) > par[0])
            tot_heads += data.sum()
            tot_data += len(data)

            # Prior is expressed as generic array
            priors = []
            priors.append(['generic', XX, YY])

            # Solve using the previous mean as initial value
            par_init = [mu]
            posterior, jumps = metropolis(data, likelihood, priors,
                                          samples=samples, par_init=par_init,
                                          width_prop=width_prop)
            mu = posterior[i0:, 0].mean()

            # Print results for each step
            print("\nStep", step+1)
            print("- head freq. = {0:.1f}%".format(100 * tot_heads / tot_data))
            print("- accepted jumps = {0:.1f}%".format(100 * jumps / samples))
            print("- <theta> mean and std = {0:.3f}, {1:.3f}"
                  .format(mu, posterior[i0:, 0].std()))

            # Use the posterior as new prior (adding 10% tails on the side)
            x_min, x_max = np.min(posterior[:, 0]), np.max(posterior[:, 0])
            xx = np.linspace(x_min, x_max, 100)
            yy = stats.gaussian_kde(posterior[:, 0])(xx)
            d = x_max - x_min
            XX = np.concatenate([[xx[0] - 0.1 * d], xx, [xx[-1] + 0.1 * d]])
            YY = np.concatenate([[0], yy, [0]])
            if (((step+1) % 3) == 0):
                plt.plot(XX, YY, label='step ' + str(step+1))

        # Plot interpolated posteriors
        plt.xlabel('x')
        plt.xticks(np.linspace(0.2, 0.5, num=7))
        plt.xlim(0.2, 0.5)
        plt.ylabel('$\Theta$')
        plt.yticks(np.linspace(0, 24, num=7))
        plt.ylim(0, 24)
        plt.axvline(1.0 - par[0], c='k', ls='--')
        plt.grid(b=True)
        plt.legend()
        plt.show()

    # Generation of random numbers given the pdf
    elif (example == 'Random'):

        def pdf_random(x, par):
            """
            Piece-wise pdf.
            """
            pdf = np.where((x > 0.0) * (x <= 1.0), 0.3 * x, 0.0)
            pdf += np.where((x > 1.0) * (x <= 2.0), -0.2 * x + 0.5, 0.0)
            pdf += np.where((x > 2.0) * (x <= 3.0), 0.1, 0.0)
            pdf += np.where((x > 3.0) * (x <= 4.0), 0.1 * x - 0.2, 0.0)
            pdf += np.where((x > 4.0) * (x <= 5.0), 0.2, 0.0)
            pdf += np.where((x > 5.0) * (x <= 7.0), -0.1 * x + 0.7, 0.0)

            return pdf

        # Parameters
        par = []
        a0, b0 = -1.0, +8.0

        # Randomly approximated pdf
        n_data = 50000
        data = random_number(pdf_random, par, a0, b0, size=n_data, n=1000)

        # Real pdf
        xx = np.linspace(a0, b0, 1000)
        yy = pdf_random(xx, par)

        # Plot
        plt.plot(xx, yy, label='Real')
        plt.hist(data, 100, histtype="step", density=True, label='Random')
        plt.xlabel('x')
        plt.xticks(np.arange(-1, 9, step=1))
        plt.xlim(-1, 8)
        plt.ylabel('pdf')
        plt.yticks(np.arange(0, 0.4, step=0.05))
        plt.ylim(0, 0.35)
        plt.grid(b=True)
        plt.legend()
        plt.show()

    else:
        print("\n", example)
        print("--> Example not found.\n")
        sys.exit(1)


import numpy as np
import scipy.linalg.decomp_lu as dl

# Considera la funzione numpy.random.rand e genera una matrice di numeri random
# A, di dimensione 7x7.
n = 7
A = np.random.rand(n, n)

# {{1}} Quanto vale il numero di condizionamento (con norma 2) della matrice? La
# matrice è ben o mal condizionata?
print('K_A = ', np.linalg.cond(A))

# {{2}} Crea il problema test con soluzione esatta x_true = [1, 1, 1, 1, 1, 1,
# 1]^T. Riporta gli elementi del vettore b, termine noto del sistema lineare.
x_true = np.ones(n)
b = A @ x_true
print('b = ', b)

# {{3}} Utilizza le funzioni di scipy.linalg.decomp_lu e fattorizza A con lu(A).
# Risulta necessario permutare le righe di A? Da cosa si evince?
p, l, u = dl.lu(A)
print('Permutation?', not np.array_equal(p, np.eye(n)))

# {{4}} Riporta il valore della norma 'fro' della differenza tra A e la sua
# fattorizzazione calcolata. A cosa è dovuto questo errore?
print('||A - l @ u||_fro = ', np.linalg.norm(A - l @ u, 'fro'))

# {{5}} Usare le funzioni scipy.linalg.solve_triangular e/o scipy.linalg.solve
# per risolvere il sistema lineare sfruttando la fattorizzazione di A. Riporta
# la soluzione ottenuta.
my_x = dl.lu_solve(dl.lu_factor(A), b)
print('my_x = ', my_x)

# {{6}} Calcola la norma 2 della differenza fra la soluzione esatta e la
# soluzione calcolata.
print('||x - my_x||_2 = ', np.linalg.norm(x_true - my_x))


import numpy as np
import matplotlib.pyplot as plt
#   data
x = np.array([1.0, 1.0, 2.0, 3.3, 3.3, 4.0, 4.0, 4.0, 4.7, 5.0, 5.6,
    5.6, 5.6, 6.0, 6.0, 6.5, 6.92])
y = np.array([10.84, 9.30, 16.35, 22.88, 24.35, 24.56, 25.86, 29.46,
    24.59, 22.25, 25.90, 27.20, 25.61, 25.45, 26.56, 21.03, 21.46])

#   fit a linear model
A = np.ones([len(x), 2])
A[:,1] = x
beta, SSe, rank, s = np.linalg.lstsq(A,y)
xl = np.linspace(min(x),max(x),num=100)
Al = np.ones([len(xl), 2])
Al[:,1] = xl
yl = np.dot(Al,beta)
yhat = np.dot(A,beta)
#   plot
plt.figure()
plt.plot(x,y,'ok')
plt.plot(xl,yl,'-b')
plt.grid(True)
plt.xlabel('$x$')
plt.ylabel('$y$')
plt.show()
#plt.savefig('dataAndFit.png', ftype='png', dpi=300)

#   compute the sum of squares of pure error
level = np.array([1.0, 2.0, 3.3, 4.0, 4.7, 5.0, 5.6, 6.0, 6.5, 6.92])
levelIndex = [y[0:2], y[2], y[3:5], y[5:8], y[8], y[9],
    y[10:13], y[13:15], y[15], y[16]]
ybarLevels = []
for i in levelIndex:
    ybarLevels.append(np.mean(i))
SSpe = 0
for i, r in enumerate(levelIndex):
    SSpe += np.sum((r-ybarLevels[i])**2)

#   compute the sum of squares lack of fit
nl = len(level)
SSlof = 0
Alevel = np.ones([nl,2])
Alevel[:,1] = level
yhatLevel = np.dot(Alevel,beta)
for i, j in enumerate(ybarLevels):
    ni = np.size(levelIndex[i])
    SSlof+= ni*((j-yhatLevel[i])**2)

#   Statistical test for lack of fit
m = len(ybarLevels)
n = len(x)
p = len(beta)
F0 = (SSlof / (m-p)) / (SSpe / (n-m))

#   test for lack of fit
from scipy.stats import f
pValue = 1.0 - f.cdf(F0,m-p,n-m)
#   since pValue is very small we reject the case


"""
This example approximates pi numerically using a Monte Carlo estimator:
I = (4/N) SUM_{i=1}^{N} h(x^(i), y^(i)) where h() is the indicator
function h(x,y) = {1 if x^2+y^2 <= 1 and 0 otherwise} and the tupels
(x^(i), y^(i)) are drawn from the uniform distribution U([0,1]x[0,1])
"""
import numpy as np
import matplotlib.pyplot as plt
from matplotlib import rc
import math
from math import log10, floor
from matplotlib.ticker import MaxNLocator

rc('font', **{'size':12, 'family':'serif', 'serif':['Computer Modern Roman']})
rc('text', usetex=True)

# Parameters for simulation
NUM_RUNS = 10
NUM_DRAW_STEPS = 5

STEP_MULTIPLIER = 10
STEP_START = 100

PRECISION = 6

PATH_FIG = 'pi_monte_carlo.pdf'
PATH_DATA = 'pi_monte_carlo.txt'

def round_sig(x, sig):
	return round(x, sig-int(floor(log10(abs(x))))-1)

def indicator_function(x, y):
	if (np.power(x, 2) + np.power(y, 2)) <= 1:
		return 1
	else:
		return 0

def approximation(num_draws):
	"""
	Approximation of pi with (num_draws) number of draws
	"""
	samples = np.random.uniform(-1, 1, (num_draws, 2))

	draws_in_circle = 0

	for i in range(0, num_draws):
		draws_in_circle += indicator_function(samples[i,0], samples[i,1])

	return 4*(draws_in_circle/num_draws)

def simulation():
	"""
	The approximation is run (NUM_RUNS)x(NUM_DRAW_STEPS)
	"""
	pi_approximations = np.zeros((NUM_DRAW_STEPS, NUM_RUNS))

	num_draws = STEP_START

	for i in range(0, NUM_DRAW_STEPS):
		for j in range(0, NUM_RUNS):
			pi_approximations[i,j] = approximation(num_draws)

		num_draws *= STEP_MULTIPLIER

	return pi_approximations

def visualization(pi_approximations):
	"""
	Plotting the expected values of the simulation
	"""
	fig, ax = plt.subplots()

	labels = [r'$10^2$', r'$10^3$', r'$10^4$', r'$10^5$', r'$10^6$']
	x = [0, 1, 2, 3, 4]

	ax.plot([-0.5, NUM_RUNS], [math.pi, math.pi], color='k', linestyle='-', linewidth=1)
	ax.plot(pi_approximations,'kx',markersize=2)

	plt.xlabel(r'Number of draws')
	plt.ylabel(r'Expected value $I(h)$')
	plt.xlim(-0.5, NUM_DRAW_STEPS-0.5)
	ax.xaxis.set_major_locator(MaxNLocator(integer=True))
	plt.xticks(x, labels)

	plt.savefig(PATH_FIG, bbox_inches='tight')
	plt.show()

def data(pi_approximations):
	"""
	Saving the data of the expected values
	"""
	data_file = open(PATH_DATA, 'w')

	for i in range(0, NUM_DRAW_STEPS):
		expecation_sum = 0

		for j in range(0, NUM_RUNS):
			roundend_expactation = round_sig(pi_approximations[i, j], PRECISION)
			expecation_sum += roundend_expactation

			data_file.write(str(roundend_expactation) + ' ')

		mean = round_sig((expecation_sum/NUM_RUNS), PRECISION)

		data_file.write('\t' + str(mean))
		data_file.write('\n')

def main():
	approximation = simulation()
	visualization(approximation)
	data(approximation)

if __name__ == "__main__":
	main()


import numpy as np
import matplotlib.pyplot as plt
from scipy.interpolate import interp1d
from scipy.integrate import simps

xs = [0.0,0.5,1.0,1.5,2.0,2.5,3.0,3.5,4.0,4.5,5.0,
    5.5,6.0,6.5,7.0,7.5,8.0,8.5,9.0,9.5,10.0]
ys = [0.2,0.165,0.167,0.166,0.154,0.134,0.117,0.108,0.092,0.06,0.031,0.028,
    0.048,0.077,0.103,0.119,0.119,0.103,0.074,0.038,0.003]

plt.scatter(xs,ys)
plt.xlabel("x")
plt.ylabel("Observed PDF")
plt.show()
#This is a continuous distribution

#span x from min to max with these data points
#interpolating the data
x = np.linspace(min(xs),max(xs),1000)
y1 = interp1d(xs,ys)(x)
#interpolating xs and ys to x line

plt.scatter(xs,ys,s=30,label="Data",c="b")
plt.plot(x,y1,label="Linear(default)")
plt.legend()
plt.show()
#linear interpolating in our data
#but if we have sparse data then it is not shown correctly
#depends on data

#another type of interpolation - nearest
x = np.linspace(min(xs),max(xs),1000)
y2 = interp1d(xs,ys,kind="nearest")(x)
#works as a step function in the graph shown below
#useful for discrete function but not very useful for
#smooth changing values

plt.scatter(xs,ys,s=30,label="Data",c="b")
plt.plot(x,y1,label="Linear(default)")
plt.plot(x,y2,label="Nearest")
plt.legend()
plt.show()

#another type of interpolation - quadratic
x = np.linspace(min(xs),max(xs),1000)
y3 = interp1d(xs,ys,kind="quadratic")(x)
#works as a step function in the graph shown below
#useful for discrete function but not very useful for
#smooth changing values

plt.scatter(xs,ys,s=30,label="Data",c="b")
plt.plot(x,y1,label="Linear(default)")
plt.plot(x,y2,label="Nearest",alpha=0.3)
plt.plot(x,y3,label="Quadratic",ls="-")
plt.legend()
plt.show()
#on the changing gradient you can see the difference
#between the nearest and quadratic

#another type of interpolation - cubic
x = np.linspace(min(xs),max(xs),1000)
y4 = interp1d(xs,ys,kind="cubic")(x)

plt.scatter(xs,ys,s=30,label="Data",c="b")
plt.plot(x,y1,label="Linear(default)")
plt.plot(x,y2,label="Nearest",alpha=0.3)
plt.plot(x,y3,label="Quadratic",ls="-")
plt.plot(x,y4,label="Cubic",ls="-")
plt.legend()
plt.show()
#the more complex interpolation,the slower
#it's(difference in quadratic and cubic) going to be

from scipy.interpolate import splev,splrep

#another type of interpolation - splev
x = np.linspace(min(xs),max(xs),1000)
y5 = splev(x,splrep(xs,ys))
#if data changing quickly, using cubic spline

plt.scatter(xs,ys,s=30,label="Data",c="b")
plt.plot(x,y1,label="Linear(default)")
plt.plot(x,y2,label="Nearest",alpha=0.3)
plt.plot(x,y3,label="Quadratic",ls="-")
plt.plot(x,y4,label="Cubic",ls="-")
plt.plot(x,y5,label="Spline",ls="-",alpha=0.5,c="#0000")
plt.legend()
plt.show()

#the red changes colour as both quadratic and spline are same

#Using the interp1d we can find the probability for any x value
#Using scipy.integrate we can calculate the CDF and probability in two bounds

#scipy.integrate.trapz = low accuracy but high speed - accuracy scales as O(h)
#scipy.integrate.simps = med accuracy but very high speed - acc scales as O(h^2)
#scipy.integrate.quad = high accuracy but low speed - arbitary accuracy

def get_prob(xs,ys,a,b): #add another variable here resolution=1000
  #a,b = bounds
  #to normalize, use the below
  x_norm = np.linspace(min(xs),max(xs),1000) #add resolution here instead of no
  y_norm = interp1d(xs,ys,kind="quadratic")(x_norm)
  normalisation = simps(y_norm,x=x_norm)
  x_vals = np.linspace(a,b,1000)#add resolution here instead of number
  y_vals = interp1d(xs,ys,kind="quadratic")(x_vals) #general solution
  #could be tweaked to gain more accuracy
  return simps(y_vals,x=x_vals)/normalisation

def get_cdf(xs,ys,v):#value of cdf at
  return get_prob(xs,ys,min(xs),v) #if xs is np array -> xs.min(),upto value v

def get_sf(xs,ys,v):
  return 1 - get_cdf(xs,ys,v)#definition of the survival function
  #OR return get_prob(xs,ys,v,max(xs))

print(get_prob(xs,ys,0,10)) #simps has some issue & needs to be normalized,
#check the normalized code above and check without normalized
#another problem is of numbers added to np.linspace,try changing
#x_vals = np.linspace(a,b,100), to solve this
#add a variable to it, use of resolution can solve the problem

v1,v2 = 6,9.3
area = get_prob(xs,ys,v1,v2)

plt.scatter(xs,ys,s=30,label="Data",color="w")
plt.plot(x,y3,linestyle="-",label="Interpolation")
plt.fill_between(x,0,y3,where=(x>=v1)&(x<=v2),alpha=0.2) #fill in the values
plt.annotate(f"p = {area}.3f",(7,0.05))
plt.legend()
plt.show()

#comparing cdf
x_new = np.linspace(min(xs),max(xs),100)
cdf_new = [get_cdf(xs,ys,i) for i in x_new] #expensive way of cdf_new

cheap_cdf =y3.cumsum()/y3.sum()
plt.plot(x_new,cdf_new,label="Interpolated CDF",c="r")
plt.plot(x,cheap_cdf,label="Super cheap CDF for specific cases",c="g")
plt.ylabel("CDF")
plt.xlabel("x")
plt.legend()
plt.show()
#for smooth data you can use the cheap cdf method
#you can see the interpolated and the cheap cdf coming together


#!/usr/bin/env python
"""Program to multiply 2  matrices together. (nxm)*(mxl)"""

import numpy as np
import timeit

speed_test = True

def check_shape(A,B):
	"""Checking if it's possible to multiply these matrices together"""
	shape_A = np.shape(A)
	shape_B = np.shape(B)
	if shape_A[1] != shape_B[0]:
		raise ValueError("shapes {0} and {1} not aligned".format(shape_A,shape_B))

def matrix_mult_nested(A,B):
	"""Matrix multiplication using nested for loops"""
	check_shape(A,B)
	rows = np.shape(A)[0]
	cols = np.shape(B)[1]
	C = np.zeros((rows,cols)) #resultant matrix after multiplication
	for i in range(rows):
		for j in range(cols): #loops give [i,j] location in matrix
			for k in range(cols): #each [i,j] location is a dot-product of a row of X and a column of Y
				C[i][j] += A[i][k]*B[k][j]
	return C

def matrix_mult_list_comp(A,B):
	"""Matrix multiplication using list comprehension"""
	check_shape(A,B)
	rows = np.shape(A)[0]
	cols = np.shape(B)[1]
	C = [[np.sum([A[i][k]*B[k][j] for k in range(cols)]) for j in range(cols)] for i in range(rows)]
	return C

def matrix_mult_numpy(A,B):
	"""Matrix multiplication using numpy dot function"""
	C = np.dot(A,B)
	return C

def time_elasped(function):
	start = timer()
	function
	end = timer()
	return end-start

#Example matrices
X = np.array([[1,2],[3,4],[5,6]])
Y = np.array([[5,6],[7,8]])

print("Example Matrices")
print("X * Y = \n{0} * \n{1}".format(X,Y))
print("Nested for loop:\n", matrix_mult_nested(X,Y))
print("List comprehension:\n", matrix_mult_list_comp(X,Y))
print("Numpy dot function:\n", matrix_mult_numpy(X,Y))

if speed_test: #speed test is optional
	#Large Matrix
	np.random.seed(1)
	large_X =  np.random.rand(20,20)
	large_X_inv = np.linalg.inv(large_X)
	print("\nSpeed test")

	#I only made these function because they don't have any arguments so I can use timeit on them easily
	def func1():
		return matrix_mult_nested(large_X,large_X_inv)
	def func2():
		return matrix_mult_list_comp(X,Y)
	def func3():
		return matrix_mult_numpy(X,Y)

	t_nested = np.min(timeit.repeat("func1()", setup="from __main__ import func1",repeat=5,number= 100))/100
	t_list_comp = np.min(timeit.repeat("func2()", setup="from __main__ import func2",repeat=5,number= 100))/100
	t_numpy = np.min(timeit.repeat("func3()", setup="from __main__ import func3",repeat=5,number= 100))/100
	print("Nested for loop:")
	print("{0:.3e} seconds".format(t_nested))
	print("List comprehension:")
	print("{0:.3e} seconds".format(t_list_comp))
	print("Numpy dot function:")
	print("{0:.3e} seconds".format(t_numpy))

#Code graveyard
	# print(timeit.repeat("func2()", setup="from __main__ import func2"))
	# print(timeit.repeat("func3()", setup="from __main__ import func3")
	# t_nested = time_elasped(matrix_mult_nested(large_X,large_X_inv))
	# t_list_comp = time_elasped(matrix_mult_list_comp(large_X,large_X_inv))
	# t_numpy = time_elasped(matrix_mult_numpy(large_X,large_X_inv))
	# print("Nested for loop:\n", t_nested*1e7)
	# #np.round(matrix_mult_nested(large_X,large_X_inv)))
	# print("List comprehension:\n", t_list_comp*1e7)
	# #np.round(matrix_mult_list_comp(large_X,large_X_inv)))
	# print("Numpy dot function:\n", t_numpy*1e7)
	# #np.round(matrix_mult_numpy(large_X,large_X_inv)))


import numpy

def cos2degree(cos) -> float:
    """
    Convert cosine to degree. Support float, List[float], numpy.ndarray[float]
    :param cos:
    :return:
    """
    return numpy.arccos(cos) / numpy.pi * 180

def cos_angle(left_side: float, right_side: float, third_side: float) -> float:
    """
    Calculate cos of angle if give 3 side of triangle
    """
    return (left_side ** 2 + right_side ** 2 - third_side ** 2) / (2 * left_side * right_side)

def degree_angle(left_side: float, right_side: float, third_side: float) -> float:
    """
    Calculate degree of angle if give 3 side of triangle
    """
    return cos2degree(cos_angle(left_side, right_side, third_side))

def cos_triangle(first_side, second_side, third_side):
    """
    Calculate 3 angles's cos of triangle
    """
    first_angle = cos_angle(second_side, third_side, first_side)
    second_angle = cos_angle(first_side, third_side, second_side)
    third_angle = cos_angle(first_side, second_side, third_side)
    return first_angle, second_angle, third_angle

def degree_triangle(first_side, second_side, third_side):
    return cos2degree(cos_triangle(first_side, second_side, third_side))