#! bin/usr/env python

import numpy as np

def make_diff_matrix(h,N):
    """This function takes in a value h which is the distance between 2
    sample points and the amount of points it would sample
    N. It will then use these parameters to create an NxN matrix that is
    used to creat N central difference
    equations to get the derivative of a function."""
    D = np.zeros((N+1,N+1))
    D[0][0] = -1.0/h
    D[0][1] = 1.0/h
    D[N][N-1] = -1.0/h
    D[N][N] = 1.0/h
    for i in range(1,N):
        for j in range(1,N+1):
            if (j==i-1):
                D[i][j] = -1.0/(2.0*h)
            elif (j==i+1):
                D[i][j] = 1/(2.0*h)
    return D

def vec_central_first_diff(f,a,b,N):
    """Takes a function and an array of equidistant inputs (all discrete)
    and returns the derivative. Achieves this using a derivative matrix.
    Takes args: function f, N sample intervals, [a,b]"""
    x_i = np.linspace(a,b,N+1)
    f_x = f(x_i)
    l = vec_central_first_diff1(x_i, f_x)
    return l

def vec_central_first_diff1(x_i, f_x):
    """This function takes in a list of points x_i and the function that is
     produced by a function acting on x_i.
    Then it takes the difference matrix defined above and applies it to the
    function to get the derivative points of the
    function."""
    N = len(x_i) - 1
    h = x_i[1]-x_i[0]
    D = make_diff_matrix(h,N)
    d1 = np.dot(D,f_x)
    return x_i, f_x, d1

def vec_central_second_diff1(x_i,f_x):
    """This function does almost the same thing as vec_central_first_diff1,
    except this is to get the second derivative.
    So the only difference here is that we multiply the difference matrix
    to itself and then apply it to the function."""
    N = len(x_i) - 1
    h = x_i[1] - x_i[0]
    D = np.dot(make_diff_matrix(h,N), make_diff_matrix(h,N))
    d1 = np.dot(D,f_x)
    return x_i, f_x, d1

def vec_central_second_diff(f, a, b, N):
    """This is the application function for the second derivative function.
    Takes in a mathematical function an interval
    [a,b] and number of sample points N, and returns the second derivative."""
    x_i = np.linspace(a,b,N+1)
    f_x = f(x_i)
    l = vec_central_second_diff1(x_i, f_x)
    return l

def vec_central_diff(f,a,b,N,order=1):
    """Application function for vcd. Takes in function f, interval [a,b]
    and sample points N and the order of the derivatives
    (default to 1)."""
    x_i = np.linspace(a,b,N+1)
    f_x = f(x_i)
    l = vcd(x_i,f_x,order)
    return l

def vcd(x_i,f_x,order):
    """Takes in list of points x_i, function points f_x and a derivative
    order. Will iterate 'order' amount of times, multiplying the difference
    matrix to itself every time and then applying D^(order) to the function
    points f_x."""
    N = len(x_i) - 1
    h = x_i[1] - x_i[0]
    D = make_diff_matrix(h,N)
    if order >= 2:
        for i in range(1,order):
            D = np.dot(make_diff_matrix(h,N),D)
    d1 = np.dot(D, f_x)
    return x_i, f_x, d1

def make_trap_matrix(h,N):
    """Takes in step size h and the number of partition points used for integration N.
    Creates an NxN integration matrix which is a lower triangular matrix. With main
    diagonal and first column equal to h/2 and the rest of the lower triangle equal to h.
    """
    I = np.zeros((N+1,N+1))
    for i in range(1,N+1):
        for j in range(0,N+1):
            if j==i:
            	I[i][j] = float(h)/2.0
            elif j==i-1:
                I[i][j] = float(h)/2.0
            I[i][j] += I[i-1][j]
    return I

def vec_trapz(f,a,b,N):
    """Parameters: Function, interval [a,b] and sample points N. Returns
    integrated function"""
    x_i = np.linspace(a,b,N+1)
    f_x = f(x_i)
    l = vt(x_i,f_x)
    return l

def vt(x_i,f_x):
    """Takes in an array of points and a vectorized function. Creates the trapezoidal matrix and acts it onto the
    vectorized function. Returns the input parameters and the integral"""
    N = len(x_i) - 1
    h = x_i[1] - x_i[0]
    I = make_trap_matrix(h,N)
    integral = np.dot(I,f_x)
    return x_i, f_x, integral

def test_vectrapz():
    """ testing the vect_trapz function with a moderately
    high error tolerance"""
    #On the interval from 0 to 10 the integral of x^3 = 2500
    xCubedInt = 2500
    a = vec_trapz(lambda x: x**3, 0,10,100)
    l = a[2]
    if(abs(l[100] - 2500) < 1):
        return True
    else:
        return False


"""Example implementation of the steepest descent algorithm using a hard coded obbjective function

    F(X1, X2) = 5X1**2 + X2**2 + 4X1X2 - 14X1 - 6X2 + 20

- At each step the value of X will be updated using
    X_k+1 = X_k + alpha * del_F(Xk)

- Initial guess of the minimiser is a point X_0 = [0, 0]
- The value of alpha will be calculated using an exact line search

    alpha = || gk || ** 2 / gk_T * A * gk

    where gk is the gradient vector at point Xk and A is the hessian matrix
"""
import math
import logging
import numpy as np

logging.basicConfig(level=logging.DEBUG)
log = logging.getLogger(__name__)

class GradientDescent:
    def __init__(self, max_iterations=100, hessian_matrix=None, linear_terms=None):
        self.Hessian: np.ndarray = hessian_matrix or np.array([[10, 4], [4, 2]])
        self.LinearCoefficients = linear_terms or np.array([-14, -6])
        self.Minimiser = np.array([0, 0])
        self.CurrentIteration = 0
        self.MaxIterations = max_iterations
        self.Epsilon = math.pow(10, -6)  # Stop if magnitude of gradient is less than this value

    @property
    def __current_gradient_value(self):
        gradient = self.del_f(self.Minimiser)
        return np.sqrt(gradient.dot(gradient))

    @staticmethod
    def f(xk, decimal_places=3):
        x1 = xk[0]
        x2 = xk[1]
        fn_value = 5*math.pow(x1, 2) + math.pow(x2, 2) + 4*x1*x2 - 14*x1 - 6*x2 + 20
        return round(fn_value, decimal_places)

    def del_f(self, xk: np.ndarray) -> np.ndarray:
        gradient_vector = np.matmul(self.Hessian, xk)
        if isinstance(self.LinearCoefficients, np.ndarray):
            gradient_vector = np.add(gradient_vector, self.LinearCoefficients)
        return gradient_vector

    def exact_step_length(self, xk: np.ndarray, decimal_places=3) -> np.float64:
        """Get the step lengh by minimising f(x + ad_k) wrt to a

        alpha(or lambda) = || gk || ** 2 / gk_T * A * gk
        """
        gk = self.del_f(xk)
        numerator = gk.dot(gk)
        denominator = gk.dot(self.Hessian).dot(gk)
        step_len = numerator/denominator
        return round(step_len, decimal_places)

    def _find_next_candidate(self):
        """Get next potential minimum using X_k+1 = X_k + alpha * del_F(Xk)"""
        xk = self.Minimiser
        dk = -1 * self.del_f(xk)
        step_length = self.exact_step_length(xk)
        x_k_plus1 = xk + (step_length * dk)
        log.debug(f"X**{self.CurrentIteration} = {xk} - {step_length} * {dk}")
        return x_k_plus1

    def execute(self):
        log.info('Gradient descent iteration started')
        # self.__preview_config()
        for k in range(self.MaxIterations):
            log.debug(f"-----------Iteration {k}--------------")
            self.CurrentIteration = k
            self.Minimiser = self._find_next_candidate()
            log.debug(f"------Minimiser = {self.Minimiser}. Gradient = {self.__current_gradient_value}--------\n")
            if self.__current_gradient_value <= self.Epsilon:
                log.warning(f"Iteration stopped at k={k}. Stopping condition reached!")
                break
        log.info(f"------Minimiser = {self.Minimiser}. Gradient = {self.__current_gradient_value}--------")
        log.info('Gradient descent completed successfully')
        return self.Minimiser, self.f(self.Minimiser), round(self.__current_gradient_value, 3)


import numpy as np
import matplotlib.pyplot as plt
import cvxpy as cp

S = np.loadtxt("S.txt")
pMean = np.loadtxt("pbar.txt")
n = pMean.shape[0]

#%%
# What is the risk of the uniform portfolio?

xU = np.ones(n)/n
riskU = xU.T @ S @ xU

print(f"uniform portfolio risk: {riskU**0.5}")

#%%
# What is the risk of an optimal portfolio with no (additional) constraints?
print("min risk with expected return equals uniform portfolio")

x = cp.Variable(n)
constraints = \
[
    cp.sum(x) == 1,
    pMean.T @ x == pMean.T @ xU
]

prob = cp.Problem(cp.Minimize(cp.quad_form(x, S)), constraints)
prob.solve()
assert prob.status == cp.OPTIMAL
print(f"optimal risk {prob.value**0.5}")

#%%
# What is the risk of a long-only portfolio 𝑥⪰0?
print("x >= 0")
prob = cp.Problem(cp.Minimize(cp.quad_form(x, S)), constraints + [x >= 0])
prob.solve()
assert prob.status == cp.OPTIMAL
print(f"optimal risk {prob.value**0.5}")

#%%
# What is the risk of a portfolio with a limit on total short position:  1𝑇(𝑥−)≤0.5 , where  (𝑥−)𝑖=max{−𝑥𝑖,0} ?
print("limit on short")
prob = cp.Problem(cp.Minimize(cp.quad_form(x, S)), constraints + [cp.sum(cp.neg(x)) <= 0.5])
prob.solve()
assert prob.status == cp.OPTIMAL
print(f"optimal risk {prob.value**0.5}")

Convex Optimization Problems/Simple Portfolio Optimization/solve.py


# -*- coding: utf-8 -*-
"""
Created on Fri Aug 23 19:07:15 2019

@author: marcus.bonifacio
"""

import numpy as np

print('1) Faça um script para estimar o valor de π utilizando o método de Monte Carlo.')

N = 10000000 #define quantidade de pontos
x = np.random.rand(N) #gera números aleatórios de 0 a 1
y = np.random.rand(N)
deltax = (0.5-x)**2 #calcula a diferença das coordenadas do ponto para o centro do círculo
deltay = (0.5-y)**2
d = np.sqrt(deltax + deltay) #calcula a distancia do ponto para o centro do círculo

circ = d[d <= 0.5] #seleciona os pontos dentro do círculo

razao = len(circ)/N #razão dos pontos dentro do círculo e todos os pontos
pi = 4*razao #aproxima pi
print('Valor aproximado de PI pelo método Monte Carlo =',pi)


import cvxpy as cp
import numpy as np
from numpy.linalg import pinv, inv, norm
from scipy.linalg import eig

def alternative_minimization(X, alpha, beta, num_iters, dual=False):

    n = X.shape[0]

    """
    helper functions
    """

    def dual_helper(Y):
        m = n * (n + 1) // 2

        # position matrix
        Q = np.zeros((n, n))
        count = 0
        for d in np.arange(0, n):
            for i in np.arange(d, n):
                j = i - d
                Q[i, j] = count
                count += 1
        for d in np.arange(1, n):
            for i in np.arange(0, n - d):
                j = i + d
                Q[i, j] = count
                count += 1

        # C = vec(Y @ Y.T)
        YYt = Y @ Y.T
        C = np.zeros(((n**2), 1))
        for i in np.arange(0, n):
            for j in np.arange(0, n):
                loc = int(Q[i, j])
                C[loc] = YYt[i, j]

        # Duplication matrix M
        nsqr = n ** 2
        M1 = np.eye(m)
        M2 = np.hstack((np.zeros((nsqr - m, n)), np.eye(m - n)))
        M = np.vstack((M1, M2))

        # Equality constraints matrix A
        A1 = np.hstack((np.ones((1, n)), np.zeros((1, m - n))))
        A2 = np.hstack((np.eye(n), np.zeros((n, m - n))))
        for i in np.arange(0, n):
            row = i
            for j in np.arange(0, i):
                col = int(Q[i, j])
                A2[row, col] = 1
            for j in np.arange(i + 1, n):
                col = int(Q[j, i])
                A2[row, col] = 1
        A = np.vstack((A1, A2))

        # vector b
        b = np.vstack((n * np.ones((1, 1)), np.zeros((n, 1))))

        # Inequality constraints matrix G
        G = np.hstack((np.zeros((m - n, n)), np.eye(m - n)))

        # vector h
        h = np.zeros((m - n, 1))

        P = 2 * beta * M.T @ M
        q = (alpha * M.T @ C).squeeze()

        return Q, P, q, G, h, A, b

    def make_L(x, Q):
        L = np.zeros((n, n))
        for i in np.arange(0, n):
            for j in np.arange(0, i + 1):
                loc = int(Q[i, j])
                L[i, j] = x[loc]

        L = L + L.T - np.diag(np.diag(L))
        return L

    """
    QP optimizations for L and Y
    """

    def optimize_L_primal(Y):
        L = cp.Variable((n,n), symmetric=True)

        obj = cp.Minimize(alpha * cp.trace(Y.T @ L @ Y) + beta * (cp.norm(L, p='fro')**2))

        constraints = [cp.trace(L) == n]
        constraints += [L >> 0]
        constraints += [L @ np.ones((n)) == np.zeros((n))]
        for i in range(n):
            for j in range(i+1,n):
                constraints += [L[i,j] <= 0]

        prob = cp.Problem(obj, constraints)

        p_star = prob.solve()
        print(f"primal value is {p_star}")
        L_opt = L.value
        return L_opt

    def optimize_L_dual(Y):
        Q, P, q, G, h, A, b = dual_helper(Y)

        lam = cp.Variable((G.shape[0]))
        nu = cp.Variable((A.shape[0]))

        Pinv = pinv(P)
        r = q + G.T @ lam + A.T @ nu

        obj_dual = cp.Maximize((-0.5) * cp.quad_form(r, Pinv) - h.T @ lam - b.T @ nu)

        constraints_dual = [lam >= 0]

        prob_dual = cp.Problem(obj_dual, constraints_dual)

        p_dual_star = prob_dual.solve()
        print(f"dual value is {p_dual_star}")
        lam_opt, nu_opt = lam.value, nu.value

        x = - Pinv @ (r.value)
        L_opt = make_L(x, Q)
        return L_opt

    def optimize_Y(L):
        Y_opt = inv(np.eye(n) + alpha * L) @ X

        return Y_opt

    """
    alternate minimization algorithm
    """

    Y_iter = X
    obj_old_val = np.inf
    for idx_iter in np.arange(num_iters):

        print(f"iteration #{idx_iter}")

        if dual is True:
            L_iter = optimize_L_dual(Y_iter)
        else:
            L_iter = optimize_L_primal(Y_iter)

        Y_iter = optimize_Y(L_iter)

        obj_val = norm(X - Y_iter, 'fro')**2 + alpha * np.trace(Y_iter.T @ L_iter @ Y_iter) + beta * (norm(L_iter, 'fro')**2);

        print(f"iteration-{idx_iter}: obj value = {obj_val}")

        if np.abs(obj_old_val - obj_val) < 1e-4:
            break
        else:
            obj_old_val = obj_val

    return L_iter, Y_iter

import numpy,from numpy,from scipy,import cvxpy


import numpy as np
from numpy import linalg as LA
from numpy import diag
from numpy import zeros
A=np.array([[-5,-5,-1,0,0,0,500,500,100],[0,0,0,-5,-5,-1,500,500,100],
           [-150,-5,-1,0,0,0,30000,1000,200],[0,0,0,-150,-5,-1,12000,400,80],
           [-150,-150,-1,0,0,0,33000,33000,220],[0,0,0,-150,-150,-1,12000,12000,80],
           [-5,-150,-1,0,0,0,500,15000,100],[0,0,0,-5,-150,-1,1000,30000,200]])
print("The A matrix:\n{}".format(A))
A_T=A.transpose()
print("The transpose of the A matrix is: \n{}".format(A_T))
A1=A.dot(A_T)
A2=A_T.dot(A)
#To find the eigen vectors and eigenvalues of the A^T*A and A*A^T.
w1, v1 = LA.eig(A1)
w2, v2 = LA.eig(A2)
#Taking the absolute values of the eigen values to keep them postive
w= abs(w2)
print("The adjusted eigen values : \n{}".format(w))
U=v1
print("The Eigen Vector U is: \n{}".format(U))
#transpose of the V eigen vector matrix
V_T=v2.transpose()
print("Eigen vectors 2\n{}".format(v2))
print("The transpose of Eigen Vector V is: \n{}".format(V_T))
#Sigma Matrix matrix can be formed using the square root of eigenvalues of A^T*A
#Sigma matrix is a diagonal matrix with the squareroot of the eigen values
sig=np.sqrt(w)
print("The eigenvectors considered for the Sigma matrix are: \n{}".format(sig))
#To form the Sigma diagonal matrix
Sigma = zeros((A.shape[0], A.shape[1]))
print(Sigma)
for i in range(len(Sigma)):
    Sigma[i][i]= sig[i]
print("The Sigma Matrix(Diagonal Matrix) is : \n{}".format(Sigma))
#SVD of A= U*Sigma*V'
SVD=U.dot(Sigma.dot(V_T))
print("The SVD of Matrix A is:")
print(SVD)
#To find the Homography matrix
H=v2[:,-1]
H1=np.reshape(H,(3,3))
print("The Homography matrix H is:")
print(H1)


# using higher order derivatives to find extrema values

from sympy import Symbol, solve, Derivative

x = Symbol( 'x' )
f = x**5 - 30*x**3 + 50*x

d1 = Derivative( f, x ).doit()

critical_points = solve( d1 )

critical_points

A = critical_points[ 2 ]
B = critical_points[ 0 ]
C = critical_points[ 1 ]
D = critical_points[ 3 ]

d2 = Derivative( f, x, 2 ).doit()

d2.subs( { x: B } ).evalf()
d2.subs( { x: C } ).evalf()
d2.subs( { x: A } ).evalf()
d2.subs( { x: D } ).evalf()

x_min = -5
x_max = 5

f.subs( { x: C } ).evalf()
f.subs( { x: A } ).evalf()

f.subs( { x: x_min } ).evalf()
f.subs( { x: x_max } ).evalf()


# define: Sigmoid function version 1
# date: 2021.12.23.
# Resource: 밑바닥부터 시작하는 인공지능(사이토고키, 2017)

def sigmoid(x):
	return 1 / (1+ np.exp(-x)) # exp is function to caculate the exponential of all elements in the input array.

if __name__ == '__main__':
	import numpy as np
	import matplotlib.pylab as plt

	x = np.arange(-5.0, 5.0, 0.1)
	y = sigmoid(x)

	plt.plot(x, y)
	plt.ylim(-0.1, 1.1)
	plt.show()


import matplotlib.pyplot as plt
import numpy as np

def numeric_better(x, h):
    return (np.cos(x + h) - np.cos(x - h)) / (2 * h)

def numeric_worse(x, h):
    return (np.cos(x + h) - np.cos(x)) / h

# line 1 points
x = 1
distances = [10 ** (-i) for i in range(1, 16)]
real_y = -np.sin(x)
numeric_better_y = [abs(numeric_better(x, distance) - real_y) for distance in distances]
numeric_worse_y = [abs(numeric_worse(x, distance) - real_y) for distance in distances]

plt.plot(distances, numeric_worse_y, label="real derivative")
plt.plot(distances, numeric_better_y, label="numeric derivative")

# plt.loglog(x, real_y, label="real derivative")

# # plotting the line 1 points
# line 2 points
# plotting the line 2 points
plt.xlabel("h")
# Set the y axis label of the current axis.
plt.ylabel("f`(x)")
# Set a title of the current axes.
plt.title("Two or more lines on same plot with suitable legends ")
# show a legend on the plot
plt.legend()
# Display a figure.
plt.show()


# Summary Statistics and More:
# mean, median, stdevs, min, max, quantiles, sum, product, etc.

# Summing Values in NumPy Arrays
import numpy as np
L = np.random.random(100)
big_array = np.random.rand(1000000)

sum(L)                  # Python   sum. 1D  arrs only
np.sum(L)               # NP ufunc sum. 2D+ arrs
sum(L) == np.sum(L)

np.sum(big_array)
big_array.sum()
# Generally, we prefer calling from object instance syntax.

# Minimum and Maximum
big_array.min(), big_array.max()

# Multi-Dimensional Aggregates
# For 2D arrays, we often want aggregates organized along rows or columns.
# We can toggle 'axis' argument of methods to perform this.
A = np.random.random((3, 4))
A
A.sum()

# NOTE: Axis controls dimension to collapse, and returns the complement.
A.min(axis=0)   # collapse row --> return min of cols
A.max(axis=1)   # collapse col --> return max of rows

# Most NumPy agg funcs have a NaN-safe equivalents (ignore NaN values).
A.sum; np.nansum(A)

# Partial List of Aggregation Functions:
np.sum
np.prod
np.mean
np.std
np.var          # compute variance (max-min)
np.min
np.max
np.argmin       # return index of min value
np.argmax       # return index of max value
np.median
np.percentile
np.any          # evaluate whether any elements are true
np.all          # evaluate whether all elements are true

# Example: Get Average Height of US Presidents
# !head -4 data/president_heights.csv
import pandas as pd
data = pd.read_csv('data/president_heights.csv')
heights = np.array(data['height(cm)'])

heights.mean()
heights.std()
heights.min()
heights.max()

np.percentile(heights, 25)
np.median(heights)
np.percentile(heights, 75)

## we can also visualize with Matplotlib
# %matplotlib inline
import matplotlib.pyplot as plt
import seaborn; seaborn.set()
plt.hist(heights)
plt.title('Height Distribution of US Presidents')
plt.xlabel('height (cm)')
plt.ylabel('number')


import numpy as np

### Functions for you to fill in ###

def polynomial_kernel(X, Y, c, p):
    """
        Compute the polynomial kernel between two matrices X and Y::
            K(x, y) = (<x, y> + c)^p
        for each pair of rows x in X and y in Y.

        Args:
            X - (n, d) NumPy array (n datapoints each with d features)
            Y - (m, d) NumPy array (m datapoints each with d features)
            c - a coefficient to trade off high-order and low-order terms (scalar)
            p - the degree of the polynomial kernel

        Returns:
            kernel_matrix - (n, m) Numpy array containing the kernel matrix
    """
    return np.power((X @ Y.T + c), p)

def rbf_kernel(X, Y, gamma):
    """
        Compute the Gaussian RBF kernel between two matrices X and Y::
            K(x, y) = exp(-gamma ||x-y||^2)
        for each pair of rows x in X and y in Y.

        Args:
            X - (n, d) NumPy array (n datapoints each with d features)
            Y - (m, d) NumPy array (m datapoints each with d features)
            gamma - the gamma parameter of gaussian function (scalar)

        Returns:
            kernel_matrix - (n, m) Numpy array containing the kernel matrix
    """
    n = X.shape[0]
    m = Y.shape[0]
    K = np.zeros((n,m))
    for i in range(n):
        for j in range(m):
            K[i,j] = np.exp(-gamma * (np.linalg.norm(X[i] - Y[j]) ** 2))
    return K
    # np.vectorize
    # n = X.shape[0]
    # m = Y.shape[0]
    # a = np.mat((X@X.T).diagonal())
    # a = np.tile(a.T, (1,m))

    # b = np.mat((Y@Y.T).diagonal())
    # b = np.tile(b, (n,1))

    # k = a + b
    # k -= 2*(X @ np.transpose(Y))

    # return np.exp(-gamma * k)

courses/MITx/MITx 6.86x Machine Learning with Python-From Linear Models to Deep Learning/project2/mnist/part1/kernel.py


import scipy.io
import os
import numpy as np
import numpy.matlib
import matplotlib.pyplot as plt

from toolbox.pcaPmtk import *

data_path = os.path.join("..", "bigData", "olivettiFaces", "olivettiFaces.mat")
mat = scipy.io.loadmat(data_path)

X = mat['faces'].T
y = np.matlib.repmat(np.arange(1, 41), 10, 1).flatten(order='F')

n, d = X.shape
h, w = 64, 64
name = 'faces'

perm = np.random.permutation(n)

f, ax_arr = plt.subplots(2, 2)
for i in range(2):
    for j in range(2):
        ax_arr[i, j].imshow(X[perm[i*2+j], :].reshape(h, w).T, cmap="gray")
        ax_arr[i, j].axis('off')
f.suptitle("pcaImages-%s-images" % name)
plt.show()

XC = X - np.mean(X, axis=0)
print('Performing PCA.... stay tuned')

V, Z, evals, _, mu = pcaPmtk(X)

f, ax_arr = plt.subplots(2, 2)
for i in range(2):
    for j in range(2):
        if i == 0 and j == 0:
            ax_arr[i, j].imshow(mu.reshape(h, w).T, cmap="gray")
            ax_arr[i, j].axis('off')
            ax_arr[i, j].set_title('mean')
        else:
            idx = i*2+j-1
            ax_arr[i, j].imshow(V[:, idx].reshape(h, w).T, cmap="gray")
            ax_arr[i, j].axis('off')
            ax_arr[i, j].set_title('principal basis %d' % idx)
f.suptitle("pcaImages-%s-basis" % name)
plt.show()

# Plot reconstructed image
ndx = 125
Ks = [5, 10, 20, np.linalg.matrix_rank(XC)]

f, ax_arr = plt.subplots(2, 2)
for ki in range(len(Ks)):
    k = Ks[ki]
    Xrecon = np.dot(Z[ndx, 0:k], V[:, 0:k].T) + mu
    i = ki//2
    j = ki%2
    ax_arr[i, j].imshow(Xrecon.reshape(h, w).T, cmap="gray")
    ax_arr[i, j].axis('off')
    ax_arr[i, j].set_title('Using %d bases' % k)
f.suptitle("pcaImages-%s-reconImages" % name)
plt.show()

Ks = []
Ks += range(1, 10)
Ks += range(10, 50, 5)
Ks += range(50, np.linalg.matrix_rank(XC), 25)
mse = []

for ki in range(len(Ks)):
    k = Ks[ki]
    Xrecon = np.dot(Z[:, 0:k], V[:, 0:k].T) + mu
    err = Xrecon - X
    mse.append(np.sqrt(np.mean(np.power(err, 2))))
plt.plot(Ks, mse, '-o')
plt.ylabel("mse")
plt.xlabel("K")
plt.title('reconstruction error')
plt.show()

plt.plot(np.cumsum(evals)/np.sum(evals), 'ko-')
plt.show()


"""Estimate pi using a parallel Monte Carlo simulation.

   Implements an MPI version of a parallel Monte Carlo simulation for estimating pi.
   MPI's default channel is used.

   Usage:
        python monte_carlo_pi.py [total_num_trials]

        Options:
               -total_num_trials    the total number of trials ( (x,y) points) for the simulation.
                                    By default it is set to 1 million since a (probabilistic) precision of 3 decimal digits
                                    requires 10^(2*3) trials (1 million trials).
                                    see https://stackoverflow.com/questions/18139934/can-a-monte-carlo-pi-calculation-be-used-for-a-world-record

"""

from mpi4py import MPI
import numpy as np
from sys import argv

def random_points_generator(trials_per_node=0):
    """Evaluate if a pair (x,y) drawn from an uniform distribution falls inside the unit circle.

    Given a random distribution between 0 and 1, it evaluates if a pair (x,y) of random points
    falls inside the unit circle. If that's the case, a counter is incremented.

    Args:
        trials_per_node (int): the number of trials/drawns for this node/process.

    Returns:
        int: a counter who keeps track of the overall number of points fallen inside the circle.

    """
    points_fallen_inside_circle = 0
    # draw a pair (x,y) from an uniform distribution
    # between 0 and 1 for each trial
    # NOTE: each child process has its own seed
    # since np.random.seed() hasn't been invoked
    # This is good for randomness but terrible for
    # reproducibility (which we don't consider in this project)
    for _ in range(trials_per_node):
        x = np.random.uniform(0,1)
        y = np.random.uniform(0,1)
        # if (x,y) falls inside the unit circle
        # increment the counter by 1
        if (x*x+y*y) <= 1.0:
            points_fallen_inside_circle += 1

    return points_fallen_inside_circle

if __name__ == "__main__":
    # Check if the number of trials has been supplied via
    # the CLI. If not, use 1 million as default value.
    total_num_trials = int(argv[1]) if len(argv) > 1 else 1e6
    # MPI_Init() is automatically invoked
    # when MPI module is imported
    # Rename default channel for convenience
    comm = MPI.COMM_WORLD
    # Get the number of nodes on comm channel
    num_nodes = comm.Get_size()
    # Get their rank
    rank = comm.Get_rank()
    # Print a start checkpoint if I am the master node
    if rank == 0:
        print("Start", flush=True)
    # Compute number of trials per node
    trials_per_node = np.ceil(total_num_trials / num_nodes).astype(int)
    # Sync all nodes here to start a 'more accurate' time measurement
    comm.Barrier()
    # Init send (i.e. elapsed_buff) and receive (i.e. longest_elapsed_buff) buffer for MAX reduction
    # as a one dimensional zero-filled arrays
    elapsed_buff         = np.zeros(1,dtype=np.float64)
    longest_elapsed_buff = np.zeros(1,dtype=np.float64)
    # Get start time
    start = MPI.Wtime()
    # Compute (x,y) random points which fall inside the circle
    points_in_circle_per_node = random_points_generator(trials_per_node)
    # Record the time when the function is done
    finish = MPI.Wtime()
    # Compute the time delta
    elapsed_buff[0] = (finish - start)

    #######################################################################################
    # If needed, print the wall-clock time for each processor
    # print("Processor {0} finished in {1:.6f}s.".format(rank,elapsed_buff[0]), flush=True)
    #######################################################################################

    # Do a MAX reduction to record the slowest processor
    comm.Reduce([elapsed_buff, MPI.DOUBLE],[longest_elapsed_buff, MPI.DOUBLE],op=MPI.MAX,root=0)
    # Get rid of the array, just need a number
    longest_elapsed = longest_elapsed_buff[0]
    # Init send (i.e. points_in_buff) and receive (i.e. tot_points_in_buff) buffer
    # for SUM reduction
    points_in_buff     = np.zeros(1,dtype=np.int32)
    tot_points_in_buff = np.zeros(1,dtype=np.int32)
    # Store points_in_circle_per_node in the buffer
    points_in_buff[0] = points_in_circle_per_node
    # Do a SUM reduction to retrieve the total number of points fallen
    # inside the circle
    comm.Reduce([points_in_buff, MPI.INT],[tot_points_in_buff, MPI.INT],op=MPI.SUM,root=0)
    # Get rid of the array, just need a number
    tot_points_in = tot_points_in_buff[0]
    # Master node estimates pi
    if rank == 0:
        # print the slowest processor
        print("Slowest processor wall-clock time: {:.6f}s".format(longest_elapsed), flush=True)
        print("End", flush=True)

        ############################################################################
        # If needed, print the estimated pi and the estimation error
        # estimate pi according to 4*(total points in the circle) / overall trials)
        # estimated_pi = 4*(tot_points_in / total_num_trials)
        # compute the error
        # error_pi = np.abs(np.pi - estimated_pi)
        # print("DEBUG: Pi is approximately: {:.6f}".format(estimated_pi))
        # print("DEBUG: Error is: {:.6f}".format(error_pi))
        ############################################################################

    # No need to MPI_Finalize() since mpi4py
    # implements an exit hook who does the job for us


import numpy as np                         # Linear algebra library
import matplotlib.pyplot as plt            # library for visualization
from sklearn.decomposition import PCA      # PCA library
import pandas as pd                        # Data frame library
import math                                # Library for math functions
import random                              # Library for pseudo random numbers

n = 1  # The amount of the correlation
x = np.random.uniform(1,2,1000) # Generate 1000 samples from a uniform random variable
y = x.copy() * n # Make y = n * x

# PCA works better if the data is centered
x = x - np.mean(x) # Center x. Remove its mean
y = y - np.mean(y) # Center y. Remove its mean

data = pd.DataFrame({'x': x, 'y': y}) # Create a data frame with x and y
plt.scatter(data.x, data.y) # Plot the original correlated data in blue

pca = PCA(n_components=2) # Instantiate a PCA. Choose to get 2 output variables

# Create the transformation model for this data. Internally, it gets the rotation
# matrix and the explained variance
pcaTr = pca.fit(data)

rotatedData = pcaTr.transform(data) # Transform the data base on the rotation matrix of pcaTr
# # Create a data frame with the new variables. We call these new variables PC1 and PC2
dataPCA = pd.DataFrame(data = rotatedData, columns = ['PC1', 'PC2'])

# Plot the transformed data in orange
plt.scatter(dataPCA.PC1, dataPCA.PC2)
plt.show()

print('Eigenvectors or principal component: First row must be in the direction of [1, n]')
print(pcaTr.components_)

print()
print('Eigenvalues or explained variance')
print(pcaTr.explained_variance_)

import matplotlib.lines as mlines
import matplotlib.transforms as mtransforms

random.seed(100)

std1 = 1     # The desired standard deviation of our first random variable
std2 = 0.333 # The desired standard deviation of our second random variable

x = np.random.normal(0, std1, 1000) # Get 1000 samples from x ~ N(0, std1)
y = np.random.normal(0, std2, 1000)  # Get 1000 samples from y ~ N(0, std2)
#y = y + np.random.normal(0,1,1000)*noiseLevel * np.sin(0.78)

# PCA works better if the data is centered
x = x - np.mean(x) # Center x
y = y - np.mean(y) # Center y

#Define a pair of dependent variables with a desired amount of covariance
n = 1 # Magnitude of covariance.
angle = np.arctan(1 / n) # Convert the covariance to and angle
print('angle: ',  angle * 180 / math.pi)

# Create a rotation matrix using the given angle
rotationMatrix = np.array([[np.cos(angle), np.sin(angle)],
                 [-np.sin(angle), np.cos(angle)]])

print('rotationMatrix')
print(rotationMatrix)

xy = np.concatenate(([x] , [y]), axis=0).T # Create a matrix with columns x and y

# Transform the data using the rotation matrix. It correlates the two variables
data = np.dot(xy, rotationMatrix) # Return a nD array

# Print the rotated data
plt.scatter(data[:,0], data[:,1])
plt.show()

plt.scatter(data[:,0], data[:,1]) # Print the original data in blue

# Apply PCA. In theory, the Eigenvector matrix must be the
# inverse of the original rotationMatrix.
pca = PCA(n_components=2)  # Instantiate a PCA. Choose to get 2 output variables

# Create the transformation model for this data. Internally it gets the rotation
# matrix and the explained variance
pcaTr = pca.fit(data)

# Create an array with the transformed data
dataPCA = pcaTr.transform(data)

print('Eigenvectors or principal component: First row must be in the direction of [1, n]')
print(pcaTr.components_)

print()
print('Eigenvalues or explained variance')
print(pcaTr.explained_variance_)

# Print the rotated data
plt.scatter(dataPCA[:,0], dataPCA[:,1])

# Plot the first component axe. Use the explained variance to scale the vector
plt.plot([0, rotationMatrix[0][0] * std1 * 3], [0, rotationMatrix[0][1] * std1 * 3], 'k-', color='red')
# Plot the second component axe. Use the explained variance to scale the vector
plt.plot([0, rotationMatrix[1][0] * std2 * 3], [0, rotationMatrix[1][1] * std2 * 3], 'k-', color='green')

plt.show()

nPoints = len(data)

# Plot the original data in blue
plt.scatter(data[:,0], data[:,1])

#Plot the projection along the first component in orange
plt.scatter(data[:,0], np.zeros(nPoints))

#Plot the projection along the second component in green
plt.scatter(np.zeros(nPoints), data[:,1])

plt.show()


"""
looking at wallis product for pi/2
"""

# import libraries
import numpy as np
import matplotlib.pyplot as plt

# close all plots
plt.close('all')

# generate numbers and sequences via an upper bound
ub = 100
num = np.append(np.repeat(np.arange(2,ub,2),2),ub)
den = np.insert(np.repeat(np.arange(3,ub+1,2),2),0,1)

# empty lists for two methods
wallis1, wallis2, wallis3 = [], [], []

### method 1 - calculate numerator, then denominator, then divide ###

# iterate through arrays and calculate
for i in range(len(num)):
    wallis1.append(np.prod(num[:i+1])/np.prod(den[:i+1]))

### method 2 - calculate num/denom of each term, then multiply ###

# iterate through arrays and calculate
for i in range(len(num)):
    wallis2.append(np.prod(num[:i+1]/den[:i+1]))

### method 3 - using production summation definition ###

# start with the first term of the series
wallis3_val = 4.0/3.0

# now start iterations from n=2
for n in range(2, ub):
    wallis3_val *= (4*n**2)/(4*n**2-1)
    wallis3.append(wallis3_val)

### create figures

fig,ax = plt.subplots()
plt.rcParams.update({'font.size': 18})
ax.plot(wallis1,label='Method 1')
ax.plot(wallis2,label='Method 2',color='g')
ax.plot(wallis3,label='Method 3',color='r')
ax.set_xlabel('# of Iterations $(n)$')
ax.set_ylabel('Value')
ax.axhline(np.pi/2.0,color='k',linestyle='--',label=r'$\pi/2$')
ax.axhline(0,color='k',linewidth=0.8)
ax.axvline(0,color='k',linewidth=0.8)
plt.legend()
ax.set_title('Wallis Product')
plt.tight_layout()


"""
Example:
Let's concider the problem
15x1 + 8x2 + 80x3 -> min        (1)
subjected to
x1 + 2x2 + 3x3 <= 15              (2)
8x1 +  15x2 +  80x3 <= 80      (3)
8x1  + 80x2 + 15x3 <=150      (4)
100x1 +  10x2 + x3 >= 800     (5)
80x1 + 8x2 + 15x3 = 750         (6)
x1 + 10x2 + 100x3 = 80           (7)
x1 >= 4                                     (8)
-8 >= x2 >= -80                        (9)
"""

from numpy import *
from openopt import LP
f = array([15,8,80])
A = mat('1 2 3; 8 15 80; 8 80 15; -100 -10 -1') # numpy.ndarray is also allowed
b = [15, 80, 150, -800] # numpy.ndarray, matrix etc are also allowed
Aeq = mat('80 8 15; 1 10 100') # numpy.ndarray is also allowed
beq = (750, 80)

lb = [4, -80, -inf]
ub = [inf, -8, inf]
p = LP(f, A=A, Aeq=Aeq, b=b, beq=beq, lb=lb, ub=ub)
#or p = LP(f=f, A=A, Aeq=Aeq, b=b, beq=beq, lb=lb, ub=ub)

#r = p.minimize('glpk') # CVXOPT must be installed
#r = p.minimize('lpSolve') # lpsolve must be installed
r = p.minimize('pclp')
#search for max: r = p.maximize('glpk') # CVXOPT & glpk must be installed
#r = p.minimize('nlp:ralg', ftol=1e-7, xtol=1e-7, goal='min', plot=1)

print('objFunValue: %f' % r.ff) # should print 204.48841578
print('x_opt: %s' % r.xf) # should print [ 9.89355041 -8.          1.5010645 ]


from __future__ import division;
import numpy as np;
from matplotlib import pyplot as plt;
from matplotlib import colors
import matplotlib as mpl;
from mpl_toolkits.mplot3d import Axes3D
from matplotlib import mlab;
from matplotlib import gridspec;
import pandas as pd
from IPython.display import display
import sklearn;
from sklearn.kernel_ridge import KernelRidge;

from notebook.services.config import ConfigManager
cm = ConfigManager()
cm.update('livereveal', {
              'theme': 'simple',
              'transition': 'none',
              'start_slideshow_at': 'selected',
});

def regression_example_draw(degree1, degree2, degree3, ifprint):
    x = np.linspace(0, 2*np.pi, 13);
    # np.random.randn generates gaussian samples
    y = np.sin(x) + np.random.randn(x.shape[0]) * 0.2;
    xx = np.linspace(0, 2*np.pi, 100);
    plt.figure(figsize=(12,7.5))
    plt.subplot(221)
    plt.plot(xx, np.sin(xx), "g", linestyle='--'); plt.hold(True)
    plt.plot(x, y, "or"); plt.hold(False)
    plt.legend(['True Curve','Data']); plt.title('Data and True Curve');

    # Here were are going to take advantage of numpy's 'polyfit' function
    # This implements a "polynomial fitting" algorithm
    # coeffs are the optimal coefficients of the polynomial
    coeffs = np.polyfit(x, y, degree1); # 0 is the degree of the poly
    # We construct poly(), the polynomial with "learned" coefficients
    poly = np.poly1d(coeffs);
    plt.subplot(222)
    plt.plot(xx, np.sin(xx), "g", linestyle='--'); plt.hold(True)
    plt.plot(x, y, "or");
    plt.plot(xx, poly(xx), color='b', linestyle='-'); plt.hold(False)
    plt.legend(['True Curve','Data','Learned Curve']); plt.title(str(degree1)+'th Order Polynomial')
    exprsn1=''
    for i in range(degree1+1):
        if i>0 and coeffs[i]>0:
            exprsn1 += '+%.3fx^%d' %(coeffs[i], i)
        elif i==0:
            exprsn1 += '%.3f' %(coeffs[i])
        else:
            exprsn1 += '%.3fx^%d' %(coeffs[i], i)

    coeffs = np.polyfit(x, y, degree2); # Now let's try degree = 1
    poly = np.poly1d(coeffs);
    plt.subplot(223)
    plt.plot(xx, np.sin(xx), "g", linestyle='--'); plt.hold(True)
    plt.plot(x, y, "or");
    plt.plot(xx, poly(xx), color='b', linestyle='-'); plt.hold(False)
    plt.legend(['True Curve','Data','Learned Curve']); plt.title(str(degree2)+'th Order Polynomial')
    exprsn2 = ''
    for i in range(degree2+1):
        if i>0 and coeffs[i]>0:
            exprsn2 += '+%.3fx^%d' %(coeffs[i], i)
        elif i==0:
            exprsn2 += '%.3f' %(coeffs[i])
        else:
            exprsn2 += '%.3fx^%d' %(coeffs[i], i)

    coeffs = np.polyfit(x, y, degree3); # Now degree = 3
    poly = np.poly1d(coeffs);
    plt.subplot(224)
    plt.plot(xx, np.sin(xx), "g", linestyle='--'); plt.hold(True)
    plt.plot(x, y, "or");
    plt.plot(xx, poly(xx), color='b', linestyle='-'); plt.hold(False)
    plt.legend(['True Curve','Data','Learned Curve']); plt.title(str(degree3)+'th Order Polynomial')
    plt.show()
    exprsn3 = ''
    for i in range(degree3+1):
        if i>0 and coeffs[i]>0:
            exprsn3 += '+%.3fx^%d' %(coeffs[i], i)
        elif i==0:
            exprsn3 += '%.3f' %(coeffs[i])
        else:
            exprsn3 += '%.3fx^%d' %(coeffs[i], i)

    if ifprint:
        print('The expression for the first polynomial is y=' + exprsn1)
        print('The expression for the second polynomial is y=' + exprsn2)
        print('The expression for the third polynomial is y=' + exprsn3)

def set_nice_plot_labels(axs):
    axs[0].set_title(r"$ \phi_j(x) = x^j$", fontsize=18, y=1.08);
    axs[0].set_xlabel("Polynomial", fontsize=18);
    axs[1].set_title(r"$ \phi_j(x) = \exp\left( - \frac{(x-\mu_j)^2}{2s^2} \right)$", fontsize=18, y=1.08);
    axs[1].set_xlabel("Gaussian", fontsize=18);
    axs[2].set_title(r"$ \phi_j(x) = (1  + \exp\left(\frac{\mu_j-x}{s}\right))^{-1}$", fontsize=18, y=1.08);
    axs[2].set_xlabel("Sigmoid", fontsize=18);

def basis_function_plot():
    x = np.linspace(-1,1,100);
    f, axs = plt.subplots(1, 3, sharex=True, figsize=(12,4));
    for j in range(8):
        axs[0].plot(x, np.power(x,j));
        axs[0].hold(True)
        axs[1].plot(x, np.exp( - (x - j/7 + 0.5)**2 / 2*5**2 ));
        axs[1].hold(True)
        axs[2].plot(x, 1 / (1 + np.exp( - (x - j/5 + 0.5) * 5)) );
        axs[2].hold(True)
    axs[0].hold(False)
    axs[1].hold(False)
    axs[2].hold(False)

    set_nice_plot_labels(axs) # I'm hiding some helper code that adds labels


import numpy as np

class DataCreator(object):
    '''
    Creates random datapoints (x,y) of the underlying function
        y = f + N(0,noise^2)
    with
        f = sin(5/2*x)*son(3/2*x)
    and
        noise = 0.15 + 0.25*(1-sin(5/2*x))^2

    '''
    def __init__(self,num_samples, x_min=0, x_max=2*np.pi,random_seed=42):
        np.random.seed(random_seed)
        self.num_samples = num_samples
        self.x_min = x_min
        self.x_max = x_max

    '''
    call to sample new data points
    '''
    def create_datapoints(self):
        self.x = np.linspace(self.x_min,self.x_max,num=self.num_samples)
        self.f = np.sin(5*self.x/2)*np.sin(3*self.x/2)
        self.noise = 0.15 + 0.25*(1-np.sin(5*self.x/2))**2
        self.y = self.f+np.random.randn(self.num_samples)*self.noise

        self.f = self.f.reshape(self.num_samples,1)
        self.noise = self.noise.reshape(self.num_samples,1)
        self.x = self.x.reshape(self.num_samples,1)
        self.y = self.y.reshape(self.num_samples,1)

    def get_data(self):
        return self.x, self.y, self.f, self.noise

    '''
    get normalized samples
    '''
    def get_normalized_data(self):
        mux = np.mean(self.x)
        sigx = np.std(self.x)
        muy = np.mean(self.y)
        sigy = np.std(self.y)

        return (self.x-mux)/sigx, (self.y-muy)/sigy, mux, sigx, muy, sigy


from math import pi, sqrt
import math
import numpy as np
import pandas as pd
np.set_printoptions(suppress=True)

def EMalgorithm(x, K, conv):
    # Function takes in the dataset, number of clusters and thee precision you want to converge with 10^-conv
    D = len(x[0]) # the Dimensions/# of features
    N = len(x)  # number of observations
    # Initializing Parameters
    x_col = np.matrix(np.mean(x, axis=0)).T # mean of column/feature
    sd_col = np.matrix(np.std(x, axis=0, ddof=1)).T # std dev of column/feature
    randn = np.random.randn(1,K) # 1xK random array
    # compute Initial Values
    mu_k = np.array(x_col*np.ones(K) + np.matmul(sd_col,randn))
    sd_k = np.mean(sd_col)*np.ones(K)
    p_k = [1/K]*K

    p_kn_1 = 0 #previous place holder for convergence
    condition = True
    i = 0
    while condition:
        print()
        print("iteration: ", i+1)
        p_kn = E_Step(x,K, D, mu_k,sd_k, p_k) # run the E_step Function
        print("p_kn: ",p_kn)
        condition = np.any(np.abs((p_kn-p_kn_1))>(10**-conv)) #update the loop condition
        p_kn_1 = p_kn # update the prev matrix
        mu_k, sd_k, p_k = M_Step(x,p_kn, K,D,N) # run the M_step Function
        print("mu_k: ",mu_k)
        print("sd_k: ",sd_k)
        print("p_k: ", p_k)
        i+=1

    return

def E_Step(x, K, D, mu_k, sd_k, p_k):
    # Function takes dataset, # of clusters, Dimensions, mean of cluster, std dev of cluster, p_k

    g = [None]*K
    k = 0 # cluster counter
    while k < K:
        g[k] = (x-mu_k[:,k])
        g[k] = np.linalg.norm(g[k],axis=1)**2
        g[k] = -1/2* (g[k]/(sd_k[k]**2))
        g[k] = np.exp(g[k])
        g[k] = 1/(((np.sqrt(2*math.pi))*sd_k[k]))**D * g[k]
        k+=1
    g = np.asarray(g) #format to array

    k=0
    p_kn =[None]*K
    while k<K:
        p_kn[k] = p_k[k]*g[k]
        k+=1
    p_kn = np.asarray(p_kn)

    p_kn = p_kn / np.sum(p_kn, axis=0)

    return p_kn

def M_Step(x,p_kn, K, D, N):
    # Function takes dataset, probability of observation being in a cluster matrix, # clusters, Dimensions, # of observations

    k = 0
    mu_i = [None]*K
    while k<K:
        mu_i[k] = np.matmul(p_kn[k],x)/np.sum(p_kn[k])
        k+=1
    mu_i= np.asarray(mu_i).T

    k = 0
    sd_i = [None]*K
    while k<K:
        norm = (x-mu_i[:,k])
        norm = np.linalg.norm(norm,axis=1)**2
        norm = np.sum(norm*p_kn[k])
        sd_i[k] = np.sqrt(1/D * norm/sum(p_kn[k]))
        k+=1

    p_i = 1/N * np.sum(p_kn, axis=1)

    return mu_i, sd_i, p_i

if __name__ == "__main__":
    Iris = pd.read_csv("iris.csv",header=0)
    IrisMatrix = Iris.to_numpy()
    IrisMatrix = IrisMatrix[:,0:4]
    IrisMatrix = IrisMatrix.astype(float)
    # print(IrisMatrix)

    x = np.array([[1,2],[4,2],[1,3],[4,3]])
    # run the Expectation maximization algorithm EMalgorithm(Data, # of clusters, 10^-precision value)
    # EMalgorithm(x,2,6)
    EMalgorithm(IrisMatrix,3,6)

Algorithms for Data Science/Homework 3/Homework3_Q1_Q2.py


import numpy as np
from matplotlib import pyplot as plt

# return the matrix A with the correct coefficients
def generate_coef(n):
	mat = np.zeros((n ** 2, n ** 2));
	for i in range(n):
		for j in range(n):
			ni = i * n + j
			# dx
			if i < n - 1:
				mat.itemset(ni, (i + 1) * n + j, 1)
			if i > 0:
				mat.itemset(ni, (i - 1) * n + j, 1)

			# dy
			if j < n - 1:
				mat.itemset(ni, i * n + j + 1, 1)
			if j > 0:
				mat.itemset(ni, i * n + j - 1, 1)

			mat.itemset(ni, ni, -4);
	return -((n + 1) ** 2) * mat

# create a n * n matrix from a value function
def func_to_matrix(f, n):
	return np.fromfunction(lambda i, j: f(i/(n - 1), j/(n - 1)), (n, n))

# create a vector of size n² from a matri of size n * n
def matrix_to_vector(m):
	return m.flatten().T

# create a matrix of size n * n from a vector of size n²
def vector_to_matrix(v):
	n = int(np.sqrt(v.size))
	return v.reshape((n, n))

# show the imge corresponding to the given function
def show_image(f, solver, N, title):
	A = generate_coef(N)

	b = matrix_to_vector(func_to_matrix(f, N))
	print("b = ")
	print(b)

	x = vector_to_matrix(solver(A, b))
	print("x = ")
	print(x)

	plt.imshow(x, cmap='hot', extent=(0, 1, 0, 1), interpolation='bilinear')
	plt.title(title)
	plt.show()

# the function f defining the initial heat of a point radiator
def point_radiator(i, j):
	return np.where((i - 0.5) ** 2 + (j - 0.5) ** 2 < 0.01, 1, 0)

# the function f defining the initial heat of a point radiator
def wall_radiator(i, j):
	return np.where(i < 0.0001, 1, 0)

def test_equation():
        N = 32
        show_image(point_radiator, np.linalg.solve, N, "Radiateur Point")
        show_image(wall_radiator, np.linalg.solve, N, "Radiateur Mur")

if __name__ == '__main__':
        test_equation()


import numpy as np
import matplotlib.pyplot as plt

def SD(A,b,x0):
    """
    Parameters:
    A - Input Matrix
    b - vector
    x0 - Initial solution
    """
    x = x0
    count = 0
    r = np.ones_like(b)
    while(np.linalg.norm(r)>1e-6):
        r = b - A*x
        alpha = np.transpose(r)*r/(np.transpose(r)*A*r)
        print(alpha)
        x_next = x + alpha*r
        x = x_next
        count +=1
    return x,count

def CG(A,b,x0):
    """
    Parameters:
    A - Input matrix
    b - Vector
    x0 - Initial Solution
    """
    assert np.all(A == np.transpose(A)) ,print('error conjugate gradient method cannot be used since Matrix is not Positive Semi Definite')
    x = x0
    count = 0
    r = np.ones_like(b)
    d = np.ones_like(b)
    while(np.linalg.norm(r)>1e-6):
        r = b - A*x
        alpha = np.transpose(r)*r/(np.transpose(d)*A*d)
        x = x + alpha*d
        r_next = r - alpha*A*d
        beta = np.transpose(r_next)*r_next/(np.transpose(r)*r)
        d = r_next + beta*d
        r = r_next
        count+=1
    return x,count

A = np.array([[3,2],[2,6]])
b = np.array([2,-8])
x0 = np.array([-3,-3])

value = SD(A,b,x0)
cg = CG(A,b,x0)
print(value)
print(cg)


import matplotlib.pyplot as plt
import numpy as np

def power_negative_3_solution(t: float, x0: float):
    """
    This function implements the solution to the differential equation

    x' = -x^3

    with initial condition x(t=0) = x0.
    """
    return np.sqrt(1.0 / (2.0 * t + 1.0 / x0**2))

def power_negative_1_solution(t: float, x0: float):
    """
    This function implements the solution to the differential equation

    x' = -x

    with initial condition x(t=0) = x0.
    """
    return x0 * np.exp(-t)

if __name__ == "__main__":
    # Get the time points for the x-axis
    ts = np.linspace(0.0, 10.0, 1000)

    # Get the solution of the differential equation x'=-x^3 evaluated at these points
    x_pm3 = power_negative_3_solution(ts, x0=10.)

    # Get the solution to the differential equation x'=-x evaluated at these points
    x_pm1 = power_negative_1_solution(ts, x0=10.)

    # Plot the solutions for comparison
    plt.plot(ts, x_pm3, label=r'$\dot{x}=-x^3$')
    plt.plot(ts, x_pm1, label=r'$\dot{x}=-x$')

    # Cosmetics
    plt.title(r"Comparisons of the solutions to the DE's $\dot{x}=-x^3$ and $\dot{x}=-x$")
    plt.xlabel("t (a.u.)")
    plt.ylabel("x (a.u.)")
    plt.legend()
    plt.show()


# -*- coding: utf-8 -*-

from __future__ import absolute_import

import numpy as np

__all__ = [
    'log_likelihood',
    'logistic_sigmoid'
]

def log_likelihood(X, y, w):
    """Compute the log-likelihood function.

    Computes the log-likelihood function over the training data.
    The key to the log-likelihood is that the log of the product of
    likelihoods becomes the sum of logs. That is (in pseudo-code),

        np.log(np.product([f(i) for i in range(N)]))

    is equivalent to:

        np.sum([np.log(f(i)) for i in range(N)])

    The log-likelihood function is used in computing the gradient for
    our loss function since the derivative of the sum (of logs) is equivalent
    to the sum of derivatives, which simplifies all of our math.

    Parameters
    ----------
    X : np.ndarray, shape=(n_samples, n_features)
        The training data.

    y : np.ndarray, shape=(n_samples,)
        The target vector of 1s or 0s.

    w : np.ndarray, shape=(n_features,)
        The vector of feature weights (coefficients)

    References
    ----------
    .. [1] For a very thorough explanation of the log-likelihood function, see
           https://www.coursera.org/learn/ml-classification/lecture/1ZeTC/very-optional-expressing-the-log-likelihood
    """
    weighted = X.dot(w)
    return (y * weighted - np.log(1. + np.exp(weighted))).sum()

def logistic_sigmoid(x):
    """The logistic function.

    Compute the logistic (sigmoid) function over a vector, ``x``.

    Parameters
    ----------
    x : np.ndarray, shape=(n_samples,)
        A vector to transform.
    """
    return 1. / (1. + np.exp(-x))


from statistics import mean
import numpy as np
import matplotlib
matplotlib.use('TkAgg')
import matplotlib.pyplot as plt

xs = [1,2,3,4,5,6]
ys = [5,4,6,5,6,7]

# plt.plot(xs,ys)
# plt.show()

# Since above are not np array they are just a python array, to change them which will give us
# more powerful way to iterate over then values in the array,

xs = np.array(xs, dtype = np.float64)  # you can also specify the data-type into your array
ys = np.array(ys, dtype = np.float64)

# first we need a function to calculate the slop as

def best_fit_slop(xs,ys):

    m = ( ((mean(xs)*mean(ys)) - mean(xs*ys)) /
          (mean(xs)**2-mean(xs**2)))

# remember the PEMDAS the order of the operations in computing
    return m

m = best_fit_slop(xs,ys)
print(m)

Machine_Learning_Old_Files/[8] linear regression P.1.py


"""
The Monty Hall Problem [http://en.wikipedia.org/wiki/Monty_Hall_problem] is a probability brain teaser that has a
rather unintuitive solution.

The gist of it, taken from Wikipedia:
Suppose you're on a game show, and you're given the choice of three doors: Behind one door is a car; behind the others,
goats. You pick a door, say No. 1 [but the door is not opened], and the host, who knows what's behind the doors, opens
another door, say No. 3, which has a goat. He then says to you, "Do you want to pick door No. 2?" Is it to your
advantage to switch your choice? (clarification: the host will always reveal a goat)

Your task is to write a function that will compare the strategies of switching and not switching over many random
position iterations. Your program should output the proportion of successful choices by each strategy. Assume that if
both unpicked doors contain goats the host will open one of those doors at random with equal probability.

If you want to, you can for simplicity's sake assume that the player picks the first door every time. The only aspect
of this scenario that needs to vary is what is behind each door.

Thanks to SleepyTurtle for posting this idea at /r/dailyprogrammer_ideas! Do you have a problem you think would be good
for us! Head on over there and post it!
"""

import numpy as np
import numpy.random as nprnd
import time

TRIALS = 1000000
doors = [1, 0, 0]

start_time = time.time()
keep = 0
switch = 0
for _ in range(TRIALS):
    """ permutes doors to create random combination of doors and then runs the keep and switch scenarios simultaneously.
    in the case of initially selecting the correct door, the choice of removing one of the other doors does not matter,
    so it just chooses the first one.
    """
    perm = nprnd.permutation(doors)
    keep += perm[0]

    zero = np.where(perm[1:] == 0)[0]
    perm = np.delete(perm, zero[0]+1)
    switch += perm[1]

print('Kept:         {}%'.format(100*keep/TRIALS))
print('Switched:     {}%'.format(100*switch/TRIALS))
print('Elapsed time: {}'.format(time.time()-start_time))

start_time = time.time()
keep = 0
switch = 0
for _ in range(TRIALS):
    """ optimization so that it doesn't actually simulate the monty hall problem. instead it is based on the realization
    that if the permutation's first door is correct, the keep scenario always wins and if the first is incorrect the
    switch scenario always wins. Time reduced to less than half of the above method.
    """
    perm = nprnd.permutation(doors)
    if perm[0]:
        keep += 1
    else:
        switch += 1

print('Kept:     {}%'.format(100*keep/TRIALS))
print('Switched: {}%'.format(100*switch/TRIALS))
print('Elapsed time: {}'.format(time.time()-start_time))


# -*- coding: utf-8 -*-
"""
Make a simple 2D Gaussian profile.  One example is in the representation of an
ideal laser beam.

"""

import numpy as np
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D
import gaussian1D_profile as gp

def gaussian_2D_profile(x_min, x_max, x_step, y_min, y_max, y_step,
                        xc, yc, sigma_x, sigma_y, amplitude):
    """Function to produce a 2D Gaussian profile.

    Parameters
    ----------

    x_min, x_max, x_step: float, float, float
        Creates a sequence (1D ndarray) of x points over which to compute the Gaussian
    y_min, y_max, y_step: float, float, float
        Creates a sequence (1D ndarray) of y points over which to compute the Gaussian
    xc,yc: float, float
        The center points in x and y of the gaussian profile
    sigma_x, sigma_y: float, float
        1/e-squared width of beam in the x and y axis respectively
    amplitude: float
        Amplitude at peak value

    Returns
    -------

    Z: ndarray
        The 2D gaussian profile amplitude values

    """

    idx = np.arange(x_min, x_max, x_step)
    idy = np.arange(y_min, y_max, y_step)

    d0x = 2*sigma_x  #beam diameter (1/e-squared) is 2 * standard deviation
    d0y = 2*sigma_y

    x = amplitude * np.e**(-2*np.power((idx-xc)/d0x,2))
    y = amplitude * np.e**(-2*np.power((idy-yc)/d0y,2))

    plt.plot(idx, x, label='x')
    plt.plot(idy, y, label='y')
    plt.title('Gaussian 1D Profiles in x and y')
    plt.legend()
    plt.show()

    #now 2d...can just multiply the two 1D curves together
    X,Y = np.meshgrid(idx,idy)

    #Z = np.e**(-2*(((X-xc)/d0x)**2 * ((Y-yc)/d0y)**2))  #interesting star type pattern

    # The following methods are equivalent e^x * e^y = e^(x+y)
    #Z = np.e**(-2*np.power((X-xc)/d0x,2))*np.e**(-2*np.power((Y-yc)/d0y,2)) #1.92ms per loop
    #Z = np.e**(-2*((X-xc)/d0x)**2) * np.e**(-2*((Y-yc)/d0y)**2) # 1.03ms per loop
    Z = np.e**(-2*(((X-xc)/d0x)**2 + ((Y-yc)/d0y)**2))          # 586us per loop (4x faster)

    return Z

def plot_2d_gaussian(Z):
    """Plot the gaussian profile as a 2D image and contour lines.

    Parameters
    ----------

    Z: ndarray
        2D Gaussian values

    """

    plt.figure()
    #plt.axis('off')
    plt.imshow(Z,extent=(-50,50,-50,50))
    plt.contour(Z)
    plt.title('2D Gaussian Profile')
    plt.show()

if __name__ == '__main__':
    Z = gaussian_2D_profile(-50,50,1,-50,50,1, 0, 0, 10, 5, 1)
    plot_2d_gaussian(Z)


""" This module implements the Fourier transforms on linear grids.

The following code approximates the continuous Fourier transform (FT) on
equidistantly spaced grids. While this is usually associated with
'just doing a fast Fourier transform (FFT)', surprisingly, much can be done
wrong.

The reason is that the correct expressions depend on the grid location. In
fact, the FT can be calculated with one DFT but in general it requires a prior
and posterior multiplication with phase factors.

The FT convention we are going to use here is the following::

    Ẽ(w) = 1/2pi ∫ E(t) exp(+i w t) dt
    E(t) =       ∫ Ẽ(w) exp(-i t w) dw

where w is the angular frequency. We can approximate these integrals by their
Riemann sums on the following equidistantly spaced grids::

    t_k = t_0 + k Δt, k=0, ..., N-1
    w_n = w_0 + n Δw, n=0, ..., N-1

and define E_k = E(t_k) and Ẽ_n = Ẽ(w_n) to obtain::

    Ẽ_n = Δt/2pi ∑_k E_k exp(+i w_n t_k)
    E_k = Δw     ∑_n Ẽ_n exp(-i t_k w_n).

To evaluate the sum using the FFT we can expand the exponential to obtain::

    Ẽ_n = Δt/2pi exp(+i n t_0 Δw) ∑_k [E_k exp(+i t_k w_0) ] exp(+i n k Δt Δw)
    E_k = Δw     exp(-i t_k w_0)  ∑_n [Ẽ_n exp(-i n t_0 Δw)] exp(-i k n Δt Δw)

Additionally, we have to require the so-called reciprocity relation for
the grid spacings::

          !
    Δt Δw = 2pi / N = ζ     (reciprocity relation)

This is what enables us to use the DFT/FFT! Now we look at the definition of
the FFT in NumPy::

     fft[x_m] -> X_k =     ∑_m exp(-2pi i m k / N)
    ifft[X_k] -> x_m = 1/N ∑_k exp(+2pi i k m / N)

which gives the final expressions::

    Ẽ_n = Δt N/2pi r_n   ifft[E_k s_k  ]
    E_k = Δw       s_k^*  fft[Ẽ_n r_n^*]

    with r_n = exp(+i n t_0 Δw)
         s_k = exp(+i t_k w_0)

where ^* means complex conjugation. We see that the array to be transformed
has to be multiplied with an appropriate phase factor before and after
performing the DFT. And those phase factors mainly depend on the starting
points of the grids: w_0 and t_0. Note also that due to our sign convention
for the FT we have to use ifft for the forward transform and vice versa.

Trivially, we can see that for ``w_0 = t_0 = 0`` the phase factors vanish and
the FT is approximated well by just the DFT. However, in optics these
grids are unusual.
For ``w_0 = l Δw`` and ``t_0 = m Δt``, where l, m are integers (i.e., w_0 and
t_0 are multiples of the grid spacing), the phase factors can be
incorperated into the DFT. Then the phase factors can be replaced by circular
shifts of the input and output arrays.

This is exactly what the functions (i)fftshift are doing for one specific
choice of l and m, namely for::

    t_0 = -floor(N/2) Δt
    w_0 = -floor(N/2) Δw.

In this specific case only we can approximate the FT by::

    Ẽ_n = Δt N/2pi fftshift(ifft(ifftshift(E_k)))
    E_k = Δw       fftshift( fft(ifftshift(Ẽ_n))) (no mistake!)

We see that the ifftshift _always_ has to appear on the inside. Failure to do
so will still be correct for even N (here fftshift is the same as ifftshift)
but will produce wrong results for odd N.

Additionally you have to watch out not to violate the assumptions for the
grid positions. Using a symmetrical grid, e.g.,::

    x = linspace(-1, 1, 128)

will also produce wrong results, as the elements of x are not multiples of the
grid spacing (but shifted by half a grid point).

The main drawback of this approach is that circular shifts are usually far more
time- and memory-consuming than an elementwise multiplication, especially for
higher dimensions. In fact I see no advantage in using the shift approach at
all. But for some reason it got stuck in the minds of people and you find the
notion of having to re-order the output of the DFT everywhere.

Long story short: here we are going to stick with multiplying the correct
phase factors. The code tries to follow the notation used above.

Good, more comprehensive expositions of the issues above can be found in
[Briggs1995]_ and [Hansen2014]_. For the reason why the first-order
approximation to the Riemann integral suffices, see [Trefethen2014]_.
"""
import numpy as np
# scipy.fftpack is still faster than numpy.fft (should change in numpy 1.17)
import scipy.fftpack as fft
from . import io
from .lib import twopi, sqrt2pi
_fft_backend = 'scipy'
try:
    import pyfftw
    _fft_backend = 'pyfftw'
except ImportError:
    pass

class FourierTransformBase(io.IO):
    """ This class implements the Fourier transform on linear grids.

    This simple implementation is mainly for educational use.

    Attributes
    ----------
    N : int
        Size of the grid
    dt : float
        Temporal spacing
    dw : float
        Frequency spacing (angular frequency)
    t0 : float
        The first element of the temporal grid
    w0 : float
        The first element of the frequency grid
    t : 1d-array
        The temporal grid
    w : 1d-array
        The frequency grid (angular frequency)
    """
    _io_store = ['N', 'dt', 'dw', 't0', 'w0']

    def __init__(self, N, dt=None, dw=None, t0=None, w0=None):
        """ Creates conjugate grids and calculates the Fourier transform.

        Parameters
        ----------
        N : int
            Array size
        dt : float, optional
            The temporal grid spacing. If ``None`` will be calculated by the
            reciprocity relation ``dt = 2 * pi / (N * dw)``. Exactly one of
            ``dt`` or ``dw`` has be provided.
        dw : float, optional
            The spectral grid spacing. If ``None`` will be calculated by the
            reciprocity relation ``dw = 2 * pi / (N * dt)``. Exactly one of
            ``dt`` or ``dw`` has be provided.
        t0 : float, optional
            The first element of the temporal grid. If ``None`` will be
            ``t0 = -floor(N/2) * dt``.
        w0 : float, optional
            The first element of the spectral grid. If ``None`` will be
            ``w0 = -floor(N/2) * dw``.
        """
        if dw is None and dt is not None:
            dw = np.pi / (0.5 * N * dt)
        elif dt is None and dw is not None:
            dt = np.pi / (0.5 * N * dw)
        else:
            raise ValueError("Exactly one of the grid spacings has to be "
                             "provided!")

        if t0 is None:
            t0 = -np.floor(0.5 * N) * dt
        if w0 is None:
            w0 = -np.floor(0.5 * N) * dw
        self.N = N
        self.dt = dt
        self.dw = dw
        self.t0 = t0
        self.w0 = w0
        self._post_init()

    def _post_init(self):
        """ Hook to initialize an object from storage.
        """
        # calculate the grids
        n = k = np.arange(self.N)
        self.t = self.t0 + k * self.dt
        self.w = self.w0 + n * self.dw
        # pre-calculate the phase factors
        # TODO: possibly inaccurate for large t0, w0
        self._fr = self.dt * self.N / twopi * np.exp(1.0j * n * self.t0 *
                                                     self.dw)
        self._fs = np.exp(1.0j * self.t * self.w0)
        # complex conjugate of the above
        self._br = np.exp(-1.0j * n * self.t0 * self.dw)
        self._bs = self.dw * np.exp(-1.0j * self.t * self.w0)

    def forward_at(self, x, w):
        """ Calculates the forward Fourier transform of `x` at the
        frequencies `w`.

        This function calculates the Riemann sum directly and has quadratic
        runtime. However, it can evaluate the integral at arbitrary
        frequencies, even if they are non-equidistantly spaced. Effectively,
        it performs a trigonometric interpolation.
        """
        Dnk = self.dt / twopi * np.exp(1.0j * w[:, None] * self.t[None, :])
        return Dnk @ x

    def backward_at(self, x, t):
        """ Calculates the backward Fourier transform of `x` at the
        times `t`.

        This function calculates the Riemann sum directly and has quadratic
        runtime. However, it can evaluate the integral at arbitrary
        times, even if they are non-equidistantly spaced. Effectively,
        it performs a trigonometric interpolation.
        """
        Dkn = self.dw * np.exp(-1.0j * t[:, None] * self.w[None, :])
        return Dkn @ x

# =============================================================================
# Fourier backend selection
# =============================================================================
if _fft_backend == "scipy":
    class FourierTransform(FourierTransformBase):

        def forward(self, x, out=None):
            """ Calculates the (forward) Fourier transform of ``x``.

            For n-dimensional arrays it operates on the last axis, which has
            to match the size of `x`.

            Parameters
            ----------
            x : ndarray
                The array of which the Fourier transform will be calculated.
            out : ndarray or None, optional
                A location into which the result is stored. If not provided or
                None, a freshly-allocated array is returned.
            """
            if out is None:
                out = np.empty(x.shape, dtype=np.complex128)
            out[:] = self._fr * fft.ifft(self._fs * x)
            return out

        def backward(self, x, out=None):
            """ Calculates the backward (inverse) Fourier transform of ``x``.

            For n-dimensional arrays it operates on the last axis, which has
            to match the size of `x`.

            Parameters
            ----------
            x : ndarray
                The array of which the Fourier transform will be calculated.
            out : ndarray or None, optional
                A location into which the result is stored. If not provided or
                None, a freshly-allocated array is returned.
            """
            if out is None:
                out = np.empty(x.shape, dtype=np.complex128)
            out[:] = self._bs * fft.fft(self._br * x)
            return out

elif _fft_backend == "pyfftw":
    class FourierTransform(FourierTransformBase):

        def _post_init(self):
            super()._post_init()
            # do not need the additional N factor
            n = np.arange(self.N)
            self._fr = self.dt / twopi * np.exp(1.0j * n * self.t0 * self.dw)
            # create the aligned arrays
            a = self._field = pyfftw.empty_aligned(self.N, dtype="complex128")
            b = self._spectrum = pyfftw.empty_aligned(self.N,
                                                      dtype="complex128")
            # instantiate the FFTW objects
            self._fft = pyfftw.FFTW(b, a, direction="FFTW_FORWARD")
            self._ifft = pyfftw.FFTW(a, b, direction="FFTW_BACKWARD")

        def forward(self, x, out=None):
            """ Calculates the (forward) Fourier transform of ``x``.

            For n-dimensional arrays it operates on the last axis, which has
            to match the size of `x`.

            Parameters
            ----------
            x : ndarray
                The array of which the Fourier transform will be calculated.
            out : ndarray or None, optional
                A location into which the result is stored. If not provided or
                None, a freshly-allocated array is returned.
            """
            if out is None:
                out = np.empty(x.shape, dtype=np.complex128)
            f, s = self._field, self._spectrum
            if x.ndim == 1:
                # fast code path for single dimension
                f[:] = x
                f *= self._fs
                self._ifft.execute()
                s *= self._fr
                out[:] = s
            else:
                # implicitly work along last axis and return copy
                for idx in np.ndindex(x.shape[:-1]):
                    f[:] = x[idx]
                    f *= self._fs
                    self._ifft.execute()
                    s *= self._fr
                    out[idx] = s
            return out

        def backward(self, x, out=None):
            """ Calculates the backward (inverse) Fourier transform of ``x``.

            For n-dimensional arrays it operates on the last axis, which has
            to match the size of `x`.

            Parameters
            ----------
            x : ndarray
                The array of which the Fourier transform will be calculated.
            out : ndarray or None, optional
                A location into which the result is stored. If not provided or
                None, a freshly-allocated array is returned.
            """
            if out is None:
                out = np.empty(x.shape, dtype=np.complex128)
            f, s = self._field, self._spectrum
            if x.ndim == 1:
                # fast code path for single dimension
                s[:] = x
                s *= self._br
                self._fft.execute()
                f *= self._bs
                out[:] = f
            else:
                # implicitly work along last axis and return copy
                for idx in np.ndindex(x.shape[:-1]):
                    s[:] = x[idx]
                    s *= self._br
                    self._fft.execute()
                    f *= self._bs
                    out[idx] = f
            return out

class Gaussian:
    """ This class can be used for testing the Fourier transform.
    """

    def __init__(self, dt, t0=0.0, phase=0.0):
        """ Instantiates a shifted Gaussian function.

        The Gaussian is calculated by::

            f(t) = exp(-0.5 (t - t0)^2 / dt^2) * exp(1.0j * phase)

        Its Fourier transform is::

            F(w) = dt/sqrt(2pi) exp(-0.5 * (w + phase)^2 * dt^2 +
                                     1j * t0 * w)

        Parameters
        ----------
        dt : float
            The standard deviation of the temporal amplitude distribution.
        t0 : float
            The center of the temporal amplitude distribution.
        phase : float
            The linear phase coefficient of the temporal distribution.
        """
        self.dt = dt
        self.t0 = t0
        self.phase = phase

    def temporal(self, t):
        """ Returns the temporal distribution.
        """
        arg = (t - self.t0) / self.dt
        return np.exp(-0.5 * arg**2) * np.exp(1.0j * self.phase * t)

    def spectral(self, w):
        """ Returns the spectral distribution.
        """
        w = w + self.phase
        arg = w * self.dt
        return (self.dt * np.exp(-0.5 * arg**2) * np.exp(1.0j * self.t0 * w) /
                sqrt2pi)


# -*- coding:utf-8 -*-

"""Strassen算法求矩阵乘法
"""

import numpy as np

def strassen(A, B):
    """Strassen算法求矩阵乘法

    Args:
        A(np.array): 矩阵1
        B(np.array): 矩阵2

    Return:
        C(np.array): 矩阵乘法结果
    """
    n, _ = A.shape

    if n == 1:
        C = np.array(A[0, 0] * B[0, 0])
        return C
    else:
        A11 = A[0:n//2, 0:n//2]
        A12 = A[0:n//2, n//2:]
        A21 = A[n//2:, 0:n//2]
        A22 = A[n//2:, n//2:]
        B11 = B[0:n//2, 0:n//2]
        B12 = B[0:n//2, n//2:]
        B21 = B[n//2:, 0:n//2]
        B22 = B[n//2:, n//2:]

        S1 = B12 - B22
        S2 = A11 + A12
        S3 = A21 + A22
        S4 = B21 - B11
        S5 = A11 + A22
        S6 = B11 + B22
        S7 = A12 - A22
        S8 = B21 + B22
        S9 = A11 - A21
        S10 = B11 + B12

        P1 = strassen(A11, S1)
        P2 = strassen(S2, B22)
        P3 = strassen(S3, B11)
        P4 = strassen(A22, S4)
        P5 = strassen(S5, S6)
        P6 = strassen(S7, S8)
        P7 = strassen(S9, S10)

        C11 = P5 + P4 - P2 + P6
        C12 = P1 + P2
        C21 = P3 + P4
        C22 = P5 + P1 - P3 - P7

        C = np.vstack((np.hstack((C11, C12)), np.hstack((C21, C22))))
        return C

if __name__ == '__main__':
    A = np.array([
        [1, 2, 3, 4],
        [4, 3, 2, 1],
        [1, 3, 5, 7],
        [7, 5, 3, 1],
    ])
    B = np.array([
        [2, 4, 6, 8],
        [8, 6, 4, 2],
        [1, 2, 3, 4],
        [4, 3, 2, 1],
    ])

    print(strassen(A, B))


#!/usr/bin/python
#   X = FEJER(N) returns N Chebyshev points X in (-1,1).
#
#   X, W = FEJER(N) returns also a vector W of weights for
#   Fejer quadrature.
#
#   X, W = FEJER(N, [A, B]) returns the nodes and weights
#   for the interval [A, B].

import numpy as np

def fejer(N, dom=[-1,1]):
    t = np.zeros(shape=(N,1))
    x = np.zeros(shape=(N,1))
    for k in range(N):
        t[k] = (0.5+k)*np.pi/N;
        x[k] = -np.cos(t[k])

    # Weights
    V_trans = np.zeros(shape=(N,N))
    for j in range(N):
        for k in range(N):
            V_trans[j][k] = np.cos(j*t[k]);
    rhs = np.zeros(shape=(N,1))
    for j in range(0, N, 2):
        rhs[j] = 2./(1.-j*j);
    c = np.linalg.solve(V_trans, rhs);

    # Scale
    a = dom[0];
    b = dom[1];
    x = .5*(b-a)*x + .5*(b+a);
    c = .5*(b-a)*c;

    return (x, c)
if __name__ == "__main__":
    x, c = fejer(5, [-1, 1])
    print 'x = ', x
    print 'c = ', c


""""
    This program calculates the inverse of a matrix by many ways.
    Marina von Steinkirch, spring/2013 (based on Mike Zingale's codes)

"""

import numpy as npy
from scipy import linalg
from inverseGauss import inverseGauss
from inverseNewton import inverseNewton

def main():
    A = npy.array([ [4, 3, 4, 10], [2, -7, 3, 0], [-2, 11, 1, 3], [3, -4, 0, 2] ], dtype=npy.float64)

    """
    print "\nInverse matrix calculated by the Numpy API:"
    AinvNpy = linalg.inv(A)
    print "NumPy: A . Ainv = \n", npy.dot(A, AinvNpy)
    print "NumPy: Ainv = \n", AinvNpy

    print "\nInverse matrix calculated by the Gauss method:"
    AinvGauss = inverseGauss(A)
    print "Gauss: A . Ainv = \n", npy.dot(A, AinvGauss)
    print "Gauss: Ainv = \n", AinvGauss

    """

    print "\nInverse matrix calculated by the Newton method:"
    AinvNewton, error, nIter = inverseNewton(A)
    print "Newtow: A . Ainv = \n", npy.dot(A, AinvNewton)
    print "Newtow: Ainv = \n", AinvNewton
    print "Number of iterations: ", nIter
    print "Error from Numpy API results: ", error

    print "\nDone!"

if __name__ == "__main__":
    main()


import numpy as np
import operator as op
from functools import reduce

def ncr(n, r):
    """n choose r"""
    r = min(r, n - r)  # This works since it's symmetric

    numer = reduce(op.mul, range(n, n - r, -1), 1)
    denom = reduce(op.mul, range(1, r + 1), 1)
    return numer / denom

def bernstein_poly(i, n, t):
    """
     The Bernstein polynomial of n, i as a function of t
    """
    return ncr(n, i) * (t ** (n - i)) * (1 - t)**i

def compute_bezier_points(points, n_times=25):
    """
        Returns a list of points that can be used to construct a bezier curve.
    """
    n_points = len(points)
    x_points, y_points = np.array([p[0] for p in points]), np.array([p[1] for p in points])

    t = np.linspace(0.0, 1.0, n_times)
    polynomial_array = np.array([bernstein_poly(i, n_points-1, t) for i in range(0, n_points)])

    x_vals, y_vals = np.dot(x_points, polynomial_array), np.dot(y_points, polynomial_array)

    return [(x, y) for x, y in zip(x_vals, y_vals)]

def colour_linear_interpolation(col_a, col_b, t):
    """
        Linearly interpolates between two colours.
    """
    col = tuple([a + (b - a) * t for a, b in zip(col_a, col_b)])
    return col

def map_from_to(x, a, b, c, d):
    """
        Maps a value x from a-b to c-d.
    """
    return (x - a) / (b - a) * (d - c) + c


import numpy as np
import math

def factorial(n):
	fact = 1
	for i in range(1,n+1):
		fact *= i
	return fact

summation = 1
for i in range(1,7):
	val = (-1)**i * (math.pi/3)**(2*i) / factorial(2*i)
	summation += val
	print "val: ", val
	print "summation: ", summation

print "Final summation: ", summation


import numpy as np
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D
import numpy as np
import pandas as pd

def entangled_two_circles(number_of_points_per_class, plot = True):
    '''
    Creates the dataset of two circles in R^3, a blue and a red one, each inside of the other.
    Inputs: number of points in each class of the dataset, if plot is true, shows the dataset in R^3
    Outputs: (dataset in 3 coordinates, labels of the dataset)
    '''
    theta = np.linspace(0,2*np.pi,number_of_points_per_class)
    x_1 = np.cos(theta)
    y_1 = np.sin(theta)
    z_1 = np.zeros(len(theta))

    x_2 = np.zeros(len(theta))
    y_2 = np.cos(theta)+np.ones(len(theta))
    z_2 = np.sin(theta)

    x = np.concatenate((x_1, x_2))
    y = np.concatenate((y_1, y_2))
    z = np.concatenate((z_1, z_2))

    target = np.concatenate((np.zeros(len(x_1)), np.ones(len(x_2))))

    x = np.reshape(x,(-1,1))
    y = np.reshape(y,(-1,1))
    z = np.reshape(z,(-1,1))
    data = np.concatenate((x,y,z), axis = 1)

    if plot == True:
        fig = plt.figure()
        ax = Axes3D(fig)

        scatter = ax.scatter(x_1,y_1,z_1, c= 'r')
        scatter = ax.scatter(x_2,y_2,z_2, c= 'b')
        plt.show()

    return (data, target)

def entangled_four_circles(number_of_points_per_class, plot = True):
    '''
    Creates the dataset of four circles in R^3, two blue and two larger red one, each entangled inside the other.
    Inputs: 4*(number of points) in each class of the dataset, if plot is true, shows the dataset in R^3
    Outputs: (dataset in 3 coordinates, labels of the dataset)
    '''
    theta = np.linspace(0,2*np.pi,number_of_points_per_class)

    x_r1 = 1/2*np.cos(theta)
    y_r1 = 1/2*np.sin(theta)+np.ones(len(theta))
    z_r1 = np.zeros(len(theta))

    x_r2 = 1/2*np.cos(theta)
    y_r2 = 1/2*np.sin(theta)-np.ones(len(theta))
    z_r2 = np.zeros(len(theta))

    x_r = np.concatenate((x_r1, x_r2))
    y_r = np.concatenate((y_r1, y_r2))
    z_r = np.concatenate((z_r1, z_r2))

    x_b1 = 1/3*np.ones(len(theta))
    y_b1 = np.cos(theta)
    z_b1 = np.sin(theta)

    x_b2 = -1/3*np.ones(len(theta))
    y_b2 = np.cos(theta)
    z_b2 = np.sin(theta)

    x_b = np.concatenate((x_b1, x_b2))
    y_b = np.concatenate((y_b1, y_b2))
    z_b = np.concatenate((z_b1, z_b2))

    x = np.concatenate((x_r, x_b))
    y = np.concatenate((y_r, y_b))
    z = np.concatenate((z_r, z_b))

    target = np.concatenate((np.zeros(len(x_r)), np.ones(len(x_b))))

    x = np.reshape(x,(-1,1))
    y = np.reshape(y,(-1,1))
    z = np.reshape(z,(-1,1))
    data = np.concatenate((x,y,z), axis = 1)

    if plot == True:
        fig = plt.figure()
        ax = Axes3D(fig)

        scatter = ax.scatter(x_r,y_r,z_r, c= 'r',  alpha = 0.5)
        scatter = ax.scatter(x_b,y_b,z_b, c= 'b')

    return (data, target)

def circle_inside_torus(number_of_points_per_class, plot = True):
    '''
    Creates a dataset of a circle of radius 2 inside a torus of inner radius 1 and outter radius 3 of number_of_points_per_class
    in each of these manifolds. If plot = True, it plots both the surface and the scatter plot for the torus sampled data
    Inputs: number_of_points_per_class; plot = True
    '''
    theta_circle = np.linspace(0,2*np.pi,number_of_points_per_class)
    x_circle = 2*np.cos(theta_circle)
    y_circle = 2*np.sin(theta_circle)
    z_circle = np.zeros(len(theta_circle))
    x_circle = np.reshape(x_circle,(-1,1))
    y_circle = np.reshape(y_circle,(-1,1))
    z_circle = np.reshape(z_circle,(-1,1))
    data_circle = np.concatenate((x_circle,y_circle,z_circle), axis = 1)

    theta_torus = 2*np.pi*np.random.rand(round(number_of_points_per_class**1/2)) #takes the square root so that each class has the same
    #number of instances
    phi_torus = 2*np.pi*np.random.rand(round(number_of_points_per_class**1/2))
    c, a = 2, 1
    x_torus = (c + a*np.cos(theta_torus)) * np.cos(phi_torus)
    y_torus = (c + a*np.cos(theta_torus)) * np.sin(phi_torus)
    z_torus = a * np.sin(theta_torus)
    x_torus = np.reshape(x_torus,(-1,1))
    y_torus = np.reshape(y_torus,(-1,1))
    z_torus = np.reshape(z_torus,(-1,1))
    data_torus = np.concatenate((x_torus,y_torus,z_torus), axis = 1)

    data =  np.concatenate((data_circle, data_torus), axis = 0)
    target = np.concatenate((np.zeros(number_of_points_per_class),np.ones(len(data_torus))), axis = 0)

    if plot == True:
        n = 100 #we will make 100 points so to make the surface plot visible
        theta_plot = np.linspace(0,2*np.pi,n)
        phi_plot = np.linspace(0, 2*np.pi, n)
        theta_plot, phi_plot = np.meshgrid(theta_plot, phi_plot)
        c, a = 2, 1
        x_torus_plot = (c + a*np.cos(theta_plot)) * np.cos(phi_plot)
        y_torus_plot = (c + a*np.cos(theta_plot)) * np.sin(phi_plot)
        z_torus_plot = a * np.sin(theta_plot)

        fig = plt.figure(figsize = [10,10])
        ax = fig.add_subplot(1, 2, 1, projection='3d')
        ax.scatter(x_circle,y_circle,z_circle, c= 'r',alpha=1)
        ax.plot_surface(x_torus_plot, y_torus_plot, z_torus_plot,
                                  color='b', alpha =0.4, rstride=5, cstride=5, edgecolors='w')
        ax.set_zlim(-3,3)

        ax = fig.add_subplot(1, 2, 2, projection='3d')
        ax.scatter(x_circle,y_circle,z_circle, c= 'r',alpha=1)
        ax.scatter(x_torus, y_torus, z_torus,
                                  color='b', alpha =0.4)
        ax.set_zlim(-3,3)

    return (data, target)

def swiss_roll(number_of_points_per_class, plot = True):
    '''
    Creates a dataset of a swiss roll of number_of_class points of half of its is in class red and half in class blue.
    If plot = True, it plots both the scatter plot for the sampled data
    Inputs: number_of_points_per_class; plot = True
    '''
    #makes the cross section of the swiss roll (ie. spiral)
    theta_blue = np.linspace(0, 5*np.pi, round(number_of_points_per_class/10))
    theta_red = np.linspace(5*np.pi, 10*np.pi, round(number_of_points_per_class/10))

    x_blue =  theta_blue*np.cos(theta_blue)
    y_blue =  theta_blue*np.sin(theta_blue)
    z_blue = np.zeros(len(x_blue))

    x_red =  theta_red*np.cos(theta_red)
    y_red =  theta_red*np.sin(theta_red)
    z_red = np.zeros(len(x_red))

    #reshape arrays
    x_blue = np.reshape(x_blue,(-1,1))
    y_blue = np.reshape(y_blue,(-1,1))
    z_blue = np.reshape(z_blue,(-1,1))

    x_red = np.reshape(x_red,(-1,1))
    y_red = np.reshape(y_red,(-1,1))
    z_red = np.reshape(z_red,(-1,1))

    blue_coord = np.concatenate((x_blue, y_blue, z_blue), axis = 1)
    red_coord = np.concatenate((x_red, y_red, z_red), axis = 1)

    #lifts the spiral to the swiss roll by making 9 extra layers of points
    for i in range(1, 10):
        z_new = i*np.ones(len(x_blue))
        z_new = np.reshape(z_new, (-1,1))

        new_blue_cord = np.concatenate((x_blue, y_blue, z_new), axis = 1)
        new_red_cord = np.concatenate((x_red, y_red, z_new), axis = 1)

        blue_coord = np.concatenate((blue_coord, new_blue_cord), axis = 0)
        red_coord = np.concatenate((red_coord, new_red_cord), axis = 0)

    #makes dataset and target
    data = np.concatenate((red_coord, blue_coord), axis = 0)

    target_red = np.zeros(len(red_coord))
    target_blue = np.ones(len(blue_coord))
    target = np.concatenate((target_red, target_blue), axis = 0)
    if plot == True:
        fig = plt.figure()
        ax = Axes3D(fig)

        scatter = ax.scatter(blue_coord[:,0],blue_coord[:,1],blue_coord[:,2], c = 'b')
        scatter = ax.scatter(red_coord[:,0],red_coord[:,1],red_coord[:,2], c = 'r')
        plt.show()

    return (data, target)


import numpy as np

class LogisticRegression:
    def __init__(self, alpha=0.01, iteration=1500, intercept=True):
        self.alpha=alpha
        self.iteration=iteration
        self.intercept=intercept

    def sigmoid(self,x):
        return 1 / (1+np.exp(-x))

    def fit(self, x, y):
        self.m = len(x)
        self.n = np.size(x,1)
        if self.intercept :
            x = np.hstack((np.ones((len(x),1)), x))

        self.theta=np.zeros(x.shape[1])

        for _ in range(self.iteration):
            self.z=np.dot(x, self.theta)
            self.h=self.sigmoid(self.z)
            grad=np.dot(x.T, (self.h-y)) / y.size
            self.theta = self.alpha * grad

    def loss(self, y):
        return ((-y*np.log(self.h))- ((1-y)*np.log(1-self.h))/self.m)

    def predict_prob(self, x):
        if self.intercept :
            x = np.hstack((np.ones((len(x),1)), x))
        return self.sigmoid(np.dot(x, self.theta))

    def predict(self, x, threshold=0.5):
        return self.predict_prob(x)>= threshold


# coding=utf-8
"""Hermite and Bezier curves using python, numpy and matplotlib"""

import numpy as np
#import matplotlib.pyplot as plt
#from mpl_toolkits.mplot3d import Axes3D

__author__ = "Daniel Calderon"
__license__ = "MIT"

def generateT(t):
    return np.array([[1, t, t**2, t**3]]).T

def hermiteMatrix(P1, P2, T1, T2):

    # Generate a matrix concatenating the columns
    G = np.concatenate((P1, P2, T1, T2), axis=1)

    # Hermite base matrix is a constant
    Mh = np.array([[1, 0, -3, 2], [0, 0, 3, -2], [0, 1, -2, 1], [0, 0, -1, 1]])

    return np.matmul(G, Mh)

def bezierMatrix(P0, P1, P2, P3):

    # Generate a matrix concatenating the columns
    G = np.concatenate((P0, P1, P2, P3), axis=1)

    # Bezier base matrix is a constant
    Mb = np.array([[1, -3, 3, -1], [0, 3, -6, 3], [0, 0, 3, -3], [0, 0, 0, 1]])

    return np.matmul(G, Mb)

def catmullRomMatrix(P0, P1, P2, P3):
    # Generate a matrix concatenating the columns
    G = np.concatenate((P0, P1, P2, P3), axis=1)

    # Camull-Rom base matrix is a constant
    Mcr = 1/2*np.array([[0, -1, 2, -1], [2, 0, -5, 3], [0, 1, 4, -3], [0, 0, -1, 1]])

    return np.matmul(G, Mcr)

def plotCurve(ax, curve, label, color=(0,0,1)):

    xs = curve[:, 0]
    ys = curve[:, 1]
    zs = curve[:, 2]

    ax.plot(xs, ys, zs, label=label, color=color)

# M is the cubic curve matrix, N is the number of samples between 0 and 1
def evalCurve(M, N):
    # The parameter t should move between 0 and 1
    ts = np.linspace(0.0, 1.0, N)

    # The computed value in R3 for each sample will be stored here
    curve = np.ndarray(shape=(N, 3), dtype=float)

    for i in range(len(ts)):
        T = generateT(ts[i])
        curve[i, 0:3] = np.matmul(M, T).T

    return curve

def evalCurveTime(M, t):
    T = generateT(t)
    curve_point = np.matmul(M, T)
    return curve_point

def matricesCRCurve(points):
    control_points = [points[0], points[len(points)-1]]
    matrices = []

    for i in range(len(points)-3):
        Mcr = catmullRomMatrix(points[i], points[i+1], points[i+2], points[i+3])
        matrices += [Mcr]
    return matrices

def evalCRCurveTime(t, matrices, times):
    N_curves = len(matrices)
    curve_point = evalCurveTime(matrices[0], t)

    for i in range(N_curves):
        if times[i] <= t <= times[i+1]:
            normalized_t = (t-times[i]) / (times[i+1]-times[i])
            curve_point = evalCurveTime(matrices[i], normalized_t)

    return curve_point

if __name__ == "__main__":

    # Number of samples to plot
    N = 500

    # Setting up the matplotlib display for 3D
    fig = plt.figure()
    ax = fig.gca(projection='3d')

    """
    Example for Catmull-Rom curve
    """

    P0 = np.array([[-0.3, -0.25, 1]]).T*25
    P1 = np.array([[-0.5, 0.25, 1.15]]).T*25
    P2 = np.array([[-0.15, 0.15, 1]]).T*25
    P3 = np.array([[0, 0.5, 0.75]]).T*25
    P4 = np.array([[0.15, 0.15, 1]]).T*25
    P5 = np.array([[0.4, 0.25, 1.15]]).T*25
    P6 = np.array([[0.15, -0.15, 1]]).T*25
    P7 = np.array([[0.4, -0.5, 0.75]]).T*25
    P8 = np.array([[0, -0.25, 1]]).T*25
    P9 = np.array([[-0.4, -0.5, 1.15]]).T*25
    P10 = np.array([[-0.25, -0.15, 1]]).T*25
    P11 = np.array([[-0.5, 0.25, 1.15]]).T*25
    P12 = np.array([[-0.15, 0.25, 1]]).T*25

    points = [P0, P1, P2, P3, P4, P5, P6, P7, P8, P9, P10, P11, P12]
    times = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
    matrices = matricesCRCurve(points)

    catmullRomCurve = np.ndarray(shape=(N, 3), dtype=float)
    ts = np.linspace(0.0, len(times)-1, N)

    for i in range(N):
        catmullRomCurve[i, 0:3] = evalCRCurveTime(ts[i], matrices, times).T
        #if 0<= ts[i] <= 1:
        #    T = generateT(ts[i])
        #    catmullRomCurve[i, 0:3] = np.matmul(matrices[0], T).T

        #elif 1 <= ts[i] <= 2:
        #    T = generateT(ts[i]-1)
        #    catmullRomCurve[i, 0:3] = np.matmul(matrices[1], T).T

        #elif 2 <= ts[i] <= 3:
        #    T = generateT(ts[i]-2)
        #    catmullRomCurve[i, 0:3] = np.matmul(matrices[2], T).T

        #elif 3 <= ts[i] <= 4:
        #    T = generateT(ts[i]-3)
        #    catmullRomCurve[i, 0:3] = np.matmul(matrices[3], T).T

    plotCurve(ax, catmullRomCurve, "Catmull-Rom curve")

    ax.set_xlabel('x')
    ax.set_ylabel('y')
    ax.set_zlabel('z')
    ax.legend()

    plt.show()


import numpy as np

def gaussian2d(shape=(3,3),sigma=0.5):
    """
    2D gaussian mask - should give the same result as MATLAB's
    fspecial('gaussian',[shape],[sigma])
    """
    m,n = [(ss-1.)/2. for ss in shape]
    y,x = np.ogrid[-m:m+1,-n:n+1]
    h = np.exp( -(x*x + y*y) / (2.*sigma*sigma) )
    h[ h < np.finfo(h.dtype).eps*h.max() ] = 0
    sumh = h.sum()
    if sumh != 0:
        h /= sumh
    return h


import numpy as np
import matplotlib.pyplot as plt

# Discrete Fourier Transform (1D)
def DFT1D(f):
  n = f.shape[0]
  F = np.zeros(f.shape, dtype=np.complex64)
  # For each frequency u=0..n-1
  for u in np.arange(n):
    # For each element of `f` x=0..n-1
    for x in np.arange(n):
      F[u] += f[x] * np.exp(-1j * (2 * np.pi) * (u * x) / n)
  # Normalize the fourer
  return F / np.sqrt(n)

# Faster Discrete Fourier Transform (1D)
def fasterDFT1D(f):
  n = f.shape[0]
  F = np.zeros(f.shape, dtype=np.complex64)
  x = np.arange(n)
  # For each frequency u=0..n-1
  for u in np.arange(n):
    F[u] += f[x] * np.exp(-1j * (2 * np.pi) * (u * x) / n)
  # Normalize the fourer
  return F / np.sqrt(n)

# Define a signal
t = np.arange(0, 1, 0.005)
f = 1 * np.sin(t * (2 * np.pi) * 2) + 0.6 * np.cos(t * (2 * np.pi) * 8) + \
    0.4 * np.cos(t * (2 * np.pi) * 16)

# Computing DFT1D of `f`
F = DFT1D(f)

fq = np.arange(200)
plt.figure(figsize=(10, 4))

# Plot the magnitude of the DFT
plt.plot(fq, np.abs(F), 'r')
plt.savefig('dft_magnitude.png')

# Due to the symmetric property of the sine and cosine functions (which are
# similar in shape, but shifted), the Fourier Transform is also symmetric with
# respect to its central coefficient. One interpretation is that both the
# positive and negative frequency sinusoids are 90 degrees out of phase, but
# the magnitude of their response will be the same. In other words, they both
# respond to real signals in the same way.

# Plot only part of the frequencies
limit = 32

fq = np.arange(limit)
plt.figure(figsize=(10, 4))

# Plot the magnitude of the DFT
plt.plot(fq, np.abs(F[:limit]), 'r')
plt.savefig('dft_magnitude_limit.png')


#Import the Libraries
import numpy as np
import matplotlib.pyplot as plt
#%matplotlib inline

#Function to genertae Loss Function (y)
def gen_y(a):
    u=np.sin(a/10)*(5*(np.cos(a+10))-np.cos(a/2000))
    return u+20*np.sin(a)

#The derivative function
def der(a):
    u=(np.sin(a/10)*((np.sin(a/2000)/2000)-5*np.sin(a/10)))
    v=(np.cos(a/10)*(5*np.cos(a+10)- np.cos(a/200)))/10
    return u + v + (20*np.cos(a))

def Gradient_Descent(W,W_prev=0,eta=0.004,tol=0.004,epochs=1):
    #Base Condition, from Equation 2
    if(W-W_prev<tol):
        print(f'Returning after {epochs} number of epochs')
        return W
    # We calculate the gradient value at W
    g=der(W)
    # We memorize the the W
    W_prev=W
    # We update the weight with the help of the previous one
    # eta is the learning rate and tol is the tolerance
    W=W_prev-eta*g
    #Itertaive Process and we also count the number of epochs
    return Gradient_Descent(W,W_prev,epochs=epochs+1)

##################################  MAIN #######################################################

#Generate loss Function
x=np.linspace(15,25,500)
y=gen_y(x)

#Visulaize the plot
fig,ax=plt.subplots(nrows=1,ncols=2,figsize=(14,8))
ax[0].plot(x,y,color='r',linewidth=3)
ax[0].set_xlabel('Weight',color='g',fontsize=20)
ax[0].set_ylabel('Loss',color='r',fontsize=20)
ax[0].set_title('Loss Function',fontsize=25,color='b')

#Choose a random starting point and use the gradient descent function to reach the minima
start=15.25
best_weight=np.round(Gradient_Descent(start),2)
print(f'The optimal weight is {best_weight}')
#To plot iterations
x_plot=np.linspace(start+0.1,best_weight-0.3,152)

##A Visual Plot of our excercise
ax[1].plot(x,y,color='r',linewidth=3)
ax[1].scatter(best_weight,gen_y(best_weight),linewidth=16,label='Optimal Wight',color='blue')
ax[1].scatter(start,gen_y(start),linewidth=16,label='Start',color='orange')
ax[1].scatter(x_plot,gen_y(x_plot),linewidth=16,label='Iteration',color='#FDDF00',alpha=0.3)
ax[1].set_xlabel('Weight',color='g',fontsize=20)
ax[1].set_ylabel('Loss',color='r',fontsize=20)
ax[1].set_title('Finding Minima with Gradient Descent',fontsize=25,color='b')
ax[1].legend(markerscale=0.2)
plt.tight_layout()


import math

import numpy as np
import utils

def execute(data, max_iterations, clusters, epsilon):
    # Number of features
    features = len(data[0])
    # Number of rows
    rows = len(data)
    # create the first membership matrix
    u_matrix = []
    # fuzzy constant
    fc = 2.00

    # create the membershipo metric using random numbers
    for i in range(rows):
        vals = np.random.rand(clusters)
        s = sum(vals)
        memb = [v / s for v in vals]
        u_matrix.append(memb)

    # iteration counter
    iteration = 0
    cntrs = []
    old_cntrs = []

    # Check if the number of iterations has reached the maximum or if the iterations have converged
    while (iteration < max_iterations) and (check_iteration(old_cntrs, cntrs, epsilon)):
        old_cntrs = cntrs
        # calcualte the Centroids
        cntrs = get_centroids(data, u_matrix, clusters, fc)
        # calculate the membership matrix
        u_matrix = calculate_membership_matrix(data, cntrs, u_matrix, fc)
        iteration += 1

    return u_matrix, cntrs

# Check if the iterations have converged
def check_iteration(c1, c2, epsilon):
    if not c1 or not c2:
        return True
    distances = [utils.distance(c1[i], c2[i]) for i in range(len(c1))]
    return not (np.amax(distances) < epsilon)

def get_centroids(data, membership, clusters, fuzzy_constant):
    centroids = []
    # Separate the memberships by cluster number
    mem = zip(*membership)

    for i in range(clusters):
        #  The cluster i memberships for every row
        mm = mem[i]
        # elevate every membership by the fuzzy constant
        mm_prod = [u ** fuzzy_constant for u in mm]
        denominator = sum(mm_prod)

        numerator = []
        # In every column calculate the ith centroid
        for j in range(len(data[0])):
            temp = []
            for k in range(len(data)):
                # uij**m * xi
                temp.append(mm_prod[k] * data[k][j])
            numerator.append(temp)

        numerator = [sum(x) for x in numerator]
        centroid = []
        # Calculate the centroid
        for n in numerator:
            centroid.append(n / denominator)
        centroids.append(centroid)

    return centroids

# Calculate a new membership function using the centroids
def calculate_membership_matrix(data, centroids, membership, fuzzy_constant):
    u_matrix = membership
    fc = 2 / (fuzzy_constant - 1)
    for i in range(len(data)):
        # Calculate the distance of the row with the centroids
        distances = [utils.distance(data[i], centroids[k]) for k in range(len(centroids))]
        for k in range(len(centroids)):
            den = sum([math.pow(float(distances[k] / distances[c]), fc) for c in range(len(centroids))])
            u_matrix[i][k] = float(1 / den)
    return u_matrix

# Calculate which cluster is the one with the highest membership for every row
def get_labels(rows, membership):
    labels = []
    for i in range(rows):
        max_membership = -1
        max_mem_index = -1
        for j in range(len(membership[i])):
            if membership[i][j] > max_membership:
                max_membership = membership[i][j]
                max_mem_index = j
        labels.append(max_mem_index)
    return labels