from mpl_toolkits.mplot3d import Axes3D
import scipy.io as sio
import matplotlib.image as img
import matplotlib.pyplot as plt
import matplotlib.gridspec as gridspec
import numpy as np
from numpy.linalg import svd

def feature_normalize(X):
    means = np.mean(X, axis=0)
    X_norm = X - means
    stds = np.std(X_norm, axis=0)
    X_norm = X_norm / stds

    return means, stds, X_norm

def compute_covariance_matrix(X):
    return X.T.dot(X) / X.shape[0]

def pca(X):
    covariance_matrix = compute_covariance_matrix(X)
    U, S, V = svd(covariance_matrix, full_matrices=True, compute_uv=True)

    return U, S

def project_data(X, U, K):
    return X.dot(U[:, :K])

def recover_data(Z, U, K):
    return Z.dot(U[:, :K].T)

def grid_plot(X, dim):
    fig = plt.figure(figsize=(6, 6))
    M, N = X.shape

    gs = gridspec.GridSpec(dim, dim)
    gs.update(bottom=0.01, top=0.99, left=0.01, right=0.99,
              hspace=0.05, wspace=0.05)

    k = 0
    for i in range(dim):
        for j in range(dim):
            ax = plt.subplot(gs[i, j])
            ax.axis('off')
            ax.imshow(-X[k].reshape(int(np.sqrt(N)), int(np.sqrt(N))).T,
                      cmap=plt.get_cmap('Greys'),  # vmin=-1, vmax=1,
                      interpolation='nearest')  # ,alpha = 1.0)
            k += 1

    plt.show()

def main():
    # 1
    file_path = 'ex7data1.mat'
    dataset = sio.loadmat(file_path)
    X = dataset["X"]

    # 2
    # X[:, 0] - first column
    plt.scatter(X[:, 0], X[:, 1], marker="o")
    plt.show()

    # 3
    covariance_matrix = compute_covariance_matrix(X)
    print(covariance_matrix)

    # 4
    # Feature normalize
    # mu, sigma
    means, stds, X_norm = feature_normalize(X)
    # Run SVD
    U, S = pca(X_norm)
    print(U, S)

    # 5
    #  Draw the eigenvectors centered at mean of data. These lines show the
    #  directions of maximum variations in the dataset.
    fig, ax = plt.subplots()
    ax.plot(X[:, 0], X[:, 1], 'o', mew=0.25)

    for i in range(len(S)):
        ax.arrow(means[0], means[1], 1.5 * S[i] * U[0, i], 1.5 * S[i] * U[1, i],
                 head_width=0.25, head_length=0.2, fc='k', ec='k', lw=2, zorder=1000)

    ax.axis([0.5, 6.5, 2, 8])
    ax.set_aspect('equal')
    ax.grid(False)

    plt.show()

    print('Top principal component: U[:, 0] = [{:.6f} {:.6f}]'.format(U[0, 0], U[1, 0]))
    print(' (you should expect to see [-0.707107 -0.707107])')

    # 6
    # Project the data onto K = 1 dimension
    K = 1
    Z = project_data(X_norm, U, K)
    print('Projection of the first example: {:.6f}'.format(Z[0, 0]))
    print('(this value should be about    : 1.481274)')

    # 7
    X_rec = recover_data(Z, U, K)
    print('Approximation of the first example: [{:.6f} {:.6f}]'.format(X_rec[0, 0], X_rec[0, 1]))
    print('       (this value should be about  [-1.047419 -1.047419])')

    # 8
    plt.figure(figsize=(6, 6))
    plt.plot(X_norm.T[0], X_norm.T[1], 'bo', mfc='none', mec='b', ms=8, label='Original Data Points')
    plt.plot(X_rec.T[0], X_rec.T[1], 'ro', mfc='none', mec='r', ms=8, label='PCA Reduced Data Points')
    plt.xlabel('x1', fontsize=14)
    plt.ylabel('x2', fontsize=14)
    plt.legend(loc=4)
    for (x, y), (x_rec, y_rec) in zip(X_norm, X_rec):
        plt.plot([x, x_rec], [y, y_rec], 'k--', lw=1)
    plt.xlim(-4, 3)
    plt.ylim(-4, 3)
    plt.show()

    # 9
    file_path = 'ex7faces.mat'
    dataset = sio.loadmat(file_path)
    X = dataset['X']

    # 10
    grid_plot(X, 10)

    # 11
    # mu, sigma
    means, stds, X_norm = feature_normalize(X)
    U, S = pca(X_norm)

    # 12, 13
    grid_plot(U.T, 6)

    # 14-15
    grid_plot(U.T, 10)

    # 16
    A = img.imread('output.jpg')
    X = A.reshape(-1, 3)

    # 17
    # Sample 1000 random indexes (since working with all the data is
    #  too expensive. If you have a fast computer, you may increase this.
    sel = np.random.choice(X.shape[0], size=1000)

    fig = plt.figure(figsize=(6, 6))
    ax = fig.add_subplot(111, projection='3d')
    idx = np.loadtxt('output.txt')
    ax.scatter(X[sel, 0], X[sel, 1], X[sel, 2], cmap='rainbow', c=idx[sel], s=10)
    ax.set_title('Pixel dataset plotted in 3D.\nColor shows centroid memberships')

    plt.show()

    # 18
    mu, sigma, X_norm = feature_normalize(X)

    # PCA and project the data to 2D
    U, S = pca(X_norm)
    Z = project_data(X_norm, U, 2)

    fig = plt.figure(figsize=(6, 6))
    ax = fig.add_subplot(111)

    ax.scatter(Z[sel, 0], Z[sel, 1], cmap='rainbow', c=idx[sel], s=32)
    ax.set_title('Pixel dataset plotted in 2D, using PCA for dimensionality reduction')
    ax.grid(False)

    plt.show()

if __name__ == '__main__':
    main()


import numpy as np

"""
This module calculates the mean value and smoothes the data.

"""
### Running mean/Moving average
def running_mean(l, N):
    sum = 0
    result = list( 0 for x in l)

    for i in range( 0, N ):
        sum = sum + l[i]
        result[i] = sum / (i+1)

    for i in range( N, len(l) ):
        sum = sum - l[i-N] + l[i]
        result[i] = sum / N

    return result

### Neighbor Mean/Moving Average
def neighbor_mean(input_list, N):
    """
    inputs:
        input_list --- []
        N --- number of neighbors to be averaged
    outputs:
        result --- []
    """
    if  type(input_list) != list:
        raise TypeError("The input list must be a list!")

    if type(N) != int:
        raise TypeError("The input type must be an int!")

    result = []
    radius = (N-1)/2

    for i in range(len(input_list)):

        if (i-radius)<0:
            result.append(np.mean(input_list[:i+1]))

        elif (i+radius)>len(input_list)-1:
            result.append(np.mean(input_list[i:]))

        else:
            temp = []
            for j in range(i-radius, i+radius+1):
                temp.append(input_list[j])
            result.append(np.mean(temp))

    return result

if __name__== "__main__":
    print "This is a test!"
    x = [i for i in range(-10,11)]
    a = [i**2 for i in range(-10,11)]
    print a
    sa = neighbor_mean(a,5)
    print sa
    import matplotlib.pyplot as plt
    plt.figure(0)
    plt.plot(x,a,'r',x,sa,'b')
    plt.show()


# softmax(yi) = exp(yi) / sum_j(exp(yj))

import numpy
import math

def softmax(x):
	array = numpy.transpose(x)
	shape = array.shape
	if len(shape) == 1:
		print "1D Array"
		temp = []
		for value in x:
			temp.append(math.exp(value))

		sum_of_x = sum(temp)

		result = []
		for value in temp:
			result.append(value / sum_of_x)

		return result
	elif len(shape) == 2:
		print "2D Array"
		rows = shape[0]
		cols = shape[1]
		result = []
		for row in range(rows):
			temp1 = []
			for col in range(cols):
				temp1.append(math.exp(x[col][row]))

			sum_of_x = sum(temp1)

			temp2 = []
			for value in temp1:
				temp2.append(value / sum_of_x)

			result.append(temp2)
		return numpy.transpose(result)
	else:
		return None

# Or using numpy more effectively:
def softmax2(x):
	return numpy.exp(x) / numpy.sum(numpy.exp(x), axis=0)

scores = [1.0, 2.0, 3.0]

print softmax2(scores), "\n"

scores = [[1, 2, 3, 6], [2, 4, 5, 6], [3, 8, 7, 6]]

print softmax2(scores)


import numpy as np

def exponential(o1, o2):
    return np.linalg.norm(o1 - o2)

def hotaling_matrix(b, dtype=np.double):
    H = np.array([[(b + 1) / 2, (b - 1) / 2],
                  [(b - 1) / 2, (b + 1) / 2]], dtype=dtype)
    return H

def hotaling(o1, o2, b):
    H = hotaling_matrix(b)
    return (o2 - o1) @ H @ (o2 - o1).T

def hotaling_S_from_D(D, phi1, phi2):
    return np.eye(D.shape[0]) - phi2 * np.exp(-phi1 * D ** 2)

def hotaling_D(M, b):
    D = np.zeros((M.shape[0], M.shape[0]))
    for i in range(M.shape[0]):
        for j in range(M.shape[0]):
            D[i][j] = hotaling(M[i], M[j], b)
    return D

def hotaling_S(M, phi1, phi2, b):
    D = hotaling_D(M, b)
    return hotaling_S_from_D(D, phi1, phi2)


import numpy as np
import matplotlib.pyplot as plt

f_wave = 2 #actual frequency of the wave
omega = 2*np.pi*f_wave
t = np.arange(0, 2, 0.001)

#signal = np.sin(omega*t)
signal = np.cos(2*omega*t)+np.sin(10*omega*t)

# plt.plot(t, signal, color='b')
# plt.savefig('Signal_Wrap/cos2+sin10.png')

min_freq = 0
max_freq = 10

f_list = np.arange(min_freq, max_freq, 0.2)

for f_idx, f in enumerate(f_list):

	r, theta = [], []

	for idx in range(len(t)):
		r.append(signal[idx])
		theta.append(2*np.pi*f*t[idx])

	x, y = [], []

	for idx in range(len(t)):
		x.append(r[idx]*np.cos(theta[idx]))
		y.append(r[idx]*np.sin(theta[idx]))

	plt.figure(figsize=(5,5))
	plt.xlim(-2.1,2.1)
	plt.ylim(-2.1,2.1)
	plt.plot(x, y, color='r')
	plt.title("Frequency = "+str(round(f_list[f_idx],1))+" Hz", size=15)
	plt.savefig('signal_wrap_%d.png'%f_idx)
	plt.clf()

#Convert to GIF in order
#convert -delay 30 signal_wrap_{0..49..1}.png sine.gif


import numpy as np
from scipy.linalg import eig
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D # <--- This is important for 3d plotting
# from mpl_toolkits import mplot3d

if __name__ == "__main__":

    X = np.loadtxt('data-dimred-X.csv', dtype=np.float, delimiter=',')
    y = np.loadtxt('data-dimred-y.csv', dtype=np.float)

    # X = X.T
    X = X-np.expand_dims(X.mean(axis=1),1)

    C = np.dot(X,X.T)/y.shape[0]
    l, U = eig(C)
    sorted_idxs = np.argsort(np.real(l))[::-1]
    l, U = l[sorted_idxs], U[:, sorted_idxs]

    # 2D PCA Projections
    dim = 2
    X_trans = np.dot(U.T[0:dim], X)

    plt.scatter(X_trans[0, y == 1], X_trans[1, y == 1], c='red', label='Class 1', alpha=0.75)
    plt.scatter(X_trans[0, y == 2], X_trans[1, y == 2], c='blue', label='Class 2', alpha=0.75)
    plt.scatter(X_trans[0, y == 3], X_trans[1, y == 3], c='green', label='Class 3', alpha=0.75)
    plt.title("PCA for dimension = 2 for 'data-dimred-X.csv'")
    plt.savefig("PCA_dim2.pdf", facecolor='w', edgecolor='w', papertype=None, format='pdf',
                transparent=False, bbox_inches='tight', pad_inches=0.1)
    plt.legend()
    plt.show()

    # 3D PCA Projections
    dim = 3
    X_trans = np.real(np.dot(U.T[0:dim], X))
    # print(X_trans)
    fig = plt.figure()
    ax = plt.axes(projection = '3d')
    ax.scatter(X_trans[0, y == 1], X_trans[1, y == 1], X_trans[2, y == 1], c='red', label='Class 1', alpha=0.75)
    ax.scatter(X_trans[0, y == 2], X_trans[1, y == 2], X_trans[2, y == 2], c='blue', label='Class 2', alpha=0.75)
    ax.scatter(X_trans[0, y == 3], X_trans[1, y == 3], X_trans[2, y == 3], c='green', label='Class 3', alpha=0.75)
    ax.set_title("PCA for dimension = 3 for 'data-dimred-X.csv'")
    plt.savefig("PCA_dim3.pdf", facecolor='w', edgecolor='w', papertype=None, format='pdf',
                transparent=False, bbox_inches='tight', pad_inches=0.1)
    ax.legend()
    plt.show()


print("-----Gauss-Jordan Elimination-----")

## in this method we transform our nxn matrix to a identity matrix and then don't need to do back substitution

import numpy as np

a=np.array([[1,3,2],
            [2,7,7],
            [2,5,2]],float)
b=np.array([2,-1,7],float)
# a= np.array([[1,3,-2,4],
#              [2,6,0,5],
#              [4,11,8,0],
#              [1,3,2,1]],float)
# b=np.array([7,5,7,-2],float)
n= len(b)

## if there is zero or negligible element on the main diagonal, then code for interchanging the rows
for k in range(n):
    if abs(a[k,k])<1.0e-10:
        for i in range(k+1,n):
            if abs(a[i,k])>abs(a[k,k]): # We can also write abs(a[k,k])=1.0e-10 because we are looking for non zero element on the main diagonal
                a[[k,i]]=a[[i,k]] ## changeing rows of matrix
                b[i,k]=b[k,i]## changeing elements in the RHS constant vector
                ##Or
                # for j in range(k,n):
                #     a[k,j],a[i,j]=a[i,j],a[k,j]
                # b[k],b[i]=b[i],b[k]

                break

    for l in range(n):
        a[k,:]=a[k,:]/a[k,k]
    b[k]=b[k]/a[k,k]

    # #Or
    # for k in range(n):
    #     pivot= a[k,k]
        # for l in range(n):
        #     a[k,l]=a[k,l]/pivot
        # b[k]=b[k]/pivot

    for i in range(n):
        # b[k],b[i]=b[i],b[k]
        if i==k or a[i,k]==0: ##we skip that row if there is zero and also i==k element(diagonal element) because we have already set that to 1.
            continue

        factor =a[i,k] ## we have to predefine the factor, because in every iteration it changes.(although sometimes it doesn't matter)
        b[i] -= b[k]*factor ## we can define this vector also after the j loop.
        for j in range(k,n):
            a[i,j]-= factor * a[k,j]

print("Transformed Identity matrix: ")
print(a)

print("The solution vector is: ")
print(b)


#!/usr/bin/env python -W ignore
import numpy as np
from scipy import integrate
import matplotlib
import matplotlib.pyplot as plt
from math import pi
from typing import Callable, List

def scalar_product_int(f1: Callable[[float], float], f2: Callable[[float], float],
    a: float, b: float, rho: Callable[[float], float]=lambda x: 1) -> float:
    """
    :param f1: first function
    :param f2: second function
    :param a: left endpoint of the integration interval
    :param b: right endpoint of the integration interval
    :return: scalar product of f1 and f2 in L2 on [a, b]
    """
    return integrate.quad(lambda x: f1(x) * f2(x) * rho(x), a, b)[0]

def root_mean_square_approximation(n: int, a0: float, b0: float,
    f: Callable[[float], float], phi: Callable[[int, float], float],
    plot: bool, rho: Callable[[float], float]=lambda x: 1) -> Callable[[float], float]:
    """
    :param n: "degree" of approximation polynomial
    :param a0: left endpoint of approximation interval
    :param b0: right endpoint of approvimation interval
    :param f: function to approximate
    :param phi: system of functions which are used in approximation
    :param plot: whether to plot the true and approximation functions
    :return: approximation function
    """
    b = np.array([
        scalar_product_int(f, lambda x: phi(i, x), a0, b0, rho)
        for i in range(n)
    ])

    a = np.array([
        [
            scalar_product_int(
                lambda x: phi(i, x),
                lambda x: phi(j, x),
                a0, b0, rho
            )
            for j in range(n)
        ]
        for i in range(n)
    ])

    c = np.linalg.solve(a, b)

    def f_sol(x):
        return np.dot(c, np.array([phi(k, x) for k in range(n)]))

    if plot:
        x = np.linspace(a0, b0, 100)
        plt.figure(figsize=(20, 10))
        plt.plot(x, f_sol(x), 'r--', label='Approximation function')
        plt.plot(x, f(x), 'b.', label='True function')
        plt.xlabel('x')
        plt.ylabel('y')
        plt.ylim((3,11))
        plt.legend()
        plt.grid(True)

    return f_sol

def root_mean_square_approximation_polinomial(n: int, a0: float, b0: float,
    f: Callable[[float], float], plot: bool) -> Callable[[float], float]:
    """
    :param n: "degree" of approximation polynomial
    :param a0: left endpoint of approximation interval
    :param b0: right endpoint of approvimation interval
    :param f: function to approximate
    :param plot: whether to plot the true and approximation functions
    :return: approximation function
    """
    def phi(i: int, x: float) -> float:
        return x**i

    if plot:
        plt.title("Polynomial approximation")

    f_sol = root_mean_square_approximation(n + 1, a0, b0, f, phi, plot)

    if plot:
        plt.savefig('../tex/polynomial.png', bbox_inches='tight')

    print(f'Polynomial approximation error = '
        f'{integrate.quad(lambda x: (f_sol(x) - f(x))**2, a0, b0)[0]}')

    return f_sol

def rescale_function(old_a: float, old_b: float, f: Callable[[float], float],
    new_a: float, new_b: float) -> Callable[[float], float]:
    """
    :param old_a: old leftpoint of rescaling interval
    :param old_b: right leftpoint of rescaling interval
    :param f: function to rescale
    :param new_a: old leftpoint of rescaled interval
    :param new_b: right leftpoint of rescaled interval
    :return: rescaled function
    """
    return lambda x: f((old_b - old_a) / (new_b - new_a) * (x - new_a) + old_a)

def root_mean_square_approximation_trigonometric(n: int, a0: float, b0: float,
    f: Callable[[float], float], plot: bool) -> Callable[[float], float]:
    """
    :param n: "degree" of approximation polynomial
    :param a0: left endpoint of approximation interval
    :param b0: right endpoint of approvimation interval
    :param f: function to approximate
    :param plot: whether to plot the true and approximation functions
    :return: approximation function
    """
    old_a, old_b = a0, b0
    a0, b0, f = -pi, pi, rescale_function(a0, b0, f, -pi, pi)

    def phi(i: int, x: float) -> float:
        return (np.sin if (i & 1) else np.cos)(((i + 1) >> 1) * x)

    if plot:
        plt.title("Trigonometric approximation")

    f_sol = root_mean_square_approximation((n << 1) + 1, a0, b0, f, phi, plot)

    if plot:
        plt.savefig('../tex/trigonometric.png', bbox_inches='tight')

    print(f'Trigonometric approximation error on [-pi, pi] = '
        f'{integrate.quad(lambda x: (f_sol(x) - f(x))**2, a0, b0)[0]}')

    f = rescale_function(-pi, pi, f, old_a, old_b)
    f_sol = rescale_function(-pi, pi, f_sol, old_a, old_b)

    print(f'Trigonometric approximation error on [{old_a}, {old_b}] = '
        f'{integrate.quad(lambda x: (f_sol(x) - f(x))**2, old_a, old_b)[0]}')

    return f_sol

def root_mean_square_approximation_exponent(n: int, a0: float, b0: float,
    f: Callable[[float], float], plot: bool) -> Callable[[float], float]:
    """
    :param n: "degree" of approximation polynomial
    :param a0: left endpoint of approximation interval
    :param b0: right endpoint of approvimation interval
    :param f: function to approximate
    :param plot: whether to plot the true and approximation functions
    :return: approximation function
    """
    def phi(i: int, x: float) -> float:
        return np.exp((((i if (i & 1) else -i) + 1) >> 1) * x)

    if plot:
        plt.title("Exponential approximation")

    f_sol = root_mean_square_approximation((n << 1) + 1, a0, b0, f, phi, plot)

    if plot:
        plt.savefig('../tex/exponent.png', bbox_inches='tight')

    print(f'Exponential approximation error = '
        f'{integrate.quad(lambda x: (f_sol(x) - f(x))**2, a0, b0)[0]}')

    return f_sol

def root_mean_square_approximation_chebyshev(n: int, a0: float, b0: float,
    f: Callable[[float], float], plot: bool) -> Callable[[float], float]:
    """
    :param n: "degree" of approximation polynomial
    :param a0: left endpoint of approximation interval
    :param b0: right endpoint of approvimation interval
    :param f: function to approximate
    :param plot: whether to plot the true and approximation functions
    :return: approximation function
    """
    a0, b0, f = -1, 1, rescale_function(a0, b0, f, -1, 1)

    def phi(i: int, x: float) -> float:
        return np.cos(i * np.arccos(x))

    if plot:
        plt.title("Chebyshev approximation")

    f_sol = root_mean_square_approximation(n + 1, a0, b0, f, phi, plot,
        lambda x: 1 / np.sqrt(1 - x**2))

    if plot:
        plt.savefig('../tex/chebyshev.png', bbox_inches='tight')

    print(f'Chebyshev approximation error = '
        f'{integrate.quad(lambda x: (f_sol(x) - f(x))**2, a0, b0)[0]}')

    return f_sol

def scalar_product_discrete(f1: Callable[[float], float], f2: Callable[[float], float],
    x: List[float]) -> float:
    """
    :param f1: first function
    :param f2: second function
    :param x: list of nodes
    :return: scalar product of f1 and f2 over x
    """
    return np.dot(list(map(f1, x)), list(map(f2, x))) / len(x)

def root_mean_square_approximation_polinomial_discrete(m: int, a0: float,
    b0: float) -> Callable[[float], float]:
    x = np.linspace(a0, b0, m + 1)

    cost = [np.inf for _ in range(15)]
    for n in range(1, 15):
        b = np.array([
            scalar_product_discrete(f, lambda x: x**i, x)
        for i in range(n + 1)])

        a = np.array([[
            scalar_product_discrete(
                lambda x: x**j,
                lambda x: x**i,
                x) for j in range(n + 1)
        ] for i in range(n + 1)])

        c = np.linalg.solve(a, b)

        def f_sol(x):
            s = 0
            x_pow = 1
            for k in range(n + 1):
                s += c[k] * x_pow
                x_pow *= x
            return s

        def diff(x):
            return f(x) - f_sol(x)

        cost[n] = scalar_product_discrete(diff, diff, x) / (m - n)

        print(f'n = {n}, cost = {cost[n]}')

    # true value of n
    n = 8  # cost.index(min(cost))

    b = np.array([
        scalar_product_discrete(f, lambda x: x**i, x)
    for i in range(n + 1)])

    a = np.array([[
        scalar_product_discrete(
            lambda x: x**j,
            lambda x: x**i,
            x) for j in range(n + 1)
    ] for i in range(n + 1)])

    c = np.linalg.solve(a, b)

    def f_sol(x):
        s = 0
        x_pow = 1
        for k in range(n + 1):
            s += c[k] * x_pow
            x_pow *= x
        return s

    def diff(x):
        return f(x) - f_sol(x)

    x = np.linspace(a0, b0, 100)
    plt.figure(figsize=(20, 10))
    plt.plot(x, f_sol(x), 'r--', label='Approximation function')
    plt.plot(x, f(x), 'b.', label='True function')
    plt.xlabel('x')
    plt.ylabel('y')
    plt.title("Polynomial discrete interpolation")
    plt.ylim((3,11))
    plt.legend()
    plt.grid(True)
    plt.savefig('../tex/discrete.png', bbox_inches='tight')

    return f_sol

def spline_interpolation(n: int, a0: float, b0: float,
    f: Callable[[float], float]) -> None:
    x = np.linspace(a0, b0, n + 1)
    h = x[1:] - x[:-1]

    rho = 3e+2
    R = rho * np.eye(n + 1)

    H = np.zeros((n + 1, n + 1))

    for i in range(1, n):
        H[i][i - 1] = 1 / h[i - 1]
        H[i][i] = - (1 / h[i - 1] + 1 / h[i])
        H[i][i + 1] = 1 / h[i]

    a = np.zeros((n + 1, n + 1))
    b = np.zeros(n + 1)

    for i in range(n + 1):
        if i == 0 or i == n:
            a[i, i] = 1
        else:
            a[i, i - 1] = h[i - 1] / 6
            a[i, i] = (h[i - 1] + h[i]) / 3
            a[i, i + 1] = h[i] / 6
            b[i] = (f(x[i + 1]) - f(x[i])) / h[i] - (f(x[i]) - f(x[i - 1])) / h[i - 1]

    m = np.linalg.solve(a + np.dot(H, np.dot(np.linalg.inv(R), H.T)), b)

    f_u = f(x) - np.dot(np.linalg.inv(R), np.dot(H.T, m).T)

    def f_sol(x0: float) -> float:
        for i in range(n):
            if x[i + 1] >= x0 >= x[i]:
                return m[i] * ((x[i + 1] - x0)**3) / (6 * h[i]) + \
                    m[i + 1] * ((x0 - x[i])**3) / (6 * h[i]) + \
                    (f_u[i] - m[i] * h[i]**2 / 6) * (x[i + 1] - x0) / h[i] + \
                    (f_u[i + 1] - m[i + 1] * h[i]**2 / 6) * (x0 - x[i]) / h[i]

    x1 = np.linspace(a0, b0, 100 + 1)
    y = [f_sol(xi) for xi in x1]
    plt.figure(figsize=(20, 10))
    plt.plot(x1, y, 'r--', label='Approximation function')
    plt.plot(x1, f(x1), 'b.', label='True function')
    plt.xlabel('x')
    plt.ylabel('y')
    plt.title(f"Spline interpolation, $\\rho_i = {rho}$")
    plt.ylim((3,11))
    plt.legend()
    plt.grid(True)
    plt.savefig(f'../tex/spline_{rho}.png', bbox_inches='tight')

    print(f'Spline approximation error = '
        f'{integrate.quad(lambda x: (f_sol(x) - f(x))**2, a0, b0)[0]}')

n, m, a0, b0 = 4, 20, -10, 10

def f(x: float) -> float:
    return (abs(x + 4) + abs(x - 4)) / 2

matplotlib.rcParams.update({'font.size': 20})

# root_mean_square_approximation_polinomial(n, a0, b0, f, True)

# root_mean_square_approximation_exponent(n, a0, b0, f, True)

root_mean_square_approximation_trigonometric(n, a0, b0, f, True)

# root_mean_square_approximation_chebyshev(n, a0, b0, f, True)

# root_mean_square_approximation_polinomial_discrete(m, a0, b0)

# spline_interpolation(m, a0, b0, f)


"""Linear Softmax Classifier."""
# pylint: disable=invalid-name
import numpy as np

from .linear_classifier import LinearClassifier

def cross_entropy_loss_naive(W, X, y, reg):
    """
    Cross-entropy loss function, naive implementation (with loops)

    Inputs have dimension D, there are C classes, and we operate on minibatches
    of N examples.

    Inputs:
    - W: A numpy array of shape (D, C) containing weights.
    - X: A numpy array of shape (N, D) containing a minibatch of data.
    - y: A numpy array of shape (N,) containing training labels; y[i] = c means
      that X[i] has label c, where 0 <= c < C.
    - reg: (float) regularization strength

    Returns a tuple of:
    - loss as single float
    - gradient with respect to weights W; an array of same shape as W
    """
    # pylint: disable=too-many-locals
    # Initialize the loss and gradient to zero.
    loss = 0.0
    dW = np.zeros_like(W)

    ############################################################################
    # TODO: Compute the cross-entropy loss and its gradient using explicit     #
    # loops. Store the loss in loss and the gradient in dW. If you are not     #
    # careful here, it is easy to run into numeric instability. Don't forget   #
    # the regularization!                                                      #
    ############################################################################

    scores = X.dot(W)
    num_train = X.shape[0]
    num_classes = W.shape[1]

    # Softmax Loss
    for i in range(num_train):
        f = scores[i] - np.max(scores[i]) # avoid numerical instability
        softmax = np.exp(f)/np.sum(np.exp(f))
        loss += -np.log(softmax[y[i]])
        # Weight Gradients
        for j in range(num_classes):
            if j != y[i]:
                dW[:,j] += X[i] * softmax[j]
        dW[:,y[i]] -= X[i]

    # Average
    loss /= num_train
    dW /= num_train

    # Regularization
    loss += 0.5 * reg * np.sum(W * W)
    dW += reg * W
    ############################################################################
    #                          END OF YOUR CODE                                #
    ############################################################################

    return loss, dW

def cross_entropy_loss_vectorized(W, X, y, reg):
    """
    Cross-entropy loss function, vectorized version.

    Inputs and outputs are the same as in cross_entropy_loss_naive.
    """
    # Initialize the loss and gradient to zero.
    loss = 0.0
    dW = np.zeros_like(W)

    ############################################################################
    # TODO: Compute the cross-entropy loss and its gradient without explicit   #
    # loops. Store the loss in loss and the gradient in dW. If you are not     #
    # careful here, it is easy to run into numeric instability. Don't forget   #
    # the regularization!                                                      #
    ############################################################################
    num_train = X.shape[0]
    scores = X.dot(W)

    # Normalize the scores to avoid computational problems with the exponential
    # Normalize with max as zero
    scores = scores - np.max(scores, axis = 1).reshape(-1,1)
    exp_scores = np.exp(scores - np.max(scores, axis=1, keepdims=True))
    probs = exp_scores/np.sum(exp_scores,axis=1,keepdims=True)
    loss = -np.sum(np.log(probs[np.arange(num_train),y]))

    # Divide the loss by the number of trainig examples
    loss /= num_train
    # Add regularization
    loss += 0.5 * reg * np.sum(W * W)

    dprobs = probs
    dprobs[np.arange(num_train),y] -= 1
    dW = X.T.dot(dprobs)
    dW /= num_train

    # Gradient regularization
    dW += reg*W

    ############################################################################
    #                          END OF YOUR CODE                                #
    ############################################################################

    return loss, dW

class SoftmaxClassifier(LinearClassifier):
    """The softmax classifier which uses the cross-entropy loss."""

    def loss(self, X_batch, y_batch, reg):
        return cross_entropy_loss_vectorized(self.W, X_batch, y_batch, reg)

def softmax_hyperparameter_tuning(X_train, y_train, X_val, y_val):
    # results is dictionary mapping tuples of the form
    # (learning_rate, regularization_strength) to tuples of the form
    # (training_accuracy, validation_accuracy). The accuracy is simply the
    # fraction of data points that are correctly classified.
    results = {}
    best_val = -1
    best_softmax = None
    all_classifiers = []
    learning_rates = [1e-7, 5e-7]
    regularization_strengths = [2.5e4, 5e4]

    ############################################################################
    # TODO:                                                                    #
    # Write code that chooses the best hyperparameters by tuning on the        #
    # validation set. For each combination of hyperparameters, train a         #
    # classifier on the training set, compute its accuracy on the training and #
    # validation sets, and  store these numbers in the results dictionary.     #
    # In addition, store the best validation accuracy in best_val and the      #
    # Softmax object that achieves this accuracy in best_softmax.              #
    #                                                                          #
    # Hint: You should use a small value for num_iters as you develop your     #
    # validation code so that the classifiers don't take much time to train;   #
    # once you are confident that your validation code works, you should rerun #
    # the validation code with a larger value for num_iters.                   #
    ############################################################################
    learning_rates = [3e-7, 3e-6]
    regularization_strengths = [1e3, 2e5]

    def get_random_params(lr_stats, reg_stats):
        lr = np.random.uniform(lr_stats[0], lr_stats[1])
        reg = np.random.uniform(reg_stats[0], reg_stats[1])
        return lr, reg

    num_experiment = 20
    for i in range(num_experiment):
        print(f"Trying {i+1:02d}/{num_experiment} subset...", end=" ")
        lr, rg = get_random_params(learning_rates, regularization_strengths)
        softmax = SoftmaxClassifier()
        softmax.train(X_train, y_train, learning_rate=lr, reg=rg, num_iters=1500)

        y_train_pred = softmax.predict(X_train)
        train_accuracy = np.mean(y_train_pred == y_train)

        y_val_pred = softmax.predict(X_val)
        val_accuracy = np.mean(y_val_pred == y_val)

        results[(lr,rg)] = (train_accuracy, val_accuracy)
        if best_val < val_accuracy:
            best_val = val_accuracy
            best_softmax = softmax

        all_classifiers.append((softmax, val_accuracy))
        print("Val Accuracy:", val_accuracy)

    ############################################################################
    #                              END OF YOUR CODE                            #
    ############################################################################

    # Print out results.
    for (lr, reg) in sorted(results):
        train_accuracy, val_accuracy = results[(lr, reg)]
        print('lr %e reg %e train accuracy: %f val accuracy: %f' % (
              lr, reg, train_accuracy, val_accuracy))

    print('best validation accuracy achieved during validation: %f' % best_val)

    return best_softmax, results, all_classifiers


"""
Frequency Change Detection
===============================

This example considers a signal which is a sinusoid whose
frequency changes abruptly at a specific point in time.
Looking at the waveform of the signal, it is very difficult
to identify the time where the frequency change occurs.
However, when we perform the wavelet decomposition of the
signal, we can clearly see the location at which the
discontinuity in frequency occurs.

We will do the following:

- Synthetically construct a sinusoid with two different frequencies
  at different times.
- Examine its Fourier spectrum which cannot reveal this discontinuity.
- Perform multilevel wavelet decomposition.
- Visually inspect the detail coefficients at multiple levels to
  locate the discontinuity
- Algorithmically locate the discontinuity by scanning the detail
  coefficients.
"""

# Configure JAX to work with 64-bit floating point precision.
from jax.config import config
config.update("jax_enable_x64", True)
# %%
# Let's import necessary libraries
import jax.numpy as jnp
# CR-Sparse libraries
import cr.sparse as crs
import cr.sparse.wt as wt
# Utilty functions to construct sinusoids
import cr.sparse.dsp.signals as signals
# Plotting
import matplotlib.pyplot as plt

# %%
# Test signal generation
# ------------------------------

# Sampling frequency in Hz
fs = 100
# Signal duration in seconds
T = 20
# Frequency of first part of signal in Hz
f = 1/8
# Start and end times of first part of signal
start_time = 0
end_time = 12
# Construct the first part of signal
t, x1 = signals.transient_sine_wave(fs, T, f, start_time, end_time)
# Frequency of second part of signal in Hz
f = 1/6
# Start and end times of first part of signal
start_time = end_time
end_time = T
# Adjust the initial phase of second part of signal to be in continuity with the first part
initial_phase=jnp.pi
# Construct the second part of signal
t, x2 = signals.transient_sine_wave(fs, T, f, start_time, end_time, initial_phase=initial_phase)
# Combine the first and second parts of signal
x = x1 + x2
# Overall signal length
n = len(x)
# Plot the parts and combined signal
fig, axs = plt.subplots(3, figsize=(12,12))
axs[0].plot(t,x1)
axs[1].plot(t,x2)
axs[2].plot(t,x)
# %%
# The last plot shows the combined signal with discontinuity at t=12 sec when
# the frequency changes from 1/8 Hz to 1/6 Hz.
# The change is very difficult to notice as well as locate in the time domain plot of the signal.

# %%
# Frequency spectrum
# ------------------------------

# Compute the FFT
f = jnp.fft.fftshift(jnp.fft.fft(x))
# Plot the magnitude
fig, axs = plt.subplots(1, figsize=(12,4))
plt.plot(jnp.abs(f[750:1250]))
# %%
# The frequencies 1/6 and 1/8 Hz are so close that it is difficult to distinguish them
# in the frequency spectrum. Of course, location of the discontinuity cannot be
# identified in the frequency spectrum.

# %%
# Multilevel wavelet decomposition
# ------------------------------

# Compute the multilevel wavelet decomposition
coeffs = wt.wavedec(x, 'db4')
# Total number of decomposition levels
levels = len(coeffs) -1
print(levels)

# %%
# First level detail coefficients
cd1 = coeffs[-1]
# Dyadic upsample them to align with the time values
cd1 = wt.dyadup_in(cd1)
# %%
# Second level detail coefficients
cd2 = coeffs[-2]
# Dyadic upsample them to align with the time values
cd2 = wt.dyadup_in(wt.dyadup_in(cd2))
# Plot the first and second level detail coefficients
fig, axs = plt.subplots(2, figsize=(12,12))
axs[0].plot(t, cd1[:n])
axs[1].plot(t, cd2[:n])
# %%
# The discontinuities are clearly visible in the plots of detail coefficients
# both at first level and second level.
# The detail coefficients have been aligned with the time values.
# Both plots should large coefficients around t=12 sec.
# Large coefficients at the boundary are due to boundary effects in DWT and can be safely ignored.

# %%
# Locate the indices of largest entries (by magnitude) in the detail
# coefficients at first level
idx, values = crs.hard_threshold_sorted(cd1, 8)
print(idx)
# %%
# After ignoring the first couple of entries for the boundary effects
# at the beginning of the data, we can see that the discontinuity
# has been correctly identified at sample 1200 which happens to
# correspond to t=12 sec.


from .model import Model
import numpy as np
from src.si.util.util import mse, add_intersect

class LinearRegression(Model):

    def __init__(self, gd=False, epochs=1000, lr=0.001):
        super(LinearRegression, self).__init__()
        self.gd = gd
        self.theta = None
        self. epochs = epochs
        self.lr = lr

    def fit(self, dataset):
        X, Y = dataset.getXy()
        X = np.hstack((np.ones((X.shape[0], 1)), X))
        self.X =X
        self.Y = Y
        #Close form or GD
        self.train_gd(X, Y) if self.gd else self.train_closed(X, Y)
        self.is_fitted = True

    def train_closed(self, X, Y):
        """
        theta = (X(T)X)**-1 * (X(T)Y)
        Uses closed form linear algebra top fit the model.
        """
        self.theta = np.linalg.inv(X.T.dot(X)).dot(X.T).dot(Y)

    def train_gd(self, X, Y):
        m = X.shape[0]
        n = X.shape[1]
        self.history = {} #ignorar por enquanto
        self.theta = np.zeros(n)
        for epoch in range(self.epochs):
            # (1/m) * (-Y(T) + theta(T)X(T)) * X ==> gradient
            grad = 1/m * (X.dot(self.theta)-Y).dot(X)
            self.theta -= self.lr * grad
            self.history[epoch] = [self.theta[:], self.cost()] #permite obter os graficos que estao no notebook para ver a veolucao do loss ao longo do tempo

    def predict(self, x):
        assert self.is_fitted, "Model must be fitted before predicting"
        if x.ndim > 1:
            res = []
            for i in x:
                _x = np.hstack(([1], i))
                res.append(np.dot(self.theta, _x))
        else:
            _x = np.hstack(([1], x))
            res = np.dot(self.theta, _x)
        return res

    def cost(self, X=None, Y=None, theta=None):
        X = add_intersect(X) if X is not None else self.X
        Y = Y if Y is not None else self.Y
        theta = theta if theta is not None else self.theta
        y_pred = np.dot(X, theta)
        return mse(Y, y_pred)/2

class LinearRegressionReg(LinearRegression): #Isto  com regularizacao

    def __init__(self, gd = False, epochs=1000, lr=0.001, lbd=1): #lbd = lambda
        """
        Linear Regression model with L2 regularization
        """
        super(LinearRegressionReg, self).__init__(gd=gd, epochs=epochs, lr=lr)
        self.lbd = lbd

    def train_closed(self, X, Y):
        """
        Uses closed form linear algebra to fit the model.
        theta = inv(XT*X+lbd*I)*XT*Y
        """
        n = X.shape[1]
        identity = np.eye(n) #matriz de identidade do tamanho do n de features
        identity[0, 0] = 0 #por cauda de bias tem que ter 0 na primeira posicao
        self.theta = np.linalg.inv(X.T.dot(X)+self.lbd*identity).dot(X.T).dot(Y)
        self.is_fitted = True

    def train_gd(self, X, Y):
        """
        Uses gradient to fit the model.
        """
        m = X.shape[0]
        n = X.shape[1]
        self.history = {}  # ignorar por enquanto
        self.theta = np.zeros(n)
        lbds = np.full(m, self.lbd) #cria um vetor de tamanho m com os valores de lambda
        lbds[0] = 0 #adiciona um 0 na posica 1 do vetor de lambdas
        for epoch in range(self.epochs):
            #(-Y(T) + theta(T)X(T)) * X ==> gradient
            grad = 1/m * (X.dot(self.theta) - Y).dot(X)
            self.theta -= (self.lr/m) * (lbds+grad)
            self.history[epoch] = [self.theta[:], self.cost()]


import numpy as np
import matplotlib.pyplot as plt

# function to create the square pulse
def square(t , T):
    f = np.logical_and( t <= T / 2.0 , t >= -T/2.0 )
    return f.astype('float')

# create time axis
def time_axis(Tmin, Tmax, N):
    n = np.arange(-N / 2.0, N / 2.0, 1)
    Dt = (Tmax - Tmin) / N
    return n * Dt

# create frequency axis
def frequency_axis(t):
    N = t.size
    Dt = t[1] - t[0]
    n = np.arange(-N / 2.0, N / 2.0, 1)
    Df = 1.0 / ( N * Dt)
    return n * Df

# Calculate spectrum of x using FFT
def spectrum(t, x):
    Dt = t[1] - t[0]
    return Dt*np.fft.fftshift(np.fft.fft(np.fft.fftshift(x)))

# Inverse fourier transform
def inv_spectrum(f, X):
    Df = f[1] - f[0]
    N = f.size
    return N * Df * np.fft.fftshift(np.fft.ifft(np.fft.fftshift(X)))

# Square filter
def square_filter(f , Fmax):
    return square(f, Fmax)

# System action
def system_action(t, x, Hcallable):
    f = frequency_axis(t)
    Hf = Hcallable(f)
    X = spectrum(t, x)
    Y = X * Hf
    return inv_spectrum(f, Y)

# Calculate energy
def energy(t, x):
    return np.trapz( np.abs(x) ** 2.0, t)

# Calculate average power
def average_power(t, x):
    ta = np.min(t)
    tb = np.max(t)
    return energy( t , x ) / ( tb - ta )

# Calculate a simple cosine sinal
def cos_signal(A, f0, t, phi = 0.0 ):
    return A * np.cos ( 2.0 * np.pi * f0 * t + phi )

# plot signal
def plot_signal(t, x, xlabel = 't', ylabel = 'x(t)', figure_no = None):
    if figure_no is None:
        plt.figure()
    else:
        plt.figure(figure_no)

    plt.plot( t, x )
    plt.xlabel( xlabel )
    plt.ylabel( ylabel )

# power spectral density
def power_density(t, x):
    T = np.max(t) - np.min(t)
    return 1.0 / T * np.abs( spectrum(t,x) ) ** 2.0

# uniform samples


import numpy as np
import math

def normalize(column):
    """
    this function takes a array and normalizes it

    input : array

    output : normalized array
    """
    sq_sum = 0
    for ele in column:
        sq_sum = sq_sum + (ele*ele)
    norm_column = column/math.sqrt(sq_sum)

    return norm_column

def gs_orthogonalization(A):
    """
    this functions performs GS orthogonalization on a matrix

    Input:
    1.A : Matrix Array

    Output:
    1.cols : columns of the input matrix
    2.on_basis : orthonormal basis found using Gram-Schmidt orthogonalization
    """
    # consider the vectors to be the columns of the matrix A.
    num_col = A.shape[1]
    cols = []
    for i in range(num_col):
        cols.append(np.array((A[:,i])))

    on_basis = []
    on_basis.append(normalize(cols[0]))

    for i in range(1,num_col):
        #finding new orthonormalbase
        new_base = cols[i]
        for element in on_basis :
             new_base = new_base - np.array(np.dot(cols[i],element)*element)
        on_basis.append(normalize(new_base))
    #  after this the on_basis contain the orthonormal basis vectors as arrays
    return cols,on_basis


import numpy as np

def square_impurity(y_train):
    """Compute the squared loss impurity of the labels

    Args:
        y_train: n-dimensional vector of labels

    Returns:
        square_impurity: squared loss impurity
    """
    (N,) = y_train.shape
    assert N > 0  # must have at least one sample

    impurity = np.sum(np.square(y_train - np.mean(y_train)))

    return impurity

def square_loss(pred, label):
    """Compute the mean squared loss

    Args:
        pred: numpy array of predictions
        label: numpy array of true labels

    Returns:
        squared loss based on predictions and true labels

    """
    return np.mean((pred - label) ** 2)

def rmse(pred, label):
    """Compute the root mean squared error

    Args:
        pred: numpy array of predictions
        label: numpy array of true labels

    Returns:
        root mean squared error based on predictions and true labels

    """
    return np.sqrt(square_loss(pred, label))

def train_test_split(x, y, train_size: float):
    xy = np.hstack((x, np.reshape(y, (len(y), -1))))
    train, test = np.array_split(xy, [int(train_size * len(xy))])
    x_train, y_train = train[:, :-1], train[:, -1]
    x_test, y_test = test[:, :-1], test[:, -1]
    return x_train, x_test, y_train, y_test

def display_seconds(seconds: int, granularity: int = 1):
    """Display seconds as a string in hours, minutes and seconds

    Args:
        seconds: number of seconds to convert
        granularity: level of granularity of output

    Returns:
        string containing seconds in hours, minutes and seconds
    """

    intervals = {
        "hours": 3600,
        "minutes": 60,
        "seconds": 1,
    }
    result = []

    for name in intervals:
        count = intervals[name]
        value = seconds // count
        if value:
            seconds -= value * count
            if value == 1:
                name = name[:-1]
            result.append(f"{value} {name}")
    return ", ".join(result[:granularity])

def param_grid(grid: dict):
    """Convert a dictionary of parameter names and values to a parameter grid

    Args:
        grid: dictionary of parameter names (str) and values (list). Parameter values
          of type str are not allowed

    Examples:
        >>> param_grid({'train_steps': [500, 1000, 2000], 'regularisation':[True, False]})
        [{'train_steps': 500, 'regularisation': 1},
         {'train_steps': 500, 'regularisation': 0},
         {'train_steps': 1000, 'regularisation': 1},
         {'train_steps': 1000, 'regularisation': 0},
         {'train_steps': 2000, 'regularisation': 1},
         {'train_steps': 2000, 'regularisation': 0}]

    Returns:
        arr_dict: list of dictionaries containing all possible parameter combinations

    """

    num_parameters = len(grid)
    parameter_names = np.array(list(grid.keys()))
    parameter_combination = np.array(np.meshgrid(*grid.values())).T.reshape(
        -1, num_parameters
    )

    arr_dict = []
    for params in parameter_combination:
        param_dict = {}
        for i, n in enumerate(parameter_names):
            param_dict[n] = params[i]
        arr_dict.append(param_dict)

    return arr_dict


import numpy as np

def sigmoid(x):
    return 1 / (1 + np.exp(-x))

class Logistic:
    def __init__(self, reg=0):
        self.reg=reg

    def fit(self, X, y, tol=1e-4):
        X = np.concatenate((np.ones_like(X[:, 0])[:, np.newaxis], X), axis=1)
        start=True
        w = np.zeros(X[0, :].T.shape)
        if start:
            w_new = w - 2*tol

        while np.linalg.norm(w_new - w) > tol:
            w = w_new.copy()
            eta = sigmoid(w.dot(X.T))
            gradl = X.T.dot(y-eta)
            Hl = -X.T.dot(np.diag(eta*(1-eta))).dot(X)
            w_new = w - 0.01*np.linalg.inv(Hl + self.reg*np.identity(Hl.shape[0])).dot(gradl) #

        self.w = w_new

    def compute_frontier(self):

        def frontier(x1):
            return (1/self.w[2])*(-self.w[1] * x1 - self.w[0])
        return frontier

    def predict_proba(self, X):
        X = np.concatenate((np.ones_like(X[:, 0])[:, np.newaxis], X), axis=1)
        return sigmoid(self.w.dot(X.T))

    def predict(self, X):
        return self.predict_proba(X) > 0.5

    def compute_misclassif_error(self, X, y):
        return np.sum(self.predict(X) != y)/len(y)


import numpy as np

def power_method(A, iter_num=1):
    """
    Calculate the first singular vector/value of a target matrix based on the power method.
    Parameters
    ----------
    A : numpy array
        Target matrix
    iter_num : int
               Number of iterations

    Returns
    -------
    u : numpy array
        first left singular vector of A
    s : float
        first singular value of A
    v : numpy array
        first right singular vector of A
    """
    # set initial vector q
    q = np.random.normal(size=A.shape[1])
    q = q / np.linalg.norm(q)

    for i in range(iter_num):
        q = np.dot(np.dot(A.T, A), q)

    v = q / np.linalg.norm(q)
    Av = np.dot(A, v)
    s = np.linalg.norm(Av)
    u = Av / s

    return u, s, v

def tridiagonalize_by_lanczos(P, m, k):
    """
    Tridiagonalize matrix by lanczos method
    Parameters
    ----------
    P : numpy array
        Target matrix
    q : numpy array
        Initial vector
    k : int
        Size of the tridiagonal matrix

    Returns
    -------
    T : numpy array
        tridiagonal matrix
    """
    # Initialize variables
    T = np.zeros((k, k))
    r0 = m
    beta0 = 1
    q0 = np.zeros(m.shape)

    for i in range(k):
        q1 = r0 / beta0
        C = np.dot(P, q1)
        alpha1 = np.dot(q1, C)
        r1 = C - alpha1 * q1 - beta0 * q0
        beta1 = np.linalg.norm(r1)

        T[i, i] = alpha1
        if i + 1 < k:
          T[i, i + 1] = beta1
          T[i + 1, i] = beta1

        q0 = q1
        beta0 = beta1
        r0 = r1

    return T

def tridiag_eigen(T, iter_num=1, tol=1e-3):
    """
    Calculate eigenvalues and eigenvectors of tridiagonal matrix
    Parameters
    ----------
    P : numpy array
        Target matrix (tridiagonal)
    iter_num : int
        Number of iterations
    tol : float
        Stop iteration if the target matrix converges to a diagonal matrix with acceptable tolerance `tol`

    Returns
    -------
    eigenvalue : numpy array
                 Calculated eigenvalues
    eigenvectors : numpy array
                 Calculated eigenvectors
    """
    eigenvectors = np.identity(T.shape[0])

    for i in range(iter_num):
        Q, R = tridiag_qr_decomposition(T)
        T = np.dot(R, Q)
        eigenvectors = np.dot(eigenvectors, Q)
        eigenvalue = np.diag(T)
        if np.all((T - np.diag(eigenvalue) < tol)):
            break

    return eigenvalue, eigenvectors

def tridiag_qr_decomposition(T):
    """
    QR decomposition for a tridiagonal matrix
    Ref. http://www.ericmart.in/blog/optimizing_julia_tridiag_qr
    Parameters
    ----------
    T : numpy array
        Target matrix (tridiagonal)

    Returns
    -------
    Qt.T : numpy array
    R    : numpy array
    """
    R = T.copy()
    Qt = np.eye(T.shape[0])

    for i in range(T.shape[0] - 1):
        u = householder(R[i:i + 2, i])
        M = np.outer(u, u)
        R[i:i + 2, :(i + 3)] -= 2 * np.dot(M, R[i:i + 2, :(i + 3)])
        Qt[i:i + 2, :(i + 3)] -= 2 * np.dot(M, Qt[i:i + 2, :(i + 3)])

    return Qt.T, R

def householder(x):
    """
    Householder projection for vector.
    Parameters
    ----------
    x : numpy array
        Target vector

    Returns
    -------
    x : numpy array
    """
    x[0] = x[0] + np.sign(x[0]) * np.linalg.norm(x)
    x = x / np.linalg.norm(x)
    return x


import numpy as np

def get_outliers(np_array):
    q1, q3 = np.percentile(np_array, [25, 75])
    iqr = q3 - q1
    low = q1 - (iqr * 1.5)
    up = q3 + (iqr * 1.5)
    outliers_indices = np.where((np_array > up) | (np_array < low))[0]
    return np_array[outliers_indices]

# caution: this function assumes that item is a member of np_array
def is_outlier(item, np_array):
    return item in get_outliers(np_array)

if __name__ == '__main__':
    S = np.array([1, 2, 2, 6, 7, 1, 1, 3, 1, 1])
    print(get_outliers(S))
    print(is_outlier(6, S))
    print(is_outlier(1, S))


import numpy as np
def subset_sum(s, t):
    n = len(s)
    A = np.empty((n, t+1))

    # POPULATE FIRST COLUMN WITH TRUE
    A[:, 0] = True

    for j in range(1, t+1):
        A[0, j] = s[0] == j

    for i in range(1, n):
        for j in range(1, t+1):
            if j >= s[i]:
                A[i, j] = A[i - 1, j] or A[i - 1, j - s[i]]
            else:
                A[i, j] = A[i - 1, j]

    return A

def subset_recover(solution, values):
    rows, cols = solution.shape
    curr_row = rows - 1
    curr_col = target = cols - 1
    curr_tot = 0
    results = []

    # TRIVIAL CASE IF LOWER RIGHT IS NOT TRUE
    # RETURN EMPTY AS NO SUBSET EXISTS
    if not solution[curr_row, curr_col]:
        return []

    while curr_tot < target:
        # IF THE VALUE OF THE PREVIOUS ROW IS FALSE
        # INCLUDE THE VALUE OF THE ROW AND MOVE OVER
        if curr_row == 0:
            results.append(values[curr_row])
            curr_tot += values[curr_row]
        if not solution[curr_row - 1, curr_col]:
            results.append(values[curr_row])
            curr_tot += values[curr_row]
            curr_col -= values[curr_row]

        curr_row -= 1

    return results

def subset_sum_w_solution(s, t):
    n = len(s)
    A = np.empty((n, t+1))

    # POPULATE FIRST COLUMN WITH TRUE
    A[:, 0] = True

    for j in range(1, t+1):
        A[0, j] = s[0] == j

    for i in range(1, n):
        for j in range(1, t+1):
            if j >= s[i]:
                A[i, j] = A[i - 1, j] or A[i - 1, j - s[i]]
            else:
                A[i, j] = A[i - 1, j]

    curr_row = n - 1
    curr_col = t
    curr_tot = 0
    results = []

    # TRIVIAL CASE IF LOWER RIGHT IS NOT TRUE
    # RETURN EMPTY AS NO SUBSET EXISTS
    if not A[curr_row, curr_col]:
        return []

    while curr_tot < t:
        # IF THE VALUE OF THE PREVIOUS ROW IS FALSE
        # INCLUDE THE VALUE OF THE ROW AND MOVE OVER
        if curr_row == 0:
            results.append(s[curr_row])
            curr_tot += s[curr_row]
        if not A[curr_row - 1, curr_col]:
            results.append(s[curr_row])
            curr_tot += s[curr_row]
            curr_col -= s[curr_row]

        curr_row -= 1

    return results

v = [2,3,5,7,9]
x = subset_sum(v, 12)

for test in range(28):
    x = subset_sum_w_solution(v,test)
    # y = subset_recover(x, v)
    print(f"Target: {test}, Solution: {x}")


import numpy

# Example 2.3
def example_2_3():
  print('''
*******************************************************************************
* Example 2.3
Where do the following lines intersect?
eq1: -1 * x  +   0 * y  +  1 = 0
eq2:  0 * x  +  -1 * y  +  1 = 0

             | i  j  k |   ( 1 )
x = l x l' = |-1  0  1 | = ( 1 )
             | 0 -1  1 |   ( 1 )

''')
  result = numpy.cross([-1, 0, 1], [0, -1, 1])
  print(result)
  print("\n")

  print("result:")
  print(result[0:2])

  print("The last field in the result should be 1, so this is a real point, not an ideal point")
  print("The first two fields should be [1,1], which is the point in R2 where the lines intersect")

  print('\n\n')

# Example 2.5
def example_2_5():
  print('''
*******************************************************************************
* Example 2.5
Where do the following parallel lines intersect?
eq1: x = 1   =>   -1 * x  +  0 * y  +  1 = 0
eq2: x = 2   =>   -2 * x  +  0 * y  +  1 = 0
             | i  j  k |   ( 0 )
x = l x l' = |-1  0  1 | = ( 1 )
             |-1  0  2 |   ( 0 )
''')
  result = numpy.cross([-1, 0, 1], [-1, 0, 2])
  print("\n")
  print("result:")
  print(result)
  print("The last field in the result should be a 0, so this is an ideal point at infinity, not a real point")
  print("This means that the two lines are parallel.")
  print("The clever aspect is that the same operation, the cross product, gave us a clearly valid answer, of the same type R^3")

print("Section 2")
example_2_3()
example_2_5()


"""
This file define the method to solve the determinant of a square matrix

# Determinant

In linear algebra, the determinant is a value that can be computed from the elements of square matrix.
The determinant of a matrix A is denoted det(A) or |A|

In the case of a 2x2 matrix the determinant may be defined as:
                | a   b |
        |A| =   | c   d |  = ad - bc

In the case of a 3x3 matrix, the determinant can be defined as:

        | a11  a12  a13 |       | a11  0   0 |    | 0   a12   0 |    |  0    0  a13 |
|A| =   | a21  a22  a23 |  =    | 0  a22 a23 | +  | a21  0  a23 | +  | a21  a22  0  |
        | a31  a32  a33 |       | 0  a32 a33 |    | a31  0  a33 |    | a31  a32  0  |

    = a11(a22a33 - a23a32) + a12(-a21a33 + a23a31) + a13(a21a32 - a22a31)
"""
import numpy as np

def det(square_matrix):
    """ calc the determinant of a square matrix

    Args:
        `square_matrix` is the numpy array of 2 dimensions, shape (n, n)

    Returns:
        out: a scalar value
    """

    m, n = square_matrix.shape
    assert m == n, "The input Matrix must be square"

    # the base case, where n equals one, then directly return the element
    if n == 1:
        return square_matrix[0][0]

    out = 0
    for i in range(n):
        # erase the first row and the ith column to get the cofactor
        factor = np.hstack((square_matrix[1:, :i], square_matrix[1:, i+1:]))
        out += square_matrix[0][i] * det(factor) * ((-1) ** i)

    return out

if __name__ == "__main__":

    matrix = np.array([[1]])

    print(det(matrix))

    matrix = np.array([[1, 3], [2, 4]])

    print(det(matrix))

    matrix = np.array([[0.0199626, 0.320843, 0.11461], [0.330681, 0.60768, 0.89497], [0.403389, 0.0243012, 0.48202]])

    print(det(matrix))


"""Kruskal-Wallis H Test"""

import numpy as np
import pandas as pd
from sys import argv
from scipy.stats import chi2

def get_data(fn):
	df = pd.read_csv(fn, header=None)
	df.columns = ['wom', 'men', 'min']
	return df

def extract_list_from_df(df):
	df = df.T
	return df.values.tolist()

def calc_H(data, ranks, sample_sizes):
	n = float(len(data))
	T_2 = lambda col: (pow(ranks[col], 2.), col)
	E = lambda tup: tup[0] / sample_sizes[tup[1]]

	Ts = list(map(T_2, list(ranks)))
	Es = list(map(E, Ts))

	a = 12. / (n*(n+1.))
	b = sum(Es)
	c = 3. * (n+1.)

	return a * b - c

def sort_data(tuples):
	return sorted(tuples, key=lambda x: x[0])

def get_tuples(df):
	tuples = list()
	for col in list(df):
		col_list = [col for _ in range(len(df[col]))]
		tuples  += zip(df[col], col_list)
	return tuples

def get_sampl_sizes(df):
	sample_sizes = {}
	for cat in list(df):
		sample_sizes[cat] = len(df[cat])
	return sample_sizes

def get_ranks(tuples, categories):
	ranks = {}
	for category in categories:
		ranks[category] = 0
	for i, tup in enumerate(tuples):
		ranks[tup[1]] += i+1
	return ranks

def calc_critical_score(alpha, dof):
	return chi2.isf(alpha, dof)

def comp_tests(H, cv):
	print(H-cv)
	if H > cv:
		print('**Medians unequal, reject null hypothesis!**')
		return
	print('Medians equal, accept null hypothesis')

def main():
	"""P-value for this example data is p=0.0347
	P-value with bonferroni correction = alpha / no-tests
	e.g. 0.05 / 6 = 0.0083
	"""
	assert len(argv) > 2, 'Usage: hwH-test.py [datafile.csv] [alpha]'
	df, alpha = get_data(argv[1]), float(argv[2])
	print('Data:')
	print(df)
	cols = list(df)
	tuples = get_tuples(df)
	sorted_data = sort_data(tuples)
	sample_sizes = get_sampl_sizes(df)
	ranks = get_ranks(sorted_data, cols)
	print('Ranks:', ranks)
	H = calc_H(sorted_data, ranks, sample_sizes)
	print('H statistic:', H)
	crit_val = calc_critical_score(alpha, 2)
	print('Critical Chi-Squared value:', crit_val)
	comp_tests(H, crit_val)

if __name__ == "__main__":
	main()


# Modules
from math import pi
import numpy as np
import matplotlib.pyplot as plt

# Inputs
ms = 50
g = 9.81
rho = 1.091
r = 0.5
A = pi*r**2
ve = 325
Cd = 0.15

t0 = 0
mp0 = 100
v0 = 0
h0 = 0

dt = 0.01
tmax = 60

# Functions
def mpp(t):
  if t < 5:
    return 20
  else:
    return 0

def f(u, dt, g, ve, rho, A, Cd, ms):
  t, mp, v = u[0]+dt, u[1], u[2]
  dh = v
  dmp = -mpp(t)
  dv = (-g + (mpp(t)*ve - 0.5*rho*v*abs(v)*A*Cd)/(ms + mp))
  return np.array([1, dmp, dv, dh])

def euler(dt, u, f, *args):
  u_new = u + dt*f(u, dt, *args)
  return u_new

def rk2(dt, u, f, *args):
  u_star = u + dt/2*f(u, dt/2, *args)
  u_new = u + dt*f(u_star, dt/2, *args)
  return u_new

# Initialisation
u0 = np.array([t0, mp0, v0, h0])
n = int(tmax/dt) + 1
u = np.zeros((n+1, 4))
u[0] = u0

# Main
for i in range(n):
  u[i+1] = euler(dt, u[i], f, g, ve, rho, A, Cd, ms)
  #u[i+1] = rk2(dt, u[i], f, g, ve, rho, A, Cd, ms)

plt.figure(1)
plt.title("mp")
plt.plot(u[:, 0], u[:, 1])
plt.show()
plt.clf()
plt.figure(2)
plt.title("v")
plt.plot(u[:, 0], u[:, 2])
plt.show()
plt.clf()
plt.figure(3)
plt.title("h")
plt.plot(u[:, 0], u[:, 3])
plt.show()
plt.clf()

ind = np.where(np.abs(u[:, 0] - 3.2) < 1e-5)[0][0]
print("mp(3.2) =", u[ind, 1])
v_max = max(u[:, 2])
print("v_max =", v_max)
ind = list(u[:, 2]).index(v_max)
print("h_v_max =", u[ind, 3])
print("t_v_max =", u[ind, 0])
h_max = max(u[:, 3])
print("h_max =", h_max)
ind = list(u[:, 3]).index(h_max)
print("t_h_max =", u[ind, 0])
ind = np.where(u[:, 3] <= 0)[0][2]
print("t_impact =", u[ind, 0])
print("v_impact =", u[ind, 2])


import matplotlib.pyplot as plt
import numpy as np
import scipy as sp
from numpy import sin,cos,tan,pi, sqrt, log, log10
from scipy import integrate

import scipy as sp
import numpy as np
import skrf as rf

#from scipy.optimize import fsolve,brentq,minimize, differential_evolution
#from scipy.optimize import newton

"""
def quad_complex(fun,a,b):
    Re = integrate.quad(lambda x:fun(x).real,a,b)[0]
    Im = integrate.quad(lambda x:fun(x).imag,a,b)[0]
    return Re + 1j*Im
"""
"""
Fourier series is an expansion of periodic signal as a linear combination of sines and cosines, while Fourier transform is the process or function used to convert signals from the time domain to frequency domain.
"""

"""
def fourier(fun,a,n):
    tol = 1e-6
    x = np.linspace(-a,a,100)
    L = a
    cn = np.array([1/(L)*(integrate.quad(lambda x:fun(x)*cos(i*x*pi/L),-L,L)[0] - 1j* \
                            integrate.quad(lambda x:fun(x)*sin(i*x*pi/L),-L,L)[0]) \
                   for i in np.arange(n)+1 ])
    approx = (fun(0+tol)+fun(0-tol))/2 + np.array([cn[i]*np.exp(1j*pi*j*x/L) for i,j in enumerate(np.arange(n)+1) ]).sum(axis = 0)
    #approx =  np.array([cn[i]*np.exp(1j*pi*j*x/L) for i,j in enumerate(np.arange(n)+1) ]).sum(axis = 0)

    #Im = integrate.quad(lambda x:fun(x).imag,a,b)[0]
    return approx,cn
"""

def fourier_expansion(fun,L,n):
    """
    fun:a periodic function
    L:interval length
    n:the number of harmonics
    """
    #tol = 1e-6
    x = np.linspace(-L,L,100)

    a0 = 1/L/2 * integrate.quad(fun,-L,L)[0]
    an = 1/L * np.array([integrate.quad(lambda x:fun(x)*cos(i*x*pi/L),-L,L)[0] for i in np.arange(n)+1])
    bn = 1/L * np.array([integrate.quad(lambda x:fun(x)*sin(i*x*pi/L),-L,L)[0] for i in np.arange(n)+1])
    approx = a0 + np.array([an[i]*cos(pi*j*x/L) + bn[i]*sin(pi*j*x/L) for i,j in enumerate(np.arange(n)+1)]).sum(axis = 0)

    """
    c0 = 1/L/2 * integrate.quad(fun,-a,a)[0]

    cn = np.array([1/(L)*(integrate.quad(lambda x:fun(x)*cos(i*x*pi/L),-L,L)[0] - 1j* \
                            integrate.quad(lambda x:fun(x)*sin(i*x*pi/L),-L,L)[0]) \
                   for i in np.arange(n)+1 ])
    approx = c0 + np.array([cn[i]*np.exp(1j*pi*j*x/L) for i,j in enumerate(np.arange(n)+1) ]).sum(axis = 0)
    """
    return approx,np.array([an,bn])

label_set = lambda ax,labels: [ax.set_xlabel(labels[0]),ax.set_ylabel(labels[1])]

def f_transform(fun,f,lim = None):
    operator_e = lambda w,t:cos(w*t)
    operator_o = lambda w,t:sin(w*t)
    if lim == None:
        even = integrate.quad(lambda t:fun(t)*operator_e(f,t),-np.inf,np.inf)[0]
        odd = integrate.quad(lambda t:fun(t)*operator_o(f,t),-np.inf,np.inf)[0]
    else:
        even = integrate.quad(lambda t:fun(t)*operator_e(f,t),lim[0],lim[1])[0]
        odd = integrate.quad(lambda t:fun(t)*operator_o(f,t),lim[0],lim[1])[0]
    return 1/(sqrt(2*pi))*(even - 1j*odd)

def if_transform(fun,t,lim = None):
    operator_e = lambda t,w:cos(w*t)
    operator_o = lambda t,w:sin(w*t)
    if lim == None:
        even = integrate.quad(lambda w:fun(w)*operator_e(w,t),-np.inf,np.inf)[0]
        odd = integrate.quad(lambda w:fun(w)*operator_o(w,t),-np.inf,np.inf)[0]
    else:
        even = integrate.quad(lambda t:fun(w)*operator_e(w,t),lim[0],lim[1])[0]
        odd = integrate.quad(lambda t:fun(w)*operator_o(w,t),lim[0],lim[1])[0]
    return 1/(sqrt(2*pi))*(even + 1j*odd)

#test_f2 = lambda t: np.exp(t) if abs(t)<2 else 0

#fig,ax = plot(lambda x:f1(x,b),x,F1,w)

def plot(f,t,F,w):
    fig,ax = plt.subplots(2,1)

    #ax[0].plot(t,list(map(lambda t: f,t)))
    ax[0].plot(t,[f(i) for i in t])
    ax[1].plot(w,F.real,label = 'real')
    ax[1].plot(w,F.imag,label = 'imag')

    ax[1].legend()
    label_set(ax[0],['x','magnitude'])
    label_set(ax[1],['w','magnitude'])
    #ax[0].set_label(ax[1].set_xlabel('frequency w')
    fig.tight_layout(pad = 1.2)
    #ax[0].set_title('Fourier')
    #fig.suptitle("Fourier")

    return fig,ax

f1 = lambda t,b: 1 if abs(t)<b else 0
a1 = lambda w,b: sqrt(2/pi)*sin(b*w)/w
b = 1

x = np.linspace(-5,5,100)
w = np.linspace(-10,10,100)

F1 = np.array([f_transform(lambda t:f1(t,b),i) for i in w])
fig,ax = plot(lambda x:f1(x,b),x,F1,w)
ax[1].plot(w,np.array(list(map(lambda w:a1(w,b),w))).real,'bo',linestyle = 'dotted',lw = 2,label = 'analytical',markevery = 10)
ax[1].legend()

f2 = lambda t,b,c: 1 if b<t<c else 0
a2 = lambda w,b,c: (np.exp(-1j*b*w) - np.exp(-1j*c*w))/(1j*w*sqrt(2*pi))

#a1 = lambda w,b: sqrt(2/pi)*sin(b*w)/w
b = 1
c = 2

x = np.linspace(-5,5,100)
w = np.linspace(-10,10,100)

F2 = np.array([f_transform(lambda t:f2(t,b,c),i) for i in w])
F2_fun = lambda w: f_transform(f2,t)
#F1 = np.array([f_transform(lambda t:f1(t,b),i) for i in w])
fig,ax = plot(lambda x:f2(x,b,c),x,F2,w)
ax[1].plot(w,np.array(list(map(lambda w:a2(w,b,c),w))).real,'bo',linestyle = 'dotted',lw = 2,label = 'analytical:real',markevery = 10)
ax[1].plot(w,np.array(list(map(lambda w:a2(w,b,c),w))).imag,'yo',linestyle = 'dotted',lw = 2,label = 'analytical:imag',markevery = 10)
ax[1].legend(loc = 'upper right',fontsize = 6)

f2 = lambda t,b,c: 1 if b<t<c else 0
a2 = lambda w,b,c: (np.exp(-1j*b*w) - np.exp(-1j*c*w))/(1j*w*sqrt(2*pi))

#a1 = lambda w,b: sqrt(2/pi)*sin(b*w)/w
b = 1
c = 2

x = np.linspace(-5,5,100)
w = np.linspace(-10,10,100)

F2 = np.array([f_transform(lambda t:f2(t,b,c),i) for i in w])
#F1 = np.array([f_transform(lambda t:f1(t,b),i) for i in w])
fig,ax = plot(lambda x:f2(x,b,c),x,F2,w)
ax[1].plot(w,np.array(list(map(lambda w:a2(w,b,c),w))).real,'bo',linestyle = 'dotted',lw = 2,label = 'analytical:real',markevery = 10)
ax[1].plot(w,np.array(list(map(lambda w:a2(w,b,c),w))).imag,'yo',linestyle = 'dotted',lw = 2,label = 'analytical:imag',markevery = 10)
ax[1].legend(loc = 'upper right',fontsize = 6)

f3_1 = lambda x: x+1 if(-1<x<0) else -x + 1 if(0<x<1) else 0
F3_1 = np.array([f_transform(lambda t:f3_1(t),i) for i in w])
#F1 = np.array([f_transform(lambda t:f1(t,b),i) for i in w])
fig,ax = plot(lambda x:f3_1(x),x,F3_1,w)
ax[1].legend(loc = 'upper right',fontsize = 12)

f3_2 = lambda x: -x-1 if(-1<x<0) else -x + 1 if(0<x<1) else 0
F3_2 = np.array([f_transform(lambda t:f3_2(t),i) for i in w])
#F1 = np.array([f_transform(lambda t:f1(t,b),i) for i in w])
fig,ax = plot(lambda x:f3_2(x),x,F3_2,w)
ax[1].legend(loc = 'upper right',fontsize = 12)

f3_3 = lambda x: x if(0<x<1) else -x + 2 if(1<x<2) else 0
F3_3 = np.array([f_transform(lambda t:f3_3(t),i) for i in w])
#F1 = np.array([f_transform(lambda t:f1(t,b),i) for i in w])
fig,ax = plot(lambda x:f3_3(x),x,F3_3,w)
ax[1].legend(loc = 'upper right',fontsize = 12)
"""
"""
f4 = lambda x,a: 1/(x**2 + a**2)
a = 1
F4 = np.array([f_transform(lambda t:f4(t,a),i,lim = [0,10]) for i in w])
#F1 = np.array([f_transform(lambda t:f1(t,b),i) for i in w])
fig,ax = plot(lambda x:f4(x,a),x,F4,w)
ax[1].legend(loc = 'upper right',fontsize = 12)

f5 = lambda x,a: np.exp(-a*x) if x>0 else 0
a = 1
a5 = lambda w,a: 1/(sqrt(2*pi)*(a + 1j*w))
F5 = np.array([f_transform(lambda t:f5(t,a),i,lim = [0,10]) for i in w])
#F1 = np.array([f_transform(lambda t:f1(t,b),i) for i in w])
fig,ax = plot(lambda x:f5(x,a),x,F5,w)
ax[1].plot(w,np.array(list(map(lambda w:a5(w,a),w))).real,'bo',linestyle = 'dotted',lw = 2,label = 'analytical:real',markevery = 10)
ax[1].plot(w,np.array(list(map(lambda w:a5(w,a),w))).imag,'yo',linestyle = 'dotted',lw = 2,label = 'analytical:imag',markevery = 10)
ax[1].legend(loc = 'upper right',fontsize = 6)

f6 = lambda x,a: np.exp(-a*x**2)
a = 1
a6 = lambda w,a: 1/sqrt(2*a)*np.exp(-w**2/4/a)
F6 = np.array([f_transform(lambda t:f6(t,a),i,lim = [-10,10]) for i in w])
#F1 = np.array([f_transform(lambda t:f1(t,b),i) for i in w])
fig,ax = plot(lambda x:f6(x,a),x,F6,w)
ax[1].plot(w,np.array(list(map(lambda w:a6(w,a),w))).real,'bo',linestyle = 'dotted',lw = 2,label = 'analytical:real',markevery = 10)
ax[1].plot(w,np.array(list(map(lambda w:a6(w,a),w))).imag,'ro',linestyle = 'dotted',lw = 2,label = 'analytical:imag',markevery = 10)
ax[1].legend(loc = 'upper right',fontsize = 6)

f7 = lambda x,a: np.sin(a*x)/x
a = 5
a7 = lambda w,a: sqrt(pi/2) if abs(w)<a else 0
F7 = np.array([f_transform(lambda t:f7(t,a),i,lim = [0,5]) for i in w])
#F1 = np.array([f_transform(lambda t:f1(t,b),i) for i in w])
fig,ax = plot(lambda x:f7(x,a),x,F7,w)
ax[1].plot(w,np.array(list(map(lambda w:a7(w,a),w))).real,'bo',linestyle = 'dotted',lw = 2,label = 'analytical:real',markevery = 10)
ax[1].plot(w,np.array(list(map(lambda w:a7(w,a),w))).imag,'ro',linestyle = 'dotted',lw = 2,label = 'analytical:imag',markevery = 10)
#ax[1].legend(loc = 'upper right',fontsize = 6)

f8 = lambda x,a: cos(x) if -a<x<a else 0
a = 10
a8 = lambda w,a: 1/sqrt(2*pi)*(sin(a*(1-w))/(1-w) + sin(a*(1+w))/(1+w))
w = np.linspace(-2,2,100)

F8 = np.array([f_transform(lambda t:f8(t,a),i) for i in w])
fig,ax = plot(lambda x:f8(x,a),x,F8,w)
ax[1].plot(w,np.array(list(map(lambda w:a8(w,a),w))).real,'bo',linestyle = 'dotted',lw = 2,label = 'analytical:real',markevery = 10)
ax[1].plot(w,np.array(list(map(lambda w:a8(w,a),w))).imag,'ro',linestyle = 'dotted',lw = 2,label = 'analytical:imag',markevery = 10)
#ax[1].legend(loc = 'upper right',fontsize = 6)
#ax[1].set_ylim([-1,1])
ax[0].set_title(r'f(x) = $\cos{x} (|x|< \pi)$')

f8 = lambda x,a: sin(x) if -a<x<a else 0
a = 10
a8 = lambda w,a: 1j*(1/sqrt(2*pi)*(-sin(a*(1-w))/(1-w) + sin(a*(1+w))/(1+w)))
w = np.linspace(-2,2,100)

F8 = np.array([f_transform(lambda t:f8(t,a),i) for i in w])
fig,ax = plot(lambda x:f8(x,a),x,F8,w)
ax[1].plot(w,np.array(list(map(lambda w:a8(w,a),w))).real,'bo',linestyle = 'dotted',lw = 2,label = 'analytical:real',markevery = 10)
ax[1].plot(w,np.array(list(map(lambda w:a8(w,a),w))).imag,'ro',linestyle = 'dotted',lw = 2,label = 'analytical:imag',markevery = 10)
ax[0].set_title(r'f(x) = $\sin{x} (|x|< \pi)$')

def plot_fourier(fun,a,n):
    """
    fun: the vectorized function
    a(float): the interval length
    n(list): the list of the required harmonics number, e.g. [1,5,10].
    """
    x = np.linspace(-a,a,100)
    fig,ax = plt.subplots()
    ax.plot(x,fun(x),label = 'exact')
    approx,cn = np.array([fourier_expansion(fun,a,i) for i in n]).T
    [ax.plot(x,j,'--', label = f'n = {n[i]}') for i,j in enumerate(approx)]
    [ax.set_xlabel('x'),ax.set_ylabel('f(x)')]
    ax.legend()
    """
    VisibleDeprecationWarning: Creating an ndarray from ragged nested sequences
    (which is a list-or-tuple of lists-or-tuples-or ndarrays with different lengths
     or shapes) is deprecated. If you meant to do this, you must specify
    'dtype=object' when creating the ndarray.
    """
    return fig,ax

"""
Solving differential equation using fourier series expansion
"""

#Q2.1 f''(x) +3f'(x) = cost x(0) = 0, x'(0) = 0

"""

There is convplution function in numpy as well. how to use is shown below
np.convolve([1, 2, 3], [0, 1, 0.5])
Out[22]: array([0. , 1. , 2.5, 4. , 1.5])
mode{‘full’, ‘valid’, ‘same’}, optional

    ‘full’:

        By default, mode is ‘full’. This returns the convolution at each point of overlap, with an output shape of (N+M-1,). At the end-points of the convolution, the signals do not overlap completely, and boundary effects may be seen.
    ‘same’:

        Mode ‘same’ returns output of length max(M, N). Boundary effects are still visible.
    ‘valid’:

        Mode ‘valid’ returns output of length max(M, N) - min(M, N) + 1. The convolution product is only given for points where the signals overlap completely. Values outside the signal boundary have no effect.

"""
x = np.linspace(0,10,100)
laplace = lambda x: integrate.quad(lambda x:x**2*np.exp(-x),0,np.inf)[0]

y = np.array([laplace(i) for i in x])
fig,ax = plt.subplots(2,1)
ax[0].plot(x)
ax[1].plot(y)

integrate.quad(lambda x:np.exp(-2*x),0,np.inf)

freq = 1
f = lambda x,freq:0 if (x >= -3 and x <= 0) else np.sin(2*pi*freq*x)  # else 1 if x >= 0 and x < 1
f = np.vectorize(f)
a = 0.3
n = [2,5,10,30]
fig,ax = plot_fourier(lambda x:f(x,freq),a,n)
fig.suptitle('f(x) = 0 (x >= -3 and x <= 1) else 10 ', fontsize=16)

f = lambda x,a:a*x# else 1 if x >= 0 and x < 1
f = np.vectorize(f)

L =2
approx,[an,bn] = fourier_expansion(lambda x:f(x,2),L,20)

dev = f(np.linspace(-L,L,100),2) - approx

fig,ax = plt.subplots(2,1)
ax[0].plot(dev)
ax[1].plot(np.abs(bn))

a = pi
n = [2,5,20]
fig,ax = plot_fourier(lambda x:f(x,2),a,n)
fig.suptitle(' f(x) = |sin(x)| ', fontsize=16)

"""
Q1.1
f = lambda x:0 if (x >= -3 and x <= 1) else 10 # else 1 if x >= 0 and x < 1
f = np.vectorize(f)
a = 3
n = [1,5,10]
fig,ax = plot_fourier(f,a,n)
fig.suptitle('f(x) = 0 (x >= -3 and x <= 1) else 10 ', fontsize=16)

Q1.2
f = lambda x:np.abs(sin(x))# else 1 if x >= 0 and x < 1
f = np.vectorize(f)
a = pi
n = [2,5,10]
fig,ax = plot_fourier(f,a,n)
fig.suptitle(' f(x) = |sin(x)| ', fontsize=16)

Q1.3
f = lambda x:np.abs(x)# else 1 if x >= 0 and x < 1
f = np.vectorize(f)
a = 2
n = [1,5,10]
fig,ax = plot_fourier(f,a,n)
fig.suptitle('f(x) = |x|', fontsize=16)
"""

"""
to fourier transform
"""
L = 2*pi
f = lambda x:sin(x) + sin(3*x) + sin(10*x)
test = fourier_expansion(f,L,20)
x = np.linspace(-L,L,100)

fig,ax = plt.subplots(2,1)
ax[0].plot(x,f(x))
[ax[0].set_xlabel('t'),ax[0].set_ylabel('magnitude')]
ax[1].plot(test[1][1],'o')

[ax[1].set_xlabel('frequency'),ax[1].set_ylabel('magnitude')]
fig.tight_layout(pad = 1.2)

L = 6.35e-3
d = 11.43e-3
t = 1.27e-3
h = 24.13e-3
w = 0.635e-3
Er = 8.875

#basis function?

Jz_d_t = lambda x: 1/sqrt(w**2 - x**2)
Jx_d_t = lambda x: x*sqrt(w**2 - x**2)
Ez = lambda x: sqrt(a**2 - x**2)
Ex = lambda x: x/sqrt(a**2 - x**2)

#We need to transform these functions into that in frequency domain
#using bessel functions

#space -> spectral domain by fourier transform

#AssertionErrorhttps://homepages.inf.ed.ac.uk/rbf/HIPR2/fourier.htm

import numpy,from numpy,import scipy,from scipy


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
BE523 Biosystems Analysis & Design
HW4 - Problem 1. Human population growth

Created on Mon Jan 25 23:19:35 2021
@author: eduardo
"""
import numpy as np
import matplotlib.pyplot as plt
import statsmodels.api as sm  # allows linear regression without intercept

P0 = 1e6  # initial population
r = 0.01  # rate of change in population
steps = 50  # years in the future
dt = 1  # time interval
dP = np.zeros(steps)

t = np.linspace(0, (steps-1)*dt, steps)  # actual time vector

# Generate the population growth data with model: P(t+1) = P[0]*(r+1)^t
P = P0 * (r + 1)**t

for i in range(1, steps):
    dP[i] = P[i] - P[i-1]  # difference between time steps

# Perform a linear regression with P and dP, then plot (dP vs P)
model = sm.OLS(dP, P)  # No intercept by default, force through the origin??
results = model.fit()
slope = results.params[0]  # grow rate of population
print("slope=", slope)  # should be close to 'r'

# Figure 1, plotting dP vs P
plt.figure(1)
plt.plot(P, dP, 'kx', P, slope*P, 'b-')
plt.legend(['data', 'linear regression $R^2$=%.2f' % results.rsquared], loc='best')
plt.xlabel('P')
plt.ylabel('dP')
plt.savefig('p1_population_linear.png', dpi=300, bbox_inches='tight')

# Generate an exponential equation (exact solution)
tdouble = np.log(2)/np.log(1+slope)*dt
print('tdouble =', tdouble)
K = np.log(2)/tdouble
Pexp = P[0] * np.exp(K*t/dt)

# Generate predictions with the model (numerical solution)
Pmodel = P[0]*(slope+1)**(t/dt)
print("The population after %d years is: %.3f" % (steps, Pmodel[-1]))

# Figure 2, plotting P (from data and model) vs t
plt.figure(2)
plt.plot(t, Pmodel, 'bo', t, Pexp, 'r-')
plt.legend(['numerical P=%g*(1+%.4f)^t' % (P[0], slope),
            'exact P=%g*exp(%.4f*t)' % (P[0], K)],
           loc='best')
plt.xlabel('Time (years)')
plt.ylabel('Population')
plt.savefig('p1_population_model.png', dpi=300, bbox_inches='tight')


import pandas as pd

df = pd.read_csv('./housing.data.txt',
                 sep='\s+')

df.columns = ['CRIM', 'ZN', 'INDUS', 'CHAS',
              'NOX', 'RM', 'AGE', 'DIS', 'RAD',
              'TAX', 'PTRATIO', 'B', 'LSTAT', 'MEDV']
df.head()

import matplotlib.pyplot as plt
import seaborn as sns

cols = ['LSTAT', 'INDUS', 'NOX', 'RM', 'MEDV']

sns.pairplot(df[cols], height=2.5)
plt.tight_layout()
plt.show()

import numpy as np

cm = np.corrcoef(df[cols].values.T)
# hm =
sns.heatmap(cm, cbar=True, annot=True, square=True, fmt='.2f', annot_kws={'size': 15},
            yticklabels=cols, xticklabels=cols)
# plt.tight_layout()
# plt.show()

class LinearRegressionGD(object):

    def __init__(self, eta=0.001, n_iter=20):
        self.eta = eta
        self.n_iter = n_iter

    def fit(self, X, y):
        self.w_ = np.zeros(1 + X.shape[1])
        self.cost_ = []

        for i in range(self.n_iter):
            output = self.net_input(X)
            errors = (y - output)
            self.w_[1:] += self.eta * X.T.dot(errors)
            self.w_[0] += self.eta * errors.sum()
            cost = (errors**2).sum() / 2.0
            self.cost_.append(cost)
        return self

    def net_input(self, X):
        return np.dot(X, self.w_[1:]) + self.w_[0]

    def predict(self, X):
        return self.net_input(X)

X = df[['RM']].values
y = df['MEDV'].values

from sklearn.preprocessing import StandardScaler

sc_x = StandardScaler()
sc_y = StandardScaler()
X_std = sc_x.fit_transform(X)
y_std = sc_y.fit_transform(y[:, np.newaxis]).flatten()

lr = LinearRegressionGD()
lr.fit(X_std, y_std)

plt.plot(range(1, lr.n_iter+1), lr.cost_)
plt.ylabel('SSE')
plt.xlabel('Epoch')
plt.show()

def lin_regplot(X, y, model):
    plt.scatter(X, y, c='steelblue', edgecolors='white', s=70)
    plt.plot(X, model.predict(X), color='black', lw=2)

lin_regplot(X_std, y_std, lr)
plt.xlabel('Average number of rooms [RM] (standardized)')
plt.ylabel('Price in $1000s [MEDV] (standardized)')
plt.show()

print('Slope: %.3f' % lr.w_[1])
print('Intercept: %.3f' % lr.w_[0])

num_rooms_std = sc_x.transform(np.array([[5.0]]))
price_std = lr.predict(num_rooms_std)
print("Price in $1000s: %.3f" % sc_y.inverse_transform(price_std))

from sklearn.linear_model import LinearRegression

slr = LinearRegression()
slr.fit(X, y)
y_pred = slr.predict(X)
print("Slope: {}".format(slr.coef_[0]))
print("Intercept: {}".format(slr.intercept_))

# Fitting a robust regression model using RANSAC

from sklearn.linear_model import RANSACRegressor

ransac = RANSACRegressor(LinearRegression(), max_trials=100, min_samples=50, loss="absolute_loss",
                         residual_threshold=5.0, random_state=0)

ransac.fit(X, y)

# show the inlier and outlier masks
inlier_mask = ransac.inlier_mask_
outlier_mask = np.logical_not(inlier_mask)

line_X = np.arange(3, 10, 1)
line_y_ransac = ransac.predict(line_X[:, np.newaxis])
plt.scatter(X[inlier_mask], y[inlier_mask], c='steelblue', edgecolors='white', marker='o',
            label='Inliers')
plt.scatter(X[outlier_mask], y[outlier_mask], c='limegreen', edgecolors='white', marker='s',
            label='Outliers')
plt.plot(line_X, line_y_ransac, color='black', lw=2)
plt.xlabel('Average number of rooms [RM]')
plt.ylabel('Price in $1000s [MEDV]')
plt.legend(loc='upper left')

plt.show()

print('Slope: %.3f' % ransac.estimator_.coef_[0])
print('Intercept: %.3f' % ransac.estimator_.intercept_)

# Evaluating the performance of linear regression models
# use all the variables

from sklearn.model_selection import train_test_split

X = df.iloc[:, :-1].values
y = df['MEDV'].values

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=0)

slr = LinearRegression()

slr.fit(X_train, y_train)
y_train_pred = slr.predict(X_train)
y_test_pred = slr.predict(X_test)

plt.scatter(y_train_pred,  y_train_pred - y_train,
            c='steelblue', marker='o', edgecolor='white',
            label='Training data')
plt.scatter(y_test_pred,  y_test_pred - y_test,
            c='limegreen', marker='s', edgecolor='white',
            label='Test data')
plt.xlabel('Predicted values')
plt.ylabel('Residuals')
plt.legend(loc='upper left')
plt.hlines(y=0, xmin=-10, xmax=50, color='black', lw=2)
plt.xlim([-10, 50])
plt.tight_layout()
plt.show()

# check mse and r2
from sklearn.metrics import r2_score
from sklearn.metrics import mean_squared_error

print('MSE train: %.3f, test: %.3f' % (
        mean_squared_error(y_train, y_train_pred),
        mean_squared_error(y_test, y_test_pred)))
print('R^2 train: %.3f, test: %.3f' % (
        r2_score(y_train, y_train_pred),
        r2_score(y_test, y_test_pred)))

# Using regularized methods for regression
from sklearn.linear_model import Lasso

lasso = Lasso(alpha=0.1)
lasso.fit(X_train, y_train)
y_train_pred = lasso.predict(X_train)
y_test_pred = lasso.predict(X_test)
print(lasso.coef_)

# In[33]:

print('MSE train: %.3f, test: %.3f' % (
        mean_squared_error(y_train, y_train_pred),
        mean_squared_error(y_test, y_test_pred)))
print('R^2 train: %.3f, test: %.3f' % (
        r2_score(y_train, y_train_pred),
        r2_score(y_test, y_test_pred)))

from sklearn.linear_model import Ridge
ridge = Ridge(alpha=1.0)

# LASSO regression:

from sklearn.linear_model import Lasso
lasso = Lasso(alpha=1.0)

# Elastic Net regression:

from sklearn.linear_model import ElasticNet
elanet = ElasticNet(alpha=1.0, l1_ratio=0.5)

# Turning a linear regression model into a curve - polynomial regression
from sklearn.preprocessing import PolynomialFeatures

X = np.array([258.0, 270.0, 294.0,
              320.0, 342.0, 368.0,
              396.0, 446.0, 480.0, 586.0])\
             [:, np.newaxis]

y = np.array([236.4, 234.4, 252.8,
              298.6, 314.2, 342.2,
              360.8, 368.0, 391.2,
              390.8])

lr = LinearRegression()
pr = LinearRegression()
quadratic = PolynomialFeatures(degree=2)
X_quad = quadratic.fit_transform(X)

# fit linear features
lr.fit(X, y)
X_fit = np.arange(250, 600, 10)[:, np.newaxis]
y_lin_fit = lr.predict(X_fit)

# fit quadratic features
pr.fit(X_quad, y)
y_quad_fit = pr.predict(quadratic.fit_transform(X_fit))

# plot results
plt.scatter(X, y, label='training points')
plt.plot(X_fit, y_lin_fit, label='linear fit', linestyle='--')
plt.plot(X_fit, y_quad_fit, label='quadratic fit')
plt.legend(loc='upper left')

plt.tight_layout()
plt.show()

y_lin_pred = lr.predict(X)
y_quad_pred = pr.predict(X_quad)

print('Training MSE linear: %.3f, quadratic: %.3f' % (
        mean_squared_error(y, y_lin_pred),
        mean_squared_error(y, y_quad_pred)))
print('Training R^2 linear: %.3f, quadratic: %.3f' % (
        r2_score(y, y_lin_pred),
        r2_score(y, y_quad_pred)))

# Modeling nonlinear relationships in the Housing Dataset

X = df[['LSTAT']].values
y = df['MEDV'].values

regr = LinearRegression()

# create quadratic features
quadratic = PolynomialFeatures(degree=2)
cubic = PolynomialFeatures(degree=3)
X_quad = quadratic.fit_transform(X)
X_cubic = cubic.fit_transform(X)

# fit features
X_fit = np.arange(X.min(), X.max(), 1)[:, np.newaxis]

regr = regr.fit(X, y)
y_lin_fit = regr.predict(X_fit)
linear_r2 = r2_score(y, regr.predict(X))

regr = regr.fit(X_quad, y)
y_quad_fit = regr.predict(quadratic.fit_transform(X_fit))
quadratic_r2 = r2_score(y, regr.predict(X_quad))

regr = regr.fit(X_cubic, y)
y_cubic_fit = regr.predict(cubic.fit_transform(X_fit))
cubic_r2 = r2_score(y, regr.predict(X_cubic))

# plot results
plt.scatter(X, y, label='training points', color='lightgray')

plt.plot(X_fit, y_lin_fit,
         label='linear (d=1), $R^2=%.2f$' % linear_r2,
         color='blue',
         lw=2,
         linestyle=':')

plt.plot(X_fit, y_quad_fit,
         label='quadratic (d=2), $R^2=%.2f$' % quadratic_r2,
         color='red',
         lw=2,
         linestyle='-')

plt.plot(X_fit, y_cubic_fit,
         label='cubic (d=3), $R^2=%.2f$' % cubic_r2,
         color='green',
         lw=2,
         linestyle='--')

plt.xlabel('% lower status of the population [LSTAT]')
plt.ylabel('Price in $1000s [MEDV]')
plt.legend(loc='upper right')

plt.show()

# Transforming the dataset

X = df[['LSTAT']].values
y = df['MEDV'].values

# transform features
X_log = np.log(X)
y_sqrt = np.sqrt(y)

# fit features
X_fit = np.arange(X_log.min()-1, X_log.max()+1, 1)[:, np.newaxis]

regr = regr.fit(X_log, y_sqrt)
y_lin_fit = regr.predict(X_fit)
linear_r2 = r2_score(y_sqrt, regr.predict(X_log))

# plot results
plt.scatter(X_log, y_sqrt, label='training points', color='lightgray')

plt.plot(X_fit, y_lin_fit,
         label='linear (d=1), $R^2=%.2f$' % linear_r2,
         color='blue',
         lw=2)

plt.xlabel('log(% lower status of the population [LSTAT])')
plt.ylabel('$\sqrt{Price \; in \; \$1000s \; [MEDV]}$')
plt.legend(loc='lower left')

plt.tight_layout()
plt.show()

# Dealing with nonlinear relationships using random forests
from sklearn.tree import DecisionTreeRegressor

X = df[['LSTAT']].values
y = df['MEDV'].values

tree = DecisionTreeRegressor(max_depth=3)
tree.fit(X, y)

sort_idx = X.flatten().argsort()

lin_regplot(X[sort_idx], y[sort_idx], tree)
plt.xlabel('% lower status of the population [LSTAT]')
plt.ylabel('Price in $1000s [MEDV]')
#plt.savefig('images/10_13.png', dpi=300)
plt.show()

# Random forest regression
from sklearn.ensemble import RandomForestRegressor

forest = RandomForestRegressor(n_estimators=1000, criterion='mse', random_state=1, n_jobs=-1)

forest.fit(X_train, y_train)
y_train_pred = forest.predict(X_train)
y_test_pred = forest.predict(X_test)

print('MSE train: %.3f, test: %.3f' % (
        mean_squared_error(y_train, y_train_pred),
        mean_squared_error(y_test, y_test_pred)))
print('R^2 train: %.3f, test: %.3f' % (
        r2_score(y_train, y_train_pred),
        r2_score(y_test, y_test_pred)))

plt.scatter(y_train_pred,
            y_train_pred - y_train,
            c='steelblue',
            edgecolor='white',
            marker='o',
            s=35,
            alpha=0.9,
            label='training data')
plt.scatter(y_test_pred,
            y_test_pred - y_test,
            c='limegreen',
            edgecolor='white',
            marker='s',
            s=35,
            alpha=0.9,
            label='test data')

plt.xlabel('Predicted values')
plt.ylabel('Residuals')
plt.legend(loc='upper left')
plt.hlines(y=0, xmin=-10, xmax=50, lw=2, color='black')
plt.xlim([-10, 50])
plt.tight_layout()

plt.show()


# -*- coding: utf-8 -*-

"""
@date: 2022/5/16 下午4:32
@file: norm.py
@author: zj
@description:
"""

import torch

import numpy as np
from sklearn.preprocessing import normalize

def l2_norm(feats: torch.Tensor) -> torch.Tensor:
    if len(feats.shape) == 1:
        feats = feats.reshape(1, -1)

    return feats / torch.norm(feats, dim=1, keepdim=True)

def l2_norm_v2(feats: torch.Tensor) -> torch.Tensor:
    if len(feats.shape) == 1:
        feats = feats.reshape(1, -1)

    return feats / torch.linalg.norm(feats, dim=1, keepdim=True)

def l2_norm_v3(feats: torch.Tensor) -> torch.Tensor:
    if len(feats.shape) == 1:
        feats = feats.reshape(1, -1)

    norm_feats = feats.numpy() / np.linalg.norm(feats.numpy(), axis=1, keepdims=True)
    return torch.from_numpy(norm_feats)

def l2_norm_v4(feats: torch.Tensor) -> torch.Tensor:
    if len(feats.shape) == 1:
        feats = feats.reshape(1, -1)

    norm_feats = normalize(feats.numpy(), norm="l2", axis=1, return_norm=False)
    return torch.from_numpy(norm_feats)

if __name__ == '__main__':
    torch.manual_seed(0)

    data = torch.randn(3, 6)
    res = l2_norm(data)
    print(res)

    res2 = l2_norm_v2(data)
    print(res2)
    assert torch.allclose(res, res2)

    res3 = l2_norm_v3(data)
    print(res3)
    assert torch.allclose(res2, res3)

    res4 = l2_norm_v4(data)
    print(res4)
    assert torch.allclose(res3, res4)


import numpy
import sympy
from mpmath import mp

from ._helpers import LineSegmentScheme

def chebyshev_gauss_1(n, mode="numpy"):
    """Chebyshev-Gauss quadrature for \\int_{-1}^1 f(x) / sqrt(1+x^2) dx.
    """
    degree = n if n % 2 == 1 else n + 1

    if mode == "numpy":
        points = numpy.cos((2 * numpy.arange(1, n + 1) - 1) / (2 * n) * numpy.pi)
        weights = numpy.full(n, numpy.pi / n)
    elif mode == "sympy":
        points = numpy.array(
            [
                sympy.cos(sympy.Rational(2 * k - 1, 2 * n) * sympy.pi)
                for k in range(1, n + 1)
            ]
        )
        weights = numpy.full(n, sympy.pi / n)
    else:
        assert mode == "mpmath"
        points = numpy.array(
            [mp.cos(mp.mpf(2 * k - 1) / (2 * n) * mp.pi) for k in range(1, n + 1)]
        )
        weights = numpy.full(n, mp.pi / n)
    return LineSegmentScheme("Chebyshev-Gauss 1", degree, weights, points)

def chebyshev_gauss_2(n, mode="numpy", decimal_places=None):
    """Chebyshev-Gauss quadrature for \\int_{-1}^1 f(x) * sqrt(1+x^2) dx.
    """
    degree = n if n % 2 == 1 else n + 1

    # TODO make explicit for all modes
    if mode == "numpy":
        points = numpy.cos(numpy.pi * numpy.arange(1, n + 1) / (n + 1))
        weights = (
            numpy.pi
            / (n + 1)
            * (numpy.sin(numpy.pi * numpy.arange(1, n + 1) / (n + 1))) ** 2
        )
    elif mode == "sympy":
        points = numpy.array(
            [sympy.cos(sympy.Rational(k, n + 1) * sympy.pi) for k in range(1, n + 1)]
        )
        weights = numpy.array(
            [
                sympy.pi / (n + 1) * sympy.sin(sympy.pi * sympy.Rational(k, n + 1)) ** 2
                for k in range(1, n + 1)
            ]
        )
    else:
        assert mode == "mpmath"
        points = numpy.array(
            [mp.cos(mp.mpf(k) / (n + 1) * mp.pi) for k in range(1, n + 1)]
        )
        weights = numpy.array(
            [
                mp.pi / (n + 1) * mp.sin(mp.pi * mp.mpf(k) / (n + 1)) ** 2
                for k in range(1, n + 1)
            ]
        )
    return LineSegmentScheme("Chebyshev-Gauss 2", degree, weights, points)


from sympy.abc import x
import sympy as sp
import numpy as np
from lagrange_interpolation import lagrange_interpolation

def split(inte, n):
    """ Split the interval.

    Args:
        inte: list/ndarray, the interval to be splited
        n: int, the number of interpolation points

    Returns:
        t: list/ndarray, the split of the given interval
    """
    # the endpoints of the interval
    a, b = inte
    t = np.linspace(a, b, n+1)
    return t

def f(t):
    return 1/(1 + t**2)

def draw_polinomial(inte, func, lst):
    """ Draw the original function and the interpolation function

    Args:
        inte: list, the interval of the function
        func: symbol object, the original function
        lst: list, the list of the endpoints number
    """
    # draw the original function
    fig = sp.plot(func(x), (x, -5, 5), line_color='r', label='Original Fuction', legend=True, show=False)
    # control color
    i = 0
    colors = ['b', 'y', 'g']
    for n in lst:
        # polinomial interpolation
        t = split(inte, n)
        y = lagrange_interpolation(t, func(t))
        print(f"The {n}-order interpolation polinomial is {y}")
        # draw polinomial interpolation
        color = colors[i]
        i += 1
        p = sp.plot(y, (x, -5, 5), line_color=color, label=f'n={n}', lengend=True, show=False)
        fig.extend(p)
    fig.save('polinomial_interpolation.png')

def piecewise_linear_interpolation(t, y):
    """ Piecewise linear interpolation.

    Args:
        t: list/ndarray, the split of the interval
        y: list/ndarray, the value of the function at every interpolation point

    Returns:
        L: list, the expresion of the interpolation at every little interval
    """
    # the number of the interpolation points
    n = len(t)

    # iteration
    L = []
    for i in range(n-1):
        l = lagrange_interpolation(t[i:i+2], y[i:i+2])
        L.append((sp.simplify(l), sp.And(x >= t[i], x <= t[i+1])))

    return sp.Piecewise(*L)

def draw_piecewise(inte, func, lst):
    """ Draw the piecewise linear interpolation function and original function

    Args:
        inte: list/ndarray, the interpolatin interval
        func: function object, the original function to be interpolated
        lst: list/ndarray, list of interpolation endpoints number
    """
    # draw the original function
    fig = sp.plot(func(x), (x, -5, 5), line_color='r', label='Original Fuction', legend=True, show=False)
    # control color
    i = 0
    colors = ['b', 'y', 'g']

    for n in lst:
        # piecewice linear interpolation
        t = split(inte, n)
        y = piecewise_linear_interpolation(t, func(t))
        # draw polinomial interpolation
        color = colors[i]
        i += 1
        p = sp.plot(y, (x, -5, 5), line_color=color, label=f'n={n}', lengend=True, show=False)
        fig.extend(p)

    fig.save('piecewise_interpolation.png')

if __name__ == '__main__':
    inte = [-5, 5]
    n_list = [5, 10, 20]

    # polinomial interpolation
    draw_polinomial(inte, f, n_list)
    # piecewise interpolation
    draw_piecewise(inte, f, n_list)


import numpy as np
np.set_printoptions(precision=3, linewidth=200, suppress=True)
from random import seed
from random import randrange
from copy import deepcopy

def feature_normalize(features, n_type='z-score'):
    """

    :param features:
    :param n_type:
    :return:
    """
    answer = np.array([])
    if n_type == 'z-score':
        mean = np.mean(features, axis=0)
        std = np.std(features, axis=0)
        if std != 0:
            answer = (features - mean) / std
        else:
            answer = features
    elif n_type == 'min-max':
        minimum = features.min(axis=0)
        maximum = features.max(axis=0)
        if maximum != minimum:
            answer = (features - minimum)/(maximum-minimum)
        else:
            answer = features
    return answer

def normalize_data(train_features, n_type='z-score'):
    """
    Feature scaling
    :param train_features: Training features
    :param n_type: Type of normalization
    :return:
    """
    row_no, col_no = train_features.shape
    normalize_train_features = deepcopy(train_features)
    for column_no in range(col_no):
        test = train_features[:, column_no]
        normalize_train_features[:, column_no] = feature_normalize(test, n_type=n_type)
    return normalize_train_features

def train_validate_split(x_axis, features, labels, split=0.75, select_features=[], verbose=False, randomize=False, random_state=7):
    """
    Split the features into train and validate set based on split ratio
    :param x_axis:
    :param features:
    :param labels:
    :param split:
    :param select_features:
    :param verbose:
    :param randomize:
    :return:
    """
    if randomize is True:
        seed(random_state)
        order = np.random.permutation(x_axis.shape[0])
        x_axis = x_axis[order]
        features = features[order]
        labels = labels[order]
    num_train_samples = int(split*labels.shape[0])
    num_validate_samples = labels.shape[0] - num_train_samples
    if verbose is True:
        print('[ DEBUG] Feature selected for training: ' + str(select_features))
        print('[ DEBUG] Split into train and validate sets: ' + str(split))
        print('[ DEBUG] Number of training samples: ' + str(num_train_samples))
        print('[ DEBUG] Number of validating samples: ' + str(num_validate_samples))
    if split != 1.0:
        train_x_axis = x_axis[:num_train_samples].reshape(num_train_samples, 1)
        train_labels = labels[:num_train_samples].reshape(num_train_samples, 1)
        validate_x_axis = x_axis[num_train_samples:].reshape(num_validate_samples, 1)
        validate_labels = labels[num_train_samples:].reshape(num_validate_samples, 1)
        if len(select_features) == 0:
            train_features = features[:num_train_samples, :]
            validate_features = features[num_train_samples:, :]
        else:
            train_features = features[:num_train_samples, select_features]
            validate_features = features[num_train_samples:, select_features]
        if verbose is True:
            print('[ DEBUG] Shape of train features: ' + str(train_features.shape))
            print('[ DEBUG] Shape of train labels: ' + str(train_labels.shape))
            print('[ DEBUG] Shape of validate features: ' + str(validate_features.shape))
            print('[ DEBUG] Shape of validate labels: ' + str(validate_labels.shape))
        return train_x_axis, train_features, train_labels, validate_x_axis, validate_features, validate_labels
    else:
        if verbose is True:
            print('[ DEBUG] Shape of features: ' + str(features.shape))
            print('[ DEBUG] Shape of labels: ' + str(labels.shape))
        return x_axis, features, labels

def cross_validate_split(x_axis, features, labels, select_features=[], num_folds=9, randomize=False, random_state=7, verbose=False):
    validate_size = int(len(features) / num_folds)
    num_examples = features.shape[0]
    start_index = 0
    validate_features = list()
    train_features = list()
    validate_labels = list()
    train_labels = list()
    for i in range(num_folds):
        end_index = start_index + validate_size
        validate_features_split = list()
        train_features_split = list()
        validate_labels_split = list()
        train_labels_split = list()
        for j in range(num_examples):
            if j >= start_index and j < end_index:
                validate_features_split.append(features[j, :])
                validate_labels_split.append(labels[j])
            else:
                train_features_split.append(features[j, :])
                train_labels_split.append(labels[j])
        start_index += validate_size
        validate_features.append(validate_features_split)
        train_features.append(train_features_split)
        validate_labels.append(validate_labels_split)
        train_labels.append(train_labels_split)
    return np.array(train_features), np.array(train_labels), np.array(validate_features), np.array(validate_labels)

def generate_dummy_data(number_of_points=1000, bias = True, randomize=True, verbose=False):
    """

    :param number_of_points:
    :param bias:
    :param randomize:
    :param verbose:
    :return:
    """
    x_axis = np.linspace(1.0, 20.0, number_of_points)[:, np.newaxis]
    if randomize is True:
        order = np.random.permutation(x_axis.shape[0])
        x_axis = x_axis[order]
    # data_y = np.sin(x_axis) + 0.1 * np.power(x_axis, 2) + 0.5 * np.random.randn(number_of_points, 1)
    data_y = -2 * np.power(x_axis + 2, 2) + np.power(x_axis - 3, 3) + 5 * np.random.randn(number_of_points, 1)
    if bias is True:
        if verbose is True:
            print('[ DEBUG] Data is generated with bias')
        data_y += 16.5
    model_order = 4
    data_x = np.power(x_axis, range(model_order))
    data_x /= np.max(data_x, axis=0)
    return x_axis, data_x, data_y

def read_data_from_csv(csv_file='./input/data.csv', label_name='', verbose=False):
    """

    :param csv_file:
    :param label_name:
    :param verbose:
    :return:
    """
    # Read data from csv
    if verbose is True:
        print('[ DEBUG] Reading data from ', csv_file)
    array = np.genfromtxt(csv_file, delimiter=',', names=True)
    column_names = array.dtype.names
    if label_name == '':
        number_of_columns = len(column_names)
        labels = []
    else:
        number_of_columns = len(column_names)-1
        labels = array[label_name]
    features = array[column_names[0]]
    for feature in range(1, number_of_columns):
        features = np.column_stack((features, array[column_names[feature]]))
    return features, labels, column_names

def write_data_to_csv(matrix, fmt, csv_file='./output/output.csv', heading_row='', verbose=False):
    if verbose is True:
        print('[ DEBUG] Writing data to ' + csv_file)
    with open(csv_file, 'wb') as f:
        f.write(bytes(heading_row, encoding='UTF-8'))
        np.savetxt(f, matrix, fmt=fmt, delimiter=',')