'''
Michael Lam
ASTP-720, Fall 2020

Cooley-Tukey Fast Fourier Transform Algorithm
'''

import numpy as np

def fft(h_n, threshold=16):
    """
    Cooley-Tukey Fast Fourier Transform Algorithm

    Parameters
    ----------
    h_n : list, np.ndarray
        Array to perform FFT, h_n
    threshold (optional) : int
        Threshold below which one does the classic DFT calculation

    Returns
    -------
    H_k : np.ndarray
        Discrete Fourier Transform, H_k
    """

    N = len(h_n)
    range_arr = np.arange(N)
    w = np.exp(-2*np.pi*1j/N)

    if N <= threshold:
        # W_kn matrix, see Lecture 19 notes
        # First, define the the range [0, 1, ... N-1]
        # Then, use meshgrid and multiply the results to get the powers
        # of w, i.e., w^(kn) power.
        n_grid, k_grid = np.meshgrid(range_arr, range_arr)
        W_kn = w**(k_grid*n_grid)
        # Now dot with h, H = W@h
        H_k = W_kn @ h_n
    else:
        # Use slice indexing to get the even and odd parts of h_n
        h_n_even = h_n[::2]
        h_n_odd = h_n[1::2]
        # Recursively call fft()
        E_k = fft(h_n_even)
        O_k = fft(h_n_odd)
        # Now combine, with w^k, make this an array for ease
        w_pow_k = w**range_arr
        H_k = np.zeros(N, dtype=np.complex)
        H_k[:N//2] = E_k + O_k*w_pow_k[:N//2] #first half-interval of H_k
        H_k[N//2:] = E_k + O_k*w_pow_k[N//2:] #second half-interval of H_k

    return H_k


from numpy import random, pi, mean

Ntrials, Nhits = 1_000_000, 0
absXPlusY = []
for n in range(Ntrials):
    x, y = random.uniform(-1, 1, 2) # draw 2 samples, each uniformly distributed over (-1,1)
    if (x**2 + y**2)**2 <= 2*abs(x*y):
        absXPlusY.append(abs(x+y))
        Nhits += 1
VL = 4 * Nhits / Ntrials
print("Monte Carlo estimator of V(L): %.5f" % (VL))
print("Monte Carlo estimator of I: %.5f" % (VL * mean(absXPlusY)))


import numpy as np

'''
    The Euclidean distance (L2 norm) is the easiest to
    understand distance calculation method derived from
    the distance formula between two points in Euclidean space
'''

objA = [22, 1, 42, 10]

objB = [20, 0, 36, 8]

npA = np.array(objA)

npB = np.array(objB)

euclidean = np.sqrt(np.sum(np.square(npA - npB)))

# euclidean = np.linalg.norm(npA - npB)

print(euclidean)


import numpy as np

def solve_matrix(augmented_matrix: np.ndarray) -> (np.ndarray, np.ndarray):
    augmented_matrix = np.array(augmented_matrix.copy())

    # Split matrix to matrix and results_vector
    results_vector: np.ndarray = augmented_matrix[:, -1]
    matrix: np.ndarray = np.delete(augmented_matrix, -1, axis=1)

    # check is matrix square
    if matrix.shape[0] != matrix.shape[1]:
        raise WrongInputException
    check_if_matrix_has_solution(augmented_matrix, matrix)

    # Create identity matrix
    reversed_matrix: np.ndarray = np.zeros(matrix.shape)
    np.fill_diagonal(reversed_matrix, 1)

    # sort rows
    matrix, reversed_matrix, results_vector = \
        sort_for_pivots(matrix, reversed_matrix, results_vector)

    for i in range(matrix.shape[1]):
        factor = matrix[i, i]
        # pivot cannot equal to zero
        if factor == 0:
            raise NoneSolutionException

        # Transform row to have one on diagonal
        matrix[i] /= factor
        results_vector[i] /= factor
        reversed_matrix[i] /= factor

        # Subtract this row from rest of the rows
        for j in range(i + 1, matrix.shape[1]):
            factor = matrix[j, i]
            matrix[j] -= matrix[i] * factor
            results_vector[j] -= results_vector[i] * factor
            reversed_matrix[j] -= reversed_matrix[i] * factor

    # Subtract rows from end to make matrix diagonal
    for i in range(matrix.shape[1])[::-1]:
        for j in range(i):
            factor = matrix[j, i]
            matrix[j] -= matrix[i] * factor
            results_vector[j] -= results_vector[i] * factor
            reversed_matrix[j] -= reversed_matrix[i] * factor

    return results_vector, reversed_matrix

def check_if_matrix_has_solution(augmented_matrix, matrix):
    # Check if matrix has solutions by comparing ranks
    matrix_rank = np.linalg.matrix_rank(matrix)
    augmented_matrix_rank = np.linalg.matrix_rank(augmented_matrix)
    if matrix_rank != augmented_matrix_rank:
        raise NoneSolutionException
    if matrix_rank == augmented_matrix_rank and matrix_rank < matrix.shape[1]:
        raise InfiniteSolutionException
    if np.linalg.det(matrix) == 0:
        raise DeterminantIsZeroException

def sort_for_pivots(matrix: np.ndarray, reversed_matrix: np.ndarray, results_vector: np.ndarray) \
        -> (np.ndarray, np.ndarray, np.ndarray):
    # perform column swap
    for col in range(matrix.shape[1] - 1):
        index_max_in_col = matrix[col:, col].argmax() + col
        swap_vector = [index_max_in_col, col]
        for change_matrix in (matrix, reversed_matrix, results_vector):
            change_matrix[swap_vector] = change_matrix[swap_vector[::-1]]

    return matrix, reversed_matrix, results_vector

class NoneSolutionException(Exception):
    pass

class InfiniteSolutionException(Exception):
    pass

class WrongInputException(Exception):
    pass

class DeterminantIsZeroException(Exception):
    pass


import numpy as np
from mpl_toolkits.mplot3d import Axes3D
from matplotlib import pyplot as plt
import tensorflow as tf

def himmelblau(x):
    # himmelblau函数实现
    return (x[0] ** 2 + x[1] - 11) ** 2 + (x[0] + x[1] ** 2 - 7) ** 2

x = np.arange(-6, 6, 0.1)
y = np.arange(-6, 6, 0.1)
print('x,y range:', x.shape, y.shape)
# 生成x-y平面采样网格点，方便可视化
X, Y = np.meshgrid(x, y)
print('X,Y maps:', X.shape, Y.shape)
Z = himmelblau([X, Y])  # 计算网格点上的函数值

# 绘制himmelblau函数曲面
fig = plt.figure('himmelblau')
ax = fig.gca(projection='3d')
ax.plot_surface(X, Y, Z)
ax.view_init(60, -30)
ax.set_xlabel('x')
ax.set_ylabel('y')
plt.show()

# 查找最小解
# 参数的初始化值对优化的影响不容忽视，可以通过尝试不同的初始化值，
# 检验函数优化的极小值情况
# 随机不同的点
# [1., 0.], [-4, 0.], [4, 0.]
# x = tf.constant([4., 0.])
# x = tf.constant([1., 0.])
# x = tf.constant([-4., 0.])
x = tf.constant([-2., 2.])

for step in range(200):  # 循环优化
    with tf.GradientTape() as tape:  # 梯度跟踪
        tape.watch([x])  # 记录梯度
        y = himmelblau(x)  # 前向传播
    # 反向传播
    grads = tape.gradient(y, [x])[0]
    # 更新参数,0.01为学习率
    x -= 0.01 * grads
    # 打印优化的极小值
    if step % 20 == 0:
        print('step {}: x = {}, f(x) = {}'
              .format(step, x.numpy(), y.numpy()))


"""ЛР 3.3, Ларькин Владимир, М8О-303Б-18"""

import numpy as np
import fire  # CLI
import matplotlib.pyplot as plt

from sem1.lab1_1.gauss import lu_decomposition, lu_solve

def f(coeffs, x):
    """Вычисление значения полинома с коэффициентами coeffs"""
    return sum([x ** i * c for i, c in enumerate(coeffs)])

def sum_squared_errors(f, y):
    """Сумма квадратов ошибок"""
    return sum((f_i - y_i) ** 2 for f_i, y_i in zip(f, y))

def lsm(x, y, n):
    """Подбор коэффициентов полинома степени n с помощью МНК"""

    N = len(x)
    mat = [[sum([x_j ** (i + j) for x_j in x]) for i in range(n + 1)] for j in range(n + 1)]
    mat[0][0] = N + 1
    b = [sum([x_j ** i * y_j for x_j, y_j in zip(x, y)]) for i in range(n + 1)]

    mat = np.array(mat)
    b = np.array(b)
    p, l, u = lu_decomposition(mat)
    b = b @ p
    coeffs = lu_solve(l, u, b)

    return coeffs

def main():
    """Аппроксимация таблично заданной функции многочленами 1-й и 2-й степеней с помощью МНК"""

    init_dict = {
        "x": [-3.0, -2.0, -1.0, 0.0, 1.0, 2.0],
        "y": [0.04979, 0.13534, 0.36788, 1.0, 2.7183, 7.3891]
    }

    x, y = init_dict["x"], init_dict["y"]

    xc = np.arange(min(x) - 1, max(x) + 1, .01)

    c1 = lsm(x, y, 1)
    y1 = f(c1, xc)

    c2 = lsm(x, y, 2)
    y2 = f(c2, xc)

    plt.plot(x, y, "o", label="Входные данные")
    plt.plot(xc, y1, label="Полином первой степени")
    plt.plot(xc, y2, label="Полином второй степени")

    plt.title("Аппроксимация МНК")
    plt.grid(True)
    plt.legend()

    plt.savefig("plot.jpg", dpi=300)
    plt.show()

    e1 = sum_squared_errors(lsm(x, y, 1), y)
    e2 = sum_squared_errors(lsm(x, y, 2), y)

    print("Сумма квадратов ошибок:")
    print("\tn = 1:", e1)
    print("\tn = 2:", e2)

    print("\nАппроксимирующие функции:")

    xs = ["", "x", "x^2"]
    for c, n in zip([c1, c2], [1, 2]):
        print(f"\tn = {n}:", " + ".join([f"{v:.6f}{x}" for v, x in zip(c, xs)]))

if __name__ == "__main__":
    fire.Fire(main)


import numpy as np

I = np.eye(2, dtype='complex128')
X = np.array([[0, 1], [1, 0]], dtype='complex128')
Y = np.array([[0, -1j], [1j, 0]], dtype='complex128')
Z = np.array([[1, 0], [0, -1]], dtype='complex128')
CX = np.array([[1, 0, 0, 0],
               [0, 1, 0, 0],
               [0, 0, 0, 1],
               [0, 0, 1, 0]], dtype='complex128')
ket_0 = np.array([[1], [0]], dtype='complex128')
ket_1 = np.array([[0], [1]], dtype='complex128')

def NKron(*args):
    """Calculate a Kronecker product over variable number of inputs. Credit
       to Jarrod McClean."""
    # 1x1 matrix to start with
    result = np.array([[1.0]])
    for op in args:
        result = np.kron(result, op)
    return result

def NDot(*args):
    """Normal matrix multiplication using np.dot(a,b)"""
    if len(args) < 2:
        raise ValueError("NDot needs at least 2 matrices to multiply")
    result = args[0]
    for op in args[1:]:
        result = np.dot(result, op)
    return result

def array_eq(a, b, tol):
    """
    Checks if matrix a is equal to matrix b by doing elementwise
    comparisons, allowing tolerance tol.
    """
    if a.shape != b.shape:
        return False
    for index, x in np.ndenumerate(a):
        if np.abs(x - b[index]) > tol:
            return False
    return True

def adjoint(u):
    """Returns conjugate transpose of u."""
    return np.transpose(np.conjugate(u))


# -*- coding: utf-8 -*-

import matplotlib.pyplot as plt
import matplotlib.lines as mlines
import numpy as np
import math as m
from array import array

### Funcao pra achar a raiz ###

def F(x):
    #return (x/2.0)**2 - m.sin(x)
    return x**2 - 3.0 * x + 2.0
    #return m.cos(x)

### Constantes ###

a = -5
b = 5
E = 0.001

## checa intervalos e procura menor com uma raiz ##
## se encontrar, coloca no array e parte pro proximo ##

intervalo = b - a

multiplicador = 2.0
np = intervalo * multiplicador
dx = intervalo / np

arr_intervalos = []

for i in range(1, int(np)):
    pa = ((i - 1) * dx) + a
    p = (i * dx) + a

    ya = F(pa)
    y = F(p)

    #print ("fx: ", ya, y)

    if((ya >= 0 and y < 0) or (ya < 0 and y >= 0)):
        arr_intervalos.append([pa, p])

def bissecao(_a, _b):

    e = _b - _a
    mx = e / 2.0
    _x = mx + _a
    iteracoes = 0

    #checa se a raiz esta nos intervalos
    if(F(_a) == 0.0):
        return _a
    if(F(_b) == 0.0):
        return _b

    while(e > E and iteracoes < 200):
        iteracoes = iteracoes + 1

        #acha meio
        e = _b - _a
        mx = e / 2.0
        _x = mx + _a

        #determina valores
        fa = F(_a)
        fb = F(_b)
        fx = F(_x)

        #se a raiz estiver entre um dos intervalos formados, ajusta intervalo (_a, _b)
        if((fa >= 0 and fx < 0) or (fa < 0 and fx >= 0)):
            #intervalo a-x
            _b = _x
        else:
            #intervalo x-b
            _a = _x

    #atingiu condicao de parada
    return _x

def FalsaPosicao(_a,_b):

    e = E #epsilon - precisão fornecida

    #Checa a condição de existencia
    if (F(_a) * F(_b) < 0):

        while (abs(F(_x)) > e and abs(_b - _a) > e):

            if(F(_a) * F(_b) < 0):
                _a = _x
            else:
                _b = _x

            _x = (_a * F(_b) + _b * F(_a)) / (F(_b) - F(_a))

        return _x

#print(arr_intervalos)

if(len(arr_intervalos) == 0):
    print("Nenhuma raiz encontrada")
else:
    for i in range(len(arr_intervalos)):
        intervalo = arr_intervalos[i]
        print("raiz #" + str(i))
        print ("pesquisar no intervalo (" + str(intervalo[0]) + ", " + str(intervalo[1]) + ")")
        raiz = bissecao(intervalo[0], intervalo[1])
        print("raiz encontrada: " + str(raiz) + "\n")


import numpy as np
import matplotlib.pyplot as plt

def drawEllipsoid(rx, ry, rz, plotFig=True, res=100):
    """
    Calculate and draw ellipsoid
    ===========================================================================
    Input       Meaning
    ----------  ---------------------------------------------------------------
    rx, ry, rz  Radius of the ellipsoid in x, y, z
    plotFig     Plot figure [True / False]
    res         Resolution, i.e. number of data points
    ===========================================================================
    Output      Meaning
    ----------  ---------------------------------------------------------------
    [x, y, z]   List of x, y, z coordinates of the ellipsoid
    Figure      Plot with ellipsoid if plotFig==True
    ===========================================================================
    """

    # Set of all spherical angles:
    u = np.linspace(0, 2 * np.pi, res)
    v = np.linspace(0, np.pi, res)

    # Cartesian coordinates that correspond to the spherical angles:
    # (this is the equation of an ellipsoid):
    x = rx * np.outer(np.cos(u), np.sin(v))
    y = ry * np.outer(np.sin(u), np.sin(v))
    z = rz * np.outer(np.ones_like(u), np.cos(v))

    # Plot:
    if plotFig:
        fig = plt.figure()
        ax = plt.axes(projection = "3d")
        ax.plot_surface(x, y, z,  rstride=4, cstride=4, color='b', alpha=0.2)
        Min = np.min([np.min(x), np.min(y), np.min(z)])
        Max = np.max([np.max(x), np.max(y), np.max(z)])
        ax.set_xlim([Min, Max])
        ax.set_ylim([Min, Max])
        ax.set_zlim([Min, Max])

    return([x, y, z])


from numpy import array, sin, cos, exp, sqrt, pi
from NIM.benchmarks.benchmark import Benchmark

class Holdertable(Benchmark):
    """dim: 2"""
    def __init__(self, lower=-10, upper=10, dimension=2):
        super(Holdertable, self).__init__(lower, upper, dimension)

    def get_optimum(self):
        return array([[8.05502, 9.66459], [8.05502, -9.66459], [-8.05502, 9.66459], [-8.05502, -9.66459]]), -19.208502567767606

    @staticmethod
    def eval(sol):
        term1 = sin(sol[0]) * cos(sol[1])
        term2 = exp(abs(1 - sqrt(sol[0] ** 2 + sol[1] ** 2) / pi))

        return -abs(term1 * term2)


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Tue Sep  1 07:29:12 2020

Numerical soluation to a 2D potential problem similar to those from class

@author: zettergm
"""

import numpy as np
from ittools import Jacobi,GaussSeidel
import matplotlib.pyplot as plt
import scipy.sparse
import scipy.sparse.linalg

# parameters of problem:
lx=24; ly=24;
N=lx*ly    # total number of grid points
a=1; b=1;   # square region

# create a 2D grid
x=np.linspace(0,a,lx)
y=np.linspace(0,b,ly)
dx=x[1]-x[0]
dy=y[1]-y[0]
[X,Y]=np.meshgrid(x,y)

# Dirichlet boundary condition for four sides of square
f1=np.ones( (lx) )
f2=np.ones( (lx) )
g1=np.zeros( (ly) )
g2=np.zeros( (ly) )

# Right-hand side of Poisson equation, viz. rho/eps0
rhs=np.zeros( (N) )

# Matrix defining finite-difference equation for laplacian operator
M=np.zeros( (N,N) )    # solutions are miserably slow using sparse storage for some reason...
for i in range(0,lx):
    for j in range(0,ly):
        k=j*lx+i    # linear index referencing i,j grid point
        if j==0:
            M[k,k]=1
            rhs[k]=f1[i]
        elif j==ly-1:
            M[k,k]=1
            rhs[k]=f2[i]
        elif i==0:
            M[k,k]=1
            rhs[k]=g1[j]
        elif i==lx-1:
            M[k,k]=1
            rhs[k]=g2[j]
        else:
            M[k,k-lx]=1/dy**2
            M[k,k-1]=1/dx**2
            M[k,k]=-2/dx**2-2/dy**2
            M[k,k+1]=1/dx**2
            M[k,k+lx]=1/dy**2
            rhs[k]=0

# Jacobi iterative solution
Phi0=np.ones( (N) )    # use a bunch of ones for the inital guess
tol=1
print("---------------------------------------------------------------------")
[PhiJ,iteration]=Jacobi(Phi0,M,rhs,tol,False)
print("---------------------------------------------------------------------")
print("Jacobi Iterative solution done...")
print(x)
print("Number of iterations required and tolerance")
print(iteration)
print(tol)

# Gauss-Seidel iterative solution, usually will use fewer iterations than Jacobi
print("---------------------------------------------------------------------")
[PhiGS,iteration]=GaussSeidel(Phi0,M,rhs,tol,False)
print("---------------------------------------------------------------------")
print("Gauss-Seidel Iterative solution done...")
print(x)
print("Number of iterations required and tolerance")
print(iteration)
print(tol)

# Solution with umfpack, note how fast this is compared to the iterative solutions :)
print("---------------------------------------------------------------------")
Msparse=scipy.sparse.csr_matrix(M)
# normally more efficient to make the csr matrix on a per-entry basis
#   but we alread have the full version....
rhssparse=scipy.sparse.csr_matrix(np.reshape(rhs,[N,1]))
PhiUMF=scipy.sparse.linalg.spsolve(Msparse,rhssparse,use_umfpack=True)
print("---------------------------------------------------------------------")
print("Solution with UMFPACK done...")

# reorganize solution data
PhiJ=np.reshape(PhiJ, (lx,ly))
PhiGS=np.reshape(PhiGS, (lx,ly))
PhiUMF=np.reshape(PhiUMF, (lx,ly))

# plot
plt.subplots(1,3,figsize=(14,6),dpi=100)
plt.subplot(1,3,1)
plt.pcolormesh(x,y,PhiJ,shading="auto")
plt.xlabel("x")
plt.ylabel("y")
plt.title("Jacobi [V]")
plt.colorbar()

plt.subplot(1,3,2)
plt.pcolormesh(x,y,PhiGS,shading="auto")
plt.xlabel("x")
plt.ylabel("y")
plt.title("Gauss-Seidel [V]")
plt.colorbar()

plt.subplot(1,3,3)
plt.pcolormesh(x,y,PhiUMF,shading="auto")
plt.xlabel("x")
plt.ylabel("y")
plt.title("Sparse LU factorization [V]")
plt.colorbar()


import numpy as np
from functools import cached_property

class MarkovChain:
    """ Discrete-time Markov chain. Provides methods to draw samples.

    The magnitudes of the products of transition rates with frame intervals
    are assumed to be <<1. (That is, we assume that the probability of two
    state transitions in one frame interval is negligible.)

    init
    ----
        TM      :   2D ndarray, transition rates. The element TM[i,j] is assumed
                    to give the transition rate from the i^th to the j^th state
                    in Hz.

        dt      :   str, frame interval (seconds)

    """
    def __init__(self, TM: np.ndarray, dt: float=1.0):
        self.dt = dt
        self.TM = np.asarray(TM)
        assert len(self.TM.shape) == 2
        assert TM.shape[0] == TM.shape[1]

        self.n_states = self.TM.shape[0]
        self.states = np.arange(self.n_states)

        # Normalize transition matrix
        self.P = self.TM.copy() * dt
        self.P[self.states, self.states] = 1.0 - \
            (self.P.sum(axis=1) - np.diagonal(self.P))

    def is_diag(self, M: np.ndarray) -> bool:
        """ Return True if a 2D ndarray is diagonal, 0 otherwise. """
        i, j = np.nonzero(M)
        return np.all(i==j)

    @cached_property
    def stat_dist(self) -> np.ndarray:
        """ Stationary distribution; 1D ndarray of length (n_states,).

        Returns
        -------
            1D ndarray of shape (n_states,), the stationary
                distribution for the Markov chain

        """
        if self.is_diag(self.P):
            return np.diag(self.TM) / self.TM.sum()
        else:
            L, V = np.linalg.eig(self.P.T)
            v = V[:,np.argmin(np.abs(L-1.0))]
            return v / v.sum()

    def __call__(self, n: int, initial: int=None) -> np.ndarray:
        """
        Simulate a single state history of length *n*.

        If no initial state is provided, the initial state is drawn
        from the stationary distribution.

        Parameters
        ----------
            n   :   int, number of timepoints to simulate

        Returns
        -------
            1D ndarray of shape (n,), the state indices at each
                timepoint

        """
        if self.is_diag(self.TM):
            return np.ones(n, dtype=np.int64) * (np.random.choice(self.states, p=self.stat_dist) if initial is None else initial)

        else:
            s = np.empty(n, dtype=np.int64)

            if initial is None:
                s[0] = np.random.choice(self.states, p=self.stat_dist)
            else:
                s[0] = initial

            for j in range(1, n):
                s[j] = np.random.choice(self.states, p=self.P[s[j-1],:])
            return s

    def __enter__(self):
        return self

    def __exit__(self, etype, evalue, traceback):
        return etype is None


"""
Solving system of equations:
x[0]**3+x[1]**3-9 = 0
x[0]-0.5*x[1] = 0
cos(x[2])+x[0]-1.5 = 0
"""

from openopt import SNLE
from numpy import asfarray, zeros, cos, sin

#f = lambda x: (x[0]**3+x[1]**3-9, x[0]-0.5*x[1], cos(x[2])+x[0]-1.5)
# or:
f = (lambda x: x[0]**3+x[1]**3-9, lambda x: x[0]-0.5*x[1], lambda x: cos(x[2])+x[0]-1.5)
# Python list, numpy.array are allowed as well:
#f = lambda x: [x[0]**3+x[1]**3-9, x[0]-0.5*x[1], cos(x[2])+x[0]-1.5]
#or f = lambda x: asfarray((x[0]**3+x[1]**3-9, x[0]-0.5*x[1], cos(x[2])+x[0]-1.5))

# start point
x0 = [8,15, 80]

#optional: gradient
df = (lambda x: [3*x[0]**2, 3*x[1]**2,  0], lambda x: [1, -0.5,  0], lambda x:[1, 0, -sin(x[2])])

#w/o gradient:
#p = SNLE(f, x0)
p = SNLE(f, x0, df = df)

#optional: user-supplied gradient check:
#p.checkdf()

#optional: graphical output, requires matplotlib installed
p.plot = 1

#r = p.solve('scipy_fsolve')
r = p.solve('nssolve')
#or using converter to nlp, try to minimize sum(f_i(x)^2):
#r = p.solve('nlp:ralg')

print('solution: %s' % r.xf)
print('max residual: %e' % r.ff)
###############################
#should print:
#solution: [  1.           2.          55.50147021] (3rd coord may differ due to cos is periodic)
#max residual: 2.72366951215e-09


""" Data in populations.txt describes the populations of hares and lynxes (and carrots) in northern Canada during 20 years. """

import numpy as np

# load data
data = np.loadtxt('populations.txt')
""" array([[ 1900., 30000.,  4000., 48300.],
       [ 1901., 47200.,  6100., 48200.],
       [ 1902., 70200.,  9800., 41500.],
       [ 1903., 77400., 35200., 38200.],
       [ 1904., 36300., 59400., 40600.],
       [ 1905., 20600., 41700., 39800.],
       [ 1906., 18100., 19000., 38600.],
       [ 1907., 21400., 13000., 42300.],
       [ 1908., 22000.,  8300., 44500.],
       [ 1909., 25400.,  9100., 42100.],
       [ 1910., 27100.,  7400., 46000.],
       [ 1911., 40300.,  8000., 46800.],
       [ 1912., 57000., 12300., 43800.],
       [ 1913., 76600., 19500., 40900.],
       [ 1914., 52300., 45700., 39400.],
       [ 1915., 19500., 51100., 39000.],
       [ 1916., 11200., 29700., 36700.],
       [ 1917.,  7600., 15800., 41800.],
       [ 1918., 14600.,  9700., 43300.],
       [ 1919., 16200., 10100., 41300.],
       [ 1920., 24700.,  8600., 47300.]]) """

year, hares, lynxes, carrots = data.T #columns to variables

carrots
""" array([48300., 48200., 41500., 38200., 40600., 39800., 38600., 42300.,
       44500., 42100., 46000., 46800., 43800., 40900., 39400., 39000.,
       36700., 41800., 43300., 41300., 47300.]) """

populations_of_hares_lynxes_carrots = data[:,1:]
""" array([[30000.,  4000., 48300.],
       [47200.,  6100., 48200.],
       [70200.,  9800., 41500.],
       [77400., 35200., 38200.],
       [36300., 59400., 40600.],
       [20600., 41700., 39800.],
       [18100., 19000., 38600.],
       [21400., 13000., 42300.],
       [22000.,  8300., 44500.],
       [25400.,  9100., 42100.],
       [27100.,  7400., 46000.],
       [40300.,  8000., 46800.],
       [57000., 12300., 43800.],
       [76600., 19500., 40900.],
       [52300., 45700., 39400.],
       [19500., 51100., 39000.],
       [11200., 29700., 36700.],
       [ 7600., 15800., 41800.],
       [14600.,  9700., 43300.],
       [16200., 10100., 41300.],
       [24700.,  8600., 47300.]]) """

# standard deviations of populations via columns
populations_of_hares_lynxes_carrots.std(axis=0)
# array([20897.90645809, 16254.59153691,  3322.50622558])

# which species has the highest population each year?

np.argmax(populations_of_hares_lynxes_carrots, axis=1)
# array([2, 2, 0, 0, 1, 1, 2, 2, 2, 2, 2, 2, 0, 0, 0, 1, 2, 2, 2, 2, 2], dtype=int64)


import numpy as np

def calc_euclidean_distance(v1: np.ndarray, v2: np.ndarray) -> float:
    """ユークリッド距離を計算する

    Args:
        v1 (np.ndarray): ベクトル
        v2 (np.ndarray): ベクトル

    Returns:
        float: ユークリッド距離
    """
    # np.linalg.norm(v1-v2)としてもよい
    dist: float = np.sqrt(np.sum(np.square(v1 - v2)))
    return dist

def calc_euclidean_similarity(v1: np.ndarray, v2: np.ndarray) -> float:
    """ユークリッド距離を利用した類似度を計算する

    Args:
        v1 (np.ndarray): ベクトル
        v2 (np.ndarray): ベクトル

    Returns:
        float: 類似度
    """
    # np.linalg.norm(v1-v2)としてもよい
    sim: float = 1 / (1 + calc_euclidean_distance(v1, v2))
    return sim

def calc_similarity_with_missing_value(v1: np.ndarray, v2: np.ndarray,
                                       metric: str = 'euclidean',
                                       missing_value: float = 0) -> float:
    """類似度を計算する関数。ベクトルのどちらかに欠損値が含まれる場合、該当idxは無視する。

    Args:
        v1 (np.ndarray): ベクトル
        v2 (np.ndarray): ベクトル
        metric (str, optional): スコア計算のメトリック
        missing_value (float, optional): 欠損値

    Returns:
        float: 類似度
    """
    idx: np.array = np.where((v1 != missing_value) & (v2 != missing_value))[0]
    if len(idx) == 0:
        return 0
    sim: float = calc_similarity(v1[idx], v2[idx], metric)
    return sim

def calc_similarity(v1: np.ndarray, v2: np.ndarray,
                    metric: str = 'euclidean') -> float:
    """類似度を計算する関数。metricによる分岐はこの関数で処理する。

    Args:
        v1 (np.ndarray): ベクトル
        v2 (np.ndarray): ベクトル
        metric (str, optional): スコア計算のメトリック

    Returns:
        float: 類似度
    """

    sim: float
    if (metric == 'euclidean'):
        sim = calc_euclidean_similarity(v1, v2)
    elif (metric == 'pearson'):
        sim = calc_pearson_correlation_coefficient(v1, v2)
    return sim

def calc_pearson_correlation_coefficient(v1: np.ndarray, v2: np.ndarray) \
        -> float:
    """ピアソン相関係数を計算する

    Args:
        v1 (np.ndarray): ベクトル
        v2 (np.ndarray): ベクトル

    Returns:
        float: ピアソン相関係数
    """
    v1_diff = (v1 - v1.mean())
    v2_diff = (v2 - v2.mean())
    square_denominator: float = np.sum(v1_diff**2) * np.sum(v2_diff**2)
    if square_denominator == 0:
        return 0
    coef: float = np.sum(v1_diff * v2_diff) / np.sqrt(square_denominator)
    return coef


import numpy as np
import matplotlib.pyplot as plt

# function to optimize
# assume input x is a numpy array, not a list
def peaks(x):
  a = 3*(1-x[0])**2*np.exp(-(x[0]**2) - (x[1]+1)**2)
  b = 10*(x[0]/5 - x[0]**3 - x[1]**5)*np.exp(-x[0]**2-x[1]**2)
  c = (1/3)*np.exp(-(x[0]+1)**2 - x[1]**2)
  return a - b - c + 6.551 # add this so objective is always positive

ub = 3.0
lb = -3.0

d = 2 # dimension of decision variable space
s = 0.5 # stdev of normal noise (if this is too big, it's just random search!)
num_seeds = 5
max_NFE = 100000
ft = np.zeros((num_seeds, max_NFE))
xt = np.zeros((2, max_NFE))
T0 = 100 # initial temperature
alpha = 0.95 # cooling parameter

# three different x's and f's we're keeping track of:
# x_current: the location right now, which can improve or worsen
# x_best: the best point found so far (can only improve)
# x_trial: a new candidate point which we may or may not accept

# simulated annealing
for seed in range(num_seeds):
  np.random.seed(seed)
  T = T0

  # random initial starting point
  x_current = np.random.uniform(lb, ub, d)
  f_current = peaks(x_current)
  x_best,f_best = x_current,f_current

  for i in range(max_NFE):

    # do not allow points outside the constraints
    x_trial = np.clip(x_current + np.random.normal(0,s,d), lb, ub)
    f_trial = peaks(x_trial)

    r = np.random.rand()
    if T > 10**-3: # protect division by zero
      P = np.min([1.0, np.exp((f_current - f_trial)/T)])
    else:
      P = 0.0

    # keep better solutions, and worse ones sometimes
    if f_trial < f_current or r < P:
      x_current = x_trial
      f_current = f_trial

    if f_trial < f_best:
      x_best = x_trial
      f_best = f_trial

    T = T0*alpha**i
    ft[seed,i] = f_current
    xt[:,i] = x_current

  # for each trial print the result (but the traces are saved in ft)
  print(x_best)
  print(f_best)

plt.subplot(1,2,1)
xx = np.arange(-3,3,0.01)
X1,X2 = np.meshgrid(xx, xx)
Z = peaks([X1,X2])
plt.contour(X1,X2,Z,50,cmap=plt.cm.Blues_r)
# if you run multiple seeds, this will only plot the last trace of xt
plt.plot(xt[0,:], xt[1,:], color='k', linewidth=2)
plt.xlabel('X1')
plt.ylabel('X2')
plt.colorbar()

plt.subplot(1,2,2)
plt.loglog(ft.T, color='steelblue', linewidth=1)
plt.xlabel('Iterations')
plt.ylabel('Objective Value')
plt.show()


# Università di Bologna
# Corso di laurea in Informatica
# 02023 - Calcolo numerico
# 29/09/2021
#
# Stefano Volpe #969766
# lab1.py

import numpy as np
import matplotlib.pyplot as plt

def es1():
    """
    Execute the following code
        import sys
        help(sys.float_info)
        print(sys.float_info)
    and understand the meaning of max, max_exp and max_10_exp.
    """
    import sys
    help(sys.float_info)
    print(sys.float_info)
    # max: maximum representable finite float
    # max_exp: maximum int e such that radix**(e-1) is representable
    # max_10_exp: maximum int e such that 10**e is representable

def es2():
    """
    Write a code to compute the machine precision ϵ in (float) default
    precision with a WHILE construct. Compute also the mantissa digits number.
    """
    epsilon = 1.0
    mant_dig = 1
    while 1.0 + epsilon / 2.0 > 1.0:
        epsilon /= 2.0
        mant_dig += 1
    print("epsilon = " + str(epsilon))
    print("mant_dig = " + str(mant_dig))

def es3():
    """
    Import NumPy (import numpy as np) and exploit the functions float16 and
    float32 in the while statement and see the differences. Check the result of
    print(np.finfo(float).eps)
    """
    print("float16:")
    epsilon = np.float16(1.0)
    mant_dig = 1
    while np.float16(1.0) + epsilon / np.float16(2.0) > np.float16(1.0):
        epsilon /= np.float16(2.0)
        mant_dig += 1
    print("  epsilon = " + str(epsilon))
    print("  mant_dig = " + str(mant_dig))

    print("float32:")
    epsilon = np.float32(1.0)
    mant_dig = 1
    while np.float32(1.0) + epsilon / np.float32(2.0) > np.float32(1.0):
        epsilon /= np.float32(2.0)
        mant_dig += 1
    print("  epsilon = " + str(epsilon))
    print("  mant_dig = " + str(mant_dig))
    print("np.finfo(float).eps = " + str(np.finfo(float).eps))

def es4():
    """
    Matplotlib is a plotting library for the Python programming language and its
    numerical mathematics extension NumPy, from https://matplotlib.org/
    Create a figure combining together the cosine and sine curves, from 0 to 10:
    - add a legend
    - add a title
    - change the default colors
    """
    linspace = np.linspace(0, 10);
    plt.subplots(constrained_layout = True)[1].secondary_xaxis(0.5);
    plt.plot(linspace, np.sin(linspace), color='blue')
    plt.plot(linspace, np.cos(linspace), color='red')
    plt.legend(['sin', 'cos'])
    plt.title('Sine and cosine from 0 to 10')
    plt.show()

def es5(n):
    """
    Write a script that, given an input number n, computes the numbers of the
    Fibonacci sequence that are less than n.
    """
    if n <= 0:
        return 0
    if n <= 1:
        return 1
    a, b = 0, 1
    cont = 2
    while a + b < n:
        b += a
        a = b - a
        cont += 1
    return cont

def r(k): # assuming k > 0
    if k <= 0:
        return 0
    a, b = 0, 1
    for _ in range(k):
        b += a
        a = b - a
    print(b / a)
    return b / a

def relative_error(k):
    phi = (1.0 + 5 ** 0.5) / 2.0
    return abs(r(k) - phi) / phi

def es6():
    """
    Write a code computing, for a natural number k, the ratio  r(k) = F(k+1) /
    / F(k), where F(k) are the Fibonacci numbers. Verify that, for a large k,
    {{rk}}k converges to the value φ=1+5√2 create a plot of the error (with
    respect to φ)
    """
    arange = np.arange(50)
    plt.plot(arange, [relative_error(i) for i in arange])
    plt.legend(['relative error'])
    plt.show()


import numpy as np
from sympy import *
import matplotlib.pyplot as plt
import time

def actual_result(func, val):
	x = Symbol('x')
	dfunc = diff(func, 'x')
	x = val
	return eval(str(dfunc))

def three_point(func, x, h):
	x = x+h
	funcxh = eval(str(func))
	x = x-2*h
	funcxmh = eval(str(func))
	return (funcxh - funcxmh)/(2*h)

def plot(Result3Point,Error3Point):
	plt.plot(Result3Point,'k:',label= "3 Point")
	plt.plot(Error3Point,'k--',label= "3 Point Error")

	legend = plt.legend(loc='upper center', shadow=True)
	frame = legend.get_frame()
	frame.set_facecolor('0.90')
	plt.title('Three Point')
	plt.show()

def threePoint(func, x ,h):
    Result3Point= []
    Error3Point = []
    start = time.time()
    actualResult = actual_result(func, x)
    for item in h:
        Result3Point.append(three_point(func, x, item))

    Error3Point = np.asarray(Result3Point) - actualResult

    end = time.time()
    timeElapsed = end-start
    print ("Elapsed time: %f" % timeElapsed)
    print (Result3Point)

    plot(Result3Point, Error3Point)

if __name__ =='__main__':
	threePoint(sin('x'),0.0,[0.1,0.01,0.001])


'''
Based on the video: https://www.youtube.com/watch?v=LJYLxEpH2vA
'''
import matplotlib.pyplot as plt
import numpy as np
import numpy.polynomial.polynomial as poly
from scipy import integrate

def plot_function():
    x_start = -1.0
    x_stop = 1.1
    increment = 0.1

    x = np.arange(start=x_start, stop=x_stop, step=increment)
    y = x**2

    plt.plot(x, y)
    plt.xlabel('x')
    plt.xlabel('x')
    plt.axis([0, 1, 0, 1])
    plt.fill_between(x, y)
    plt.show()

def trapezoid_rule():
    a = 0
    b = 1
    N = 100

    x = np.linspace(a, b, N+1)
    y = x**2

    y_right = y[1:]
    y_left = y[:-1]

    dx = (b-a)/N
    A = (dx/2)*np.sum(y_right+y_left)

    print(A)
    print(np.trapz(y, x, dx))
    print(integrate.quad(lambda x: x**2, a, b))

    plt.plot(x, y)
    plt.xlim([-1, 1])
    plt.ylim([-1, 1])
    plt.show()

def polynomial_integration():
    # p(x) = x^3 + 2x^2 - x + 3
    # p(x) = 3 - x + 2x^2 + x^3
    p = [3, -1, 2, 1]

    # Find indefinite integral
    I = poly.polyint(p)
    print(f'I = {I}')

    # Find definite integral
    a = -1
    b = 1
    A = poly.polyval(b, I) - poly.polyval(a, I)
    I = poly.polyint(p)
    print(f'A = {A}')

def main():
    # plot_function()
    # trapezoid_rule()
    polynomial_integration()

if __name__ == "__main__":
    main()


""" Some sample functions for various tests

Test functions taken from Wikipedia:
https://en.wikipedia.org/wiki/Test_functions_for_optimization
"""
import numpy as np

def rosenbrock(x):
    """ Rosenbrock test function """
    assert len(x) > 1, "Must pass a vector to rosenbrock"

    value = 0
    for ii in range(len(x) - 1):
        value += 100 * (x[ii + 1] - x[ii] ** 2) ** 2 + (1 - x[ii]) ** 2
    return value

def circle(x):
    """ Circle test function """
    return np.sum(x ** 2, axis=1)

def rastrigin(x):
    """ Rastrigin function """
    A = 10
    return A * x.shape[1] + np.sum(x ** 2 - A * np.cos(2 * np.pi * x), axis=1)

def forrester_high_fidelity(x):
    """ The high-fidelity function from Forrester et al.

    doi:10.1098/rspa.2007.1900
    """
    return (6 * x - 2) ** 2 * np.sin(12 * x - 4)

def forrester_low_fidelity(x, y=None):
    """ The low-fidelity function from Forrester et al.

    doi:10.1098/rspa.2007.1900
    """
    if y is None:
        y = forrester_high_fidelity(x)
    A = np.ptp(y, axis=0) / 40.0
    B = np.ptp(y, axis=0) / 2.0
    C = -np.ptp(y, axis=0) / 4.0
    return A * y + B * (np.sum(x, axis=1) - 0.5) + C


import numpy as np
import matplotlib.pyplot as plt
import sys

def inv(A):
    #Augment with an identity matrix of required order: Here it's 2 :
    if A.shape(0)!= A.shape(1):
       sys.exit('Not Invertible!')
    n = A.shape(0)
    a = np.concatenate((A,np.identity(n,dtype = float)),axis = 1)
    # Applying Gauss Jordan Elimination:
    for i in range(n):
        if a[i][i] == 0.0:
           sys.exit('Not Invertible!')
        for j in range(n):
            if i != j:
               ratio = a[j][i]/a[i][i]
               for k in range(2*n):
                   a[j][k] = a[j][k] - ratio*a[i][k]
    #Row operation to make principal diagonal element to 1:
    for i in range(n):
        divisor = a[i][i]
        for j in range(2*n):
            a[i][j] = a[i][j]/divisor
    #Displaying the Inverted matrix:
    #print('\nINVERTED MATRIX IS:')
    #for i in range(n):
    #    for j in range(n,2*n):
    #        print(a[i][j],end ='\t')
    #    print()
    #Returning the inverted matrix
    b = np.zeros([n,n])
    for i in range(n):
        for j in range(n,2*n):
            b[i-n][j-n] = a[i][j]
    return b

def linreg_all(step,abs_err,name):
    data_x = (np.array(step).reshape(len(step),1))
    data_y = (np.array(abs_err).reshape(len(step),1))
    #Adding a column of all ones (intercept term) to data_x:
    data_X = np.concatenate((np.ones([len(data_x),1]),data_x),axis = 1)
    #We now apply the Normal Equation:
    X_transpose =np.transpose(data_X)
    theta = np.linalg.inv(X_transpose.dot(data_X)).dot(X_transpose).dot(data_y)
    #Plotting the linear regression:
    plt.plot(data_X[:,1],data_X.dot(theta),'-',label=name)
    #Plotting raw data to compare:
    plt.scatter(data_x,data_y,s=8,marker = 'x')
    #plt.xlabel('X')
    #plt.ylabel('Y')
    plt.autoscale()
    #plt.show()
    #return theta[0],theta[1]

def linreg_ind(step,abs_err):
    data_x = (np.array(step).reshape(len(step),1))
    data_y = (np.array(abs_err).reshape(len(step),1))
    #Adding a column of all ones (intercept term) to data_x:
    data_X = np.concatenate((np.ones([len(data_x),1]),data_x),axis = 1)
    #We now apply the Normal Equation:
    X_transpose =np.transpose(data_X)
    theta = np.linalg.inv(X_transpose.dot(data_X)).dot(X_transpose).dot(data_y)
    #Plotting the linear regression:
    plt.plot(data_X[:,1],data_X.dot(theta),'-',label='Linear Regression',color='b')
    #Plotting raw data to compare:
    plt.scatter(data_x,data_y,s=8,marker = 'x',color = 'r',label = 'Raw Data')
    #plt.xlabel('X')
    #plt.ylabel('Y')
    plt.autoscale()
    #plt.show()
    #return theta[0],theta[1]


import numpy as np
from collections import defaultdict
from scipy.optimize import minimize_scalar, root_scalar, bracket
from scipy.special import logsumexp

def em_worst_expected_error(n=2, eps=1, delta=1):
    def foo(p):
        return np.log(p) * (1 - 1 / (1 + (n-1)*p))
    a = -minimize_scalar(lambda p: foo(p), bounds=(0,1), method='bounded').fun
    return a * 2 * delta / eps

def pf_worst_expected_error(n=2, eps=1, delta=1):
    def foo(p):
        return np.log(p) * (1 - (1 - (1-p)**n) / (n*p))
    a = -minimize_scalar(lambda p: foo(p), bounds=(0,1), method='bounded').fun
    return a * 2 * delta / eps

def pf_pmf(q, eps=1.0, sensitivity=1.0, monotonic=False):
    coef = 1.0 if monotonic else 0.5
    p = np.exp(coef*eps/sensitivity*(q - q.max()))
    n = q.size

    # first we will calculate
    # sum(prod(p_i, i in S), |S| = k) for each k

    subsets = np.zeros(n)
    curr = np.cumsum(p)
    subsets[0] = curr[-1]
    for j in range(1,n):
        curr[j:] = np.cumsum(curr[j-1:-1]*p[j:])
        subsets[j] = curr[-1]

    # coefficient vector: (-1)^k / (k+1) for k = 1..n
    coef = (np.arange(n) % 2 * 2 - 1) / (np.arange(n)+2)

    # we will now calculate
    # sum(prod(p_i, i in S), |S| = k, r not in S)
    # and compute the final probabilities

    ans = np.zeros(n)
    for i in range(n):
        new = np.copy(subsets)
        new[0] -= p[i]
        for j in range(1,n):
            new[j] -= new[j-1]*p[i]

        ans[i] = p[i] * (1 + new @ coef)

    return ans

def em_pmf(q, eps=1.0, sensitivity=1.0, monotonic=False):
    coef = 1.0 if monotonic else 0.5
    q = q - q.max()
    logits = coef*eps/sensitivity*q
    return np.exp(logits - logsumexp(logits))
    #p = np.exp(coef*eps/sensitivity*q)
    #return p / p.sum()

def em(q, eps=1.0, sensitivity=1.0, prng=np.random, monotonic=False):

    coef = 1.0 if monotonic else 0.5

    q = q - q.max()
    p = np.exp(coef*eps/sensitivity*q)
    p /= p.sum()

    return prng.choice(p.size, p=p)

def pf(q, eps=1.0, sensitivity=1.0, prng=np.random, monotonic=False):

    coef = 1.0 if monotonic else 0.5

    q = q - q.max()
    p = np.exp(coef*eps/sensitivity*q)

    for i in prng.permutation(p.size):
        if prng.rand() <= p[i]:
            return i

def expected_error(q, eps, pmf=em_pmf):
    # compute the expected error of the mechanism (given it's probability mass function)
    ans = q.max() - pmf(q,eps) @ q
    maxerr = q.max() - q.mean()
    if ans > maxerr or ans < 0:
        return maxerr
    return ans

def variance(q, eps, pmf=em_pmf):
    e = expected_error(q, eps, pmf)
    return pmf(q, eps) @ (q.max() - q)**2 - e**2

def expected_epsilon(q, err, bounds=None, pmf=em_pmf):
    # computed the epsilon required to achieve given expected error
    foo = lambda eps: expected_error(q, eps, pmf) - err

    if bounds is None:
        eps = 1.0
        while foo(eps) > 0:
            eps *= 2
        while foo(eps) < 0:
            eps /= 2.0
        bounds = [eps,2*eps]

    return root_scalar(foo,bracket=bounds,method='bisect').root

def max_epsilon_ratio(q):
    def foo(eps):
        err = expected_error(q, eps, pf_pmf)
        eps2 = expected_epsilon(q, err, [eps, 2*eps])
        return -eps2/eps
    br = bracket(foo, 1e-3, 1.0)[0:3]
    ans = minimize_scalar(foo, bracket=br, method='brent')
    eps0 = ans.x
    err = expected_error(q, eps0, pf_pmf)
    eps1 = expected_epsilon(q, err, [eps0, 2*eps0])
    return eps0, err, eps1


import numpy as np
import matplotlib.pyplot as plt
import sys
sys.path.append("../Unsupervised_Learning/")
from PCA import PCA

#activations
def sigmoid(x,derivative=False):
	'''
	   sigmoid function, set derivative = true to get the derivative
	'''
	if derivative == True:
		return 1/(1+np.e**-(x*1.0))*(1-(1/(1+np.e**-(x*1.0))))
	else:
		return 1/(1+np.e**-(x*1.0))

def softmax(x,derivative=False):
    '''
        stable softmax function, set derivative = true to get the derivative
    '''
    vecs = np.exp(x  - np.max(x))
    if derivative == True:
        if len(x.shape) > 1:
            s = vecs/(np.sum(vecs,axis=1).reshape(x.shape[0],1))
            return s*(1-s)
        else:
            s = vecs/(np.sum(vecs))
            return s*(1-s)
    else:
        if len(x.shape) > 1:
            return vecs/(np.sum(vecs,axis=1).reshape(x.shape[0],1))
        else:
            return vecs/(np.sum(vecs))

def relu(x,derivative=False):
    '''
        relu function, set derivative = true to get the derivative
    '''
    rel = np.maximum(x,0)
    if derivative == True:
        rel[x < 0] = 0
        rel[x >= 0] = 1
    return rel

#similarity measurements
def compute_covariance(X,col=True, correlation=False):
    '''
        computes covariance using definition 1/n(XXT) - mumuT
        change col to True for 1/n(XTX) - mumuT
        change correlation to True for the correlation matrix
    '''
    if col == False:
        newx = ((X-X.mean(axis=1)).dot(X-X.mean(axis=1)).T)/X.shape[1]
    else:
        newx = ((X-X.mean(axis=0)).T.dot(X-X.mean(axis=0)))/X.shape[0]
    if correlation == True:
        stdev = np.diag(1/np.sqrt(np.diag(newx)))
        newx = stdev.dot(newx).dot(stdev)
    return newx

def l2distance(X,y):
    '''
	   gets euclidian distance between vector X and matrix(or vector) y
    '''
    if len(X.shape) < 2:
        X = X.reshape(1,X.shape[0])
    ones = np.ones(y.shape[0]).reshape(y.shape[0],1)
    X = ones.dot(X)
    dist = (y - X)

    return np.linalg.norm(dist,axis=1)

def standardize(X):
	'''
	    z-score standardization, (x -mu)/std(x), standardizing the columns
	'''
	mu = np.mean(X,axis=0)
	mumat = np.outer(mu,np.ones(X.shape[0])).T
	newx = X - mumat

	var = np.sqrt(1/np.var(X,axis=0).reshape((X.shape[0],1)))
	ones = np.ones(X.shape[0]).reshape((1,X.shape[0]))
	varmatrix = var.dot(ones)/X.shape[0]

	return newx.dot(varmatrix)

def cosine_distance(x,y):

    num = np.dot(x,y)
    denom = np.linalg.norm(x)*np.linalg.norm(y)
    return 1 - num/denom

def gen_distance_similarities(X,y,dist='l2'):
    '''
       calculates distance similarities measurements, such as l2, l1, and cosine
    '''
    if dist == 'l2':
        ones = np.ones(y.shape[0]).reshape(y.shape[0],1)
        X = ones.dot(X)
        dist = (y - X)**2
        dist = 1/(1 + np.sqrt(np.sum(dist,axis=1)))
    elif dist == 'l1':
        ones = np.ones(y.shape[0]).reshape(y.shape[0],1)
        X = ones.dot(X)
        dist = np.abs(y - X)
        dist = 1/(np.sum(dist,axis=1) + 1)
    elif dist == 'cosine':
        dist = np.zeros(y.shape[0])
        for i in range(y.shape[0]):
            dist[i] = cosine_distance(X,y[i])
    return dist

def generate_similarity_matrix(X,sim_type='l2'):

    n = X.shape[0]
    p = X.shape[1]

    newmatrix = np.zeros((n,n))
    for i in range(n):
        newmatrix[i] = (gen_distance_similarities(X[i].reshape(1,p),X,sim_type)).reshape(1,n)
    return newmatrix

#EDA and data manipulation
def cross_val_split_set(X,portion,y=None):
    '''
    use:
        X = iris.data
        y = iris.target
        X_train, X_test, y_train, y_test = split_set(X,0.1,y)
    '''
    X = np.array(X)
    y = np.array(y)
    size = int(X.shape[0]*portion)
    indexlist = np.arange(X.shape[0])
    testinds = np.random.choice(indexlist, size, replace=False)
    traininds = np.array([x for x in range(X.shape[0]) if x not in testinds])
    if np.all(y == None):
        return X[traininds],X[testinds]
    else:
        return X[traininds],X[testinds],y[traininds],y[testinds]

def bucket(data):
    """
        buckets continuous data by percentiles
    """
    upper = np.percentile(data,75)
    mid = np.percentile(data,50)
    lower = np.percentile(data,25)
    data[(data <= lower)] = 0
    data[(data > lower) & (data <= mid)] = 1
    data[(data < upper) & (data > mid)] = 2
    data[(data >= upper)] = 3
    return data

#Accuracy and loss measurements
def calc_accuracy(predictions,ytest):
    """
        Calculates accuracy for classification tasks
    """
    if len(ytest.shape) > 1:
        ytest = np.argmax(ytest,axis=1)
    acc = ytest - predictions
    return np.where(acc == 0)[0].shape[0]/ytest.shape[0]

def cross_entropy(predictions,y):
    """
        categorical cross entropy
    """
    m = predictions.shape[0]
    loss = -1*(1/m)*np.sum(y*np.log(predictions))
    return loss

def mean_squared_error(predictions,y):
    """
        mean squared error
    """
    m = predictions.shape[0]
    loss = np.square(np.subtract(y, predictions)).mean()
    return loss

#plotting vals
def plot2D(X,y,classes=False,title="Plotted Vals"):

    fig, ax = plt.subplots()
    if len(X.shape) >= 2:
        pca = PCA(X)
        if classes == True:
            X = pca.project(2)
            for class_val in np.unique(y):
                i = np.where(y == class_val)[0]
                ax.scatter(x=X[i,0],y=X[i,1], label=class_val)
                plt.ylabel("PC 2")
                plt.xlabel("PC 1")
        else:
            X = pca.project(1)
            ax.scatter(X,y)
            plt.ylabel("Y")
            plt.xlabel("X")
    else:
        if classes == True:
            for class_val in np.unique(y):
                i = np.where(y == class_val)[0]
                ax.scatter(x=X,y=y, label=class_val)
                plt.ylabel("PC 2")
                plt.xlabel("PC 1")
        else:
            ax.scatter(x=X,y=y)
            plt.ylabel("Y")
            plt.xlabel("X")

    ax.legend()
    plt.title(title)
    plt.figure(figsize=(20,20))
    plt.show()


import numpy as np

def norm(v):
    """
    Norm of a vector

    Parameter
    ---------
    v: numpy array
       vector

    Return
    ------
    numpy float
    norm of the vector v
    """
    return np.sqrt(v.dot(v))

def normalize(vectors):
    """
    Return the unit vector of the vectors (could be one or several vectors)

    Parameter
    ---------
    vector: numpy array
       vector

    Return
    ------
    numpy array
    The unit vector of vectors
    """
    if len(vectors.shape) == 1:
        unit_vectors = vectors/norm(vectors)
    elif len(vectors.shape) == 2:
        unit_vectors = [v/norm(v) for v in vectors]
        unit_vectors = np.array(unit_vectors)
    else:
        return NotImplemented
    return unit_vectors

def screw_motion(quaternions, translations):
    """
    Compute screw parameters from quaternions and translation
    (adapted from the original code in screwframe.ap)

    Parameters
    ----------
    quaternions: np.ndarray
       all normalized quaternion with q[0]>0, describing the successive rotations between frenet bases
    translations: np.ndarray
       all translation vectors between C-alpha atoms

    Returns
    -------
    list of tuple (r0, axis, phi, d):
        r0: point on the screw axis
        axis: normalized vector indicating the axis direction
        phi: angle of the rotation around the axis
        d: scalar displacement along the axis
    """
    results = []
    for q, t in zip(quaternions, translations):
        cosphi2 = q[0]
        if abs(cosphi2-1.) < 1.e-6:
            # No rotation, pure translation.
            # We arbitrarily choose the axis parallel to the translation.
            phi = 0.
            r0 = np.array([0., 0., 0.])
            d = norm(t)
            axis = t/d
        else:
            # From the definition of np.arccos, and with
            # cosphi2 > 0, we get 0 <= phi <= pi.
            phi = 2.*np.arccos(cosphi2)
            # We choose sinphi2 as the positive solution,
            # which is consistent with the range of phi.
            sinphi2 = np.sqrt(1.-cosphi2*cosphi2)
            axis = q[1:]/sinphi2
            # We choose the direction of the axis such that the
            # scalar displacement parameter is positive.
            d = np.dot(t, axis)
            if d < 0:
                d = -d
                axis = -axis
                phi = -phi
                sinphi2 = -sinphi2
            # r0 is the axis point closest to the reference point for the
            # rotation, relative to the reference point of the rotation.
            x = t-d*axis
            r0 = 0.5*(x - (cosphi2/sinphi2)*np.cross(axis, x))
        results.append((r0, axis, phi, d))
    return results


from cv2 import imread, Canny
from math import exp, sqrt, atan2, pi
import numpy as np
from matplotlib.pyplot import imshow, show, title

def gauss_func(sig, x, y):
    return exp(-((x**2 + y**2)/(2 * (sig**2))))

def create_gaussian_filter(sig, size):
    f = np.zeros((size, size))
    for x in range(size):
        for y in range(size):
            f[x, y] = gauss_func(sig, x - (size - 1) / 2, y - (size - 1) / 2)

    # I return this to make the sum equal to 1
    return f / np.sum(f)

def gauss_filter(I, sig, size):
    # since f is symmetric from all angles, we do not need to reverse it
    f = create_gaussian_filter(sig, size)
    I = np.lib.pad(I, ((0, size), (0, size)), 'reflect')

    h = np.size(I, axis=0)
    w = np.size(I, axis=1)
    newimage = np.zeros((h, w))
    newimage[:size, :size] = f

    FI = np.fft.fft2(I)
    Fnewimage = np.fft.fft2(newimage)
    newimage = np.fft.ifft2(FI * Fnewimage)

    return abs(newimage[(size+1)//2:-(size-1)//2, (size+1)//2:-(size-1)//2])

def gradient(I, tau):
    h = np.size(I, axis=0)
    w = np.size(I, axis=1)
    M = np.zeros((h, w))
    A = np.zeros((h, w))

    for x in range(1, h - 1):
        for y in range(1, w - 1):
            Ix = (I[x + 1, y] - I[x - 1, y]) / 2
            Iy = (I[x, y + 1] - I[x, y - 1]) / 2
            dist = sqrt(Ix**2 + Iy**2)
            M[x, y] = dist if dist > tau else 0
            angle = atan2(Iy, Ix)
            angle += pi / 2
            # to keep the angle between [0, 2pi]
            if angle < 0:
                angle += 2 * pi
            if angle > 2*pi:
                angle -= 2 * pi
            A[x, y] = angle

    return M, A

def nonmaxima(M, A):
    h = np.size(M, axis=0)
    w = np.size(M, axis=1)
    m = np.zeros((h, w))
    s = pi/8
    for x in range(1, h - 1):
        for y in range(1, w - 1):
            angle = A[x, y]
            if (angle < s or angle >= 15*s) or (angle < 9*s and angle >= 7*s): #lr
                if(M[x, y] >= M[x, y + 1] and M[x, y] >= M[x, y - 1]):
                    m[x, y] = M[x, y] > 0
            elif (angle < 5*s and angle >= 3*s) or (angle < 13*s and angle >= 11*s): #ud
                if(M[x, y] >= M[x + 1, y] and M[x, y] >= M[x - 1, y]):
                    m[x, y] = M[x, y] > 0
            elif (angle < 3*s and angle >= s) or (angle < 11*s and angle >= 9*s): #ur
                if(M[x, y] >= M[x - 1, y + 1] and M[x, y] >= M[x + 1, y - 1]):
                    m[x, y] = M[x, y] > 0
            elif (angle < 7*s and angle >= 5*s) or (angle < 15*s and angle >= 13*s): #ul
                if(M[x, y] >= M[x + 1, y + 1] and M[x, y] >= M[x - 1, y - 1]):
                    m[x, y] = M[x, y] > 0

    return m

def canny(I, sig, tau):
    I = gauss_filter(I, sig, 9)
    M, A = gradient(I, tau)
    E = nonmaxima(M, A)
    return E, M, A

wirebond = imread('images/Fig2wirebond_mask.jpg', 0)
E, M, A = canny(wirebond, sqrt(0.5), 5)

title('wirebond (sigma)^2 = 0.5')
imshow(E, cmap='gray')
show()

E, M, A = canny(wirebond, sqrt(1), 5)

title('wirebond (sigma)^2 = 1')
imshow(E, cmap='gray')
show()

E, M, A = canny(wirebond, sqrt(3), 5)

title('wirebond (sigma)^2 = 3')
imshow(E, cmap='gray')
show()

bottles = imread('images/Fig3bottles.jpg', 0)
E, M, A = canny(bottles, sqrt(50), 7.5)

title('Edge map Bottles (sigma)^2 = 50, tau = 7.5')
imshow(E, cmap='gray')
show()

img = imread('images/mustafa.jpeg')
grayimg = np.dot(img[:, :, :3], [0.299, 0.587, 0.114])

M, A = gradient(grayimg, 10)

title('Mustafa image, Gradient with threshold 10')
imshow(M, cmap='gray')
show()

# Built-in Canny
cannyimg = Canny(img, 100, 200)

title('Mustafa image, Edge with Built-in Canny')
imshow(cannyimg, cmap='gray')
show()


import sys

import numpy as np

A = -2.0
B = 2.0
n = 5
x = np.arange(A, B + 0.1, (B - A) / n)
F_SECOND_DER = -0.0239496  # f''(a)=f''(b)

def f(x):
    return float(x * (3 ** x + 1) ** (-1))

def get_c():
    rhs = [3 * ((f(x[i]) - f(x[i - 1])) / (x[i] - x[i - 1]) - (f(x[i - 1]) - f(x[i - 2])) / (x[i - 1] - x[i - 2]))
           for i in range(2, n + 1)]
    rhs.insert(0, F_SECOND_DER)
    rhs.append(F_SECOND_DER)

    lhs = np.zeros((n + 1, n + 1), float)
    np.fill_diagonal(lhs, [1.0] + [2 * (x[i] - x[i - 2]) for i in range(2, n + 1)] + [1.0])  # main diag
    lhs += np.diagflat([x[i] - x[i - 1] for i in range(1, n + 1)], 1)  # upper diag
    lhs += np.diagflat([x[i - 1] - x[i - 2] for i in range(2, n + 2)], -1)  # lower diag
    return np.linalg.solve(lhs, rhs)

def get_d(c):
    return [(c[i + 1] - c[i]) / (3 * (x[i] - x[i - 1])) for i in range(1, n)] + [-c[n] / (3 * (x[n] - x[n - 1]))]

def get_b(c):
    return [(f(x[i]) - f(x[i - 1])) / (x[i] - x[i - 1]) - (1 / 3) * (x[i] - x[i - 1]) * (c[i + 1] + 2 * c[i])
            for i in range(1, n)] + [(f(B) - f(x[n - 1])) / (x[n] - x[n - 1]) - 2 / 3 * c[n] * (x[n] - x[n - 1])]

def get_a():
    return [f(x[i - 1]) for i in range(1, n + 1)]

def spline(x0, i):
    return a[i] + b[i] * (x0 - x[i]) + c[i] * (x0 - x[i]) ** 2 + d[i] * (x0 - x[i]) ** 3

def accuracy():
    return [abs(f((x[i] + x[i - 1]) / 2) - spline((x[i] + x[i - 1]) / 2, i - 1)) for i in range(1, n + 1)]

def sign_print(el, val):
    sys.stdout.write(('{0:.' + str(val) + 'g}').format(el) + ', ')

def out(k, v):
    sys.stdout.write(k + ': [')
    map(sign_print, v, [3] * len(v))
    sys.stdout.write(']\n')

c = get_c()
a = get_a()
b = get_b(c)
d = get_d(c)

map(out, ('a', 'b', 'c', 'd'), (a, b, c, d))
sys.stdout.write('r: ')
map(sign_print, accuracy(), [1] * len(accuracy()))


# simple example of linear regression.  We make up some "experimental"
# data by calling a random number generator to perturb a line and then
# fit to that line
#
# M. Zingale (2013-03-10)

import numpy
import pylab

def yExperiment(a1, a2, sigma, x):
    """ return the experimental data in a linear + random fashion a1
        is the intercept, a2 is the slope, and sigma is the error """

    N = len(x)

    # randn gives samples from the "standard normal" distribution
    r = numpy.random.randn(N)

    y = a1 + a2*x + sigma*r

    return y

def yExperiment2(a1, a2, a3, sigma, x):
    """ return the experimental data in a quadratic + random fashion,
        with a1, a2, a3 the coefficients of the quadratic and sigma is
        the error.  This will be poorly matched to a linear fit for
        a3 != 0 """

    N = len(x)

    # randn gives samples from the "standard normal" distribution
    r = numpy.random.randn(N)

    y = a1 + a2*x + a3*x*x + sigma*r

    return y

def linearRegression(x, y, sigma, sigmaa=0):

    N = len(x)

    S = numpy.sum(1.0/sigma**2)

    xi_1 = numpy.sum(x/sigma**2)
    xi_2 = numpy.sum(x*x/sigma**2)

    eta = numpy.sum(y/sigma**2)
    mu = numpy.sum(x*y/sigma**2)

    a2 = (S*mu - xi_1*eta)/(xi_2*S - xi_1**2)
    a1 = (eta*xi_2 - mu*xi_1)/(xi_2*S - xi_1**2)

    chisq = numpy.sum( (a1 + a2*x - y)**2/sigma**2)
    chisq /= N-2

    if not sigmaa:
        return a1, a2, chisq

    else:
        sigmaFit = numpy.array([xi_2/(S*xi_2 - xi_1**2),
                                S/(S*xi_2 - xi_1**2)])
        sigmaFit = numpy.sqrt(sigmaFit)

        return a1, a2, chisq, sigmaFit

#-----------------------------------------------------------------------------
N = 40
x = numpy.linspace(0.0, 100.0, N)

#-----------------------------------------------------------------------------
# test 1 -- linear data

print "Linear fit to linear data\n"

# make up the experimental data with errors
sigma = 25.0*numpy.ones(N)

y = yExperiment(10.0, 3.0, sigma, x)

pylab.scatter(x,y)
pylab.errorbar(x, y, yerr=sigma, fmt=None)

# do the linear regression
a1, a2, chisq, sigmaFit = linearRegression(x, y, sigma, sigmaa=1)

pylab.plot(x, a1 + a2*x)

print "reduced chisq = ", chisq
print " a1 = %f +/- %f\n a2 = %f +/- %f\n" % (a1, sigmaFit[0], a2, sigmaFit[1])

pylab.savefig("linear-regression.png")

#-----------------------------------------------------------------------------
# test 2 -- quadratic data

pylab.clf()

print "Linear fit to quadratic data\n"

# make up the experimental data with errors
sigma = 5.0*numpy.ones(N)

y = yExperiment2(2.0, 1.50, -0.02, sigma, x)

pylab.scatter(x,y)
pylab.errorbar(x, y, yerr=sigma, fmt=None)

# do the linear regression
a1, a2, chisq = linearRegression(x, y, sigma)

pylab.plot(x, a1 + a2*x)

print "reduced chisq = ", chisq

pylab.savefig("linear-regression-quad.png")


#  O(n^3) matrix multiplication

import time
import numpy as np

def matrixmul(A,B):
    I,K = A.shape
    J = B.shape[1]
    C = np.zeros((I,J), dtype=A.dtype)
    for i in range(I):
        for j in range(J):
            for k in range(K):
                C[i,j] += A[i,k]*B[k,j]
    return C

A = np.array([[1,2,3],[4,5,6],[7,8,9],[10,11,12]])
B = np.array([[1,2],[3,4],[5,6]])
N = 100000

s = time.time()
for i in range(N):
    C = np.matmul(A,B)
e = time.time()
print("np.matmul: %0.6f" % (e-s,))

s = time.time()
for i in range(N):
    C = matrixmul(A,B)
e = time.time()
print("matrixmul: %0.6f" % (e-s,))


# Proceduri Laborator 5

import numpy as np

def metSubDesc(A, b, tol):
    """

    Parameters
    ----------
    A : matrice pătratică, superior triunghiulară, cu toate elementele de pe diagonala principală nenule.
    b : vectorul termenilor liberi.
    tol : toleranță = valoare numerică foarte mică în raport cu care vom compara numerele apropiate de 0.

    Returns
    -------
    x = Soluția Sistemului.

    """

    # Verificăm dacă matricea este pătratică
    m, n = np.shape(A)
    if m!= n:
        print("Matricea nu este pătratică. Introduceți altă matrice.")
        x = None
        return x

    # Verificăm dacă matricea este superior triunghiulară
    for i in range(m):
       for j in range(i):
           if abs(A[i][j]) > tol:
               print("Matricea nu este superior triunghiulară.")
               x = None
               return x

    # Verificam dacă toate elementele de pe diagonala principală sunt nenule => Si. este compatibil ddeterminat (adică am soluție unică)
    for i in range(n):
        if A[i][i] == 0:
            print("Sistemul nu este compatibil determinat.")
            x = None
            return x

    x = np.zeros((n, 1))
    x[n - 1] = b[n - 1] / A[n - 1][n - 1]

    k = n - 2
    while k >= 0:
        s = 0
        for j in range(k + 1, n):
            s += x[j] * A[k][j]

        x[k] = (1 / A[k][k]) * (b[k] - s)
        k -= 1

    return x

def metSubAsc(A, b, tol):
    """

    Parameters
    ----------
    A : matrice inferior triunghiulară.
    b : vectorul termenilor liberi.
    tol : toleranța.

    Returns
    -------
    soluția.

    """

    # Verificăm dacă matricea este pătratică
    m, n = np.shape(A)
    if m!= n:
        print("Matricea nu este pătratică. Introduceți altă matrice.")
        x = None
        return x

    # Verificăm dacă matricea este superior triunghiulară
    for i in range(m):
       for j in range(i):
           if abs(A[j][i]) > tol:
               print("Matricea nu este inferior triunghiulară.")
               x = None
               return x

    # Verificam dacă toate elementele de pe diagonala principală sunt nenule => Si. este compatibil ddeterminat (adică am soluție unică)
    for i in range(n):
        if abs(A[i][i]) <= tol:
            print("Sistemul nu este compatibil determinat.")
            x = None
            return x

    x = np.zeros((m, 1))
    x[0] = b[0] / A[0][0]

    for k in range(1, n):

        sum = 0
        for j in range(k):
            sum += A[k][j] * x[j]

        x[k] = (1 / A[k][k]) * (b[k] - sum)

    return x

def FactLU(A, b, tol):
    """

    Parameters
    ----------
    A : matrice pătratică.
    b : vectorul termenilor liberi.
    tol = toleranța

    Returns
    -------
    L, U = matrici
    w = vector

    """

    n = np.shape(A)[0]
    L = np.eye(n,n)
    U = np.copy(A)
    w = np.arange(0, n)

    for k in range(n - 1):

        max = abs(U[k][k])
        p = k
        for i in range(k + 1, n):
            if abs(U[i][k]) > max:
                max = abs(U[i][k])
                p = i

        if max <= tol:
            print("A nu admite factorizarea LU.")
            return None, None, None

        if p != k:
            U[[p, k], :] = U[[k, p], :]
            w[[p, k]] = w[[k, p]]

            if k > 0:
                L[[p, k], 0:k] = L[[k, p], 0:k]

        for l in range(k + 1, n):
            L[l][k] = U[l][k] / U[k][k]
            U[l][k] = 0
            U[l, k+1:n] =U[l, k+1:n] - L[l][k] * U[k, k+1:n]

    if abs(U[n - 1][n - 1]) <= tol:
        print("A nu admite factorizarea LU.")
        return None, None, None

    return L, U, w

# Gauss cu Pivotare Parțială
def GaussPP(A, b, tol):
    """

    Parameters
    ----------
    A : matrice pătratică.
    b : vectori termeni liberi.
    tol : toleranță => valoare cu care comparăm numerele nenule.

    Returns
    -------
    x = Soluția Sistemului.

    """

    # Verificăm dacă matricea este pătratică
    m, n = np.shape(A)
    if m != n:
        print("Matricea nu este pătratică. Introduceți altă matrice.")
        x = None
        return x

    A_extins = np.concatenate((A, b), axis = 1) # axis = 1  =>  pentru concatenare coloane; dacă aveam 0, concatenam linii
    # SAU: A_extins = np.column_stack((A, b)) # concatenare special pentru coloane

    for k in range(n - 1):       # coloane
        max = A_extins[k][k]
        p = k
        for j in range(k + 1, n):
            if abs(A_extins[j][k]) > abs(max):
                max = A_extins[j][k]
                p = j

        if abs(max) <= tol:
            print("Sistemul nu admite solutie unică.")
            x = None
            return x

        if p != k:
            A_extins[[p, k]] = A_extins[[k, p]]   # swap linia p cu linia k

        for j in range(k + 1, n):
            A_extins[j] = A_extins[j] - (A_extins[j][k] / A_extins[k][k]) * A_extins[k]

    if abs(A_extins[n - 1][n - 1]) <= tol:
        print("Sistemul nu admite soluție unică.")
        x = None
        return x

    x = metSubDesc(A_extins[:, 0:n], A_extins[:, n], tol)
    return x

Calcul Numeric (CN)/Laborator/Laborator 5/proceduri.py


# Native

# Site-packages
import numpy as np

# Locals
from losses import MSE
from utils import Algorithm
from regularizers import Regularizer, Placeholder, L1, L2, ElasticNet

class LinearRegression(Algorithm):
    def __init__(self):
        pass

    def fit(self, X_train: np.array, y_train: np.array):
        X_train_1 = np.hstack((X_train, np.ones((X_train.shape[0], 1))))
        moore_penrose = np.linalg.inv(X_train_1.T @ X_train_1) @ X_train_1.T
        self.W = moore_penrose @ y_train
        self.train_score = MSE(y_train, self.predict(X_train))

    def predict(self, X: np.array) -> np.array:
        X = np.hstack((X, np.ones((X.shape[0], 1))))
        return X.dot(self.W)

class GradientDescentRegression(Algorithm):
    def __init__(self, lr: float=1e-5, epochs: int=100, regularizer: Regularizer=None):
        self.lr = lr
        self.epochs = epochs
        if not regularizer:
            self.regularizer = Placeholder()
        else:
            self.regularizer = regularizer

    def fit(self, X_train: np.array, y_train: np.array):
        X_train = np.hstack((X_train, np.ones((X_train.shape[0], 1))))
        self.history = {}
        n_rows, n_columns = X_train.shape
        self.W = np.random.uniform((1 / -n_columns), 1 / n_columns, (n_columns,))
        for i in range(self.epochs):
            current_pred = X_train.dot(self.W)
            diff = (current_pred - y_train)
            dW = X_train.T.dot(diff) * (2 / n_rows)
            self.W -= self.lr * dW + self.regularizer.grad(self.W)

            self.history[i] = ((y_train - current_pred)**2 + self.regularizer(self.W)).mean()

    def predict(self, X: np.array) -> np.array:
        X = np.hstack((X, np.ones((X.shape[0], 1))))
        return X.dot(self.W)

class LassoRegression(GradientDescentRegression):
    def __init__(self, lambda_: float, lr: float=1e-5, epochs: int=100):
        self.regularizer = L1(lambda_)
        self.lambda_ = lambda_
        super(LassoRegression, self).__init__(lr=lr,
                                              epochs=epochs,
                                              regularizer=self.regularizer)

class RidgeRegression(GradientDescentRegression):
    def __init__(self, omega_: float, lr: float=1e-5, epochs: int=100):
        self.regularizer = L2(omega_)
        self.omega_ = omega_
        super(RidgeRegression, self).__init__(lr=lr,
                                              epochs=epochs,
                                              regularizer=self.regularizer)

class ElasticNetRegression(GradientDescentRegression):
    def __init__(self, lambda_1: float, lambda_2: float, lr: float=1e-5, epochs: int=100):
        self.regularizer = ElasticNet(lambda_1, lambda_2)
        self.lambdas = (lambda_1, lambda_2)
        super(ElasticNetRegression, self).__init__(lr=lr,
                                              epochs=epochs,
                                              regularizer=self.regularizer)


"""
Collect functions related to the stereographic projection.

A stereographic projection is a mapping between a direction in 3D space and a
position in a 2D plane. The direction can be described in polar coordinates
by (theta,phi), where theta denotes the angle between the direction and
the z axis, and phi denotes the azimuthal angle. The point in the 2D
plane can be described by its Cartesian coordinates (x,y) or its polar
coordinates (r,phi). phi is the same for the direction and the point.
"""
import numpy as np

def stereographic_projection(theta, phi=None):
    """
    Perform the stereographic projection (theta,phi) -> (r,phi).

    :param theta: Polar angle (rad)
    :param phi: Azimuthal angle (rad)
    :return (r,phi): Polar coordinates in the plane
    """
    r = 2 * np.tan(0.5*theta)

    if phi is None:
        return r
    else:
        return r, phi

def inverse_stereographic_projection(r, phi=None):
    """
    Perform the inverse stereographic projection (r,phi) -> (theta,phi).

    :param r: Distance from origin
    :param phi: Azimuthal angle (rad)
    :return (theta,phi): polar coordinates of the direction
    """
    theta = 2 * np.arctan(0.5*r)

    if phi is None:
        return theta
    else:
        return theta, phi

def cartesian(r, phi):
    """
    Convert polar to Cartesian coordinates.

    :param r: Distance from origin
    :param phi: Azimutal angle
    :return (x,y): Cartesian coordinates
    """
    x = r * np.cos(phi)
    y = r * np.sin(phi)

    return x, y

def polar(x, y):
    """
    Convert Cartesian to polar coordinates.

    :param x: Coordinate along the x axis
    :param y: Coordinate along the y axis
    :return (r,phi): (radius, azimuthal angle)
    """
    r = np.hypot(x, y)
    phi = np.arctan2(y, x)

    return r, phi

# not sure where we need this:
def intersect_bounding_box(circle, bbox):
    xc, yc, r = circle
    xmin, ymin, xmax, ymax = bbox
    intersections = []
    # intersections with x=xmin and x=xmax
    for x in (xmin, xmax):
        discriminant = r**2 - (x-xc)**2
        if discriminant >= 0:
            for sign in (1, -1):
                y = yc + sign * np.sqrt(discriminant)
                if y >= ymin and y <= ymax:
                    intersections.append((x, y))
    # intersections with y=ymin and y=ymax
    for y in (ymin, ymax):
        discriminant = r**2 - (y-yc)**2
        if discriminant >= 0:
            for sign in (1, -1):
                x = xc + sign * np.sqrt(discriminant)
                if x >= xmin and x <= xmax:
                    intersections.append((x, y))
    # remove duplicates (could be in the corners)
    intersections = tuple(set(intersections))
    return intersections

if __name__ == '__main__':
    import matplotlib.pyplot as plt
    xs, ys = get_wedge()
    plt.plot(xs, ys)
    xs, ys = get_triangle(10)
    plt.plot(xs, ys, '--')
    plt.gca().set_aspect('equal')
    plt.show()


import numpy as np
from matplotlib import pyplot as plt
from numpy import ndarray
from scipy.linalg import svd

def explained_var(Σ: ndarray) -> ndarray:
	ΣΣ: ndarray = Σ * Σ
	return ΣΣ / ΣΣ.sum()

def plot_explained_variance(ρ: ndarray, threshold = 0.9, save_to_file: str = ""):
	plot_range = range(1, len(ρ) + 1)
	f = plt.figure()
	plt.plot(plot_range, ρ, 'x-')
	plt.plot(plot_range, np.cumsum(ρ), 'o-')
	plt.plot([1, len(ρ)], [threshold, threshold], 'k--')
	plt.xlabel('Principal component')
	plt.ylabel('Variance explained')
	plt.legend(['Individual', 'Cumulative', 'Threshold'])
	plt.grid()
	if save_to_file != '':
		plt.savefig(save_to_file)
	return f

def plot_data_projected_unto_principal_components(projected_data: ndarray, class_labels: ndarray):
	classes: ndarray = np.unique(class_labels)
	num_principal_components = projected_data.shape[1]
	num_rows = 2
	num_columns = int(np.ceil((num_principal_components-1)*2/num_rows))
	fig, subplots = plt.subplots(num_rows, num_columns)
	plt.title('Data projected onto Principal components')
	for d1 in range(num_principal_components):
		for d2 in range(d1 + 1, num_principal_components):
			index = d1 + d2 - 1
			for klass in classes:
				class_mask: ndarray = (class_labels == klass)
				subplots[int(np.floor(index / num_columns)), index % num_columns].plot(data_projected[class_mask, d1], data_projected[class_mask, d2], 'o')
			#subplots[d1, d2].plt.legend(['With Parkinson\'s', 'Without Parkinson\'s'])
			#subplots[d1, d2].xlabel(f'PC{d1}')
			subplots[(d1 + d2 - 1) % num_rows, (d1 + d2 - 1) % num_columns ].set(xlabel=f'PC{d1}', ylabel=f'PC{d2}')

def project_data_onto_pcs(data: ndarray, threshold) -> ndarray:
	if threshold < 0 or threshold > 1:
		raise ValueError('Threshold must be between 0 and 1')

	(U, Σ, Vh) = svd(data)
	V = Vh.T

	ρ: ndarray = explained_var(Σ)
	num_pc_to_threshold = (np.cumsum(ρ) < threshold).sum() + 1
	data_projected = data @ V[:, :num_pc_to_threshold]  # Data projected onto {num_pc_to_threshold} components
	#plot_explained_variance(ρ, threshold, "PCA_explained_variance.pdf")
	#print(f"Acceptable threshold: {threshold}\nRequired number of components: {num_pc_to_threshold}")
	#plot_data_projected_unto_principal_components(data @ V[:, :4], class_labels)
	return data_projected


# Jessica Trawick,Gage Kizzar,Monica Hiemer
# 2300326
# trawick@chapman.edu

import numpy as np
import matplotlib.pyplot as plt

def gradient(a,b,n):
    D=np.zeros((n,n))
    x = np.linspace(a,b,n)
    dx = (b-a)/(n-1)
    ones = np.ones(len(D)-1)
    np.fill_diagonal(D[1:],-ones)
    np.fill_diagonal(D[:, 1:],ones)
    D[0][0]=-2
    D[0][1]= 2
    D[n-1][n-1]=2
    D[n-1][n-2]=-2

    D= (1/(2*dx) * D)
    return D

def f_plot(a, b, n):
    x = np.linspace(a,b,n)
    fx = [i**2 for i in x ]
    Df = gradient(a, b , n)
    f_grad= Df@fx
    plt.plot(x, fx , label = "f(x)")
    plt.plot(x, f_grad , label = "f'(x)")
    plt.legend()
    plt.title("$x^2$ function and the derivative")

def f_plot_grad(a,b , n ):
    x = np.linspace(a,b,n)
    fx = [i**2 for i in x ]
    f_grad = np.gradient(fx)
    plt.plot(x, fx , label ="f(x)")
    plt.plot(x, f_grad , label ="f'(x)")
    plt.legend()
    plt.title ("$x^2$ function and the derivative using np.gradient")

def f_second(a,b, n):
    x = np.linspace(a,b,n)
    fx = [i**2 for i in x ]
    D = gradient(a, b , n)
    Df= D@D
    f_grad= D@fx
    f_2_grad= Df@fx
    plt.plot(x, fx , label ="f(x)")
    plt.plot(x, f_grad , label ="f(x)")
    plt.plot(x, f_2_grad, label = "f''(x)")
    plt.legend()
    plt.title ("$x^2$ function and it's first and second derivatives")

def s_plot(a, b , n):
    x = np.linspace(a,b,n)
    fx = [np.sin(i) for i in x ]
    Df = gradient(a, b , n)
    f_grad= Df@fx
    plt.plot(x, fx , label ="f(x)")
    plt.plot(x, f_grad , label ="f'(x)")
    plt.legend()
    plt.title ("Sin(x) fucntion and it's derivative")

def s_polt_grad(a, b, n ):
    x = np.linspace(a,b,n)
    fx = [np.sin(i) for i in x ]
    Df = np.gradient(fx)
    f_grad = np.gradient(fx)
    plt.plot(x, fx , label ="f(x)")
    plt.plot(x, f_grad , label ="f'(x)")
    plt.legend()
    plt.title ("sin(x) function and it's derivative using np.gradient")

def g_plot(a, b , n ):
    x = np.linspace(a,b,n)
    fx = [(np.divide((np.exp(np.divide(-(i**2), 2))), np.sqrt(2*np.pi))) for i in x]
    Df = gradient(a, b , n)
    f_grad= Df@fx
    plt.plot(x, fx , label ="f(x)")
    plt.plot(x, f_grad , label ="f'(x)")
    plt.legend()
    plt.title (" $e^{-x^2/2}/\sqrt{2\pi}$ function and it's derivative")

def g_plot_grad(a, b, n):
    x = np.linspace(a, b, n)
    fx = [(np.divide((np.exp(np.divide(-(i**2), 2))), np.sqrt(2*np.pi))) for i in x]
    f_grad = np.gradient(fx)
    plt.plot(x, fx, label = "f(x)")
    plt.plot(x, f_grad, label = "f'(x)")
    plt.legend()
    plt.title(" $e^{-x^2/2}/\sqrt{2\pi}$ function and its derivative using np.gradient")


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Thu Aug 20 15:53:41 2020

@author: Joel Eugênio - joelecjr@gmail.com
"""
import numpy as np
import matplotlib.pyplot as plt

# Lognormal parameters
S = 1
MU = 0
log_mean = np.exp(MU+(S**2/2))
log_sd = np.sqrt(np.exp(2*MU+S**2)*(np.exp(S**2)-1))

# Lognormal PDF
def p_lognorm(x, sigma=1, mu=0):
    pdf = np.zeros(x.shape)
    i = 0
    for j in x:
        if (j > 0):
            pdf[i] = (1/(j*sigma*np.sqrt(2*np.pi)))*np.exp(-.5*((np.log(j)-mu)/sigma)**2)
        i += 1
    return pdf

# Normal PDF
def p_norm(x, sd=1, m=0):
    return (1/(sd*np.sqrt(2*np.pi)))*np.exp(-.5*((x-m)/sd)**2)

# Laplace PDF
def p_lap(x, sd=1, m=0):
    a = np.sqrt(2)/sd
    return  (a/2)*np.exp(-a*np.abs(x-m))

n = np.linspace(-10,10,1000)

print('Lognormal pdf with mean {:.2f} ({:.2f}) and SD {:.2f}'.format(0,log_mean, log_sd))
p1 = p_lognorm(n+log_mean)
print('Gaussian pdf with mean {:.2f} and SD {:.2f}'.format(0, log_sd))
p2 = p_norm(n, sd=log_sd)
print('Laplace pdf with mean {:.2f} and SD {:.2f}'.format(0, log_sd))
p3 = p_lap(n, sd=log_sd)

plt.plot(n,p1,label='lognormal')
plt.plot(n,p2,label='gaussiana')
plt.plot(n,p3,label='laplace')
plt.axvline(0, 0, 1, ls='--', color='k',linewidth=1)
plt.legend()
plt.show()


# why using t-dependent test?
# for example, the same group student attend a test before
# and after some training
# the key is `the same`
import numpy as np
from scipy import stats

def t_value_dependent(X, X_):
    """
    D = X - X_
                        sum(D)
    t = ---------------------------------------
        sqrt([n * sum(D^2) - sum(D)^2] / (n-1))
    """
    D = X - X_
    n = X.shape[0]
    t = np.sum(D) / np.sqrt((n * np.sum(D**2) - np.sum(D)**2) / (n - 1))
    return np.abs(t)

if __name__ == '__main__':
    X = np.array([
        3, 5, 4, 6, 5, 5, 4, 5, 3, 6, 7, 8,
        7, 6, 7, 8, 8, 9, 9, 8, 7, 7, 6, 7, 8,
    ])
    X_ = np.array([
        7, 8, 6, 7, 8, 9, 6, 6, 7, 8, 8, 7,
        9, 10, 9, 9, 8, 8, 4, 4, 5, 6, 9, 8, 12,
    ])
    t = t_value_dependent(X, X_)
    p = 0.05
    # for we test one side, so p * 2, this is python trick
    left, right = stats.t.interval(1-p*2, df=X.shape[0]-1)
    # and we just get the positive one
    threshold = right
    if t > threshold:
        msg = "reject original hypothesis."
    else:
        msg = 'cannot reject original hypothesis'
    # so our result is
    result = 't[24] = %.2f, p < .05' % t
    print(result)


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Fri Jul 20 14:48:53 2018

@author: sjeet
"""

import numpy as np
import matplotlib.pyplot as plt

def make_list():
    x=list(input('Provide array of numbers (Provide space between numbers)'))
    blank=''
    for i in x:
        blank=blank+i
    x_list=[]
    for i in blank.split(' '):
        x_list.append(int(i))
    return x_list

def tri_mf(x,defn):
    a=defn[0]
    b=defn[1]
    c=defn[2]
    res=[]
    for i in x:
        if i<=a or i>=c:
            res.append(0)
        if a<i<b:
            res.append((i-a)/(b-a))
        if b<i<c:
            res.append((c-i)/(c-b))
        if i==b:
            res.append(1)
    return res

def trap_mf(x,defn):
    a=defn[0]
    b=defn[1]
    c=defn[2]
    d=defn[3]
    res=[]
    for i in x:
        if i<=a or i>=d:
            res.append(0)
        if a<i<b:
            res.append((i-a)/(b-a))
        if b<=i<=c:
            res.append(1)
        if c<i<d:
            res.append((d-i)/(d-c))
    return res

def gauss_mf(x,c,s):
    res=[]
    for i in x:
        res.append(np.exp((-1/2)*((i-c)/s)**2))
    return res

def gen_bell(x,a,b,c):
    res=[]
    for i in x:
        res.append(1/(1+(i-c)/a)**(2*b))
    return res

def sigmoid(x,a,c):
    res=[]
    for i in x:
        res.append(1/(1+np.exp(-a*(x-c))))
    return res

def main():
    x=np.array(make_list())
    atm=int(input('Press 1 for triangular membership function or 2 for trapezoidal membership function 3 for Gaussian MF, 4 for generalised bell MF, 5 for sigmoid.'))
    if atm==1:
        print('Enter list of a,b,c.')
        b=make_list()
        res_ans=tri_mf(x,b)
    if atm==2:
        print('Enter list of a,b,c,d.')
        b=make_list()
        res_ans=trap_mf(x,b)
    if atm==3:
        print('Enter mean and sd for Gaussian MF')
        c=float(input('Value of mean '))
        s=float(input('Value of standard deviation '))
        res_ans=gauss_mf(x,c,s)
    if atm==4:
        print('Enter values of a,b,c')
        a=float(input('Value of a '))
        b=float(input('Value of b '))
        c=float(input('Value of c '))
        res_ans=gen_bell(x,a,b,c)
    if atm==5:
        print('Enter values of a,c')
        a=float(input('Value of a '))
        c=float(input('Value of c '))
        res_ans=sigmoid(x,a,c)
    else:
        print('Invalid input.')
    print('Your desired output is '+'\n',res_ans)
    print('\n'+'Plot is given below.'+'\n')
    plt.plot(res_ans)
    return res_ans

if __name__=='__main__':
    main()


#! /Users/michael/anaconda3/bin/python
# @Date:   2019-03-16 10:41:49

"""
仿照 sklearn 的风格用自己的代码来实现梯度下降算法，从而创建线性回归模型
代码只是为了说明整个算法的思路，要确保代码保持尽可能简洁，因此没有过多处理异常
"""

import numpy as np

class LinearRegression:

    def __init__(self, alpha=0.001, precision=5):
        self.alpha = alpha
        self.precision = precision

    def fit(self, X, y):
        X = np.c_[np.ones((X.shape[0],1)), X]  # 增加一维作为截距
        self.theta = np.zeros((X.shape[1], 1)) # 初始值为全零的参数向量
        count = 0 # 迭代次数计数器，是评估算法优劣的指标之一
        while True:
            count += 1
            last_theta = np.copy(self.theta)
            error = np.dot(X, self.theta)-y
            gradient = np.dot(X.T, error)
            self.theta = self.theta - self.alpha * gradient
            if abs(self.theta - last_theta).mean() < 0.00000001:
                break
        print('迭代次数:', count)
        print('coef_ =', [round(i[0], self.precision) for i in self.theta[1:]])
        print('intercept_ =', round(self.theta[0][0], self.precision))

    def predict(self, x): # 传入变量向量，给出预测值
        x = np.array(x)
        # import ipdb; ipdb.set_trace()
        x = np.r_[[1], x] # 首位加一个1，用于和截距相乘
        return round(np.dot(x, self.theta)[0], self.precision)

class StandardScaler:

    def __init__(self):
        self.stds = []
        self.means = []

    def fit(self, X): # 计算每个特征的均值和标准差
        X = np.array(X) # 传入列表的话，自动转ndarray对象
        for col in X.T: # 转置后遍历列
            self.stds.append(col.std())
            self.means.append(col.mean())

    def transform(self, X): # 标准化每一列
        X = np.array(X)
        cols_trans = []
        for index, col in enumerate(X.T):
            cols_trans.append((col - self.means[index])/self.stds[index])
        return np.array(cols_trans).T

print('没有作标准化：')

X = np.array([[1, 3], [2, 4]])
y = np.array([[10], [15]])

lr1 = LinearRegression()
lr1.fit(X, y)
print(lr1.predict([1,3]))
print(lr1.predict([3,5]))

print('\n----华丽的分界线----\n')
print('经过标准化的训练集：')

scaler = StandardScaler()
scaler.fit(X)
X = scaler.transform(X)
lr2 = LinearRegression()
lr2.fit(X, y)
print(lr2.predict(scaler.transform([1, 3])))
print(lr2.predict(scaler.transform([3, 5])))

"""
运行结果如下：

没有作标准化：
迭代次数: 50947
coef_ = [2.49992, 2.50002]
intercept_ = 5e-05
10.00004
19.99991

----华丽的分界线----

经过标准化的训练集：
迭代次数: 6811
coef_ = [1.25, 1.25]
intercept_ = 12.49999
9.99999
19.99999

可以看到经过标准化之后的训练集能够更快收敛，提高训练效率。
"""


import numpy as np
from numa import utils

def solveGauss(A, b):
    """Solves the given linear system of equations using the Gauss method. A post-iteration is implemented as well
to reduce the error.

Parameters
----------
A: numpy.arrays
    Coefficient matrix
b: numpy.array
    Column vector of constant terms

Returns
-------
x: numpy.array
    Solution vector

Notes
-----
The pivot element for the i-th row in the i-th column is choosen through

.. math::

    a_p = max_{k=i,...,n}|a_{ki}|

Rounding errors can lead to less accurate results so

.. math::

    A \cdot (x + \Delta x) = b + \Delta b

is calculated instead. This can be rewritten

.. math::

    \Delta b = A \cdot (x + \Delta x) - b

.. math::

    A \cdot \Delta x = \Delta b

.. math::

    x_0 = (x + \Delta x)

.. math::

    x_1 = x_0 - \Delta x

"""
    utils._checkDimensions(A, b)
    if utils.isSingular(A):
        raise utils.SingularityError("Input matrix is singular.")
    x_calculated = _gauss(A, b)

    acc = 10e-14
    accuracy_achieved = False
    while not accuracy_achieved:
        delb = b - np.matmul(A, x_calculated)
        delX = _gauss(A, delb)
        x_calculated = np.subtract(x_calculated, delX)
        if [x < acc for x in x_calculated]:
            accuracy_achieved = True
    return x_calculated

def _gauss(A, b):
    """Python implementation of the Gauß method to solve systems of linear equations.

Parameters
----------
A: numpy.arrays
    Coefficient matrix
b: numpy.array
    Column vector of constant terms

Returns
-------
x: numpy.array
    Solution vector

"""
    M = np.column_stack((A, b))
    m, n = M.shape
    for j in range(n-1):
        # i - row, j - column
        pivot_elements = [abs(M[l][j]) for l in range(j, len(M))]
        pivot_index = pivot_elements.index(max(pivot_elements)) + j
        pivot_row, j_row = np.copy(M[pivot_index]), np.copy(M[j])
        M[j], M[pivot_index] = pivot_row, j_row                     # Swap

        for i in range(j+1, m):
            f = M[i][j] / M[j][j]
            for k in range(j + 1, n):
                M[i][k] -= M[j][k] * f
            M[i][j] = 0

    x = []
    for i in range(m - 1, -1, -1):
        x.insert(0, M[i][m] / M[i][i])
        for k in range(i - 1, -1, -1):
            M[k][m] -= M[k][i] * x[0]
    return x


'''
Regression Concepts and Code in Python
'''

import os
os.chdir("C:/Users/tprabhan/Documents/My_Books/RSADBE_2.0/R_and_Python_Programs/Python/Chapter_06")
import matplotlib.pyplot as plt
import pandas as pd
import numpy as np
import statsmodels.api as sm
import statsmodels.formula.api as smf
from statsmodels.formula.api import ols
from statsmodels.regression.linear_model import OLS
import scipy.stats as st

IOT = pd.read_csv("Data/IO_Time.csv",delimiter=',')
IO = IOT.No_of_IO
CPUT = IOT.CPU_Time
plt.plot(IO,CPUT)
plt.xlabel("No of IO"); plt.ylabel("CPU Time")
plt.show()

y = CPUT
X = sm.add_constant(IO)
CPU_lm = sm.OLS(y,X).fit()
predict_CPU = CPU_lm.predict(X)
CPU_lm.summary()

'''
Without Intercept Model
###### Do Not RUN #######
CPU_lm = sm.OLS(CPUT,IO).fit()
predict_CPU = CPU_lm.predict(IO)
CPU_lm.summary()
'''

# Getting ANOVA and Confidence intervals
CPU_lm2 = ols('CPU_Time~No_of_IO',data=IOT).fit()
print(sm.stats.anova_lm(CPU_lm2,typ=2))
print(CPU_lm.conf_int(0.05))

'''
Model Validataion and Residuals
'''
CPU_Resid = CPU_lm.resid
plt.subplot(3,2,1)
plt.plot(IO,CPU_Resid)
plt.xlabel("Predictor Variable")
plt.ylabel("Residuals")
plt.title("Plot of Residuals Vs Predictor Variable")
plt.subplot(3,2,2)
plt.plot(IO,abs(CPU_Resid))
plt.xlabel("Predictor Variable")
plt.ylabel("Absolute Residuals")
plt.title("Plot of Absolute Residual Values Vs Predictor Variable")
plt.subplot(3,2,3)
plt.plot(IO,CPU_Resid**2)
plt.xlabel("Predictor Variable")
plt.ylabel("Squared Residuals")
plt.title("Plot of Squared Residual Values Vs Predictor Variable")
plt.subplot(3,2,4)
plt.plot(predict_CPU,CPU_Resid)
plt.xlabel("Fitted Values")
plt.ylabel("Residuals")
plt.title("Plot of Residuals Vs Predicted Values")
plt.subplot(3,2,5)
plt.plot(np.arange(1,len(CPU_Resid)+1),CPU_Resid)
plt.xlabel("Sequence Number")
plt.ylabel("Residuals")
plt.title("Sequence Plot of the Residuals")
plt.subplot(3,2,6)
plt.boxplot(CPU_Resid)
plt.title("Box Plot of the Residuals")
plt.show()

'''
Multiple Linear Regression Model
'''
Gasoline = pd.read_csv("Gasoline.csv",delimiter=',')
Mileage_lm = smf.ols(formula='y ~ x1 + x2 + x3 + x4 + x5 + x6 + x7 + x8 + x9 + x10 + x11',data=Gasoline).fit()
Mileage_lm.params
Mileage_lm.summary()
print(sm.stats.anova_lm(Mileage_lm,typ=2))
print(Mileage_lm.conf_int(0.05))

plt.subplot(2,1,1)
plt.plot(Mileage_lm.fittedvalues,Mileage_lm.resid,'.')
plt.xlabel("Fitted Values")
plt.ylabel("Residuals")
plt.title("Plot of Residuals Vs Fitted Values")
plt.subplot(2,1,2)
plt.plot(Mileage_lm.fittedvalues,Mileage_lm.resid_pearson,'.')
plt.xlabel("Fitted Values")
plt.ylabel("Pearson Residuals")
plt.title("Plot of Pearson Residuals Vs Fitted Values")
plt.show()

# Alternate way of fitting linear regression model
X = Gasoline[['x1', 'x2', 'x3', 'x4', 'x5', 'x6', 'x7', 'x8', 'x9', 'x10']]
y = Gasoline[['y']]
X = sm.add_constant(X)
X = X.drop(X.index[[18]])
y = y.drop(y.index[[18]])
Gasoline_lm = sm.OLS(y,X)

# http://www.statsmodels.org/dev/_modules/statsmodels/stats/outliers_influence.html

'''
Hatvalues Function
'''

def hatvalues(fitted_lm):
	X=fitted_lm.exog
	hat_mat = np.dot(X,np.dot(np.linalg.inv(np.dot(X.T,X)),X.T))
	hat_values = np.diagonal(hat_mat)
	return(hat_values)
hatvalues(Gasoline_lm)

'''
Cook's distance
'''
Gasoline_lm_ols = smf.ols(formula='y~x1+x2+x3+x4+x5+x6+x7+x8+x9+x10+x11',data=Gasoline)
Gasoline_fitted = Gasoline_lm_ols.fit()
CD = Gasoline_fitted.get_influence()
(cook,p) = CD.cooks_distance
cook
(dffits,p) = CD.dffits
dffits
CD.dfbetas

'''
Variance Inflatino Factor
'''
X2 = X
del X2['const']
del X2['x11']
from statsmodels.stats.outliers_influence import variance_inflation_factor
vif = pd.DataFrame()
vif["VIF Factor"] = [variance_inflation_factor(X2.values, i) for i in range(X2.shape[1])]
vif["features"] = X2.columns
vif.round(1)

'''
Subset Selection
http://www.science.smith.edu/~jcrouser/SDS293/labs/lab8/Lab%208%20-%20Subset%20Selection%20in%20Python.pdf
'''
import os
os.chdir("C:/Users/tprabhan/Documents/My_Books/RSADBE_2.0/R_and_Python_Programs")
import pandas as pd
%matplotlib inline
import numpy as np
import itertools
import time
import statsmodels.api as sm
import matplotlib.pyplot as plt

df = pd.read_csv('Hitters.csv',delimiter=',')
df.head()

print(df["Salary"].isnull().sum())
print(df.shape)
df = df.dropna().drop('Player', axis=1)
print(df.shape)
print(df["Salary"].isnull().sum())
dummies = pd.get_dummies(df[['League', 'Division', 'NewLeague']])
y = df.Salary
X_ = df.drop(['Salary', 'League', 'Division', 'NewLeague'], axis=1).astype('float64')
X = pd.concat([X_, dummies[['League_N', 'Division_W', 'NewLeague_N']]], axis=1)
def processSubset(feature_set):
    # Fit model on feature_set and calculate RSS
    model = sm.OLS(y,X[list(feature_set)])
    regr = model.fit()
    RSS = ((regr.predict(X[list(feature_set)]) - y) ** 2).sum()
    return {"model":regr, "RSS":RSS}
def getBest(k):
    tic = time.time()
    results = []
    for combo in itertools.combinations(X.columns, k):
        results.append(processSubset(combo))
    # Wrap everything up in a nice dataframe
    models = pd.DataFrame(results)
    # Choose the model with the highest RSS
    best_model = models.loc[models['RSS'].argmin()]
    toc = time.time()
    print("Processed ", models.shape[0], "models on", k, "predictors in", (toc-tic), "seconds.")
    # Return the best model, along with some other useful information about the model
    return best_model

models = pd.DataFrame(columns=["RSS", "model"])
tic = time.time()
for i in range(1,8):
    models.loc[i] = getBest(i)
toc = time.time()
print("Total elapsed time:", (toc-tic), "seconds.")

import numpy,import scipy,import statsmodels,from statsmodels


import numpy as np

# %%
__all__ = [
'angle',
'binarize',
'ceil',
'floor',
'moving_mean',
'standardize',
]

# %%
def floor(x, decimals=0):
    if decimals==0:
        return np.floor(x)
    else:
        return np.floor(x * 10**decimals) / 10**decimals

def ceil(x, decimals=0):
    if decimals==0:
        return np.ceil(x)
    else:
        return np.ceil(x * 10**decimals) / 10**decimals

def standardize(array, axis=None, ep=1e-20):
    return (array - array.mean(axis=axis))/(array.std(axis=axis)+ep)

def binarize(array, threshold):
    result = np.zeros_like(array)
    result[array>=threshold] = 1
    return result

def moving_mean(x, w):
    odd = bool(w%2)
    edge = w//2
    if odd:
        x = np.pad(x, w//2+1, mode='edge')
        x = np.cumsum(x).astype(np.float64)
        x = (x[w:] - x[:-w])/w
        return x[:-1]
    else:
        x = np.pad(x, w//2, mode='edge')
        x = np.cumsum(x).astype(np.float64)
        x = (x[w:] - x[:-w])/w
        return x

def angle(x1, x2):
    '''angle between two vectors, derived from cosine rule
    return theta within range of [0,np.pi]'''
    theta = np.arccos(x1@x2/(np.linalg.norm(x1)*np.linalg.norm(x2)))
    return theta

def sigmoid(x):
    return 1/(1+np.exp(-x))

# %%
if __name__ == '__main__':
    pass


"""Generalized Pauli matrices."""
import numpy as np

from toqito.matrices import shift
from toqito.matrices import clock

def gen_pauli(k_1: int, k_2: int, dim: int) -> np.ndarray:
    r"""
    Produce generalized Pauli operator [WikGenPaul]_.

    Generates a `dim`-by-`dim` unitary operator. More specifically, it is the
    operator `X^k_1*Z^k_2`, where X and Z are the "shift" and "clock" operators
    that naturally generalize the Pauli X and Z operators. These matrices span
    the entire space of `dim`-by-`dim` matrices as `k_1` and `k_2` range from 0
    to `dim-1`, inclusive.

    Note that the generalized Pauli operators are also known by the name of
    "discrete Weyl operators". [WatrousLec6]_

    Examples
    ==========

    The generalized Pauli operator for `k_1 = 1`, `k_2 = 0` and `dim = 2` is
    given as the standard Pauli-X matrix

    .. math::
        G_{1, 0, 2} = \begin{pmatrix}
                         0 & 1 \\
                         1 & 0
                      \end{pmatrix}.

    This can be obtained in `toqito` as follows.

    >>> from toqito.matrices import gen_pauli
    >>> dim = 2
    >>> k_1 = 1
    >>> k_2 = 0
    >>> gen_pauli(k_1, k_2, dim)
    [[0.+0.j, 1.+0.j],
     [1.+0.j, 0.+0.j]])

    The generalized Pauli matrix `k_1 = 1`, `k_2 = 1`, and `dim = 2` is given as
    the standard Pauli-Y matrix

    .. math::
        G_{1, 1, 2} = \begin{pmatrix}
                        0 & -1 \\
                        1 & 0
                      \end{pmatrix}.

    This can be obtained in `toqito` as follows.`

    >>> from toqito.matrices import gen_pauli
    >>> dim = 2
    >>> k_1 = 1
    >>> k_2 = 1
    >>> gen_pauli(k_1, k_2, dim)
    [[ 0.+0.0000000e+00j, -1.+1.2246468e-16j],
     [ 1.+0.0000000e+00j,  0.+0.0000000e+00j]])

    References
    ==========
    .. [WikGenPaul] Wikipedia: Generalizations of Pauli matrices
        https://en.wikipedia.org/wiki/Generalizations_of_Pauli_matrices

    .. [WatrousLec6] Lecture 6: Further remarks on measurements and channels
        https://cs.uwaterloo.ca/~watrous/LectureNotes/CS766.Fall2011/06.pdf

    :param k_1: (a non-negative integer from 0 to `dim-1` inclusive).
    :param k_2: (a non-negative integer from 0 to `dim-1` inclusive).
    :param dim: (a positive integer indicating the dimension).
    :return: A generalized Pauli operator.
    """
    gen_pauli_x = shift(dim)
    gen_pauli_z = clock(dim)

    gen_pauli_w = np.linalg.matrix_power(gen_pauli_x, k_1) @ np.linalg.matrix_power(
        gen_pauli_z, k_2
    )

    return gen_pauli_w


# AP Ruymgaart
# simulate ARMA process and plot ACF
import random as rnd, numpy as np
abs, rndm, exp = np.abs, np.random.normal, np.exp
from statsmodels.tsa.stattools import acf
import statsmodels.api as sm
import matplotlib.pyplot as plt

n = 100
noise = 0.5
rn = rndm(0,noise,[n])
#print(rn)

#=== ARMA(0,0) ===
# just noise
if False:
	ACF = acf(rn)

	plt.bar(np.arange(len(ACF)), ACF)
	plt.title('ARMA(0,0)')
	plt.ylabel('Autocorrelation')
	plt.xlabel('Lag')
	plt.show()
else:
	fig, ax = plt.subplots(2,1)
	fig = sm.graphics.tsa.plot_acf(rn, lags=30, ax=ax[0])
	fig = sm.graphics.tsa.plot_pacf(rn, lags=30, ax=ax[1])
	#plt.title('ARMA(0,0) White noise ACF, PACF')
	ax[1].set_xlabel('Lag')
	plt.show()

#=== ARMA(1,0) ===
# Y_t = noise_t + a_1 Y_{t-1}
'''
Note: AR(1) with a1=1 is a random walk (same as ARIMA(0,1,0))
'''
a1 = 0.8
Y = []
ym1 = 0.0
for i in range(len(rn)):
	noise_t = rn[i]
	y = noise_t + a1*ym1
	Y.append(y)
	ym1 = y

ACF = acf(Y)

plt.bar(np.arange(len(ACF)), ACF)
plt.title('ARMA(1,0)')
plt.ylabel('Autocorrelation')
plt.xlabel('Lag')
plt.show()


import sys
import numpy as np
from timeit import default_timer as timer

def gauss_jordan_method(A, b, partial_pivot = False):
    start = timer()
    N = A.shape[0]  # Número de linhas da matriz A.
    M = A.shape[1]  # Número de colunas da matriz A.
    I = np.identity(N)  # Matriz identidade N x N. Por padrão, dtype é float.

    print("Matrix A:")
    print(A)

    print("\nVector b:")
    print(b)

    for j in range(M):  # Colunas que têm elementos a serem zerados.
        if partial_pivot:
            biggest_pivot = abs(A[j, j])
            line_biggest_pivot = j

            for k in range(j + 1, N):  # Encontrando o pivô com maior módulo.
                if abs(A[k, j]) > biggest_pivot:
                    biggest_pivot = abs(A[k, j])
                    line_biggest_pivot = k

            if line_biggest_pivot != j:  # Troca de linhas.
                line = A[j].copy()
                A[j] = A[line_biggest_pivot]
                A[line_biggest_pivot] = line

                line = I[j].copy()
                I[j] = I[line_biggest_pivot]
                I[line_biggest_pivot] = line

                line = b[0, j].copy()
                b[0, j] = b[0, line_biggest_pivot]
                b[0, line_biggest_pivot] = line

        auxiliary = A[j, j].copy()

        for k in range(M):
            A[j, k] = A[j, k] / auxiliary
            I[j, k] = I[j, k] / auxiliary

        b[0, j] = b[0, j] / auxiliary

        for i in range(N):  # Linhas nas quais estes elementos (a serem zerados) estão.
            if i != j:
                m = (-1) * A[i, j]

                for k in range(M):  # Atualização da linha.
                    A[i, k] = m * A[j, k] + A[i, k]
                    I[i, k] = m * I[j, k] + I[i, k]

                b[0, i] = m * b[0, j] + b[0, i]

    print("\nMatrix A as identity:")
    print(A)

    print("\nSolution found:")
    print(b)

    print("\nInverse matrix of A:")
    print(I)

    end = timer()
    print("\nRuntime: %.6e seconds." % (end - start))
    return b

gauss_jordan_method(A = np.array(np.mat(sys.argv[1]), dtype = float),
                    b = np.array(np.mat(sys.argv[2]), dtype = float),
                    partial_pivot = str(sys.argv[3]).lower() == "pp")


# -*- coding: utf-8 -*-

from math import *
from numpy import *

def f(x):
	#return(x)
	return(exp(x)+4)
#print(f(0))

def ITrap(a,b,f,n):
	if (n<2):
		return (0)
	else:
		I=(f(a)+f(b))/2
		if (n>2):
			x=linspace(a,b,n)
			for i in range(1,n-1):
				I=I+f(x[i])
		I=I*float(b-a)/(n-1)
		return (I)

def Aitken(X,Y,x):
	n=min(len(X),len(Y))
	A=[]
	for i in range(n):
		A.append([Y[n-i-1]])
	for j in range(1,n):
		for i in range(n-j):
			val=(X[n-i-j-1]-x)*A[i][j-1]
			val-=(X[n-i-1]-x)*A[i+1][j-1]
			val/=float(X[n-i-j-1]-X[n-i-1])
			A[i].append(val)
			#A[i].append(0)
	#print(A)
	return (A[0][n-1])

def IRom(a,b,f,n):
	if (n<=0):
		return(ITrap(a,b,f,2))
	else:
		H=[float(b-a)/(2**i) for i in range(n+1)]
		I=[ITrap(a,b,f,(2**i)+1) for i in range(n+1)]
		#print(H)
		#print(I)
		return (Aitken(H,I,0))

def IRom2(a,b,f,n):
	H=[b-a]
	I=[ITrap(a,b,f,2)]
	#print(H)
	#print(I)
	if (n>0):
		for i in range(1,n+1):
			H.append((b-a)/(2**i))
			s=0
			for j in range(1,2**(i-1)+1):
				s+=f(a+(2*j-1)*H[i])
			I.append(0.5*I[i-1]+H[i]*s)
			#print(H)
			#print(I)
	return (Aitken(H,I,0))

a=-5
b=6
n=5

#X=[0,1,2,3]
#Y=[1,2,5,10]
#print(Aitken(X,Y,4))

print(ITrap(a,b,f,(2**n)+1))
print(IRom(a,b,f,n))
print(IRom2(a,b,f,n))
#print(float(b**2-a**2)/2)
print(exp(b)-exp(a)+b-a)