import numpy as np
import matplotlib.pyplot as plt

# plotting Rosenbrock funtion
x, y = np.meshgrid(np.linspace(-50, 50, 200),np.linspace(-500, 1000, 200))
z = (1-x)**2 + 100*((y-x**2)**2)

fig = plt.figure()
ax = fig.add_subplot(1, 1, 1, projection='3d')
ax.plot_surface(x, y, z, rstride=1, cstride=1, cmap = 'jet', alpha=0.7)

plt.xlabel("x")
plt.ylabel("y")
plt.title("Rosenbrock Function", fontsize='small')
plt.grid(color='silver', linestyle='-', linewidth=0.2)
ax.scatter(1, 1, marker='o', color='red')
plt.tight_layout()
plt.show()

# plot the contour
fig = plt.figure()
ax = fig.add_subplot(1, 1, 1)
ax.contour(x, y, z, 200, cmap = 'jet')

ax.scatter(1, 1, marker='o', color='red')

plt.xlabel("x")
plt.ylabel("y")
plt.title("Rosenbrock Function", fontsize='small')
plt.grid(color='silver', linestyle='-', linewidth=0.2)
plt.tight_layout()
plt.show()

# using autodiff to calculate the gradient and perform gradient descent
from autograd import grad
import autograd.numpy as np
import numpy

def rosenbrock(x):
    z = (1-x[0])**2 + 100*((x[1]-x[0]**2)**2)
    return z
# testing how to calculate the gradient!
gradient = grad(rosenbrock)
# it works!
gradient(np.array([3., 5.]))

# now doing regular gradient descent
w = np.array([2., 3.])
gamma = 10e-4
epochs = 1000
trace_x = np.zeros(1 + epochs)
trace_y = np.zeros(1 + epochs)
trace_z = np.zeros(1 + epochs)
trace_x[0] = w[0]
trace_y[0] = w[1]
trace_z[0] = rosenbrock(w)

for e in range(epochs):
  grad_eval = gradient(w)
  w = w - gamma*grad_eval
  trace_x[e+1] = w[0]
  trace_y[e+1] = w[1]
  trace_z[e+1] = rosenbrock(w)
  #print(w)
print(trace_z)

# plotting the trace
import numpy as nmp
import matplotlib.pyplot as plt

# plotting Rosenbrock funtion
x, y = nmp.meshgrid(nmp.linspace(-50, 50, 200),nmp.linspace(-500, 1000, 200))
z = (1-x)**2 + 100*((y-x**2)**2)

fig = plt.figure()
ax = fig.add_subplot(1, 1, 1, projection='3d')
ax.plot_surface(x, y, z, rstride=1, cstride=1, cmap = 'jet', alpha=0.7)

plt.xlabel("x")
plt.ylabel("y")
plt.title("Rosenbrock Function", fontsize='small')
plt.grid(color='silver', linestyle='-', linewidth=0.2)
ax.scatter(trace_x, trace_y, trace_z, marker='o', color='red')
plt.tight_layout()
plt.show()

# plot the contour
fig = plt.figure()
ax = fig.add_subplot(1, 1, 1)
ax.contour(x, y, z, 200, cmap = 'jet')
ax.scatter(trace_x, trace_y, marker='o', color='red')

plt.xlabel("x")
plt.ylabel("y")
plt.title("Rosenbrock Function", fontsize='small')
plt.grid(color='silver', linestyle='-', linewidth=0.2)
plt.tight_layout()
plt.show()

# now doing gradient descent with momentum
# I used some code from http://people.duke.edu/~ccc14/sta-663-2018/notebooks/S09G_Gradient_Descent_Optimization.html
beta = 0.9
#w = np.array([2., 3.])
w = np.array([-2., 0.])
v=0
trace_x = np.zeros(1 + epochs)
trace_y = np.zeros(1 + epochs)
trace_z = np.zeros(1 + epochs)
trace_x[0] = w[0]
trace_y[0] = w[1]
trace_z[0] = rosenbrock(w)
gamma = 10e-3
for i in range(epochs):
  v = beta*v + (1-beta)*gradient(w)
  vc = v/(1+beta**(i+1))
  w = w - gamma * vc
  trace_x[i+1] = w[0]
  trace_y[i+1] = w[1]
  trace_z[i+1] = rosenbrock(w)
  #print(w)
print(trace_z)
print(trace_x)
print(trace_y)

# plotting the trace
import numpy as nmp
import matplotlib.pyplot as plt

# plotting Rosenbrock funtion
x, y = nmp.meshgrid(nmp.linspace(-5, 5, 200),nmp.linspace(-5, 5, 200))
z = (1-x)**2 + 100*((y-x**2)**2)

fig = plt.figure()
ax = fig.add_subplot(1, 1, 1, projection='3d')
ax.plot_surface(x, y, z, rstride=1, cstride=1, cmap = 'jet', alpha=0.7)
ax.view_init(elev=10, azim=60)

plt.xlabel("x")
plt.ylabel("y")
plt.title("Rosenbrock Function", fontsize='small')
#plt.grid(color='silver', linestyle='-', linewidth=0.2)
ax.scatter(trace_x, trace_y, trace_z, marker='x', color='red', s=10)
plt.tight_layout()
plt.show()

# plot the contour
fig = plt.figure()
ax = fig.add_subplot(1, 1, 1)
ax.contour(x, y, z, 200, cmap = 'jet')
ax.plot(trace_x, trace_y, color='blue')
ax.scatter(trace_x, trace_y, marker='x', color='red', s=5)
ax.scatter(1, 1, marker='o', color='magenta', s=30)

plt.xlabel("x")
plt.ylabel("y")
plt.title("Rosenbrock Function", fontsize='small')
plt.grid(color='silver', linestyle='-', linewidth=0.2)
plt.tight_layout()
plt.show

# plot the contour
fig = plt.figure()
ax = fig.add_subplot(1, 1, 1)
ax.scatter(trace_x, trace_y, marker='x', color='red', s=5)

plt.xlabel("x")
plt.ylabel("y")
plt.title("Rosenbrock Function", fontsize='small')
plt.grid(color='silver', linestyle='-', linewidth=0.2)
plt.tight_layout()
plt.show


# Linear System Solvers
#
# Available Methods:
# - LU (Own Implementation)
# - LU_sparse (Own Implementation)
# - LU_compiled (Own Implementation)
# - LU_scipy_sparse (Scipy Implementation)
# - QR (Own Implementation)
# - QR_compiled (Own Implementation)
# - QR_scipy (Scipy Implementation)
# - scipy (Scipy Implementation)
# - jacobi (Own Implementation)
# - jacobi_compiled (Own Implementation)
# - jacobi_sparse (Own Implementation)
# - gauss_seidel (Own Implementation)
# - gauss_seidel_compiled (Own Implementation)
# - gauss_seidel_sparse (Own Implementation)
# - sor (Own Implementation)
# - sor_compiled (Own Implementation)
# - sor_sparse (Own Implementation)
#
# Assignment 1 for ME4233
# Author: Giovani Hidalgo Ceotto
# Prof: Mengqi Zhang

import numpy as np
from scipy.sparse import dia_matrix, csr_matrix, csc_matrix, lil_matrix, identity, tril, triu
from scipy.sparse.linalg import splu
from scipy.sparse.linalg import inv as sparse_inv
from scipy.linalg import solve, inv, qr
from numba import jit

# Wrapper for all solvers
def solve_algebraic_system(A, b, method, initial_guess=0, true_solution=0, tol=1e-6, w=1):
    """ Solves the algebraic system A*x = b, for the vector x.

        Parameters
        ----------
        A : array
            Matrix A corresponding to the system A*x = b. Must be in sparse
            format.
        b : array
            Column vector b corresponding to the system A*x = b.
        method : string
            Mehtod to be used. Available methods are:
                - LU (Own Implementation)
                - LU_sparse (Own Implementation)
                - LU_compiled (Own Implementation)
                - LU_scipy_sparse (Scipy Implementation)
                - QR (Own Implementation)
                - QR_compiled (Own Implementation)
                - QR_scipy (Scipy Implementation)
                - scipy (Scipy Implementation)
                - jacobi (Own Implementation)
                - jacobi_compiled (Own Implementation)
                - jacobi_sparse (Own Implementation)
                - gauss_seidel (Own Implementation)
                - gauss_seidel_compiled (Own Implementation)
                - gauss_seidel_sparse (Own Implementation)
                - sor (Own Implementation)
                - sor_compiled (Own Implementation)
                - sor_sparse (Own Implementation)
        initial_guess : array, optional
            Initial solution used for iterative solvers.
        true_solution : array, optional
            If given, iterative solver residuals will be calculated considering
            this.
        tol : float, optional
            Convergence tolerance for iterative solvers.
        w : float, optiona
            Relaxation parameter to be used with the SOR methods.

        Returns
        -------
        x : array
            Solution vector corresponding to A*x = b.
        residue : array
            Residue array given by iterative solver.
    """
    if method == "LU":
        L_prime, U = LU_factorization(A.toarray())
        return solve_upper_triangular_system(U, L_prime.dot(b))
    elif method == "LU_sparse":
        L_prime, U = LU_factorization_sparse(A)
        return solve_upper_triangular_system(U.toarray(), L_prime.dot(b))
    elif method == "LU_compiled":
        L, U = LU_factorization_compiled(A.toarray())
        y = solve_lower_triangular_system(L, b)
        return solve_upper_triangular_system(U, y)
    elif method == "LU_scipy_sparse":
        LU = splu(csc_matrix(A))
        return LU.solve(b)
    elif method == "QR":
        Q, R = QR_factorization(A.toarray())
        return solve_upper_triangular_system(R, np.dot(Q.T, b))
    elif method == "QR_compiled":
        Q, R = QR_factorization_compiled(A.toarray())
        return solve_upper_triangular_system(R, np.dot(Q.T, b))
    elif method == "QR_scipy":
        Q, R = qr(A.toarray())
        return solve_upper_triangular_system(R, np.dot(Q.T, b))
    elif method == "jacobi":
        return jacobi(A.toarray(), b, initial_guess, true_solution, tol)
    elif method == "jacobi_compiled":
        return jacobi_compiled(A.toarray(), b, initial_guess, true_solution, tol)
    elif method == "jacobi_sparse":
        return jacobi_sparse(A, b, initial_guess, true_solution, tol)
    elif method == "gauss_seidel":
        return gauss_seidel(A.toarray(), b, initial_guess, true_solution, tol)
    elif method == "gauss_seidel_compiled":
        return gauss_seidel_compiled(A.toarray(), b, initial_guess, true_solution, tol)
    elif method == "gauss_seidel_sparse":
        return gauss_seidel_sparse(A, b, initial_guess, true_solution, tol)
    elif method == "sor":
        return sor(A.toarray(), b, initial_guess, true_solution, tol, w)
    elif method == "sor_compiled":
        return sor_compiled(A.toarray(), b, initial_guess, true_solution, tol, w)
    elif method == "sor_sparse":
        return sor_sparse(A, b, initial_guess, true_solution, tol, w)
    elif method == "scipy":
        return solve(A.toarray(), b)

# Direct triangular system Solvers
def solve_upper_triangular_system(U, b):
    """ Solves the linear algebraic equation U*x = b, where b is a column
    vector, x is the unknown column vector and U is an upper triangular
    matrix.

    Parameters
    ----------
    U : array
        Square upper triangular matrix.
    b : array
        Column vector with the same number of lines as U.

    Returns
    -------
    x : array
        Column vector with the same number of lines as U which solves U*x=b.
    """
    # Retrieve matrix dimension
    n, m = U.shape

    # Initialize solution vector x:
    x = np.zeros(n)

    # Iterate line by line, beginning from the last one.
    for i in range(n-1, -1, -1):
        x[i] = (b[i] - sum([U[i, j]*x[j] for j in range(i+1, n)]))/U[i, i]

    # Return the solution vector x
    return x

def solve_lower_triangular_system(L, b):
    """ Solves the linear algebraic equation L*y = b, where b is a column
    vector, y is the unknown column vector and L is a lower triangular
    matrix.

    Parameters
    ----------
    L : array
        Square lower triangular matrix.
    b : array
        Column vector with the same number of lines as L.

    Returns
    -------
    y : array
        Column vector with the same number of lines as L which solves L*y=b.
    """
    # Retrieve matrix dimension
    n, m = L.shape

    # Initialize solution vector y
    y = np.zeros(n)

    # Iterate line by line, beginning from the first one.
    for i in range(n):
        y[i] = (b[i] - sum([L[i, j]*y[j] for j in range(i)]))/L[i, i]

    # Return the solution vector y
    return y

# Factorization methods
def LU_factorization(A, return_L=False):
    """ Decomposes the given A matrix into its LU factors, that is A=LU,
    where L is a lower triangular matrix, with diagonal entries equal to 1 and
    U is an upper triangular matrix. By default, it returns U and L^(-1).

    Parameters
    ----------
    A : array
        Matrix to be decomposed into A=LU. A must be a square matrix with no
        diagonal entries equal to 0.
    return_L : bool, optional
        If True, returns U and L. If false, which it is by default, return U
        and L^(-1).

    Returns
    -------
    L : array
        Lower triangular matrix with the same shape as A, equal to L^(-1) if
        return_L is False. If return_L is true, equal to L.
    U : array
        Upper triangular matrix with the same shape as A.

    """
    # Retrieve matrix dimension
    n, m = A.shape

    # Initialize L prime (L^(-1)) and U
    L_prime = np.eye(n, n, dtype=np.float64)
    U = A.copy()

    # Iterate column by column, except for the last column
    for j in range(m - 1):
        # Create temporary L_1
        L_temp = np.eye(n, n, dtype=np.float64)
        L_temp[(j+1):, j] = U[(j+1):, j]/(-U[j, j])

        # Multiply L_temp by current version of U to get new U
        U = L_temp.dot(U)

        # Multiply L_temp by current version of L_prime to get new L_prime
        L_prime = L_temp.dot(L_prime)

    # Return L and U
    if return_L:
        return inv(L_prime), U
    else:
        return L_prime, U

def LU_factorization_sparse(A, return_L=False):
    """ Decomposes the given A matrix into its LU factors, that is A=LU,
    where L is a lower triangular matrix, with diagonal entries equal to 1 and
    U is an upper triangular matrix. By default, it returns U and L^(-1). It
    makes use of sparse properties to optimize speed.

    Parameters
    ----------
    A : array
        Matrix to be decomposed into A=LU. A must be a square matrix with no
        diagonal entries equal to 0.
    return_L : bool, optional
        If True, returns U and L. If false, which it is by default, return U
        and L^(-1).

    Returns
    -------
    L : array
        Lower triangular matrix with the same shape as A, equal to L^(-1) if
        return_L is False. If return_L is true, equal to L.
    U : array
        Upper triangular matrix with the same shape as A.

    """
    # Retrieve matrix dimension
    n, m = A.shape

    # Initialize L prime (L^(-1)) U
    L_prime = identity(n, np.float64, 'csr')
    U = csc_matrix(A)
    L_temp1 = identity(n, np.float64, 'lil')

    # Iterate column by column, except for the last column
    for j in range(m - 1):
        # Create temporary Ls
        L_temp1[(j+1):, j] = U[(j+1):, j]/(-U[j, j])
        L_temp2 = csr_matrix(L_temp1)

        # Roll back L_temp1
        L_temp1[(j+1):, j] = 0

        # Multiply L_temp by current version of U to get new U
        U = L_temp2.dot(U)

        # Multiply L_temp by current version of L_prime to get new L_prime
        L_prime = L_temp2.dot(L_prime)

    # Return L and U
    if return_L:
        return Inv(L_prime), U
    else:
        return L_prime, U

@jit(nopython=True)
def LU_factorization_compiled(A):
    """ Decomposes the given A matrix into its LU factors, that is A=LU,
    where L is a lower triangular matrix, with diagonal entries equal to 1 and
    U is an upper triangular matrix. The function is compiled the first time it
    is ran by numba.

    Parameters
    ----------
    A : array
        Matrix to be decomposed into A=LU. A must be a square matrix with no
        diagonal entries equal to 0 and in numpy format.
    return_L : bool, optional
        If True, returns U and L. If false, which it is by default, return U
        and L^(-1).

    Returns
    -------
    L : array
        Lower triangular matrix with the same shape as A.
    U : array
        Upper triangular matrix with the same shape as A.

    """
    # Retrieve matrix dimension
    n, m = A.shape

    # Initialize L and U
    L = np.eye(n, n)
    U = np.eye(n, n)

    # Create first column of L and first row of U
    L[:, 0] = A[:, 0]/A[0, 0]
    U[0, :] = A[0, :]

    # Iterate line by line, except for the first and last ones
    for i in range(1, n):
        #  Complete ith row of U and ith column of L
        for j in range(i, n):
            sum1 = 0
            for k in range(i):
                sum1 += L[i, k]*U[k, j]
            U[i, j] = A[i, j] - sum1
            sum2 = 0
            for k in range(i):
                sum2 += L[j, k]*U[k, i]
            L[j, i] = (A[j, i] - sum2)/U[i, i]

    # Return L and U
    return L, U

def QR_factorization(A):
    """Decomposes the given A matrix into its QR factors, that is A=QR,
    where Q is a orthogonal matrix and R is an upper triangular matrix.

    Parameters
    ----------
    A : array
        Matrix to be decomposed into A=QR. A must be a square matrix with no
        diagonal entries equal to 0.

    Returns
    -------
    Q : array
        Orthogonal matrix with the same shape as A.
    R : array
        Upper triangular matrix with the same shape as A.
    """
    # Retrieve matrix dimension
    n, m = A.shape

    # Initialize Q and R matrix
    Q = A.copy()
    R = np.eye(n, n)

    # Compute columns of Q matrix
    for j in range(n):
        # Subtract projections on previous columns
        temp1 = Q[:, j]
        temp2 = temp1 - sum([np.dot(temp1.T, Q[:, k])*Q[:, k] for k in range(j)])
        # Normalize result
        Q[:, j] = temp2/np.linalg.norm(temp2)

    # Compute lines of R matrix
    for i in range(n):
        R[i, i:] = [np.dot(A[:, j], Q[:, i]) for j in range(i, n)]

    # Return Q and R
    return Q, R

@jit(nopython=True)
def QR_factorization_compiled(A):
    """Decomposes the given A matrix into its QR factors, that is A=QR,
    where Q is a orthogonal matrix and R is an upper triangular matrix.

    Parameters
    ----------
    A : array
        Matrix to be decomposed into A=QR. A must be a square matrix with no
        diagonal entries equal to 0.

    Returns
    -------
    Q : array
        Orthogonal matrix with the same shape as A.
    R : array
        Upper triangular matrix with the same shape as A.
    """
    # Retrieve matrix dimension
    n, m = A.shape

    # Initialize Q and R matrix
    Q = A.copy()
    R = np.eye(n, n)

    # Compute columns of Q matrix
    for j in range(n):
        # Subtract projections on previous columns
        current_column = Q[:, j].copy()
        for k in range(j):
            reference_column = Q[:, k]
            # Project current column on reference column
            proj = 0
            for l in range(n):
                proj += current_column[l] * reference_column[l]
            # Subtract projection from current column
            current_column -= proj*reference_column
        # Calculate norm
        norm = 0.0
        for k in range(n):
            norm += current_column[k]**2
        norm = (norm)**0.5
        # Normalize result
        Q[:, j] = current_column/norm

    # Compute lines of R matrix
    for i in range(n):
        for j in range(i, n):
            # Scalar projection of jth column from A into ith column from Q
            proj = 0
            A_column = A[:, j]
            Q_column = Q[:, i]
            for k in range(n):
                proj += A_column[k] * Q_column[k]
            R[i, j] = proj

    # Return Q and R
    return Q, R

# Iterative Solvers
def jacobi(A, b, initial_guess, true_solution, tol):
    """ Solves the algebraic system A*x=b using a jacobi iterative solver.

    Parameters
    ----------
    A : array
        Matrix A corresponding to the system A*x = b.
    b : array
        Column vector b corresponding to the system A*x = b.
    initial_guess : array, optional
        Initial solution.
    true_solution : array, optional
        Used for residual calculation
    tol : float, optional
        Convergence tolerance.

    Returns
    -------
    x : array
        Solution vector corresponding to A*x = b.
    residue : array
        Residue array.
    """
    print("Initializing Jacobi Solver", end='\r')

    # Retrieve matrix dimension
    n, m = A.shape

    # Flatten b
    b = b.flatten()

    # Initialize u
    u = initial_guess.flatten()
    u_true = true_solution.flatten()
    u_new = u.copy()

    # Initialize residue
    res = np.linalg.norm(u - true_solution)
    res_array = [res]

    # Iterate
    while res > tol:
        print("Current Iteration and Residue: {:06d} - {:05.4E}".format(len(res_array), res), end='\r')

        for i in range(n):
            sum_factor = 0
            for j in range(i):
                sum_factor += A[i, j] * u[j]

            for j in range(i + 1, n):
                sum_factor += A[i, j] * u[j]

            u_new[i] = (1/A[i, i]) * (b[i] - sum_factor)

        # Update u
        u = u_new.copy()

        res = np.linalg.norm(u - true_solution)
        res_array += [res]

    print("\nJacobi solver converged after ", len(res_array), " iterations.")
    return u, res_array

@jit(nopython=True)
def jacobi_compiled(A, b, initial_guess, true_solution, tol):
    """ Solves the algebraic system A*x=b using a jacobi iterative solver.

    Parameters
    ----------
    A : array
        Matrix A corresponding to the system A*x = b.
    b : array
        Column vector b corresponding to the system A*x = b.
    initial_guess : array, optional
        Initial solution.
    true_solution : array, optional
        Used for residual calculation
    tol : float, optional
        Convergence tolerance.

    Returns
    -------
    x : array
        Solution vector corresponding to A*x = b.
    residue : array
        Residue array.
    """
    # Retrieve matrix dimension
    n, m = A.shape

    # Flatten b
    b = b.flatten()

    # Initialize u
    u = initial_guess.flatten()
    u_true = true_solution.flatten()
    u_new = u.copy()

    # Initialize residue
    res = np.linalg.norm(u - true_solution)
    res_array = [res]

    # Iterate
    while res > tol:
        for i in range(n):
            sum_factor = 0
            for j in range(i):
                sum_factor += A[i, j] * u[j]

            for j in range(i + 1, n):
                sum_factor += A[i, j] * u[j]

            u_new[i] = (1/A[i, i]) * (b[i] - sum_factor)

        # Update u
        u = u_new.copy()

        # Update residue
        res = np.linalg.norm(u - true_solution)
        res_array += [res]

    return u, res_array

def jacobi_sparse(A, b, initial_guess, true_solution, tol):
    """ Solves the algebraic system A*x=b using a jacobi iterative solver.

    Parameters
    ----------
    A : array
        Matrix A corresponding to the system A*x = b. Should be in sparse
        format.
    b : array
        Column vector b corresponding to the system A*x = b.
    initial_guess : array, optional
        Initial solution.
    true_solution : array, optional
        Used for residual calculation
    tol : float, optional
        Convergence tolerance.

    Returns
    -------
    x : array
        Solution vector corresponding to A*x = b.
    residue : array
        Residue array.
    """
    # print("Initializing Jacobi Sparse Solver", end='\r')
    # Retrieve matrix dimension
    n, m = A.shape

    # Define D by extracting main diagonal from A
    main_diagonal = A.diagonal()
    D = dia_matrix(([main_diagonal], [0]), shape=(n, n), dtype=np.float64)

    # Compute R
    R = A - D

    # Compute D^-1
    D_inverse = dia_matrix(([1/main_diagonal], [0]), shape=(n, n), dtype=np.float64)

    # Compute (D^-1)*b
    D_inverse_dot_b =  D_inverse.dot(b.flatten())

    # Compute -(D^-1)*R
    D_inverse_dot_R = D_inverse.dot(R)

    # Initialize u
    u = initial_guess.flatten()
    u_true = true_solution.flatten()

    # Initialize residue
    res = np.linalg.norm(u - true_solution)
    res_array = [res]

    # Iterate
    while res > tol:
        # print("Current Iteration and Residue: {:06d} - {:05.4E}".format(len(res_array), res), end='\r')

        # Update u
        u = D_inverse_dot_b - D_inverse_dot_R.dot(u)

        # Update residue
        res = np.linalg.norm(u - true_solution)
        res_array += [res]

    # print("\nJacobi solver converged after ", len(res_array), " iterations.")
    return u, res_array

def gauss_seidel(A, b, initial_guess, true_solution, tol):
    """ Solves the algebraic system A*x=b using a Gauss-Seidel iterative solver.

    Parameters
    ----------
    A : array
        Matrix A corresponding to the system A*x = b.
    b : array
        Column vector b corresponding to the system A*x = b.
    initial_guess : array, optional
        Initial solution.
    true_solution : array, optional
        Used for residual calculation
    tol : float, optional
        Convergence tolerance.

    Returns
    -------
    x : array
        Solution vector corresponding to A*x = b.
    residue : array
        Residue array.
    """
    print("Initializing Gauss-Seidel Solver", end='\r')

    # Retrieve matrix dimension
    n, m = A.shape

    # Flatten b
    b = b.flatten()

    # Initialize u
    u = initial_guess.flatten()
    u_true = true_solution.flatten()

    # Initialize residue
    res = np.linalg.norm(u - true_solution)
    res_array = [res]

    # Iterate
    while res > tol:
        print("Current Iteration and Residue: {:06d} - {:05.4E}".format(len(res_array), res), end='\r')

        for i in range(n):
            sum_factor = 0
            for j in range(i):
                sum_factor += A[i, j] * u[j]

            for j in range(i + 1, n):
                sum_factor += A[i, j] * u[j]

            u[i] = (1/A[i, i]) * (b[i] - sum_factor)

        res = np.linalg.norm(u - true_solution)
        res_array += [res]

    print("\nGauss-Seidel solver converged after ", len(res_array), " iterations.")
    return u, res_array

@jit(nopython=True)
def gauss_seidel_compiled(A, b, initial_guess, true_solution, tol):
    """ Solves the algebraic system A*x=b using a Gauss-Seidel iterative solver.
    This function takes advantage of just in time compilation to speed up
    evaluations.

    Parameters
    ----------
    A : array
        Matrix A corresponding to the system A*x = b.
    b : array
        Column vector b corresponding to the system A*x = b.
    initial_guess : array, optional
        Initial solution.
    true_solution : array, optional
        Used for residual calculation
    tol : float, optional
        Convergence tolerance.

    Returns
    -------
    x : array
        Solution vector corresponding to A*x = b.
    residue : array
        Residue array.
    """
    # Retrieve matrix dimension
    n, m = A.shape

    # Flatten b
    b = b.flatten()

    # Initialize u
    u = initial_guess.flatten()
    u_true = true_solution.flatten()

    # Initialize residue
    res = np.linalg.norm(u - true_solution)
    res_array = [res]

    # Iterate
    while res > tol:
        for i in range(n):
            sum_factor = 0
            for j in range(i):
                sum_factor += A[i, j] * u[j]

            for j in range(i + 1, n):
                sum_factor += A[i, j] * u[j]

            u[i] = (1/A[i, i]) * (b[i] - sum_factor)

        # Update residue
        res = np.linalg.norm(u - true_solution)
        res_array += [res]

    return u, res_array

def gauss_seidel_sparse(A, b, initial_guess, true_solution, tol):
    """ Solves the algebraic system A*x=b using a Gauss-Seidel iterative solver.
    This function makes use of the sparsity of the A matrix to speed up
    function evaluation.

    Parameters
    ----------
    A : array
        Matrix A corresponding to the system A*x = b. Should be in sparse
        format.
    b : array
        Column vector b corresponding to the system A*x = b.
    initial_guess : array, optional
        Initial solution.
    true_solution : array, optional
        Used for residual calculation
    tol : float, optional
        Convergence tolerance.

    Returns
    -------
    x : array
        Solution vector corresponding to A*x = b.
    residue : array
        Residue array.
    """
    # print("Initializing Gauss-Seidel Sparse Solver", end='\r')
    # Retrieve matrix dimension
    n, m = A.shape

    # Compute L by extracting lower triangular matrix from A
    L = tril(A)

    # Compute U by extracting upper triangular matrix from A
    U = triu(A, k=1)

    # Compute L^-1
    L_inverse = sparse_inv(csc_matrix(L))

    # Compute (L^-1)*b
    L_inverse_dot_b =  L_inverse.dot(b.flatten())

    # Compute -(L^-1)*U
    L_inverse_dot_U = L_inverse.dot(U)

    # Initialize u
    u = initial_guess.flatten()
    u_true = true_solution.flatten()

    # Initialize residue
    res = np.linalg.norm(u - true_solution)
    res_array = [res]

    # Iterate
    while res > tol:
        # print("Current Iteration and Residue: {:06d} - {:05.4E}".format(len(res_array), res), end='\r')

        # Update u
        u = L_inverse_dot_b - L_inverse_dot_U.dot(u)

        # Update residue
        res = np.linalg.norm(u - true_solution)
        res_array += [res]

    # print("\nGauss-Seidel solver converged after ", len(res_array), " iterations.")
    return u, res_array

def sor(A, b, initial_guess, true_solution, tol, w):
    """ Solves the algebraic system A*x=b using a Gauss-Seidel Successive
    Over Relaxation iterative solver.

    Parameters
    ----------
    A : array
        Matrix A corresponding to the system A*x = b.
    b : array
        Column vector b corresponding to the system A*x = b.
    initial_guess : array, optional
        Initial solution.
    true_solution : array, optional
        Used for residual calculation
    tol : float, optional
        Convergence tolerance.
    w : float
        Relaxation parameter.

    Returns
    -------
    x : array
        Solution vector corresponding to A*x = b.
    residue : array
        Residue array.
    """
    print("Initializing Gauss-Seidel SOR Solver", end='\r')

    # Retrieve matrix dimension
    n, m = A.shape

    # Flatten b
    b = b.flatten()

    # Initialize u
    u = initial_guess.flatten()
    u_true = true_solution.flatten()

    # Initialize residue
    res = np.linalg.norm(u - true_solution)
    res_array = [res]

    # Iterate
    while res > tol:
        print("Current Iteration and Residue: {:06d} - {:05.4E}".format(len(res_array), res), end='\r')

        for i in range(n):
            sum_factor = 0
            for j in range(i):
                sum_factor += A[i, j] * u[j]

            for j in range(i + 1, n):
                sum_factor += A[i, j] * u[j]

            u[i] = (1-w)*u[i] + w*(1/A[i, i]) * (b[i] - sum_factor)

        res = np.linalg.norm(u - true_solution)
        res_array += [res]

    print("\nGauss-Seidel Successive Over Relaxation solver converged after ", len(res_array), " iterations.")
    return u, res_array

@jit(nopython=True)
def sor_compiled(A, b, initial_guess, true_solution, tol, w):
    """ Solves the algebraic system A*x=b using a Gauss-Seidel Successive
    Over Relaxation iterative solver. This function takes advantage of just
    in time compilation to speed up evaluations.

    Parameters
    ----------
    A : array
        Matrix A corresponding to the system A*x = b.
    b : array
        Column vector b corresponding to the system A*x = b.
    initial_guess : array, optional
        Initial solution.
    true_solution : array, optional
        Used for residual calculation
    tol : float, optional
        Convergence tolerance.
    w : float
        Relaxation parameter.

    Returns
    -------
    x : array
        Solution vector corresponding to A*x = b.
    residue : array
        Residue array.
    """
    # Retrieve matrix dimension
    n, m = A.shape

    # Flatten b
    b = b.flatten()

    # Initialize u
    u = initial_guess.flatten()
    u_true = true_solution.flatten()

    # Initialize residue
    res = np.linalg.norm(u - true_solution)
    res_array = [res]

    # Iterate
    while res > tol:
        for i in range(n):
            sum_factor = 0
            for j in range(i):
                sum_factor += A[i, j] * u[j]

            for j in range(i + 1, n):
                sum_factor += A[i, j] * u[j]

            u[i] = (1-w)*u[i] + w*(1/A[i, i]) * (b[i] - sum_factor)

        res = np.linalg.norm(u - true_solution)
        res_array += [res]

    return u, res_array

def sor_sparse(A, b, initial_guess, true_solution, tol, w):
    """ Solves the algebraic system A*x=b using a Gauss-Seidel Successive
    Over Relaxation iterative solver. This function makes use of the sparsity
    of the A matrix to speed up function evaluation.

    Parameters
    ----------
    A : array
        Matrix A corresponding to the system A*x = b. Should be in sparse
        format.
    b : array
        Column vector b corresponding to the system A*x = b.
    initial_guess : array, optional
        Initial solution.
    true_solution : array, optional
        Used for residual calculation
    tol : float, optional
        Convergence tolerance.
    w : float
        Relaxation parameter.

    Returns
    -------
    x : array
        Solution vector corresponding to A*x = b.
    residue : array
        Residue array.
    """
    # print("Initializing Gauss-Seidel Successive Over Relaxation Sparse Solver", end='\r')
    # Retrieve matrix dimension
    n, m = A.shape

    # Compute D by extracting main diagonal from A
    main_diagonal = A.diagonal()
    D = dia_matrix(([main_diagonal], [0]), shape=(n, n), dtype=np.float64)

    # Compute L by extracting lower triangular matrix from A
    L = tril(A, k=-1)

    # Compute U by extracting upper triangular matrix from A
    U = triu(A, k=1)

    # Compute (D + w*L)^-1
    D_plus_w_L_inverse = sparse_inv(csc_matrix(D + w*L))

    # Compute w*b
    w_b =  w*b.flatten()

    # Compute w*U + (w-1)*D
    w_U_plus_w_minus_1_D = w*U + (w-1)*D

    # Compute ((D + w*L)^-1)*(w*U + (w-1)*D)
    D_plus_w_L_inverse_dot_w_U_plus_w_minus_1_D = D_plus_w_L_inverse.dot(w_U_plus_w_minus_1_D)

    # Compute ((D + w*L)^-1)*w*b0
    D_plus_w_L_inverse_dot_w_b = D_plus_w_L_inverse.dot(w_b)

    # Initialize u
    u = initial_guess.flatten()
    u_true = true_solution.flatten()

    # Initialize residue
    res = np.linalg.norm(u - true_solution)
    res_array = [res]

    # Iterate
    while res > tol:
        # print("Current Iteration and Residue: {:06d} - {:05.4E}".format(len(res_array), res), end='\r')

        # Update u
        u = D_plus_w_L_inverse_dot_w_b - D_plus_w_L_inverse_dot_w_U_plus_w_minus_1_D.dot(u)

        # Update residue
        res = np.linalg.norm(u - true_solution)
        res_array += [res]

    # print("\nGauss-Seidel Successive Over Relaxation solver converged after ", len(res_array), " iterations.")
    return u, res_array


"""
Static methods that all take a float parameter meant to be between 0 and 1 (but can be any), and return a float between
0 and 1. Meant to be used as fill color value in order to have custom shadings. Some return a list of 3 floats
for rgb values.
"""
import math
from typing import Callable

from scipy.stats import norm

# import numpy as np

def _clamp(x: float, lower_limit: float = 0.0, upper_limit: float = 1.0) -> float:
    if x > upper_limit:
        return upper_limit
    if x < lower_limit:
        return lower_limit
    return x

def zero(ratio: float) -> float:
    return 0.0

def _power(exponent: float) -> Callable[[float], float]:
    return lambda x: _clamp(x ** exponent)

def identity(ratio: float) -> float:
    return _power(1.0)(ratio)

def square(ratio: float) -> float:
    return _power(2)(ratio)

def cube(ratio: float) -> float:
    return _power(3)(ratio)

def sinus(ratio: float) -> float:
    return _clamp(math.sin(2 * ratio / math.pi))

def _gauss(mean: float = 0.5, sd: float = 0.3) -> Callable[[float], float]:
    return lambda x: norm.cdf(x, mean, sd)

def gauss(ratio: float) -> float:
    return _gauss()(ratio)

def gauss_heavy(ratio: float) -> float:
    return _gauss(0.8, 0.4)(ratio)

def _logistic_curve(midpoint: float = 0.5, steepness: float = 1.0, max_value: float = 1.0) -> Callable[[float], float]:
    return lambda x: max_value / (1 + math.e ** (-steepness * (x - midpoint)))

def logistic_curve(ratio: float) -> float:
    return _logistic_curve(0.5, 5.0, 1.0)(ratio)

def smooth_step(ratio: float, edge_1: float = 0.0, edge_2: float = 1.0) -> float:
    x = _clamp((ratio - edge_1) / (edge_2 - edge_1))
    return x * x * (3 - 2 * x)

def smoother_step(ratio: float, edge_1: float = 0.0, edge_2: float = 1.0) -> float:
    x = _clamp((ratio - edge_1) / (edge_2 - edge_1))
    return x * x * x * (x * (x * 6 - 15) + 10)

def general_smooth_step(N, x):
    x = _clamp(x, 0.0, 1.0)
    result = 0

    for i in range(N):
        result += pascal_triangle(-N - 1, i) * pascal_triangle(2 * N + 1, N - i) * math.pow(x, N + i + 1)
    return result

def pascal_triangle(a, b):
    result = 1
    for i in range(b):
        result *= (a - i) / (i + 1)
    return result


from scipy.integrate import quad
import numpy as np

import matplotlib.pyplot as plt

#-----------------------------------
#INTEGRATION EXAMPLE
#----------------------------------

#ANTIDEVIATIVE OF x^2--> x^3/3
def integrand(x):
    return x**2

print(1/3-0,quad(integrand, 0, 1))

#-----------------------------------
#NORMAL DISTIBUTION ENTROPY
#----------------------------------

s=1; u=0
def p(x):
	return np.exp(-0.5*((x-u)/s)**2)/(s*np.sqrt(2*3.1415))

def I(x): return -p(x)*np.log(p(x))

S=quad(I, -20, 20)
print(S, np.log(2*3.1415*np.exp(1)*s)) #,np.exp(1))

#-----------------------------------
#KL DIVERGENCE AND CROSS ENTROPY
#----------------------------------

u1=0; s1=1
u2=3; s2=1

#INTEGRATION LIMITS
x1=min(u1,u2)-4*max(s1,s2)
x2=max(u1,u2)+4*max(s1,s2); print(x1,x2)

#DEFINE TWO NORMAL DISTRIBUTINO
def p1(x): return np.exp(-0.5*((x-u1)/s1)**2)/(s1*np.sqrt(2*3.1415))
def p2(x): return np.exp(-0.5*((x-u2)/s2)**2)/(s2*np.sqrt(2*3.1415))

def I_S1(x): return -p1(x)*np.log(p1(x))  		#P1 ENTROPY
def I_S2(x): return -p2(x)*np.log(p2(x))  		#P2 ENTROPY
def I_CE(x): return -p1(x)*np.log(p2(x))  		#CROSS ENTROPY
def I_KL(x): return -p1(x)*np.log(p2(x)/p1(x))  #D_KL

def plot(KLD,S1,S2,CE):
	fig = plt.figure(figsize=(20,12))
	ax = fig.add_subplot(111)
	x=np.linspace(x1,x2,500)
	plt.plot(x, p1(x), label="p(x)", linewidth=4)
	plt.plot(x, p2(x), label="q(x)", linewidth=4)
	plt.plot(x, I_S1(x), label="I_S1", linewidth=4)
	# plt.plot(x, I_S2(x), label="I_S2", linewidth=4)
	plt.plot(x, I_CE(x), label="I_CE", linewidth=4)
	plt.plot(x, I_KL(x), label="S1,S2,CE,KLD="+str(S1)+' '+str(S2)+' '+str(CE)+' '+str(KLD), linewidth=4)
	plt.legend(loc="best")
	ax.set_xlabel('x')
	ax.set_ylabel('probablity density function')
	plt.show()

S1=round(quad(I_S1, x1, x2)[0],3)
S2=round(quad(I_S1, x1, x2)[0],3)
KLD=round(quad(I_KL, x1, x2)[0],3)
CE=round(quad(I_CE, x1, x2)[0],3)
print(CE,S1+KLD)
plot(KLD,S1,S2,CE)

exit()

#-----------------------------------
#KL DIVERGENCE: MOVIE
#----------------------------------

fig = plt.figure(figsize=(20,12))
ax = fig.add_subplot(111)

for var in [15,1.42,3,4,5,6,7,8,9]:
	u1=0; s1=1
	u2=0; s2=var

	#INTEGRATION LIMITS
	x1=min(u1,u2)-4*max(s1,s2)
	x2=max(u1,u2)+4*max(s1,s2); print(x1,x2)

	#DEFINE TWO NORMAL DISTRIBUTINO
	def p1(x): return np.exp(-0.5*((x-u1)/s1)**2)/(s1*np.sqrt(2*3.1415))
	def p2(x): return np.exp(-0.5*((x-u2)/s2)**2)/(s2*np.sqrt(2*3.1415))

	#INTEGRAND
	def I(x): return -p1(x)*np.log(p2(x)/p1(x))

	def plot(KLD):

		x=np.linspace(x1,x2,500)
		ax.clear()

		ax.plot(x, p1(x), label="p(x)", linewidth=4)
		ax.plot(x, p2(x), label="q(x)", linewidth=4)
		ax.plot(x, I(x), label="p(x)log(p(x)/q(x)): KLD="+str(KLD), linewidth=4)
		ax.legend(loc="best")
		ax.set_xlabel('x')
		ax.set_ylabel('probablity density function')
		plt.pause(1)

	KLD=quad(I, x1, x2)[0]
	plot(KLD)

plt.show()


import numpy as np

def func1(x):
    # function f(x)= 10 + sum_i(-x_i^2)
    # for 2D: f(x)= 10 - x1^2 - x2^2
    return 10 + np.sum(-1*np.power(x, 2), axis=1)

def mc_integrate(func, a, b, dim, n = 1000):
    # Monte Carlo integration of given function over domain from a to b (for each parameter)
    # dim: dimensions of function

    x_list = np.random.uniform(a, b, (n, dim))
    print(x_list)
    y = func(x_list)
    print(y)

    y_mean =  y.sum()/len(y)
    domain = np.power(b-a, dim)

    integ = domain * y_mean

    return integ

# Examples
print("For f(x)= 10 - x1\u00b2 - x2\u00b2, integrated from -2 to 2 (for all x's)")
print(f"Monte Carlo solution for : {mc_integrate(func1, -2, 2, 2, 1000000): .3f}")
print(f"Analytical solution: 117.333")

print("For f(x)= 10 - x1\u00b2 - x2\u00b2 - x3\u00b2, integrated from -2 to 2 (for all x's)")
print(f"Monte Carlo solution: {mc_integrate(func1, -2, 2, 3, 1000000): .3f}")
print(f"Analytical solution: 384.000")


import numpy as np

def heuristic_aux(x, y, x_g, y_g, name):
  """Admissible heuristics

  Parameters
  ----------
  x, y: int
    state coordinate
  x_g, y_g: int
    goal coordinate
  name: str
    heuristic function name

  Return
  ------
  Float:
    Corresponding heuristic value for a single coordinate
  """
  if name == 'tiles-out':
    return x != x_g or y != y_g
  elif name == 'manhattan':
    return abs(x-x_g) + abs(y-y_g)
  elif name == 'euclidean':
    return np.sqrt((x-x_g) ** 2 + (y-y_g) ** 2)
  elif name == 'uniform-cost':
    return 0

def heuristic(goal, state, name='euclidean'):
  """Heuristic function

  Parameters
  ----------
  goal: list
    Puzzle state we want to achieve
  state: list
    Current Puzzle state
  name: str (Default: 'euclidean')
    Name of heuristic function to use ∈ ['euclidean', 'manhattan', 'tiles-out', 'uniform-cost']

  Returns
  -------
  One of
    'Tiles-out': float
      Number of tiles in the wrong position
    'Manhattan': float
      Sum of the horizontal and vertical distances between,
      current position and desired position, i.e. ∑ |state - goal|
    'Euclidean': float
      Sum of the distance between tiles
      ∑ √(x - x_g)² + (y - y_g)²
  """
  size = int(np.sqrt(len(goal)))

  coord_goal  = np.zeros((size ** 2, 2)) # in index i coordinates of tile i in goal
  coord_state = np.zeros((size ** 2, 2)) # in index i coordinates of tile i in state
  for x in range (size):
    for y in range (size):
      coord_goal[ goal[ x * size + y]] = [x, y]
      coord_state[state[x * size + y]] = [x, y]

  return np.sum([heuristic_aux(*coord_state[i], *coord_goal[i], name) for i in range(size ** 2)])


from matplotlib import pyplot as plt
import pandas as pd
import numpy as np
import math
import scipy.stats as stats

from .. import samp

def samp_size(s_p, width, alpha):
    """
    Input: s_p (sample proportion), width, alpha
    Output: Estimated sample size
    """
    z_cv = stats.norm.ppf(1 - alpha / 2)
    return (z_cv * math.sqrt(s_p * (1 - s_p)) / width) ** 2

def con_level(s_p, n, alpha, show=True, Wilson=False, N=False, correction=True):
    """
    Caution: np̂ > 5 and n(1 - p̂) > 5
    Input: s_p, n, alpha, show=True, Wilson=False, N=False, correction=True
    Output: {"lat": lat, "lcl": lcl, "ucl": ucl, "z_cv": z_cv}

    Note:
        z_cv = stats.norm.ppf(1 - alpha / 2)
        lat = z_cv * math.sqrt(s_p * (1 - s_p)/n)
        or for Wilson: lat = z_cv * math.sqrt(s_p * (1 - s_p)/(n + 4))

    Just in case:
    if there is no need for correction, but is is corrected
    go through 'lat' and do:
        lcl = s_p - lat
        ucl = s_p + lat
    """
    con_coef = 1 - alpha
    z_cv = stats.norm.ppf(1 - alpha / 2)
    if not Wilson and not samp.check5(n, s_p):
        print('Not satisfying np̂ > 5 and n(1 - p̂) > 5...')
    if Wilson:
        # make sure that you have arrange s_p to (x + 2) / (n + 4)
        lat = z_cv * math.sqrt(s_p * (1 - s_p)/(n + 4))
    else:
        lat = z_cv * math.sqrt(s_p * (1 - s_p)/n)

    lcl = s_p - lat
    ucl = s_p + lat

    if N:
        if n / N > 0.5 and correction:
            print("Corrected...")
            fpcf = math.sqrt((N - n)/(N - 1))
            lcl = s_p - lat * fpcf
            ucl = s_p + lat * fpcf
        elif correction:
            print("Corrected...")
            fpcf = math.sqrt((N - n)/(N - 1))
            lcl = s_p - lat * fpcf
            ucl = s_p + lat * fpcf
        if lcl < 0:
            lcl = 0
        if ucl < 0:
            ucl = 0
        result = f"""{con_coef * 100:.1f}% Confidence Interval: N [{lcl:.4f}, {ucl:.4f}] = [{N * lcl:.4f}, {N * ucl:.4f}]
p̂: {s_p:.4f}
Sample Size: {n}
z_cv (Critical value): {z_cv:.4f}
    """
    else:
        if lcl < 0:
            lcl = 0
        if ucl < 0:
            ucl = 0
        result = f"""{con_coef * 100:.1f}% Confidence Interval: [{lcl:.4f}, {ucl:.4f}]
p̂: {s_p:.4f}
Sample Size: {n}
z_cv (Critical value): {z_cv:.4f}
    """
    if show:
        print(result)
    return {"lat": lat, "lcl": lcl, "ucl": ucl, "z_cv": z_cv}

def rejection_region_method(s_p, h0_p, nsize, alpha, option='left', precision=4, show=True, ignore=False):
    """
    Input: s_p, h0_p, nsize, alpha, option='left', precision=4, show=True, ignore=False
    Output:
        if opt == 't':
            return p_l, p_u
        else:
            return p_c
    """
    opt = option.lower()[0]
    if not samp.check5(nsize, h0_p):
        print('Not satisfying np_0 > 5 and n(1 - p_0) > 5...')
    if opt == 't':
        option = 'Two-Tail Test'
        z_cv = stats.norm.ppf(1 - alpha/2)
        p_u = h0_p + z_cv * math.sqrt(h0_p * (1 - h0_p)/nsize)
        p_l = h0_p - z_cv * math.sqrt(h0_p * (1 - h0_p)/nsize)
        flag = s_p < p_l or s_p > p_u
        if not ignore:
            result = f'''======= The Rejection Region Method =======
Significant Level (alpha) = {alpha:.{precision}f}
z (Critical value) = {z_cv:.{precision}f}

Using {option}:
p̂ =  {s_p:.{precision}f}
p_l (Lower bound for the critical value) = {p_l:.{precision}f}
p_u (Upper bound for the critical value) = {p_u:.{precision}f}
Reject H_0 → {flag}
            '''
        else:
            result = f'''======= The Rejection Region Method =======
Significant Level (alpha) = {alpha:.{precision}f}
z (Critical value) = {z_cv:.{precision}f}

Using {option}:
p_l (Lower bound for the critical value) = {p_l:.{precision}f}
p_u (Upper bound for the critical value) = {p_u:.{precision}f}
            '''

    else:
        z_cv = stats.norm.ppf(1 - alpha)
        if opt == 'l':
            # left tail
            option = 'One-Tail Test (left tail)'
            p_c = h0_p - z_cv * math.sqrt(h0_p * (1 - h0_p)/nsize)
            flag = s_p < p_c
        elif opt == 'r':
            option = 'One-Tail Test (right tail)'
            p_c = h0_p + z_cv * math.sqrt(h0_p * (1 - h0_p)/nsize)
            flag = s_p > p_c
        if not ignore:
            result = f'''======= The Rejection Region Method =======
Significant Level (alpha) = {alpha:.{precision}f}
z (Critical value) = {z_cv:.{precision}f}

Using {option}:
p̂ =  {s_p:.{precision}f}
p_c (Critical value) = {p_c:.{precision}f}
Reject H_0 → {flag}
            '''
        else:
            result = f'''======= The Rejection Region Method =======
Significant Level (alpha) = {alpha:.{precision}f}
z (Critical value) = {z_cv:.{precision}f}

Using {option}:
p_c (Critical value) = {p_c:.{precision}f}
            '''

    if show:
        print(result)

    if opt == 't':
        return p_l, p_u
    else:
        return p_c

def testing_statistic_method(s_p, h0_p, nsize, alpha, option='left', precision=4, ignore=False):
    """
    Input: s_p, h0_p, nsize, alpha, option='left', precision=4, ignore=False
    Output: z_stats, z_cv
    """
    opt = option.lower()[0]
    z_stats = (s_p - h0_p)/math.sqrt(h0_p * (1 - h0_p)/nsize)
    if not samp.check5(nsize, h0_p):
        print('Not satisfying np_0 > 5 and n(1 - p_0) > 5...')

    if opt == 't':
        z_cv = stats.norm.ppf(1 - alpha / 2)
        option = 'Two-Tail Test'
        flag = z_stats < -z_cv or z_stats > z_cv

        if not ignore:
            result = f'''======= Testing Statistic Method =======
Significant Level (alpha) = {alpha:.{precision}f}
z_cv (Critical value) = {z_cv:.{precision}f}

Using {option}:
z_stats (Observed value) =  {z_stats:.{precision}f}
-z_cv (Lower bound for the critical value) = {-z_cv:.{precision}f}
z_cv (Upper bound for the critical value) = {z_cv:.{precision}f}
Reject H_0 → {flag}
            '''
        else:
            result = f'''======= Testing Statistic Method =======
Significant Level (alpha) = {alpha:.{precision}f}
z_cv (Critical value) = {z_cv:.{precision}f}

Using {option}:
-z_cv (Lower bound for the critical value) = {-z_cv:.{precision}f}
z_cv (Upper bound for the critical value) = {z_cv:.{precision}f}
            '''

    else:
        z_cv = stats.norm.ppf(1 - alpha)
        if opt == 'l':
            # left tail
            option = 'One-Tail Test (left tail)'
            z_cv = -z_cv
            flag = z_stats < z_cv
        elif opt == 'r':
            option = 'One-Tail Test (right tail)'
            flag = z_stats > z_cv

        if not ignore:
            result = f'''======= Testing Statistic Method =======
Significant Level (alpha) = {alpha:.{precision}f}
z_cv (Critical value) = {z_cv:.{precision}f}

Using {option}:
z_stats (Observed value) =  {z_stats:.{precision}f}
z_cv (Critical value) = {z_cv:.{precision}f}
Reject H_0 → {flag}
            '''

        else:
            result = f'''======= Testing Statistic Method =======
Significant Level (alpha) = {alpha:.{precision}f}
z_cv (Critical value) = {z_cv:.{precision}f}

Using {option}:
z_cv (Critical value) = {z_cv:.{precision}f}
            '''

    print(result)

    return z_stats, z_cv

def inter_p_value(p_value):
    # interpretation
    if p_value >= 0 and p_value < 0.01:
        inter_p = 'Overwhelming Evidence'
    elif p_value >= 0.01 and p_value < 0.05:
        inter_p = 'Strong Evidence'
    elif p_value >= 0.05 and p_value < 0.1:
        inter_p = 'Weak Evidence'
    elif p_value >= .1:
        inter_p = 'No Evidence'
    return inter_p

def p_value_method(s_p, h0_p, nsize, alpha, option='left', precision=4):
    """
    Input: s_p, h0_p, nsize, alpha, option='left', precision=4
    Output: z_cv, z_stats, p_value
    """
    opt = option.lower()[0]
    z_stats = (s_p - h0_p)/math.sqrt(h0_p * (1 - h0_p)/nsize)
    if not samp.check5(nsize, h0_p):
        print('Not satisfying np_0 > 5 and n(1 - p_0) > 5...')
    if opt == 't':
        # two-tail test
        option = 'Two-Tail Test'
        if s_p > h0_p:
            p_value = stats.norm.sf(z_stats) * 2
        else:
            p_value = stats.norm.cdf(z_stats) * 2

        z_cv = stats.norm.ppf(1 - alpha/2)
        flag = p_value < alpha
        sub_result = f'''Using {option}:
Difference = {s_p - h0_p}
z_cv (Critical value) = {-z_cv:.{precision}f}, {z_cv:.{precision}f}
z_stats (Observed value) = {z_stats:.{precision}f}
p-value = {p_value:.{precision}f} ({inter_p_value(p_value)})
Reject H_0 → {flag}
        '''
    else:
        if opt == 'l':
            option = 'One-Tail Test (left tail)'
            p_value = stats.norm.cdf(z_stats)
            z_cv = -stats.norm.ppf(1 - alpha)
        elif opt == 'r':
            option = 'One-Tail Test (right tail)'
            p_value = stats.norm.sf(z_stats)
            z_cv = stats.norm.ppf(1 - alpha)
        flag = p_value < alpha
        sub_result = f'''Using {option}:
Difference = {s_p - h0_p}
z_cv (Critical value) = {z_cv:.{precision}f}
z_stats (Observed value) = {z_stats:.{precision}f}
p-value = {p_value:.{precision}f} ({inter_p_value(p_value)})
Reject H_0 → {flag}
        '''

    result = f"""======= p-value Method =======
p̂ = {s_p:.{precision}f}
Number of Observation = {nsize:.{precision}f}
Hypothesized Proportion (H0 Mean) = {h0_p:.{precision}f}
Significant Level (alpha) = {alpha:.{precision}f}

""" + sub_result

    print(result)

    return z_cv, z_stats, p_value


import matplotlib.pyplot as plt
import numpy as np

x = [1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0]
y = [0.99, 2.3, 2.9, 4.01, 4.85, 5.80, 7.2]
size = 7

plt.plot(x, y, 'go')
plt.ylabel('some numbers')

# Algorithm goes here
xsum = 0
x2sum = 0
ysum = 0
xysum = 0
slope = 0
intercept = 0

for i in range(size):
    xsum = xsum + x[i]
    ysum = ysum + y[i]
    x2sum = x2sum + x[i]**2
    xysum = xysum + x[i] * y[i]

slope = (size * xysum - xsum * ysum) / (size * x2sum - xsum * xsum)
intercept = (x2sum * ysum - xsum * xysum) / (x2sum * size - xsum * xsum)
y_fit = []

for i in range(size):
    y_fit.append(slope * x[i] + intercept)

plt.plot(x, y_fit, 'r')

plt.show()

#t = np.arange(0., 5., 0.2)
#plt.plot(t, t, 'r--', t, t**2, 'bs', t, t**3, 'g^')
#plt.show()


import numpy as np

def CorBrownian(mu, E, sampleSize):
    # Algorithm generates samples of increments from a correlated Brownian motion with a given mean and Variance-Covariance matrix (E).
    # The algorithm uses the fact that if you have n independent brownian motions, the samples given by "mu+ C*Z" are distributed as N(mu,E), where mu is the vector of means and C is the square root of the Variance-Covariance matrix.
    # For calculating the square root of the VarCovar matrix, the Cholesky decomposition is implemented.
    # Arguments:
    #     mu        = Array with n elements containing the mean of each BM
    #     E         = n x n numpy array with the Variance-Covariance matrix
    #    sampleSize = integer representing the number of samples
    #
    # Returns:
    #     sampleSize x n numpy array containing sampled increments for the correlated Brownian motion
    #
    # Note:
    #  The algorithm is not optimized for speed and no testing of inputs is implemented. If this would be usefull to you, let us know and we can extend the code.
    #
    # Example of use:
    # import numpy as np
    # mu = [1,2]
    # VarCovar = np.matrix('1,0.8; 0.8,3')
    # sampleSize = 5
    # out = CorBrownian(mu,VarCovar, sampleSize)
    # > [ 2.83211068  4.50021193]
    #   [ 0.26392619  1.56450446]
    #   [-0.25928109  0.97167124]
    #   [ 1.52038489  1.76274556]]
    #

    def Cholesky(X):
        # Cholesky–Banachiewicz algorithm decomposes a Hermitian matrix into a product of a lower triangular matrix and its conjugate transpose.
        # Arguemnts:
        #    X = n x n ndarray representing a Hermitian matrix that the user wants to decompose
        # Returns:
        #    n x n ndarray lower triangular matrix such that the matrix product between it and its conjugate transpose returns X
        #
        # More info on: https://en.wikipedia.org/wiki/Cholesky_decomposition#The_Cholesky.E2.80.93Banachiewicz_and_Cholesky.E2.80.93Crout_algorithms

        L = np.zeros_like(X)
        n = X.shape[0]

        for i in range(0, n):
            for j in range(0, i+1):
                sum = 0
                for k in range(0, j):
                    sum = sum+ L[i,k]*L[j,k]
                if (i==j):
                    L[i,j] = np.sqrt(X[i,i]-sum)
                else:
                    L[i,j] = 1.0/L[j,j] * (X[i,j]-sum)
        return L

    dim = E.shape[0]                                         # Guess the number of Brownian motions (dimension) from the size of the Var-Covar matrix
    Z = np.random.default_rng().normal(0, 1, (sampleSize, dim)) # Generate independent increments of a simpleSize dimensional Brownian motion
    Y = np.zeros((sampleSize, dim))                          # Predefine the final output
    L = Cholesky(E)                                          # Calculate the square root of the Var-Covar matrix

    for iSample in range(sampleSize): # For each sample, calculate mu + L*Z
        Y[iSample] = np.transpose(mu) +  L @ np.transpose(Z[iSample])
    return Y


# PROJECT: Cross-Entropy Script
# PROGRAMMER: Carlos Mertens - Udacity Student
#				Udacity Instructors (Machine Learning Engineer Nanodegree)
# DATE CREATED: (DD/MM/YY) 10/01/2019
# REVISED DATE: (DD/MM/YY)
# PURPOSE: To show the algorithm to implement Cross-Entropy from the scratch.
# USAGE: ...

import numpy as np

# Write a function that takes as input two lists Y, P,
# and returns the float corresponding to their cross-entropy.
def cross_entropy(y, p):
    y = np.float_(y)
    p = np.float_(p)
    result = -np.sum(y * np.log(p) + (1 - y) * np.log(1 - p))
    return result

y = [1,0,1,1]
p = [0.4,0.6,0.1,0.5]

# Call function to calculate the cross-entropy
my_cross_entropy = cross_entropy(y, p)

print(my_cross_entropy)


import numpy as np

def mse(e):
    return 1/2 * np.mean(e**2)

def rmse(e):
    return np.sqrt(2 * mse(e))

def mae(e):
    return np.mean(np.abs(e))

def sigmoid(z):
    z = np.clip(z, -1000, 1000)
    return 1.0 / (1 + np.exp(-z))

def compute_mse(y, tx, w):
    """Compute the loss by MSE (Mean Square Error)."""
    e = y - tx.dot(w)
    return mse(e)

def compute_rmse(y, tx, w):
    """Compute the loss by RMSE (Root Mean Square Error)."""
    e = y - tx.dot(w)
    return rmse(e)

def compute_mae(y, tx, w):
    """Compute the loss by MAE (Mean Absolute Error)."""
    e = y - tx.dot(w)
    return mae(e)

def compute_log_likelihood_error(y, tx, w):
    """Compute the loss of the log-likelihood cost function."""
    tx_dot_w = tx.dot(w)
    return np.sum(np.log(1 + np.exp(tx_dot_w))) - y.dot(tx_dot_w)


import numpy as np

def gaussian_elimination_with_pivot(m):

    # forward elimination
    n = len(m)
    print(n)
    for k in range(n):
        max = -1e100
        for r in range(k, n):
            if max < abs(m[r][k]):
                max_row = r
                max = abs(m[r][k])
        m[k], m[max_row] = m[max_row], m[k]
        print(m)
        for i in range(k+ 1, n):
            m[i] = [m[i][j] - m[k][j] * m[i][k] / m[k][k] for j in range(n + 1)]

    if m[n - 1][n - 1] == 0: raise ValueError('No unique solution')

    print(m)

    # backward substitution
    x = [0] * n
    for i in range(n - 1, -1, -1):
        s = sum(m[i][j] * x[j] for j in range(i, n))
        x[i] = (m[i][n] - s) / m[i][i]
    return x

'''
# shorter way to pivot but cannot run in trinket
def pivot(m, n, i):
  max_row = max(range(i, n), key=lambda r: abs(m[r][i]))
  m[i], m[max_row] = m[max_row], m[i]
'''

def pivot(m, n, i):
    max = -1e100
    for r in range(i, n):
        if max < abs(m[r][i]):
            max_row = r
            max = abs(m[r][i])
    m[i], m[max_row] = m[max_row], m[i]

if __name__ == '__main__':
    # m = [[0,-2,6,-10], [-1,3,-6,5], [4,-12,8,12]]
    # m = [[1,-1,3,2], [3,-3,1,-1], [1,1,0,3]]
    m = [[0.5, 1.1, 3.1, 6], [5, 0.96, 6.5, 0.96], [2, 4.5, 0.36, 0.02]]
    print(gaussian_elimination_with_pivot(m))
    print(len(m))

    """
    m = [[4,4,0,400], [-1,4,2,400], [0,-2,4,400]]   # aj Montri p80  [50, 50, 125]
    print(gaussian_elimination_with_pivot(m))
    """


#Problem sheet 1

print 'Problem Sheet 1'

from gaussElimin import *
import numpy as np
from scipy import linalg

print 'Q1a'
#4v1-v2-v3-v4=v+
#-v1+3v2+0v3-v4=0
#-v1+0v2+3v3-v4=v+
#-v1-v2-v3+4v4=0
print '4v1-v2-v3-v4=v+'
print '-v1+3v2+0v3-v4=0'
print '-v1+0v2+3v3-v4=v+'
print '-v1-v2-v3+4v4=0'

print 'Q1b'
x=np.array([[4.0,-1.0,-1.0,-1.0],[-1.0,3.0,0.0,-1.0],[-1.0,0.0,3.0,-1.0],[-1.0,-1.0,-1.0,4.0]], dtype = float)
print x
y=np.array([[5.0],[0.0],[5.0],[0.0]], dtype = float) #creates a matrix
print y

z=gaussElimin(x,y)
print z

print 'Q1c - Using LuDecomposition of our matrix'

from LUdecomp import *

P,L,U=linalg.lu(x) #in this case P is just an identitiy matrix because its just 1

print 'The original matrix'
print (x)
print ''
print 'The L matrix'
print (L)
print ''
print 'The U matrix'
print (U)
print ''

print 'Q1D'

x1=np.array([[5.0],[0.0],[5.0],[0.0]]) #vector b when vo is 0
x2=np.array([[5.0],[1.0],[5.0],[1.0]]) #vector b when v0 is now 1

print 'original vector b when v0=0'
print x1
print "Below are the vectors b where instead of V=0 V0=1:"
print x2

x=np.array([[4.0,-1.0,-1.0,-1.0],[-1.0,3.0,0.0,-1.0],[-1.0,0.0,3.0,-1.0],[-1.0,-1.0,-1.0,4.0]])

P,L,U=linalg.lu(x)
k=gaussElimin(L,x2)#solves Lz=b where b is new vector and L is from part c, which redefines b when you use gaussElimin solving as Ux=z it puts in your new b in this case x2 and spits out your original answer
J=gaussElimin(U,k)
M=gaussElimin(L,x1)
N=gaussElimin(U,M)
print 'when v0=1'
print J #Solutions match
print 'When v0=0'
print M #Solutions match from before with part b


import numpy as np

# ステップ関数の実装
def step_function(x):
    return np.array(x > 0, dtype=np.int32)

#シグモイド関数の実装
def sigmoid(x):
    return 1 / (1 + np.exp(-x))

def sigmoid_grad(x):
    return (1.0 - sigmoid(x)) * sigmoid(x)

#ReLU関数の実装
def relu(x):
    return np.maximum(0, x)

#恒等関数(あくまでも例なので値を戻しているだけ)
def identity_function(x):
    return x

#この書き方ではオーバーフローの問題が出てくるので、このままでは利用できない
#def softmax(a):
#    exp_a = np.exp(a) #指数関数
#    sum_exp_a = np.sum(exp_a) #指数関数の和
#    y = exp_a / sum_exp_a
#
#    return y

#ソフトマックス関数
#※ソフトマックス関数はニューラルネットワークの学習時に使用するものであり、
#  推論(分類)時には省略するのが一般的
def softmax(x):
    #c = np.max(a)
    #exp_a = np.exp(a - c) #オーバーフロー対策
    #sum_exp_a = np.sum(exp_a)
    #y = exp_a / sum_exp_a
    #return y
    x = x - np.max(x, axis=-1, keepdims=True)   # オーバーフロー対策
    return np.exp(x) / np.sum(np.exp(x), axis=-1, keepdims=True)

#2乗和誤差
def sum_squareed_error(y, t):
    return 0.5 * np.sum((y - t)**2)

#交差エントロピー誤差(バッチ対応版)
def cross_entropy_error(y, t):
    if y.ndim == 1:
        t = t.reshape(1, t.size)
        y = y.reshape(1, y.size)

    # 教師データがone-hot-vectorの場合、正解ラベルのインデックスに変換
    if t.size == y.size:
        t = t.argmax(axis=1)

    batch_size = y.shape[0]
    return -np.sum(np.log(y[np.arange(batch_size), t] + 1e-7)) / batch_size

#数値微分
def numerical_diff(f, x):
    h = 1e-4 #0.0001
    return (f(x + h) - f(x - h)) / (2 * h)

#2次関数(簡単な微分)
def function_1(x):
    return 0.01*x**2 + 0.1*x

#偏微分
def function_2(x):
    #return x[0]**2 + x[1]**2
    #または return np.sum(x**2)
    if x.ndim == 1:
        return np.sum(x**2)
    else:
        return np.sum(x**2, axis=1)

#勾配
def _numerical_gradient_no_batch(f, x):
    h = 1e-4 #0.0001
    grad = np.zeros_like(x) #Xと同じ形状の配列を生成

    for idx in range(x.size):
        tmp_val = x[idx]
        # f(x + h)の計算
        x[idx] = tmp_val + h
        fxh1 = f(x)

        # f(x - h)の計算
        x[idx] = tmp_val - h
        fxh2 = f(x)

        grad[idx] = (fxh1 - fxh2) / (2 * h)
        x[idx] = tmp_val #値を元に戻す

    return grad

def numerical_gradient(f, x):
    #if X.ndim == 1:
    #    return _numerical_gradient_no_batch(f, X)
    #else:
    #    grad = np.zeros_like(X)
    #
    #    for idx, x in enumerate(X):
    #        grad[idx] = _numerical_gradient_no_batch(f, x)
    #
    #    return grad
    h = 1e-4 # 0.0001
    grad = np.zeros_like(x)

    it = np.nditer(x, flags=['multi_index'], op_flags=['readwrite'])
    while not it.finished:
        idx = it.multi_index
        tmp_val = x[idx]
        x[idx] = tmp_val + h
        fxh1 = f(x) # f(x+h)

        x[idx] = tmp_val - h
        fxh2 = f(x) # f(x-h)
        grad[idx] = (fxh1 - fxh2) / (2*h)

        x[idx] = tmp_val # 値を元に戻す
        it.iternext()

    return grad

#勾配降下法
# f : 最適化したい関数
# init_x : 初期値
# lr : learning rate
# step_num : 勾配法による繰り返しの数
def gradient_descent(f, init_x, lr=0.01, step_num=100):
    x = init_x
    x_history = []

    for i in range(step_num):
        x_history.append(x.copy())

        grad = numerical_gradient(f, x)
        x -= lr * grad

    return x, np.array(x_history)


#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Time     : 2018/10/31 17:03
# @Author   : Iydon
# @File     : 5.9.py

import numpy as np

def linspace(start, end, step=None):
    """
    Args:
        step: start:step:end
    """
    length = (end-start)/step
    return [start+i*step for i in range(int(length[0,0])+1)]

def runge_kutta(dy, dy0, start, end, step, order:int=2):
    """
    Runge-Kutta Methods of Order Two.
    """
    if order not in [1,2,3,4]:
        raise ValueError
    xs = linspace(start, end, step=step)
    ys = [dy0 for i in xs]
    for i in range(len(xs)-1):
        k1 = step * dy(xs[i],ys[i])
        if order<2:
            ys[i+1] = ys[i] + k1
        elif order<3:
            k2 = step * dy(xs[i]+step/2,ys[i]+k1/2)
            ys[i+1] = ys[i] + k2
        elif order<4:
            k2 = step * dy(xs[i]+step/2,ys[i]+k1/2)
            k3 = step * dy(xs[i]+step,ys[i]-k1+2*k2)
            ys[i+1] = ys[i] + (k1+4*k2+k3)/6
        elif order<5:
            k2 = step * dy(xs[i]+step/2,ys[i]+k1/2)
            k3 = step * dy(xs[i]+step/2,ys[i]+k2/2)
            k4 = step * dy(xs[i]+step,ys[i]+k3)
            ys[i+1] = ys[i] + (k1+2*k2+2*k3+k4)/6
    return ys

t1,t2 = lambda t: -(2*t*t+1)*np.exp(2*t), lambda t: (t*t+2*t-4)*np.exp(2*t)
dy    = lambda t,y: np.matrix([[t1(t[0,0])],[t2(t[1,0])]]) + np.matrix([[3,2],[4,1]])*y
dy0   = np.matrix([[1],[1]])
start = np.matrix([[0],[0]])
end   = np.matrix([[1],[1]])
step  = 0.2
result = runge_kutta(dy, dy0, start, end, step, order=4)
print(result)
u1 = lambda t: np.exp(5*t)/3-np.exp(-t)/3+np.exp(2*t)
u2 = lambda t: np.exp(5*t)/3+2*np.exp(-t)/3+t*t*np.exp(2*t)
t  = [0, 0.2, 0.4, 0.6, 0.8, 1]
print([u1(i) for i in t])
print([u2(i) for i in t])

t1,t2 = lambda t: np.cos(t)+4*np.sin(t), lambda t: -3*np.sin(t)
dy    = lambda t,y: np.matrix([[t1(t[0,0])],[t2(t[1,0])]]) + np.matrix([[-4,-2],[3,1]])*y
dy0   = np.matrix([[0],[-1]])
start = np.matrix([[0],[0]])
end   = np.matrix([[2],[2]])
step  = 0.1
result = runge_kutta(dy, dy0, start, end, step, order=4)
print(result)
u1 = lambda t: 2*np.exp(-t)-2*np.exp(-2*t)+np.sin(t)
u2 = lambda t: -3*np.exp(-t)+2*np.exp(-2*t)
t  = [i/10 for i in range(21)]
print([u1(i) for i in t])
print([u2(i) for i in t])


import numpy as np

# Helper function to calculate the sigmoid
def sigmoid(z):
    sig = 1 / (1 + np.exp(-z))

    return sig

# Helper function to initialize weights and bias with zero
def initialize_parameters(dimension):
    weights = np.zeros((dimension, 1))
    bias = 0

    return weights, bias

# Helper function to do a single forward pass
def forward_propagate(X, Y, weights, bias):
    # Get the number of training examples
    m = X.shape[1]

    # Calculate the z
    z = np.dot(weights.T, X) + bias

    # Calculate activation A using sigmoid activation
    A = sigmoid(z)

    # Calculate for the cost
    cost = (-1/m) * np.sum(Y * np.log(A) + (1 - Y) * np.log(1 - A))

    cost = np.squeeze(cost)

    return A, cost

# Helper function to do a single backward pass
def backward_propagate(X, Y, A):
    # Calculate for the derivatives
    m = X.shape[1]
    dz = A - Y
    dw = (1/m) * np.dot(X, dz.T)
    db = (1/m) * np.sum(dz)

    return dw, db

# Helper function to optimize weights and bias using gradient descent algorithm
def gradient_descent(X, Y, weights, bias, iterations, learning_rate, print_cost=False):
    costs = []

    for i in range(iterations):
        # Compute for cost and gradient using propagate
        A, cost = forward_propagate(X, Y, weights, bias)
        dw, db = backward_propagate(X, Y, A)

        # Update weights and bias parameters
        weights = weights - learning_rate * dw
        bias = bias - learning_rate * db

        # Record cost
        if i % 100 == 0:
            costs.append(cost)

        # Print the cost every 100 iterations
        if print_cost and i % 100 == 0:
            print ("Cost after iteration %i: %f" %(i, cost))

    return weights, bias, dw, db, costs

def predict(X, weights, bias):
    m = X.shape[1]
    Y_pred = np.zeros((1, m))

    A = sigmoid(np.dot(weights.T, X) + bias)

    for i in range(A.shape[1]):
        if A[0,i] <= 0.5:
            Y_pred[0,i] = 0
        else:
            Y_pred[0,i] = 1

    return Y_pred


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Fri Jul  9 08:04:05 2021

@author: alessandro
"""

import numpy as np
import sympy as sym
from sympy.utilities.lambdify import lambdify
import matplotlib.pyplot as plt
from scipy.optimize import fsolve

x = sym.symbols('x')
fx = x - 2 * sym.sqrt(x - 1)
dfx = sym.diff(fx, x, 1)

f = lambdify(x, fx, np)
df = lambdify(x, dfx, np)

a = 1
b = 3
zero = 2

xx = np.linspace(a, b, 100)
y = f(xx)
d = df(xx)
plt.ylim((-1, 1))
plt.plot(xx, y)
plt.plot(xx, d)
plt.plot([zero, ], f(np.array([zero])), 'o')
plt.plot([zero, ], df(np.array([zero])), 'o')
plt.plot(xx, 0*xx)
plt.show()

# Serve newton di grado due poichè la derivata prima vale zero in alpha
# mentre la derivata seconda non si annulla

def newton(f, df, m, x0, tolx, tolf, nmax=2048):
    def delta(x): return f(x) / df(x) if abs(df(x)) > np.spacing(1) else print("Derivata nulla")
    def prossimax(x): return x - m * delta(x)

    x = prossimax(x0)
    fx = f(x)
    it, xk = 1, [x]
    while it < nmax and abs(fx) > tolf and abs(delta(x)) > tolx * abs(x):
        x = prossimax(x)
        xk.append(x)
        fx = f(x)
        it += 1

    return x, it, xk

alpha, it, xk = newton(f, df, 2, 3, 1e-12, 1e-12)
print(alpha)

# devo stabilire l'ordine di convergenza

def stimaordine(xk):
    k = len(xk) - 4
    n = np.log(np.abs(xk[k + 3] - xk[k + 2]) / np.abs(xk[k + 2] - xk[k + 1]))
    d = np.log(np.abs(xk[k + 2] - xk[k + 1]) / np.abs(xk[k + 1] - xk[k]))
    return n / d

ordine = stimaordine(xk)
print(f"L'ordine è {ordine}")

# stampo le iterate
plt.title("Iterazioni")
plt.semilogy(range(1, it + 1), xk, 'o')
plt.semilogy(range(1, it + 1), xk)

# provo a partire da x0 = 1
a = newton(f, df, 2, 1, 1e-12, 1e-12)
print(a)

import numpy,from scipy,import sympy,from sympy


r"""

Newton's method is a rather popular iterative root finding algorithm. Starting
at an initial guess :math:`x_0` it tries to find better and better
approximations of the root of a function :math:`f(x)`. For this it uses the
first derivative :math:`f'(x)` of the function. The process

.. math::

    x_{n+1} = x_n - \frac{f(x_n)}{f'(x_n)}

is repeated until a value :math:`f(x_n)` is reached that is within a predefined
tolerance to zero. For further information see the `Wikipedia`_ page.

The way it is supposed to work is as follows:

>>> def f(x):
...     return x**2 - 2
...
>>> def df_dx(x):
...     return 2*x
...
>>> newtons_method_1d(f, df_dx, x0=4, tol=1e-8)  # doctest:+SKIP
1.4142135623730951

Hence, implement a function following the given definition:

.. autofunction:: newtons_method_1d

Start by downloading the
:exercise:`exercise template </exercises/newtons_method_1d.py>` and
editing this file. You can run tests via

.. code-block:: console

    $ python newtons_method_1d.py test

to check whether you got a correct solution. You can also take a look at
:solution:`one possible solution </exercises/newtons_method_1d.py>`.

.. _Wikipedia: https://en.wikipedia.org/wiki/Newton's_method

"""

def newtons_method_1d(f, df_dx, x0, tol):
    """Return the root of `f` within `tol` by using Newton's method.

    Parameters
    ----------
    f : callable[[float], float]
        The function of which the root should be found.
    df_dx : callable[[float], float]
        The derivative of `f`.
    x_0 : float
        The initial guess for the algorithm.
    tol : float
        The tolerance of the method.

    Returns
    -------
    float
        The root of `f` within a tolerance of `tol`.

    """
    # begin solution
    x = x0
    while abs(f(x)) > tol:
        x -= f(x) / df_dx(x)
    return x
    # end solution

# -----------------------------------------------------------------------------
# In the following the tests for this exercise are given---do not modify them.
import sys
import pytest
from numpy.testing import assert_allclose

@pytest.mark.parametrize(
    ('f', 'df_dx', 'x0', 'tol', 'expected'),
    [
        (lambda x: x**2 - 2, lambda x: 2*x, 4, 1e-8, 2**(1/2)),
        (lambda x: x**2 - 2, lambda x: 2*x, -4, 1e-8, -2**(1/2)),
        (lambda x: x**3 - 5, lambda x: 3*x**2, 4, 1e-8, 5**(1/3))
    ])
def test_correct_result(f, df_dx, x0, tol, expected):
    assert_allclose(
        newtons_method_1d(f, df_dx, x0, tol),
        expected,
        rtol=0,
        atol=tol)

if __name__ == '__main__':
    if len(sys.argv) == 2 and sys.argv[-1] == 'test':
        pytest.main([__file__])


"""
 sum

The sum tool returns the sum of array elements over a given axis.

import numpy

my_array = numpy.array([ [1, 2], [3, 4] ])

print numpy.sum(my_array, axis = 0)         #Output : [4 6]
print numpy.sum(my_array, axis = 1)         #Output : [3 7]
print numpy.sum(my_array, axis = None)      #Output : 10
print numpy.sum(my_array)                   #Output : 10

By default, the axis value is None. Therefore, it performs a sum over all the dimensions of the input array.

prod

The prod tool returns the product of array elements over a given axis.

import numpy

my_array = numpy.array([ [1, 2], [3, 4] ])

print numpy.prod(my_array, axis = 0)            #Output : [3 8]
print numpy.prod(my_array, axis = 1)            #Output : [ 2 12]
print numpy.prod(my_array, axis = None)         #Output : 24
print numpy.prod(my_array)                      #Output : 24

By default, the axis value is None. Therefore, it performs the product over all the dimensions of the input array.

Task

You are given a 2-D array with dimensions
X.
Your task is to perform the tool over axis and then find the

of that result.

Input Format

The first line of input contains space separated values of
and .
The next lines contains

space separated integers.

Output Format

Compute the sum along axis

. Then, print the product of that sum.

Sample Input

2 2
1 2
3 4

Sample Output

24

Explanation

The sum along axis
= [ ]
The product of this sum =
"""

import numpy

N, M = map(int, input().split())
A = numpy.array([input().split() for _ in range(N)],int)
print(numpy.prod(numpy.sum(A, axis=0), axis=0))


import numpy as np

def mean_square_error(y_true: np.ndarray, y_pred: np.ndarray) -> float:
    """
    Calculate MSE loss

    Parameters
    ----------
    y_true: ndarray of shape (n_samples, )
        True response values
    y_pred: ndarray of shape (n_samples, )
        Predicted response values

    Returns
    -------
    MSE of given predictions
    """
    squared_error = np.square(y_true-y_pred)
    mean2 = np.sum(squared_error)/y_true.shape[0]
    return mean2

def misclassification_error(y_true: np.ndarray, y_pred: np.ndarray, normalize: bool = True) -> float:
    """
    Calculate misclassification loss

    Parameters
    ----------
    y_true: ndarray of shape (n_samples, )
        True response values
    y_pred: ndarray of shape (n_samples, )
        Predicted response values
    normalize: bool, default = True
        Normalize by number of samples or not

    Returns
    -------
    Misclassification of given predictions
    """
    t = y_pred * y_true
    error_num = np.count_nonzero(t<0)
    if normalize:
      return float(error_num/y_true.shape[0])
    else:
        return error_num

def accuracy(y_true: np.ndarray, y_pred: np.ndarray) -> float:
    """
    Calculate accuracy of given predictions

    Parameters
    ----------
    y_true: ndarray of shape (n_samples, )
        True response values
    y_pred: ndarray of shape (n_samples, )
        Predicted response values

    Returns
    -------
    Accuracy of given predictions
    """

    true_predictions = np.count_nonzero(y_pred == y_true)
    predictions = y_pred.shape[0]
    return float(true_predictions)/predictions

def cross_entropy(y_true: np.ndarray, y_pred: np.ndarray) -> float:
    """
    Calculate the cross entropy of given predictions

    Parameters
    ----------
    y_true: ndarray of shape (n_samples, )
        True response values
    y_pred: ndarray of shape (n_samples, )
        Predicted response values

    Returns
    -------
    Cross entropy of given predictions
    """
    raise NotImplementedError()


import numpy as np
import matplotlib.pyplot as plt

def pca(x, varRetained = 0.95):
    ''' varRetained is the original data variance retained in the new dataset. New number of dimensions
     is computed based on the desired fraction of the original variance.
    '''
    n = x.shape[0]
    d = x.shape[1]
    sigma = 1.0/d*np.dot(x.T,x)
    U, S, V = np.linalg.svd(sigma, full_matrices=True)
    k = 0
    total_var = np.sum(S)
    var_cum_sums = np.array([np.sum(S[:i+1])/total_var*100 for i in range(d)])
    k = len(var_cum_sums[var_cum_sums<(varRetained*100)])
    U_reduced = U[:, : k]
    return np.dot(x, U_reduced), k

class FLDA:
    def fit(self, x, y):
        n = x.shape[0]
        x = np.concatenate([x, np.ones([n,1])], axis = 1)
        classes = set(y)
        n0= n1= 0
        for i in y:
            if(y==0):
                M0 += x[i]
                n0 += 1
            else:
                M1 += x[i]
                n1 += 1
        M0 = M0/n0
        M1 = M1/n1
        Sw = []
        for i in y:
            if(y==0):
                Sw += np.dot((x[i] - M0), (x[i] - M0).T)
            else:
                Sw += np.dot((x[i] - M1), (x[i] - M1).T)
        w = np.dot(np.linalg.inv(Sw), (M0- M1))
        return w
    def predict(self, x, w):
        h = np.dot(w, x)
        if(h>0):
            return 0
        else:
            return 1


# Author: Dimitrios Damopoulos
# MIT license (see LICENCE.txt in the top-level folder)

# TODO: documentation

import numpy as np

def sigmoid(z):
    """
    Computes the sigmoid function

    Args:
        z (np.ndarray): the values where the sigmoid should be calculated on

    Returns:
        An np.ndarray with the same shape as the input `z' with the values of
        sigmoid
    """
    sigm = 1. / (1. + np.exp(-z))
    return sigm

def cross_entropy(p_hat, p):
    """
    Computer the cross entropy between two distributions

    Args:
        p_hat (np.ndarray of shape m,): The "predicted" probability distribution
        p (np.ndarray of shape m,): The "true" probability distribution

    Returns:
        The cross-entropy between the two distributions
    """
    # get values which are very close to zero a bit higher, in order to avoid
    # NaNs in the computation of the logarithm later.
    eps = np.finfo(float).eps
    p_hat[p_hat < eps] = eps

    ce = - np.log(p_hat).T.dot(p)
    return ce

def cross_entropy_with_one_hot_vector(p_hat, p):
    """
    Same as math_utils.cross-entropy, but for a "true" probability distribution
    has arose for a binary random variable (i.e., a process that outputs True
    or False for every sample. It should output the same as
    math_utils.cross_entropy, but faster.

    Args:
        p_hat (np.ndarray of shape m,): The "predicted" probability distribution
        p (np.ndarray of shape m,): The "true" probability distribution. Its
            values are converted to binary before the computation of the
            cross-entropy, by just comparing them with 0: anything above zero is
            treated as have value "1" and the rest as having value "0"

    Returns:
        The cross-entropy between the two distributions
    """
    eps = np.finfo(float).eps
    p1 = p > 0
    p0 = np.logical_not(p1)
    part1 = np.log(p_hat[p1] + eps).sum()
    part0 = np.log(1. - p_hat[p0] + eps).sum()

    return -(part0 + part1)

def rmse(y_hat, y):
    """
    Returns the Root Mean Square error between two series of measurements
    """
    losses = y_hat - y
    se = losses.T.dot(losses)
    return np.sqrt(se / y.shape[0])

def gradient_of_rmse(y_hat, y, Xn):
    """
    Returns the gradient of the Root Mean Square error with respect to the
    parameters of the linear model that generated the prediction `y_hat'.
    Hence, y_hat should have been generated by a linear process of the form
    Xn.T.dot(theta)

    Args:

       y_hat (np.array of shape N,): The predictions of the linear model
       y (np.array of shape N,): The "ground-truth" values.

    Returns:
        The RMSE between y_hat and y
    """

    N = y.shape[0]
    assert N > 0, ('At least one sample is required in order to compute the '
                  'RMSE loss')

    losses = y - y_hat
    gradient = - 2 * Xn.T.dot(losses) / N

    return gradient


import numpy as np
import matplotlib.pyplot as plt

def f(x):
    """"Returns the function where you're looking for zero spots"""
    return np.sin(x)**2 + np.sin((x+1)**2)

class FindZeroSpot:
    def __init__(self, func, x1=0.8, x2=1.2, precision=1e-3):
        """
        :param func: defined function to examine
        :param x1: the beginning of the area under investigation
        :param x2: end of the area under investigation
        :param precision: precision of searching for zero place
        """
        N = 300
        self.x_min = -5
        self.x_max = 5
        self.precision = precision
        self.function = func
        self.interval_list = [[x1, x2, 0]]

        self.x_points = np.linspace(self.x_min, self.x_max, N)
        self.y_points = self.function(self.x_points)

    def steps(self, act_interval):
        f1 = self.function(act_interval[0])
        f2 = self.function(act_interval[1])
        x0 = (act_interval[0] * f2 - act_interval[1] * f1) / (f2 - f1)
        f0 = self.function(x0)

        if f0 * f1 > 0:
            return [x0, act_interval[1], x0]
        else:
            return [act_interval[0], x0, x0]

    def count(self):
        """
        counts the coordinates of the zero place
        :return:x coordinate, y coordinate
        """

        def new_interval(interval_list):
            return interval_list.append(self.steps(interval_list[-1]))

        def check_precision_condition(interval_list, precision):
            new_interval(interval_list)
            if abs(interval_list[-1][2] - interval_list[-2][2]) < precision:
                return False
            return True

        while check_precision_condition(self.interval_list, self.precision):
            pass

        x_result = self.interval_list[-1][2]
        y_result = self.function(x_result)
        print(self.interval_list)
        return x_result, y_result

    def figure(self, x_result, y_result):
        """
        draws a graph of functions, interval, point

        :param x_result: x coordinate of the result
        :param y_result: y coordinate of the result
        :return: null
        """
        fig = plt.figure()
        ax = fig.add_subplot(111)
        ax.plot(self.x_points, self.y_points, "-b")
        ax.plot([self.x_points[0], self.x_points[-1]], [0, 0], '-k')
        ax.plot([self.interval_list[0][0], self.interval_list[0][0]], [-0.5, 0.5], '-y')
        ax.plot([self.interval_list[0][1], self.interval_list[0][1]], [-0.5, 0.5], '-y')
        ax.plot(x_result, y_result, 'or')
        plt.show()

example = FindZeroSpot(f)
x, y = example.count()
print("coordinates of it \nx:{:5.2f} y:{:5.2f}".format(x, y))
example.figure(x, y)


#%% ================ Introduction: load packages ================
import numpy as np
import matplotlib.pyplot as plt
import pandas as pd

from featureNormalize import featureNormalize
from gradientDescentMulti import gradientDescentMulti
from normalEqn import normalEqn

#%% ================ Part 1: Feature Normalization ================

print('\n -------------------------- \n')
print('Loading data ...')

# Load Data
path = 'ex1data2.txt'
data = pd.read_csv(path, header=None, names=['HouseSize', 'NbOfBedrooms', 'Price'])
data.head()

# Résumé des données
data.describe()

# set X (training data) and y (target variable)
nbCol = data.shape[1]
X = data.iloc[:,0:nbCol-1]
y = data.iloc[:,nbCol-1:nbCol]

# convert from data frames to numpy arrays
X = np.array(X.values)
y = np.array(y.values)
m = X.shape[0]

# Print out some data points
print('\n -------------------------- \n')
print('First 10 examples from the dataset:')
print(np.column_stack( (X[:10], y[:10]) ))

# Scale features and set them to zero mean
print('\n -------------------------- \n')
print('Normalizing Features ...')

X, mu, sigma = featureNormalize(X)
print('[mu] [sigma]')
print(mu, sigma)

# Add intercept term to X
X = np.concatenate((np.ones((m, 1)), X), axis=1)

#%% ================ Part 2: Gradient Descent ================
#
# ====================== YOUR CODE HERE ======================
# Instructions: We have provided you with the following starter
#               code that runs gradient descent with a particular
#               learning rate (alpha).
#
#               Your task is to first make sure that your functions -
#               computeCost and gradientDescent already work with
#               this starter code and support multiple variables.
#
#               After that, try running gradient descent with
#               different values of alpha and see which one gives
#               you the best result.
#
#               Finally, you should complete the code at the end
#               to predict the price of a 1650 sq-ft, 3 br house.
#
#
# Hint: At prediction, make sure you do the same feature normalization.
#

print('\n -------------------------- \n')
print('Running gradient descent ...')

# Choose some alpha value
alpha = 0.01
num_iters = 400

# Init Theta and Run Gradient Descent
n = X.shape[1]
theta = np.zeros((n,1))
theta, cost_history, theta_history = gradientDescentMulti(X, y, theta, alpha, num_iters)

# Plot the convergence graph
fig = plt.figure()
ax = plt.gca()
ax.plot(np.arange(num_iters), cost_history, color="blue", linewidth=2.0, linestyle="-")
ax.grid()
ax.set_xlabel('iteration number')
ax.set_ylabel(r'Cost J($\theta$)')
ax.set_title('Error vs. Training Epoch (number of iters)')
fig.show()
plt.show()

# Display gradient descent's result
print('\n -------------------------- \n')
print('Theta computed from gradient descent: ')
print(theta)

# Estimate the price of a 1650 sq-ft, 3 br house
price = np.array([[1, 1650, 3 ]]).dot(theta)

print('\n -------------------------- \n')
print('Predicted price of a 1650 sq-ft, 3 br house')
print("(using gradient descent):\n $%f\n" % price)

#%% ================ Part 3: Normal Equations ================

# ====================== YOUR CODE HERE ======================
# Instructions: The following code computes the closed form
#               solution for linear regression using the normal
#               equations. You should complete the code in
#               normalEqn.py
#
#               After doing so, you should complete this code
#               to predict the price of a 1650 sq-ft, 3 br house.
#
print('\n -------------------------- \n')
print('Solving with normal equations...')

# Load Data
data = np.loadtxt('ex1data2.txt', delimiter=',')
path = 'ex1data2.txt'
data = pd.read_csv(path, header=None, names=['HouseSize', 'NbOfBedrooms', 'Price'])
data.head()

# set X (training data) and y (target variable)
nbCol = data.shape[1]
X = data.iloc[:,0:nbCol-1]
y = data.iloc[:,nbCol-1:nbCol]

# convert from data frames to numpy arrays
X = np.array(X.values)
y = np.array(y.values)
m = X.shape[0]

# Add intercept term to X
X = np.concatenate((np.ones((m, 1)), X), axis=1)

# Calculate the parameters from the normal equation
theta = normalEqn(X, y)

# Display normal equation's result
print('Theta computed from the normal equations:')
print(' %s \n' % theta)

# Estimate the price of a 1650 sq-ft, 3 br house
price = np.array([[1, 1650, 3 ]]).dot(theta)

print("Predicted price of a 1650 sq-ft, 3 br house ")
print('(using normal equations):\n $%f\n' % price)

# ============================================================


# Computational Solutions
## Introduction to Linear Programming

Computational environments provide organizations with the ability to implement solutions to complex time, in real time.

[Scikit Learn, Pulp, CPLEX, and Gurobi](https://medium.com/opex-analytics/optimization-modeling-in-python-pulp-gurobi-and-cplex-83a62129807a) are  Python packages which provide capabilities for Linear programming and optimization.

!pip install pulp

#Import some required packages.
import numpy as np
import matplotlib.pyplot as plt
%matplotlib inline

## Product mix problem - Farmers Fields

Problem: How much of each brand to purchase to minimize total cost of fertilizer given following data?

Product resource requirements and unit profit:

Two brands of fertilizer available – Super-gro, Crop-quick.

Field requires at least 16 pounds of nitrogen and 24 pounds of phosphate.

Super-gro costs: `$6 per bag`

Crop-quick: `$3 per bag`

Decision Variables:

$x_{1}$ = number of bags of Super-gro

$x_{2}$ = number of bags of Crop-quick

Cost (Z) minimization

Z = 6$x_{1}$ + 3$x_{2}$

Nitrogen Constraint

2$x_{1}$ + 4$x_{2}$ >= 16

Phosphate Constraint Check

4$x_{1}$ + 3$x_{2}$ >= 24

Non-negativitiy Constraint

$x_{1}$ > 0

$x_{2}$ > 0

#Initialize the model as a minimization problem.
import pulp as pl
opt_model = pl.LpProblem("MIPModel", pl.LpMinimize)

#Set the variables. Notice this is where we put
# the "non-negativity" constraint
x1 =pl.LpVariable(cat=pl.LpInteger, lowBound=0, name="$x_{1}$")
x2 =pl.LpVariable(cat=pl.LpInteger, lowBound=0, name="$x_{2}$")

#Set the objective function
opt_model += 6 * x1 + 3 * x2

#Set the Constraints
opt_model += 2 * x1  + 4* x2  >= 16

opt_model += 4 * x1 + 3 * x2 >= 24

## Review Model

Now that we have created the model we can review it.

opt_model

## Markdown of output
If we copy the above text into a markdown cell you will see the implications of the varous models.

MIPModel:

MINIMIZE

6*$x_{1}$ + 3*$x_{2}$ + 0

SUBJECT TO

_C1: 2 $x_{1}$ + 4 $x_{2}$ >= 16

_C2: 4 $x_{1}$ + 3 $x_{2}$ >= 24

VARIABLES

0 <= $x_{1}$ Integer

0 <= $x_{2}$ Integer

## Solve

We now solve the system of equations with the solve command.

#Solve the program
opt_model.solve()

## Check the Status

Here are 5 status codes:
* **Not Solved**: Status prior to solving the problem.
* **Optimal**: An optimal solution has been found.
* **Infeasible**: There are no feasible solutions (e.g. if you set the constraints x <= 1 and x >=2).
* **Unbounded**: The constraints are not bounded, maximising the solution will tend towards infinity (e.g. if the only constraint was x >= 3).
* **Undefined**: The optimal solution may exist but may not have been found.

pl.LpStatus[opt_model.status]

for variable in opt_model.variables():
    print(variable.name," = ", variable.varValue)

## Hurray!
We got the same answer as we did before.

## Exercise

Solve the LP problem for Beaver Creek Pottery using the maximization model type (`pl.LpMaximize`).

### Product mix problem - Beaver Creek Pottery Company
How many bowls and mugs should be produced to maximize profits given labor and materials constraints?

Product resource requirements and unit profit:

Decision Variables:

$x_{1}$ = number of bowls to produce per day

$x_{2}$ = number of mugs to produce per day

Profit (Z)  Mazimization

Z = 40$x_{1}$ + 50$x_{2}$

Labor Constraint Check

1$x_{1}$ + 2$x_{2}$ <= 40

Clay (Physicial Resource) Constraint Check

4$x_{1}$ + 3$x_{2}$ <= 120

Negative Production Constaint Check

$x_{1}$ > 0

$x_{2}$ > 0

## Sensitivity Analysis

for name, c in opt_model.constraints.items():
    print (name, ":", c, "\t", c.pi, "\t\t", c.slack)


import numpy as np

def kmeans(puntos, k):

    """
    K-means algorithm:
    Input = A 2D matrix of coordinates we want to make a 2-means clustering.
    Output = A 2x2 matrix with the resulting means
    Process = Iteratively assign the coordinates to a certain mean. After all points are assigned, the clustering means are reestimated based on the
    points assigned to each of these means.
    """

    eps = 0 # Tolerance for the while

    # Initialization of variables, we initialize with data points
    centroides = puntos[np.random.choice(puntos.shape[0], size=k, replace=False), :] # Initial means
    new_centroides = np.zeros((centroides.shape))

    aux_matrix = np.zeros((puntos.shape[0], k+1)) # Auxiliary matrix for calulations

    dif_norm = 1

    while dif_norm > eps:

        for idx in range(puntos.shape[0]):
            aux_matrix[idx,0:k] = np.linalg.norm((puntos[idx,:]-centroides), ord=2, axis=1)

            if len(np.where(aux_matrix[idx,0:k]==np.min(aux_matrix[idx,0:k]))[0]) == 1:
                aux_matrix[idx, k] = np.where(aux_matrix[idx,0:k]==np.min(aux_matrix[idx,0:k]))[0]
            else:
                aux_matrix[idx, k] = np.where(aux_matrix[idx,0:k]==np.min(aux_matrix[idx,0:k]))[0][0]

        for clu in range(k):
            new_centroides[clu,:] = np.mean(puntos[aux_matrix[:,k]==clu], axis=0)

        dif_norm = np.linalg.norm((centroides-new_centroides), ord=1)
        centroides = new_centroides

    return centroides


import numpy as np
import math as ma

def _normalize(v):
    norm = np.linalg.norm(v)
    if norm == 0.0:
       return v
    return v / norm

def writhe(curve):
    '''
    Returns writhe of given closed parametric curve.

    Parameters
    ----------
    curve: np.array
        An ``N`` by 3 array describing locations of endpoints of ``N`` linear segments approximating the curve. All locations should be distinct.

    Returns
    -------
    float
        Value of writhe for closed curve comprised of linear segments given.

    Example
    -------
    >>> import numpy as np
    >>> curve = np.array([[-1,0,-1],[1,0,1],[0,-1,1],[0,1,1]])
    >>> pywrithe.writhe(curve)
    0.366

    '''
    segments = np.transpose(np.array([np.roll(curve,1,axis=0),curve]),(1,0,2)) # (segment,begin/end,coordinate) order
    n = len(segments)
    omegas = np.zeros((n,n))
    for i in range(n):
        for j in range(n):
            if i-j == 0 or i-j == 1 or i-j == -1 or i-j == n-1 or i-j == -n+1:
                omegas[i][j] = 0
            else:
                segment_a = segments[i]
                segment_b = segments[j]
                r13 = segment_a[0] - segment_b[0]
                r14 = segment_a[0] - segment_b[1]
                r23 = segment_a[1] - segment_b[0]
                r24 = segment_a[1] - segment_b[1]

                r34 = segment_b[0] - segment_b[1]
                r12 = segment_a[0] - segment_a[1]

                n1 = _normalize(np.cross(r13,r14))
                n2 = _normalize(np.cross(r14,r24))
                n3 = _normalize(np.cross(r24,r23))
                n4 = _normalize(np.cross(r23,r13))

                d1 = np.clip(np.dot(n1,n2),-1,1)
                d2 = np.clip(np.dot(n2,n3),-1,1)
                d3 = np.clip(np.dot(n3,n4),-1,1)
                d4 = np.clip(np.dot(n4,n1),-1,1)

                a1 = ma.asin(d1)
                a2 = ma.asin(d2)
                a3 = ma.asin(d3)
                a4 = ma.asin(d4)

                omega_star = (a1+a2+a3+a4)

                omega = omega_star * ma.copysign(1.0, np.dot( np.cross( r34, r12) , r13) )

                omegas[i][j] = omega

    #print(omegas)

    return (1.0 / (4.0 * ma.pi)) * np.sum(omegas)


import math
import numpy as np
def derivative(f):
    '''returns a function that is the derivative of f with respect to x'''
    def dfdx(x):
        epsilon = 1e-3
        df = f(x+epsilon) - f(x-epsilon)
        dx = 2*epsilon
        return df/dx
    return dfdx
def integrate(f,a,b):
    '''returns definite integral of f(x) with respect to x from a to b
    uses average of left, right, and trapezoidal riemann sum.
    might not work with vertical asymptotes'''
    epsilon = 1e-3
    samples = ( max(a,b)-min(a,b) ) / epsilon
    X = np.linspace(min(a,b),max(a,b),samples)
    left = 0
    for x in X[:-1]:
        left += f(x) * epsilon
    right = 0
    for x in X[1:]:
        right += f(x) * epsilon
    trap = 0
    for i in range(len(X)-1):
        trap += .5 * ( f(X[i]) + f(X[i+1]) ) * epsilon
    ans = (left+right+trap) / 3
    if b > a:
        return ans
    return -1 * ans
def firstOrderODE(dydx,xi,yi,xf):
    '''expects dydx(x,y) and (xi,yi) which is the initial condition
    example: firstOrderODE(f(x,y),2,1)
    uses euler approximation'''
    ic = (xi,yi)
    dx = .001
    X = []
    Y = []
    x = ic[0]
    y = ic[1]
    while x < xf and y < 1e99:
        dy = dydx(x,y)*dx
        x += dx
        y += dy
    return y
if __name__ == '__main__':
    #for testing
    def f(x,y):
        return x+y
    print(firstOrderODE(f,2,1,6))


# Exercise 6
# Verify that the CDF inversion formula for the generation of exponential random variates of average value equal
# to 2 actually generates exponential variates.
# 1. Extract Ntr exponential random variates with the same average value equal to 2 using through the CDF inversion method.
# 2. Draw a QQ-plot to compare your draws against the quantiles of the exponential distribution with the same average value.
# 3. What happens if, instead, you draw your QQ-plot against the quantiles of an exponential distribution with a different average value?
# And what if you increase (e.g., 2×, 4×, . . . ) the number of exponential draws? Discuss.

# import
import numpy as np
from matplotlib import pyplot as plt
from numpy import mean, min, max, median, quantile
from scipy.stats import binom, norm, t as student, expon, poisson, chi2
from math import sqrt, pow, floor, ceil, exp, log
from statsmodels.api import qqplot

Ntr = 10000
# use the inverted cdf: x = - ln(U) / mean
average = 2
lam = 1 / average  # lambda = 1/avarage
results = -np.log(np.random.rand(Ntr)) / lam
plt.hist(results, bins=20)

# draw qq plot
qqplot(results, dist=expon, scale=1 / lam, line='45')

# draw qq against different average values: 5 and 0.5
qqplot(results, dist=expon, scale=5, line='45')
qqplot(results, dist=expon, scale=0.5, line='45')

# try with more draws (x2, x4, x20)
Ntr_2 = Ntr * 2
results = -np.log(np.random.rand(Ntr_2)) / lam
qqplot(results, dist=expon, scale=1 / lam, line='45')

Ntr_4 = Ntr * 4
results = -np.log(np.random.rand(Ntr_4)) / lam
qqplot(results, dist=expon, scale=1 / lam, line='45')

Ntr_20 = Ntr * 20
results = -np.log(np.random.rand(Ntr_20)) / lam
qqplot(results, dist=expon, scale=1 / lam, line='45')

Ntr_1_100 = Ntr // 100
results = -np.log(np.random.rand(Ntr_1_100)) / lam
qqplot(results, dist=expon, scale=1 / lam, line='45')

plt.show()

import numpy,from numpy,from scipy,from statsmodels


import numpy as np
from scipy.integrate import odeint
import matplotlib.pyplot as plt
from abc import ABC, abstractmethod # for later

def derive(y,t,N, beta, gamma):
    """The SIR model differential equations."""

    S, I, R = y
    dSdt = -beta * S * I / N
    dIdt = beta * S * I / N - gamma * I
    dRdt = gamma * I
    return dSdt, dIdt, dRdt

class SIR:

    def __init__(self, N=1000, I0=1, R0=0, beta=0.4, gamma=1./10, days=250):
        """Setup for the model.

        N:     Total population, N.
        I0,R0: Initial number of infected and recovered individuals.
        beta:  Contact rate, beta,
        gamma: Mean recovery rate,(in 1/days).
        t:     A grid of time points (in days)
        """

        self.N  = N
        self.I0 = I0
        self.R0 = R0
        self.S = 0
        self.I = 0
        self.R = 0
        self.beta = beta
        self.gamma = gamma
        self.t = np.linspace(0, days, days)

    def projection(self):
        """Expected change in the disease's distribution in the population. """

        S0 = self.N - self.I0 - self.R0 # Everyone else, S0, is susceptible to infection initially.
        # Initial conditions vector
        y0 = S0, self.I0, self.R0
        # Integrate the SIR equations over the time grid, t.
        ret = odeint(derive, y0, self.t, args=(self.N, self.beta, self.gamma))
        self.S, self.I, self.R = ret.T

    def plot_projection(self):
        """Plot the data on three separate curves for S(t), I(t) and R(t)"""

        fig = plt.figure(facecolor='w')
        ax = fig.add_subplot(111, facecolor='#dddddd', axisbelow=True)
        ax.plot(self.t, self.S/1000, 'b', alpha=0.5, lw=2, label='Susceptible')
        ax.plot(self.t, self.I/1000, 'r', alpha=0.5, lw=2, label='Infected')
        ax.plot(self.t, self.R/1000, 'g', alpha=0.5, lw=2, label='Recovered with immunity')
        ax.set_xlabel('Time /days')
        ax.set_ylabel('Number (1000s)')
        ax.set_ylim(0,1.2)
        ax.yaxis.set_tick_params(length=0)
        ax.xaxis.set_tick_params(length=0)
        ax.grid(b=True, which='major', c='w', lw=2, ls='-')
        legend = ax.legend()
        legend.get_frame().set_alpha(0.5)
        for spine in ('top', 'right', 'bottom', 'left'):
            ax.spines[spine].set_visible(False)
        plt.show()

class City(SIR):

    def __init__(self, name, position, area, air, port):
        super().__init__()
        self.name = name
        self.position = position
        self.area = area
        self.air = air
        self.port = port

    def get_name(self):
        return self.name

    def get_position(self):
        return self.position

    def __str__(self):
        return str(self.__dict__)