"""
2017 A. R. Malipeddi
A simple 2D geometric multigrid solver for the homogeneous Dirichlet Poisson problem on Cartesian grids and unit square. Cell centered 5-point finite difference operator.

Thanks Dr. Malpeddi!
"""

import numpy as np
from scipy.sparse.linalg import LinearOperator

def Jacrelax(level,nx,ny,u,f,iters=1,pre=False,r=0):
  '''
  under-relaxed Jacobi method smoothing
  '''

  dx=1.0/nx; dy=1.0/ny
  Ax=1.0/dx**2; Ay=1.0/dy**2
  Ap=1.0/(2.0*(Ax+Ay))

  #Dirichlet BC
  u[ 0,:] = -u[ 1,:]
  u[-1,:] = -u[-2,:]
  u[:, 0] = -u[:, 1]
  u[:,-1] = -u[:,-2]
  #
  # if(r):
  #     u[int(nx/2),int(ny/2)] = 1

  T = u.copy()
  for it in range(iters):
    for i in range(1,nx+1):
      for j in range(1,ny+1):
         u[i,j]= Ap*( Ax*(u[i+1,j]+u[i-1,j])
                     + Ay*(u[i,j+1]+u[i,j-1]) - f[i,j])

    u[ 0,:] = -u[ 1,:]
    u[-1,:] = -u[-2,:]
    u[:, 0] = -u[:, 1]
    u[:,-1] = -u[:,-2]

#  if(not pre):
#    return u,None

  res=np.zeros([nx+2,ny+2])
  for i in range(1,nx+1):
    for j in range(1,ny+1):
      res[i,j]=f[i,j] - ((Ax*(u[i+1,j]+u[i-1,j])+Ay*(u[i,j+1]+u[i,j-1]) - 2.0*(Ax+Ay)*u[i,j]))

  # if(r):
  #     res[int(nx/2),int(ny/2)] = 0

  return u,res

def restrict(nx,ny,v):
  '''
  restrict 'v' to the coarser grid
  '''
  v_c=np.zeros([nx+2,ny+2])

  for i in range(1,nx+1):
    for j in range(1,ny+1):
      v_c[i,j]=0.25*(v[2*i-1,2*j-1]+v[2*i,2*j-1]+v[2*i-1,2*j]+v[2*i,2*j])

  return v_c

def prolong(nx,ny,v):
  '''
  interpolate 'v' to the fine grid
  '''
  v_f=np.zeros([2*nx+2,2*ny+2])

  for i in range(1,nx+1):
      for j in range(1,ny+1):
        v_f[2*i-1,2*j-1] = 0.5625*v[i,j]+0.1875*(v[i-1,j]+v[i,j-1])+0.0625*v[i-1,j-1]
        v_f[2*i  ,2*j-1] = 0.5625*v[i,j]+0.1875*(v[i+1,j]+v[i,j-1])+0.0625*v[i+1,j-1]
        v_f[2*i-1,2*j  ] = 0.5625*v[i,j]+0.1875*(v[i-1,j]+v[i,j+1])+0.0625*v[i-1,j+1]
        v_f[2*i  ,2*j  ] = 0.5625*v[i,j]+0.1875*(v[i+1,j]+v[i,j+1])+0.0625*v[i+1,j+1]

  return v_f

def V_cycle(nx,ny,num_levels,u,f,level=1):
  '''
  V cycle
  '''
  if(level==num_levels):#bottom solve
    u,res=Jacrelax(level,nx,ny,u,f,iters=50,pre=True)
    return u,res

  #Step 1: Relax Au=f on this grid
  u,res=Jacrelax(level,nx,ny,u,f,iters=2,pre=True,r=1)

  #Step 2: Restrict residual to coarse grid
  res_c=restrict(nx//2,ny//2,res)

  #Step 3:Solve A e_c=res_c on the coarse grid. (Recursively)
  e_c=np.zeros_like(res_c)
  e_c,res_c=V_cycle(nx//2,ny//2,num_levels,e_c,res_c,level+1)

  #Step 4: Interpolate(prolong) e_c to fine grid and add to u
  u+=prolong(nx//2,ny//2,e_c)

  #Step 5: Relax Au=f on this grid
  u,res=Jacrelax(level,nx,ny,u,f,iters=1,pre=False,r=1)
  return u,res

def FMG(nx,ny,num_levels,f,nv=1,level=1):

  if(level==num_levels):#bottom solve
    u=np.zeros([nx+2,ny+2])
    u,res=Jacrelax(level,nx,ny,u,f,iters=50,pre=True)
    return u,res

  #Step 1: Restrict the rhs to a coarse grid
  f_c=restrict(nx//2,ny//2,f)

  #Step 2: Solve the coarse grid problem using FMG
  u_c,_=FMG(nx//2,ny//2,num_levels,f_c,nv,level+1)

  #Step 3: Interpolate u_c to the fine grid
  u=prolong(nx//2,ny//2,u_c)

  #step 4: Execute 'nv' V-cycles
  for _ in range(nv):
    u,res=V_cycle(nx,ny,num_levels-level,u,f)
  return u,res


# Highly divisible triagular number
#
# The sequence of triangle numbers is generated by adding the natural numbers.
# So the 7th triangle number would be 1 + 2 + 3 + 4 + 5 + 6 + 7 = 28.
# The first ten terms would be:
#
#   1, 3, 6, 10, 15, 21, 28, 36, 45, 55, ...
#
# Lets us list the factors of the first seven triangle number:
#
#  1: 1
#  3: 1, 3
#  6: 1, 2, 3, 6
# 10: 1, 2, 5, 10
# 15: 1, 3, 5, 15
# 21: 1, 3, 7, 21
# 28: 1, 2, 4, 7, 14, 28
#
# We can see that 29 is the first triangle number to have over five divisors.
#
# What is the value of the first triangle number to have over five hundred divisors?
import numpy as np

# input: number to be factored
# output: N/A
# return: array of factors
def factor(factor_number):
    factors = 0
#    factors = np.array([])
#    factors = []
    for i in range(1,factor_number+1):
        if factor_number % i == 0:
            factors += 1
#            factors = np.append(factors,[i])
#            factors.append(i)

    return factors

# input: number to triangle
# output: N/A
# return: triangular number
def triangular(triangular_number):
    triangular_value = 0
    for x in range(1,triangular_number+1):
        triangular_value += x

    return triangular_value

#
#   Main Program Execution
#
if __name__ == "__main__":
    count = 0
    t_number = 0
    num_factors = 0
#    for x in range(1,100):
#        print("{}, value {}".format(x,triangular(x)))
#        print("Factors: {}".format(factor(triangular(x))))

    while num_factors < 500:
        count += 1
        t_number = triangular(count)
#        factors = factor(t_number)
#        print(factors.size)
        num_factors = factor(t_number)
#        print(t_number)
        print(num_factors)
    print(t_number)


'''
Power Iteration method of finding dominant eigenvector.
	source : https://www.wikiwand.com/en/Power_iteration
'''
import numpy as np
from numpy.linalg import norm

def rayleigh_quotient( A, b ):
	bt = conjugate_transpose( b )
	return ( np.dot( np.dot( bt, A ), b ) )/( np.dot( bt, b ) )

def conjugate_transpose( b ):
	return np.conj( b.T )

def __test():
	# same A as in power_iteration_eigenvector.py
	A = np.array( [ [ 0.5, 0.5 ], [ 0.2, 0.8 ] ] )

	# value estimated by test case in power_iteration_eigenvector.py
	b = np.array([0.70710679, 0.70710677])
	return rayleigh_quotient( A, b )#, np.linalg.eig( A )

if __name__ == '__main__':
	print __test()


from autodiff import operations
from autodiff.structures import Number
from autodiff.structures import Array
import numpy as np
from copy import deepcopy

def newtons_method(func, initial_guess, iterations=100,tolerance = 10**-7,verbose = False, show_fxn=False):

    """Use Newton's method to find the root of the function
    Args:
        func: the function that the user wants to find root for
        initial_guess: a number object for the initial guess
        iterations: number of maximum iterations
        show_fxn: if true, return function value at the final xstar

    Returns:
        xn: the x value of the root
        jacobians: the jacobian at each step of the root_finding
        fxn: func(xn). If root_finding is successful, this value should be 0
    """
    if isinstance(initial_guess,Number):
        #scalar case
        jacobians = []

        x0 = initial_guess

        fxn = func(initial_guess)

        fpxn = fxn.jacobian(initial_guess)

        x1 = x0 - fxn/fpxn

        jacobians.append(fpxn)

        for i in range(iterations):
            if abs(fxn.val) > 1e-7:
                x0 = x1

                fxn = func(x0)

                fpxn = fxn.jacobian(x0)

                jacobians.append(fpxn)

                x1 = x0- fxn / fpxn

        if show_fxn:
            return x1, jacobians,fxn
        else:
            return x1, jacobians
    elif isinstance(initial_guess,Array):
        jacobians = []
        if isinstance(func(initial_guess),tuple):
            for i in range(iterations):

                if i == 0:
                    x0= initial_guess
                else:
                    x0 = x1

                fxn = func(x0)
                fpxn = []
                if verbose:
                    print(i,x0,fxn)

                vector = []
                for n in range(len(fxn)):
                    vector.append(fxn[n].val)

                if np.linalg.norm(vector)< tolerance:
                    break

                for k in range(len(fxn)):
                    fpxn_row = []
                    for j in range(len(x0)):
                        fpxn_row.append(fxn[k].jacobian(x0[j]))
                    fpxn.append(fpxn_row)
                jacobians.append(fpxn)
                x1 = x0 - np.dot(np.linalg.inv(fpxn),fxn)

            if show_fxn:
                return x1, jacobians,fxn
            else:
                return x1, jacobians

        else:
            for i in range(iterations):

                if i == 0:
                    x0= initial_guess
                else:
                    x0 = x1

                fxn = func(x0)
                fpxn = []
                if verbose:
                    print(i,x0,fxn)

                if abs(fxn.val)< tolerance:
                    break

                for j in range(len(x0)):
                    fpxn.append(fxn.jacobian(x0[j]))
                jacobians.append(fpxn)
                print(fpxn)
                x1 = x0 - np.dot(np.reciprocal(fpxn),fxn)

            if show_fxn:
                return x1, jacobians,fxn
            else:
                return x1, jacobians


"""
Use the Gram Schmidt procedure to reorient a set of points such that one point is at the origin, another point lies
on one axis (x1, 0, ..., 0) and the next point lies on a plane spanned by the two axes  (x2, x3, 0, ..., 0), and
so on. This constrains the minimization function that looks for the points of best fit, as rotations and
translations are made redundant.

This can also take into effect the noise parameter as we do not constrain where on the first axis, the second
point lies.
"""
import numpy as np

def choose_basis_vectors(M):
    """
    Given a matrix of points M (m by n), where m is the number of dimensions and n is the number of points,
    choose m vectors that are linearly independent from each other.

    Since the chosen submatrix would be m by m, a simple way to do this is by calculating the determinant.
    If the determinant is 0, the vectors are not linearly independent
    """
    # first check if M has full rank
    n_dim, n_points = M.shape
    if np.linalg.matrix_rank(M) < n_dim:
        raise ValueError("Matrix is rank-deficient. Points lie in a subspace of rank smaller than {} dim".format(n_dim))
    submatrix = M[:, 0:n_dim]
    if np.linalg.det(submatrix) == 0:
        return NotImplementedError("Edge case not handled. Points will need reordering if other vectors"
                                   "are selected.")
    return submatrix

def gram_schmidt(B):
    """
    Take in an m by m matrix A and return a matrix with orthonormal vectors that span the same space as A
    In this case, we assume that the input matrix A is square and has linearly independent vectors.
    """
    G = np.zeros(B.shape)
    n_row, n_col = B.shape
    for j in range(n_col):
        vector_j = B[:, j]
        for k in range(0, j):
            # subtract the projection of j onto the previous vector from vector j
            scalar = (np.dot(G[:, k], B[:, j]) / np.dot(G[:, k], G[:, k]))
            projection = G[:, k] * scalar
            vector_j = vector_j - projection
        G[:, j] = vector_j
    norms = np.linalg.norm(G, axis=0)
    G = G / norms
    return G

def anchor_points(points):
    """
    points: 37 by d matrix
    Given a set of 37 points, find the rotation needed using Gram Schmidt and then
    transform the  coordinates to the format needed - one point is zero and the rest are partly in a triangular form
    """
    # make the first point the origin (translate all by first vector)
    points = points - points[0, :]
    # transpose the matrix so that each point is a column
    points = points.T
    # choose d independent vectors
    subset_points = choose_basis_vectors(points[:, 1:])
    Q = gram_schmidt(subset_points)
    # transform the 37 points
    R = Q.T @ points
    # put rotated points in the correct format
    return R.T


import numpy as np

class Optimizer:

    def __init__(self, precision=1e-6) -> None:
        super().__init__()
        self.has_converged = False
        self.precision = precision

    def next(self, X, dX):
        _X = self._next(X, dX)

        if np.abs(_X - X).mean() < self.precision:
            self.has_converged = True

        return _X

class GradientDescent(Optimizer):

    def __init__(self, learning_rate=0.01, **kwargs) -> None:
        super().__init__(**kwargs)
        self.learning_rate = learning_rate

    def _next(self, X, dX):
        return X - self.learning_rate * dX

class Adam(Optimizer):

    def __init__(self, alpha=0.01, beta_1=0.9, beta_2=0.999, epsilon=1e-8, **kwargs) -> None:
        super().__init__(**kwargs)

        self.alpha = alpha
        self.beta_1 = beta_1
        self.beta_2 = beta_2
        self.epsilon = epsilon

        self.m_t = 0
        self.v_t = 0
        self.t = 0

    def _next(self, X, dX):
        self.t += 1
        beta_1, beta_2 = self.beta_1, self.beta_2

        # update moving average of gradient and squared gradient
        self.m_t = beta_1 * self.m_t + (1 - beta_1) * dX
        self.v_t = beta_2 * self.v_t + (1 - beta_2) * (dX * dX)

        # calculates the bias-corrected estimates
        m_cap = self.m_t / (1 - (beta_1 ** self.t))
        v_cap = self.v_t / (1 - (beta_2 ** self.t))

        # do the gradient update
        _X = X - (self.alpha * m_cap) / (np.sqrt(v_cap) + self.epsilon)

        return _X


try:
    from scipy.special import gamma as _gamma, gammainc as _gammainc, digamma as _digamma
    import numpy as _np
    from math import sqrt as _sqrt, log as _log, exp as _exp
    from typing import Union, Dict, List
    from univariate._base import SemiInfinite
except Exception as e:
    print(f"some modules are missing {e}")

class Gamma(SemiInfinite):
    """
    This class contains methods concerning a variant of Gamma distribution [#]_.

    .. math::
        \\text{Gamma}(x;a,b) = \\frac{1}{b^a \\Gamma(a)} \\ x^{a-1} e^{\\frac{-x}{b}}

    Args:

        shape(float): shape parameter (:math:`a`) where shape > 0
        scale(float): scale parameter (:math:`b`) where scale > 0
        x(float): random variable where x > 0

    References:
        .. [#] Matlab(2020). Gamma Distribution. https://www.mathworks.com/help/stats/gamma-distribution.html
    """

    def __init__(self, shape: float, b: float, x: float):
        if shape < 0:
            raise ValueError('shape should be greater than 0.')
        if b < 0:
            raise ValueError('scale should be greater than 0.')
        self.shape = shape
        self.scale = b

    def pdf(self, x: Union[List[float], _np.ndarray, float]) -> Union[float, _np.ndarray]:
        """
        Args:
            x (Union[List[float], numpy.ndarray, float]): random variable(s)

        Raises:
            ValueError: when there exist a value of x that is less than 0

        Returns:
            Union[float, numpy.ndarray]: evaluation of pdf at x
        """
        shape = self.shape
        scale = self.scale

        if isinstance(x, (_np.ndarray, List)):
            if not type(x) is _np.ndarray:
                x = _np.array(x)
            if _np.any(x < 0):
                raise ValueError('random variable should be greater than 0.')
            return (1 / (pow(scale, shape) * _gamma(shape))) * _np.log(x, shape - 1) * _np.exp(-x / scale)

        if x < 0:
            raise ValueError('random variable should be greater than 0.')
        return (1 / (pow(scale, shape) * _gamma(shape))) * _log(x, shape - 1) * _exp(-x / scale)

    def cdf(self, x: Union[List[float], _np.ndarray, float]) -> Union[float, _np.ndarray]:
        """
        Args:
            x (Union[List[float], numpy.ndarray, float]): data point(s) of interest

        Returns:
            Union[float, numpy.ndarray]: evaluation of cdf at x
        """
        shape = self.shape
        scale = self.scale

        # there is no apparent explanation for reversing gammainc's parameter, but it works quite perfectly in my prototype
        def __generator(shape: float, b: float, x: Union[float, _np.ndarray]) -> Union[float, _np.ndarray]:
            return 1 - _gammainc(shape, x / b)

        if isinstance(x, (_np.ndarray, List)):
            if not type(x) is _np.ndarray:
                x = _np.array(x)
            return __generator(shape, scale, x)
        return __generator(shape, scale, x)

    def mean(self) -> Union[float, int]:
        """
        Returns: Mean of the Gamma distribution
        """
        return self.shape * self.scale

    def median(self) -> str:
        """
        Returns: Median of the Gamma distribution.
        """
        return "No simple closed form."

    def mode(self) -> Union[float, int]:
        """
        Returns: Mode of the Gamma distribution
        """
        return (self.shape - 1) * self.scale

    def var(self) -> Union[float, int]:
        """
        Returns: Variance of the Gamma distribution
        """
        return self.shape * pow(self.scale, 2)

    def std(self) -> float:
        """
        Returns: Standard deviation of the Gamma distribution
        """
        return _sqrt(self.var())

    def skewness(self) -> float:
        """
        Returns: Skewness of the Gamma distribution
        """
        return 2 / _sqrt(self.shape)

    def kurtosis(self) -> float:
        """
        Returns: Kurtosis of the Gamma distribution
        """
        return 6 / self.shape

    def entropy(self) -> float:
        """
        Returns: differential entropy of the Gamma distribution

        Reference: Park, S.Y. & Bera, A.K.(2009). Maximum entropy autoregressive conditional heteroskedasticity model. Elsivier.
        link: http://wise.xmu.edu.cn/uploadfiles/paper-masterdownload/2009519932327055475115776.pdf
        """
        k = self.shape
        theta = self.scale
        return k + _log(theta)+_log(_gamma(k))-(1-k)*_digamma(k)

    def summary(self) -> Dict[str, Union[float, int, str]]:
        """
        Returns:
            Dictionary of Gamma distirbution moments. This includes standard deviation.
        """
        return {
            'mean': self.mean(), 'median': self.median(), 'mode': self.mode(),
            'var': self.var(), 'std': self.std(), 'skewness': self.skewness(), 'kurtosis': self.kurtosis()
        }


"""
Activations functions for fundl.
"""
import jax.numpy as np

def sigmoid(x, version="tanh"):
    if version not in ["tanh", "exp"]:
        raise ValueError("version must be one of ['tanh' or 'exp']")

    sigmoids = {
        "tanh": lambda x: 0.5 * np.tanh(x) + 0.5,
        "exp": lambda x: 1 / (1 + np.exp(-x)),
    }
    return sigmoids[version](x)

def relu(x, alpha=0.1):
    return x * (x > 0) + alpha * x * (x <= 0)

def tanh(x):
    return np.tanh(x)

def leaky_relu(x, a):
    """Leaky ReLu"""
    pass

def identity(x):
    return x

def softmax(x, axis: int = -1):
    return np.exp(x) / np.sum(np.exp(x), axis=axis).reshape(-1, 1)


from __future__ import print_function

import numpy as np

def meanclip(indata, clipsig=3.0, maxiter=5, converge_num=0.02, verbose=1, return_array=0, return_median=0):
    """Computes an iteratively sigma-clipped mean on a data set.

    Clipping is done about median, but mean is returned by default
    (use return_median to return the clipped median value).

    Parameters:
        indata : array_like
            Input data.

        clipsig : float
            Number of sigma at which to clip.

        maxiter : int
            Ceiling on number of clipping iterations.

        converge_num : float
            If the proportion of rejected pixels is less than
            this fraction, the iterations stop.

        verbose : {0, 1}
            Print messages to screen?

        return_array : {0, 1}
            Return the final array indices that were used to compute
            statistics.

        return_median : {0, 1}
            Return the median if 1. Return the mean if 0.

    Returns:
        val : float
            The N-sigma clipped mean or median.
        sigma : float
            The standard deviation of remaining pixels.

    Outputs:
        Prints to screen mean or median statistics.

    History:
        * 21/10/1998 Written by RSH, RITSS
        * 20/01/1999 Added SUBS, fixed misplaced paren on float call, improved doc. RSH
        * 24/11/2009 Converted to Python. PLL.
        * 08/01/2013 Added option to return the array indices of non-clipped pixels. DMH
        * Added option to return median of the clipped array. DMH

    Notes:
        This is based on MYMEANCLIP routine from ACS library.

    Examples:

    >>> mean, sigma = meanclip(indata)
    """
    # Flatten array
    skpix = indata.reshape( indata.size, )

    # initialize array to store indices of pixels used to compute stats
    arrind = np.arange(0,skpix.size)

    ct = indata.size
    iter = 0; c1 = 1.0 ; c2 = 0.0

    while (c1 >= c2) and (iter < maxiter):
        lastct = ct
        medval = np.median(skpix)
        sig = np.std(skpix)
        wsm = np.where( abs(skpix-medval) < clipsig*sig )
        ct = len(wsm[0])
        if ct > 0:
            skpix = skpix[wsm]
            arrind = arrind[wsm]

        c1 = abs(ct - lastct)
        c2 = converge_num * lastct
        iter += 1

    if return_median:
        val = np.median(skpix)
        val_type = 'median'
    else:
        val  = np.mean( skpix )
        val_type = 'mean'
    sigma = np.std( skpix )

    if verbose:
        if return_median:
            prf = 'MEDIANCLIP:'
            print('{} {}.1f-sigma clipped median'.format(prf, clipsig))
            print('{} Median computed in {} iterations'.format(prf, iter))
            print('P{} Median = {}, sigma = {}'.format(prf, val, sigma))

        else:
            prf = 'MEANCLIP:'
            print('{} {}.1f-sigma clipped mean'.format(prf, clipsig))
            print('{} Mean computed in {} iterations'.format(prf, iter))
            print('{} Mean = {}, sigma = {}'.format(prf, val, sigma))

    if return_array:
        return np.copy(arrind)
    else:
        return val, sigma


# https://www.hackerrank.com/challenges/np-mean-var-and-std/problem

# Variance: Mean of all the squared deviations from mean of the set of numbers
# Standard Deviation: Square root of variance

import numpy as np

n, m = list(map(int, input().split(' ')))

myList = []
for _ in range(n):
    myList += [input().split(' ')]
myArr = np.array(myList, int)
myArr.shape = (n, m)

print(np.mean(myArr, axis=1))
print(np.var(myArr, axis=0))
print(round(float(np.std(myArr)), 11))  # Rounded for compatibility in the challenge


import numpy as np

array1 = [[1,2],[3,4]]
array2 = [[5,6],[7,8]]

print("Concatenate")
print("_________")

array3_0 = np.concatenate((array1,array2), axis=0)
print(array3_0)
"""
[[1 2]
 [3 4]
 [5 6]
 [7 8]]
"""

array3_1 = np.concatenate((array1,array2), axis=1)
print(array3_1)
"""
[[1 2 5 6]
 [3 4 7 8]]
"""

print("")
print("Stack")
print("_________")

array1 = [1,2, 3,4]
array2 = [5,6,7,8]

array3_0 = np.stack((array1,array2), axis=0)
print(array3_0)
"""
[[1 2 3 4]
 [5 6 7 8]]
"""

array3_1 = np.stack((array1,array2), axis=1)
print(array3_1)
"""
[[1 5]
 [2 6]
 [3 7]
 [4 8]]
"""


import numpy as np

def projsplx(y):
    """projsplx projects a vector to a simplex
    by the algorithm presented in
    (Chen an Ye, "Projection Onto A Simplex", 2011)"""
    assert len(y.shape) == 1
    N = y.shape[0]
    y_flipsort = np.flipud(np.sort(y))
    cumsum = np.cumsum(y_flipsort)
    t = (cumsum - 1) / np.arange(1,N+1).astype('float')

    t_iter = t[:-1]
    t_last = t[-1]
    y_iter = y_flipsort[1:]

    if np.all((t_iter - y_iter) < 0):
        t_hat = t_last
    else:
        # find i such that t>=y
        eq_idx = np.searchsorted(t_iter - y_iter, 0, side='left')
        t_hat = t_iter[eq_idx]

    x = y - t_hat
    # there may be a numerical error such that the constraints are not exactly met.
    x[x<0.] = 0.
    x[x>1.] = 1.
    assert np.abs(x.sum() - 1.) <= 1e-5
    assert np.all(x >= 0) and np.all(x <= 1.)
    return x


import sys
from operator import add
from pyspark.sql import SparkSession
from pyspark import SparkContext
import pyspark
from pyspark.ml.linalg import Vectors
import numpy as np
from sklearn.linear_model import LinearRegression
from pyspark.sql.types import *
from pyspark.sql import functions as func
from pyspark.sql.functions import *
from pyspark.sql import SQLContext
import matplotlib.pyplot as plt
import time
from pandas import Series,DataFrame
import pandas as pd
import re
from collections import Counter
from sklearn.linear_model import LinearRegression
from pyspark.ml.classification import LogisticRegression
from pyspark.ml.evaluation import MulticlassClassificationEvaluator
from pyspark.ml.classification import LinearSVC

# building functions

def isfloat(value):
	try:
		float(value)
		return True
	except:
		return False

def correctRows(p):
	if isfloat(p[3]) and isfloat(p[4]) and isfloat(p[6]) and isfloat(p[7]) and isfloat(p[9]):
		return p

def to_list(a):
	return [a]

def addToList(x, y):
	x.append(y)
	return x

def extend(x,y):
	x.extend(y)
	return x

if __name__ == "__main__":
	if len(sys.argv) != 3:
		print("Usage: wordcount <file> <output> ", file=sys.stderr)
		exit(-1)

	spark = SparkSession.builder.master("local[*]").getOrCreate()
	sc = SparkContext.getOrCreate()
	sqlContext = SQLContext(sc)

	# load data set
	lines2 = sc.textFile("Google-Playstore.csv")

	# generate test case file
	df = pd.read_csv("Google-Playstore.csv")
	test_case = df.sample(n = 10000)
	test_case.to_csv('Google-Playstore_test.csv', index=False)

	"""

	Simple Linear Regression

	"""

	print("##### Finding Simple Linear Regression Equation #####")

	# data pre-processing

	correctLine = lines2.map(lambda x: x.split(','))
	cleaned = correctLine.filter(correctRows)

	max_install = cleaned.map(lambda p: (float(p[7])))
	rating = cleaned.map(lambda p: (float(p[3])))

	# apply linear regression
	x = np.array(max_install.collect())
	y = np.array(rating.collect())

	X = np.stack([x], axis = 1)

	reg = LinearRegression(fit_intercept=True).fit(X, y)

	print("The m (coefficient) =",reg.coef_)
	print("The b (y-intercept) =",reg.intercept_)
	print("The equation is: y = "+str(reg.coef_[0])+"X + "+str(reg.intercept_))

	"""

	Gradient Descent for parameters

	"""

	print("##### Finding the parameters using gradient descent #####")

	start1 = time.time()
	df = np.stack([y, x], axis=1)
	dff = map(lambda x: (float(x[0]), Vectors.dense(x[1:])), df)
	mydf = spark.createDataFrame(dff, schema=["Money", "Distance"])
	myRDD=mydf.rdd.map(tuple).map(lambda x: (float(x[0]), np.array(x[1]) ))

	learningRate = 0.00001
	num_iteration = 100
	size = float(len(y))
	beta = np.array([0.1])
	costs = []

	for i in range(num_iteration):
		gradientCost=myRDD.map(lambda x: (x[1], (x[0] - x[1] * beta) ))\
							   .map(lambda x: (x[0]*x[1], x[1]**2 )).reduce(lambda x, y: (x[0] +y[0], x[1]+y[1] ))
		cost= gradientCost[1]
		gradient=(-1/float(size))* gradientCost[0]
		print(i, "Beta", beta, " Cost", cost)
		beta = beta - learningRate * gradient
		costs.append(cost[0])

	end1 = time.time()

	print(f"Computation time of BGD is {(end1 - start1)/60} Minutes")

	# making plot
	xValues = [i for i in range(len(costs))]
	plt.plot(xValues, costs, 'o', markersize=2)
	plt.xlabel("Number of Iteration")
	plt.ylabel("Cost")
	plt.title("Cost with the number of iteration")
	plt.show()

	"""

	Multi-Linear Regression

	"""

	print("##### Finding the parameters of multi-linear regression using gradient descent #####")

	start2 = time.time()

	rating = cleaned.map(lambda p: (p[0], p[3]))
	rating_count = cleaned.map(lambda p: (p[0], p[4]))
	min_install = cleaned.map(lambda p: (p[0], p[6]))
	max_install = cleaned.map(lambda p: (p[0], p[7]))
	price = cleaned.map(lambda p: (p[0], p[9]))

	rating = rating.combineByKey(to_list, addToList, extend)
	rating = rating.collect()
	rating_count = rating_count.combineByKey(to_list, addToList, extend)
	rating_count = rating_count.collect()
	min_install = min_install.combineByKey(to_list, addToList, extend)
	min_install = min_install.collect()
	max_install = max_install.combineByKey(to_list, addToList, extend)
	max_install = max_install.collect()
	price = price.combineByKey(to_list, addToList, extend)
	price = price.collect()

	ratingKey = []
	ratingValue = []
	for i in range(len(rating)):
		ratingKey.append(rating[i][0])

		rate = 0
		total = 0
		for j in [float(i) for i in rating[i][1]]:
			rate += j
			total += 1

		ratingValue.append(rate/total)

	ratingCountKey = []
	ratingCountValue = []
	for i in range(len(rating_count)):
		ratingCountKey.append(rating_count[i][0])

		rate = 0
		for j in [float(i) for i in rating_count[i][1]]:
			rate += j

		ratingCountValue.append(rate)

	min_installKey = []
	min_installValue = []
	for i in range(len(min_install)):
		min_installKey.append(min_install[i][0])

		count = 0
		for j in [float(i) for i in min_install[i][1]]:
			if j != 0:
				count += 1

		min_installValue.append(count)

	max_installKey = []
	max_installValue = []
	for i in range(len(max_install)):
		min_installKey.append(max_install[i][0])

		count = 0
		for j in [float(i) for i in max_install[i][1]]:
			if j != 0:
				count += 1

		max_installValue.append(count)

	priceKey = []
	priceValue = []
	for i in range(len(price)):
		priceKey.append(price[i][0])

		amount = 0
		for j in [float(i) for i in price[i][1]]:
			amount += j

		priceValue.append(amount)

	app = ratingKey
	rating = ratingValue
	countOfRating = ratingCountValue
	mi_install = min_installValue
	Price = priceValue

	ma_install = max_installValue

	x = []
	y = []
	for i in range(len(app)):
		x.append([float(rating[i]), float(countOfRating[i]), float(mi_install[i]), float(Price[i])])
		y.append(float(ma_install[i]))

	learningRate = 0.000001
	num_iteration = 100
	size = len(y)
	costs = []
	beta = np.array([0.1, 0.1, 0.1, 0.1])

	data = {'y':y, 'x':x}
	df = DataFrame(data)
	spark_df_from_pandas = spark.createDataFrame(df, schema=['x', 'y'])
	myRDD=spark_df_from_pandas.rdd.map(lambda x: (float(x[0]), np.array(x[1])))

	for i in range(num_iteration):
		gradientCost=myRDD.map(lambda x: (x[1], (x[0] - x[1] * beta)))\
							   .map(lambda x: (x[0]*x[1], x[1]**2 )).reduce(lambda x, y: (x[0] +y[0], x[1]+y[1] ))

		cost = 0
		for j in gradientCost[1]:
			cost += j

		gradient=(-1/float(size))* gradientCost[0]
		print(i, "Beta", beta, " Cost", cost)
		beta = beta - learningRate * gradient

		costs.append(cost)

	end2 = time.time()

	print(f"Computation time of multi-linear regression by BGD is {(end2 - start2)/60} minutes")

	xValues = [i for i in range(len(costs))]

	plt.plot(xValues, costs, 'o', markersize=2)
	plt.xlabel("Number of Iteration")
	plt.ylabel("Cost")
	plt.title("Cost with the number of iteration")
	plt.show()

	"""

	Logistic Regression

	"""

	print("##### gradient descent algorithm to learn a logistic regression model #####")

	start3 = time.time()

	total_install = cleaned.map(lambda p: (p[0], p[7]))
	tuples = total_install.collect()

	appWords = []

	for i in app:
		words = i.split(" ")
		for j in words:
			j = re.sub('[^A-Za-z0-9]+', '', j)
			appWords.append(j)

	appWords = ' '.join(appWords).split()

	allWords = sc.parallelize(appWords)
	allCount = allWords.map(lambda x: (x, 1)).reduceByKey(add)
	topWords = allCount.top(20000, lambda x: x[1])

	topWordsK = sc.parallelize(range(20000))
	dictionary = topWordsK.map(lambda x: (topWords[x][0], x))

	def TF(words_list, top_words):
		words_dict = dict(Counter(words_list))
		tf_vector = []
		for word in top_words:
			if word in words_dict.keys():
				tf = words_dict[word]
				tf_vector.append(tf)
			else:
				tf_vector.append(0)
		return tf_vector

	key_id = [i for i in range(len(tuples))]
	key_values = {tuples[i][0]: i for i in key_id}
	topWordsBC = sc.broadcast(dictionary.keys().collect())
	feat = cleaned.map(lambda x: (key_values[x[0]], TF(x[1], topWordsBC.value)))
	labels = cleaned.map(lambda x: (key_values[x[0]], int(x[0][0] == 'A' and x[0][1] == 'U')))
	trainRDD = feat.join(labels)

	learningRate = 0.0003
	num_iteration = 5
	lambda_cof = 0.01
	size = len(tuples)

	loss_list = list()

	parameter_vector = np.random.normal(0, 0.1, (dictionary.count(), 1))

	def sigmoid(x):
		return 1.0 / (1 + np.exp(-x))

	def loss_func(feat_line, y, parameter_vector):
		feat_line = np.array(feat_line)
		pred = sigmoid(np.dot(feat_line, parameter_vector))
		return -y * np.log(pred + 1e-12) - (1 - y) * np.log((1 - pred) + 1e-12)

	def accuracy_score(feat_line, y, parameter_vector):
		feat_line = np.array(feat_line)
		pred = sigmoid(np.dot(feat_line, parameter_vector))
		pred = 1 if pred >= 0.5 else 0
		acc = int(pred == y)
		return acc

	def grad_func(feat_line, y, parameter_vector):
		feat_line = np.array(feat_line)
		pred = sigmoid(np.dot(feat_line, parameter_vector))
		grad = (pred - y) @ feat_line[None, :]
		return grad

	acc_his = []
	loss_his = []
	grad_his = []
	prev_param_norm = 0

	for i in tqdm.trange(num_iteration):
		parameter_vector_BC = sc.broadcast(parameter_vector)

		loss = trainRDD.map(lambda x: loss_func(x[1][0], x[1][1], parameter_vector_BC.value)).reduce(add) / size
		acc = trainRDD.map(lambda x: accuracy_score(x[1][0], x[1][1], parameter_vector_BC.value)).reduce(add) / size
		grad = trainRDD.map(lambda x: grad_func(x[1][0], x[1][1], parameter_vector_BC.value)).reduce(add) / size

		parameter_vector = parameter_vector - learningRate * grad[:, None]

		if np.abs(np.linalg.norm(parameter_vector) - prev_param_norm) < 1e-7:
			print('Break')
			break

		prev_param_norm = np.linalg.norm(parameter_vector)
		# L2
		parameter_vector = parameter_vector - 2 * lambda_cof * parameter_vector

		acc_his.append(acc)
		loss_his.append(loss)
		grad_his.append(np.linalg.norm(grad))

	end3 = time.time()

	print(f"Computation time of multi-linear regression by BGD is {(end3 - start3)/60} minutes")

	fig, ax = plt.subplots(3, figsize=(13, 13))
	ax[0].set_title('Accurary')
	ax[0].plot(acc_his)
	ax[1].set_title('Loss')
	ax[1].plot(loss_his)
	ax[2].set_title('GradNorm')
	ax[2].plot(grad_his)
	plt.savefig("TrainingProcess.png")
	plt.show()

	print('The five words with the largest coefficients',
		  np.array(topWordsBC.value)[np.argsort(parameter_vector[:, 0])[-5:]])

	"""

	Logistic Regression Model Evaluation

	"""

	print("##### model evaluation #####")

	start4 = time.time()

	t_tuples = total_install.collect()
	key_id = [i for i in range(len(tuples))]
	key_values = {tuples[i][0]: i for i in key_id}
	topWordsBC = sc.broadcast(dictionary.keys().collect())
	feat = cleaned.map(lambda x: (key_values[x[0]], TF(x[1], topWordsBC.value)))
	labels = cleaned.map(lambda x: (key_values[x[0]], int(x[0][0] == 'A' and x[0][1] == 'U')))
	testRDD = feat.join(labels)

	# val
	def TP_func(feat_line, y, parameter_vector):
		feat_line = np.array(feat_line)
		pred = sigmoid(np.dot(feat_line, parameter_vector))
		pred = 1 if pred >= 0.5 else 0
		TP = int(pred == 1 and y == 1)
		return TP

	def FP_func(feat_line, y, parameter_vector):
		feat_line = np.array(feat_line)
		pred = sigmoid(np.dot(feat_line, parameter_vector))
		pred = 1 if pred >= 0.5 else 0
		FP = int(pred == 1 and y != 1)
		return FP

	def FN_func(feat_line, y, parameter_vector):
		feat_line = np.array(feat_line)
		pred = sigmoid(np.dot(feat_line, parameter_vector))
		pred = 1 if pred >= 0.5 else 0
		FN = int(pred != 1 and y == 1)
		return FN

	def TN_func(feat_line, y, parameter_vector):
		feat_line = np.array(feat_line)
		pred = sigmoid(np.dot(feat_line, parameter_vector))
		pred = 1 if pred >= 0.5 else 0
		TN = int(pred != 1 and y != 1)
		return TN

	parameter_vector_BC = sc.broadcast(parameter_vector)
	acc = testRDD.map(lambda x: accuracy_score(x[1][0], x[1][1], parameter_vector_BC.value)).reduce(add) / size
	TP = testRDD.map(lambda x: TP_func(x[1][0], x[1][1], parameter_vector_BC.value)).reduce(add)
	FP = testRDD.map(lambda x: FP_func(x[1][0], x[1][1], parameter_vector_BC.value)).reduce(add)
	FN = testRDD.map(lambda x: FN_func(x[1][0], x[1][1], parameter_vector_BC.value)).reduce(add)
	TN = testRDD.map(lambda x: TN_func(x[1][0], x[1][1], parameter_vector_BC.value)).reduce(add)

	F1 = 2 * TP / (2 * TP + FN + FP)
	print('The Acc of Test: ', acc)
	print('The F1 score of Test: ', F1)

	print(f"Computation time of multi-linear regression by BGD is {(end4 - start4)/60} minutes")

	"""

	SVM Model

	"""

	print("##### SVM model #####")

	start5 = time.time()

	d_corpus = sc.textFile("Google-Playstore.csv")
	d_keyAndText = d_corpus.map(lambda x: (x[x.index('id="') + 4: x.index('" url=')], x[x.index('">') + 2:][:-6]))
	regex = re.compile('[^a-zA-Z]')

	d_keyAndListOfWords = d_keyAndText.map(lambda x: (str(x[0]), regex.sub(' ', x[1]).lower().split())).sortByKey(False)

	tuples = d_keyAndListOfWords.collect()
	allWordsList = []

	for i in range(len(tuples)):
		for j in tuples[i][1]:
			allWordsList.append(j)

	allWords = sc.parallelize(allWordsList)
	allCount = allWords.map(lambda x: (x, 1)).reduceByKey(add)
	topWords = allCount.top(20000, lambda x: x[1])

	topWordsK = sc.parallelize(range(20000))
	dictionary = topWordsK.map(lambda x: (topWords[x][0], x))

	def TF(words_list, top_words):
		words_dict = dict(Counter(words_list))
		tf_vector = []
		for word in top_words:
			if word in words_dict.keys():
				tf = words_dict[word]
				tf_vector.append(tf)
			else:
				tf_vector.append(0)
		return Vectors.dense(tf_vector)

	key_id = [i for i in range(len(tuples))]
	key_values = {tuples[i][0]: i for i in key_id}
	topWordsBC = sc.broadcast(dictionary.keys().collect())
	feat = d_keyAndListOfWords.map(lambda x: (key_values[x[0]], TF(x[1], topWordsBC.value)))
	labels = d_keyAndListOfWords.map(lambda x: (key_values[x[0]], int(x[0][0] == 'A' and x[0][1] == 'U')))
	train_feat_df = sqlContext.createDataFrame(feat, ['ind', 'features'])
	train_labels_df = sqlContext.createDataFrame(labels, ['ind', 'labels'])
	train_df = train_feat_df.join(train_labels_df, on=['ind']).sort(['ind'])
	train_df.cache()

	svc = LinearSVC(labelCol='labels')
	svg_model = svc.fit(train_df)

	st_test_read = time.time()
	t_corpus = sc.textFile('Google-Playstore_test.csv')
	t_keyAndText = t_corpus.map(lambda x: (x[x.index('id="') + 4: x.index('" url=')], x[x.index('">') + 2:][:-6]))
	regex = re.compile('[^a-zA-Z]')
	t_keyAndListOfWords = t_keyAndText.map(lambda x: (str(x[0]), regex.sub(' ', x[1]).lower().split())).sortByKey(False)
	t_tuples = t_keyAndListOfWords.collect()
	t_key_id = [i for i in range(len(t_tuples))]
	t_key_values = {t_tuples[i][0]: i for i in t_key_id}
	t_feat = t_keyAndListOfWords.map(lambda x: (t_key_values[x[0]], TF(x[1], topWordsBC.value)))
	t_labels = t_keyAndListOfWords.map(
		lambda x: (t_key_values[x[0]], int(x[0][0] == 'A' and x[0][1] == 'U')))
	test_feat_df = sqlContext.createDataFrame(t_feat, ['ind', 'features'])
	test_labels_df = sqlContext.createDataFrame(t_labels, ['ind', 'labels'])
	test_df = test_feat_df.join(test_labels_df, on=['ind']).sort(['ind'])
	test_df.cache()

	st_test = time.time()
	test_pred = svg_model.evaluate(test_df).predictions
	evaluator = MulticlassClassificationEvaluator(labelCol='labels')

	print('Acc of Test: ', evaluator.evaluate(test_pred, {evaluator.metricName: "accuracy"}))
	print('F1 of Test:', evaluator.evaluate(test_pred, {evaluator.metricName: "f1"}))

	end5 = time.time()

	print(f"Computation time of SVM regression by BGD is {(end5 - start5)/60} minutes")


# coding: utf-8
import numpy as np
import matplotlib.pyplot as plt

X = 2 * np.random.rand(100, 1) # pass in desired shape. In this case, (100,1)
y = 4 + 3 * X + np.random.randn(100, 1)
# NOTE: I'm using two different random functions. rand() gives a uniform distribution, randn() gives a Gaussian distribution
plt.scatter(X, y, s=20)
plt.axis(ymin=0.0)
plt.show()

# Linear Regression

X_b = np.c_[np.ones((100,1)), X] # add the bias term to every instance
print(X.shape)
print(X_b.shape)

# Cost Function
#
# Before we start training our model, we need to define an error function to minimize.
# One of the most common cost functions used in regression is the Mean Squared Error, or MSE.

def MSE_error(theta):
    err = 0
    m = X.shape[0]
    for i in range(m):
        y_pred = theta.T.dot(X_b[i])[0]
        err += (y_pred - y[i])**2
    return err/m

# # Training Algorithm: Gradient Descent
#
# * https://en.wikipedia.org/wiki/Gradient_descent
# * Generic Optimization Algorithm - used in many domains outside of ML
#
# A common analogy to explain Gradient Descent is to image that you are standing on hilly terrain covered in fog.
# You want to find the deepest valley (minimum), but can only see your immediate surroundings.
# Your best strategy is to look at the steepness of the terrain at your feet. Find the direction of steepest descent,
# walk some distance in that direction, and repeat. Eventually you will find yourself in a valley.
#
# We are trying to find the parameter values that minimize our cost function (MSE). To do this, we approximate the derivative of
# the cost function with respect to our model parameters. When we have a multivariate (more than one input variable) function,
# we need to compute the *Gradient* of the function. The Gradient is the multivariable generalization of the derivative.
# It is a vector containing *partial derivatives.*
# A partial derivative is obtained by holding all other input variables constant, and differentiating our function with respect to just one variable.
# The Gradient, then, is a vector containing all the partial derivatives. In this case, our gradient vector will have two components.
#

learn_rate = 0.1
n_iter = 20
m = X.shape[0]

MSE_list = np.zeros(shape=(n_iter,))

theta = np.random.randn(2,1) # random initialization of theta
print('starting value of theta:\n', theta)

plt.scatter(X, y, s=20)
for iter in range(n_iter):
    gradients = 2/m * X_b.T.dot(X_b.dot(theta) - y) # compute the gradient
    theta = theta - learn_rate * gradients # update theta

    y_0 = theta.T.dot((1,0)) # should be roughly 4
    y_2 = theta.T.dot((1,2)) # should be roughly 10
    plt.plot([0,2], [y_0,y_2])
    MSE_list[iter] = MSE_error(theta)

plt.show()

print('optimized value of theta:\n', theta)

print('MSE error at each iteration:')
plt.plot(np.arange(n_iter), MSE_list)
plt.show()

# In[6]:

y_0 = theta.T.dot((1,0)) # should be roughly 4
y_2 = theta.T.dot((1,2)) # should be roughly 10

plt.scatter(X, y, s=20)
plt.plot([0,2], [y_0,y_2])
plt.show()


from math import ceil, sqrt

import numpy as np

def hoeffding_n_given_t_and_p_one_sided(t:np.double, p:np.double, C=0.5) -> int:
    """
    Return n such that with probability at least p, P(E[X] < \bar X_n + t).

    Where \bar X_n is the mean of n samples.

    Parameters
    ----------
    t : double
        one sided confidence interval width
    p : double
        probability of bound holding
    C : double
        Width of sample support domain. E.g. 0.5 if all samples fall in
            [0.5, 1.0]

    Returns
    -------

    """
    return int(ceil(C ** 2 * np.log(1 - p) / (-2 * t ** 2)))

def hoeffding_n_given_t_and_p_two_sided(t:np.double, p:np.double, C=0.5) -> int:
    """
    Return n such that with probability at least p, P(|E[X] - \bar X_n| <= t).

    Where \bar X_n is the mean of n samples.

    Parameters
    ----------
    t : double
        two sided confidence interval width
    p : double
        probability of bound holding
    C : double
        Width of sample support domain. E.g. 0.5 if all samples fall in
            [0.5, 1.0]

    Returns
    -------

    """
    return int(ceil(C ** 2 * np.log( 0.5*(1 - p) ) / (-2 * t ** 2)))

def chebyshev_k_from_upper_bound_prob(p_bound_holds:np.double) -> int:
    """
    Return k such with with probability at least p_bound_holds X will be < mu + k*sigma

    Parameters
    ----------
    p_bound_holds : double

    Returns
    -------

    """
    p_bound_violated = 1 - p_bound_holds
    return int(ceil(sqrt(1 / p_bound_violated)))

def accuracy_to_statistical_distance(accuracy):
    return (accuracy - 0.5) * 2

def statistical_distance_to_accuracy(statistical_distance):
    return 0.5 + 0.5 * statistical_distance


import numpy as np
from numpy.linalg import *

def leverage(X):
    """ compute leverage of a design matrix

    Parameters: X (nd-array) - the design matrix (with intercept)

    Returns: (array) - the leverage
    """
    # the orthogonal projection onto the column space of X
    H = np.dot(np.dot(X, np.linalg.inv(np.dot(X.T, X))), X.T)
    # diagonal of hat matrix, aka, leverage
    H_diag = H.diagonal()

    return H_diag

def internally_studentized(residuals, h, p, n):
    """ compute the Internally Studentized Residuals

    Parameters: residual (array-like) - the raw residuals
                h (array-like) - leverae of the design matrix
                p (int) - coefficients dimensions
                n (int) - total numbers of observations

    Returns: (array) - the studentized residual
    """
    sigma = np.sqrt((1/(n-p))*np.dot(residuals.T, residuals))
    standardized_residuals = residuals / (sigma*np.sqrt(1-h))
    return standardized_residuals

def cooks_distance(standard_residuals, h, p):
    """ compute cook's distance

    Parameters: standard_residuals (array) - studentized residauls
                h (array) - leverage
                p (int) - coefficients dimensions

    Returns: (array) - an array of Cook's distance
    """
    cooks = (h / (1-h)) * (standard_residuals**2 / p)
    return cooks

################### for t test ###################
def coef_standard_error(residuals, X):
    """ find standard errors of regression coefficients

    Parameters: residuals (array) - raw residuals
                X (nd-array) - design matrix, including intercept

    Returns: (nd-array) - the standard errors of regression coefficients
    """
    sse = np.dot(residuals.T, residuals) / (X.shape[0] - X.shape[1])
    se = np.sqrt(np.diagonal(sse * np.linalg.inv(np.dot(X.T, X))))
    return se

################### r2 & adj r2 ###################
def r_squared(y, residuals, p):
    """ compute r-squared and adjusted r-squared

    Parameters: y (array) - raw response
                residuals (array) - raw residuals
                p (int) - coefficients dimensions, including intercept

    Returns: r2, adj_r2 (float), (float)
    """
    n = len(y)
    sse = np.sum(residuals**2)
    sst = np.sum((y - np.mean(y))**2)
    r2 = 1 - (sse / sst)
    adj_r2 = 1 - ((sse / (n-p)) / (sst/(n-1)))
    return r2, adj_r2

################### for f test ###################
def f_stat(residuals, fitted_values, y, p):
    """ compute f statistics for testing regression model slope

    Parameters: residuals (array) - raw residuals
                fitted_values (array) - the model fitted values
                y (array) - raw response
                p (int) - coefficients dimesnion, including intercept

    Returns: (float) - the f statistics for the model
    """
    n = len(y)
    sse = np.sum(residuals**2)
    ssr = np.sum((fitted_values - np.mean(y))**2)
    f_stat = (ssr / (p-1)) / (sse / (n-p))
    return f_stat


import sympy as sym
# Mesh: |--------|-------|
#       0      pi/2      pi
#
# Basis functions:
#
#   phi_0   phi_1   phi_2
#     \      /\      /
#      \    /  \    /
#       \  /    \  /
#        \/      \/
#     |-------|-------|
#     0      pi/2     pi

x = sym.Symbol('x')
A = sym.zeros(3,3)
f = sym.sin

phi_0 = 1 - (2*x)/sym.pi
phi_1l = 2*x/sym.pi          # left part of phi_1
phi_1r = 2 - (2*x)/sym.pi    # right part of phi_1
phi_2 = x/(sym.pi/2) - 1
node_0 = 0
node_1 = sym.pi/2
node_2 = sym.pi

# Diagonal terms
A[0,0] = sym.integrate(phi_0**2,  (x, node_0, node_1))
A[1,1] = sym.integrate(phi_1l**2, (x, node_0, node_1)) + \
         sym.integrate(phi_1r**2, (x, node_1, node_2))
A[2,2] = sym.integrate(phi_2**2,  (x, node_1, node_2))

# Off-diagonal terms
A[0,1] = sym.integrate(phi_0*phi_1l, (x, node_0, node_1))
A[1,0] = A[0,1]

A[1,2] = sym.integrate(phi_1r*phi_2, (x, node_1, node_2))
A[2,1] = A[1,2]

print('A:\n', A)  # Can compare with general matrix, h=pi/2

b = sym.zeros(3,1)

b[0] = sym.integrate(phi_0*f(x),  (x, node_0, node_1))
b[1] = sym.integrate(phi_1l*f(x), (x, node_0, node_1)) + \
       sym.integrate(phi_1r*f(x), (x, node_1, node_2))
b[2] = sym.integrate(phi_2*f(x),  (x, node_1, node_2))

print('b:\n', b)

c = A.LUsolve(b)
print('c:\n', c)

for i in range(len(c)):
    print('c[%d]=%g' % (i, c[i].evalf()))
print('u(pi/2)=%g' % c[1])

# For reports
print(sym.latex(A))
print(sym.latex(b))
print(sym.latex(c))


##%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
##                    Feature Engineering (quadratic fit via linear model)
## %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
#%% read data
import numpy as np
data = np.loadtxt('quadratic_raw_data.csv', delimiter=',')
x = data[:,0,None]; y = data[:,1,None]

# plot
import matplotlib.pyplot as plt
plt.figure()
plt.plot(x,y, 'o')
plt.xlabel('x'), plt.ylabel('y')

#%% generate quadratic features
from sklearn.preprocessing import PolynomialFeatures

poly = PolynomialFeatures(degree=2, include_bias=False)
X_poly = poly.fit_transform(x) # X_poly: 1st column is x, 2nd column is x^2

#%% scale model inputs
from sklearn.preprocessing import StandardScaler

scaler = StandardScaler()
X_scaled = scaler.fit_transform(X_poly)

#%% linear fit & predict
from sklearn.linear_model import LinearRegression

model = LinearRegression()
model.fit(X_poly, y)
y_predicted = model.predict(X_poly)

#%% check predictions
# plot
plt.figure()
plt.plot(x,y, 'o', label='raw data')
plt.plot(x,y_predicted, label='quadratic fit')
plt.legend()
plt.xlabel('x'), plt.ylabel('y')

# accuracy
from sklearn.metrics import r2_score
print('Fit accuracy = ', r2_score(y, y_predicted))


from math import exp

import numpy as np

class Utils:
    @staticmethod
    def cast_int(s):
        try:
            return int(s)
        except:
            return None

    @staticmethod
    def cast_float(s):
        try:
            return float(s)
        except:
            return None

    @staticmethod
    def softmax(x):
        """
            https://medium.com/data-science-bootcamp/understand-the-softmax-function-in-minutes-f3a59641e86d
            Compute softmax values for each sets of scores in x.
        """
        return np.exp(x) / np.sum(np.exp(x), axis=0)

    @staticmethod
    def sigmoid(x):
        """
            https://machinelearningmastery.com/choose-an-activation-function-for-deep-learning/
            :param x:
            :return:
        """
        return 1.0 / (1.0 + exp(-x))

    @staticmethod
    def tanh(x):
        """
            https://machinelearningmastery.com/choose-an-activation-function-for-deep-learning/
            :param x:
            :return:
        """
        return (exp(x) - exp(-x)) / (exp(x) + exp(-x))


"""
  *** Hussain Salih Mahdi ***
  _________Shubbair__________

TODO Gradiant Descent find (m,b) from student degree @m = coefficient , @b = Intercept

inputs : csv file of student degree
output : predict the value should be close to the m an b

"""

import numpy as np
import pandas as pd
from sklearn import linear_model
import math

def sklearn_predict():
    data = pd.read_csv('input.csv')
    reg = linear_model.LinearRegression()
    reg.fit(data[['math']], data.cs)
    return reg.coef_, reg.intercept_

def gradiant_descent(x, y):
    m_curr = b_curr = 0
    iterations = 1000000
    learning_rate = 0.0002
    cost_previous = 0

    for i in range(iterations):
        py = m_curr * x + b_curr
        cost = (1 / len(x)) * sum([value ** 2 for value in (y - py)])
        md = -(2 / len(x)) * sum(x * (y - py))
        bd = -(2 / len(x)) * sum(y - py)
        m_curr = m_curr - learning_rate * md
        b_curr = b_curr - learning_rate * bd
        if math.isclose(cost, cost_previous, rel_tol=1e-20):
            break
        cost_previous = cost
        print("m {}, b {}, cost {}, iteration {}".format(m_curr, b_curr, cost, i))
    return m_curr, b_curr

if __name__ == "__main__":
    degree = pd.read_csv('input.csv')
    x = np.array(degree.math)
    y = np.array(degree.cs)

    m, b = gradiant_descent(x, y)
    print('Using Gradiant Descent : Coefficient {} , Intercept {}'.format(m, b))

    sklearn_m, sklearn_b = sklearn_predict()
    print('Using Sklearn : Coefficient {} , Intercept {}'.format(sklearn_m, sklearn_b))


import numpy as np
# DO NOT add any library

def train_transformation(X, Y,R, train_steps=100, learning_rate=0.0003):
    #please write your code here
    # find best R using Gradient Descent
    #WRITE your code here
    m = len(Y)
    prediction = X.dot(R)
    for _ in range(train_steps):
        R = R - np.transpose((learning_rate / m * (np.transpose(prediction - Y))).dot(X))

    #END of your code
    return R

def nearest_neighbor(v, candidates, k=1):
    # find k best similar vectors index. please sort them in order max to min and return index
    # for your similarity function please use cosine similarity
    similarity_l = []
    #WRITE your code here
    for candidate in candidates:
        ss = v.dot(np.transpose(candidate)) / np.sqrt(v.dot(np.transpose(v)) * candidate.dot(np.transpose(candidate)))
        similarity_l = np.append(similarity_l, ss)
    #END of your code
    sorted_ids = np.argsort(similarity_l)

    k_idx = sorted_ids[-k:]
    return k_idx

# This is how this code is called
# X, y = np.load('X_train.npy'), np.load('Y_train.npy')
# print(train_transformation(X, y, np.random.randn(300,300)))


import numpy as np
from math import tau
from scipy.special import binom

import matplotlib.pyplot as plt

def perimeter_bessel(a, b=1, k=10):
    h = (a-b)**2 / (a+b)**2
    return np.pi * (a + b) * [(binom(0.5, range(k)) ** 2 * v**np.arange(k)).sum() for v in h]

def perimeter_parker(a, b=1):
    return 3 * tau * (a / 5 + b / 8)

def perimeter_jcop(a, b=1):
    return tau * (7/45 * a + 27/41 * np.sqrt((a**2 + b**2)/2) + 7/37 * b)

def main():
    space = np.linspace(1, 75, 500)

    parker = perimeter_parker(space)
    bessel = perimeter_bessel(space)
    jcop = perimeter_jcop(space)

    err_parker = np.abs(1 - parker / bessel) * 100
    err_jcop = np.abs(1 - jcop / bessel) * 100

    plt.figure(figsize=(12, 4))
    plt.subplot(121)
    plt.plot(space[:40], err_parker[:40], "k-", label="parker")
    plt.plot(space[:40], err_jcop[:40], "m-", label="jcop")
    plt.legend()

    plt.subplot(122)
    plt.plot(space, err_parker, "k-", label="parker")
    plt.plot(space, err_jcop, "m-", label="jcop")
    plt.legend()

    plt.tight_layout()
    plt.show()

if __name__ == "__main__":
    main()


#########################
##                     ##
## Irving Gomez Mendez ##
##  February 12, 2021  ##
##                     ##
#########################

import numpy as np
import matplotlib.pyplot as plt
from scipy.stats import t

# The general formula of a ellipse is given by
# x^2 + By^2 + Cxy + Dx + Ey + F = 0
# i.e. By^2 + Cxy + Dx + Ey + F = -x^2

# We can see this problem as finding the function
# f(x,y) = x^2 + By^2 + Cxy + Dx + Ey + F
# which minimizes the data:
# x  | y  | f(x,y)
# x1 | y1 | 0
# x2 | y2 | 0

points = np.array([[0,2],[2,1],[1,-1],[-1,-2],[-3,1],[-3,-2]])

points_x = points[:,0]
points_y = points[:,1]
points_x2 = points_x**2
points_y2 = points_y**2
points_xy = points_x*points_y

n = points.shape[0]
yy = -points_x2
XX = np.vstack([points_y2, points_xy, points_x, points_y, np.ones(len(points_x))]).T
p = XX.shape[1]

# We get the coeff.
B, C, D, E, F = np.linalg.lstsq(XX, yy, rcond=None)[0]
SSE = np.linalg.lstsq(XX, yy, rcond=None)[1]

# We plot the ellipse
x = np.linspace(-4.5, 2.5, 50)
y = np.linspace(-2.5, 2.5, 50)
x, y = np.meshgrid(x, y)

plt.plot(points_x, points_y, 'o', markersize=7)
CS = plt.contour(x, y, (x**2+B*y**2+C*x*y+D*x+E*y+F), [0], colors='purple')
plt.title('Best fit ellipse by least squares')

# These are the points on the ellipse
points_ellipse = CS.allsegs[0][0]

ellipse_x = points_ellipse[:,0]
ellipse_y = points_ellipse[:,1]
ellipse_y2 = ellipse_y**2
ellipse_x2 = ellipse_x**2
ellipse_xy = ellipse_x*ellipse_y

ellipse_matrix = np.vstack([ellipse_y2, ellipse_xy, ellipse_x, ellipse_y, np.ones(len(ellipse_x))]).T

# Confidence interval
# Remember that the points on the adjusted ellipse satisfied: \hat{f}(x,y)=0
# So the interval is just given by +- t_{alpha/2,n-p}\sqrt{SSE/(n-p)*X_0(X^TX)^{-1}X_0^T}
# which represent the level on which the conic surface should be cut.

alpha = 0.05
aux_t = np.sqrt(SSE/(n-p)*(np.diag(ellipse_matrix @ np.linalg.inv(XX.T @ XX) @ ellipse_matrix.T)))
upp_contour = np.mean(t.ppf(1-alpha/2,1)*aux_t)
low_contour = np.mean(-t.ppf(1-alpha/2,1)*aux_t)

x = np.linspace(-5, 3, 50)
y = np.linspace(-3, 3, 50)
x, y = np.meshgrid(x, y)

plt.plot(points_x, points_y, 'o', markersize=7)
CS = plt.contour(x, y, (x**2+B*y**2+C*x*y+D*x+E*y+F), [low_contour, 0, upp_contour], colors=['red', 'purple', 'green'])
plt.title('Best fit ellipse by least squares')


import numpy as np
import pytest

from galois_field.core import gcd

@pytest.mark.parametrize('poly1, poly2, p, expected', [
    (np.poly1d([1, 0, 1]), np.poly1d([1, 2]), 5, np.poly1d([1, 2])),
    (np.poly1d([1, 0, 1, 1]), np.poly1d([1, 2]), 5, np.poly1d([1])),
    (np.poly1d([1, 0, 1]), np.poly1d([1, 4, 2, 1]), 5, np.poly1d([1, 2])),
    (np.poly1d([1, 0, 0, 0, 0, 0, 1, 1]),
     np.poly1d([1, 0, 0, 0, 4, 0]), 5, np.poly1d([2]))
])
def test_gcd_poly(poly1, poly2, p, expected):
    assert gcd.gcd_poly(poly1, poly2, p) == expected


# For this part of the assignment, please implement your own code for all computations,
# Do not use inbuilt functions like fft from either numpy, opencv or other libraries
import math
import numpy as np

class DFT:

    def forward_transform(self, matrix):
        """Computes the forward Fourier transform of the input matrix
        takes as input:
        matrix: a 2d matrix
        returns a complex matrix representing fourier transform"""

        # Using direct function
        # matrix1 = np.fft.fft2(matrix)

        # Without using direct function
        rows, columns = np.shape(matrix)
        matrix1 = np.zeros((rows, columns), dtype=complex)
        for u in range(rows):
            for v in range(columns):
                a = []
                for i in range(rows):
                    for j in range(columns):
                        omega = np.exp(-2 * math.pi * 1J * (((u * i) / rows) + ((v * j) / columns)))
                        a.append(matrix[i, j] * omega)
                matrix1[u, v] = sum(a)

        return matrix1

    def inverse_transform(self, matrix):
        """Computes the inverse Fourier transform of the input matrix
        matrix: a 2d matrix (DFT) usually complex
        takes as input:
        returns a complex matrix representing the inverse fourier transform"""

        # Using direct function
        # matrix1 = np.fft.ifft2(matrix)

        # Without using direct function
        rows, columns = np.shape(matrix)
        matrix1 = np.zeros((rows, columns), dtype=complex)
        for u in range(rows):
            for v in range(columns):
                a = []
                for i in range(rows):
                    for j in range(columns):
                        omega = np.exp(2 * math.pi * 1J * (((u * i) / rows) + ((v * j) / columns)))
                        a.append(matrix[i, j] * omega)
                matrix1[u, v] = sum(a)
                # Normalized value of inverse dft
                # matrix1[u, v] = (1 / (rows * columns)) * sum(a)

        return matrix1

    def discrete_cosine_tranform(self, matrix):
        """Computes the discrete cosine transform of the input matrix
        takes as input:
        matrix: a 2d matrix
        returns a matrix representing discrete cosine transform"""

        rows, columns = np.shape(matrix)
        matrix1 = np.zeros((rows, columns), dtype=float)
        for u in range(rows):
            for v in range(columns):
                a = []
                for i in range(rows):
                    for j in range(columns):
                        temp = math.cos(2 * math.pi * (((u * i) / rows) + ((v * j) / columns)))
                        a.append(matrix[i, j] * temp)
                matrix1[u, v] = sum(a)

        return matrix1

    def discrete_sine_tranform(self, matrix):
        """Computes the discrete cosine transform of the input matrix
        takes as input:
        matrix: a 2d matrix
        returns a matrix representing discrete cosine transform"""

        rows, columns = np.shape(matrix)
        matrix1 = np.zeros((rows, columns), dtype=float)
        for u in range(rows):
            for v in range(columns):
                a = []
                for i in range(rows):
                    for j in range(columns):
                        temp = math.sin(2 * math.pi * (((u * i) / rows) + ((v * j) / columns)))
                        a.append(matrix[i, j] * temp)
                matrix1[u, v] = sum(a)

        return matrix1

    def magnitude(self, matrix):
        """Computes the magnitude of the DFT
        takes as input:
        matrix: a 2d matrix
        returns a matrix representing magnitude of the dft"""

        # method 1
        # matrix1 = np.abs(matrix)

        # method 2
        rows, columns = np.shape(matrix)
        matrix1 = np.zeros((rows, columns), dtype=float)
        for i in range(rows):
            for j in range(columns):
                # matrix1[i, j] = abs(matrix[i, j])
                matrix1[i, j] = math.sqrt(matrix[i, j].real ** 2 + matrix[i, j].imag ** 2)

        return matrix1


import numpy as np
from matplotlib import pyplot as plt
import math

def f1(a0,a1,x):
    return (a0+(a1*x))
def f2(a0,a1,a2,x):
    return (a0+(a1*x)+(a2*x*x))
def f3(a0,a1,a2,a3,x):
    return (a0+(a1*x)+(a2*x*x)+(a3*x*x*x))

inpo=open('data.txt','r').readlines()
n=len(inpo)
x=[0 for i in range(n)]
y=[0 for i in range(n)]
for i in range(n):
    spl=inpo[i].split(' ')
    x[i]=float(spl[0])
    y[i]=float(spl[1])
plt.scatter(x,y,s=3)
plt.xlabel('X Label')
plt.ylabel('Y label')
plt.legend()
plt.grid()
#deg 1
sumx=0
for i in range(n):
    sumx=sumx+x[i]

sumy=0
for i in range(n):
    sumy=sumy+y[i]
sumx2=0
for i in range(n):
    sumx2=sumx2+(x[i]*x[i])
sumy2=0
for i in range(n):
    sumy2=sumy2+(y[i]*y[i])
sumxy=0
for i in range(n):
    sumxy=sumxy+(x[i]*y[i])
a1=((n*sumxy)-(sumx*sumy))/((n*sumx2)-(sumx*sumx))
ybar=sumy/n
xbar=sumx/n
a0=ybar-(a1*xbar)
r=((n*sumxy)-(sumx*sumy))/(math.sqrt((n*sumx2)-(sumx*sumx))*math.sqrt((n*sumy2)-(sumy*sumy)))
print('for order 1')
print('a0=',a0)
print('a1=',a1)
print('r=',r)
#order 2
sumx3=0
for i in range(n):
    sumx3=sumx3+(x[i]*x[i]*x[i])
sumx4=0
for i in range(n):
    sumx4=sumx4+(x[i]*x[i]*x[i]*x[i])

meta= [[0 for i in range(3)] for j in range(3)]
metb=[0 for i in range(3)]
for i in range(3):
    for j in range(3):
        if i==0 and j==0:
            meta[i][j]=n
        else :
            k=i+j
            if k==1:
                meta[i][j]=sumx
            elif k==2:
                meta[i][j]=sumx2
            elif k==3:
                meta[i][j]=sumx3
            elif k==4:
                meta[i][j]=sumx4
sumx2y=0
for i in range(n):
    sumx2y=sumx2y+(x[i]*x[i]*y[i])
metb[0]=sumy
metb[1]=sumxy
metb[2]=sumx2y
#print(meta)
aord2=np.linalg.solve(meta,metb)
st=0
sr=0
for i in range (n):
    sr=sr+((y[i]-f2(aord2[0],aord2[1],aord2[2],x[i]))*(y[i]-f2(aord2[0],aord2[1],aord2[2],x[i])))
ybar=sumy/n
for i in range(n):
    st=st+((ybar-y[i])*(ybar-y[i]))
r=math.sqrt((st-sr)/st)
print('For order 2')
print('a0=',aord2[0])
print('a1=',aord2[1])
print('a2=',aord2[2])
print('r=',r)

#deg3
meta1= [[0 for i in range(4)] for j in range(4)]
metb1=[0 for i in range(4)]
sumx5=0
for i in range(n):
    sumx5=sumx5+(x[i]*x[i]*x[i]*x[i]*x[i])
sumx6=0
for i in range(n):
    sumx6=sumx6+(x[i]*x[i]*x[i]*x[i]*x[i]*x[i])

for i in range(4):
    for j in range(4):
        if i==0 and j==0:
            meta1[i][j]=n
        else :
            k=i+j
            if k==1:
                meta1[i][j]=sumx
            elif k==2:
                meta1[i][j]=sumx2
            elif k==3:
                meta1[i][j]=sumx3
            elif k==4:
                meta1[i][j]=sumx4
            elif k==5:
                meta1[i][j]=sumx5
            elif k==6:
                meta1[i][j]=sumx6
sumx3y=0
for i in range(n):
    sumx3y=sumx3y+(x[i]*x[i]*x[i]*y[i])
metb1[0]=sumy
metb1[1]=sumxy
metb1[2]=sumx2y
metb1[3]=sumx3y
aord3=np.linalg.solve(meta1,metb1)
#print(meta1)
st=0
sr=0
for i in range (n):
    sr=sr+((y[i]-f3(aord3[0],aord3[1],aord3[2],aord3[3],x[i]))*(y[i]-f3(aord2[0],aord2[1],aord2[2],aord3[3],x[i])))
ybar=sumy/n
for i in range(n):
    st=st+((ybar-y[i])*(ybar-y[i]))
r=math.sqrt((st-sr)/st)
print('For order 3')
print('a0=',aord3[0])
print('a1=',aord3[1])
print('a2=',aord3[2])
print('a3=',aord3[3])
print('r=',r)
z1=x
for i in range(n):
    for j in range(i+1,n):
        if z1[j]<z1[i]:
            t=z1[i]
            z1[i]=z1[j]
            z1[j]=t
z2=[0 for i in range(n)]
for i in range(n):
    z2[i]=f1(a0,a1,z1[i])
plt.plot(z1,z2,label='order 1 curve',color='red',linewidth=3)
plt.legend()
plt.grid()

z3=[0 for i in range(n)]
for i in range(n):
    z3[i]=f2(aord2[0],aord2[1],aord2[2],z1[i])
plt.plot(z1,z3,label='order 2 curve',color='black',linewidth=3)
plt.legend()
plt.grid()

z4=[0 for i in range(n)]
for i in range(n):
    z4[i]=f3(aord3[0],aord3[1],aord3[2],aord3[3],z1[i])
plt.plot(z1,z4,label='order 3 curve',color='purple',linewidth=3)
plt.legend()
plt.grid(True)
plt.title('this is graph of different order curve')


"""
Exercise4
Find the length of a curve between two points
"""

from sympy import Derivative, Integral, Symbol, sqrt, SympifyError, sympify

def find_length(fx, var, a, b):
    deriv = Derivative(fx, var).doit()
    return Integral(sqrt(1 + deriv ** 2), (var, a, b)).doit().evalf()

def validate_function(function):

    try:
        return sympify(function)
    except SympifyError:
        print("Invalid function entered")
        sys.exit(1)

    return None

if __name__ == "__main__":
    function = input("Provide a function of one variable: ")
    function = validate_function(function)

    variable = input("Enter the variable to differentiate with respect to: ")
    variable = Symbol(variable)

    lower_bound = float(input("Enter the lower limit of the variable: "))
    upper_bound = float(input("Enter the upper limit of the variable: "))

    print(
        "Length of {} between {} and {} is {} ".format(
            function,
            lower_bound,
            upper_bound,
            find_length(function, variable, lower_bound, upper_bound),
        )
    )


import numpy as np
import control
import matplotlib.pyplot as plt

def predict_lti(b, a, y, u, k=1, d=0):
    """
    Returns the k-step ahead prediction for the LTI with pulse-transfer function
       H(z) = B(z)/A(z)
    The coefficients of the numerator polynomial is contained in the array b, and those of
    the denominator polynomial in the array a.

    Input
       b  ->  numerator coefficients (1 x m) row vector
       a  ->  denominator coefficients (1 x n) row vector
       u  ->  input data (N x 1) column vector
       y  ->  output data (N x 1) column vector
       k  ->  prediction horizon (integer > 0)
       d  ->  input delay (integer >= 0)

    Output
       ypredk  <-  predicted output ((N-k) x 1) column vector
       tk      <-  for convenience, discrete time vector for predicted values
                   tk = [1+k, 2+k, ..., N]

    """

    # Kjartan Halvorsen
    # 2019-10-03

    # Adjust coefficient arrays in case of input delay
    aa = np.concatenate( (a, np.zeros(d)), axis=None)
    bb = np.concatenate( (np.zeros(len(aa)-len(b)), b), axis=None)

    n = len(aa) - 1 # System order
    N = len(u)      # Length of data

    print("Order n=%d" %n)
    print("Data points N=%d" %N)

    yy = np.concatenate( (np.zeros(n), y), axis=None ) # Prepad with initial values
    uu = np.concatenate( (np.zeros(n), u), axis=None )  # Prepad with initial values

    tk = np.arange(k, N)

    ypredk = np.zeros(len(tk))
    for i in range(N-k):
        ysim = yy[i:i+n+k+1] # First n are initial values, last k will be overwritten
        usim = uu[i:i+n+k+1]
        for l in range(k):
            ysim[n+l+1] = - np.dot(aa[1:], ysim[n+l:l:-1])
            ysim[n+l+1] += np.dot(bb, usim[n+l+1:l:-1])
        ypredk[i] = ysim[-1]

    return (ypredk, tk)

def predict_unit_test():
    # Generate some data and test method
    b = np.array([0, 0, 1,0.9])
    a = np.array([1, -np.sqrt(3), 1, 0])

    N = 50
    u = np.random.randn(N)
    t = np.arange(N)
    (t, y, x) = control.forced_response( control.tf(b,a,1.0), t, u)
    y = np.ravel(y)

    k = 10;
    (ypred, tpred) = predict_lti(b[2:], a[:-1], y,  u, k, 1);

    plt.figure
    plt.plot(tpred, ypred, 'rx')
    plt.plot(t, y, 'bo')

    return (ypred, tpred)


'''
This Code transforms a matrix into Row Reduced Echelon Form using the Row Echelon Form of a matrix
Stage : Developmental
Many test cases are checked, yet to check some corner cases. If you notice a corner case that gives
error, please create a branch and update accordingly
or mail me at: de.arkadipta05@gmail.com, ai20mtech14002@iith.ac.in , arkadipta.de@gcettb.ac.in

Author: Arkadipta De
'''
import numpy as np

def Row_Echelon(A):
  '''
  Function to transform a matrix into Row-Echelon Form
  '''
  row = A.shape[0]
  column = A.shape[1]

  if row==0 or column==0:
      return A
  for i in range(len(A)) :
      if A[i,0] !=0 :
          break
  else:
      A_Below = Row_Echelon(A[:,1:])
      return np.hstack([A[:,:1],A_Below])

  if i>0:
      ith_row = A[i].copy()
      A[i] = A[0]
      A[0] = ith_row

  pivot = A[0,0]

  A[0] = A[0]/pivot                 #Normalize Pivot's Row
  A[1:] = A[1:] - A[0]*A[1:,0:1]    #Make Pivot Column 0

  A_Below = Row_Echelon(A[1:,1:])
  A_Stacked = np.hstack([A[1:,:1],A_Below])
  return np.vstack([A[:1],A_Stacked])

def Reduced_Row_Echelon_Form(A):
  '''
  Function to transform a matrix into Reduced-Row-Echelon Form
  '''
  A = Row_Echelon(A)
  row = A.shape[0]-1
  col = A.shape[1]
  while (row > 0):
      i = row
      for j in range(col):
        if A[i,j]==1:
          while (i > 0):
            A[i-1] -= A[row]*A[i-1,j]
            i-=1
      row-=1
  return A

def Inverse(A):
  '''
  Function to Extract Inverse Matrix from Augmented RRE of a Matrix
  Input: Augmented RRE Form of a Matrix
  '''
  row = A.shape[0]
  col = A.shape[1]
  if (row!=col):
    print("\nInverse Does not exist")
  elif np.linalg.det(A)==0:
    print("\nInverse doesn't exist.")
  else:
    AI = np.hstack([A,np.identity(row)])
    AI_RREF = Reduced_Row_Echelon_Form(AI)
    AI_row = AI_RREF.shape[0]
    AI_col = AI_RREF.shape[1]
    return AI_RREF[:,int(AI_col/2):]

# Driver Code
A1 = np.array([[1,0,0],[0,1,0],[0,0,1]])
A2 = np.array([[0,0,1],[1,0,0],[0,1,0]])
A3 = np.array([[0,0,1,5],[0,1,2,1],[0,1,2,1]])
A4 = np.array([[1,3,1],[0,2,3],[34,-2,-1]])
A5 = np.array([[1,2,2],[-2,9,17],[1,0,0],[0,1,0]])
A6 = np.array([[1,1,1,1,1.0],[1,1,1,1,1],[1,1,1,1,1]])
A7 = np.array([[1,1,1,1,1.0],[1,1,2,2,2],[1,1,1,4,5]])
A8 = np.array([[1,1,1,1],[1,1,1,1],[1,2,3,4]])
A9 = np.array([[1,1,1],[1,1,2],[2,3,1]])

A = A5

RE = Row_Echelon(A)
print('Row Echelon Form = \n',RE)
RRE = Reduced_Row_Echelon_Form(A)
print('\nReduced Row Echelon Form = \n',RRE)
print("Inverse = \n",Inverse(A))


# Write a program called plottask.py that displays a plot of the functions f(x)=x, g(x)=x^2 and h(x)=x^3 in the range [0, 4] on the one set of axes.
# Author : Conor McCaffrey

import numpy as np, matplotlib.pyplot as plt   # importing libraries and assiging aliases

x = np.arange(0.0,4.0,1.0)   # setting the range as described in task and in RealPython reference

y1 = x   # plotting the points on y-axis
y2 = x**2 #plotting the points on y-axis
y3 = x**3 #plotting the points on y-axis

font = {'family': 'serif', 'color':'darkred','weight': 'normal', 'size':20} # adding in fonts
font1 = {'family':'serif','color':'navy','size':10}   # adding in fonts

plt.plot(y1, color='#575daa', ls='-.', lw = 2,
         marker='s', ms = '5', mec='black', mfc='hotpink', label='f(x)=x')  # plotting each point and making the plot pretty
plt.plot(y2, color='#57de5a', ls='-.', lw = 2,
         marker='o', ms ='5', mec='black', mfc = 'hotpink', label='g(x)=x^2')  # plotting each point and making the plot pretty
plt.plot(y3, color='#ff0202', ls='-.', lw = 2,
         marker='o', ms ='5', mec='black', mfc = 'hotpink', label='h(x)=x^3')  # plotting each point and making the plot pretty

plt.xlabel('x-axis', fontdict=font1)  # adding labels and setting the font
plt.ylabel('y-axis', fontdict=font1)   # adding labels and setting the font
plt.title('Plot Task', fontdict=font, loc = 'left')  # adding a title and properties
plt.legend()   # adding a legend for clarity
plt.grid(axis = 'y', color = 'green' , lw = 0.1) # adding a y=axis grid and setting properties
#plt.show()   # displaying the graph
plt.savefig('plottask.png')   # adding code here that would be used to save the file if we did not want to show it


# coding: utf-8

# # Linear Regression
#
# In this tutorial we will implement a linear regression model. We will also implement a function that splits the available data into a training and a testting part.
#
# ## Problem Setting
#
# We will use the Boston Housing Dataset. This dataset contains information collected by the U.S Census Service concerning housing in the city of Boston in the state of Massachusetts in 1978. Our goal is to predict the median value of the houses in a particular town in the city of Boston given its attributes. Check the file ’housing.names’ for more information on the attributes.

# In[ ]:

import urllib
import pandas as pd
import numpy as np
# for auto-reloading external modules
# see http://stackoverflow.com/questions/1907993/autoreload-of-modules-in-ipython
get_ipython().magic(u'load_ext autoreload')
get_ipython().magic(u'autoreload 2')

from sklearn.datasets import load_boston
boston=load_boston()
testfile = urllib.URLopener()
testfile.retrieve("https://archive.ics.uci.edu/ml/machine-learning-databases/housing/housing.names", "housing.names")
df=pd.DataFrame(boston.data)
df.columns=['crime_rate','res_land_zoned','industry','charles_river','nox','avg_num_rooms','prop_bf_1940','dst_emply_center','rd_highway_idx','tax_rate','stdnt_tchr_ratio','prop_blacks','low_status_pct']
X=boston.data
y=boston.target

# In[ ]:

df.head(10)

# ### Exercise 1
#
# Write the *split_train_test(X,y,split,seed)*, given an instance matrix $X \in \mathbb{R}^{N \times D}$, labels $y \in Y^N$, a split ratio in $[0, 1]$ and a random seed $\in \mathbb{Z}$. Split the dataset in $(split×100)\%$ of the instances for training our model and the rest for testing, i.e.
#
# $$ \left|X_{\text{train}}\right| = \lceil \text{split} \cdot N \rceil, \qquad |X_{\text{train}}| + |X_{\text{test}}| = N. $$
# Make sure you use the given random number generator seed so we all get the same results. The function is supposed to return:
#
# - X_train, y_train: the training instances and labels;
# - X_test, y_test: the test instances and labels,
#
# in the same order as was mentioned.
#
# Hint: It may be helpful to use shuffling functionality (e.g. np.random.shuffle).

# In[ ]:

def split_train_test(X,y,split,seed):
    ##################
    #INSERT CODE HERE#
    ##################
    return None # X_train, y_train, X_test, y_test

# ### Exercise 2
#
# Write the function *train_linear_reg(X_train,y_train,lmbd)*.
# Implement the ridge regression model (slide 24). The function should output the learned weight vector $\theta \in \mathbb{R}^D$ or $\mathbb{R}^{D+1}$ (depending on whether you are adding *bias*).

# In[ ]:

def train_linear_reg(X, y, lmbd):
    ##################
    #INSERT CODE HERE#
    ##################
    return None # theta

# ### Exercise 3
#
# Write the function *predict(X,theta)* which predicts housing values vector pred for a dataset X and a previously trained parameter vector $\theta$.

# In[ ]:

def predict(X, theta):
    ##################
    #INSERT CODE HERE#
    ##################
    return None # y_pred

# ### Exercise 4
#
# Write the function *mean_abs_loss(y_true,y_pred)* which computes the mean of the absolute differences between our prediction vector $y\_pred$ and the real housing values $y\_true$.

# In[ ]:

def mean_abs_loss(y_true,y_pred):
    ##################
    #INSERT CODE HERE#
    ##################
    return 0

# ### Exercise 5
#
# Evaluate your solutions by running the following code.
#
# Moreover, answer the following questions: What is the most important feature in your model? Are there features that are not so important? What happens if you remove them? Are there outliers with a high absolute loss?

# In[ ]:

seed = 3
lmbd=1
split=0.7
X_train,y_train,X_test,y_test=split_train_test(X,y,split,seed)
theta=train_linear_reg(X_train,y_train,lmbd)
y_pred=predict(X_test,theta)
mae=mean_abs_loss(y_test,y_pred)
print 'The mean absolute loss is {loss:0.3f}'.format(loss=mae*1000)


"""
Functions for measurements and building bond lists.
"""

import numpy as np

# example of numpy style docstring
# pytest can run examples section of docstring
# $ pytest -v --doctest-modules "dir"
def calculate_distance(rA, rB):
    """
    Calculate the distance between two points.

    Parameters
    ----------
    rA, rB : np.ndarray
        The coordinates of each point.

    Returns
    -------
    distance : float
        The distance between the two points.

    Examples
    --------
    >>> rA = np.array([0.0, 0.0, 0.0])
    >>> rB = np.array([0.0, 0.0, 1.0])
    >>> calculate_distance(rA, rB)
    1.0

    """
    # can access doc string with
        # >>> help(molecool.calculate_distance)
        # >>> molecool.calculate_distance.__doc__
        # >>> print(molecool.calculate_distance.__doc__)

    # This function calculates the distance between two points given as numpy arrays.

    #if isinstance(rA, np.ndarray) is False or isinstance(rB, np.ndarray is False):
     #   raise TypeError("rA abd rB must be numpy arrays")

    d=(rA-rB)
    dist=np.linalg.norm(d)

    """
    if dist == 0.0:
        raise Exception("Two atoms are located at the same point")
    """

    return dist

def calculate_angle(rA, rB, rC, degrees=False):
    """
    Calculate the angle between three points.
    Answer is given in radians by default, but can be given in degrees
    by setting degrees=True.

    Parameters
    ----------
    rA, rB, rC : np.ndarray
    degrees : True/False

    Returns
    -------
    theta : float
        The angle between three points.

    """
    # Calculate the angle between three points. Answer is given in radians by default, but can be given in degrees
    # by setting degrees=True
    AB = rB - rA
    BC = rB - rC
    theta=np.arccos(np.dot(AB, BC)/(np.linalg.norm(AB)*np.linalg.norm(BC)))

    if degrees:
        return np.degrees(theta)
    else:
        return theta


import numpy as np
from math import *

# Global variables
NMAX = 20
TOLERANCE = 0.00001

def fixedPoint(p0,  g, test):
    iteri = 0
    while iteri < NMAX:
        p = g(p0)
        if not test:
            print("p = \t" + str(p))
        error = abs(p0 - p)
        if error < TOLERANCE:
            break
        p0 = p
        if not test:
            print("p0 = \t" + str(p0))
            print()
        iteri += 1

    # Salida
    if iteri > NMAX:
        print("The method does not converge")
    else:
        if not test:
            print("The point of the function g(x) is: \t" + str(p))
        else:
            return p

def g_test_1(x):
    return (2 - (np.exp(x)) + x**2) / 3

if __name__ == "__main__":
    p0 = 0
    fixedPoint(p0, g_test_1, False)

# Tests
def test_1():
    p0 = 0
    assert fixedPoint(p0, g_test_1, True) == 0.25753180626753985


import numpy as np

def softmax(x, axis=-1):
    e_x = np.exp(x - np.max(x, axis=axis, keepdims=True))
    return e_x / np.sum(e_x, axis=axis, keepdims=True)

def sigmoid(x):
    return 1 / (1 + np.exp(-x))


'''
description of sigma(h) state function which is a piecewise linear function with the slopes k and lambda'
'''

import numpy as np

def s(lmbd, k, h):
    bounds = (-1.0 / lmbd, 1.0 / lmbd)
    c = (1 - k / lmbd) #
    x = 1.0 * (h <= bounds[0])
    z = 1.0 * (h >= bounds[1])
    y = (np.ones_like(h) - x - z)
    return (k * h - c) * x + (lmbd * h) * y + (k * h + c) * z

def der_s(lmbd, k, h):
    bounds = (-1.0 / lmbd, 1.0 / lmbd)
    x = 1.0 * (h <= bounds[0])
    z = 1.0 * (h >= bounds[1])
    y = (np.ones_like(h) - x - z)
    return k * (x + z) + lmbd * y

def inv_s(lmbd, k, s):
    x = 1.0 * (s <= -1)
    z = 1.0 * (s >= 1)
    y = (np.ones_like(s) - x - z)
    c = (1 - k / lmbd)
    return ((s + c) / k) * x + (s / lmbd) * y + ((s - c) / k) * z

if __name__ == '__main__':
    from matplotlib import pyplot as plt
    fig = plt.figure(figsize = (10, 3))
    x = np.linspace(-10,10, 100)
    plt.grid(True)
    plt.plot(x, s(0.5, 0.1, x))
    plt.xlabel("x")
    plt.ylabel("s(lmbd, k, x)")
    # plt.plot(x, inv_s(0.5, 0.1, s(0.5, 0.1, x)))
    plt.show()


import numpy as np

def compute_euclidean_distance(point1, point2):
    """
    Compute Euclidean distance(s) between points.

    Args:
        point1: Left-hand side `numpy.ndarray`.
        point2: Right-hand side `numpy.ndarray`.

    Returns:
        The computed Euclidean distance(s).
    """

    return np.sqrt(np.sum(np.square(point1 - point2), axis=0, keepdims=True))

def compute_gaussian(distance, sigma):
    """
    Compute Gaussian with given `distance` and `sigma`.

    Args:
        distance: Euclidean distance.
        sigma: Gaussian standard deviation.

    Returns:
        The computed Gaussian value.
    """

    return np.exp(-0.5 * np.square(distance / sigma)) / (sigma * np.sqrt(2 * np.pi))


# -*- coding: utf-8 -*-
"""hw1_310555029.ipynb

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/1JRZxfH5avZg0896ddvxZSfm2lUtMGKSu

#Linear Regression

##(0) Load the data  -> Standardize data -> Split the dataset
"""

import numpy as np
import pandas as pd

data_x_df = pd.read_csv('data_X.csv')
data_t_df = pd.read_csv('data_T.csv')

X = np.array(data_x_df)
T = np.array(data_t_df)

# standardize
X = (X-X.mean(axis=0))/X.std(axis=0)
#T = (T-T.mean(axis=0))/T.std(axis=0)

# 80% training dataset and 20% test dataset
X_SIZE = X.shape[0]
ratio = 8/10
trainRange = int(X_SIZE*ratio) #num of training data
x_train = X[:trainRange]
x_test = X[trainRange:]
y_train = T[:trainRange]
y_test = T[trainRange:]

print("x_train shape {} and size {}".format(x_train.shape,x_train.size))
print("y_train shape {} and size {}".format(y_train.shape,y_train.size))
print("x_test shape {} and size {}".format(x_test.shape,x_test.size))
print("y_test shape {} and size {}".format(y_test.shape,y_test.size))

"""##(1) Feature selection

### Define model
"""

class LinearRegression:
    def __init__(self,x,M):
      self.M = M
      self.size = x.shape[0]
      self.feature_num = x.shape[1]
      self.weight = None

    def closed_form(self, x, y, _lambda=0, regularize = False):
      if not regularize:
      # Using Normal Equation to find w = (xT x)^(-1) xT y
        xTx = np.dot(x.T, x)
        inv = np.linalg.inv(np.matrix(xTx))
        w = np.dot(np.dot(inv, x.T), y)
        self.weight = w
      else:
      # w = (lambdaI + xT x)^(-1) xT y
        xTx = np.dot(x.T, x)
        inv = np.linalg.inv(np.eye(xTx.shape[0]) * _lambda + xTx)
        w = np.dot(np.dot(inv, x.T), y)
        self.weight = w

    def predict(self, x):
      pre = np.dot(x, self.weight)
      return pre

    def poly_features(self,x):
      #PolynomialFeatures
      # find the phi matrix
      # array_new = [[1 x1 x1^2 ...]
      #        [1 x2 x1^2 ...]
      #          .
      #          .
      #        [1 xn xn^2 ...]]
      if(self.M == 1):
        temp_matrix1 = np.ones([x.shape[0],1])
        temp_matrix2 = x[:x.shape[0],:]
        phi_matrix = np.hstack((temp_matrix1,temp_matrix2))
        return phi_matrix

      if(self.M == 2):
        temp_matrix1 = np.ones([x.shape[0],1])
        temp_matrix2 = x[:x.shape[0],:]
        phi_matrix = np.hstack((temp_matrix1,temp_matrix2))
        position = 1 + self.feature_num
        for i in range(x.shape[1]):
          for j in range(i,x.shape[1]):
            if i <= j:
              phi_matrix = np.insert(phi_matrix, position, (x[:,i]*x[:,j]), axis=1)
              position+=1
        return phi_matrix

def RMSE(y_predict,y_test):
    y_predict = np.squeeze(np.asarray(y_predict)) # numpy.matrix to numpy.ndarray
    y_test = np.squeeze(np.asarray(y_test))
    return np.sqrt(np.mean((y_predict - y_test)**2))

"""####(a) In the feature selection stage, please apply polynomials of order M = 1 and M = 2 over the dimension D = 8 input data. Please evaluate the corresponding RMS error on the training set and valid set. (15%) Code Result"""

print("M = ",1)
model_1 = LinearRegression(x_train,1)
x_new = model_1.poly_features(x_train)
model_1.closed_form(x_new,y_train) #find weight
# RMSE error
y_predict = model_1.predict(x_new)
train_rmse = RMSE(y_predict,y_train)
print("training RMSE: {}".format(train_rmse))
x_new = model_1.poly_features(x_test)
y_predict = model_1.predict(x_new)
test_rmse = RMSE(y_predict,y_test)
print("testing RMSE: {}".format(test_rmse))

print("M = ",2)
model_2 = LinearRegression(x_train,2)
x_new = model_2.poly_features(x_train)
model_2.closed_form(x_new,y_train) #find weight
# RMSE error
y_predict = model_2.predict(x_new)
train_rmse = RMSE(y_predict,y_train)
print("training RMSE: {}".format(train_rmse))
x_new = model_2.poly_features(x_test)
y_predict = model_2.predict(x_new)
test_rmse = RMSE(y_predict,y_test)
print("testing RMSE: {}\n".format(test_rmse))

"""####(b)How will you analysis the weights of polynomial model M = 1 and select the most contributive feature? Code Result, Explain (10%)

**The most contributive feature is median income**
"""

for i in range(len(features)):
  x = x_train
  print("【Without ",features[i],"】")
  x = np.delete(x, i, 1)
  model = LinearRegression(x,1)
  x_new = model_1.poly_features(x)
  model.closed_form(x_new,y_train) #find weight
  # RMSE error
  y_predict = model.predict(x_new)
  train_rmse = RMSE(y_predict,y_train)
  print("training RMSE: {}".format(train_rmse))
  x = x_test
  x = np.delete(x, i, 1)
  x_new = model.poly_features(x)
  y_predict = model.predict(x_new)
  test_rmse = RMSE(y_predict,y_test)
  print("testing RMSE: {}\n".format(test_rmse))

"""##(2) Maximum likelihood approach

####(b) Introduce the basis function you just decided in (a) to linear regression model and analyze the result you get.

使用 **sigmoid basis function** 做轉換，再進行linear regression分析
"""

def sigmoid(x):
    return 1 / (1 + np.exp(-x))

X_sigmoid = X
T_sigmoid = T

# sigmoid
X_sigmoid = sigmoid(X_sigmoid)

# 80% training dataset and 20% test dataset
X_SIZE = X.shape[0]
ratio = 8/10
trainRange = int(X_SIZE*ratio) #num of training data
x_train = X_sigmoid[:trainRange]
x_test = X_sigmoid[trainRange:]
y_train = T[:trainRange]
y_test = T[trainRange:]

print("【Using sigmoid function】")
M=2
for i in range(1,M+1):
  print("M = ",i)
  model = LinearRegression(x_train,i)
  x_new = model.poly_features(x_train)
  model.closed_form(x_new,y_train) #find weight
  # RMSE error
  x_new = model.poly_features(x_train)
  y_predict = model.predict(x_new)
  train_rmse = RMSE(y_predict,y_train)
  print("training RMSE: {}".format(train_rmse))
  x_new = model.poly_features(x_test)
  y_predict = model.predict(x_new)
  test_rmse = RMSE(y_predict,y_test)
  print("testing RMSE: {}".format(test_rmse))

"""#### (c)**N-fold cross-validation** in your training stage to select at least one hyperparameter for model

"""

X_sigmoid = X
T_sigmoid = T

# sigmoid
X_sigmoid = sigmoid(X_sigmoid)

X_SIZE = X_sigmoid.shape[0]
N = 10
ratio = 1/N
testRange = int(X_SIZE*ratio)
M=2
print("【N-fold cross-validation】")
for i in range(1,M+1):
  train_rmse = []
  test_rmse = []
  for r in range(N):
    # data split into 1 set(test) and N-1 sets(train)
    x_test = X_sigmoid[range((testRange*r),(testRange*(r+1))),:]
    y_test = T_sigmoid[range((testRange*r),(testRange*(r+1))),:]
    x_train = np.delete(X_sigmoid, [range((testRange*r),(testRange*(r+1)))], axis=0)
    y_train = np.delete(T_sigmoid, [range((testRange*r),(testRange*(r+1)))], axis=0)

    model = LinearRegression(x_train,i)
    x_new = model.poly_features(x_train)
    model.closed_form(x_new,y_train) #find weight
    # train RMSE error
    x_new = model.poly_features(x_train)
    y_predict = model.predict(x_new)
    train_rmse.append(RMSE(y_predict,y_train))
    # test RMSE error
    x_new = model.poly_features(x_test)
    y_predict = model.predict(x_new)
    test_rmse.append(RMSE(y_predict,y_test))

  average_train_rmse = sum(train_rmse) / N
  average_test_rmse = sum(test_rmse) / N
  print(f'M = {i}, Average_train_rmse = {average_train_rmse}, Average_test_rmse = {average_test_rmse}')

"""## (3) Maximum a *posteriori* approach

####(b) Use maximum a posteriori approach method to retest the model in 2 you designed. You could choose Gaussian distribution as a prior.

使用 Maximum a posteriori 做 linear regression , 設Lambda = 0.1
"""

_lambda = 28

X_sigmoid = X
T_sigmoid = T

# sigmoid
X_sigmoid = sigmoid(X_sigmoid)

# 80% training dataset and 20% test dataset
X_SIZE = X_sigmoid.shape[0]
ratio = 8/10
trainRange = int(X_SIZE*ratio) #num of training data
x_train = X_sigmoid[:trainRange]
x_test = X_sigmoid[trainRange:]
y_train = T_sigmoid[:trainRange]
y_test = T_sigmoid[trainRange:]

M=2
for i in range(1,M+1):
  print("M = ",i)
  model = LinearRegression(x_train,i)
  x_new = model.poly_features(x_train)
  model.closed_form(x_new,y_train,_lambda,True) #find weight
  # RMSE error
  x_new = model.poly_features(x_train)
  y_predict = model.predict(x_new)
  train_rmse = RMSE(y_predict,y_train)
  print("training RMSE: {}".format(train_rmse))
  x_new = model.poly_features(x_test)
  y_predict = model.predict(x_new)
  test_rmse = RMSE(y_predict,y_test)
  print("testing RMSE: {}".format(test_rmse))

"""使用 Maximum a posteriori 做 N-fold cross-validation """

X_sigmoid = X
T_sigmoid = T

# sigmoid
X_sigmoid = sigmoid(X_sigmoid)

X_SIZE = X_sigmoid.shape[0]
N = 10
ratio = 1/N
testRange = int(X_SIZE*ratio)
M=2

for i in range(1,M+1):
  train_rmse = []
  test_rmse = []
  for r in range(N):
    # data split into 1 set(test) and N-1 sets(train)
    x_test = X_sigmoid[range((testRange*r),(testRange*(r+1))),:]
    y_test = T_sigmoid[range((testRange*r),(testRange*(r+1))),:]
    x_train = np.delete(X_sigmoid, [range((testRange*r),(testRange*(r+1)))], axis=0)
    y_train = np.delete(T_sigmoid, [range((testRange*r),(testRange*(r+1)))], axis=0)

    model = LinearRegression(x_train,i)
    x_new = model.poly_features(x_train)
    model.closed_form(x_new,y_train,_lambda,True) #find weight
    # train RMSE error
    x_new = model.poly_features(x_train)
    y_predict = model.predict(x_new)
    train_rmse.append(RMSE(y_predict,y_train))
    # test RMSE error
    x_new = model.poly_features(x_test)
    y_predict = model.predict(x_new)
    test_rmse.append(RMSE(y_predict,y_test))

  average_train_rmse = sum(train_rmse) / N
  average_test_rmse = sum(test_rmse) / N
  print(f'『M = {i}』 Average_train_rmse = {average_train_rmse}, Average_test_rmse = {average_test_rmse}')


# -*- coding: utf-8 -*-
"""
Created on Mon May 30 17:29:28 2016

@author: Michael

How does gaussian white noise behave when it is input to a running average filter,
i.e. a perfect low pass filter (except for discretization errors)?

I assume ergodicity, i.e. marginal probability densities can
be estimated by averaging over time. Pretty sure that assumption is justified.
"""

import numpy as np
import scipy.fftpack

import matplotlib.pyplot as pyplot

def gaussian(x, m, sigma):
  return 1./np.sqrt(2 * np.pi)/sigma * np.exp(-np.square((x - m)/sigma)*0.5)

NG = 1024 * 10
gauss_noise_iid = np.random.normal(0., 1., NG)
gauss_fft = scipy.fftpack.fft(gauss_noise_iid)

sigmas = []
cutoffs = []
for cutoff in [1,2,3,4,5,6,7,8,9, 9.5, 9.9]:
  NFILT = int(512 * cutoff)
  RELATIVE_FREQ_LIMIT = 1.*(NG/2-1-NFILT) / (NG/2-1)
  gauss_fft_limited = gauss_fft.copy()
  gauss_fft_limited[NG/2-1-NFILT:NG/2+NFILT] = 0.
  gauss_limited = scipy.fftpack.ifft(gauss_fft_limited).real
  cutoffs.append(RELATIVE_FREQ_LIMIT)
  sigmas.append(np.std(gauss_limited))
  # low pass frequency limit = 6. / 8.
  # correlation length = 8. / 6.
  # -> sigma_limit = sigma * sqrt(6. / 8.)

  print 'cutoff %s, sigma: white %s, limited %s' % (RELATIVE_FREQ_LIMIT, np.std(gauss_noise_iid), np.std(gauss_limited))
  if 0:
    pyplot.subplot(4, 1, 1)
    pyplot.plot(gauss_noise_iid)
    pyplot.plot(gauss_limited)
    pyplot.gca().set_ylim((-4., 4.))
    pyplot.subplot(4, 1, 2)
    pyplot.plot(np.abs(gauss_fft))
    pyplot.plot(np.abs(gauss_fft_limited))
    pyplot.subplot(4, 1, 3)
    y = np.linspace(-4., 4., 128)
    h1 = gaussian(y, 0., 1.)
    h2 = gaussian(y, 0., 0.44)
    pyplot.hist(gauss_noise_iid, bins = 32, range = (-4., 4.), histtype='step', normed = True)
    pyplot.hist(gauss_limited, bins = 32, range = (-4., 4.), histtype='step', normed = True)
    pyplot.plot(y, h1)
    pyplot.plot(y, h2)
    pyplot.subplot(4, 1, 4)
    pyplot.show()

T = np.reciprocal(np.asarray(cutoffs, dtype = np.float))
print T
pyplot.plot(T, sigmas)
pyplot.gca().set(xscale = 'log')
x = np.logspace(0, 2, 1024)
pyplot.plot(x, np.sqrt(1./x), color = 'red')
pyplot.show()
# The curves should agree very well.
# This means that the variance of the smoothed signal decreases as T0/T, where
# T0 is the correlation length of the input signal and T the width of the
# moving average filter.

# What happens when T < T0? Or in other words when the low pass filter
# corresponding to the moving average over T comprises all frequencies of the
# input. Then of course the signal is preserved. In particular, sigma
# is the same.

# How about super sampling a signal at distances smaller than its correlation
# length? Seems to preserve gaussian processes. Seems intuitive that it
# cannot change the (marginal) distribution of the input.


import numpy as np
import matplotlib.pyplot as plt

def main():
    ''' Read list of cities, solve with nearest neighbour,
        write solution on a file and make plot'''

    csvfile = 'cities.csv'

    cities = read_input_file(csvfile)

    city_order = nearest_neighbour(cities)

    print(city_order)
    plot_tour(city_order, cities)

def read_input_file(csvfile):
    ''' Read an input file with cities
    '''
    return np.genfromtxt(csvfile, skip_header=1, delimiter=',')

def euclidean_distance(x1, y1, x2, y2):
    ''' Computes Euclidean distance between (x1, y1) and (x2, y2)
    '''
    return np.sqrt((x1 - x2)**2 + (y1 - y2) ** 2)

def city_distance(i, j, cities):
    ''' Computes Euclidean distance between city i and j
    using the coordinates given in the 2D array cities
    '''
    return euclidean_distance(cities[i, 0],
                              cities[i, 1],
                              cities[j, 0],
                              cities[j, 1])

def plot_cites(cities, style='ro'):
    ''' Plots a list of cities with the style given in 'style'
    '''
    plt.plot(cities[:,0], cities[:,1], style)

def plot_tour(tour, cities):
    ''' Plots the tour given in 'tour' based on the city coordinates
    given in 'cities'
    '''
    n_cities = cities.shape[0]
    new_cities = np.empty((n_cities + 1, 2))
    new_cities[:n_cities,:] = cities[tour,:]
    new_cities[n_cities:,:] = cities[tour[0],:]
    plot_cites(new_cities, style='ro-')

def nearest_neighbour(cities):
    ''' Returns a tour of N cities constructed by adding the nearest neighbouring city
    iteratively.
    Arguments:
        cities - 2D array of coordinates for the cities (N rows, 2 Columns)
    '''

    # Initialise variables
    n_cities = cities.shape[0]
    city_order = np.full(n_cities, -1, dtype=np.int32)
    city_visited = np.full(n_cities, False, dtype=np.bool)

    # Add first city (the first on the list)
    city_order[0] = 0
    city_visited[0] = True

    for next_city in range(1, n_cities):

        last_visit = city_order[next_city - 1]
        nearest_city = -1
        nearest_dist = np.inf

        # Loop 2, fill distances
        for p_des in range(n_cities):
            if city_visited[p_des]:
                continue

            distance = city_distance(last_visit,
                                     p_des,
                                     cities)
            if distance < nearest_dist:
                nearest_city = p_des
                nearest_dist = distance

        # Add next city
        city_order[next_city] = nearest_city
        city_visited[nearest_city] = True

    return city_order

if __name__ == '__main__':
    main()


from random import randrange
from secrets import randbelow
import numpy as np

SMALL_PRIMES = [3,5,7,11,13,17,19,23,29,31,37,41,43,47,53,59,61,67,71,73,79,83,89,97
                   ,101,103,107,109,113,127,131,137,139,149,151,157,163,167,173,179
                   ,181,191,193,197,199,211,223,227,229,233,239,241,251,257,263,269
                   ,271,277,281,283,293,307,311,313,317,331,337,347,349,353,359,367
                   ,373,379,383,389,397,401,409,419,421,431,433,439,443,449,457,461
                   ,463,467,479,487,491,499,503,509,521,523,541,547,557,563,569,571
                   ,577,587,593,599,601,607,613,617,619,631,641,643,647,653,659,661
                   ,673,677,683,691,701,709,719,727,733,739,743,751,757,761,769,773
                   ,787,797,809,811,821,823,827,829,839,853,857,859,863,877,881,883
                   ,887,907,911,919,929,937,941,947,953,967,971,977,983,991,997]

# Attempts to prove that X is a composite number using A as a witness
# d2^s must equal n - 1
def attempt_prove_composite(x, a, s, d):
	# Calculate v = a^d % x
	v = pow(a, d, x) # pow(base, exponent, modulus)

	# If first term v = 1 or x - 1, A won’t prove X is composite
	if v in (1, x - 1):
		return False

	# Keep going through the sequence till we find something that proves not composite
	## For i from 0 to s (exclusive)
	for i in range(1, s):
		# v = v^2 mod x
		v = pow(v, 2, x)

		# If v = x - 1, X is not composite
		if v == x - 1:
			return False

	# If we don't find anything, it's probably composite
	return True

# Returns if a number looks prime after a series of tests.
# Higher k gives a higher confidence.
def is_prime(x, k=40):
	# Try some divisions by small primes to weed out easy ones
	## For each small prime
	for small in SMALL_PRIMES:
		## If x % small prime is 0, X isn't prime
		if x % small == 0:
			return False

	# Find s and d such that d2^s = n - 1
	## Start with d = x - 1 and s = 0 divisions by 2 done.
	d = np.int(x - 1)
	s = np.int(0)
	# While we can roundly divide d by 2,
	while d % 2 == 0:
		# Divide d by 2
		d //= 2

		# Increment the number of times we've divided by 2
		s += 1

	# Repeat k times:
	for _ in range(0, k):
		# Get a random number a (0 < a < x)
		a = np.int(randrange(0, x - 1))

		# Try to prove x is composite with a, s and d
		if attempt_prove_composite(x, a, s, d):
			# If it is composite, X is not prime
			return False

	# If we can't prove it's composite, it's probably prime
	return True

# Generates a prime number in [minimum, maximum)
def generate_prime(minimum, maximum):
	# Since x = 6r +- 1, min r = min / 6, etc
	r_min = minimum // 6
	r_max = maximum // 6

	# Until a prime is found
	while True:
		# Get a random number r
		r = np.int(randbelow(r_max - r_min) + r_min)

		# Check if 6r + 1 is probably prime
		x = (6 * r) + 1
		if is_prime(x):
			# If so, return it
			return x

		# Check if 6r - 1 is probably prime
		x = (6 * r) - 1
		if is_prime(x):
			# If so, return it.
			return x