#!/usr/bin/env python
# coding: utf-8

# In[1]:

import numpy as np

# In[2]:

# Quadratic Integral Approximation 5.4
def GausIntegral(f, a, b, n):
    def g(t):
        return f(((b-a)*t+(b+a))/2) * ((b-a)/2)
    if n == 1:
        I = g(-(1/3)**0.5)             + g((1/3)**0.5)
        return I
    if n == 2:
        I = (5/9)*g(-(3/5)**0.5)             + (8/9)*g(0)             + (5/9)*g((3/5)**0.5)
        return I
    if n == 3:
        I = ((1/2)+(1/12)*((10/3)**.5))*g(-((1/7)*(3-4*(0.3**0.5)))**0.5)             + ((1/2)-(1/12)*((10/3)**.5))*g(-((1/7)*(3+4*(0.3**0.5)))**0.5)             + ((1/2)+(1/12)*((10/3)**.5))*g(((1/7)*(3-4*(0.3**0.5)))**0.5)             + ((1/2)-(1/12)*((10/3)**.5))*g(((1/7)*(3+4*(0.3**0.5)))**0.5)
        return I
    if n == 4:
        I = (0.3*((-0.7+5*(0.7**0.5))/(-2+5*(0.7**0.5))))*g(-((1/9)*(5-(2*(((10/7)**0.5)))))**0.5)             + (0.3*((0.7+5*(0.7**0.5))/(2+5*(0.7**0.5))))*g(-((1/9)*(5+(2*(((10/7)**0.5)))))**0.5)             + (128/225)*g(0)             + (0.3*((-0.7+5*(0.7**0.5))/(-2+5*(0.7**0.5))))*g(((1/9)*(5-(2*(((10/7)**0.5)))))**0.5)             + (0.3*((0.7+5*(0.7**0.5))/(2+5*(0.7**0.5))))*g(((1/9)*(5+(2*(((10/7)**0.5)))))**0.5)
        return I

# In[3]:

def f(x):
    return 1/(x**0.5)
a = 0
b = 1
I = GausIntegral(f, a, b, 1)
print(I)
print(I - 1.4183)
# Book solution of 1.4183 is wrong?

# In[4]:

# Example 3 approximation for e^(-x^2) from x = 0 to x = 1
def f(x):
    return np.exp(-(x**2))
a = 0
b = 1
I = GausIntegral(f, a, b, 2)
print(I)
print(I-0.746814584)
#Correct!!!

# In[ ]:

Chapter5/Quadratic Integral Approx 5.4.py


import numpy as np

def sigmoid(z):
    """Sigmoid or logistic activation function"""
    return 1 / (1 + np.exp(-z))

# def softmax(z):
#     """Softmax activation function (generalized sigmoid)"""
#     z -= np.max(z)
#     return np.exp(z) / np.sum(np.exp(z), axis=0)

def softmax(arr, axis=None):
    X = np.atleast_2d(arr)
    if axis is None:
        axis = next(j[0] for j in enumerate(X.shape) if j[1] > 1)
    X = X - np.expand_dims(np.max(X, axis=axis), axis)
    X = np.exp(X)
    probs = X / np.expand_dims(np.sum(X, axis=axis), axis)
    if len(arr.shape) == 1:
        return probs.flatten()
    else:
        return probs

def tanh(z):
    """Hyperbolic tangent activation function"""
    sinh = np.exp(z) - np.exp(-z)
    cosh = np.exp(z) + np.exp(-z)
    return sinh / cosh

def relu(z):
    """Rectified Linear Unit activation function"""
    return np.maximum(z, 0, z)

def leaky_relu(z, a=0.01):
    """Leaky Rectified Linear Unit activation function"""
    return np.maximum(z, a * z, z)


''' mbinary
#########################################################################
# File : solve-linear-by-iteration.py
# Author: mbinary
# Mail: zhuheqin1@gmail.com
# Blog: https://mbinary.xyz
# Github: https://github.com/mbinary
# Created Time: 2018-10-02  21:14
# Description:
#########################################################################
'''

'''
#########################################################################
# File : solve-linear-by-iteration.py
# Author: mbinary
# Mail: zhuheqin1@gmail.com
# Blog: https://mbinary.xyz
# Github: https://github.com/mbinary
# Created Time: 2018-05-04  07:42
# Description:
#########################################################################
'''
import numpy as np
from operator import le,lt

def jacob(A,b,x,accuracy=None,times=6):
    ''' Ax=b,  arg x is the init val, times is the time of iterating'''
    A,b,x = np.matrix(A),np.matrix(b),np.matrix(x)
    n,m = A.shape
    if n!=m:raise Exception("Not square matrix: {A}".format(A=A))
    if b.shape !=( n,1) : raise Exception('Error: {b} must be {n} x1 in dimension'.format(b = b,n=n))
    D = np.diag(np.diag(A))
    DI = np.zeros([n,n])
    for i in range(n):DI[i,i]= 1/D[i,i]
    R = np.eye(n) - DI * A
    g = DI * b
    print('R =\n{}'.format(R))
    print('g =\n{}'.format(g))
    last = -x
    if accuracy != None:
        ct=0
        while 1:
            ct+=1
            tmp = x-last
            last = x
            mx = max ( abs(i) for i in tmp)
            if mx<accuracy:return x
            x = R*x+g
            print('x{ct} =\n{x}'.format(ct = ct,x=x))
    else:
        for i in range(times):
            x = R*x+g
            print('x{ct} =  \n{x}'.format(ct=i+1,x=x))
    print('isLimitd: {}'.format(isLimited(A)))
    return x
def gauss_seidel(A,b,x,accuracy=None,times=6):
    ''' Ax=b,  arg x is the init val, times is the time of iterating'''
    A,b,x = np.matrix(A),np.matrix(b),np.matrix(x)
    n,m = A.shape
    if n!=m:raise Exception("Not square matrix: {A}".format(A=A))
    if b.shape !=( n,1) : raise Exception('Error: {b} must be {n} x1 in dimension'.format(b = b,n=n))
    D =np. matrix(np.diag(np.diag(A)))
    L = np.tril(A) - D  # L = np.triu(D.T) - D
    U = np.triu(A) - D
    DLI = (D+L).I
    S = - (DLI) * U
    f = (DLI)*b
    print('S =\n{}'.format(S))
    print('f =\n{}'.format(f))
    last = -x
    if accuracy != None:
        ct=0
        while 1:
            ct+=1
            tmp = x-last
            last = x
            mx = max ( abs(i) for i in tmp)
            if mx<accuracy:return x
            x = S*x+f
            print('x{ct} =\n{x}'.format(ct=ct,x=x))
    else:
        for i in range(times):
            x = S*x+f
            print('x{ct} =  \n{x}'.format(ct=i+1,x=x))
    print('isLimitd: {}'.format(isLimited(A)))
    return x

def isLimited(A,strict=False):
    '''通过检查A是否是[严格]对角优来判断迭代是否收敛, 即对角线上的值是否都大于对应行(或者列)的值'''
    diag = np.diag(A)
    op = lt if strict else le
    if op(A.max(axis=0),diag).all(): return True
    if op(A.max(axis=1), diag).all(): return True
    return False

testcase=[]
def test():
    for func,A,b,x,*args in testcase:
        acc =None
        times = 6
        if args !=[] :
            if isinstance(args[0],int):times = args[0]
            else : acc = args[0]
        return func(A,b,x,acc,times)

if __name__ =='__main__':
    A = [[2,-1,-1],
         [1,5,-1],
         [1,1,10]
        ]
    b = [[-5],[8],[11]]
    x = [[1],[1],[1]]
    #testcase.append([gauss_seidel,A,b,x])

    A = [[2,-1,1],[3,3,9],[3,3,5]]
    b = [[-1],[0],[4]]
    x = [[0],[0],[0]]
    #testcase.append([jacob,A,b,x])

    A = [[5,-1,-1],
         [3,6,2],
         [1,-1,2]
        ]
    b=  [[16],[11],[-2]]
    x = [[1],[1],[-1]]
    testcase.append([gauss_seidel,A,b,x,0.001])
    test()


"""
Eigenvalue demonstration

The revolving red hand is the input vector and the blue hand is the linearly
transformed vector.

Four times every revolution the two hands are parallel (or anti-parallel),
twice to each eigenvector of the matrix A.  The ratio of lengths, blue hand
over red hand, is the corresponding eigenvalue.  The eigenvalue will be
negative if the hands are anti-parallel.

"""
import sys
import numpy as np
import matplotlib.pyplot as plt
import matplotlib.animation as animation
from math import pi, sin, cos

def eigdemo(A):

    print('matrix A =')
    print(A)
    e, x = np.linalg.eig(A)

    print(f"λ1 = {e[0]:.3f}, x1 = {np.real(x[:,0].flatten())}")
    print(f"λ2 = {e[1]:.3f}, x2 = {np.real(x[:,1].flatten())}")

    s = np.max(np.abs(e))

    fig, ax = plt.subplots()
    plt.axis([-s, s, -s, s])
    plt.grid(True)
    plt.title('Eigenvector demonstration')
    plt.xlabel('x')
    plt.ylabel('y')
    plt.axis('equal')
    plt.xlim(-s, s)
    plt.ylim(-s, s)
    ax.set_aspect('equal')

    l1, = plt.plot([0, 0], [0, 0], color='r', linewidth=1.5)  # input vector
    l2, = plt.plot([0, 0], [0, 0], color='b', linewidth=1.5)  # transformed vector

    plt.legend(['$x$', r'${\bf A} x$'])

    def animate(theta):

        x = np.r_[cos(theta), sin(theta)]
        y = A @ x

        l1.set_xdata([0, x[0]])
        l1.set_ydata([0, x[1]])

        l2.set_xdata([0, y[0]])
        l2.set_ydata([0, y[1]])

        return l1, l2

    myAnimation = animation.FuncAnimation(
        fig, animate, frames=np.linspace(
            0, 2 * pi, 400), blit=True, interval=20, repeat=True)

    plt.show(block=True)

def main():

    def help():
        print("eigdemo          uses default matrix [1 2; 3 4]")
        print("eigdemo a b c d  uses matrix [a b; c d]")
        sys.exit(0)

    if len(sys.argv) == 5:

        try:
            vec = [float(a) for a in sys.argv[1:5]]
            A = np.reshape(vec, (2,2))
        except:
            help()

    elif sys.argv == 1:
        A = np.array([
            [1, 2],
            [3, 3]
        ])

    else:
        help()

    eigdemo(A)

if __name__ == "__main__":
    main()


import numpy as np

def create_matrix():
    A = np.mat("0 1 2; 1 0 3;4 -3 8")
    print("A\n",A)

    inverse = np.linalg.inv(A)
    print("inverse of A\n", inverse)
    print("Check\n", A * inverse)

def solution():
    A = np.mat("1 -2 1; 0 2 -8; -4 5 9")
    print("A\m",A)
    b = np.array([0, 8, -9])
    print("b\n",b)
    x = np.linalg.solve(A,b)
    print("Solution:",x)
    print("Check\n",np.dot(A,x))

def matrix_det():
    A = np.mat("2 3; 4 5")
    print("|A|(Determinant) = \n",np.linalg.det(A))

def matrix_sigenvalue():
    A = np.mat("3 -2;1 0")
    print("A\n",A)
    print("Eigenvalues\n", np.linalg.eigvals(A))

    eigenvalues, eigenvectors = np.linalg.eig(A)
    print("First tuple of eig\n", eigenvalues)
    print("Second tuple of eig\n", eigenvectors)

    for i in range(len(eigenvalues)):
        print("Left\n", np.dot(A, eigenvectors[:,i]))
        print("Right\n", eigenvalues[i] * eigenvectors[:i])
        print("")

if __name__ == '__main__':
    # create_matrix()
    # solution()
    # matrix_det()
    matrix_sigenvalue()


import numpy as np
from math import acos,sin,cos,pi,radians

def spherical_dist(lon_1,lat_1,lon_2,lat_2):
    """
    Calculate the distance of two postions and return distance in degree
    """

    lon_1,lat_1,lon_2,lat_2 = map(radians,[lon_1,lat_1,lon_2,lat_2])
    a=acos(sin(lat_1)*sin(lat_2)+cos(lat_1)*cos(lat_2)*cos(lon_2-lon_1))
    return a*180/pi

def in_ellipse(xy_list,width,height,angle=0,xy=[0,0]):
    """
    Find data points inside an ellipse and return index list

    Parameters:
        xy_list: Points needs to be deteced.
        width: Width of the ellipse
        height: Height of the ellipse
        angle: anti-clockwise rotation angle in degrees
        xy: the origin of the ellipse
    """
    if isinstance(xy_list,list):
        xy_list = np.array(xy_list)
    if not isinstance(xy_list,np.ndarray):
        raise Exception(f"Unrecoginzed data type: {type(xy_list)}, should be list or np.ndarray")
    new_xy_list = xy_list.copy()
    new_xy_list = new_xy_list - xy

    #------------ define coordinate conversion matrix----------
    theta = angle/180*np.pi         # degree to radians
    con_mat = np.zeros((2,2))
    con_mat[:,0] = [np.cos(theta),np.sin(theta)]
    con_mat[:,1] = [np.sin(theta),-np.cos(theta)]

    tmp = np.matmul(con_mat,new_xy_list.T)
    con_xy_list = tmp.T

    #------------ check one by one ----------------------------
    idxs = []
    for i,[x,y] in enumerate(con_xy_list):
        if ((x/(width/2))**2+(y/(height/2))**2) < 1:
            idxs.append(i)

    return idxs

def loc_by_width(lon1,lat1,lon2,lat2,width,direction='right'):
    """
    Calculate the points of a rectangle with width and two tips provided.

    Parameters:
      lon1,lat1: longitude and latitude of tip 1
      lon2,lat2: longitude and latitude of tip 2
    """
    sphe_dist = spherical_dist(lon1,lat1,lon2,lat2)
    dlon = lon2 - lon1
    dlat = lat2 - lat1

    if direction == "right":  # extend width to the right
    	delta_lat = -width*(dlon/sphe_dist)    # cos_theta
    	delta_lon =  width*(dlat/sphe_dist)    # sin_theta

    if direction == "left":
    	delta_lat =  width*(dlon/sphe_dist)    # cos_theta
    	delta_lon = -width*(dlat/sphe_dist)    # sin_theta

    new_lon1 = lon1 + delta_lon
    new_lat1 = lat1 + delta_lat
    new_lon2 = lon2 + delta_lon
    new_lat2 = lat2 + delta_lat

    return new_lon1,new_lat1,new_lon2,new_lat2


"""
@author: Rowan Parker
@date: 03/05/2022
"""
import numpy as np

def power_method(A, x, k, scale=True, debug=False):
    """
    Power method for finding dominant eigenvectors and eigenvalues.

    Parameters
    ----------
    A : n × n matrix
        Must have a dominant eigenvalue.
    x : n × 1 non-zero vector
        An initial approximation for the dominant eigenvector.
    k : number of iterations to perform
    scale : boolean, optional
        Use scaling. The default is True.
    debug : boolean, optional
        Enable debugging print statements. The default is False.

    Returns
    -------
    (dominant eigenvalue, dominant eigenvector)
    """
    for i in range(k):
        if debug:
            print("step_{}: x = \n{}".format(i, x))

        x = A*x
        if scale:
            x = np.divide(x, np.max(x))

    eigenvalue = np.dot(np.squeeze(np.asarray(A*x)),
                        np.squeeze(np.asarray(x)))
    eigenvalue /= np.dot(np.squeeze(np.asarray(x)),
                         np.squeeze(np.asarray(x)))

    return eigenvalue, x

def swaprows(i, j, Ab):
    (n, m) = Ab.shape
    temp_row = np.zeros(m)

    temp_row[:] = Ab[i, :]
    Ab[i, :] = Ab[j, :]
    Ab[j, :] = temp_row[:]

    return(Ab)

def gauss_pp(Ab):
    (n, m) = Ab.shape
    # Row reduce
    for k in range(0, n-1):
        # Find the best pivot using partial pivoting
        MAX = abs(Ab[k, k])
        I = k
        for i in range(k+1, n):
            m = abs(Ab[i, k])
            if m > MAX:
                print("Row %d switched with Row %d:" % ((i+1), (k+1)))
                MAX = m
                I = i

        swaprows(I, k, Ab)

        # Continue with row reduction after finding the best pivot
        for j in range(k+1, n):
            c = Ab[j, k]/Ab[k, k]
            Ab[j, :] = Ab[j, :] - c*Ab[k, :]

        print(Ab, "\n")

    # back-substitute [A:b]
    (n, m) = Ab.shape
    for i in range(n-1, -1, -1):
        # subtract constants from solution
        for j in range(m-1):
            if j != i:
                Ab[i, m-1] -= Ab[i, j]
                Ab[i, j] = 0

        # divide both sides by coefficent
        Ab[i, m-1] /= Ab[i, i]
        Ab[i, i] /= Ab[i, i]

        # substitute value back into other rows
        for k in range(i-1, -1, -1):
            Ab[k, i] *= Ab[i, m-1]

        # print augmented matrix
        print("After solving for x%d:\n" % (i+1), Ab, "\n")

    # print solution
    print("Solution:", (Ab[:, -1]).reshape(-1))

    return Ab

def gauss_npp(Ab):
    (n,m) = Ab.shape
    #Row Reduce [A:b]
    for k in range(0, n-1): #produce k-th column of zeros
        for j in range(k+1, n): #j-th row operation
            c = Ab[j,k]/Ab[k,k]
            Ab[j,:] = Ab[j,:] - c*Ab[k,:]

    # print augmented matrix after row reduction
    print("After row reduction:\n", Ab, "\n")

    # back-substitute [A:b]
    for i in range(n-1, -1, -1):
        # subtract constants from solution
        for j in range(m-1):
            if j != i:
                Ab[i, m-1] -= Ab[i, j]
                Ab[i, j] = 0

        # divide both sides by coefficent
        Ab[i, m-1] /= Ab[i, i]
        Ab[i, i] /= Ab[i, i]

        # substitute value back into other rows
        for k in range(i-1, -1, -1):
            Ab[k, i] *= Ab[i, m-1]

        # print augmented matrix
        print("After solving for x%d:\n" % (i+1), Ab, "\n")

    # print solution
    print("Solution:", (Ab[:, -1]).reshape(-1))

    return Ab

def LUdecomp(A):
    (n,m) = A.shape
    L = eye(n,m)
    U = zeros((n,m))

    U = A
    for k in range(n-1):
        for j in range(k+1, n):
            L[j, k] = U[j, k] / U[k, k]
            U[j, :] = U[j, :] - L[j, k] * U[k, :]
    return (L,U)

def LUsolve(L, b):
    n = L.shape[0]
    y = zeros(n)
    x = zeros(n)

    for i in range(n):
        # calculate y[i]
        y[i] = b[i]
        for j in range(n):
            if j != i:
                y[i] -= L[i, j]

        # substitute forward
        for k in range(i+1, n):
            L[k, i] *= y[i]

    for i in range(n-1, -1, -1):
        # calculate x[i]
        x[i] = y[i]
        for j in range(n):
            if j != i:
                x[i] -= U[i, j]
        x[i] /= U[i, i]

        # substitute back
        for k in range(i-1, -1, -1):
            U[k, i] *= x[i]

    return x


from numpy.random import uniform, seed
from matplotlib.mlab import griddata
import matplotlib.pyplot as plt
import numpy as np
# make up data.
#npts = int(raw_input('enter # of random points to plot:'))
seed(-1)
npts = 200
x = uniform(-2,2,npts)
y = uniform(-2,2,npts)
z = x*np.exp(-x**2-y**2)
# define grid.
xi = np.linspace(-2.1,2.1,100)
yi = np.linspace(-2.1,2.1,100)
# grid the data.
zi = griddata(x,y,z,xi,yi)
# contour the gridded data, plotting dots at the nonuniform data points.
CS = plt.contour(xi,yi,zi,15,linewidths=0.5,colors='k')
CS = plt.contourf(xi,yi,zi,15,cmap=plt.cm.jet)
plt.colorbar() # draw colorbar
# plot data points.
plt.scatter(x,y,marker='o',c='b',s=5)
plt.xlim(-2,2)
plt.ylim(-2,2)
plt.title('griddata test (%d points)' % npts)
plt.show()

# test case that scikits.delaunay fails on, but natgrid passes..
#data = np.array([[-1, -1], [-1, 0], [-1, 1],
#                 [ 0, -1], [ 0, 0], [ 0, 1],
#                 [ 1, -1 - np.finfo(np.float_).eps], [ 1, 0], [ 1, 1],
#                ])
#x = data[:,0]
#y = data[:,1]
#z = x*np.exp(-x**2-y**2)
## define grid.
#xi = np.linspace(-1.1,1.1,100)
#yi = np.linspace(-1.1,1.1,100)
## grid the data.
#zi = griddata(x,y,z,xi,yi)


# gaussian elimination with scaled pivoting
#
# M. Zingale (2013-02-25)

import numpy

def gaussElim(A, b, returnDet=0):
    """ perform gaussian elimination with pivoting, solving A x = b A
        is an NxN matrix, x and b are an N-element vectors.  Note: A
        and b are changed upon exit to be in upper triangular (row
        echelon) form """

    # b is a vector
    if not b.ndim == 1:
        print "ERROR: b should be a vector"
        return None

    N = len(b)

    # A is square, with each dimension of length N
    if not (A.shape[0] == N and A.shape[1] == N):
        print "ERROR: A should be square with each dim of same length as b"
        return None

    # allocation the solution array
    x = numpy.zeros((N), dtype=A.dtype)

    # find the scale factors for each row -- this is used when pivoting
    scales = numpy.max(numpy.abs(A), 1)

    # keep track of the number of times we swapped rows
    numRowSwap = 0

    printAb(A, b)

    # main loop over rows
    for k in range(N):

        # find the pivot row based on the size of column k -- only consider
        # the rows beyond the current row
        rowMax = numpy.argmax(A[k:, k]/scales[k:])
        if (k > 0): rowMax += k  # we sliced A from k:, correct for total rows

        # swap the row with the largest scaled element in the current column
        # with the current row (pivot) -- do this with b too!
        if not rowMax == k:
            A[[k, rowMax],:] = A[[rowMax, k],:]
            b[[k, rowMax]] = b[[rowMax, k]]
            numRowSwap += 1

        # do the forward-elimination for all rows below the current
        for i in range(k+1, N):
            coeff = A[i,k]/A[k,k]

            for j in range(k+1, N):
                A[i,j] += -A[k,j]*coeff

            A[i,k] = 0.0
            b[i] += -coeff*b[k]
        #print A, "\n"
        printAb(A, b)

    # back-substitution

    # last solution is easy
    x[N-1] = b[N-1]/A[N-1,N-1]

    for i in reversed(range(N-1)):
        sum = b[i]
        for j in range(i+1,N):
            sum += -A[i,j]*x[j]
        x[i] = sum/A[i,i]

    # determinant
    det = numpy.prod(numpy.diagonal(A))*(-1.0)**numRowSwap

    if not returnDet:
        return x
    else:
        return x, det

# for debugging:
# output: numpy.savetxt("test.out", a, fmt="%5.2f", delimiter="  ")
# convert -font Courier-New-Regular -pointsize 20 text:test.out test.png

def printAb(A, b):
    """ printout the matrix A and vector b in a pretty fashion.  We
        don't use the numpy print here, because we want to make them
        side by side"""

    N = len(b)

    openT = "/"
    closeT = "\\"

    openB = "\\"
    closeB = "/"

    # numbers take 6 positions + 2 spaces
    aFmt = " %6.3f "
    space = 8*" "

    line = "|" + N*aFmt + "|" + space + "|" + aFmt + "|"
    top = openT + N*space + closeT  + space + openT + space + closeT
    bottom = openB + N*space + closeB + space + openB + space + closeB + "\n"

    print top
    for i in range(N):
        out = tuple(A[i,:]) + (b[i],)
        print line % out
    print bottom


#!/usr/bin/python3
import numpy as np
import random as rd
from math import *
import matplotlib.pyplot as plt

########################################
####### Méthode des rectangles #########
########################################
tests = False

### Exercice 1-a ###
def Irg(f, a, b, n):
    ints = np.linspace(a, b, n + 1)
    summ = 0
    for i in range(len(ints) - 1):
        summ += f(ints[i]) * (ints[i+1] - ints[i])
    return summ

### Exercice 1-b ###
# sin x croissante sur [0; \pi/2] : Irg donne une
# minoration de l'intégrale.
sing = Irg(sin, 0, pi/2, 10)

### Exercice 1-c ###
def Ird(f, a, b, n):
    ints = np.linspace(a, b, n + 1)
    summ = 0
    for i in range(len(ints) - 1):
        summ += f(ints[i+1]) * (ints[i+1] - ints[i])
    return summ

### Exercice 1-d ###
# sin x croissante sur [0; \pi/2] : Ird donne une
# majoration de l'intégrale.
sind = Ird(sin, 0, pi/2, 10)

### Exercice 1-e ###
# On peut considérer que c'est un valeur
# moyenne entre la valeur minimale et la
# valeur maximale : probablement plus
# précise.
sina = .5 * (sind + sing)

### Exercice 2 ###
def Er(meth, f, a, b, n, intex, m1):
    intc = meth(f, a, b, n)
    err = (b-a) * (b-a) / (2*n) * m1
    return abs(intex - intc) <= err

if tests:
    countg = 0
    countd = 0
    for n in range(1, 1001):
        if Er(Irg, sin, 0, pi/2, n, 1, 1):
            countg += 1
        if Er(Ird, sin, 0, pi/2, n, 1, 1):
            countd += 1
    print("Validg : ", countg)
    print("Validd : ", countd)

### Exercice 3-a ###
def Irm(f, a, b, n):
    ints = np.linspace(a, b, n + 1)
    summ = 0
    for i in range(len(ints) - 1):
        summ += f((ints[i] + ints[i+1])/2) * (ints[i+1] - ints[i])
    return summ

### Exercice 3-b ###
def Er2(meth, f, a, b, n, intex, m2):
    intc = meth(f, a, b, n)
    err = (b-a) * (b-a) * (b-a) / (24*n*n) * m2
    return abs(intex - intc) <= err
if tests:
    countm = 0
    for n in range(1, 1001):
        if Er2(Irm, sin, 0, pi/2, n, 1, 1):
            countm += 1
    print("Validm : ", countm)

### Exercice 3-c ###
def f1(x): return x + 2
def f2(x): return x*x
if tests:
    print(Irm(f1, 0, 1, 10))
    print(Irm(f2, 0, 1, 10))

########################################
######## Méthode des trapèzes ##########
########################################
tests = False

### Exercice 1 ###
def It(f, a, b, n):
    ints = np.linspace(a, b, n+1)
    summ = 0
    for i in range(len(ints) - 1):
        summ += (ints[i+1] - ints[i]) * (f(ints[i]) + f(ints[i+1])) / 2
    return summ

### Exercice 2 ###
def Er3(meth, f, a, b, n, intex, m2):
    intc = meth(f, a, b, n)
    err = (b-a) * (b-a) * (b-a) / (12*n*n) * m2
    return abs(intex - intc) <= err
if tests:
    count = 0
    for n in range(1, 1001):
        if Er3(It, sin, 0, pi/2, n, 1, 1):
            count += 1
    print("Valid trap : ", count)

### Exercice 3 ###
if tests:
    print(It(f1, 0, 1, 10))
    print(It(f2, 0, 1, 10))

########################################
######### Méthode de simpson ###########
########################################
tests = False

### Exercice 1 ###
def Isimp(f, a, b, n):
    ints = np.linspace(a, b, n+1)
    summ = 0
    for i in range(len(ints) - 1):
        summ += (ints[i+1] - ints[i]) * (f(ints[i])/6 + 2*f((ints[i]+ints[i+1])/2)/3 + f(ints[i+1])/6)
    return summ

### Exercice 2 ###
def Er4(meth, f, a, b, n, intex, m4):
    intc = meth(f, a, b, n)
    err = (b-a) * (b-a) * (b-a) * (b-a) * (b-a) / (2880*n*n*n*n) * m4
    return abs(intex - intc) <= err
if tests:
    count = 0
    for n in range(1, 1001):
        if Er4(Isimp, sin, 0, pi/2, n, 1, 1):
            count += 1
    print("Valid simp : ", count)

### Exercice 3 ###
def f3(x): return x*x*x
def f4(x): return x*x*x*x
if tests:
    print(Isimp(f3, 0, 1, 10))
    print(Isimp(f4, 0, 1, 10))

########################################
####### Méthode de monte-carlo #########
########################################
tests = False
def Imont(f, a, b, m, n):
    count = 0
    for i in range(n):
        x = rd.uniform(a, b)
        y = rd.uniform(0, m)
        if y < f(x):
            count += 1
    return m * (b-a) * count / n

def f(x): return sqrt(1 - x*x)
if tests:
    mpi = 4 * Imont(f, 0, 1, 1, 1000000)
    print(mpi)

########################################
########## Graphes d'erreur ############
########################################
tests = True

def plotfn(f, a, b, n, cl = 'blue', lb = ''):
    xs = np.linspace(a, b, n)
    ys = [f(x) for x in xs]
    plt.plot(xs, ys, linewidth = 1, color = cl)
    plt.text(xs[-1], ys[-1], lb)

def createfn(m, p):
    return lambda x: m*x + p

def calcerr(meth, f, a, b, n, intex):
    return log(abs(meth(f, a, b, n) - intex))

def errgraph(mts, f, a, b, n, intex, l):
    N = 100
    for m in l:
        plotfn(createfn(-m, 0), 0, log(n), N, 'red', str(-m))

    i = 0
    cls = ['blue', 'green', 'orange', 'salmon', 'cyan']
    my = 0
    for m in mts:
        meth = m[0]
        ys = [calcerr(meth, f, a, b, i, intex) for i in range(1, n+1)]
        xs = [log(i) for i in range(1, n+1)]
        plt.plot(xs, ys, linewidth = 2, color = cls[i])
        plt.text(xs[-1], ys[-1], m[1])
        i = divmod(i+1, len(cls))[1]
        my = max(my, ys[0])

    plt.title("Error graph")
    plt.axis([0, log(n), ys[-1] - 1, my])
    plt.show()

if tests:
     errgraph([[Ird,"ird"], [Irm,"irm"], [It,"trap"], [Isimp, "simp"]],
             sin, 0, pi/2, 1000, 1, [.5, 1, 2, 4])


# logistic distribution is used to describe growth.
# used extensively in machine learning in logistic regression, neural networks etc.
# it has three parameters:
# loc = mean, where peak is. Default = 0.
# scale = standard deviation, flatness of distribution. Default 1.
# size = Shape of returned array.

from numpy import random
import matplotlib.pyplot as plt
import seaborn as sns

arr1 = random.logistic(loc=10, scale=2, size=10)
print(arr1)
arr2 = random.normal(loc=10, scale=2, size=10)
print(arr2)
sns.distplot(arr1, hist=False, label='logistic')
sns.distplot(arr2, hist=False, label='normal')
plt.show()

# both are near identical, but logistic has more area under tails.
# i.e. possibility of more occurrence of number that are futher away from mean

Python/Numpy/numpy 24 - logistic distribution.py


import tensorflow as tf
node1 = tf.constant(3.0, tf.float32)
node2 = tf.constant(4.0, tf.float32)
sess = tf.Session()
# print (sess.run([node1, node2]))
node3 = tf.add(node1, node2)
print("node3: ", node3)
print("sess.run(node3): ", sess.run(node3))

a = tf.placeholder(tf.float32)
b = tf.placeholder(tf.float32)
adder_node = a + b

print(sess.run(adder_node, {a: 3, b: 4.5}))
print(sess.run(adder_node, {a: [1, 3], b: [2, 4]}))

add_and_tripple = adder_node * 3
print(sess.run(add_and_tripple, {a: 3, b: 4.5}))

W = tf.Variable([.3], tf.float32)
b = tf.Variable([-.3], tf.float32)
x = tf.placeholder(tf.float32)
linear_model = W * x + b

init = tf.global_variables_initializer()
sess.run(init)

print(sess.run(linear_model, {x: [1, 2, 3, 4]}))

y = tf.placeholder(tf.float32)
squared_deltas = tf.square(linear_model - y)
loss = tf.reduce_sum(squared_deltas)
print(sess.run(loss, {x: [1,2,3,4], y:[0, -1, -2, -3]}))

fixW = tf.assign(W, [-1.])
fixb = tf.assign(b, [1.])
sess.run([fixW, fixb])
print(sess.run(loss, {x: [1,2,3,4], y: [0, -1, -2, -3]}))

optimizer = tf.train.GradientDescentOptimizer(0.01)
train = optimizer.minimize(loss)
sess.run(init)
for i in range(1000):
    sess.run(train, {x: [1,2,3,4], y:[0, -1, -2, -3]})
print(sess.run([W, b]))

#
# Linear regression
#
import numpy as numpy
import tensorflow as tf

# Model parameters
W = tf.Variable([.3], tf.float32)
b = tf.Variable([-.3], tf.float32)
# Model input and output
x = tf.placeholder(tf.float32)
linear_model = W * x + b
y = tf.placeholder(tf.float32)
# loss
loss = tf.reduce_sum(tf.square(linear_model - y))
# optimizer
optimizer = tf.train.GradientDescentOptimizer(0.01)
train = optimizer.minimize(loss)
# training data
x_train = [1, 2, 3, 4]
y_train = [0, -1, -2, -3]
# training loop
init = tf.global_variables_initializer()
sess = tf.Session()
sess.run(init)
for i in range(1000):
    sess.run(train, {x: x_train, y: y_train})

# evaluate training accuracy
curr_W, curr_b, curr_loss = sess.run([W, b, loss], {x: x_train, y: y_train})
print("W: %s b: %s loss: %s" % (curr_W, curr_b, curr_loss))


import numpy as np

def get_entropy(x):
    """
    Also known as Shanon Entropy
    Reference: https://en.wikipedia.org/wiki/Entropy_(information_theory)
    """
    unique, count = np.unique(x, return_counts=True, axis=0)
    probability = count/len(x)
    entropy = np.sum((-1)*probability*np.log2(probability))
    return entropy

def get_joint_entropy(x, y):
    """
    H(Y;X)
    Reference: https://en.wikipedia.org/wiki/Joint_entropy
    """
    xy = np.c_[x, y]
    return get_entropy(xy)

def get_conditional_entropy(y, x):
    """
    conditional entropy = Joint Entropy - Entropy of X
    H(Y|X) = H(Y;X) - H(X)
    Reference: https://en.wikipedia.org/wiki/Conditional_entropy
    """
    return get_joint_entropy(y, x) - get_entropy(x)

def get_information_gain(y, x):
    """
    Information Gain, I(Y;X) = H(Y) - H(Y|X)
    Reference: https://en.wikipedia.org/wiki/Information_gain_in_decision_trees#Formal_definition
    """
    return get_entropy(y) - get_conditional_entropy(y, x)


import math
import numpy as np
def sigmoid(x):
    return 1/(1 + np.exp(-x))

def sigmoid_prime(x):
    sx = sigmoid(x)
    return np.multiply(sx, (1 - sx))

def relu(x):
    return np.maximum(x, 0)

def relu_prime(x):
    return x >= 0

class NeuralNetwork:
    def __init__(self, in_dim, hid_dim, out_dim):
        # np.random.seed(10) # for generating the same results
        # self.W1 = np.random.rand(hid_dim, in_dim)
        self.W1 = np.eye(hid_dim, in_dim)
        # self.W2 = np.random.rand(out_dim, hid_dim)
        self.W2 = np.eye(out_dim, hid_dim)
        self.x = None
        self.W1x = None
        self.x_1 = None
        self.W2x_1 = None
        self.x_2 = None

    def forward(self, x):
        # print(x)
        self.x = x
        # print(self.W1.shape, x.shape)
        self.W1x = np.dot(self.W1, x)
        # print(self.W1x)
        # self.x_1 = relu(self.W1x)
        self.x_1 = sigmoid(self.W1x)
        # print(self.x_1)
        self.W2x_1 = np.matmul(self.W2, self.x_1)
        # print(self.W2x_1)

        self.x_2 = sigmoid(self.W2x_1)
        # print(self.x_2)

        return self.x_2

    def loss(self, X, Y):
        lse = 0
        for j in range(len(X)):
            x = X[j]
            y = Y[j]
            self.forward(x)

            lse += np.linalg.norm(y - self.x_2)**2
        return lse

    def gradient_descent(self, X, Y, iterations):
        for i in range(iterations):
            loss = self.loss(X, Y)
            print("loss", i, loss)
            G2 = np.zeros(self.W2.shape)
            G1 = np.zeros(self.W1.shape)

            for j in range(len(X)):
                x = X[j]
                y = Y[j]
                self.forward(x)

                # gradients for hidden to output weights
                # print((y - self.x_2).shape, sigmoid_prime(self.W2x_1).shape)
                mu2 = np.multiply((y - self.x_2), sigmoid_prime(self.W2x_1))
                g2 = np.matmul(mu2, np.transpose(self.x_1))
                # print("w1x", self.W1x)

                # mu1a = relu_prime(self.W1x)
                mu1a = sigmoid_prime(self.W1x)
                # print("mu1a", mu1a)

                mu1b = np.dot(np.transpose(self.W2), mu2)
                # print("mu1b", mu1b)

                mu1 = np.multiply(mu1a, mu1b)
                # print("mu1", mu1)

                g1 = np.dot(mu1, np.transpose(self.x))

                # print("g2", g2)
                # print("g1", g1)

                G1 += g1
                G2 += g2

            self.W2 = self.W2 + G2 #/ len(X)
            self.W1 = self.W1 + G1 #/ len(X)

# small example

# x = np.random.rand(10, 2, 1)
# x = [np.random.rand(2, 1) for _ in range(10)]
x = [np.matrix([[1],[0]]), np.matrix([[0.5],[-0.3]])]
print(x)

nn = NeuralNetwork(2, 4, 2)
y_hat = nn.forward(x[0])
print("before", y_hat)
y_hat = nn.forward(x[1])
print("before", y_hat)

# y = [np.random.rand(2, 1) for _ in range(10)]
y = [np.matrix([[1], [0]]), np.matrix([[0], [1]])]
# print(y)
nn.gradient_descent(x, y, 10)
y_hat = nn.forward(x[0])
print("after", y_hat)
y_hat = nn.forward(x[1])
print("after", y_hat)

# framingam example

#     def gradient_descent(self, x, y, iterations):
#         for i in range(iterations):
#             Xi = x
#             Xj = self.sigmoid(Xi, self.wij)
#             yhat = self.sigmoid(Xj, self.wjk)
#             # gradients for hidden to output weights
#             g_wjk = np.dot(Xj.T, (y - yhat) * self.sigmoid_derivative(Xj, self.wjk))
#             # gradients for input to hidden weights
#             g_wij = np.dot(Xi.T, np.dot((y - yhat) * self.sigmoid_derivative(Xj, self.wjk), self.wjk.T) * self.sigmoid_derivative(Xi, self.wij))
#             # update weights
#             self.wij += g_wij
#             self.wjk += g_wjk
#         print('The final prediction from neural network are: ')
#         print(yhat)
#
# if __name__ == '__main__':
# neural_network = NeuralNetwork()
# print('Random starting input to hidden weights: ')
# print(neural_network.wij)
# print('Random starting hidden to output weights: ')
# print(neural_network.wjk)
# X = np.array([[0, 0, 1], [1, 1, 1], [1, 0, 1], [0, 1, 1]])
# y = np.array([[0, 1, 1, 0]]).T
# neural_network.gradient_descent(X, y, 10000)


# -*- coding: utf-8 -*-
"""
Created on Wed Apr 24 08:51:47 2019

@author: mhossa12
"""
## https://www.kaggle.com/andyxie/k-means-clustering-implementation-in-python
# Import necessary libraries
from copy import deepcopy
import numpy as np # linear algebra
from matplotlib import pyplot as plt
# Set three centers, the model should predict similar results
center_1 = np.array([1,1])
center_2 = np.array([5,5])
center_3 = np.array([8,1])

# Generate random data and center it to the three centers
data_1 = np.random.randn(200, 2) + center_1
data_2 = np.random.randn(200,2) + center_2
data_3 = np.random.randn(200,2) + center_3

data = np.concatenate((data_1, data_2, data_3), axis = 0)

plt.scatter(data[:,0], data[:,1], s=7)
plt.grid()
plt.show()

# Number of clusters
k = 3
# Number of training data
n = data.shape[0]
# Number of features in the data
c = data.shape[1]

# Generate random centers, here we use sigma and mean to ensure it represent the whole data
mean = np.mean(data, axis = 0)
std = np.std(data, axis = 0)
centers = np.random.randn(k,c)*std + mean

# Plot the data and the centers generated as random
plt.scatter(data[:,0], data[:,1], s=7)
plt.scatter(centers[:,0], centers[:,1], marker='*', c='g', s=150)

plt.grid()
plt.show()

centers_old = np.zeros(centers.shape) # to store old centers
centers_new = deepcopy(centers) # Store new centers

data.shape
clusters = np.zeros(n)
distances = np.zeros((n,k))

error = np.linalg.norm(centers_new - centers_old)

# When, after an update, the estimate of that center stays the same, exit loop
while error != 0:
    # Measure the distance to every center
    for i in range(k):
        distances[:,i] = np.linalg.norm(data - centers[i], axis=1)
    # Assign all training data to closest center
    clusters = np.argmin(distances, axis = 1)

    centers_old = deepcopy(centers_new)
    # Calculate mean for every cluster and update the center
    for i in range(k):
        centers_new[i] = np.mean(data[clusters == i], axis=0)
    error = np.linalg.norm(centers_new - centers_old)
centers_new

# Plot the data and the centers generated as random
plt.scatter(data[:,0], data[:,1], s=7)
plt.scatter(centers_new[:,0], centers_new[:,1], marker='*', c='g', s=150)

plt.grid()
plt.show()

Clustering K-Means/00Develope and Verify a K-Means++ Clustering Model with a Randomly Generated Dataset in Python.py


import sympy as sp

if __name__ == '__main__':
    j = complex(0, 1)
    A = sp.Matrix([
        [1, 0, -1],
        [0, 1, j],
        [-1, -j, 0]
    ])

    A_eigenvectors = A.eigenvects()

    print(f"A := {A}", '-'*25, sep='\n')
    for eigenvalue, multiplicity, eigenvector in A_eigenvectors:
        print(f"lambda = {eigenvalue}, m = {multiplicity}", eigenvector, '-'*175, sep='\n')

    E, a, theta = sp.symbols("E a theta")
    H = sp.Matrix([
        [E, a, 0],
        [a, E, a],
        [0, a, E]
    ])
    H_eigenvectors = H.eigenvects()

    print(f"H := {H}", '-' * 25, sep='\n')
    for eigenvalue, multiplicity, eigenvector in H_eigenvectors:
        print(f"lambda = {eigenvalue}, m = {multiplicity}", eigenvector, '-'*175, sep='\n')

    R = sp.Matrix([
        [1, 0, 0],
        [0, sp.cos(theta), -sp.sin(theta)],
        [0, sp.sin(theta), sp.cos(theta)]
    ])
    RHRinv = R*H*R.inv()
    RHRinv_eigenvectors = RHRinv.eigenvects()

    print(f"RHRinv := {RHRinv}", '-' * 25, sep='\n')
    for eigenvalue, multiplicity, eigenvector in RHRinv_eigenvectors:
        print(f"lambda = {eigenvalue}, m = {multiplicity}", eigenvector, '-' * 175, sep='\n')

    print(H.eigenvals(), RHRinv.eigenvals(), sep='\n')
    check = all([e in RHRinv.eigenvals() for e in H.eigenvals()])
    print(f"H.eigenvals() == RHRinv.eigenvals(): {check}")


"""Probability function"""
from scipy.stats import norm

# Finding the probability of a discrete value
def prob_norm_single_value(discrete_value=None, mean=None,stdev=None):
    """Probability function for a discrete value"""
    x_value = norm(loc = mean, scale= stdev).cdf(discrete_value)
    return x_value

# Finding the probability between two ranges
def prob_norm_between_values(lower_val=None, upper_val=None, mean=None,stdev=None):
    """Probability between two values"""
    l_value = norm(loc = mean, scale= stdev).cdf(lower_val)
    u_value = norm(loc = mean, scale= stdev).cdf(upper_val)
    delta = u_value - l_value
    return delta

# Finding the probability above a discrete value
def prob_norm_above_value(discrete_value=None, mean=None,stdev=None):
    """Probability above a discrete value"""
    x_value = norm(loc = mean, scale= stdev).cdf(discrete_value)
    delta = 1 - x_value
    return delta

def main():
    """Main logic flow"""
    func_type = input ('Enter the function type (single/range/above): ')
    lower_val = float ( input("Enter the single or lower range (value): ") )
    if func_type == 'range':
        upper_val = float ( input("Enter the upper range (value): ") )
    mean = float ( input("Enter the mean (value): ") )
    stdev = float ( input("Enter the standard deviation (value): ") )
    if func_type == 'range':
        return_value = prob_norm_between_values(
                lower_val=lower_val,
                upper_val=upper_val,
                mean=mean,
                stdev=stdev
                )
        print(f'Probability between {lower_val} and {upper_val} is: {return_value}')
    elif func_type == 'above':
        return_value = prob_norm_above_value(
                discrete_value=lower_val,
                mean=mean,
                stdev=stdev
                )
        print(f'Probability above {lower_val} is: {return_value}')
    else:
        return_value = prob_norm_single_value(
                discrete_value=lower_val,
                mean=mean,
                stdev=stdev
                )
        print(f'Probability of {lower_val} is: {return_value}')

if __name__ == "__main__":
    main()


from pyomo.environ import *
infinity = float('inf')

# Creation of a Concrete Model
model = AbstractModel()

#DEFINE SETS
# Products
model.F = Set()
# Nutrients
model.N = Set()

# DEFINE PARAMETERS
# Cost
model.c = Param(model.F, within = PositiveReals, doc = 'price in $')

# Amount of nutrient
model.a    = Param(model.F, model.N, within = NonNegativeReals)

# Max and Min for each Nutrient
model.Nmin = Param(model.N, within = NonNegativeReals, default = 0.0)
model.Nmax = Param(model.N, within = NonNegativeReals, default = infinity)

# Number of servings
model.x = Var(model.F, within = NonNegativeIntegers)

# Minimize z(cost)
def cost(model):
    return sum(model.c[i]*model.x[i] for i in model.F)
model.cost = Objective(rule=cost)

# LIMITS

#Max
def nutrients_max(model, j):
    value = sum(model.a[i,j]*model.x[i] for i in model.F)
    return value <= model.Nmax[j]
model.nutrient_limit_max = Constraint(model.N, rule=nutrients_max)
#Min
def nutrient_min(model, j):
    value = sum(model.a[i,j]*model.x[i] for i in model.F)
    return model.Nmin[j] <= value
model.nutrient_limit_min = Constraint(model.N, rule=nutrient_min)

#def pyomo_postprocess(options=None, instance=None, results=None):
#    model.x.display()


# Question 4 Lab Assignment 2
# AB Satyaprakash - 180123062

# imports ----------------------------------------------------------------------
from math import log, tan, factorial
from sympy import *
import numpy as np

# functions --------------------------------------------------------------------

def f(x):
    return log(tan(x))/log(10)

def thirdLagrange(x0, x1, x2, x3, val):
    # Using the values given in the quesiton
    f0, f1, f2, f3 = 0.1924, 0.2414, 0.2933, 0.3492

    # compute l0(x) with x = val
    l0 = ((val-x1)*(val-x2)*(val-x3))/((x0-x1)*(x0-x2)*(x0-x3))
    l1 = ((val-x0)*(val-x2)*(val-x3))/((x1-x0)*(x1-x2)*(x1-x3))
    l2 = ((val-x0)*(val-x1)*(val-x3))/((x2-x0)*(x2-x1)*(x2-x3))
    l3 = ((val-x0)*(val-x1)*(val-x2))/((x3-x0)*(x3-x1)*(x3-x2))
    px = f0*l0 + f1*l1 + f2*l2 + f3*l3
    return px

def derivative(x0, n):
    x = symbols('x')
    f = log(tan(x), 10)
    fn = f.diff(x, n)
    fn = lambdify(x, fn)
    return fn(x0)

def maxError(nodes, x):
    err = 1
    for n in nodes:
        err *= (x-n)
    err /= factorial(len(nodes))
    a = min(nodes)
    b = max(nodes)
    l = np.linspace(a, b, 250)
    ret = 0
    for z in l:
        der = derivative(z, len(nodes))
        ret = max(ret, abs(err*der))
    return ret

# ------------------------------------------------------------------------------
# Use the following values and four-digit rounding arithmetic to construct a third Lagrange polynomial approximation to f(1.09).
# The function being approximated is f(x) = log10(tan x). Use this knowledge to find a bound for the error in the approximation.
# f(1.00) = 0.1924, f(1.05) = 0.2414, f(1.10) = 0.2933, f(1.15) = 0.3492.
# Using f(1.00) = 0.1924, f(1.05) = 0.2414, f(1.10) = 0.2933, f(1.15) = 0.3492
# and four-digit rounding arithmetic to construct a third Lagrange polynomial approximation for f(1.09)
x0, x1, x2, x3, val = 1, 1.05, 1.1, 1.15, 1.09
res = thirdLagrange(x0, x1, x2, x3, val)
print('The third Lagrange polynomial approximation for f(1.09) is {}'.format(res))
print('The third Lagrange polynomial approximation for f(1.09) rounded to 4 decimal places is {}'.format(round(res, 4)))

# The bound for error in approximation:
print('The bound for the error in this approximation is {}'.format(maxError([x0, x1, x2, x3], val)))

Semester 6/MA 322 (Scientific Computing Theory)/Labs/Lab 2/Code/q4.py


import argparse
from math import inf

import matplotlib.pyplot as plt
import numpy as np

# Create k random initial center points
def randCent(dataSet, k):
    n = np.shape(dataSet)[1]
    cpoints = np.zeros((k, n))
    # create centroid mat
    for j in range(n):
        # create random cluster centers, within bounds of each dimension
        minJ = np.min(dataSet[:, j])
        rangeJ = float(np.max(dataSet[:, j]) - minJ)
        cpoints[:, j] = minJ + rangeJ * np.random.rand(k)
    return cpoints

# Euclidean distance
def eucDist(pA, pB):
    return np.linalg.norm(pA - pB)

# K-means main method.
def kmeans(dataSet, k, distMeasure=eucDist, centerGen=randCent):
    m = np.shape(dataSet)[0]  # number of data
    cluResult = np.zeros((m, 2))  # center and distance of each point
    centers = centerGen(dataSet, k)  # the k centers
    cluChanged = True
    while cluChanged:
        cluChanged = False
        for i in range(m):
            minDist = inf
            minIndex = -1
            for j in range(k):
                # Step 1: find the nearest center for each point
                newDist = distMeasure(centers[j, :], dataSet[i, :])
                if newDist < minDist:
                    minDist = newDist
                    minIndex = j
            if cluResult[i, 0] != minIndex:
                cluChanged = True
                cluResult[i, :] = minIndex, minDist
        # Step 2: re-evaluate the center points
        for cent in range(k):
            ptsInClust = dataSet[np.where(cluResult[:, 0] == cent)]
            centers[cent, :] = np.mean(ptsInClust, axis=0)
    return centers, cluResult

def main():
    data = np.loadtxt('rawdata.csv', delimiter=',')
    parser = argparse.ArgumentParser(
        description='K-means clustering program.')
    parser.add_argument('-k', type=int, default=3,
                        help='The number of clusters. Default is 3.')
    args = parser.parse_args()
    dataSet = data[:, 0:-1]
    centers, cluResult = kmeans(dataSet, args.k)
    print('Centers:', centers)
    print('Clustering result:', cluResult)
    # Drawing
    colors=['black', 'blue', 'green', 'lime', 'maroon', 'olive', 'orange', 'purple', 'red', 'teal', 'yellow']
    np.random.shuffle(colors)
    for i in range(len(centers)):
        center=centers[i]
        cluPoint=dataSet[np.where(cluResult[:,0]==i)]
        plt.scatter(center[0],center[1],c=colors[i],marker='*')
        plt.scatter(cluPoint[:,0],cluPoint[:,1], c=colors[i])
    plt.show()

if __name__ == '__main__':
    main()


# coding: utf-8

# # Solving nonlinear equations (finding roots of functions)
# Find $x$ such that $f(x) = 0$

import numpy as np
import matplotlib.pyplot as plt

tol = 1e-10
max_steps = 100
f = lambda x: x**2 - 2.0
df = lambda x: 2*x
print("root of x^2 - 2:", np.sqrt(2))

# ## Newtons method
# From:
# $$ f(x^*) = 0 = f(x_0) + f'(x_0)(x^* - x_0) + \mathcal{O}(|x^* - x_0|^2) $$
# we get:
# $$ x_{n+1} \leftarrow x_n - \frac{f(x_n)}{f'(x_n)}$$

def newton(f, df, x0, tol=tol, max_steps=max_steps):
    x = x0
    step = 0
    while True:
        x_new = x - f(x)/df(x)
        if abs(x - x_new) <= tol or step == max_steps:
            return x_new, step
        x = x_new
        step += 1

print("newton:", newton(f, df, 10.0))

# ## Bisection
# We have two points $a$ and $b$. Now we compute the point between thous $c = \frac{a + b}{2}$. If $f(c)$ has the same sign as $f(a)$ then we can use $c$ as our new $a$, if $f(c)$ as the same sign as $f(b)$ we can use $c$ as our new $b$.

def shrink_interval(f, a, b, c):
    if np.sign(f(c)) == np.sign(f(a)):
        return c, b
    else:
        return a, c

def bisect(f, a, b, tol=tol, max_steps=max_steps):
    step = 0
    assert np.sign(f(a)) != np.sign(f(b))
    while True:
        c = (a + b)/2
        if abs(a - b) <= tol or step == max_steps or f(c) == 0.0:
            return c, step
        a, b = shrink_interval(f, a, b, c)
        step += 1

print("bisection:", bisect(f, 0.0, 10.0, tol=1e-15, max_steps=200))

# ## Secant method
# We have two points $a$ and $b$. Then we can approx. the root between these points by the root of the secant between the points.
# The secant hits $(a, f(a))$ and $(b, f(b))$.
# $$m = \frac{f(a) - f(b)}{a - b}$$
# $$b = f(a) - m a$$
# $$ 0 = mx + b \Rightarrow x = -\frac{b}{m}$$

def secant_root(f, a, b):
    m = (f(a) - f(b))/(a - b)
    d = f(a) - m*a
    return -d/m

def secant(f, x0, x1, tol=tol, max_steps=max_steps):
    step = 0
    while True:
        x2 = secant_root(f, x0, x1)
        if abs(x1 - x2) <= tol or step == max_steps:
            return x2, step
        x0, x1 = x1, x2
        step += 1

print("secant:", secant(f, 0.0, 10.0))

# ## Regula falsi (false position)
# Lets combine secant and bisection!
#
# Start with an interval $(a_0,b_0)$ where $sgn(f(a_0)) \neq sgn(f(b_0))$
# - Find root of the secant $c$ between $a_0$ and $b_0$
# - We are done if $f(c) = 0$ or we reached some tolerance.
# - Choose $a_1 = c$ if $sgn(f(c)) = sgn(f(a))$ otherwise choose $b_1 = c$ if $sgn(f(c)) = sgn(f(b))$

def regula_falsi(f, a, b, tol=tol, max_steps=max_steps):
    assert np.sign(a) != np.sign(b)
    step = 0
    while True:
        c = secant_root(f, a, b)
        if min(abs(a - c), abs(b - c)) <= tol or c == 0.0 or step == max_steps:
            return c, step
        a, b = shrink_interval(f, a, b, c)
        step += 1

print("regula_falsi:", regula_falsi(f, 0.0, 10.0))

# ## Fixpoint iteration
# Instead of solving $f(x) = 0$ for $x$, we turn this equation into $g(x) = x$ and then compute $x_{n + 1} = g(x_n)$ for some sufficently large $n$.

def fixpoint(g, x, tol=tol, max_steps=max_steps, debug=False):
    step = 0
    while True:
        new_x = g(x)
        if debug:
            print(x, new_x)
        if abs(x - new_x) <= tol or step == max_steps:
            return new_x, step
        x = new_x
        step += 1

# $$f(x) = x^2 - 2$$
# $$x^2 - 2 = 0$$
# $$g(x) = x^2 + x - 2$$
# $$x = x^2 + x - 2$$

fixpoint(lambda x: x**2 + x - 2., 1.4, max_steps=10, debug=True)

# $$f(x) = sin(x)$$
# $$sin(x) = 0$$
# $$g(x) = sin(x) + x$$
# $$x = sin(x) + x$$

# In[15]:

fixpoint(lambda x: np.sin(x) + x, 1.0, debug=True)


import numpy as np
import scipy.sparse as spar
import scipy.linalg as la
from scipy.sparse import linalg as sla

def to_matrix(filename,n):
    '''
    Return the nxn adjacency matrix described by datafile.
    INPUTS:
    datafile (.txt file): A .txt file describing a directed graph. Lines
        describing edges should have the form '<from node>\t<to node>\n'.
        The file may also include comments.
    n (int): The number of nodes in the graph described by datafile
    RETURN:
        Return a SciPy sparse `dok_matrix'.
    '''
    pass

def calculateK(A,N):
    '''
    Compute the matrix K as described in the lab.
    Input:
        A (array): adjacency matrix of an array
        N (int): the datasize of the array
    Return:
        K (array)
    '''
    pass

def iter_solve(adj, N=None, d=.85, tol=1E-5):
    '''
    Return the page ranks of the network described by `adj`.
    Iterate through the PageRank algorithm until the error is less than `tol'.
    Inputs:
    adj - A NumPy array representing the adjacency matrix of a directed graph
    N (int) - Restrict the computation to the first `N` nodes of the graph.
            Defaults to N=None; in this case, the entire matrix is used.
    d     - The damping factor, a float between 0 and 1.
            Defaults to .85.
    tol  - Stop iterating when the change in approximations to the solution is
        less than `tol'. Defaults to 1E-5.
    Returns:
    The approximation to the steady state.
    '''
    pass

def eig_solve( adj, N=None, d=.85):
    '''
    Return the page ranks of the network described by `adj`. Use the
    eigenvalue solver in \li{scipy.linalg} to calculate the steady state
    of the PageRank algorithm
    Inputs:
    adj - A NumPy array representing the adjacency matrix of a directed graph
    N - Restrict the computation to the first `N` nodes of the graph.
            Defaults to N=None; in this case, the entire matrix is used.
    d     - The damping factor, a float between 0 and 1.
            Defaults to .85.
    Returns:
    The approximation to the steady state.
    '''
    pass

def team_rank(filename='ncaa2013.csv'):
    '''
    Use your iterative PageRank solver to predict the rankings of the teams in
    the given dataset of games.
    The dataset should have two columns, representing winning and losing teams.
    Each row represents a game, with the winner on the left, loser on the right.
    Parse this data to create the adjacency matrix, and feed this into the
    solver to predict the team ranks.
    Inputs:
    filename (optional) - The name of the dataset.
    Returns:
    ranks - A list of the ranks of the teams in order "best" to "worst"
    teams - A list of the names of the teams, also in order "best" to "worst"
    '''
    pass


from __future__ import print_function

import numpy as np
from scipy import stats
import matplotlib.pyplot as plt

# ========== HW07 SOLUTION [Python2/3] ========== #

# ========== 1 ========== #

a = np.array([0.5507979, 0.70814782, 0.29090474, 0.51082761, 0.89294695, 0.89629309, 0.12558531, 0.20724288, 0.0514672, 0.44080984])
b = np.array([-0.04381817, -0.47721803, -1.31386475, 0.88462238, 0.88131804, 1.70957306, 0.05003364, -0.40467741, -0.54535995, -1.54647732])

# K-S 2 sample test
# null hypothesis: a and b are sampled from the same distribution
kstest = stats.ks_2samp(a, b)
kstest.pvalue < 0.05  # True; reject null hypothesis.
# conclude a and b are NOT from the same distribution.

# a histogram can show visually that they are not from the same distribution.
plt.figure()
plt.hist(a)
plt.hist(b)
plt.show()

# ========== 2 ========== #

np.random.seed(121)
binom_mars = stats.binom(100, 0.0925)
mars_rv = binom_mars.rvs(10)
print(np.mean(mars_rv))
print(binom_mars.cdf(10.0))

# ========== 3 ========== #

rv = np.load('Aluminum_youngs_moduli.npy')
est_loc = np.mean(rv)
est_scale = np.std(rv)

# fitting different distributions
norm_param = stats.norm.fit(rv)
laplace_param = stats.laplace.fit(rv)
maxwell_param = stats.maxwell.fit(rv)
logistic_param = stats.logistic.fit(rv)

# plot PDFs
x = np.linspace(10, 150, 200)
plt.figure()
plt.hist(rv, bins=21, normed=True)
plt.plot(x, stats.norm.pdf(x, *norm_param), label='norm')
plt.plot(x, stats.maxwell.pdf(x, *maxwell_param), label='maxwell_param')
plt.plot(x, stats.logistic.pdf(x, *logistic_param), label='logistic_param')
plt.plot(x, stats.laplace.pdf(x, *laplace_param), label='laplace')
plt.legend()
plt.show()

# plot CDFs
plt.figure()
plt.hist(rv, cumulative=True, bins=21, normed=True)
plt.plot(x, stats.norm.cdf(x, *norm_param), label='norm')
plt.plot(x, stats.maxwell.cdf(x, *maxwell_param), label='maxwell_param')
plt.plot(x, stats.logistic.cdf(x, *logistic_param), label='logistic_param')
plt.plot(x, stats.laplace.cdf(x, *laplace_param), label='laplace')
plt.legend()
plt.show()

# visually, we can eliminate the Maxwell distribution

# K-S tests
ks_norm = stats.kstest(rv, 'norm', norm_param)
ks_laplace = stats.kstest(rv, 'laplace', laplace_param)
ks_maxwell = stats.kstest(rv, 'maxwell', maxwell_param)
ks_logistic = stats.kstest(rv, 'logistic', logistic_param)

# ks_norm.pvalue -> 0.00016185237120902585
# ks_laplace.pvalue -> 0.75696987136141258
# ks_maxwell,pvalue -> 0.0
# ks_logistic.pvalue -> 0.14502969211826544

# with alpha=0.05, accept either the laplace or logistic.
# however, it is clear to be laplace due to the high p-value.

# note that we are looking for a LARGE p-value
# to accept the null hypothesis of coming from that distribution.


import numpy as np

# Core functions for logistics regression

def sigmoid(z):
    return 1 / (1 + np.exp(-z))

def hypothesis(X, theta):
    return X @ theta

def cost(theta,X,y):
    m,n = X.shape
    hx = sigmoid(hypothesis(X,theta.reshape(len(theta),1)))
    return np.sum((-y * np.log(hx)) - ((1-y) * np.log(1-hx))) * 1/m

def gradient(theta, X,y):
    m,n = X.shape
    hx = sigmoid(hypothesis(X,theta.reshape(len(theta),1)))
    g = ((hx - y).T @ X) * 1/m
    return g.flatten() if theta.ndim==1 else g

def map_features(x1,x2,degree):
    features = np.ones((len(x1),1))
    for i in range(1,degree+1):
        for j in range(0,i+1):
            z = np.power(x1 , (i-j)) * np.power(x2 , j)
            features = np.hstack([features,z])
    return features

def cost_regularized(theta, X, y, lmbda):
    m,n = X.shape
    n_cost = cost(theta,X,y)
    n_cost += np.sum((theta[2:len(theta):] ** 2)) * (lmbda/(2*m))
    return n_cost

def gradient_regularized(theta,X,y,lmbda):
    grad = gradient(theta,X,y)
    m,n = X.shape
    return grad + np.hstack([[[0]], ((lmbda/m) *  theta[1:,:]).T])


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# @Date    : Aug-13-20 17:12
# @Author  : Your Name (you@example.org)
# @RefLink : https://numpy.org/doc/stable/reference/generated/numpy.linalg.eig.html#numpy.linalg.eig
# @RefLink : https://numpy.org/doc/stable/reference/generated/numpy.linalg.solve.html

import numpy as np

def solve_test():
    """solve equation a * x = b
    """
    a = np.array([[3, 1], [1, 2]])
    b = np.array([9, 8])
    x = np.linalg.solve(a, b)
    print(f"x:{x}")

def eigen_test():
    """
    v: The normalized (unit “length”) eigenvectors, such that the column v[:,i] is the eigenvector corresponding to the eigenvalue w[i].
    """
    # A = np.matrix([[1, 2, 3], [4, 5, 6], [7, 8, 9]])
    A = np.asarray([[1, 2], [3, 4]])
    print(f"A:{A}")

    w, v = np.linalg.eig(A)

    print(f"w:{w}")
    print(f"v:{v}")
    A_restore = np.dot(np.dot(v, np.diag(w)), np.linalg.inv(v))
    print(f"A_restore:{A_restore}")

    error = np.sum(A_restore-A)
    print(f"error:{error}")

def main():
    eigen_test()

if __name__ == "__main__":
    main()


import numpy as np

def unit_step(t):
    """ Takes time as argument and returns a unit-step function """
    return 1*(t>=0)

def dirac_delta(t,Delta=1e-3):

	x_t = (1/Delta)*(unit_step(t+Delta/2) - unit_step(t-Delta/2) )
	return x_t

def Fourier_Transfrom(x_t,t):
	""" Computes CTFT using FFT algorithm.
	Frequency and amplitude are scaled appropriately.
	It is assumed that the time varible is equispaced.
	Numpy FFT algorithm is used to compute the Fourier Transfrom,
	therefore it is also assumed that the signal is periodic in time
	with a period T_range.
	Note that you should define the time from -T_range/2 to +T_range/2"""

	N_pts = np.size(x_t)
	T_range = t.max() - t.min()

	delta_t = T_range/N_pts

	X_fft = np.fft.fft(x_t)
	X_fft = np.fft.fftshift(X_fft)
	X_omega = X_fft*delta_t

	omega = (2*np.pi/delta_t)*np.fft.fftshift(np.fft.fftfreq(N_pts))
	#omega = (2*np.pi/delta_t)*np.fft.fftfreq(N_pts)

	return X_omega, omega

def Inverse_Fourier_Transform(X_omega,omega):
	"""Computes and returns continuous inverse Fourier Transfrom using numpy
	ifft function. Time and frequency are scaled appropriately."""

	omega_range = omega.max() - omega.min()
	delta_omega = omega_range/np.size(omega)

	x_t_ifft = np.fft.ifft(np.fft.fftshift(X_omega))*omega_range/(2*np.pi)
	t_ifft = (2*np.pi/delta_omega)*np.fft.fftshift(np.fft.fftfreq(np.size(omega)))
	return x_t_ifft, t_ifft


# -*- coding: utf-8 -*-
"""Quadrature implementations"""

from __future__ import division

import numpy as np

from typing import Tuple
from .typing import ArrayLike

def clenshaw_curtis(n: int, a: float = -1.0, b: float = 1.0) -> Tuple[ArrayLike,ArrayLike]:
    """
    Computes the points and weights for a Clenshaw-Curtis integration
    from a to b. In other words, for the approximation to the integral

    \int_a^b f(x) dx \approx \sum_{i=0}^{n} w_i f(x_i)

    with the Clenshaw-Curtis quadrature, this function returns the
    positions x_i and the weights w_i.
    """
    assert b > a and n > 1

    npoints = n
    nsegments = n - 1
    theta = np.pi * np.flip(np.arange(npoints)) / nsegments
    xx = np.cos(theta) * 0.5 * (b - a) + 0.5 * (a + b)

    wcc0 = 1.0/(nsegments*nsegments - 1 + (nsegments%2))

    # build v vector
    v = np.zeros(nsegments)
    v[:nsegments//2] = 2.0/(1.0 - 4.0 * np.arange(nsegments//2)**2)
    v[nsegments//2] = (nsegments - 3) / (2 * (nsegments//2) - 1) - 1

    kk = np.arange(1, npoints//2)
    v[nsegments-kk] = np.conj(v[kk])

    # build g vector
    g = np.zeros(nsegments)
    g[:nsegments//2] = -wcc0
    g[nsegments//2] = wcc0 * ( (2 - (nsegments%2)) * nsegments - 1 )
    g[nsegments-kk] = np.conj(g[kk])

    h = v + g
    wcc = np.fft.ifft(h)

    # sanity check
    imag_norm = np.linalg.norm(np.imag(wcc))
    assert imag_norm < 1e-14

    out = np.zeros(npoints)
    out[:nsegments] = np.real(wcc)
    out[nsegments] = out[0]
    out = np.flip(out) # might be redundant, but for good measure
    out *= 0.5 * (b - a)

    return xx, out

def midpoint(n: int, a: float = -1.0, b: float = 1.0) -> Tuple[ArrayLike,ArrayLike]:
    """
    Returns the points and weights for a midpoint integration
    from a to b. In other words, for the approximation to the integral

    \int_a^b f(x) dx \approx \frac{b-a}{n} \sum_{i=0}^n f((x_0 + x_1)/2)
    """
    assert b > a and n > 1

    weights = np.ones(n) * (b - a) / n
    points = a + ((b - a) / n * (np.arange(n) + 0.5))

    return points, weights

def trapezoid(n: int, a: float = -1.0, b: float = 1.0) -> Tuple[ArrayLike,ArrayLike]:
    """
    Returns the points and weights for a trapezoid integration
    from a to b. In other words, for the approximation to the integral

    \int_a^b f(x) dx \approx \frac{b-a}{n} \sum_{i=0}^n f((x_0 + x_1)/2)
    """
    assert b > a and n > 1

    ninterval = n - 1

    weights = np.ones(n) * (b - a) / ninterval
    weights[0] *= 0.5
    weights[-1] *= 0.5
    points = a + ((b - a) / ninterval) * np.arange(n)

    return points, weights

def simpson(n: int, a: float = -1.0, b: float = 1.0) -> Tuple[ArrayLike,ArrayLike]:
    """
    Returns the points and weights for a simpson rule integration
    from a to b. In other words, for the approximation to the integral

    \int_a^b f(x) dx \approx \frac{b-a}{n} \sum_{i=0}^n f((x_0 + x_1)/2)
    """
    assert b > a and n > 1

    if n%2 != 1:
        raise Exception("Simpson's rule must be defined with an odd number of points (even number of intervals)")

    ninterval = n - 1

    weights = np.ones(n)
    for i in range(1, ninterval-1, 2):
        weights[i] = 4.0
        weights[i+1] = 2.0
    weights[ninterval-1] = 4.0
    weights *= (b-a) / ninterval / 3.0

    points = a + ((b - a) / ninterval) * np.arange(n)

    return points, weights

def quadrature(n: int, a: float = -1.0, b: float = 1.0, method: str = "gl") -> Tuple[ArrayLike,ArrayLike]:
    """
    Returns a quadrature rule for the specified method and bounds
    """
    if method.lower() == "cc" or method.lower() == "clenshaw-curtis":
        return clenshaw_curtis(n, a, b)
    elif method.lower() == "gl" or method.lower() == "gauss-legendre":
        points, weights = np.polynomial.legendre.leggauss(n)
        points = points * 0.5 * (b - a) + 0.5 * (a + b)
        weights *= 0.5
        return points, weights
    elif method.lower() == "midpoint" or method.lower() == "mp":
        return midpoint(n, a, b)
    elif method.lower() == "trapezoid":
        return trapezoid(n, a, b)
    elif method.lower() == "simpson":
        return simpson(n, a, b)
    else:
        raise Exception("Unrecognized quadrature choice")


# %%
import numpy as np
import sympy as sym

# %%

def lagrange_fn(x_array, y_array, out_type):
    # Inputs-
    # x_array: evaluation points
    # y_array: function values at evaluation points
    #          (associated with the evaluation points of the same index)
    # out_type: 3 options - "fun" returns a function, "sym" returns symbolic
    # math object, "lat" returns latex code for the function

    # for consistency ensure arrays are numpy arrays
    x_array = np.asarray(x_array)
    x_array = np.asarray(x_array)

    # catch easy error
    if len(x_array) != len(y_array):
        raise ValueError('arrays not same length')

    # find length of array
    n = len(x_array)
    # create x
    x = sym.symbols('x')
    # create var
    expr = 0

    for i in range(n):
        # reset term
        term = 1
        for j in range(n):
            if i != j:
                # Add factor
                term = term * ((x - x_array[j]) / (x_array[i] - x_array[j]))

        # addend new term with forrect function value
        expr = expr + y_array[i] * term

    # simplify expression (effect will probably depend on terms)
    # Presumably will convert to the monomial basis(if you aren't a fan you
    # can comment this out)
    expr = sym.simplify(expr)

    # Function case
    if out_type == "fun":
        fun_out = sym.lambdify(x, expr)

    # symbolic math object case
    if out_type == "sym":
        fun_out = expr

    # Latex code case
    if out_type == "lat":
        fun_out = sym.latex(expr)

    return fun_out

    raise ValueError('bad type (need to set out_type as fun, sym, or lat)')


import math
import pandas as pd
import numpy as np
from sklearn import preprocessing
from sklearn.linear_model import LogisticRegression

FILE_PATH = "data/spam.csv"
THRESHOLD = 0.5
EPSILON = 1e-10

def read_file(filename):
    read_data = pd.read_csv(filename, header=None)
    return read_data

def standardize(df_to_std):
    x = df_to_std.iloc[:, 0:57].values
    y = df_to_std.iloc[:, 57]

    standard_scaler = preprocessing.StandardScaler()
    x_scaled = standard_scaler.fit_transform(x)

    df_x = pd.DataFrame(x_scaled)
    df_scaled = df_x.join(y)

    return df_scaled

def sigmoid(z):
    return 1 / (1 + np.exp(-z))

def loss(p, y, w, lambda_reg):
    return -np.average(y * np.log(p + EPSILON) + (1 - y) * np.log(1 - p + EPSILON)) \
           + lambda_reg * np.average(np.linalg.norm(w)) / 2

def get_x(df):
    return df.iloc[:, 0:-1].values

def get_y(df):
    return df.iloc[:, -1].values

def predict_probability(x, w):
    z = np.dot(x, w)
    return sigmoid(z)

def train(df, iterations=1500, learning_rate=0.1, lambda_reg=0.01, verbose=True):
    x = get_x(df)
    y = get_y(df)

    w_train = np.zeros(x.shape[1])

    for it in range(iterations):
        predictions = predict_probability(x, w_train)

        reg_w_train = w_train.copy()
        reg_w_train[0] = 0
        gradient = np.dot(x.T, (predictions - y)) / y.size
        regularization = lambda_reg * reg_w_train / y.size

        w_train -= learning_rate * (gradient + regularization)

        if verbose and it % (iterations/5) == 0:
            print("It. %4d\t|\tLoss: %0.4f" % (it, loss(predictions, y, w_train, lambda_reg)))
    return w_train

def predict(x, w, threshold=0.5):
    prob = predict_probability(x, w)
    return prob >= threshold

def accuracy(df, w, verbose=True):
    x = get_x(df)
    y = get_y(df)
    predictions = predict(x, w, threshold=THRESHOLD)

    acc = np.average(predictions == y)

    if verbose:
        print("Accuracy: %0.4f\n" % acc)
    return acc

def log_reg_sklearn(df):
    x = get_x(df)
    y = get_y(df)

    model = LogisticRegression(C=1e20)
    model.fit(x, y)

    predictions = model.predict(x)
    acc = np.average(predictions == y)
    print("sklearn: %0.4f" % acc)

    return model.coef_

def get_block_data(df, fold, tot_folds):
    fold_size = math.floor(df.shape[0] / tot_folds)

    start_index = fold_size * fold
    end_index = start_index + fold_size

    df_test = df.loc[start_index:end_index]
    df.drop(df.loc[start_index:end_index].index, inplace=True)

    return df, df_test

def shuffle(df):
    return df.sample(frac=1).reset_index(drop=True)

def cross_validation(df, l_r, l_reg, folds=10, verbose=True, sklearn=True):
    avg_acc = 0
    sk_acc = 0

    for i in range(folds):
        if verbose:
            print("Fold number " + str(i+1)
        tr_data, test_data = get_block_data(df.copy(), i, folds)
        w = train(
            tr_data,
            learning_rate=l_r,
            lambda_reg=l_reg,
            verbose=verbose
        )
        avg_acc += accuracy(test_data, w, verbose=verbose)

        if sklearn:
            x = get_x(tr_data)
            y = get_y(tr_data)

            model = LogisticRegression(C=1e20)
            model.fit(x, y)

            predictions = model.predict(get_x(test_data))
            sk_acc += np.average(predictions == get_y(test_data))

    avg_acc /= folds
    sk_acc /= folds

    print(BColors.OK_GREEN + "AVG acc: %0.4f" % avg_acc)
    print("AVG sklearn: %0.4f" % sk_acc)

if __name__ == '__main__':
    df = read_file(FILE_PATH)
    df = standardize(df)

    df.insert(0, 'A', pd.Series(np.ones(df.shape[0]), index=df.index))
    df = shuffle(df)

    cross_val = False

    if cross_val:

        cross_validation(
            df,
            l_r=0.1,
            l_reg=0.01
        )

        # for rate in [0.1, 0.05, 0.01, 0.005, 0.001]:
        #     for reg in [0.1, 0.05, 0.01, 0.005, 0.001]:
        #         print(str(rate), str(reg))
        #         cross_validation(
        #             df_shuffled,
        #             verbose=False,
        #             l_r=rate,
        #             l_reg=reg,
        #             sklearn=False
        #         )
    else:
        weights = train(df)

        accuracy(df, weights)

        w2 = log_reg_sklearn(df)


import numpy as np
import numpy.polynomial as pol
from typing import Callable, List, Union
Vector = List[float]
Function = Callable[[float], float]
Polynomial = pol.Polynomial

def lagrange2(X: Vector, Y: Vector) -> Function:
    """Returns the lagrange interpolation
    Args:
        X (Vector): X-data
        Y (Vector): Y-data
    Returns:
        Callable[[Vector], Vector]: returns the function that evaluates the lagrange interpolation
    """
    X = np.asarray(X)

    def lagran(x):
        # Checks the input's type
        if type(x) is np.ndarray:
            x = x.reshape(-1, 1)

        if isinstance(x, list):
            x = np.array(x).reshape(-1, 1)

        else:
            x = np.array([x]).reshape(-1, 1)

        out = 0
        for i in range(len(X)):
            #pi_x = (x - np.array([(X[X != xi]).reshape(-1)] * len(x))) because X[X != xi] autoreshape (-1)
            pi_x = x - np.array([(X[X != X[i]])] * len(x))

            out += Y[i] * (pi_x / (X[i] - X[X != X[i]])).prod(axis = 1)

        return out

    return lagran

def lagrange(X: Vector, Y: Vector) -> Polynomial:
    p = 0
    for i in range(len(X)):
        qn = pol.Polynomial.fromroots(X[X != X[i]])
        p += Y[i] * qn / qn(X[i])
    return p


# coding: utf-8

#
Table of Contents

#

1  Linear Regression

1.1  Baseline your model

1.1.1  Plot the initial dataset

1.1.2  Split datasets, fit a linear regression and measure $R^2$ over the test set

1.1.3  Interpretation

1.1.4  Visualization

1.1.5  Cross Validation

1.1.6  Main pipeline call

1.1.7  Visualize Validation Scores

1.1.8  Increase the number of folds in train and test

1.2  Using SKLEARN Pipelines

1.3  Features preprocessing

1.3.1  Simple Cross Validation score

1.4  Multiple Linear Regression

1.4.1  What's important in Multiple linear regression

1.5  Bias and variance over training

1.5.1  Plot the polynomial expression.

1.5.2  Training size effect

1.5.3  What about the degree of the polynomial?

# # Linear Regression
# Let's practice with basic linear regression concepts in Python. We will read the Advertising dataset used in the ISLR book, and we will perform linear with single and multiple coefficients.

# In[1]:

import random
import sys
import pandas as pd
import matplotlib.pyplot as plt
import numpy as np
import seaborn as sns
import statsmodels.api as sm

from sklearn.pipeline import Pipeline, make_pipeline
from sklearn.preprocessing import PolynomialFeatures, StandardScaler
from sklearn.model_selection import cross_validate, cross_val_score,         train_test_split, learning_curve, validation_curve
from sklearn import datasets, linear_model
from sklearn.metrics import mean_squared_error, r2_score
from sklearn.model_selection import cross_val_score, ShuffleSplit
from sklearn_pandas import DataFrameMapper
from statsmodels.api import add_constant

import warnings
warnings.filterwarnings('ignore')

# ## Baseline your model
#
# Start by reading the data into a pandas DataFrame

# In[2]:

url="http://www-bcf.usc.edu/~gareth/ISL/Advertising.csv"
adv=pd.read_csv(url)
adv.head(5)

# Remove the first column containing the index of each row as it's not necessary when using Pandas.

# In[3]:

adv = adv.drop(adv.columns[0], axis=1)

# There're only have fours predictors or variables or dimensions to play with. Our **target** variable is **Sales**: We want to predict the amount of sales that we will get, basen on the money that we invest in TV, radio or newspaper advertising channels.

# ### Plot the initial dataset
#
# Plot the two variables to see how they look like.

# In[4]:

plt.scatter(adv.TV, adv.sales,  color='grey')
plt.xlabel('TV'); plt.ylabel('Sales')
plt.show();

# ### Split datasets, fit a linear regression and measure $R^2$ over the test set
#
# As usual, we must start by splitting the dataset into training & test. This time I will do once again to get a validation split. To do so, I will appply the `train_test_split` twice over the training set to get the validation from there.
#
# The idea is to finally evaluate the model using the validation set.

# In[5]:

X = pd.DataFrame(adv.TV)
y = pd.DataFrame(adv, columns=['sales'])

X_train, X_test, y_train, y_test = train_test_split(
    X, y, test_size=0.2, random_state=25)
X_train, X_val, y_train, y_val = train_test_split(
    X_train, y_train, test_size=0.2, random_state=12)

lm = linear_model.LinearRegression()
lm.fit(X_train, y_train)
y_pred = lm.predict(X_test)

print('Intercept:', lm.intercept_)
print('Coefficients:', lm.coef_)
print('Mean squared error (MSE): {:.2f}'.format(mean_squared_error(y_test, y_pred)))
print('Variance score (R2): {:.2f}'.format(r2_score(y_test, y_pred)))

# ### Interpretation
#
# - The intercept is the expected mean value of the target variable when all predictors are set to 0.
# - The coeffient is the slope of the line that explains the regression.
# - The R2 is the percentage of explained variance of the predictions.
#
# But, to really see how good is our model, we must evaluate the score ($R^2$) over a validation split --a different split of the data set. We will understand later, why this is important. I'm getting $R^2=0.51$ over this specific test set.
#
# However, if you check the results below, you'll see that the $R^2$ got when predicting over the validation dataset are slightlit better: from $R^2=0.51$ to $R^2=0.64$ !!
#
# **Why is that?** In this case, for this specific splits, it turns out that the validations split is probably more similar to the training set, than the test set. And that is why our model is better predicting values from that split, than from the test split.

# In[6]:

y_generalize = lm.predict(X_val)

print('Intercept:', lm.intercept_)
print('Coefficients:', lm.coef_)
print('Mean squared error: {:.2f}'.format(mean_squared_error(y_val, y_generalize)))
print('Variance score (R2): {:.2f}'.format(r2_score(y_val, y_generalize)))

# ### Visualization
#
# Now, plot the original points used to train the linear regression (grey) and the line representing the function that better approximates this relationship between TV and sales that we got from applying the linear regression.
#
# If you look at the code, you'll see that the way to obtain the values of Y later plotted using `plt.plot` come from directly applying the function describing the linear regression:
#
# $$\hat{y} = \beta_0 + \beta_1 X_{val}$$
#
# Correspondence:
# - $\beta_0$ is `lm.intercept_`
# - $\beta_1$ is `lm.coef_[0]`

# In[7]:

plt.scatter(X_train, y_train, color='gray')
x_vals = X_val.TV.values
y_vals = lm.intercept_ + lm.coef_[0] * x_vals
plt.plot(x_vals, y_vals, '--');

# ### Cross Validation
#
# As you might possible have figured out, we haven't used the validation dataset, and our model has learnt only from the first (and only) split that we made between training and test. In order to improve our results, a good approach is to repeat the splitting process again and again to obtain the best possible subset of individuals that lead to the best possible model.
#
# To do so, we should repeat the split and validation in a loop to select the best one. Here you can find a possible way of accomplishing that task.

# In[8]:

def split_datasets(X, y, seed,
                   split_test_size=0.2,
                   validation_split=True):
    """
    Split X and y dataframes into training, test and validation datasets,
    using the provided random seed.
    Returns a dictionary with the datasets
    """
    X_train, X_test, y_train, y_test = train_test_split(
        X, y, test_size=split_test_size, random_state=seed)

    split = dict()
    if validation_split is True:
        X_train, X_val, y_train, y_val = train_test_split(
            X_train, y_train, test_size=split_test_size, random_state=seed)
        split['X_val'] = X_val
        split['y_val'] = y_val

    split['X_train'] = X_train
    split['X_test'] = X_test
    split['y_train'] = y_train
    split['y_test'] = y_test

    return split

# In[9]:

def prepare_datasets(data, features, target,
                     seed=1024,
                     test_size=0.2,
                     validation_split=False):
    """
    From an input dataframe, separate features from target, and
    produce splits (with or without validation).
    """
    X = pd.DataFrame(adv, columns=features)
    y = pd.DataFrame(adv.loc[:, target])
    split = split_datasets(X, y, seed=seed,
                           split_test_size=test_size,
                           validation_split=validation_split)
    return split

# In[10]:

def train_and_predict(X, y, num_folds=1):
    """
    Call the splitting of the dataset, train a linear regression
    and predict with the test set.
    Returns the model, the splits used and the R2 score.
    """
    lm = linear_model.LinearRegression()
    test_r2 = 0.0
    for i in range(num_folds):
        seed = random.randrange(2**32 - 1)
        split = prepare_datasets(adv, ['TV'], 'sales',
                                 seed=seed, validation_split=True)

        lm.fit(split['X_train'], split['y_train'])
        y_pred = lm.predict(split['X_test'])
        test_r2 += r2_score(split['y_test'], y_pred)

    test_r2 = test_r2/num_folds
    return lm, split, test_r2

# In[11]:

def validate(model, X_val, y_val):
    """
    Scores a model over the validation datasets, by measuring the R2 score.
    Returns the r2.
    """
    y_generalized = lm.predict(X_val)
    r2 = r2_score(y_val, y_generalized)
    return r2

# In[12]:

def my_pipeline(X, y, max_iterations, num_folds=1):
    """
    Repeats during `max_iterations` the process of
      1) Splitting a dataset, training a linear regression
         and measuring its score over test set
      2) Store its R2 and compute its R2 over the validation dataset
    """
    history = dict()
    history['test_r2'] = np.array([])
    history['val_r2'] = np.array([])
    for num_iterations in range(max_iterations):
        model, split, test_r2 = train_and_predict(X, y, num_folds=num_folds)
        val_r2 = validate(model, split['X_val'], split['y_val'])
        history['test_r2'] = np.append(history['test_r2'], test_r2)
        history['val_r2'] = np.append(history['val_r2'], val_r2)
    return history

# ### Main pipeline call
#
# We are ready now to call the pipeline. The required arguments are the X and y dataframes, and the numner of iterations we want it to run. We can also specify how many runs we want each model to be trained over the same split of training and test, which by default is 1. We will keep it like that for the moment.
#
# It's quite impressive to check how our simple linear regression jumps from a bad 0.64 validation $R^2$ to an impressive $0.84$ ! And only by repeating the shuffling of the training and test sets.

# In[13]:

scores = my_pipeline(X, y, max_iterations=100)
max_val_r2 = np.amax(scores['val_r2'])
print('Best Validation R2: {:.2f}'.format(max_val_r2))

# ### Visualize Validation Scores
#
# To get an idea of what we got, let's draw a density plot with the validation score values saved along the different iterations. The density plot illustrates how important is to shuffle the training, test and validation sets to find out what is the one which is producing the best generalization.

# In[14]:

sns.distplot(scores['val_r2'], hist = False, kde = True,
             kde_kws = {'shade': True, 'linewidth': 3});

#
# ### Increase the number of folds in train and test
#
# What if I increase the number of folds from 1 to 10? The results are plotted below and there's n significant improvement in the results of the validation scores.
#
# Why? Well, It's namely because of the way we implemented our train & predict method: we're training `num_folds` linear regression without keeping it for test scoring. If we do that, we will explore which model will better generalize over different test sets, but keeping a single training split. And that will for sure, be a better method to assess over the the validation split.

# **Important**
#
# By partitioning the available data into three sets, we drastically reduce the number of samples which can be used for learning the model, and the results can depend on a particular random choice for the pair of (train, validation) sets.
#
# A test set should still be held out for final evaluation, but the validation set is no longer needed when doing CV. In the basic approach, called k-fold CV, the training set is split into k smaller sets.

# In[15]:

score_2 = my_pipeline(X, y, max_iterations=100, num_folds=10)
max_val_r2 = np.amax(score_2['val_r2'])
print('Best Validation R2: {:.2f}'.format(max_val_r2))

fig, ax = plt.subplots()
sns.distplot(scores['val_r2'], hist = False, kde = True,
             label='1-fold',
             kde_kws = {'shade': True, 'linewidth': 3}, ax=ax);
sns.distplot(score_2['val_r2'], hist = False, kde = True,
             label='10-fold',
             kde_kws = {'shade': True, 'linewidth': 3}, ax=ax);
plt.legend(loc='best')
plt.show();

# ## Using SKLEARN Pipelines
#
# Instead of having you to program everything from scratch, you can also access `sklearn` API for building preprocessing and cross validation pipelines. In the example below, we put in practice the very same approach used before, but using only three lines of code (!).
#
# I strongly recommend you to explore deeper the capabilities of the cross validation functions in SKLearn.

# ## Features preprocessing
#
# Before we go deeper, we must ensure that our features fulfill the set of requirements that are imposed by the linear regression method: centered and with variance = 1, following a normal ditribution and no correlation among them, namely. To do so, we will build a transformation pipeline. For more details, please, refere to the page: https://www.kaggle.com/baghern/a-deep-dive-into-sklearn-pipelines
#
# This case we're using the sklearn-pandas integration library `sklearn_pandas` which makes transformation pipelines much easier to read and understand. In the example below we're simply applying a standard scaler.

# In[16]:

mapper = DataFrameMapper([(list(adv), StandardScaler())], df_out=True)
res = np.round(mapper.fit_transform(adv), 2)
res.columns = list(adv)  # Set the columns names as they were.

# ### Simple Cross Validation score
#
# As we said, instead of having to program every single function in charge of splitting the datasets, setting the random seed, feeding a model with the training part and evaluating with test part, we will explore the `cross_val_score` method. It will be able to produce a single scoring (if you want more than one, you must use `cross_validate`) and will allow you to decide what CV strategy to use, or build a custom one.
#
# In the example below, we feed the `cross_val_score` with
# - the model (or pipeline) to be fit (`model`),
# - the features (`X`),
# - the target datasets (`y`),
# - the cross validation strategy (`cv`), and
# - the validation metric we want as a result (`r2`)
#
# This will run the cross validation with the model over the features and target using a ShuffleSplit (similar to _k-fold_ but with random splits), _n_splits_ (1000) times, evaluating with a 20% split as a test set, and returning the R2 achieved.

# In[17]:

model = linear_model.LinearRegression()
cv = ShuffleSplit(n_splits=100, test_size=0.2, random_state=666)
scores = cross_val_score(model, X, y, cv=cv, scoring='r2')
print('Best Validation R2: {:.2f}'.format(max(scores)))

sns.distplot(scores, hist = False, kde = True,
             kde_kws = {'shade': True, 'linewidth': 3});

# ## Multiple Linear Regression
#
# We want to consider all possible predictors available when it comes to train model. So, it's time to use the other two columns in the original dataset: radio and newspapers.

# In[18]:

X = pd.DataFrame(adv.loc[:, ['TV','radio','newspaper']])
y = pd.DataFrame(adv, columns=['sales'])
X.head(2)

# To train a multiple linear regression, the only thing that we must do is to feed the model with all the variables that you want to consider for the final equation. Obviously, building a linear model with 3 features, implies that our model is better described in 3+1 dimensions, which makes this task slightly more difficult to plot.

# In[19]:

mlm, split, test_r2 = train_and_predict(X, y)
print('Multiple linear regression R2: {:.2f}'.format(test_r2))
print('Multiple linear regression coeffcients: {}'.format(mlm.coef_))

# Repeating our CV experiment with SKLearn, this time replacing the single-feature $X$ by a 3-features dataframe $X$, leads to a similar result ($0.96$).

# In[20]:

clf = linear_model.LinearRegression()
cv = ShuffleSplit(n_splits=100, test_size=0.2, random_state=0)
scores = cross_val_score(clf, X, y, cv=cv, scoring='r2')
print('Best Validation R2: {:.2f}'.format(max(scores)))

sns.distplot(scores, hist = False, kde = True,
             kde_kws = {'shade': True, 'linewidth': 3});

# ### What's important in Multiple linear regression
#
# There's two concepts that are important when running a multiple linear regression: _p_values_ and the R-squared. Take a look to the values below in order to understand if this multiple linear regression is a better approximation to the original problem than the single-predictor linear regression.

# In[21]:

X = add_constant(X)
model = sm.OLS(y, X).fit()
print(model.summary())

# ## Bias and variance over training
#
# One of the advantages of linear regression is that we can make it as much complicated or fitted to data as we want. We simply add polynomial interactions between variables the following way:
#
# - Split datasets to hold out a test set, and use the training set for a cross-validation model selection
# - Build an sklearn _pipeline_ to randomly produce polynomial expressions before fitting a linear regression
# - Perform a CV validation score over the test set to check how do the models perform.

# We will extract some parts of the code to separate functions for the sake of code clarity, and better readibility. The first function (`xy_values`) will simply produce a dataframe with columns `x` and `y` after applying the prediction stage to an input dataframe. The second function (`polynomial_pipeline`) will construct a pipeline in which at first, a polynomial is found over input dataset, and second, a linear regression is fit (OLS) with that polynomial.

# In[22]:

def xy_values(pipeline, X):
    """ Returns the x and y values sorted by X in a dataframe """
    y_pred = pipeline.predict(X)
    return pd.DataFrame({'x': list(X.values), 'y': list(y_pred)}).                        sort_values(by=['x'])

# In[23]:

def polynomial_pipeline(X, y, degree):
    """
    Build a pipeline with polinomial expressions and linear regression over it.
    """
    cv = ShuffleSplit(n_splits=100, test_size=0.2, random_state=555)
    poly = PolynomialFeatures(degree=degree, interaction_only=False,
                              include_bias=False)
    lm = linear_model.LinearRegression()
    pipeline = make_pipeline(poly, lm)
    pipeline.fit(X, y)
    return pipeline

# In[24]:

split = prepare_datasets(adv, ['TV'], 'sales')
pipeline = polynomial_pipeline(split['X_train'], split['y_train'], degree=2)
scores = cross_val_score(pipeline, split['X_test'], split['y_test'],
                         scoring="r2", cv=cv)
print('The polynomial features used are:\n',
      pipeline.get_params('polynomialfeatures')['polynomialfeatures'].\
      get_feature_names())
print('Best Validation R2: {:.2f}'.format(max(scores)))

# ### Plot the polynomial expression.
#
# As you can easily check for this simulation, the higher degree polynomial means a better fit to the training but poorer generalization. It seems that here we might be overfitting a bit.

# In[25]:

f, (ax1, ax2) = plt.subplots(nrows=1, ncols=2)
f.set_figwidth(11)
ax1.scatter(split['X_train'].values, split['y_train'].values, color='grey')

pred = xy_values(pipeline, split['X_test'])
ax1.plot(pred.x.values, pred.y.values, label='prediction',
         linewidth=3, color='green')
ax1.grid()
ax1.set_title('2 degree polynomial prediction')

ax2.hist(scores[scores>0], 50, normed=1, alpha=0.5);
ax2.set_title('histogram of scores')
plt.show();

# ### Training size effect
#
# A learning curve shows the validation and training score of an estimator for varying numbers of training samples. It is a tool to find out how much we benefit from adding more training data and whether the estimator suffers more from a variance error or a bias error.
#
# A cross-validation generator splits the whole dataset $k$ times in training and test data. Subsets of the training set with varying sizes will be used to train the estimator and a score for each training subset size and the test set will be computed. Afterwards, the **scores will be averaged** over all $k$ runs for each training subset size.
#
# Can you interpret what is the number of samples you need to feed your model in training, in order to get the best possible result? When adding more samples does not add significant advantage?

# In[26]:

def plot_training_curve():
    plt.plot(train_sizes, np.mean(train_scores, axis=1), '-o',
             label='train_scores')
    plt.plot(train_sizes, np.mean(valid_scores, axis=1), '-o',
             label='validation_scores')
    plt.title('Training curve')
    plt.legend(loc='best')
    plt.xlabel('Training set size')
    plt.ylabel('Explained variance ($R^2$)')
    plt.grid()
    plt.show()

# In[27]:

split = prepare_datasets(data=adv, features=['TV'], target='sales', test_size=0.0)
cv = ShuffleSplit(n_splits=100, test_size=0.2, random_state=666)
train_sizes, train_scores, valid_scores = learning_curve(
    lm, split['X_train'], split['y_train'],
    train_sizes=[50, 75, 100, 125, 150, 175],
    cv=10)
plot_training_curve()

# ### What about the degree of the polynomial?
#
# Let's work on the `sklearn.model_selection.validation_curve` to see how the degree of the polynomial affects the R2 score. We pass an array of possible degrees for the polynomials to the `validation_curve` method.
#
# Compute scores for an estimator with different values of a specified parameter (degree of polynomial). This is similar to grid search with one parameter. However, this will also compute training scores and is merely a utility for plotting the results.

# In[28]:

def plot_validation_curve():
    # Plot the mean train error and validation error across folds
    plt.figure(figsize=(6, 4))
    plt.plot(degrees, train_scores.mean(axis=1), '-o',
             lw=2, label='training')
    plt.plot(degrees, validation_scores.mean(axis=1), '-o',
             lw=2, label='cross-validation')
    plt.title('Validation curve')
    plt.legend(loc='best')
    plt.xlabel('degree of polynomial')
    plt.ylabel('explained variance ($R^2$)')
    plt.tight_layout()
    plt.grid()
    plt.show()

# In[29]:

split = prepare_datasets(data=adv, features=['TV'], target='sales', test_size=0.0)
lm = linear_model.LinearRegression()
degrees = np.arange(1, 10)
model = make_pipeline(PolynomialFeatures(), lm)
cv = ShuffleSplit(n_splits=100, test_size=0.2, random_state=666)

train_scores, validation_scores = validation_curve(
                 model, split['X_train'], split['y_train'],
                 param_name='polynomialfeatures__degree',
                 cv=cv,
                 param_range=degrees)
plot_validation_curve()

# It seems that increasing the degree of the polynomial beyond '2' produces a decrease in validation accuracy.
#
# Now try to iterate over different maximum degrees for the polinomials to be generated to see how they affect the overall score of the model.

# In[30]:

plt.figure(figsize=(6, 4))
plt.scatter(split['X_train'].values, split['y_train'].values, color='grey')

split = prepare_datasets(adv, ['TV'], 'sales', seed=1024)
cv = ShuffleSplit(n_splits=100, test_size=0.2, random_state=666)

for degree in range(1, 6):
    pipeline = polynomial_pipeline(split['X_train'], split['y_train'], degree)
    scores = cross_val_score(pipeline, split['X_test'], split['y_test'],
                             scoring="r2", cv=10)
    pred = xy_values(pipeline, split['X_test'])
    plt.plot(pred.x.values, pred.y.values,
             label='deg.{}'.format(degree))

plt.title('Linear regression with different polynomial degrees')
plt.legend(loc='best')
plt.grid()
plt.show()

import numpy,import statsmodels,from statsmodels


import pathlib  # needed to create folder
import matplotlib.pyplot as plt  # needed for graphs
from tqdm import tqdm
import numpy as np

print('Program started running!')

pi = np.pi
dpisiz = 1000  # for images resolution

#######################################
# Lambda anonymous functions          #
#######################################

f = lambda x, e, k: x - k - e * np.sin(x)
df = lambda x, e: 1 - e * np.cos(x)

########################################
# Creating Our Newton-Raphson function #
########################################
def newton_raphson(f, df, x, e, TOL, k):
    error = 1
    iterations = 0
    while error > TOL:
        new_x = x - f(x, e, k) / df(x, e)
        error = abs(new_x - x)
        x = new_x
        iterations += 1
        # print(f'x{iterations}: {x:.15f} \t | \t error: {error} \t | \t f(x):{f(x, e,k)} \t | \t df(x):{df(x, e)} \t | \t line:{df(x, e)} * x+{f(x, e,k)}')
    # print(f"Newton's Estimate = {x:.15f}\nIterations: {iterations}")
    return x

#####################################
# Function to save images to folder #
#####################################
def savim(dir, name):
    path = pathlib.Path(f"./{dir}")
    path.mkdir(exist_ok=True,  # Without exist_ok=True,FileExistsError show up if folder already exists
               parents=True)  # Missing parents of the path are created.
    plt.savefig(f'./{dir}/{name}.png', dpi=dpisiz)
    print(f'Saved image at location: /{dir}/{name}.png')

numberoftries = 10000
i = np.empty((numberoftries, 6))
i[:, 0] = np.linspace(0, 2 * pi, numberoftries)
e = [0.1, 0.3, 0.5, 0.7, 0.9]

for j in tqdm(range(1, 6)):
    for k in range(numberoftries):
        i[k, j] = newton_raphson(f, df, 1, e[j - 1], 1e-15, i[k, 0])  # Adding NP-estimates to array
e = [0.1, 0.3, 0.5, 0.7, 0.9]

# Plotting our function
fig = plt.figure()
ax = plt.axes()
lines = ax.plot(i[:, 0], i[:, 0:5])
ax.set_ylabel('E')
ax.set_xlabel('M')
ax.set_title('Project 1 - Task 1 - Newton-Raphson method')
labels = [0] * 5
for o in range(0, 5):
    labels[o] = f'E(M) for e={e[o]}'
ax.legend(lines, labels)
savim('./pr1_task1/images', 'plot')

print('Program finished running!')

##############################################################################
#    A faster convergence could be achieved with a better initial value.     #
#          Current code uses a standard initial value of one.                #
##############################################################################


# -*- coding: utf-8 -*-
"""
Created on Wed Oct  7 08:25:49 2020
@author: Ashlee
ODE Example 1
$\frac{dx}{dt} = b1-b2*x
"""

import numpy as np
from scipy.optimize import curve_fit
from scipy.integrate import odeint
import matplotlib.pyplot as plt

# Data for example 1
xaxisData = np.array( [0.0, 0.5, 1.0, 5.0, 30.0] ) # time, independent variable
yaxisData = np.array( [0.0, 0.5, 1.2, 2.5, 2.7] ) # x, dependent variable

# guesses for parameters
b1guess = 1.0
b2guess = 1.0
parameterguesses = np.array([b1guess, b2guess])

# Need two functions for our model
# 1. to define the system of ODE(s)
# 2. to solve the ODE(s) and return ypredicted values in same shape as yaxisData

# 1. define ODE
def system_of_ODEs(x,t,parameters): # yvar, xvar, args
    # unpack the parameters
    b1 = parameters[0]
    b2 = parameters[1]
    dxdt = b1-b2*x
    return dxdt
# end of function

# 2. Solve ODEs at xaxisData points
    # and return calculated yaxisCalculated
    # using current values of the parameters
def model(xaxisData,*params):
    # initial condition(s) for the ODE(s)
    yaxis0 = 0.0 # should include a decimal
    yaxisCalc = np.zeros(xaxisData.size)
    for i in np.arange(0,len(xaxisData)):
        if xaxisData[i] == 0.0: # should include a decimal
            yaxisCalc[i] = yaxis0
        else:
            xaxisSpan = np.linspace(0,xaxisData[i],101)
            ySoln = odeint(system_of_ODEs,yaxis0,xaxisSpan,args = (params,)) # soln for entire xaxisSpan
            yaxisCalc[i] = ySoln[-1] # calculated y at the end of the xaxisSpan
    return yaxisCalc
    # end of for loop
# end of model function

# Estimate the parameters
parametersoln, pcov = curve_fit(model,xaxisData,yaxisData,p0=parameterguesses)
print(parametersoln)
plt.plot(xaxisData,yaxisData,'o',label='data')
yaxis0 = 0.0
xaxisForPlotting = np.linspace(0,xaxisData[-1],101)
yaxisCalcFromGuesses = odeint(system_of_ODEs,yaxis0,xaxisForPlotting,args = (parameterguesses,))
yaxisCalc = odeint(system_of_ODEs,yaxis0,xaxisForPlotting,args = (parametersoln,))
plt.plot(xaxisForPlotting,yaxisCalcFromGuesses,'r-',label='output with parameter guesses') # before fitting
plt.plot(xaxisForPlotting,yaxisCalc, 'g--', label='output with estimated parameters') # at soln parameters
plt.xlabel('t')
plt.ylabel('x')
plt.legend()
plt.show()

CHEclassFa20/In Class Problem Solutions/Python/ODEParamEstimExample1.py