"""
    Projects the given Hermitian matrix on the set of density matrices (positive, trace one operators).

    Author: Akshay Seshadri
"""

import numpy as np

import project_root # noqa
from src.optimization.project_simplex import project_on_simplex_bisection

def project_on_density_matrices_flattened(V):
    """
        Projects a Hermitian matrix V on the set of density matrices X = {A \in C^{N x N} | A^\dagger = A, A >=0, tr(A) = 1}.

        Note that a Hermitian matrix must be provided. Further, the input can be the matrix or its flattened vector (row-major/C-style flattening
        assumed, which is the default of numpy). Checks are not done to ensure this is the case.

        The output is a (row-major) flattened vector of the projection.
    """
    n = int(np.sqrt(V.size))                                        # dimension of V

    V = V.reshape((n, n))                                           # V as a matrix

    V_eigvals, V_eigvecs = np.linalg.eigh(V)                        # get the spectral decomposition of V

    # project on to the density matrices by projecting the eigenvalues of V on the probability simplex
    Pi_V_eigvals = project_on_simplex_bisection(V_eigvals)

    # construct the projection form the eigevectors of V and the projected eigenvalues: Pi_V = U D_Pi U^\dagger
    Pi_V = V_eigvecs.dot( np.diag(Pi_V_eigvals).dot( V_eigvecs.T.conj() ) )

    # return the (row-major) flattened output
    return Pi_V.ravel()


"""
This script provides functions for linear least square fitting
"""

import numpy as np
import sys
from matplotlib import pyplot

def lls_fit(X, Y):
    """Given X and Y, solve A that optimizes the 2-norm difference between
    A.T @ X and Y.

    Attributes:
        X (np.ndarray):
            Dimension: n-by-N. X = [x^1, x^2, \cdots{}, x^N] where x^i is the
            i-th data point vector.
        Y (np.ndarray):
            Dimension: m-by-N. Y = [y^1, y^2, \cdots{}, y^N] where y^i is the
            i-th target vector.

    Returns:
        A (np.ndarray):
            Dimension: n-by-m.
    """
    if X.ndim == 1:
        A = X@Y / (X@X)
    else:
        A = np.linalg.pinv(X@X.T) @ X@Y.T
    return A

def lls_predict(A, X):
    """Given X and A, compute predicted value Y = A.T@X
    """
    if X.ndim == 1:
        return A * X
    else:
        return A.T @ X

if __name__ == "__main__":
    if len(sys.argv) != 3:
        print("Usage: dataX dataY")
        sys.exit(1)

    # read data
    X = np.loadtxt(sys.argv[1])
    Y = np.loadtxt(sys.argv[2])

    # sort data
    if X.ndim == 1:
        inds = np.argsort(X)
        Xnew, Ynew = [], []
        for i in inds:
            Xnew.append(X[i])
            Ynew.append(Y[i])
        X = np.array(Xnew)
        Y = np.array(Ynew)

    # put data into the desired format
    if X.ndim != 1:
        X = X.T
        Y = Y.T

    # fit
    A = lls_fit(X, Y)

    # predict
    x = np.arange(0, 20, 0.5)
    pred_Y = lls_predict(A, x)

    # plot and compare
    if X.ndim == 1:
        pyplot.plot(X, Y, 'ko', label="data")
        pyplot.plot(x, pred_Y, 'r', label='fitted')
        pyplot.show()


import numpy as np
import math

def get_scaling_factor(n):
    k = n // 2
    if n % 2 == 0:
        scaling_factor = np.sqrt(2 / (np.pi * (2 * k - 1))) \
                * (2 ** (2 * k - 2) * math.factorial(k-1)**2) / math.factorial(2 * k - 2)
    else:
        scaling_factor = np.sqrt(np.pi / k) * math.factorial(2 * k - 1) / \
                (2 ** (2 * k - 1) * math.factorial(k - 1)**2)
    return scaling_factor

def ustd(x):
    """
    See https://en.wikipedia.org/wiki/Unbiased_estimation_of_standard_deviation#Bias_correction
    """
    # Reshape to handle both 1d and 2d arrays
    if x.ndim == 1:
        n = x.size
        unbiased_variance = np.var(x, ddof=1)
    else:
        n = x.shape[1]
        unbiased_variance = np.var(x, ddof=1, axis=1)

    unbiased_std = np.sqrt(unbiased_variance) / get_scaling_factor(n)

    return unbiased_std

if __name__ == "__main__":
    assert abs(get_scaling_factor(2) - 0.7978845608) < 1e-6
    assert abs(get_scaling_factor(3) - 0.8862269255) < 1e-6
    assert abs(get_scaling_factor(4) - 0.9213177319) < 1e-6
    # test that this actually works
    x = np.random.normal(size=(100000,4))
    sigma = ustd(x)
    assert abs(np.mean(sigma) - 1) < 0.005


import numpy as np

def energy(X,W_fixed):
    """
    Input X is a vector (N by 1), and the shape of W_fixed is N by N.\n
    Return a 2D array E with shape (1,1)
    """
    return (-1/2) * np.dot(np.dot(np.transpose(X),W_fixed),X)

def f(W,X):
    """
    W : W_fixed after storaging phase with shape (N,N) \n
    X : testing pattern X with shape (N,1) \n
    Return X_new with shape (N,1)
    """
    i = 0
    v = np.dot(W,X)
    for value in v :
        if value > 0 :
            v[i,0] = 1
        elif value < 0 :
            v[i,0] = -1
        else :
            v[i,0] = X[i,0]
        i += 1
    return v

X0 = np.array([ 1,-1, 1,-1, 1,-1])
X2 = np.array([-1, 1,-1, 1,-1, 1])
X4 = np.array([ 1, 1, 1, 1, 1, 1])
X6 = np.array([-1,-1,-1,-1,-1,-1])

X0      = X0.reshape(1,X0.size)
X2      = X2.reshape(1,X2.size)
X4      = X4.reshape(1,X4.size)
X6      = X6.reshape(1,X6.size)
X       = np.array([X0,X2,X4,X6])
W_fixed = np.zeros([X0.shape[1],X0.shape[1]])

for p in range(X.shape[0]):
    W_fixed = W_fixed + np.dot(X[p,:,:].reshape(-1,1),X[p,:,:].reshape(1,-1))
for i in range(X.shape[2]):
    for j in range(X.shape[2]):
        if i==j:
            W_fixed[i,j]=0

X_test = X0.reshape(-1,1)                    #-- change label index
E = energy(X_test,W_fixed)
#------------  obtain new X  -------------
E_cycle = [E]
print('-------------------------------------')
print('E(0) : ',E[0,0])
print('X(0) : ',X_test[:,0])
X_new = X_test

for cycle in range(100):
    X_new = f(W_fixed,X_new)
    E = energy(X_new,W_fixed)
    E_cycle.append(E[0,0])

    print('-------------------------------------')
    print('E({}) : {}'.format(cycle+1,E[0,0]))
    print('X({}) : {}'.format(cycle+1,X_new[:,0]))

    if E_cycle[cycle+1] == E_cycle[cycle] :
        print('-------------------------------------')
        print('cycle :',cycle+1)
        print('E converges : {}'.format(E[0,0]))
        print('X final : {}'.format(X_new[:,0]))
        break


from scipy.linalg import lstsq
import pandas as pd
import numpy as np
import argparse

def landmark_registration_ls(pts_f1, pts_f2, out_f, delimiter="\t"):

    points1 = pd.read_csv(pts_f1, delimiter=delimiter)
    points2 = pd.read_csv(pts_f2, delimiter=delimiter)

    src = np.concatenate([np.array(points1['x']).reshape([-1,1]),
                          np.array(points1['y']).reshape([-1,1])],
                         axis=-1)
    dst = np.concatenate([np.array(points2['x']).reshape([-1,1]),
                          np.array(points2['y']).reshape([-1,1])],
                         axis=-1)

    A = np.zeros((src.size,6))
    A[0:src.shape[0],[0,1]] = src
    A[0:src.shape[0],2] = 1
    A[src.shape[0]:,[3,4]] = src
    A[src.shape[0]:,5] = 1
    b = dst.T.flatten().astype('float64')
    x = lstsq(A,b)

    tmat = np.eye(3)
    tmat[0,:] = x[0].take([0,1,2])
    tmat[1,:] = x[0].take([3,4,5])
    pd.DataFrame(tmat).to_csv(out_f, header=None, index=False, sep="\t")

if __name__ == "__main__":

    parser = argparse.ArgumentParser(description="Estimate transformation from points using least squares")

    parser.add_argument("fn_pts1", help="File name src points")
    parser.add_argument("fn_pts2", help="File name dst points")
    parser.add_argument("fn_tmat", help="File name transformation matrix")
    args = parser.parse_args()

    landmark_registration_ls(args.fn_pts1, args.fn_pts2, args.fn_tmat)


#!/usr/bin/env python3

"""
    Title:  ML Workshop: Multiple Linear Regression (MLR)
    Date:   May 15, 2021
    Author: Mario Raciti
"""

import matplotlib.pyplot as plt
import numpy as np
from sklearn.linear_model import LinearRegression
from sklearn.model_selection import cross_validate

np.random.seed(0)

# Create a dataset
# Vars
obs_n = 200
b0, b1, b2, b3 = 5, 8, -5, 0.1
fluctuations = 1
# Features
X1 = 1 + 2 * np.random.random(size=(obs_n, 1))  # in (1, 3)
X2 = 2 * np.random.random(size=(obs_n, 1))      # in (0, 2)
X3 = 4 + 3 * np.random.random(size=(obs_n, 1))  # in (4, 7)
X = np.concatenate((X1, X2, X3), axis=1)    # Gather features
# # Plot features to chart
# plt.plot(X1.squeeze(), "r+")
# plt.plot(X2.squeeze(), "b*")
# plt.plot(X3.squeeze(), "go")
# plt.suptitle('X1, X2, X3', fontsize=16)
# plt.show()
# Targets
Y = b0 + b1 * X1.squeeze() + b2 * X2.squeeze() + b3 * X3.squeeze() + \
    fluctuations * np.random.randn(obs_n)
# # Plot dataset to chart
# plt.plot(X1.squeeze(), Y, 'b+')
# plt.suptitle('Y vs X1', fontsize=16)
# plt.show()
# plt.plot(X2.squeeze(), Y, 'b+')
# plt.suptitle('Y vs X2', fontsize=16)
# plt.show()
# plt.plot(X3.squeeze(), Y, 'b+')
# plt.suptitle('Y vs X3', fontsize=16)
# plt.show()

# Create an MLR model
mlr_model = LinearRegression()
# Training
mlr_model.fit(X, Y)
print(mlr_model.coef_)
print(mlr_model.intercept_)
# Results
print(f'Source b0 = {b0}\t Model b0 = {mlr_model.intercept_:.2f}\tRelative difference {100*(b0-mlr_model.intercept_)/mlr_model.intercept_:.2f}%')
print(f'Source b1 = {b1}\t Model b1 = {mlr_model.coef_[0]:.2f}\tRelative difference {100*(b1-mlr_model.coef_[0])/mlr_model.coef_[0]:.2f}%')
print(f'Source b2 = {b2}\t Model b2 = {mlr_model.coef_[1]:.2f}\tRelative difference {100*(b2-mlr_model.coef_[1])/mlr_model.coef_[1]:.2f}%')
print(f'Source b3 = {b3}\t Model b3 = {mlr_model.coef_[2]:.2f}\tRelative difference {100*(b3-mlr_model.coef_[2])/mlr_model.coef_[2]:.2f}%')

# Predictions
new_obs = 100
i = 0
feature_n = np.shape(X)[1]
X_pred = np.empty([new_obs, feature_n])
for feature in [X1, X2, X3]:
    first, last = feature.min(), feature.max()
    X_pred[:, i] = np.linspace(first, last, new_obs)
    i += 1

# Predict
Y_pred = mlr_model.predict(X=X_pred)

# Plot predictions to chart
plt.plot(X, Y, '+')
plt.plot(X_pred, Y_pred, 'r-')
plt.show()

# Model Evaluation
# K-fold Cross Validation
K = 5
print(f'Number of folds = {K}\n')
error = 'neg_mean_squared_error'    # NOTE: MSE = RSS/N
results = cross_validate(mlr_model, X, Y, cv=K, scoring=error)

# Error for each fold and average error over the folds:
errors = -results['test_score']
print(f'Error for each fold: {errors}')
print(f'Error averaged on all folds: {errors.mean()}')


from math import sin, pi
import numpy as np
from sympy import Symbol, integrate
import matplotlib.pyplot as plt
# function for the Rectangular rule
def Rectangular(a,b,f,n):
   h = (b-a)/float(n)
   s = 0
   for i in range(0,n,1):
       x = (i+0.5)*h
       s = s+ f(x)
   return h*s
#  function to compute pi
def function(x):
    return sin(2*pi*x)
# define integration limits and integration points
a = 0.0; b = 0.5;
n = 100
Exact = 1./pi
print("Relative error= ", abs( (Rectangular(a,b,function,n)-Exact)/Exact))


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Thu Apr 25 10:49:26 2019

@author: self-driver
"""

import numpy as np
from scipy.misc import comb

def bernstein_poly(i, n, t):
    """
     The Bernstein polynomial of n, i as a function of t
     (i)
     (n) * t^i * (1-t)^(n-i)
    """
    return comb(n, i) * ( (1-t)**(n-i) ) * (t)**i

def bezier_curve(points,nTimes=1000):
    """
       Given a set of control points, return the
       bezier curve defined by the control points.

       points should be a list of lists, or list of tuples
       such as [ [1,1], [2,3], [4,5], ..[Xn, Yn] ]
        nTimes is the number of time steps, defaults to 1000

        See http://processingjs.nihongoresources.com/bezierinfo/
    """
    npoints = len(points)
    xPoints = np.array([p[0] for p in points])
    yPoints = np.array([p[1] for p in points])

    t = np.linspace(0.0, 1.0, nTimes)

    polynomial_array = np.array([ bernstein_poly(i,npoints-1, t) for i in range(0, npoints)])

    xvals = np.dot(xPoints, polynomial_array)
    yvals = np.dot(yPoints, polynomial_array)

    return xvals, yvals

def bezier_curve_D(points,n,nTimes=1000):
    """
       Given a set of control points, return the
       bezier curve defined by the control points.

       points should be a list of lists, or list of tuples
       such as [ [1,1], [2,3], [4,5], ..[Xn, Yn] ]
        nTimes is the number of time steps, defaults to 1000

        See http://processingjs.nihongoresources.com/bezierinfo/
    """
    npoints = len(points)
    xPoints = np.array([p[0] for p in points])
    yPoints = np.array([p[1] for p in points])

    t = np.linspace(0.0, 1.0, nTimes)

    polynomial_array = np.array([ bernstein_poly(i,npoints-1, t) for i in range(0, npoints)])

    xvals = np.dot(xPoints, polynomial_array)
    yvals = np.dot(yPoints, polynomial_array)

    return xvals, yvals


import numpy as np

def SS(data:np.ndarray, eq_params:tuple|list) -> float:
    """
    Calculate the sum of sqared error for a given dataset and line.

    Args:
        data: Row major numpy array with target variable in last column
        eq_params: coefficients for slope-intercept form of line, ex: if eq is y = 1 + 2x -> eq_params is (1, 2)
                   Importantly, order of the coefficients in eq_params must be the same as the variables in data

    Returns:
        sum_squared_diffs: Float value, sum of squared error from data to prediction line
    """
    # calculate the predicted values of the target variable using the equation parameters and all non-target dimensions
    y_intercept, *dim_coefficients = eq_params
    coefficients_applied_to_data = data[:, :-1] * dim_coefficients
    predicted_target_values = np.sum(coefficients_applied_to_data, axis=1) + y_intercept

    # calculate, square, and sum the distances from each point to its position predicted by the line
    diffs = predicted_target_values - data[:, -1]
    squared_diffs = np.square(diffs)
    sum_squared_diffs = np.sum(squared_diffs)

    return sum_squared_diffs

# def determine_line_of_best_fit_for_vis():
#     pass

def determine_LBF(data: np.array) -> tuple:
    """
    Iteratively calculate the line of best fit (LBF) for given data

    Args:
        data: Row major representation of data in np.array w/ target variable in last column
        precision: # of decimal places to calculate the paramaters for LBF to

    Returns:
        line_params: tuple of parameters for LBF, ex: if line is y = 1 + 2x -> line_params is (1, 2)
    """
    pass

if __name__ == "__main__":
    points = np.array([[0, 0], [1, 2], [2, 4], [3, 6]])
    line_params = [0, 2]
    result = SS(points, line_params)
    print(result)
    points_t = np.transpose(points)

    print(np.matmul(points_t,points))


# -*- coding: utf-8 -*-
"""
Created on Sat Apr 18 13:03:24 2020

@author: Mohammad Asif Zaman

Code for testng differentiation matrix operators

"""

from __future__ import print_function

import time
import math
import numpy as np
import pylab as py
import scipy.sparse as sp                 # import sparse matrix library

py.rcParams.update({'font.size': 14})

# import the file where the differentiation matrix operators are defined
from diff_matrices import Diff_mat_1D, Diff_mat_2D

def my_contourf(x,y,F,ttl):
    py.contourf(x,y,F,41,cmap = 'inferno')
    py.colorbar()
    py.xlabel('x'); py.ylabel('y'); py.title(ttl)
    return 0

# Define independent variables
Nx = 80                         # No. of grid points along x direction
Ny = 70                         # No. of grid points along y direction
x = np.linspace(-3,3,Nx)        # x variables in 1D
y = np.linspace(-3,3,Ny)        # y variable in 1D

dx = x[1] - x[0]                # grid spacing along x direction
dy = y[1] - y[0]                # grid spacing along y direction

X,Y = np.meshgrid(x,y)          # 2D meshgrid

# Test funcitons
F = np.exp(-X**2-3*Y**2)        # 2D Gaussian test function
Fx = np.multiply(X,F)*(-2)      # Analytical value of dF/dx
Fy = np.multiply(Y,F)*(-6)      # Analytical value of dF/dy

F2x = np.multiply(2*X**2-1,F)*(2)      # Analytical value of d2F/dx2
F2y = np.multiply(6*Y**2-1,F)*(6)      # Analytical value of d2F/dy2

G = np.exp(-x**2)
Gx = np.multiply(x,G)*(-2)
G2x = np.multiply(2*x**2-1,G)*(2)

Dx_1d, D2x_1d = Diff_mat_1D(Nx)                     # Calling 1D matrix operators from function
Dx_2d, Dy_2d, D2x_2d, D2y_2d = Diff_mat_2D(Nx,Ny)   # Calling 2D matrix operators from funciton

# Differentiation using matrix operators. The division by 2dx and 2dy are needed as they are not defined within the funcitons

dGx =  Dx_1d*G/(2*dx)     # dF/dx from matrix operator
d2Gx =  D2x_1d*G/(dx**2)     # dF/dx from matrix operator

dFx = (Dx_2d*F.ravel()).reshape([Ny,Nx])/(2*dx)     # dF/dx from matrix operator
dFy = (Dy_2d*F.ravel()).reshape([Ny,Nx])/(2*dy)     # dF/dy from matrix operator

d2Fx = (D2x_2d*F.ravel()).reshape([Ny,Nx])/(dx**2)     # d2F/dx2 from matrix operator
d2Fy = (D2y_2d*F.ravel()).reshape([Ny,Nx])/(dy**2)     # d2F/dy2 from matrix operator

# Plotting 1D differntiations
py.close('all')
#py.contourf(x,y,F,41,cmap = 'plasma')

py.figure(figsize = (14,4.8))
py.subplot(1,2,1)
py.plot(x,Gx,'o',label = 'Analytical')
py.plot(x,dGx,label = 'Matrix')
py.legend()
py.xlabel('x'); py.ylabel(r'$dG/dx$'); py.title('First derivative')

py.subplot(1,2,2)
py.plot(x,G2x,'o',label = 'Analytical')
py.plot(x,d2Gx,label = 'Matrix')
py.legend()
py.xlabel('x'); py.ylabel(r'$d^2G/dx^2$'); py.title('Second derivative')

# Plotting 2D differentiations

# First partial derivatives
py.figure(figsize = (14,4.8))
py.subplot(1,2,1); my_contourf(x,y,Fx,r'Analytical $dF/dx$')
py.subplot(1,2,2); my_contourf(x,y,dFx,r'Matrix operator $dF/dx$')

py.figure(figsize = (14,4.8))
py.subplot(1,2,1); my_contourf(x,y,Fy,r'Analytical $dF/dy$')
py.subplot(1,2,2); my_contourf(x,y,dFy,r'Matrix operator $dF/dy$')

# Second partial derivatives
py.figure(figsize = (14,4.8))
py.subplot(1,2,1); my_contourf(x,y,F2x,r'Analytical $d^2F/dx^2$')
py.subplot(1,2,2); my_contourf(x,y,d2Fx,r'Matrix operator $d^2F/dx^2$')

py.figure(figsize = (14,4.8))
py.subplot(1,2,1); my_contourf(x,y,F2y,r'Analytical $d^2F/dy^2$')
py.subplot(1,2,2); my_contourf(x,y,d2Fy,r'Matrix operator $d^2F/dy^2$')


import numpy as np
import matplotlib.pyplot as plt

'''
Adam Bashforth m step method:
    Function Name: adam_bashforth

    Inputs: f: defining function ( from y' = f(t,y) )
            a: starting Time
            b: ending Time
            alpha: Initial Condition
            h: stepsize
            type: # of steps (the 'm' in m-step scheme)
                                m = 2,3,4 or 5
            N = (optional) to override the number of iterations
                default: N = 0 for normal Adam Bashforth explicit
                    reset N to any finite number i to get i iterations.
                        (i >= type-1, for obvious reasons)
            plot: Binary input
                Default plot = 0: No plotting
                        plot = 1: plots the solution

    Outputs: [t, y]: where
                t: time axis
                y: solution array
'''

#--------------------- RK 4th order method ---------------------------

def RK4_method(f, a, b, t, y, h, N=0):
    if(N == 0):
        N = int((b-a)/h)

    for i in range(N):
        k1 = h*f(t[-1], y[-1])
        yp2 = y[-1] + k1/2
        k2 = h*f(t[-1]+h/2, yp2)
        yp3 = y[-1] + k2/2
        k3 = h*f(t[-1]+h/2, yp3)
        yp4 = y[-1] + k3
        k4 = h*f(t[-1]+h, yp4)
        val = (1/6)*(k1 + 2*k2 + 2*k3 + k4)
        y.append(y[-1] + val)
        t.append(t[-1] + h)
    return t, y

#----------------- Adam Bashforth explicit methods --------------------

#Adam Bashforth handler
def adam_bashforth(f, a, b, alpha, h, type, N = 0, plot = 0):
    t = [a]
    y = [alpha]
    t, y = RK4_method(f, a, b, t, y, h, type - 1)
    if(N == 0):
        N = int(((b-a)/h) - (type-1))

    if(type == 2):
        t, y = abf2(f, y, t, h, N)
        if(plot == 1):
            plt.plot(t, y, label = "AB %s step solution"%type)
            plt.legend()
            plt.show()

    if(type == 3):
        t, y = abf3(f, y, t, h, N)
        if(plot == 1):
            plt.plot(t, y, label = "AB %s step solution"%type)
            plt.legend()
            plt.show()

    if(type == 4):
        t, y = abf4(f, y, t, h, N)
        if(plot == 1):
            plt.plot(t, y, label = "AB %s step solution"%type)
            plt.legend()
            plt.show()

    if(type == 5):
        t, y = abf5(f, y, t, h, N)
        if(plot == 1):
            plt.plot(t, y, label = "AB %s step solution"%type)
            plt.legend()
            plt.show()

    return [t, y]

#2 step Adam Bashforth
def abf2(f, y, t, h, N):
    for i in range(N):
        y.append(y[-1] + (h/2)*(3*f(t[-1], y[-1]) - f(t[-2], y[-2])))
        t.append(t[-1] + h)
    return t, y

#3 step Adam Bashforth
def abf3(f, y, t, h, N):
    for i in range(N):
        y.append(y[-1] + (h/12)*(23*f(t[-1], y[-1]) - 16*f(t[-2], y[-2]) \
                                + 5*f(t[-3], y[-3])))
        t.append(t[-1] + h)
    return t, y

#4 step Adam Bashforth
def abf4(f, y, t, h, N):
    for i in range(N):
        y.append(y[-1] + (h/24)*(55*f(t[-1], y[-1]) - 59*f(t[-2], y[-2]) \
                                    + 37*f(t[-3], y[-3]) - 9*f(t[-4], y[-4])))
        t.append(t[-1] + h)
    return t, y

#5 step Adam bashforth
def abf5(f, y, t, h, N):
    for i in range(N):
        y.append(y[-1] + (h/720)*(1901*f(t[-1], y[-1]) - 2774*f(t[-2], y[-2]) \
                                + 2616*f(t[-3], y[-3]) - 1274*f(t[-4], y[-4]) \
                                +251*f(t[-5], y[-5])))
        t.append(t[-1] + h)
    return t, y


#!/usr/bin/env python

__author__ = "Adeel Ahmad"
__email__ = "adeelahmad14@hotmail.com"
__status__ = "Production"

import numpy as np
from mpl_toolkits.mplot3d import Axes3D
import matplotlib.pyplot as plt
import skimage.io as io

def edgesMarrHildreth(img, sigma):
	"""
		finds the edges using MarrHildreth edge detection method...
		:param im : input image
		:param sigma : sigma is the std-deviation and refers to the spread of gaussian
		:return:
		a binary edge image...
	"""
	size = int(2*(np.ceil(3*sigma))+1)

	x, y = np.meshgrid(np.arange(-size/2+1, size/2+1), np.arange(-size/2+1, size/2+1))

	normal = 1 / (2.0 * np.pi * sigma**2)

	kernel = ((x**2 + y**2 - (2.0*sigma**2)) / sigma**4) * np.exp(-(x**2+y**2) / (2.0*sigma**2)) / normal # LoG filter

	kern_size = kernel.shape[0]
	log = np.zeros_like(img, dtype=float)

	# applying filter
	for i in range(img.shape[0]-(kern_size-1)):
		for j in range(img.shape[1]-(kern_size-1)):
			window = img[i:i+kern_size, j:j+kern_size] * kernel
			log[i,j] = np.sum(window)

	log = log.astype(np.int64, copy=False)

	zero_crossing = np.zeros_like(log)

	# computing zero crossing
	for i in range(log.shape[0]-(kern_size-1)):
		for j in range(log.shape[1]-(kern_size-1)):
			if log[i][j] == 0:
				if (log[i][j-1] < 0 and log[i][j+1] > 0) or (log[i][j-1] < 0 and log[i][j+1] < 0) or (log[i-1][j] < 0 and log[i+1][j] > 0) or (log[i-1][j] > 0 and log[i+1][j] < 0):
					zero_crossing[i][j] = 255
			if log[i][j] < 0:
				if (log[i][j-1] > 0) or (log[i][j+1] > 0) or (log[i-1][j] > 0) or (log[i+1][j] > 0):
					zero_crossing[i][j] = 255

	# plotting images
	fig = plt.figure()
	a =fig.add_subplot(1,2,1)
	imgplot = plt.imshow(log, cmap='gray')
	a.set_title('Laplacian of Gaussian')
	a = fig.add_subplot(1,2,2)
	imgplot = plt.imshow(zero_crossing, cmap='gray')
	string = 'Zero Crossing sigma = '
	string += (str(sigma))
	a.set_title(string)
	plt.show()

	return zero_crossing


# solutions.py
"""Volume 1A: SVD and Image Compression. Solutions file."""

# solutions.py
from scipy import linalg as la
import numpy as np
from matplotlib import pyplot as plt

# Problem 1
def truncated_svd(A,k=None,tol=10**-6):
    """Computes the truncated SVD of A. If r is None or equals the number
        of nonzero singular values, it is the compact SVD.
    Parameters:
        A: the matrix
        k: the number of singular values to use
        tol: the tolerance for zero
    Returns:
        U - the matrix U in the SVD
        s - the diagonals of Sigma in the SVD
        Vh - the matrix V^H in the SVD
    """
    m,n = A.shape
    eigs,evecs = la.eig(A.conj().T.dot(A))

    s = np.sqrt(eigs)
    sort_indices = np.argsort(s)[::-1] #Sort from greatest to least
    # Sort singular values
    s = s[sort_indices]
    # Sort eigenvectors the same way
    evecs = evecs[:,sort_indices]
    if k is not None:
        if k > len(s):
            raise ValueError("k is too large")
        else:
            # Keep only the largest k singular values
            s = s[:k]
    # Remove all zero singular values
    s = s[s>tol]

    #calculate V
    V = evecs[:,:len(s)]

    #calculate U
    U = np.empty((m,len(s)))
    for i in xrange(len(s)):
        U[:,i] = (1./s[i])*A.dot(V[:,i])

    return U, s, V.conj().T

# Problem 2
def visualize_svd():
    S = np.load("circle.npz")["circle"]
    vec = np.load("circle.npz")["unit_vectors"]

    A = np.array([3,1,1,3]).reshape((2,2))
    U,s,Vh = la.svd(A)

    VhS = Vh.dot(S)
    SigVhS = np.diag(s).dot(VhS)
    USigVhS = U.dot(SigVhS)

    Vhvec = Vh.dot(vec)
    SigVhvec = np.diag(s).dot(Vhvec)
    USigVhvec = U.dot(SigVhvec)

    border1 = [-1.1, 1.1, -1.1, 1.1]
    border2 = [-4.5, 4.5, -4.5, 4.5]
    plt.subplot(2,2,1)
    plt.plot(S[0],S[1],vec[0],vec[1])
    plt.axis(border1)
    plt.axis("equal")
    plt.subplot(2,2,2)
    plt.plot(VhS[0],VhS[1],Vhvec[0],Vhvec[1])
    plt.axis(border1)
    plt.axis("equal")
    plt.subplot(2,2,3)
    plt.plot(SigVhS[0],SigVhS[1],SigVhvec[0],SigVhvec[1])
    plt.axis(border2)
    plt.axis("equal")
    plt.subplot(2,2,4)
    plt.plot(USigVhS[0],USigVhS[1],USigVhvec[0],USigVhvec[1])
    plt.axis(border2)
    plt.axis("equal")
    plt.show()

# Problem 3
def svd_approx(A, k):
    """Returns best rank k approximation to A with respect to the induced 2-norm.

    Inputs:
    A - np.ndarray of size mxn
    k - rank

    Return:
    Ahat - the best rank k approximation
    """
    U,s,Vt = la.svd(A, full_matrices=False)
    S = np.diag(s[:k])
    u1,s1,vt1 = U[:,:k],S[:k,:k],Vt[:k,:]
    diff = (u1.nbytes + np.diag(s1).nbytes + vt1.nbytes) - A.nbytes
    if diff > 0:
        print "WARNING: Given parameters do not result in compressed data."
    Ahat = U[:,:k].dot(S).dot(Vt[:k,:])
    return Ahat

# Problem 4
def lowest_rank_approx(A,e):
    """Returns the lowest rank approximation of A with error less than e
    with respect to the induced 2-norm.

    Inputs:
    A - np.ndarray of size mxn
    e - error

    Return:
    Ahat - the lowest rank approximation of A with error less than e.
    """
    U,s,Vt = la.svd(A, full_matrices=False)
    if e<=s[-1]:
        print "Warning: Matrix cannot be approximated below this error bound"
        return A
    k = np.where(s<e)[0][0]
    Ahat = U[:,:k].dot(S).dot(Vt[:k,:])
    return Ahat

# Problem 5
def compress_image(filename,k):
    """Plot the original image found at 'filename' and the rank k approximation
    of the image found at 'filename.'

    filename - jpg image file path
    k - rank
    """
    orig_img = plt.imread(filename).astype(float)
    red = orig_img[:,:,0]
    green = orig_img[:,:,1]
    blue = orig_img[:,:,2]

    img = np.zeros(orig_img.shape)
    img[:,:,0] = svd_approx(red,k)
    img[:,:,1] = svd_approx(green,k)
    img[:,:,2] = svd_approx(blue,k)

    img = np.round(img)/255.
    orig_img = np.round(orig_img)/255.
    img[img<0] = 0.
    img[img>1] = 1.

    plt.subplot(1,2,1)
    plt.title("Original Image")
    plt.imshow(orig_img)
    plt.subplot(1,2,2)
    plt.title("Rank " + str(k) + " Approximation")
    plt.imshow(img)
    plt.show()


import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
%matplotlib inline

# import sklearn

"""Math behind Large Margin Classification"""

u = np.array([10, 20])
v = np.array([30, 70])
np.dot(u, v)

np.transpose(u)

# Gaussian Kernel
from mpl_toolkits.mplot3d import Axes3D

x_lin = np.array([1,2,3,4,5,6,7,8,9,10])
y_lin = np.array([2,2,3,2,2,9,6,8,8,9])
label_lin = np.array([0,0,0,0,0,1,1,1,1,1])

fig = plt.figure()
ax=fig.add_subplot(111)
plt.scatter(x_lin, y_lin, c=label_lin, s=60)
plt.plot([-2.5, 10], [12.5, -2.5], 'k-', lw=2)
ax.set_xlim([-5,15])
ax.set_ylim([-5,15])
plt.show()

# In the figure below, we plot a dataset which is not linearly separable. If we draw a straight line, most of the points will be not be classified in the correct class.
#
# One way to tackle this problem is to take the dataset and transform the data in another feature map. It means, you will use a function to transform the data in another plan, which should be linearable.

x = np.array([1,1,2,3,3,6,6,6,9,9,10,11,12,13,16,18])
y = np.array([18,13,9,6,15,11,6,3,5,2,10,5,6,1,3,1])
label = np.array([1,1,1,1,0,0,0,1,0,1,0,0,0,1,0,1])

fig = plt.figure()
plt.scatter(x, y, c=label, s=60)
plt.show()

### illustration purpose
def mapping(x, y):
	x = np.c_[(x, y)]
	if len(x) >	2:
		x_1 = x[:,0]**2
		x_2 = np.sqrt(2)*x[:,0]*x[:,1]
		x_3 = x[:,1]**2
	else:
		x_1 = x[0]**2
		x_2 = np.sqrt(2)*x[0]*x[1]
		x_3 = x[1]**2
	trans_x = np.array([x_1, x_2, x_3])
	return trans_x

x_1  = mapping(x, y)
x_1.shape

# Let's make a new plot with 3 axis, x, y and z respectively.

# plot
fig = plt.figure()
ax = fig.add_subplot(111, projection='3d')
ax.scatter(x_1[0], x_1[1], x_1[2], c=label, s=60)
ax.view_init(30, 185)
ax.set_xlabel('X Label')
ax.set_ylabel('Y Label')
ax.set_zlabel('Z Label')
plt.show()

# We see an improvement but if we change the orientation of the plot, it is clear that the dataset is now separable

# plot
fig = plt.figure()
ax = fig.add_subplot(111, projection='3d')
ax.scatter(x_1[0], x_1[1], x_1[1], c=label, s=60)
ax.view_init(0, -180)
ax.set_ylim([150,-50])
ax.set_zlim([-10000,10000])
ax.set_xlabel('X Label')
ax.set_ylabel('Y Label')
ax.set_zlabel('Z Label')
plt.show()

"""What is a Kernel in machine learning?

The idea is to use a higher-dimension feature space to make the data almost linearly separable as shown in the figure above.

There are plenty of higher dimensional spaces to make the data points separable. For instance, we have shown that the polynomial mapping is a great start.

We have also demonstrated that with lots of data, these transformation is not efficient. Instead, you can use a kernel function to modify the data without changing to a new feature plan. """

"""Let's see an example to understand the concept of Kernel. You have two vectors, x1 and x2. The objective is to create a higher dimension by using a polynomial mapping. The output is equal to the dot product of the new feature map. From the method above, you need to:

    1.Transform x1 and x2 into a new dimension
    2.Compute the dot product: common to all kernels
    3.Transform x1 and x2 into a new dimension

You can use the function created above to compute the higher dimension.
"""
## Kernel
x1 = np.array([3,6])
x2 = np.array([10,10])

x_1 = mapping(x1, x2)
print(x_1)
## Compute the dot product

# You can use the object dot from numpy to compute the dot product between the first and second vector stored in x_1.

print(np.dot(x_1[:,0], x_1[:,1]))

# Instead, you can use the polynomial kernel to compute the dot product without transforming the vector. This function computes the dot product of x1 and x2 as if these two vectors have been transformed into the higher dimension. Said differently, a kernel function computes the results of the dot product from another feature space.

# You can write the polynomial kernel function in Python as follow.

def polynomial_kernel(x, y, p=2):
	return (np.dot(x, y)) ** p

polynomial_kernel(x1, x2, p=2)

# Type of Kernel Methods

# There are lots of different kernels available. The simplest is the linear kernel. This function works pretty well for text classification. The other kernel is:

    # Polynomial kernel
    # Gaussian Kernel


from scipy import signal

from nnlib.layers import *

def rel_error(x, y):
    """ returns relative error """
    return np.max(np.abs(x - y) / (np.maximum(1e-8, np.abs(x) + np.abs(y))))

x = [1, 2, 3, 5, 1, -1, 2, 3, 5, 8, 3, 9, 1, 2, 5, 1]
x = np.array(x)
print("input x shape: ", x.shape)
filters = [4, 5, 3, 4]
filters = np.array(filters)
b = np.array([0])

stride = 1

mode = "full"
if mode == "valid":
    padding = 0
elif mode == "full":
    padding = len(filters) - 1

scipy_correlate = signal.correlate(x, filters, mode=mode)
print("correlate scipy:", scipy_correlate)
print("correlate scipy shape:", scipy_correlate.shape)

scipy_fft = signal.correlate(x, filters, mode=mode, method="fft")
print("fft scipy:", scipy_fft)
print("fft scipy shape:", scipy_fft.shape)

np_correlate = np.correlate(x, filters, mode=mode)
print("correlate numpy:", np_correlate)
print("correlate numpy shape:", np_correlate.shape)

print("x: ", x)
print("filters: ", filters)
xfft = np.fft.fft(x)
filterfft = np.conj(np.fft.fft(filters, len(xfft)))
# element-wise multiplication in the frequency domain
out = xfft * filterfft
# take the inverse of the output from the frequency domain and return the modules of the complex numbers
out = np.fft.ifft(out)
output = np.real(out)[:xfft.shape[-1]]
out_W = np.int(((len(x) + 2 * padding - len(filters)) / stride) + 1)
# output = np.absolute(out)
output = output[:out_W]

print("output of cross-correlation via fft: ", output)
print("output of cross-correlation via fft shape: ", output.shape)

x = x.reshape(1, 1, -1)
filters = filters.reshape(1, 1, -1)

print("size of padding: ", padding)
conv_param = {'stride': stride, 'pad': padding}
outnaive, _ = conv_forward_naive_1D(x, filters, b, conv_param)
print("out naive conv: ", outnaive)
print("out naive conv shape: ", outnaive.shape)

print("x: ", x)
print("filters: ", filters)

outfft, _ = conv_forward_fft_1D(x, filters, b, conv_param)
print("outfff: ", outfft)
print("outfff shape: ", outfft.shape)

print("is the fft cross_correlation for convolution correct with respect to naive: ",
      np.allclose(outfft, outnaive, atol=1e-12))
print("relative error fft naive: ", rel_error(outfft, outnaive))
print("absolute error fft: ", np.sum(np.abs(outfft - outnaive)))
print("absolute error scipy: ", np.sum(np.abs(scipy_correlate - outnaive)))
print("absolute error numpy: ", np.sum(np.abs(np_correlate - outnaive)))
print("absolute error scipy fft: ", np.sum(np.abs(scipy_fft - outnaive)))


import numpy as np
import pylab as pl
import time

def plot_exact_solution():
    # Advection velocity:
    a=1

    x=np.linspace(-1,1,1000)
    pl.clf()
    pl.hold(False)
    for t in np.linspace(0,1,100):
        pl.plot(x,eta(x-a*t))
        pl.draw()
        time.sleep(0.0001)

def eta(x,IC='gaussian'):
    # Initial condition:
    x=np.remainder(x,1.)
    if IC=='gaussian':
        return np.exp(-100*(x-0.5)**2)
    elif IC=='square':
        return 1.0*(x>0.05)*(x<0.25)

def FD_solution(method='Lax-Friedrichs',a=1.,cflnum=0.9,m=50,IC='gaussian'):
    method=method.lower()

    x=np.linspace(0,1,m+1);
    u=eta(x,IC)

    h=x[1]-x[0]
    k=cflnum*h/a

    if method=='leapfrog':
        # Take one step with Euler:
        uold=u.copy()
        u=eta(x-a*k,IC); u[0]=eta(1.-a*k,IC)
        u[1:-1]=u[1:-1]-0.5*cflnum*(u[2:]-u[:-2])
        u[-1]=u[-1]-0.5*cflnum*(u[0]-u[-2])
        u[0] =u[0] -0.5*cflnum*(u[1]-u[-1])
        unew=np.zeros(np.size(u))

    pl.clf()
    pl.hold(False)
    T=2.
    N=int(round(T/k))

    for n in xrange(N+1):
        t=n*k
        pl.plot(x,u,'-o',x,eta(x-t*a,IC),'-k')
        pl.ylim([-0.1,1.])
        pl.legend(['computed','exact'],loc=2)
        pl.draw()
        time.sleep(0.0001)
        if method=='centered':
            u[1:-1]=u[1:-1]-0.5*cflnum*(u[2:]-u[:-2])
            u[0] = u[0] -0.5*cflnum*(u[1]-u[-1])
            u[-1]= u[-1]-0.5*cflnum*(u[0]-u[-2])
        elif method=='leapfrog':
            unew[1:-1]=uold[1:-1]-cflnum*(u[2:]-u[:-2])
            unew[0]=uold[0]-cflnum*(u[1]-u[-1])
            unew[-1]=uold[-1]-cflnum*(u[0]-u[-2])
            uold=u.copy()
            u=unew.copy()
        elif method=='lax-friedrichs':
            u[1:-1]=0.5*(u[2:]+u[:-2])-0.5*cflnum*(u[2:]-u[:-2])
            u[0] =  0.5*(u[1] +u[-1]) -0.5*cflnum*(u[1]-u[-1])
            u[-1]=  0.5*(u[0] +u[-2]) -0.5*cflnum*(u[0]-u[-2])
        elif method=='lax-wendroff':
            u[1:-1]=u[1:-1]-0.5*cflnum*(u[2:]-u[:-2])+0.5*cflnum**2 * (u[2:]-2*u[1:-1]+u[:-2])
            u[0]=   u[0]   -0.5*cflnum*(u[1]-u[-1])+0.5*cflnum**2 * (u[1]-2*u[0]   +u[-1])
            u[-1]=  u[-1]  -0.5*cflnum*(u[0]-u[-2])+0.5*cflnum**2 * (u[0]-2*u[-1]   +u[-2])
        elif method=='beam-warming':
            u[2:]=u[2:]-0.5*cflnum*(3*u[2:]-4*u[1:-1]+u[:-2])+0.5*cflnum**2 * (u[2:]-2*u[1:-1]+u[:-2])
            #u[0]=u[0]-0.5*cflnum*(3*u[0]-4*u[-1]+u[-2])+0.5*cflnum**2 * (u[0]-2*u[-1]+u[:-2])
        elif method=='upwind':
            u[1:]=u[1:]-cflnum*(u[1:]-u[:-1])
            u[0]= u[0] -cflnum*(u[0] -u[-1])
        else:
            raise Exception('unrecognized method')

    t=(N+1)*k
    #pl.plot(x,u)
    pl.plot(x,u,'-o',x,eta(x-t*a,IC),'-k')
    pl.ylim([-0.1,1.])
    pl.legend(['computed','exact'],loc=2)
    pl.draw()


#!/usr/bin/python3
import scipy.fftpack
import numpy as np
import matplotlib.pyplot as plt

# Number of samplepoints -- FFT size
N = 512
# sampling frequency : fs
# maximum frequency that can be measured "fs/2" Nyquist's theorem
fs= 1000 #Hz
# sampling time period = 1/fs
T = 1/fs
# length of the signal in time
L = N*T
# Vector with sampling time-stamps
t = np.linspace(0.0,L,N)
# input signal
# test signal -- replace this --
y = 0.5 * np.sin(150 * 2.0 * np.pi * t) + np.sin(400 * 2.0 * np.pi * t)

fig, (datain,dataout) = plt.subplots(nrows=1,ncols=2,sharex=False,sharey=False)

yf = scipy.fftpack.fft(y)

# yfabs[0] gives the DC component of the input signal
# If the FFT is input is just real numbers (true for all real world apps.)
#   second half of fft-out is complex conjugate of first half of the fft-out
#<VVI> Why are we scaling the output ??
yfabs = np.abs(yf[0:int(N/2)]) * 2/N
fd = np.array(list(range(0,int(N/2)))).astype(float) * fs/N
print(y)
print(yfabs)

datain.plot(t,y)
dataout.plot(fd,yfabs)
#yinv = scipy.fftpack.ifft(yf)
#dataout.plot(t,yinv)

#xf = np.linspace(0.0, 1.0/(2.0*T), N/2)

#fig, ax = plt.subplots()
#ax.plot(xf[1:], 2.0/N * np.abs(yf[1:N/2]))
plt.show()


"""
Created on Mon Feb  1 10:08:31 2016
"""
#------------------------------------------------------------------------------
#CHAPTER 6:  The Finite-Element Method
#------------------------------------------------------------------------------
import numpy as np
import matplotlib.pyplot as plt

#%% CODE 07: Listing 6.1 1D static elasticity - Pag 140
#------------------------------------------------------------------------------
# FINITE ELEMENTS SOLUTION
#------------------------------------------------------------------------------
# [...]
# Basic parameters
nx = 20             # Number of boundary points
u = np.zeros(nx)    # Solution vector
f = np.zeros(nx)    # Source vector
mu = 1              # Constant shear modulus
# Element boundary points
x = np.linspace(0, 1, nx) # x in [0,1]
h = x[2] - x[1] # Constant element size
# Assemble stiffness matrix K_ij
K = np.zeros((nx, nx))
for i in range(1, nx-1):
    for j in range(1, nx-1):
        if i == j:
            K[i, j] = 2*mu/h
        elif i == j + 1:
            K[i, j] = -mu/h
        elif i + 1 == j:
            K[i, j] = -mu/h
        else:
            K[i, j] = 0
# Souce term is a spike at i = 15
f[15] = 1
# Boundary condition at x = 0
u[0] = 0.15 ; f[1] = u[0]/h
# Boundary condition at x = 1
u[nx-1] = 0.05 ; f[nx-2] = u[nx-1]/h
# finite element solution
u[1:nx-1] = np.linalg.inv(K[1:nx-1, 1:nx-1]) @ f[1:nx-1].T
# [...]

# Plotting
plt.plot(x,u, color='r', lw=3.)
plt.xlabel('x')
plt.ylabel('u(x)')
plt.axis([0, 1, 0.04, .28])

#%% CODE 08: Listing 6.2 1D static elasticity with finite differences - Pag 141
#------------------------------------------------------------------------------
# FINITE DIFFERENCES SOLUTION
#------------------------------------------------------------------------------
# Poisson's equation with relaxation method
# non-zero boundary conditions
u  = np.zeros(nx) # set u to zero
du = np.zeros(nx) # du/dx
f  = np.zeros(nx) # forcing

nt = 500
isnap = 25

# [...]
# Forcing
f[15] = 1/h  # force vector
for it in range(nt):
    # Calculate the average of u (omit boundaries)
    for i in range(1, nx-1):
        du[i] =u [i+1] + u[i-1]
    u = 0.5*( f*h**2/mu + du )
    u[0] = 0.15    # Boundary condition at x=0
    u[nx-1] = 0.05 # Boundary condition at x=1
# [...]

    # visualization
    fd = u
    xfd = np.arange(0, nx)*h
    if np.mod(it+1, isnap) == 0:
        plt.plot(xfd, fd, color='k', ls='-.')

plt.savefig('Fig_6.7.png')
plt.show()


from typing import Tuple

import numpy
from scipy.special import expit
from sklearn.linear_model import LogisticRegression

from assignment_2_3_4.oracles import AbstractOracle
from assignment_2_3_4.oracles.abstract_oracle import Matrix

class LogisticRegressionOracle(AbstractOracle):
    _eps = numpy.finfo(numpy.float64).eps
    name: str = "logistic regression"

    def __init__(self, x: Matrix, y: numpy.ndarray):
        super().__init__(x, y)
        self._x_t = self._x.T

    def get_true_minimum(self, tol: float, max_iter: float) -> float:
        sklearn_logit_reg = LogisticRegression(
            tol=tol, max_iter=max_iter, random_state=7, fit_intercept=False, penalty="none"
        )
        sklearn_logit_reg.fit(self._x, self._y)
        minimum_point = sklearn_logit_reg.coef_[0]
        return self.value(minimum_point)

    # ========== Logistic Regression Calculations ==========

    def _get_loss(self, logit: numpy.ndarray) -> numpy.float:
        return -numpy.mean(numpy.log(numpy.where(self._y != 0, logit + self._eps, 1 - logit + self._eps)))

    def _get_grad(self, logit: numpy.ndarray) -> numpy.ndarray:
        return (logit - self._y) @ self._x / self._n_samples

    def _get_hessian(self, logit: numpy.ndarray) -> numpy.ndarray:
        logit = (logit * (1 - logit)).reshape(-1, 1) / self._n_samples
        if isinstance(self._x_t, numpy.ndarray):
            return self._x_t @ (self._x * logit) / self._n_samples
        else:
            return (self._x_t @ (self._x.multiply(logit))).toarray()

    def _get_hessian_dot_vector(self, logit: numpy.ndarray, d: numpy.ndarray) -> numpy.ndarray:
        logit = logit * (1 - logit) / self._n_samples
        return ((self._x @ d) * logit) @ self._x

    # ========== Oracle Interface ==========

    def value(self, weights: numpy.ndarray) -> numpy.float:
        self._call_counter += 1
        return self._get_loss(expit(self._x.dot(weights)))

    def grad(self, weights: numpy.ndarray) -> numpy.ndarray:
        self._call_counter += 1
        return self._get_grad(expit(self._x.dot(weights)))

    def hessian(self, weights: numpy.ndarray) -> numpy.ndarray:
        self._call_counter += 1
        return self._get_hessian(expit(self._x.dot(weights)))

    def hessian_vec_product(self, weights: numpy.ndarray, d: numpy.ndarray) -> numpy.ndarray:
        self._call_counter += 1
        return self._get_hessian_dot_vector(expit(self._x.dot(weights)), d)

    def fuse_value_grad(self, weights: numpy.ndarray) -> Tuple[numpy.float, numpy.ndarray]:
        self._call_counter += 1
        logit = expit(self._x.dot(weights))
        return self._get_loss(logit), self._get_grad(logit)

    def fuse_value_grad_hessian(self, weights: numpy.ndarray) -> Tuple[numpy.float, numpy.ndarray, numpy.ndarray]:
        self._call_counter += 1
        logit = expit(self._x.dot(weights))
        return self._get_loss(logit), self._get_grad(logit), self._get_hessian(logit)

    def fuse_value_grad_hessian_vec_product(
        self, weights: numpy.ndarray, d: numpy.ndarray
    ) -> Tuple[numpy.float, numpy.ndarray, numpy.ndarray]:
        self._call_counter += 1
        logit = expit(self._x.dot(weights))
        return (
            self._get_loss(logit),
            self._get_grad(logit),
            self._get_hessian_dot_vector(logit, d),
        )


import numpy as np
import sklearn.datasets as skdata
import sklearn.metrics as skmetrics
import sklearn.preprocessing as skpreprocess
from sklearn.linear_model import LinearRegression
from matplotlib import pyplot as plt

'''
Name: Kanagaraj, Kanimozhi

Summary:

TODO:
(1) In polynomial Feature exapnsion :
    a. we add an extra dimension to seprate the data at higher dimensions(mapping from linear to non-linear space)
    b. Given dataset{X^n,y^n} , where n = 1,...,N
    c. choose the order of function 1st or 2nd or 3rd polynomial called phi
        polynomial Feature Exapansion algorithn:
            phi_p(X) = (x_0,x_1,x_2,..,x_d,x_1^p,..,x_d^p)
            phi_p(X) = w_0.x_0 + w_1.x_1+...+w_d.x_d + w_d+x_1^p+..+w_D.x_d^p
    d. Map the features from X to Z using function phi
    e. optimize weights by minimizing the loss function to find the best linear hypothesis in non-linear space
    f. for hypothesis we take transpose of weight and phi(X) (in regrssion)

(2) We use this polynomial algortihm:
    a. As a framework for non-linear mapping and to hand-craft features for real world problems
    b. As it is not possible to seprate the data in 2-dimensions
       we add an extra dimension to seprate the data in non-linear mapping

(3) Negative learning the algorithm is prone to ar:
    Overfitting: (when model fits the data too well)
    It happens when model learns to fit both data and noise present, where it has negative impact on new data
    Resulting in incorrect predictions for new data

TODO: Report your scores here. For example,

Results using scikit-learn LinearRegression model with linear features
Training set mean squared error: 23.2560
Training set r-squared scores: 0.7323
Validation set mean squared error: 17.6111
Validation set r-squared scores: 0.7488
Testing set mean squared error: 17.1465
Testing set r-squared scores: 0.7805
Results of LinearRegression model using scikit-learn order-2 polynomial expansion features
Features after polynomial transform order-2: 105
Training set mean squared error: 6.6954
Training set r-squared scores: 0.9229
Validation set mean squared error: 9.5225
Validation set r-squared scores: 0.8642
Testing set mean squared error: 28.2841
Testing set r-squared scores: 0.6378
Results of LinearRegression model using scikit-learn order-3 polynomial expansion features
Features after polynomial transform order-3: 560
Training set mean squared error: 0.0000
Training set r-squared scores: 1.0000
Validation set mean squared error: 131226.9258
Validation set r-squared scores: -1870.9922
Testing set mean squared error: 119705.0919
Testing set r-squared scores: -1531.7112
Results for LinearRegression model using our implementation of order-2 polynomial expansion features
Features after polynomial transform order-2: 105
Training set mean squared error: 6.6954
Training set r-squared scores: 0.9229
Validation set mean squared error: 9.5225
Validation set r-squared scores: 0.8642
Testing set mean squared error: 28.2841
Testing set r-squared scores: 0.6378
Results for LinearRegression model using our implementation of order-3 polynomial expansion features
Features after polynomial transform order-3: 560
Training set mean squared error: 0.0000
Training set r-squared scores: 1.0000
Validation set mean squared error: 131094.4582
Validation set r-squared scores: -1869.1026
Testing set mean squared error: 119633.2842
Testing set r-squared scores: -1530.7918
'''

'''
Implementation of our polynomial expansion for nonlinear mapping
'''
class PolynomialFeatureExpansion(object):

    def __init__(self, degree):
        '''
        Args:
            degree : int
                order or degree of polynomial for expansion
        '''
        # Degree or order of polynomial we will expand to
        self.__degree = degree

        # List of boolean lists (True, False) to represent which polynomials we will create
        # Examples of polynomials:
        # [
        #   [x1^2, x1x2, x2x1, x2^2]  2nd order
        #   [...]
        # ]
        # Corresponding polynomials terms to create:
        # [
        #   [True, True, False, True]  2nd order
        #   [...]
        # ]
        self.__polynomial_terms = []

    def transform(self, X):
        '''
        Computes up to p-order (degree) polynomial features and augments them to the data

        Args:
            X : numpy
                N x d feature vector

        Returns:
            polynomial expanded features in Z space
        '''

        # Initialize the bias
        # TODO: What is the shape of bias and why do we select this shape?
        '''
            shape of bias is (N, 1) , we select this as we need to augmnets up to p-order polynomial
            (ie),as X is a (N,d), we need N number if biases so we create (N,1)
        '''
        bias = np.ones([X.shape[0], 1])
        # Initialize polynomial expansion features Z
        # TODO: Suppose x = [x1, x2], what terms are in Z?
        '''
            Z will have values : [x_0,x_1,x_2,x_3,..x_d,..x_1^p,..x_d^p]
            (Z is agumented to X)
            Z has the bias, terms of X ,polynomial terms
        '''
        Z = [bias, X]

        # If degree is less than 2, then return the original features
        if self.__degree < 2:
            Z = np.concatenate(Z, axis=1)
            return Z

        # Split X into it's d dimensions separately
        linear_features = np.split(X, indices_or_sections=X.shape[1], axis=1)

        if self.__degree == 2:
            # Keep a list of new polynomial features that we've accumulated
            new_polynomial_features = []

            # Keep track of the polynomial terms that we will keep
            polynomial_terms = []

            # For every linear feature
            for l1 in range(len(linear_features)):

                # Multiply it by every linear feature
                for l2 in range(len(linear_features)):

                    # TODO: Suppose x = [x_1, x_2]
                    # write the polynomial terms after each iteration
                    # for 2nd order polynomial
                    '''
                    First Iteration
                        a. x_1 * x_1 = x_1^2
                        b. x_1 * x_2 = x_1 x_2
                    Second Iteration
                        a. x_2 * x_1 = x_2 x_1
                        b. x_2 * x_2 = x_2^2

                    '''
                    polynomial_feature = linear_features[l1] * linear_features[l2]

                    # Check if we have already found the polynomial terms to keep
                    if len(self.__polynomial_terms) < self.__degree - 1:

                        # If we have not, then iterate through the expansion
                        keep_polynomial_term  = True

                        # Check if we already have the feature created
                        for feature in new_polynomial_features:
                            if np.sum(polynomial_feature - feature) == 0.0:
                                keep_polynomial_term = False
                                break

                        # Keep track of whether we keep or discard (True/False) the term
                        polynomial_terms.append(keep_polynomial_term)

                        if keep_polynomial_term:
                            # And append the result to the new set of polynomial features
                            new_polynomial_features.append(polynomial_feature)
                    else:
                        # Check if the current polynomial term was kept
                        # l1 * len(linear_features) + l2 indexes into the term we are creating
                        keep_polynomial_term = self.__polynomial_terms[0][l1 * len(linear_features) + l2]

                        if keep_polynomial_term:
                            # And append the result to the new set of polynomial features
                            new_polynomial_features.append(polynomial_feature)

            # If we've never processed the polynomial terms before, save the list of terms to keep
            if len(self.__polynomial_terms) < self.__degree - 1:
                self.__polynomial_terms.append(polynomial_terms)

            # Add the new polynomial features to Z
            Z.append(np.concatenate(new_polynomial_features, axis=1))

        if self.__degree > 2:
            # Start off with X as both the set of linear and current polynomial features
            linear_features = np.split(X, indices_or_sections=X.shape[1], axis=1)
            current_polynomial_features = linear_features

            # Since we will be taking the difference of the sum of features at every
            # iteration of the inner loop, let's compute their sums first to save compute
            sum_Z_features = [
                np.sum(f) for f in linear_features
            ]

            # For every degree expansion
            for d in range(0, self.__degree - 1):
                # Initialize a list to hold the new polynomial features
                new_polynomial_features = []

                # Keep track of the polynomial terms that we will keep
                polynomial_terms = []

                # Since expanding a polynomial (x1 + x2)^2 to a higher order (x1 + x2)^3 is just
                # multiplying by linear terms e.g. (x1 + x2)^3 = (x1 + x2)^2 (x1 + x2)
                # we treat the outer loop as the current polynomial term (x1 + x2)^2
                # that we are processing

                # For every polynomial feature

                for p in range(len(current_polynomial_features)):
                    # Multiply it by every linear feature
                    for l in range(len(linear_features)):
                        # TODO: Suppose x = [x_1, x_2]
                        # write the polynomial terms after each iteration
                        # for 3rd order polynomial
                        '''
                        First Iteration:
                            1. x_1 * x_1 = x_1^2
                            2. x_1 * x_2 = x1 x_2
                            3. (x_1 * x_1)* x_1 = x_1^3
                            4. (x_1 * x_1) * x_2 = x_1^2 x_2
                            5. (x_1 * x_2) * x_1 = x_1^2 x_2
                            6. (x_1 * x_2) * x_2 = x_2^2 x_1
                        Second Iteration:
                            1. x_2 * x_1 = x_2 x_1
                            2. x_2 * x_2 =  x_2^2
                            3. (x_2 * x_1) * x_1 = x_1^2 x_2
                            4. (x_2 * x_1)* x_2 = x_2^2 x_1
                            5. (x_2 * x_2) * x_1 = x_2^2 x_1
                            6. (x_2 * x_2)* x_2 = x_2^3
                        '''

                        polynomial_feature = current_polynomial_features[p] * linear_features[l]

                        # Check if we have already found the polynomial terms to keep
                        if len(self.__polynomial_terms) < self.__degree - 1:

                            # If we have not, then iterate through the expansion
                            keep_polynomial_term  = True

                            # Check if we already have the feature created
                            # To save some compute sum this once before going into loop
                            sum_polynomial_feature = np.sum(polynomial_feature)

                            for sum_Z_feature in sum_Z_features:
                                # We check if the absolute difference of the sums is less than a small epsilon
                                if np.abs(sum_polynomial_feature - sum_Z_feature) < 1e-9:
                                    keep_polynomial_term = False
                                    break

                            # Keep track of whether we keep or discard (True/False) the term
                            polynomial_terms.append(keep_polynomial_term)

                            if keep_polynomial_term:
                                # And append the result to the new set of polynomial features
                                new_polynomial_features.append(polynomial_feature)
                                sum_Z_features.append(sum_polynomial_feature)

                        else:
                            # Check if the current polynomial term was kept
                            # p * len(linear_features) + l indexes into the term we are creating
                            # TODO: What is d referring to?
                            '''
                                d refers to range of one less than order of degree
                                eg : if range of degree is 0,1,2 then d is 0,1
                            '''
                            # TODO: For third degree expansion of x = [x1, x2],
                            # What terms are we indexing to if we just use p * len(linear_features) instead?
                            '''
                                we will be indexing only current polynomial terms(2nd order terms)
                                ie., l the third term wont be indexed, won't index the created terms
                            '''
                            keep_polynomial_term = self.__polynomial_terms[d][p * len(linear_features) + l]

                            if keep_polynomial_term:
                                # And append the result to the new set of polynomial features
                                new_polynomial_features.append(polynomial_feature)

                # If we've never processed the polynomial terms before, save the list of terms to keep
                if len(self.__polynomial_terms) < self.__degree - 1:
                    self.__polynomial_terms.append(polynomial_terms)

                # Add the new polynomial features to Z
                # TODO: Why do we concatenate along the 1st axis?
                '''
                    new polynomial feature is a list of polynomial features
                    We concatenate along axis 1 to get (N,D) matrices
                    Eg :  degree of 2 we get (N,3), dgree of 3 we get(N,7)

                '''
                Z.append(np.concatenate(new_polynomial_features, axis=1))

                # TODO: For 3rd order polynomial expansion, what does Z contain after
                # each iteration of the outer for loop (d)
                '''
                    Z will contain bias , X(linear features) and polynomial features
                    2nd polynomial feature in first iteration and 3rd order polynomial in next iteration
                '''
                # Set the new polynomial features to curren polynomial features and repeat
                current_polynomial_features = new_polynomial_features

        # Concatenate every term into the feature vector
        Z = np.concatenate(Z, axis=1)

        return Z

if __name__ == '__main__':

    boston_housing_data = skdata.load_boston()
    x = boston_housing_data.data
    y = boston_housing_data.target

    # 80 percent train, 10 percent validation, 10 percent test split
    train_idx = []
    val_idx = []
    test_idx = []
    for idx in range(x.shape[0]):
        if idx and idx % 10 == 9:
            val_idx.append(idx)
        elif idx and idx % 10 == 0:
            test_idx.append(idx)
        else:
            train_idx.append(idx)

    x_train, x_val, x_test = x[train_idx, :], x[val_idx, :], x[test_idx, :]
    y_train, y_val, y_test = y[train_idx], y[val_idx], y[test_idx]

    '''
    Trains and tests linear regression from scikit-learn
    '''
    # Variables to hold training, validation and testing scores for linear up to p-order polynomial expansion
    scores_mse_train = []
    scores_mse_val = []
    scores_mse_test = []
    scores_r2_train = []
    scores_r2_val = []
    scores_r2_test = []

    # Initialize scikit-learn linear regression model
    model = LinearRegression()

    # Trains scikit-learn linear regression model
    model.fit(x_train, y_train)

    print('Results using scikit-learn LinearRegression model with linear features')

    # Test model on training set
    predictions_train = model.predict(x_train)

    score_mse_train = skmetrics.mean_squared_error(predictions_train, y_train)
    print('Training set mean squared error: {:.4f}'.format(score_mse_train))

    score_r2_train = model.score(x_train, y_train)
    print('Training set r-squared scores: {:.4f}'.format(score_r2_train))

    # TODO: Save MSE and R-square scores on training set
    scores_mse_train.append(score_mse_train)
    scores_r2_train.append(score_r2_train)

    # Test model on validation set
    predictions_val = model.predict(x_val)

    score_mse_val = skmetrics.mean_squared_error(predictions_val, y_val)
    print('Validation set mean squared error: {:.4f}'.format(score_mse_val))

    score_r2_val = model.score(x_val, y_val)
    print('Validation set r-squared scores: {:.4f}'.format(score_r2_val))

    # TODO: Save MSE and R-square scores on validation set
    scores_mse_val.append(score_mse_val)
    scores_r2_val.append(score_r2_val)

    # Test model on testing set
    predictions_test = model.predict(x_test)

    score_mse_test = skmetrics.mean_squared_error(predictions_test, y_test)
    print('Testing set mean squared error: {:.4f}'.format(score_mse_test))

    score_r2_test = model.score(x_test, y_test)
    print('Testing set r-squared scores: {:.4f}'.format(score_r2_test))

    # TODO: Save MSE and R-square scores on testing set
    scores_mse_test.append(score_mse_test)
    scores_r2_test.append(score_r2_test)

    # TODO: Set the degrees/orders of polynomials to be 2 and 3 for nonlinear mapping
    degrees_polynomial = [2, 3]

    '''
    Trains and tests linear regression from scikit-learn with scikit-learn polynomial features
    '''
    for degree in degrees_polynomial:

        print('Results of LinearRegression model using scikit-learn order-{} polynomial expansion features'.format(degree))

        # TODO: Initialize polynomial expansion
        poly_transform = skpreprocess.PolynomialFeatures(degree=degree)

        # TODO: Compute the polynomial terms needed for the data
        # Generates x_1^2, x_1 x_2, x_1 x_3, ..., x_d^2
        poly_transform.fit(x_train)

        # TODO: Transform the data by nonlinear mapping
        # Applies all the polynomial terms to the data and augments it to x
        # Computes the values for x_0, x_1, x_2, ..., x_1^2, x_1 x_2, ... x_d^2
        # x_1 = 2, x_2 = 4 : x -> (1, 2, 4, ..., 4, 8, ..., x_d^2)
        x_poly_train = poly_transform.transform(x_train)
        x_poly_val = poly_transform.transform(x_val)
        x_poly_test = poly_transform.transform(x_test)

        # Initialize scikit-learn linear regression model
        model_poly = LinearRegression()

        # TODO: Trains scikit-learn linear regression model using p-order polynomial expansion
        print('Features after polynomial transform order-{}: {}'.format(degree, x_poly_train.shape[1]))
        model_poly.fit(x_poly_train, y_train)

        # TODO: Test model on training set
        predictions_poly_train = model_poly.predict(x_poly_train)

        score_mse_poly_train = skmetrics.mean_squared_error(predictions_poly_train, y_train)
        print('Training set mean squared error: {:.4f}'.format(score_mse_poly_train))

        score_r2_poly_train = model_poly.score(x_poly_train, y_train)
        print('Training set r-squared scores: {:.4f}'.format(score_r2_poly_train))

        # TODO: Save MSE and R-square scores on training set
        scores_mse_train.append(score_mse_poly_train)
        scores_r2_train.append(score_r2_poly_train)

        # TODO: Test model on validation set
        predictions_poly_val = model_poly.predict(x_poly_val)

        score_mse_poly_val = skmetrics.mean_squared_error(predictions_poly_val, y_val)
        print('Validation set mean squared error: {:.4f}'.format(score_mse_poly_val))

        score_r2_poly_val = model_poly.score(x_poly_val, y_val)
        print('Validation set r-squared scores: {:.4f}'.format(score_r2_poly_val))

        # TODO: Save MSE and R-square scores on validation set
        scores_mse_val.append(score_mse_poly_val)
        scores_r2_val.append(score_r2_poly_val)

        # TODO: Test model on testing set
        predictions_poly_test = model_poly.predict(x_poly_test)

        score_mse_poly_test = skmetrics.mean_squared_error(predictions_poly_test, y_test)
        print('Testing set mean squared error: {:.4f}'.format(score_mse_poly_test))

        score_r2_poly_test = model_poly.score(x_poly_test, y_test)
        print('Testing set r-squared scores: {:.4f}'.format(score_r2_poly_test))

        # TODO: Save MSE and R-square scores on testing set
        scores_mse_test.append(score_mse_poly_test)
        scores_r2_test.append(score_r2_poly_test)

    # TODO: Convert each scores to NumPy arrays
    scores_mse_train = np.array(scores_mse_train)
    scores_mse_val = np.array(scores_mse_val)
    scores_mse_test = np.array(scores_mse_test)
    scores_r2_train = np.array(scores_r2_train)
    scores_r2_val = np.array(scores_r2_val)
    scores_r2_test = np.array(scores_r2_test)

    # TODO: Clip each set of MSE scores between 0 and 50
    scores_mse_train = np.clip(scores_mse_train, 0.0, 50.0)
    scores_mse_val = np.clip(scores_mse_val, 0.0, 50.0)
    scores_mse_test = np.clip(scores_mse_test, 0.0, 50.0)

    # TODO: Clip each set of R-squared scores between 0 and 1
    scores_r2_train = np.clip(scores_r2_train, 0.0, 1.0)
    scores_r2_val = np.clip(scores_r2_val, 0.0, 1.0)
    scores_r2_test = np.clip(scores_r2_test, 0.0, 1.0)

    n_experiments = len(scores_mse_train)

    # TODO: Create figure for training, validation and testing scores for different features
    fig = plt.figure()

    # TODO: Create subplot for MSE for training, validation, testing
    ax = fig.add_subplot(1, 2, 1)

    x_values = [range(1, n_experiments + 1)] * n_experiments
    y_values = [scores_mse_train, scores_mse_val, scores_mse_test]
    labels = ['Training', 'Validation', 'Testing']
    colors = ['blue', 'red', 'green']

    # TODO: Plot MSE scores for training, validation, testing sets
    for x, y, label, color in zip(x_values, y_values, labels, colors):
        ax.plot(x, y, marker='o', color=color, label=label)
        ax.legend(loc='best')

    # TODO: Set y limits between 0 and 50, set x limits to 0 to number experiments + 1
    ax.set_ylim([0.0, 50.0])
    ax.set_xlim([0.0, n_experiments + 1])

    # TODO: Set y label to 'MSE', set x label to 'p-degree'
    ax.set_ylabel('MSE')
    ax.set_xlabel('p-degree')

    # TODO: Create subplot for R-square for training, validation, testing

    ax = fig.add_subplot(1, 2, 2)
    x_values = [range(1, n_experiments + 1)] * n_experiments
    y_values = [scores_r2_train,scores_r2_val,scores_r2_test]
    labels = ['Training', 'Validation', 'Testing']
    colors = ['blue', 'red', 'green']

    # TODO: Plot R-squared scores for training, validation, testing sets

    for x,y,label,color in zip(x_values,y_values,labels,colors):
        ax.plot(x, y,marker='o', color = color, label=label)
        ax.legend(loc='best')

    # TODO: Set y limits between 0 and 1, set x limits to 0 to number experiments + 1

    ax.set_ylim([0.0, 1.0])
    ax.set_xlim([0.0, n_experiments + 1])

    # TODO: Set y label to 'R-squared', set x label to 'p-degree'

    ax.set_ylabel('R-squared')
    ax.set_xlabel('p-degree')

    # TODO: Create super title 'Scikit-learn Polynomial Expansion on Training, Validation and Testing Sets'
    plt.suptitle('Scikit-learn Polynomial Expansion on Training, Validation and Testing Sets')

    '''
    Trains and tests linear regression from scikit-learn with our implementation of polynomial features
    '''
    # TODO: Instantiate lists containing the training, validation and testing
    # MSE and R-squared scores obtained from linear regression without nonlinear mapping
    scores_mse_train = [0]
    scores_mse_val = [0]
    scores_mse_test = [0]
    scores_r2_train = [0]
    scores_r2_val = [0]
    scores_r2_test = [0]

    for degree in degrees_polynomial:

        print('Results for LinearRegression model using our implementation of order-{} polynomial expansion features'.format(degree))

        # TODO: Transform the data by nonlinear mapping using our implementation of polynomial expansion
        poly_transform = PolynomialFeatureExpansion(degree=degree)

        # Transform x_train to x_poly_train with p-degree expansion
        x_poly_train = poly_transform.transform(x_train)
        x_poly_val = poly_transform.transform(x_val)
        x_poly_test = poly_transform.transform(x_test)

        # Initialize scikit-learn linear regression model
        model_poly = LinearRegression()

        # TODO: Trains scikit-learn linear regression model using p-order polynomial expansion
        print('Features after polynomial transform order-{}: {}'.format(degree, x_poly_train.shape[1]))
        model_poly.fit(x_poly_train, y_train)

        # TODO: Test model on training set

        prediction_poly_train = model_poly.predict(x_poly_train)

        score_mse_poly_train = skmetrics.mean_squared_error(prediction_poly_train, y_train)
        print('Training set mean squared error: {:.4f}'.format(score_mse_poly_train))

        score_r2_poly_train = model_poly.score(x_poly_train, y_train)
        print('Training set r-squared scores: {:.4f}'.format(score_r2_poly_train))

        # TODO: Save MSE and R-square scores on training set

        scores_mse_train.append(score_mse_poly_train)
        scores_r2_train.append(score_r2_poly_train)

        # TODO: Test model on validation set
        prediction_poly_val = model_poly.predict(x_poly_val)

        score_mse_poly_val = skmetrics.mean_squared_error(prediction_poly_val,y_val)
        print('Validation set mean squared error: {:.4f}'.format(score_mse_poly_val))

        score_r2_poly_val = model_poly.score(x_poly_val,y_val)
        print('Validation set r-squared scores: {:.4f}'.format(score_r2_poly_val))

        # TODO: Save MSE and R-square scores on validation set

        scores_mse_val.append(score_mse_poly_val)
        scores_r2_val.append(score_r2_poly_val)

        # TODO: Test model on testing set
        prediction_poly_test = model_poly.predict(x_poly_test)

        score_mse_poly_test = skmetrics.mean_squared_error(prediction_poly_test,y_test)
        print('Testing set mean squared error: {:.4f}'.format(score_mse_poly_test))

        score_r2_poly_test = model_poly.score(x_poly_test,y_test)
        print('Testing set r-squared scores: {:.4f}'.format(score_r2_poly_test))

        # TODO: Save MSE and R-square scores on testing set
        scores_mse_test.append(score_mse_poly_test)
        scores_r2_test.append(score_r2_poly_test)

    # TODO: Convert each scores to NumPy arrays
    scores_mse_train = np.array(scores_mse_train)
    scores_mse_val = np.array(scores_mse_val)
    scores_mse_test = np.array(scores_mse_test)
    scores_r2_train = np.array(scores_r2_train)
    scores_r2_val = np.array(scores_r2_val)
    scores_r2_test = np.array(scores_r2_test)

    # TODO: Clip each set of MSE scores between 0 and 50
    scores_mse_train = np.clip(scores_mse_train, 0.0, 50.0)
    scores_mse_val = np.clip(scores_mse_val, 0.0, 50.0)
    scores_mse_test = np.clip(scores_mse_test, 0.0, 50.0)

    # TODO: Clip each set of R-squared scores between 0 and 1scores_r2_train = np.clip(scores_r2_train, 0.0, 1.0)
    scores_r2_val = np.clip(scores_r2_val, 0.0, 1.0)
    scores_r2_test = np.clip(scores_r2_test, 0.0, 1.0)
    scores_r2_train = np.clip(scores_r2_train, 0.0, 1.0)
    scores_r2_val = np.clip(scores_r2_val, 0.0, 1.0)
    scores_r2_test = np.clip(scores_r2_test, 0.0, 1.0)

    n_experiments = len(scores_mse_train)

    # TODO: Create figure for training, validation and testing scores for different featuresfig_2 = plt.figure()

    fig_2 = plt.figure()

    # TODO: Create subplot for MSE for training, validation, testing

    ax = fig_2.add_subplot(1, 2, 1)

    x_values = [range(1, n_experiments + 1)] * n_experiments
    y_values = [scores_mse_train,scores_mse_val,scores_mse_test]
    labels = ['Training', 'Validation', 'Testing']
    colors = ['blue', 'red', 'green']

    # TODO: Plot MSE scores for training, validation, testing sets
    for x, y, label, color in zip(x_values, y_values, labels, colors):
        ax.plot(x, y, marker='o', color=color, label=label)
        ax.legend(loc='best')

    # TODO: Set y limits between 0 and 50, set x limits to 0 to number experiments + 1

    ax.set_ylim([0.0, 50.0])
    ax.set_xlim([0.0, n_experiments + 1])

    # TODO: Set y label to 'MSE', set x label to 'p-degree'

    ax.set_ylabel('MSE')
    ax.set_xlabel('p-degree')

    # TODO: Create subplot for R-square for training, validation, testing

    ax = fig_2.add_subplot(1, 2, 2)

    x_values = [range(1, n_experiments + 1)] * n_experiments
    y_values = [scores_r2_train,scores_r2_val,scores_r2_test]
    labels = ['Training', 'Validation', 'Testing']
    colors = ['blue', 'red', 'green']

    # TODO: Plot R-squared scores for training, validation, testing sets

    for x, y, label,color in zip(x_values,y_values,labels,colors):
        ax.plot(x, y, marker='o', color=color, label=label)
        ax.legend(loc='best')

    # TODO: Set y limits between 0 and 1, set x limits to 0 to number experiments + 1

    ax.set_ylim([0.0, 1.0])
    ax.set_xlim([0.0, n_experiments + 1])

    # TODO: Set y label to 'R-squared', set x label to 'p-degree'

    ax.set_ylabel('R-squarred')
    ax.set_xlabel('p-degree')

    # TODO: Create super title 'Our Polynomial Expansion on Training, Validation and Testing Sets'

    plt.suptitle('Our Polynomial Expansion on Training, Validation and Testing Sets')

    plt.show()


import numpy as np
import matplotlib.pyplot as plt
import os

def solve(m1, m2, std1, std2):
    a = 1 / (2 * std1 ** 2) - 1 / (2 * std2 ** 2)
    b = m2 / (std2 ** 2) - m1 / (std1 ** 2)
    c = m1 ** 2 / (2 * std1 ** 2) - m2 ** 2 / (2 * std2 ** 2) - np.log(std2 / std1)
    return np.roots([a, b, c])

def main():
    N = 100000000  # Number of samples
    bins = 1000  # Number of bins
    np.random.seed(100)  # Ensure fixed seed

    # generate two distributions
    dist1 = np.random.normal(0, 1, N)
    dist2 = np.random.normal(1, np.sqrt(2), N)

    result = solve(0, 1, 1, np.sqrt(2))

    # plot figure
    fig = plt.figure(figsize=(6, 4))
    ax = fig.add_subplot(111)
    y1_, x1_, p1 = ax.hist(dist1, bins=bins, color='red', alpha=0.7, density=True, label=r'p(X|L=1) ~ $N(0, 1)$')
    y2_, x2_, p2 = ax.hist(dist2, bins=bins, color='blue', alpha=0.6, density=True,
                           label=r'p(X|L=2) ~ $N(\mu, \sigma^{2})$')
    ax.legend()
    ax.set_title("Gaussian Class Conditioned Probability")
    ax.set_ylabel("Density")
    ax.set_xlabel("x")
    ax.axvline(x=result[0], color='k', linestyle='--')
    ax.text(result[0] - 3.0, 0.40, r'$lower={0:}$'.format(np.round(result[0], 2)))
    ax.axvline(x=result[1], color='k', linestyle='--')
    ax.text(result[1] - 3.0, 0.40, r'$upper={0:}$'.format(np.round(result[1], 2)))
    plt.savefig('results/q4.png')
    plt.show()

    s = 1
    scaling_factor = 0.5 / s
    fig = plt.figure(figsize=(6, 4))
    ax = fig.add_subplot(111)
    y1, x1 = np.histogram(dist1, bins=bins, density=True)
    y2, x2 = np.histogram(dist2, bins=bins, density=True)
    ax.bar((x1[1:] + x1[:-1]) * .5, y1 * scaling_factor,
           width=(x1[1] - x1[0]), color="red", alpha=0.7, label=r'p(L=1|x) ~ $N(0, 1)$')
    ax.bar((x2[1:] + x2[:-1]) * .5, y2 * scaling_factor,
           width=(x2[1] - x2[0]), color="blue", alpha=0.6, label=r'p(L=2|x) ~ $N(\mu, \sigma^{2})$', )

    ax.legend()
    ax.set_title("Class Posterior Probability")
    ax.set_ylabel("Density")
    ax.set_xlabel("x")
    ax.axvline(x=result[0], color='k', linestyle='--')
    ax.text(result[0] - 1.0, 0.20, r'$lower={0:}$'.format(np.round(result[0], 2)))
    ax.axvline(x=result[1], color='k', linestyle='--')
    ax.text(result[1] - 1.0, 0.20, r'$upper={0:}$'.format(np.round(result[1], 2)))
    plt.savefig('results/q4_posterior.png')
    plt.show()

if __name__ == '__main__':
    dirName = "results"
    if not os.path.exists(dirName):
        try:
            os.mkdir(dirName)
        except OSError:
            print("Creation of the directory %s failed" % dirName)
        else:
            print("Successfully created the directory %s " % dirName)
    else:
        print("Directory already exists.")
    main()


from sympy import Symbol
import matplotlib.pyplot as plt

def geraFuncao(a, b):
    func = 0
    var = Symbol('x')  # Simbolo para montar o polinomio interpolador

    return a + b * var

def calculaReta(a, b, x):
    Y = a + b * x

    return Y

def tabelaQuadradosMinimos(x, y):
    x2 = 0
    xy = 0
    xi = 0
    yi = 0
    for i in range(len(x)):
        x2 += x[i] ** 2
        xy += x[i] * y[i]
        xi += x[i]
        yi += y[i]

    b = ((xi * yi) - (len(x) * xy)) / (xi ** 2 - (len(x) * x2))
    a = (yi - (b * xi)) / len(x)

    return xi, yi, xy, x2, a, b

def calculaDesvio(x, y, a, b):
    D = 0
    for i in range(len(x)):
        D += (y[i] - (a + b * x[i])) ** 2

    return D

def calculaAjuste(x, y, a, b):
    num = 0
    den1 = 0
    den2 = 0
    for i in range(len(x)):
        num += (y[i] - a - b * x[i]) ** 2
        den1 += y[i] ** 2
        den2 += y[i]

    return 1 - num / (den1 - (1 / len(x)) * den2 ** 2)

def mostraGrafico(x, y, a, b, funcao, val):
    vetorXRetaAjuste = [61, 81]
    vetorYRetaAjuste = [calculaReta(a, b, 61), calculaReta(a, b, 81)]

    if val != 'null':
        pontoY = round(a + b * val, 2)
        plt.scatter(val, pontoY, color='yellow')
        plt.text(val, pontoY, f"P({val},{pontoY})", fontsize=8, horizontalalignment='right')

    plt.scatter(x[0], y[0], color='yellow')
    plt.scatter(x[1], y[1], color='yellow')
    plt.scatter(x[2], y[2], color='yellow')
    plt.scatter(x[3], y[3], color='yellow')
    plt.scatter(x[4], y[4], color='yellow')
    # plt.plot(x, y, marker='o', label='Curva real')
    plt.plot(vetorXRetaAjuste, vetorYRetaAjuste, label=f'f(x) = {funcao}')
    plt.title('Ajuste de Curvas')
    plt.xlabel('eixo X')
    plt.ylabel('eixo Y')
    plt.grid(True)
    plt.legend(loc=0)

    plt.show()

def main():
    # x = [0.3, 2.7, 4.5, 5.9, 7.8]
    # y = [1.8, 1.9, 3.1, 3.9, 3.3]

    x = [0.5, 1.2, 2.1, 3.5, 5.4]
    y = [5.1, 3.2, 2.8, 1.0, 0.4]

    xi, yi, xy, x2, a, b = tabelaQuadradosMinimos(x, y)

    funcao = geraFuncao(round(a, 4), round(b, 4))

    resp = float(input('Deseja avaliar um ponto da reta? (1 para SIM) ou (0 para NÃO): '))

    if resp == 1:
        val = float(input('Informe o valor que deseja calcular: '))
    else:
        val = 'null'

    mostraGrafico(x, y, a, b, funcao, val)

    a = 4.3271
    b = -0.7272

    D = calculaDesvio(x, y, a, b)

    r2 = calculaAjuste(x, y, a, b)

    print(f'a = {a}')
    print(f'b = {b}')
    print(f'\nCurva: {funcao}')
    print(f'Valor do desvio = {D}')
    print(f'Qualidade do ajuste = {r2}')

if __name__ == '__main__':
    main()


#!/usr/bin/env python3
# -*- coding: utf-8 -*-

###
# Name: Jacob Anabi, Grady Lynch
# Student ID: 2294644, 02297574
# Email: anabi@chapman.edu, grlynch@chapman.edu
# Course: PHYS220/MATH220/CPSC220 Fall 2018
# Assignment: CW09
###

"""
array_calculus.py Module Description:

Computes the derivive of some function f (Df = gradient(x) @ f(x))

Functions:
gradient(x) - computes a gradient matrix based on an array x of domain points
derivative(f, n=100) - computes the derivative of a function func at n domain points

Additonal Notes:
Forward finite difference: (f_i+1 - f_i)/dx
Backward finite difference: (f_i - f_i-1)/dx
Central finite difference: (f_i+1 - f_i-1)/2dx
"""

import numpy as np
from matplotlib import pyplot as plt

def gradient(x):
    """
    gradient(x) function description:

    computes the gradient matrix based on an array x of domain points

    Args:
        x - an array containing the domain points of the function

    Returns:
        2D numpy arry - the gradient matrix, D, which is of the form [[-2, 2, ..., 0][-1, 0, 1, ..., 0], ..., [0, ..., -2, 2]]/(2*dx)
    """
    dx = x[1] - x[0] # dx should be same for each point

    # Creating diagonal arrays to be used in the matrix
    main_diag = np.zeros(len(x)) # main diagonal for the gradient
    pos_diag = np.ones(len(x)-1) # diagonal at index + 1 for the gradient, positive numbers
    neg_diag = np.ones(len(x)-1) # diagonal at index - 1 for the gradient, negative numbers
    main_diag[0], main_diag[-1] = -2, 2 # main diagonal boundarys are -2 and 2 based on forward and backward finite difference
    pos_diag[0] = 2 # positive diagonal front boundary is 2, based on forward finite difference
    neg_diag = neg_diag*-1 # converting each element in the negative diagonal to negative numbers
    neg_diag[-1] = -2 # negative diagonal back boundary is -2, based on backward finite difference

    D = np.diag(main_diag, k=0) + np.diag(pos_diag, k=1) + np.diag(neg_diag, k=-1) # our gradient matrix, D, which is of the form [[-2, 2, ..., 0][-1, 0, 1, ..., 0], ..., [0, ..., -2, 2]]
    return D/(2*dx) # multipling our matrix by the scalar 1/(2*dx)

def derivative(func, xmax=100, n=100, power=1):
    """
    derivative(func, n=100) function description:

    computes the the derivative to some power of some function func for n domain points

    Args:
        func - the function to compute the derivative of
        n - the number of domain points (defaults to 100)
        power - the power to compute the derivative to (defaults to 1) (ex. power = 2 => 2nd derivative)

    Returns:
        1D numpy array - an array of the function func evaluted at n domain points
    """
    x = np.linspace(0, xmax, n, endpoint=True) # domain
    f = np.vectorize(func) # vectorize our function
    Df = np.linalg.matrix_power(gradient(x), power) @ f(x) # matrix multiplication of the gradient(x) to some power and the f(x), our vectorized function
    return Df # a 1D array of each derivative of our function evaluated at each domain point to some power

def plot(x, y, xlabel="", ylabel="", title=""):
    """
    plot(x, y, xlabel="", ylabel="") function description:

    plots a function given a domain and range

    Args:
        x - the domain of values
        y - the range of values
        xlabel - the label of the x-axis (defaults to "")
        ylabel - the label of the y-axis (defaults to "")
    """
    # Plotting
    fig = plt.figure(figsize=(8,6)) # Setting funciton figure size (width, height)
    axes = plt.axes() # Creating function plot axes

    axes.plot(x, y, label=ylabel) # plotting graph
    axes.legend() # axes legend
    plt.xlabel(xlabel) # x-axis label for graph
    plt.ylabel(ylabel) # y-axis label for graph
    plt.title(title) # the title of the graph

    plt.show() # show the graph


from math import ceil
import numpy as np
from scipy import linalg

def lowess(x, y, f, iterations):
    n = len(x)
    r = int(ceil(f * n))
    h = [np.sort(np.abs(x - x[i]))[r] for i in range(n)]
    w = np.clip(np.abs((x[:, None] - x[None, :]) / h), 0.0, 1.0)
    w = (1 - w ** 3) ** 3
    yest = np.zeros(n)
    delta = np.ones(n)
    for iteration in range(iterations):
        for i in range(n):
            weights = delta * w[:, i]
            b = np.array([np.sum(weights * y), np.sum(weights * y * x)])
            A = np.array([[np.sum(weights), np.sum(weights * x)],[np.sum(weights * x), np.sum(weights * x * x)]])
            beta = linalg.solve(A, b)
            yest[i] = beta[0] + beta[1] * x[i]

        residuals = y - yest
        s = np.median(np.abs(residuals))
        delta = np.clip(residuals / (6.0 * s), -1, 1)
        delta = (1 - delta ** 2) ** 2

    return yest

import math
n = 100
x = np.linspace(0, 2 * math.pi, n)
y = np.sin(x) + 0.3 * np.random.randn(n)
f =0.25
iterations=3
yest = lowess(x, y, f, iterations)

import matplotlib.pyplot as plt
plt.plot(x,y,"r.")
plt.plot(x,yest,"b-")

7th SEM/MACHINE LEARNING LABORATORY/10-Locally Weighted Regression Algorithm/LAB10.py


import numpy as np
import pandas as pd

# -----------------------------------
# Regression
# -----------------------------------
# rmse

from sklearn.metrics import mean_squared_error

# y_true are the true values、y_pred are the predictions
y_true = [1.0, 1.5, 2.0, 1.2, 1.8]
y_pred = [0.8, 1.5, 1.8, 1.3, 3.0]

rmse = np.sqrt(mean_squared_error(y_true, y_pred))
print(rmse)
# 0.5532

# -----------------------------------
# Binary classification
# -----------------------------------
# Confusion matrix

from sklearn.metrics import confusion_matrix

# True values and predicted values are binary, i.e. either 0 or 1
y_true = [1, 0, 1, 1, 0, 1, 1, 0]
y_pred = [0, 0, 1, 1, 0, 0, 1, 1]

tp = np.sum((np.array(y_true) == 1) & (np.array(y_pred) == 1))
tn = np.sum((np.array(y_true) == 0) & (np.array(y_pred) == 0))
fp = np.sum((np.array(y_true) == 0) & (np.array(y_pred) == 1))
fn = np.sum((np.array(y_true) == 1) & (np.array(y_pred) == 0))

confusion_matrix1 = np.array([[tp, fp],
                              [fn, tn]])
print(confusion_matrix1)
# array([[3, 1],
#        [2, 2]])

# Can also be created using the confusion_matrix() function from scikit-learn's metrics, but
# be aware that the arrangement of the confusion matrix elements may be different
confusion_matrix2 = confusion_matrix(y_true, y_pred)
print(confusion_matrix2)
# array([[2, 1],
#        [2, 3]])

# -----------------------------------
# accuracy

from sklearn.metrics import accuracy_score

# True values and predicted values are binary, i.e. either 0 or 1
y_true = [1, 0, 1, 1, 0, 1, 1, 0]
y_pred = [0, 0, 1, 1, 0, 0, 1, 1]
accuracy = accuracy_score(y_true, y_pred)
print(accuracy)
# 0.625

# -----------------------------------
# logloss

from sklearn.metrics import log_loss

# True values are binary (0 or 1), predicted values are probabilities
y_true = [1, 0, 1, 1, 0, 1]
y_prob = [0.1, 0.2, 0.8, 0.8, 0.1, 0.3]

logloss = log_loss(y_true, y_prob)
print(logloss)
# 0.7136

# -----------------------------------
# Multi-class classification
# -----------------------------------
# multi-class logloss

from sklearn.metrics import log_loss

# True values are 3-class classifiers, predicted values are probabilities for each class
y_true = np.array([0, 2, 1, 2, 2])
y_pred = np.array([[0.68, 0.32, 0.00],
                   [0.00, 0.00, 1.00],
                   [0.60, 0.40, 0.00],
                   [0.00, 0.00, 1.00],
                   [0.28, 0.12, 0.60]])
logloss = log_loss(y_true, y_pred)
print(logloss)
# 0.3626

# -----------------------------------
# Multi-label classification
# -----------------------------------
# mean_f1, macro_f1, micro_f1

from sklearn.metrics import f1_score

# For calculating performance metric of multi-label classification, it is easier to handle the true / predicted values as binary matrices of record x class
# True values - [[1,2], [1], [1,2,3], [2,3], [3]]
y_true = np.array([[1, 1, 0],
                   [1, 0, 0],
                   [1, 1, 1],
                   [0, 1, 1],
                   [0, 0, 1]])

# Predicted values - [[1,3], [2], [1,3], [3], [3]]
y_pred = np.array([[1, 0, 1],
                   [0, 1, 0],
                   [1, 0, 1],
                   [0, 0, 1],
                   [0, 0, 1]])

# mean_f1 is the mean of the F1-scores for each record
mean_f1 = np.mean([f1_score(y_true[i, :], y_pred[i, :]) for i in range(len(y_true))])

# macro_f1 is the mean of the F1-scores for each class
n_class = 3
macro_f1 = np.mean([f1_score(y_true[:, c], y_pred[:, c]) for c in range(n_class)])

# micro-f1 is the F1-score calculate using the true/predicted values for each record-class pair
micro_f1 = f1_score(y_true.reshape(-1), y_pred.reshape(-1))

print(mean_f1, macro_f1, micro_f1)
# 0.5933, 0.5524, 0.6250

# Can also be calculated using a scikit-learn function
mean_f1 = f1_score(y_true, y_pred, average='samples')
macro_f1 = f1_score(y_true, y_pred, average='macro')
micro_f1 = f1_score(y_true, y_pred, average='micro')

# -----------------------------------
# Multi-class classification with ordered classes
# -----------------------------------
# quadratic weighted kappa

from sklearn.metrics import confusion_matrix, cohen_kappa_score

# Function for calculating quadratic weighted kappa
def quadratic_weighted_kappa(c_matrix):
    numer = 0.0
    denom = 0.0

    for i in range(c_matrix.shape[0]):
        for j in range(c_matrix.shape[1]):
            n = c_matrix.shape[0]
            wij = ((i - j) ** 2.0)
            oij = c_matrix[i, j]
            eij = c_matrix[i, :].sum() * c_matrix[:, j].sum() / c_matrix.sum()
            numer += wij * oij
            denom += wij * eij

    return 1.0 - numer / denom

# y_true is the true class list, y_pred is the predicted class list
y_true = [1, 2, 3, 4, 3]
y_pred = [2, 2, 4, 4, 5]

# Calculate the confusion matrix
c_matrix = confusion_matrix(y_true, y_pred, labels=[1, 2, 3, 4, 5])

# Calculate quadratic weighted kappa
kappa = quadratic_weighted_kappa(c_matrix)
print(kappa)
# 0.6153

# Can also be calculated using a scikit-learn function
kappa = cohen_kappa_score(y_true, y_pred, weights='quadratic')

# -----------------------------------
# Recommendation
# -----------------------------------
# MAP@K

# K=3, with 5 records and 4 class types
K = 3

# True values for each record
y_true = [[1, 2], [1, 2], [4], [1, 2, 3, 4], [3, 4]]

# Predicted values for each record - as K=3, usually predict order of 3 records for each class
y_pred = [[1, 2, 4], [4, 1, 2], [1, 4, 3], [1, 2, 3], [1, 2, 4]]

# Function to calculate the average precision for each record
def apk(y_i_true, y_i_pred):
    # Length of y_pred must be less than or equal to K, and all elements must be unique
    assert (len(y_i_pred) <= K)
    assert (len(np.unique(y_i_pred)) == len(y_i_pred))

    sum_precision = 0.0
    num_hits = 0.0

    for i, p in enumerate(y_i_pred):
        if p in y_i_true:
            num_hits += 1
            precision = num_hits / (i + 1)
            sum_precision += precision

    return sum_precision / min(len(y_i_true), K)

# Function for calculating MAP@K
def mapk(y_true, y_pred):
    return np.mean([apk(y_i_true, y_i_pred) for y_i_true, y_i_pred in zip(y_true, y_pred)])

# Calculate MAP@K
print(mapk(y_true, y_pred))
# 0.65

# Even if the number of true values is the same, if the order is different then the score will be different
print(apk(y_true[0], y_pred[0]))
print(apk(y_true[1], y_pred[1]))
# 1.0, 0.5833


import numpy as np

X = np.array(([2, 9], [1, 5], [3, 6])) # Hours Studied, Hours Slept
y = np.array(([92], [86], [89])) # Test Score

y = y/100 # max test score is 100

# Sigmoid Function
def sigmoid(x): #this function maps any value between 0 and 1
    return 1/(1 + np.exp(-x))

# Derivative of Sigmoid Function
def derivatives_sigmoid(x):
    return x * (1 - x)

# Variable initialization
epoch = 10000 # Setting training iterations
lr = 0.1 #Setting learning rate
inputlayer_neurons = 2 # number of features in data set
hiddenlayer_neurons = 3 # number of hidden layers neurons
output_neurons = 1 # number of neurons of output layer

# weight and bias initialization
weight_hidden = np.random.uniform(size=(inputlayer_neurons, hiddenlayer_neurons))
# bias matrix to the hidden layer
bias_hidden = np.random.uniform(size=(1, hiddenlayer_neurons))
# weight matrix to the output layer
weight_output = np.random.uniform(size=(hiddenlayer_neurons, output_neurons))
# matrix to the output layer
bias_output = np.random.uniform(size=(1, output_neurons))

for i in range(epoch):
    # Forward Propogation
    hinp1 = np.dot(X, weight_hidden)
    hinp = hinp1 + bias_hidden # bias_hidden GRADIENT Descent
    hlayer_activation = sigmoid(hinp)

    outinp1 = np.dot(hlayer_activation, weight_output)
    outinp = outinp1 + bias_output
    output = sigmoid(outinp)

    # Backpropagation
    # Compare prediction with actual output and calculate the gradient of error
    # (Actual – Predicted)
    EO = y - output
    # Compute the slope/gradient of hidden and output layer neurons
    outgrad = derivatives_sigmoid(output)

    # Compute change factor(delta) at output layer, dependent on the gradient
    # of error multiplied by the slope of output layer activation
    d_output = EO * outgrad

    # At this step, the error will propagate back into the network which
    # means error at hidden layer. we will take the dot product of output
    # layer delta with weight parameters of edges between the hidden and
    # output layer (weight_hidden.T).
    EH = d_output.dot(weight_output.T)

    # how much hidden layer weight contributed to error
    hiddengrad = derivatives_sigmoid(hlayer_activation)
    d_hiddenlayer = EH * hiddengrad

    # update the weights
    # dot product of nextlayererror and currentlayerop
    weight_output += hlayer_activation.T.dot(d_output) * lr
    bias_output += np.sum(d_output, axis=0, keepdims=True) * lr

    weight_hidden += X.T.dot(d_hiddenlayer) * lr
    bias_hidden += np.sum(d_hiddenlayer, axis=0, keepdims=True) * lr

print("Input:\n" + str(X))
print("Actual Output:\n" + str(y))
print("Predicted Output:\n", output)


import numpy as np
import matplotlib.pyplot as plt
from IPython.display import clear_output

def Tfunc(n_row,n_col):
    T=np.zeros([n_row,n_col])
    for x_i in range(n_row):
        for y_j in range(n_col):

            if y_j==0:
                T[x_i,y_j] = T[x_i,y_j+1]

            elif y_j==n_col-1:
                T[x_i,y_j] = T[x_i,y_j-1]

            if x_i==0:
                T[x_i,y_j]=0

            if x_i==n_row-1:
                T[x_i,y_j]=1
    return (T)

def Tmap(T,iters,Errors):
    fig1 = plt.figure(1)
    ax1 = fig1.add_subplot(121)
    ax2 = fig1.add_subplot(122)

    clear_output(wait=True)

    ax1.clear()
    heatmap = ax1.pcolor(T, cmap=plt.cm.gist_heat)
    plt.colorbar(heatmap,ax=ax1)
    ax1.set_xlabel('$Y_{j}$')
    ax1.set_ylabel('$X_{i}$')
    ax1.set_title("Temperature profile")

    ticks=[round(i/mesh,2) for i in ax1.get_xticks()]

    ax1.set_xticklabels(ticks)
    ax1.set_yticklabels(ticks)

    ax2.clear()
    ax2.plot(iters,Errors)
    ax2.set_xlabel('Iteration')
    ax2.set_ylabel('Error')
    ax2.set_title("Convergence History")

    title='method = ' + method+'  mesh size= '+str(mesh)+' w= '+str(w)
    fig1.suptitle(title)

    plt.show()

def FiniteDiff(Temp,mesh,method,w):

    n_col = mesh
    n_row = mesh

    dy=(1/n_col)
    dx=(1/n_row)

    a=(dx**2)
    b=(dy**2)

    T=Temp.copy()

    if method == 'Jacobi':
        Tbuffer=T.copy()

    else:
        Tbuffer=T

    Tnew=T.copy()

    for x_i in range(1,n_row-1):

        for y_j in range(1,n_col-1):

            Tbuffer[x_i,y_j]=((100*a*b-b*(T[x_i+1,y_j]+T[x_i-1,y_j])-a*(T[x_i,y_j+1]+T[x_i,y_j-1]))/(-2*(a+b)))
            if method =='SOR':
                Tbuffer[x_i,y_j]=Tnew[x_i,y_j]+w*(Tbuffer[x_i,y_j]-Tnew[x_i,y_j])
            else:
                continue

    T=Tnew.copy()
    Tnew=Tbuffer.copy()

    for x_i in range(n_row):
        for y_j in range(n_col):

            if y_j==0:
                Tnew[x_i,y_j] = Tnew[x_i,y_j+1]

            elif y_j==n_col-1:
                Tnew[x_i,y_j] = Tnew[x_i,y_j-1]

            if x_i==0:
                Tnew[x_i,y_j]=0

            if x_i==n_row-1:
                Tnew[x_i,y_j]=1

    A=abs(Tnew-T)

    maxError=np.amax(A)

    return (Tnew,A,maxError)

def Temp_Profile(mesh,method,w=1):

    n_col = mesh
    n_row = mesh

    dy=(1/n_col)
    dx=(1/n_row)

    a=(dx**2)
    b=(dy**2)

    tolerance=0.001

    iterations=[]
    Errors=[]

    iteration=1

    y=[j*dy for j in range(n_col)]
    x=[i*dx for i in range(n_row)]

    Temp=Tfunc(n_row,n_col)

    Tmap(Temp,iterations,Errors)

    T_Jacobi,A,Error=FiniteDiff(Temp,mesh,method,w)

    Errors.append(Error)
    iterations.append(iteration)

    Tmap(T_Jacobi,iterations,Errors)

    print('Initial Error = ', Error,'\n\n')

    while Error>tolerance:

        T_Jacobi,A,Error=FiniteDiff(T_Jacobi,mesh,method,w)

        iteration+=1

        Errors.append(Error)
        iterations.append(iteration)

        Tmap(T_Jacobi,iterations,Errors)

        print('iter = ',iteration,'\tError = ',Error)

#testing
'''

Meshes=[9,17,33]
Methods=['Jacobi','Gauss','SOR']

for mesh in Meshes:
    for method in Methods:
        print('**********mesh = ',mesh,'\tmethod = ',method,'**********\n\n')
        if method=='SOR':
            for w in [0.5,1.5,2]:
                print('***W = ',w,' ***')
                Temp_Profile(mesh,method,w)
        else:
            Temp_Profile(mesh,method)

'''


# solve a 2-d Poisson equation by differentiating the discretized Poisson
# equation and then substituting in the inverse Fourier transform and solving
# for the amplitudes in Fourier space.
#
# This is the way that Garcia and NR do it.
#
# Note: we need a periodic problem for an FFT
#
# M. Zingale (2013-04-02)

import pylab
import numpy
import math

# the analytic solution
def true(x,y):
    pi = math.pi
    return numpy.sin(2.0*pi*x)**2*numpy.cos(4.0*pi*y) + \
        numpy.sin(4.0*pi*x)*numpy.cos(2.0*pi*y)**2

# the righthand side
def frhs(x,y):
    pi = math.pi
    return 8.0*pi**2*numpy.cos(4.0*pi*y)*(numpy.cos(4.0*pi*x) -
                                          numpy.sin(4.0*pi*x)) - \
           16.0*pi**2*(numpy.sin(4.0*pi*x)*numpy.cos(2.0*pi*y)**2 +
                       numpy.sin(2.0*pi*x)**2 * numpy.cos(4.0*pi*y))

# create the domain
xmin = 0.0
xmax = 1.0

ymin = 0.0
ymax = 1.0

Nx = 64
Ny = 64

dx = (xmax - xmin)/Nx
dy = (ymax - ymin)/Ny

x = (numpy.arange(Nx) + 0.5)*dx
y = (numpy.arange(Ny) + 0.5)*dy

x2d = numpy.repeat(x, Ny)
x2d.shape = (Nx, Ny)

y2d = numpy.repeat(y, Nx)
y2d.shape = (Ny, Nx)
y2d = numpy.transpose(y2d)

# create the RHS
f = frhs(x2d, y2d)
print f

# compatibility conditions require that the RHS sum to zero
print "sum of RHS: ", numpy.sum(f)

print x2d.shape

# FFT of RHS
F = numpy.fft.fft2(f)

# get the wavenumbers -- we need these to be physical, so multiply by Nx
kx = Nx*numpy.fft.fftfreq(Nx)
ky = Ny*numpy.fft.fftfreq(Ny)

# make 2-d arrays for the wavenumbers
kx2d = numpy.repeat(kx, Ny)
kx2d.shape = (Nx, Ny)

ky2d = numpy.repeat(ky, Nx)
ky2d.shape = (Ny, Nx)
ky2d = numpy.transpose(ky2d)

# here the FFT frequencies are in the order 0 ... N/2-1, -N/2, ...
# the 0 component is not a physical frequency, but rather it is the DC
# signal.  Don't mess with it, since we'll divide by zero
oldDC = F[0,0]
F = 0.5*F*dx*dx/(numpy.cos(2.0*math.pi*kx2d/Nx) +
                 numpy.cos(2.0*math.pi*ky2d/Ny) - 2.0)# + 1.e-20)

F[0,0] = oldDC

# transform back to real space
fsolution = numpy.real(numpy.fft.ifft2(F))

#pylab.subplot(121)
#pylab.imshow(fsolution, origin="lower", interpolation="nearest")
#pylab.colorbar()

#pylab.subplot(122)
pylab.imshow(true(x2d,y2d))
pylab.colorbar()

#pylab.tight_layout()

pylab.savefig("poissonFFT.png")

# error
print Nx, Ny, numpy.sqrt(dx*dx*numpy.sum( ( (fsolution - true(x2d,y2d))**2).flat))