import numpy as np
import pandas as pd
from datetime import timedelta
from datetime import datetime
from io import StringIO
from urllib import request as url_request
import os
import sys

import matplotlib.pyplot as plt
import matplotlib.dates as mdates

from scipy.optimize import curve_fit

# This stuff because pandas or matplot lib complained...
from pandas.plotting import register_matplotlib_converters
register_matplotlib_converters()
from scipy import stats,integrate
from scipy.optimize import curve_fit

from scipy import stats,integrate

def logistic(x, L, k, x0, y0):
    """
    General Logistic function.

    Args:
        x    float or array-like, it represents the time
        L    float, the curve's maximum value
        k    float, the logistic growth rate or steepness of the curve.
        x0   float, the x-value of the sigmoid's midpoint
        y0   float, curve's shift in the y axis
    """
    y = L / (1 + np.exp(-k*(x-x0))) + y0
    return y

def logistic_derivative(x, L, k, x0):
    """
    General Gaussian-like function (derivative of the logistic).

    Args:
        x    float or array-like, it represents the time
        L    float, the curve's integral (area under the curve)
        k    float, the logistic growth rate or steepness of the curve.
        x0   float, the x-value of the max value
    """
    y = k * L * (np.exp(-k*(x-x0))) / np.power(1 + np.exp(-k*(x-x0)), 2)
    return y

def fit_curve(curve, ydata, title, ylabel, last_date, coeff_std, do_imgs=False,plt_forecast=False,show_every=5):

    xdata = -np.flip(np.arange(len(ydata)))

    days_past = -2 # days beyond the start of the data to plot
    days_future = 40 # days after the end of the data to predict and plot
    #show_every = 3 # int value that defines how often to show a date in the x axis. (used not to clutter the axis)

    myFmt = mdates.DateFormatter('%d/%m') # date formatter for matplotlib

    total_xaxis = np.array(range(-len(ydata) + days_past, days_future)) + 1

    date_xdata = [last_date + timedelta(days=int(i)) for i in xdata]
    date_total_xaxis = [last_date + timedelta(days=int(i)) for i in total_xaxis]

    future_axis = total_xaxis[len(ydata) - days_past:]
    date_future_axis = [last_date + timedelta(days=int(i)) for i in future_axis]

            # Plotting
    fig, ax = plt.subplots(figsize=(15,8))
    ax.xaxis.set_major_formatter(myFmt)
    fig.autofmt_xdate()

    start = (len(ydata) - days_past - 1) % show_every
    ax.set_xticks(date_total_xaxis[start::show_every])
    ax.set_xlabel('Date')
    ax.set_ylabel(ylabel)
    ax.set_title(title + ': ' + str(last_date.strftime("%d-%m-%Y")))

    ax.grid(True)

    if curve.__name__ == 'logistic':
        p0_1=ydata[-1]
        p0=[p0_1, 0.5, 1, 0]
        bounds=([0, 0, -100, 0], [2*p0_1, 10, 100, 1])
        #bounds=([0,0,-np.inf,0],[np.inf,np.inf,np.inf,np.inf])
        params_names = ['L', 'k', 'x0', 'y0']
    elif curve.__name__ == 'logistic_derivative':
        p0_1=3*max(ydata)
        p0=[p0_1, 0.5, 1]
        bounds=([0, 0, -100], [10*p0_1, 10, 100])
        #bounds=([0,0,-np.inf],[np.inf,np.inf,np.inf])
        params_names = ['L', 'k', 'x0']
    else:
        print('this curve is unknown')
        return -1

    popt, pcov = curve_fit(curve, xdata, ydata, p0=p0, bounds=bounds,maxfev=20000)

    print(title)
    descr = '    fit: '
    for i, param in enumerate(params_names):
        descr = descr + "{}={:.3f}".format(param, popt[i])
        if i < len(params_names) - 1:
            descr = descr + ', '
    print(descr)

    perr = np.sqrt(np.diag(pcov))
    print('perr',perr)

    pworst = popt + coeff_std*perr
    pbest = popt - coeff_std*perr

    # Plotting
    # fig, ax = plt.subplots(figsize=(15,8))
#     ax.xaxis.set_major_formatter(myFmt)
#     fig.autofmt_xdate()

    total_xaxis = np.array(range(-len(ydata) + days_past, days_future)) + 1
    date_total_xaxis = [last_date + timedelta(days=int(i)) for i in total_xaxis]

    date_xdata = [last_date + timedelta(days=int(i)) for i in xdata]

    future_axis = total_xaxis[len(ydata) - days_past:]
    date_future_axis = [last_date + timedelta(days=int(i)) for i in future_axis]

    #print('pbest',pbest)
    #print('pworst',pworst)

    if plt_forecast==True:
        ax.plot(date_total_xaxis, curve(total_xaxis, *popt), 'g-', label='prediction')
        ax.fill_between(date_future_axis, curve(future_axis, *pbest), curve(future_axis, *pworst),
        facecolor='red', alpha=0.2, label='std')
    #print('Integral=',np.trapz(curve(total_xaxis, *popt)))

    ax.scatter(date_xdata, ydata, color='blue', label='real data',s=8)
    ax.plot(date_xdata, ydata, color='blue',alpha=0.5)

   #  start = (len(ydata) - days_past - 1) % show_every
#     ax.set_xticks(date_total_xaxis[start::show_every])
#     ax.set_xlabel('Date')
#     ax.set_ylabel(ylabel)
#     ax.set_title(title + ': ' + str(last_date.strftime("%d-%m-%Y")))
    ax.legend(loc='upper left')
#     ax.grid(True)

    fig=plt.gcf()
    plt.show()
    if do_imgs:
        fig.savefig('plots/'+ title + '.png', dpi=200)

    return popt, perr

def fit_curve_evolution(curve, ydata1, title, ylabel, last_date, coeff_std, do_imgs=False,plt_forecast=False,
                daily=True,days_past = -2,days_future = 40,show_every = 3,day_start=30):

    xdata = -np.flip(np.arange(len(ydata1)))

    #days_past = -2 # days beyond the start of the data to plot
    #days_future = 40 # days after the end of the data to predict and plot
    #show_every = 3 # int value that defines how often to show a date in the x axis. (used not to clutter the axis)

    myFmt = mdates.DateFormatter('%d/%m') # date formatter for matplotlib

    total_xaxis = np.array(range(-len(ydata1) + days_past, days_future)) + 1

    date_xdata = [last_date + timedelta(days=int(i)) for i in xdata]
    date_total_xaxis = [last_date + timedelta(days=int(i)) for i in total_xaxis]

    future_axis = total_xaxis[len(ydata1) - days_past:]
    date_future_axis = [last_date + timedelta(days=int(i)) for i in future_axis]

            # Plotting
    fig, ax = plt.subplots(figsize=(15,8))
    ax.xaxis.set_major_formatter(myFmt)
    fig.autofmt_xdate()

    start = (len(ydata1) - days_past - 1) % show_every
    ax.set_xticks(date_total_xaxis[start::show_every])
    ax.set_xlabel('Date')
    ax.set_ylabel(ylabel)
    ax.set_title(title + ': ' + str(last_date.strftime("%d-%m-%Y")))

    ax.grid(True)

    coln=len(np.arange(day_start,len(ydata1)+1))
    colors = plt.cm.YlGn(np.linspace(0,1,coln))

    for j in np.arange(day_start,len(ydata1)+1):
        #print('STEP',j)
        ydata=ydata1[:j]
        xdata = -np.flip(np.arange(len(ydata)))
        #print(len(xdata),len(ydata))

        if curve.__name__ == 'logistic':
            p0_1=ydata[-1]
            p0=[p0_1, 0.5, 1, 0]
            bounds=([0, 0, -100, 0], [2*p0_1, 10, 100, 1])
            #bounds=([0,0,-np.inf,0],[np.inf,np.inf,np.inf,np.inf])
            params_names = ['L', 'k', 'x0', 'y0']
        elif curve.__name__ == 'logistic_derivative':
            p0_1=3*max(ydata)
            p0=[p0_1, 0.5, 1]
            bounds=([0, 0, -100], [10*p0_1, 10, 100])
            #bounds=([0,0,-np.inf],[np.inf,np.inf,np.inf])
            params_names = ['L', 'k', 'x0']
        else:
            print('this curve is unknown')
            return -1

        popt, pcov = curve_fit(curve, xdata, ydata, p0=p0, bounds=bounds,maxfev=20000)

        perr = np.sqrt(np.diag(pcov))

        pworst = popt + coeff_std*perr
        pbest = popt - coeff_std*perr

        total_xaxis = np.array(list(range(-len(ydata) + days_past, days_future))) + 1

        if plt_forecast==True:
            ax.plot(date_total_xaxis[:len(total_xaxis)], curve(total_xaxis, *popt), color=colors[j-day_start])
            #ax.fill_between(date_future_axis, curve(future_axis, *pbest), curve(future_axis, *pworst),
            #facecolor='red', alpha=0.2)
            #print('Integral=',np.trapz(curve(total_xaxis, *popt)))

    if daily:
        sm = plt.cm.ScalarMappable(cmap='YlGn', norm=plt.Normalize(vmin=day_start+1, vmax=len(ydata1)+1))
    else:
        sm = plt.cm.ScalarMappable(cmap='YlGn', norm=plt.Normalize(vmin=day_start, vmax=len(ydata1)))

    plt.colorbar(sm,label='No. of days in data')

    ax.scatter(date_xdata, ydata1, color='blue', label='real data',s=8)
    ax.plot(date_xdata, ydata1, color='blue',alpha=1.0)
    ax.legend(loc='upper left')

    #plt.ylim(bottom=0, top=1.1*max(ydata1))

    fig=plt.gcf()
    plt.show()
    if do_imgs:
        fig.savefig('plots/evolution'+ title + '.png', dpi=200)

    return


import sympy as sp
x, y = sp.symbols('x y')
sp.init_printing()

# Here we simply give the input needed as The Following:
# ( Equation ) First Specify if the input is an equation ( 1 ) or only a single application of the rule ( 0 )
# ( X ) and ( Y ) The initial Values
# If The Equation was set to "0" you need to provide The Values:
    # First Derivative in terms of X ( drX ).
    # First Derivative in terms of Y ( drY ).
    # Second Derivative of "drX" in terms of X ( drXX ).
    # Second Derivative of "drY" in terms of Y ( drYY ).
    # Second Derivative of "drX" in terms of Y ( drXY ) or the opposite.
    ############# if not selected you can keep 0 ###############
# If The Equation was set to "1" you need to provide The Equation ( F ). if not selected you can keep 0

Equation = 1

X = 10

Y = 6.1

drX = 0

drY =  0

drXX = -6.4

drYY = -6.6

drXY =  3.5

F =  9.8*x**7*y**3 - 9*x**2*y**8 +2.6*x + 5.6*y

if Equation == 1:
    drXVal = sp.diff(F, x).subs(x, X).subs(y, Y).evalf()
    drYVal = sp.diff(F, y).subs(x, X).subs(y, Y).evalf()
    drXXVal = sp.diff(sp.diff(F, x), x).subs(x, X).subs(y, Y).evalf()
    drYYVal = sp.diff(sp.diff(F, y), y).subs(x, X).subs(y, Y).evalf()
    drXYVal = sp.diff(sp.diff(F, x), y).subs(x, X).subs(y, Y).evalf()

if Equation == 0 :
    drXVal = drX
    drYVal = drY
    drXXVal = drXX
    drYYVal = drYY
    drXYVal = drXY

H = drXXVal*drYYVal - sp.Pow(drXYVal, 2)
HVal = H.subs(x, X).subs(y, Y)
GradVal = "Not Critical"
if HVal > 0 and drXXVal > 0 : print("\n  Local Min")
elif HVal > 0 and drXXVal < 0 : print("\n  Local Max")
elif HVal < 0 : print("\n  Saddle Point")

print("___________________________\n   Fx      | {0:.2f} " .format(float(drXVal)))
print("___________________________\n   Fy      | {0:.2f} ".format(float(drYVal)))
print("___________________________\n   Fxx     | {0:.2f} ".format(float(drXXVal)))
print("___________________________\n   Fyy     | {0:.2f} ".format(float(drYYVal)))
print("___________________________\n   Fxy     | {0:.2f} ".format(float(drXYVal)))
print("___________________________\n   H       | {0:.2f} ".format(float(HVal)))
print("___________________________\nGrad(ai)   | {0:.2f} ".format(float(drXVal)))
print("___________________________\nGrad(bj)   | {0:.2f} \n".format(float(drYVal)))

# Finally Displaying if The next point is a Critical point or not then the solution with each derivative


import numpy as np
from PIL import Image as im

print()

Mmax = -5000
Mmin = 0

F = open("KAU_Height.txt", 'r')
F = F.readlines()

xlen = int(F[-1].split(",")[0]) + 1
ylen = int(F[-1].split(",")[1].split(":")[0]) + 1

arr = np.zeros((ylen, xlen))

# print(arr.shape)

for line in range(len(F)):

    s = F[line].split(",")
    x = int(s[0])
    c = s[1].split(":")
    y = int(c[0])
    v = float(c[1])

    # print(x,y)
    arr[y][x] = v

    if v > Mmax: Mmax = v

    if v > -5000 and v < Mmin: Mmin = v

L_range = Mmax - Mmin
print("MAX = ", Mmax, "MIN = ", Mmin, "Range : ", L_range)
# norm = np.linalg.norm(arr)

# n_arr = arr / norm * 255 + 100

m_arr = (arr-Mmin)/L_range*255

# print(n_arr)

data = im.fromarray(m_arr)
data = data.convert("L")
data.save('KAU_Hmap.png')

# for x in range(xlen):
#     for y in range(ylen):
#         arr[x][y] = F[][]

# arr = np.zeros((ylen, xlen))
# arr = np.zeros((6, 4))
# arr[1][1:3] = 5
# print(arr)


"""Mixture model using EM"""
from typing import Tuple
import numpy as np
from common import GaussianMixture

def estep(X: np.ndarray, mixture: GaussianMixture) -> Tuple[np.ndarray, float]:
    """E-step: Softly assigns each datapoint to a gaussian component

    Args:
        X: (n, d) array holding the data
        mixture: the current gaussian mixture

    Returns:
        np.ndarray: (n, K) array holding the soft counts
            for all components for all examples
        float: log-likelihood of the assignment
    """
    dim = X.shape[1]
    exponents = np.sum((X[:, np.newaxis, :] - mixture.mu) ** 2, axis=2) / (2 * mixture.var[np.newaxis, :])
    weighted_likelihoods = np.transpose(1 / ((2 * np.pi * mixture.var[:, np.newaxis]) ** (dim / 2))) * np.exp(
        -exponents) * mixture.p
    post = np.transpose(weighted_likelihoods.T / np.sum(weighted_likelihoods, axis=1))
    log_likelihood = np.sum(np.log(np.sum(weighted_likelihoods, axis=1))).astype(float)
    return post, log_likelihood

def mstep(X: np.ndarray, post: np.ndarray) -> GaussianMixture:
    """M-step: Updates the gaussian mixture by maximizing the log-likelihood
    of the weighted dataset

    Args:
        X: (n, d) array holding the data
        post: (n, K) array holding the soft counts
            for all components for all examples

    Returns:
        GaussianMixture: the new gaussian mixture
    """
    dim = X.shape[1]
    weight = np.sum(post, axis=0) / X.shape[0]
    mean = np.transpose(np.transpose(np.dot(np.transpose(post), X)) / np.sum(post, axis=0))
    var = np.sum(np.sum((X[:, np.newaxis, :] - mean) ** 2, axis=2) * post, axis=0) / (dim * np.sum(post, axis=0))
    return GaussianMixture(mu=mean, var=var, p=weight)

def run(X: np.ndarray, mixture: GaussianMixture,
        post: np.ndarray) -> Tuple[GaussianMixture, np.ndarray, float]:
    """Runs the mixture model

    Args:
        X: (n, d) array holding the data
        post: (n, K) array holding the soft counts
            for all components for all examples

    Returns:
        GaussianMixture: the new gaussian mixture
        np.ndarray: (n, K) array holding the soft counts
            for all components for all examples
        float: log-likelihood of the current assignment
    """
    precision = 1e-6
    error = 2 * precision
    posterior, prev_log_likelihood = estep(X, mixture)
    mixture = mstep(X, posterior)
    while error > precision:
        posterior, log_likelihood = estep(X, mixture)
        mixture = mstep(X, posterior)
        error = (log_likelihood - prev_log_likelihood) / abs(log_likelihood)
        prev_log_likelihood = log_likelihood

    return mixture, posterior, prev_log_likelihood


# -*- coding: utf-8 -*-
"""
Created on Fri Oct 11 14:55:20 2019

@author: tunde
"""
import numpy as np
import warnings

def bic(log_like, n, k):

    """
    Compute the bayesian information criteria

    Parameters:
    -----------

    log_like: array-like;
        The maximized value of the likelihood function of the model M.

    n: array-like;
        Number of data points in the observation.

    k: array-like;
        Number of parameters estimated by the model

    Returns:
    --------

    BIC: array-like input;
        The value of the Bayesian information Criterion. Model with lowest bic is preferred

    """
    import numpy as np
    return -2. * log_like + k * np.log(n)

def red_chisquare(data, model, error, npar):
    """
    Calculate the reduced chisquare. x2_red.
    if x2_red ~ 1, the model fits the data well.
    if x2_red << 1, the errors are overestimated, or the model is overfitting the data.
    if x2_red >> 1, the errors are underestimated, or the model is underfitting the data.

    Parameters:
    -----------

    data : array;
        the observed data.

    model : array-like data;
        calculated model to explain the data.

    error : array-like data;
        error on the observed data points.

    npar : int;
        number of fitted parameters in the model.

    Returns:
    --------

    x2_red : float;
        the reduced chisquare given by sum(((data-model)/error)**2) / (len(data)-npar)

    """

    return np.sum(((data-model)/error)**2) / (len(data)-npar)

def aic(log_like, n, k):
    """
    The Aikake information criterion.
    A model comparison tool based of infomormation theory. It assumes that N is large i.e.,
    that the model is approaching the CLT.

    Parameters:
    -----------

    log_like: array-like;
        The maximized value of the likelihood function of the model M.

    n: array-like;
        Number of data points in the observation.

    k: array-like;
        Number of parameters estimated by the model

    Returns:
    --------

    AIC: array-like input;
        The value of the Akaike Information Criterion. Model with lowest aic is preferred
    """

    val = -2. * log_like + 2 * k
    val += 2 * k * (k + 1) / float(n - k - 1)

    if not np.isfinite(val):
        val = 0

        warnings.warn('AIC was NAN. Recording zero, but you should examine your fit.')

    return val

def rmse(error):
    """
    Calculate the root-mean-square of the imputed error array

    Parameters:
    -----------

    error : array-like;
            array of error or residual values (observed - calculated)

    Return:
    --------
    rmse : float;
            root mean square error value

    """

    return np.sqrt(np.mean(np.square(error)))

def sse(error):

    return np.sum( np.square(error) )

def mse(error):
    """
    Calculate the mean-square error of imputed error array

	Parameters:
    ----------
    error : array-like;
            array of error or residual values (observed - calculated)

    Return:
    --------
    mse : float;
            mean square error value
    """
    return np.mean(np.square(error))

def mae(error):
    """
    Calculate the mean-absolute error of imputed error array

	Parameters:
    ----------
    error : array-like;
            array of error or residual values (observed - calculated)

    Return:
    --------
    mae : float;
            mean absolute error value
    """
    return np.mean(np.abs(error))

def rse(obs, calc):
    """
    Calculate the relative-square error from the observed and calculated values

	Parameters:
    ----------
    obs : array-like;
            array of observed values
    calc : array-like;
            array of calculated values e.g from model
    Return:
    --------
    rse : float;
            relative square error value
    """
    return np.mean(np.square((obs-calc)/obs))

def rae(obs, calc):
    """
    Calculate the relative-absolute error from the observed and calculated values

	Parameters:
    ----------
    obs : array-like;
            array of observed values
    calc : array-like;
            array of calculated values e.g from model
    Return:
    --------
    rae : float;
            relative absolute error value
    """
    return np.mean(np.abs((obs-calc)/obs))

def r_squared(obs, calc):
    """
    Calculate the R2_score commonly referred to as coefficient of determination. It measure how close the regression line is to the  observed values. Best possible value is 1.0

	Parameters:
    ----------
    obs : array-like;
            array of observed values
    calc : array-like;
            array of calculated values e.g from model
    Return:
    --------
    r2 : float;
            r2_score value
    """

    return 1 - (np.sum(np.square(obs-calc)) / np.sum(np.square(obs-np.mean(obs))) )


import numpy as np
from scipy.stats import norm
import matplotlib.pyplot as plt

class BayesModel:
	def __init__(self, data, prior_range):
		self.data = data
		def pdata(data, mu):
			r"""
			Probability of :math:`x` given :math:`theta`, assuming :math:`\sigma = 1`.
			"""
			return norm.pdf(data, loc=mu).prod()
		self.pdata = pdata
		def priorprob(mu):
			r"""
			Flat prior on the mean
			"""
			return 1/(prior_range[1] - prior_range[0])
		self.priorprob = priorprob

		def samplemu(size=1):
			r"""
			Sample the parameter from the flat prior on the mean
			"""
			return np.random.uniform(prior_range[0], prior_range[1], size=size)
		self.samplemu = samplemu

def metropolis(model, N):
	mu = model.samplemu()
	posterior = []

	for _ in range(int(N)):
		new_mu = model.samplemu()

		old_posterior = model.pdata(data, mu)*model.priorprob(mu)
		new_posterior = model.pdata(data, new_mu)*model.priorprob(new_mu)

		paccept = new_posterior/old_posterior

		if (np.random.uniform(0, 1) < paccept):
			mu = new_mu
		posterior.append(mu)

	return np.array(posterior)

if __name__ == '__main__':
	Ndata = 100
	N = 10000

	data = np.random.normal(0.0, 1.0, Ndata)
	prior_range = [-2.0, 2.0]

	bayes = BayesModel(data, prior_range)

	posterior = metropolis(bayes, N=N)

	plt.figure(figsize=(10, 5))
	ax = plt.subplot(121)
	ax.hist(data,
		bins=max(10, int(np.sqrt(len(data)))),
		color='k',
		density=True,
		histtype='stepfilled',
		label='Data',
		alpha=0.4)
	ax.legend(fontsize=10,
		loc='upper left',
		title_fontsize=12,
		title=r'$N_{\mathrm{data}} =$' + r'${}$'.format(len(data)))
	ax.set_title('Data')
	ax.set_xlabel(r'$x$')
	ax.set_ylabel(r'$p(\mathrm{data} \sim \mathcal{N}(0, 1)$')

	ax = plt.subplot(122)
	ax.hist(bayes.samplemu(size=1000),
		bins=100,
		density=True,
		histtype='stepfilled',
		label='Prior',
		alpha=0.3)
	ax.hist(posterior,
		bins=int(np.sqrt(len(posterior))),
		density=True,
		histtype='stepfilled',
		label='Posterior',
		alpha=0.3)
	ax.legend(fontsize=10,
		loc='upper left',
		title=r'$\hat{\mu} =$' + r'${:.2f} \pm {:.2f},$'.format(np.mean(posterior), np.std(posterior)) + '\n' + r'$N = {}$'.format(len(posterior)),
		title_fontsize=12)
	ax.set_title('Posterior Distribution')
	ax.set_xlabel(r'$\mu$')
	ax.set_ylabel(r'$p(\mu | \mathrm{data} \sim \mathcal{N}(0, 1)$')
	plt.savefig('bayes.png')


from __future__ import print_function
import numpy as np
import matplotlib.pyplot as plt
from scipy.optimize import curve_fit

#optimizing fn generation

def gauss(x, K, B, x0, stddev):
    return K + B * np.exp(-(x - x0) ** 2 / (2 * stddev ** 2))

def gaussian_curve_fit(x, y):
    mean = sum(x * y) / sum(y)
    stddev = np.sqrt(sum(y * (x - mean) ** 2) / sum(y))
    popt, pcov = curve_fit(gauss, x, y, p0=[min(y), max(y), mean, stddev])
    return popt

#data creation

np.random.seed(104)
xdata = np.linspace(3, 10, 100)

#Perfect gaussian

ydata_perfect = gauss(xdata, 20, 5, 6, 1)
ydata = np.random.normal(ydata_perfect, 1, 100)

K, B, x0, stddev = gaussian_curve_fit(xdata, ydata)

plt.plot(xdata, ydata, 'c', label='Noisy Data')
plt.plot(xdata, ydata_perfect, '-.k', label='Perfect Data')
plt.plot(xdata, gauss(xdata, *gaussian_curve_fit(xdata, ydata)), ':r', label='fit')
plt.legend()
plt.title('Gaussian fit')
plt.xlabel('X Data')
plt.ylabel('Y Data')
plt.show()


import numpy as np

def compare_entries(A, B, sig_figs):
    """
    Compares that every entry in array `A` and `B` match to a certain number
    of *significant figures*.  Note that this is NOT THE SAME AS the number of
    decimal places (from zero); it is actually a more useful check than
    checking decimal places.

    0.5412 == 0.5414       is True if sig_figs = 3, but False if sig_figs = 4
    1.5412 == 1.5414       is True if sig_figs = 4, but False if sig_figs = 5
    1.5412E-5 == 1.5414E-5 is True if sig_figs = 4, but False if sig_figs = 5
    1.5412E+5 == 1.5414E+5 is True if sig_figs = 4, but False if sig_figs = 5

    This function checks that:
    base = np.ceil(np.log10(A[i,j,k]) * np.sign(A[i,j,k]))
    np.abs(A[i,j,k] - B[i,j,k])*1E(base) < 1E(-sig_figs)

    Return
    ------
    Returns a (long) list of boolean comparisons of the entries in `A` and `B`.
    It can then be subsequently checked that np.all(...) entries in this output
    are True, to ensure that the comparison succeeded.
    """
    if not (isinstance(A, np.ndarray)):
        A = np.array([A])
    if not (isinstance(B, np.ndarray)):
        B = np.array([B])
    assert np.prod(A.shape) == np.prod(B.shape)

    check = pow(10, -sig_figs)
    out = []
    for a, b in zip(A.flat, B.flat):
        base = np.ceil(np.log10(a * np.sign(a)))
        out.append(np.abs(a - b) * pow(10, base) < check)

    return out


'''
A Neural Network for the IDENTITY operator, i.e
the output is the same as the input.

input x | output y

  0     |   0
  1     |   1

or the NOT operator, i.e.
the output is the opposite of the input.

input x | output y

  0     |   1
  1     |   0
'''
import numpy as np
import random
# initialize and populate X, the input vector
x0 = np.zeros( (2,1) )
x0[0,0] = 0
x0[1,0] = 1

# append a vector of ones to x0 to account for the bias
X = np.concatenate(  (np.ones((x0.shape[0],1) ),x0) ,axis=1)

# initialize y
# for the IDENTITY operator, it is  the same a x0
yId = x0

# for the NOT operator
yNot = np.zeros( (2,1) )
yNot[0,0] = 1
yNot[1,0] = 0

# initialize w and populate it with random values
w = np.zeros( (2,1) )
w[0,0] = random.uniform(-1,1)*10
w[1,0] = random.uniform(-1,1)*10

# choose a gradient descent step (a.k.a learning rate)
alpha = .01;

# chose the IDENTITY (Y = Yid) or NOT operator (Y = Ynot)
y = yId

print('X is \n',X,'\n')
print('y is \n',y,'\n')
print('w is \n',w,'\n')

# GRADIENT DESCENT
for i in range(0,10000):

    w = w + alpha*np.dot(X.T, y - np.dot(X,w))

print('the solution is: \n',w)

beta = np.linalg.solve(np.dot(X.T,X),np.dot(X.T,y))
print('\nUsing LS normal equations we obtain w = \n',beta)


# -*- coding: utf-8 -*-
"""Demo163_EffectOfDistributions.ipynb

## Variable distributions and their effects on Models

Reference
[https://www.statisticssolutions.com/homoscedasticity/]

### Linear Regression Assumptions

- Linear relationship with the outcome Y
- Homoscedasticity
- Normality
- No Multicollinearity

## Linear Assumption

- The X variable is linearly related to the dataset
- Pearson correlation coefficient can determine the linearity magnitude  between variables

## Normality Assumption

- The variable X follows a normal or gaussian distribution

## Homoscedasticity Assumption

- Homogeneity of variance

- Homoscedasticity describes a situation in which the error term (that is, the “noise” or random disturbance in the relationship between the independent variables and the dependent variable) is the same across all values of the independent variables

### Unaffected models

- Neural Networks
- Support Vector Machines
- Trees
"""

import matplotlib.pyplot as plt
import numpy as np
import math
import seaborn as sns
sns.set()
import pandas as pd

import scipy.stats as stats
mu = 0
variance = 1
sigma = math.sqrt(variance)
x = np.linspace(mu - 5*sigma, mu + 5*sigma, 100)
plt.plot(x, stats.norm.pdf(x, mu, sigma))
plt.show()

mu = 0
variance = 1
sigma = math.sqrt(variance)
x = np.linspace(mu - 5*sigma, mu + 5*sigma, 1000)
alpha = 0
y = stats.skewnorm.pdf(x, alpha)
sns.lineplot(x,y)

mu = 0
variance = 1
sigma = math.sqrt(variance)
x = np.linspace(mu - 5*sigma, mu + 5*sigma, 1000)
alpha = 2
y = stats.skewnorm.pdf(x, alpha)
sns.lineplot(x,y)

mu = 0
variance = 1
sigma = math.sqrt(variance)
x = np.linspace(mu - 5*sigma, mu + 5*sigma, 1000)
alpha = -2
y = stats.skewnorm.pdf(x, alpha)
sns.lineplot(x,y)

mu = 0
variance = 1
sigma = math.sqrt(variance)
x = np.linspace(mu - 5*sigma, mu + 5*sigma, 1000)
fig, ax = plt.subplots()
y = stats.skewnorm.pdf(x, 0)
sns.lineplot(x,y, ax=ax)
y = stats.skewnorm.pdf(x, 3)
sns.lineplot(x,y, ax=ax)
y = stats.skewnorm.pdf(x, -3)
sns.lineplot(x,y, ax=ax)
ax.legend(["Normal", "Negative Skew", "Positive Skew"])

"""## Let's explore the distributions of the variables"""

from google.colab import drive
drive.mount('/content/gdrive')
data = pd.read_csv("gdrive/My Drive/Colab Notebooks/FeatureEngineering/train_date.csv")

data.head()

"""#### Age"""

sns.set_color_codes()
sns.distplot(data['Age'], color='r')

"""Make a QQ-plot:

- Quantiles on vertical axis
- Quantiles of distributin on horizontal axis
- The points on the graph will form a 45 degree line if gaussian
"""

fig, ax = plt.subplots()
stats.probplot(data['Age'].dropna(), dist="norm", plot=ax)
ax.set_title("QQPlot")

sns.set_color_codes()
sns.distplot(data['Age']**(1/1.5), color='g')

fig, ax = plt.subplots()
stats.probplot(data['Age'].dropna()**(1/1.5), dist="norm", plot=ax)
ax.set_title("QQPlot")

"""#### Fare"""

sns.set_color_codes()
sns.distplot(data['Fare'], color='r')

"""Make a QQ-plot:

- Quantiles on vertical axis
- Quantiles of distributin on horizontal axis
- The points on the graph will form a 45 degree line if gaussian
"""

fig, ax = plt.subplots()
stats.probplot(data['Fare'].dropna(), dist="norm", plot=ax)
ax.set_title("QQPlot")

sns.set_color_codes()
sns.distplot(data['Fare']**(1/3), color='g')

fig, ax = plt.subplots()
stats.probplot(data['Fare'].dropna()**(1/1.5), dist="norm", plot=ax)
ax.set_title("QQPlot")

"""## Effect on Models"""

from sklearn.model_selection import train_test_split
X_train, X_test, y_train, y_test = train_test_split(
          data[['Age', 'Fare', 'Pclass',]].fillna(data.mean()),
          data['Survived'],
          test_size=0.2)

X_train.shape, X_test.shape

from sklearn.preprocessing import StandardScaler
obj = StandardScaler()
X_train = obj.fit_transform(X_train)
X_test = obj.transform(X_test)

data_processed = data.copy()
data_processed['Age'] = data_processed['Age']**(1/2)
data_processed['Fare'] = data_processed['Fare']**(1/3)
from sklearn.model_selection import train_test_split
X_train_processed, X_test_processed, y_train_processed, y_test_processed = train_test_split(
          data_processed[['Age', 'Fare', 'Pclass',]].fillna(data.mean()),
          data_processed['Survived'],
          test_size=0.2)

X_train_processed.shape, X_test_processed.shape

from sklearn.preprocessing import StandardScaler
obj = StandardScaler()
X_train_processed = obj.fit_transform(X_train_processed)
X_test_processed = obj.transform(X_test_processed)

from sklearn.metrics import accuracy_score

from sklearn.linear_model import LogisticRegression
classifier = LogisticRegression()
classifier.fit(X_train,y_train)
y_pred = classifier.predict(X_test)
y_pred = np.round(y_pred).flatten()
print(accuracy_score(y_test, y_pred))

classifier.fit(X_train_processed,y_train_processed)
y_pred_processed = classifier.predict(X_test_processed)
y_pred_processed = np.round(y_pred_processed).flatten()
print(accuracy_score(y_test_processed, y_pred_processed))

from sklearn.linear_model import RidgeClassifierCV
classifier = RidgeClassifierCV()
classifier.fit(X_train,y_train)
y_pred = classifier.predict(X_test)
y_pred = np.round(y_pred).flatten()
print(accuracy_score(y_test, y_pred))

classifier.fit(X_train_processed,y_train_processed)
y_pred_processed = classifier.predict(X_test_processed)
y_pred_processed = np.round(y_pred_processed).flatten()
print(accuracy_score(y_test_processed, y_pred_processed))

from sklearn.linear_model import RidgeClassifierCV
classifier = RidgeClassifierCV()
classifier.fit(X_train,y_train)
y_pred = classifier.predict(X_test)
y_pred = np.round(y_pred).flatten()
print(accuracy_score(y_test, y_pred))

classifier.fit(X_train_processed,y_train_processed)
y_pred_processed = classifier.predict(X_test_processed)
y_pred_processed = np.round(y_pred_processed).flatten()
print(accuracy_score(y_test_processed, y_pred_processed))

from sklearn.svm import SVC
classifier = SVC()
classifier.fit(X_train,y_train)
y_pred = classifier.predict(X_test)
y_pred = np.round(y_pred).flatten()
print(accuracy_score(y_test, y_pred))

classifier.fit(X_train_processed,y_train_processed)
y_pred_processed = classifier.predict(X_test_processed)
y_pred_processed = np.round(y_pred_processed).flatten()
print(accuracy_score(y_test_processed, y_pred_processed))

from sklearn.neural_network import MLPClassifier
classifier = MLPClassifier()
classifier.fit(X_train,y_train)
y_pred = classifier.predict(X_test)
y_pred = np.round(y_pred).flatten()
print(accuracy_score(y_test, y_pred))

classifier.fit(X_train_processed,y_train_processed)
y_pred_processed = classifier.predict(X_test_processed)
y_pred_processed = np.round(y_pred_processed).flatten()
print(accuracy_score(y_test_processed, y_pred_processed))

from sklearn.svm import LinearSVC
classifier = LinearSVC()
classifier.fit(X_train,y_train)
y_pred = classifier.predict(X_test)
y_pred = np.round(y_pred).flatten()
print(accuracy_score(y_test, y_pred))

classifier.fit(X_train_processed,y_train_processed)
y_pred_processed = classifier.predict(X_test_processed)
y_pred_processed = np.round(y_pred_processed).flatten()
print(accuracy_score(y_test_processed, y_pred_processed))

from sklearn.ensemble import RandomForestClassifier
classifier = RandomForestClassifier()
classifier.fit(X_train,y_train)
y_pred = classifier.predict(X_test)
y_pred = np.round(y_pred).flatten()
print(accuracy_score(y_test, y_pred))

classifier.fit(X_train_processed,y_train_processed)
y_pred_processed = classifier.predict(X_test_processed)
y_pred_processed = np.round(y_pred_processed).flatten()
print(accuracy_score(y_test_processed, y_pred_processed))

from sklearn.tree import DecisionTreeClassifier
classifier = DecisionTreeClassifier()
classifier.fit(X_train,y_train)
y_pred = classifier.predict(X_test)
y_pred = np.round(y_pred).flatten()
print(accuracy_score(y_test, y_pred))

classifier.fit(X_train_processed,y_train_processed)
y_pred_processed = classifier.predict(X_test_processed)
y_pred_processed = np.round(y_pred_processed).flatten()
print(accuracy_score(y_test_processed, y_pred_processed))

from sklearn.ensemble import GradientBoostingClassifier
classifier = GradientBoostingClassifier()
classifier.fit(X_train,y_train)
y_pred = classifier.predict(X_test)
y_pred = np.round(y_pred).flatten()
print(accuracy_score(y_test, y_pred))

classifier.fit(X_train_processed,y_train_processed)
y_pred_processed = classifier.predict(X_test_processed)
y_pred_processed = np.round(y_pred_processed).flatten()
print(accuracy_score(y_test_processed, y_pred_processed))

from sklearn.linear_model import SGDClassifier
classifier = SGDClassifier()
classifier.fit(X_train,y_train)
y_pred = classifier.predict(X_test)
y_pred = np.round(y_pred).flatten()
print(accuracy_score(y_test, y_pred))

classifier.fit(X_train_processed,y_train_processed)
y_pred_processed = classifier.predict(X_test_processed)
y_pred_processed = np.round(y_pred_processed).flatten()
print(accuracy_score(y_test_processed, y_pred_processed))

from sklearn.linear_model import Perceptron
classifier = Perceptron()
classifier.fit(X_train,y_train)
y_pred = classifier.predict(X_test)
y_pred = np.round(y_pred).flatten()
print(accuracy_score(y_test, y_pred))

classifier.fit(X_train_processed,y_train_processed)
y_pred_processed = classifier.predict(X_test_processed)
y_pred_processed = np.round(y_pred_processed).flatten()
print(accuracy_score(y_test_processed, y_pred_processed))

from sklearn.naive_bayes import GaussianNB
classifier = GaussianNB()
classifier.fit(X_train,y_train)
y_pred = classifier.predict(X_test)
y_pred = np.round(y_pred).flatten()
print(accuracy_score(y_test, y_pred))

classifier.fit(X_train_processed,y_train_processed)
y_pred_processed = classifier.predict(X_test_processed)
y_pred_processed = np.round(y_pred_processed).flatten()
print(accuracy_score(y_test_processed, y_pred_processed))

from sklearn.neighbors import KNeighborsClassifier
classifier = KNeighborsClassifier()
classifier.fit(X_train,y_train)
y_pred = classifier.predict(X_test)
y_pred = np.round(y_pred).flatten()
print(accuracy_score(y_test, y_pred))

classifier.fit(X_train_processed,y_train_processed)
y_pred_processed = classifier.predict(X_test_processed)
y_pred_processed = np.round(y_pred_processed).flatten()
print(accuracy_score(y_test_processed, y_pred_processed))


#!/usr/bin/env python
# coding: utf-8

# # Intro to Linear Algebra
#
#

# In[1]:

import numpy as np

# ## Matrix multiplication with NumPy
#
# Element-wise multiplication

# In[2]:

# Element-wise multiplication
matrix_1 = np.array([
    [2,4,6],
    [1,2,3]
])
matrix_2 = np.array([
    [2,2,2],
    [3,3,3]
])
# is multiplying them with a * operator the same as np.multiply?
matrix_1 * matrix_2 == np.multiply(matrix_1, matrix_2)

# Matrix multiplication
#
# Output matrix has:
# - same no of rows as the first matrix
# - same no of cols as the second matrix

# In[3]:

# (2,2) matrix
matrix_a = np.array([
    [1,1,1],
    [1,1,1]
])

# (2, 4)
matrix_b = np.array([
    [2,2],
    [2,2],
    [2,2]
])

np.matmul(matrix_a, matrix_b)

# Doing `Matmul` is essentially the same as doing the Dot product, if the matrices are 2D. That is why you will see it often.

# In[4]:

np.dot(matrix_a, matrix_b)


import numpy as np

def flux2mag(flux, zp, flux_err=0.0):
    """Converts fluxes to magnitudes, propagating errors if given.

    Note: if there are negative or zero fluxes, these are converted to NaN values.

    Parameters
    ----------
    flux : array
        Array of fluxes.
    zp : float or array
        Zero points.
    flux_err : array, default ``0.0``
        Array of flux errors.

    Returns
    -------
    mag : array
        Fluxes converted to magnitudes.
    mag_err : array
        Flux errors converted to errors in magnitudes.
    """

    if type(flux)==np.ndarray:
        flux_ = np.array([f if f>=0.0 else np.nan for f in flux ])  # turns negative and 0.0 values to NaN
    elif flux<=0.0:
        flux_ = np.nan
    else:
        flux_ = flux

    mag = -2.5*np.log10(flux_) + zp
    mag_err = np.abs( 2.5*flux_err/(flux_*np.log(10)) )

    return mag, mag_err

def mag2flux(mag, zp, mag_err=0.0):
    """Converts magnitudes to fluxes, propagating errors if given.

    Parameters
    ----------
    mag : array
        Array of magnitudes.
    zp : float or array
        Zero points.
    mag_err : array, default ``0.0``
        Array of magnitude errors.

    Returns
    -------
    flux : array
        Magnitudes converted to fluxes.
    flux_err : array
        Magnitude errors converted to errors in fluxes.
    """

    flux = 10**( -0.4*(mag-zp) )
    flux_err =  np.abs( flux*0.4*np.log(10)*mag_err )

    return flux, flux_err

def change_zp(flux, zp, new_zp):
    """Converts flux units given a new zero-point.

    **Note:** this assumes that the new zero-point is in the same magnitude system as the current one.

    Parameters
    ----------
    flux : float or array
        Fluxes.
    zp : float or array
        Current zero-point for the given fluxes.
    new_zp : float or array
        New zero-point to convert the flux units.

    Returns
    -------
    new_flux : float or array
        Fluxes with with a new zero-point.

    """

    new_flux = flux*10**( -0.4*(zp - new_zp) )

    return new_flux

def trim_filters(response):
    """Trim the leading and trailing zeros from a 1-D array or sequence, leaving
    one zero on each side. This is a modified version of numpy.trim_zeros.

    Parameters
    ----------
    response : 1-D array or sequence
        Input array.

    Returns
    -------
    first : int
        Index of the last leading zero.
    last : int
        Index of the first trailing zero.
    """

    first = 0
    for i in response:
        if i != 0.:
            if first == 0:
                first += 1  # to avoid filters with non-zero edges
            break
        else:
            first = first + 1

    last = len(response)
    for i in response[::-1]:
        if i != 0.:
            if last == len(response):
                last -= 1  # to avoid filters with non-zero edges
            break
        else:
            last = last - 1

    first -= 1
    last += 1

    return first, last

def extrapolate_mangling_edges(x, y, yerr, x_edges, extra_extension=0.0):
    """"Extrapolates the edges of y according to the giving edges in x, x_edges.

    Parameters
    ----------
    x_data : array
        Independent values.
    y_data : array
        Dependent values.
    yerr_data : array, int
        Dependent value errors.
    x_edges: array-like
        Minimum and maximum x-axis values. These are used to extrapolate both edges.
    extra_extension: float, default ``0.0``
        This value is added to extend the edges even more. It might look redundant, but it has its purpose.

    Returns
    -------
    x : array
        Extrapolated independent values.
    y : array
        Extrapolated dependent values.
    yerr : array, int
        Extrapolated dependent value errors.
    """

    x_min, x_max = x_edges
    x_min -= extra_extension
    x_max += extra_extension

    # linear extrapolation
    pmin = np.polyfit(x[:2], y[:2], deg=1)
    pmax = np.polyfit(x[-2:], y[-2:], deg=1)
    y_left, y_right = np.poly1d(pmin)(x_min), np.poly1d(pmax)(x_max)

    # avoid negative values
    if y_left<0:
        y_left=0
    if y_right<0:
        y_right=0
    y = np.r_[y_left, y, y_right]
    x = np.r_[x_min, x, x_max]
    try:
        yerr_min = np.sqrt(yerr[0]**2 + yerr[1]**2)
        yerr_max = np.sqrt(yerr[-1]**2 + yerr[-2]**2)
        yerr = np.r_[yerr_min, yerr, yerr_max]
    except:
        pass

    return x, y, yerr


#
#  file:  gd_1d.py
#
#  1D example of GD
#
#  RTK, 14-Feb-2021
#  Last update:  14-Feb-2021
#
################################################################

import sys
import os
import numpy as np
import matplotlib.pylab as plt

#  The function and its derivative
def f(x):
    return 6*x**2 - 12*x + 3

def d(x):
    return 12*x - 12

#  Show the function, derivative, and minimum
x = np.linspace(-1,3,1000)
y = f(x)
plt.plot(x,y,color='#1f77b4')
x = np.linspace(0,3,10)
z = d(x)
plt.plot(x,z,color='#ff7f0e')
plt.plot([-1,3],[0,0],linestyle=(0,(1,1)),color='k')
plt.plot([1,1],[-10,25],linestyle=(0,(1,1)),color='k')
plt.plot([1,1],[f(1),f(1)],marker='o',color='#1f77b4')
plt.xlabel("$x$")
plt.ylabel("$y$")
plt.tight_layout(pad=0, w_pad=0, h_pad=0)
plt.savefig("gd_1d_plot.png", dpi=300)
#plt.show()
plt.close()

#  Show a series of gradient descent steps
x = np.linspace(-1,3,1000)
plt.plot(x,f(x))

x = -0.9
eta = 0.03
for i in range(15):
    plt.plot(x, f(x), marker='o', color='r')
    x = x - eta * d(x)

plt.xlabel("$x$")
plt.ylabel("$y$")
plt.tight_layout(pad=0, w_pad=0, h_pad=0)
plt.savefig("gd_1d_steps.png", dpi=300)
#plt.show()
plt.close()
print("Minimum at (%0.6f, %0.6f)" % (x, f(x)))

#  Show oscillation if step size too large
x = np.linspace(0.75,1.25,1000)
plt.plot(x,f(x))
x = xold = 0.75
for i in range(14):
    plt.plot([xold,x], [f(xold),f(x)], marker='o', linestyle='dotted', color='r')
    xold = x
    x = x - 0.15 * d(x)

plt.xlabel("$x$")
plt.ylabel("$y$")
plt.tight_layout(pad=0, w_pad=0, h_pad=0)
plt.savefig("gd_1d_oscillating.png", dpi=300)
#plt.show()


import numpy as np

def affine_forward(x, w, b):
    """
    This function computes the forward pass for an affine (fully-connected) layer.

    The input x has shape (N, d) and contains a N data examples each of which has d attributes.

    Inputs:
    x: input data, an array of shape (N, d)
    w: weights, an array of shape (d, M)
    b: biases, an array of shape (M,)

    Outputs:
    out: output, an array of shape (N, M)
    cache: (x, w, b)
    """
    out = None
    #############################################################################
    #                            START OF YOUR CODE                            #
    #############################################################################
    out = np.matmul(x, w) + b
    #############################################################################
    #                             END OF YOUR CODE                              #
    #############################################################################
    cache = (x, w, b)
    return out, cache

def affine_backward(dout, cache):
    """
    This function computes the backward pass for an affine (fully-connected) layer.

    Inputs:
    dout: derivatives coming from the next layer, an array of shape (N, M)
    cache: Tuple of:
      - x: input data, an array of shape (N, d)
      - w: weights, an array of shape (d, M)
      - b: biases, an array of shape (M,)

    Outputs:
    dx: gradient with respect to x, an array of shape (N, d)
    dw: gradient with respect to w, an array of shape (d, M)
    db: gradient with respect to b, an array of shape (M,)
    """
    x, w, b = cache
    dx, dw, db = None, None, None
    #############################################################################
    #                            START OF YOUR CODE                            #
    #############################################################################
    dx = np.matmul(dout, w.T)
    dw = np.matmul(x.T, dout)
    db = np.sum(dout, axis=0)
    #############################################################################
    #                             END OF YOUR CODE                              #
    #############################################################################
    return dx, dw, db

def relu_forward(x):
    """
    This function computes the forward pass for a layer of rectified linear units (ReLUs).

    Input:
    x: inputs, an array of any shape

    Outputs:
    out: Output, an array of the same shape as x
    cache: x
    """
    out = None
    #############################################################################
    #                            START OF YOUR CODE                            #
    #############################################################################
    out = np.maximum(0, x)
    #############################################################################
    #                             END OF YOUR CODE                              #
    #############################################################################
    cache = x
    return out, cache

def relu_backward(dout, cache):
    """
    This function computes the backward pass for a layer of rectified linear units (ReLUs).

    Inputs:
    dout: derivatives coming from the next layer, an array of any shape
    cache: input x, an array of same shape as dout

    Outputs:
    dx: gradient with respect to x
    """
    dx, x = None, cache
    #############################################################################
    #                            START OF YOUR CODE                            #
    #############################################################################
    dx = (x > 0) * dout
    #############################################################################
    #                             END OF YOUR CODE                              #
    #############################################################################
    return dx

def L2_loss(x, y):
    """
    This functin computes the loss and gradient using L2 norm.

    Inputs:
    x: input data, an array of shape (N,) where x[i] is the regression output for the ith input.
    y: vector of labels, an array of shape (N,) where y[i] is the label for x[i].

    Outputs:
    loss: scalar giving the loss
    dx: gradient of the loss with respect to x
    """
    loss = None
    dx = None
    #############################################################################
    #                            START OF YOUR CODE                            #
    #############################################################################
    loss = np.sum(np.square(y - x)) / 2 / x.shape[0]
    dx = -(y - x) / x.shape[0]
    #############################################################################
    #                             END OF YOUR CODE                              #
    #############################################################################
    return loss, dx


# LUPFactors.py
#
# Daniel R. Reynolds
# SMU Mathematics
# Math 4315

# imports
import numpy

def LUPFactors(A):
    """
    usage: [L,U,P] = LUPFactors(A)

    Row-oriented LU factorization with partial pivoting; constructs the factorization
          P A = LU  <=>  A = P^T L U
    This function checks that A is square, and attempts to catch the case where A
    is singular.

    Inputs:
      A - square n-by-n matrix

    Outputs:
      L - unit-lower-triangular matrix (n-by-n)
      U - upper-triangular matrix (n-by-n)
      P - permutation matrix (n-by-n)
    """

    # check input
    m, n = numpy.shape(A)
    if (m != n):
        raise ValueError("LUPFactors error: matrix must be square")

    # set singularity tolerance
    tol = 1000*numpy.finfo(float).eps

    # create output matrices
    U = A.copy()
    L = numpy.eye(n)
    P = numpy.eye(n)
    for k in range(n-1):             # loop over pivots
      s = k                          # determine pivot row
      for i in range(k+1,n):
        if (numpy.abs(U[i,k]) > numpy.abs(U[s,k])):
          s = i
      if (numpy.abs(U[s,k]) < tol):  # check for singularity
        raise ValueError("LUPFactors error: matrix is [close to] singular")
      for j in range(k,n):           # swap rows in U
        tmp = U[k,j]
        U[k,j] = U[s,j]
        U[s,j] = tmp
      for j in range(k):             # swap rows in L
        tmp = L[k,j]
        L[k,j] = L[s,j]
        L[s,j] = tmp
      for j in range(n):             # swap rows in P
        tmp = P[k,j]
        P[k,j] = P[s,j]
        P[s,j] = tmp
      for i in range(k+1,n):         # loop over remaining rows
        L[i,k] = U[i,k]/U[k,k]       # compute multiplier
        for j in range(k,n):         # update remainder of matrix row
          U[i,j] -= L[i,k]*U[k,j]

    return [L, U, P]

# end function


# Taken from: https://github.com/nghiaho12/rigid_transform_3D/blob/master/rigid_transform_3D.py

import numpy as np

def rigid_transform_3D(A, B):
    assert A.shape == B.shape

    num_rows, num_cols = A.shape
    if num_rows != 3:
        raise Exception(f"matrix A is not 3xN, it is {num_rows}x{num_cols}")

    num_rows, num_cols = B.shape
    if num_rows != 3:
        raise Exception(f"matrix B is not 3xN, it is {num_rows}x{num_cols}")

    # find mean column wise
    centroid_A = np.mean(A, axis=1)
    centroid_B = np.mean(B, axis=1)

    # ensure centroids are 3x1
    centroid_A = centroid_A.reshape(-1, 1)
    centroid_B = centroid_B.reshape(-1, 1)

    # subtract mean
    Am = A - centroid_A
    Bm = B - centroid_B

    H = Am @ np.transpose(Bm)

    # find rotation
    U, S, Vt = np.linalg.svd(H)
    R = Vt.T @ U.T

    # special reflection case
    if np.linalg.det(R) < 0:
        print("det(R) < R, reflection detected!, correcting for it ...")
        Vt[2,:] *= -1
        R = Vt.T @ U.T

    t = -R @ centroid_A + centroid_B

    return R, t

# Test with random data

# Random rotation and translation
R = np.random.rand(3,3)
t = np.random.rand(3,1)

# make R a proper rotation matrix, force orthonormal
U, S, Vt = np.linalg.svd(R)
R = U@Vt

# remove reflection
if np.linalg.det(R) < 0:
   Vt[2,:] *= -1
   R = U@Vt

# number of points
n = 10

A = np.random.rand(3, n)
B = R@A + t

# Recover R and t
ret_R, ret_t = rigid_transform_3D(A, B)

# Compare the recovered R and t with the original
B2 = (ret_R@A) + ret_t

# Find the root mean squared error
err = B2 - B
err = err * err
err = np.sum(err)
rmse = np.sqrt(err/n)

print("Points A")
print(A)
print("")

print("Points B")
print(B)
print("")

print("Ground truth rotation")
print(R)

print("Recovered rotation")
print(ret_R)
print("")

print("Ground truth translation")
print(t)

print("Recovered translation")
print(ret_t)
print("")

print("RMSE:", rmse)

if rmse < 1e-5:
    print("Everything looks good!")
else:
    print("Hmm something doesn't look right ...")


from scipy import *
from scipy.interpolate import lagrange
from numpy import *

def equispaced(order):
    '''
    Takes input d and returns the vector of d equispaced points in [-1,1]
    And the integral of the basis functions interpolated in those points
    '''
    nodes= linspace(-1,1,order)
    w= zeros(order)
    for k in range(order):
        yy= zeros(order)
        yy[k]=1.
        zz=lagrange(nodes,yy)
        pp=zz.integ()
        w[k]=pp(1)-pp(-1)

    return nodes, w

def lglnodes(n,eps):
    '''
    Python translation of lglnodes.m

    Computes the Legendre-Gauss-Lobatto nodes, weights and the LGL Vandermonde
    matrix. The LGL nodes are the zeros of (1-x^2)*P'_N(x). Useful for numerical
    integration and spectral methods.

    Parameters
    ----------
    n : integer, requesting an nth-order Gauss-quadrature rule on [-1, 1]

    Returns
    -------
    (nodes, weights) : tuple, representing the quadrature nodes and weights.
                       Note: (n+1) nodes and weights are returned.

    Example
    -------
    >>> from lglnodes import *
    >>> (nodes, weights) = lglnodes(3)
    >>> print(str(nodes) + "   " + str(weights))
    [-1.        -0.4472136  0.4472136  1.       ]   [0.16666667 0.83333333 0.83333333 0.16666667]

    Notes
    -----

    Reference on LGL nodes and weights:
      C. Canuto, M. Y. Hussaini, A. Quarteroni, T. A. Tang, "Spectral Methods
      in Fluid Dynamics," Section 2.3. Springer-Verlag 1987

    Written by Greg von Winckel - 04/17/2004
        Contact: gregvw@chtm.unm.edu

    Translated and modified into Python by Jacob Schroder - 9/15/2018
    '''

    w = zeros((n+1,))
    x = zeros((n+1,))
    xold = zeros((n+1,))

    # The Legendre Vandermonde Matrix
    P = zeros((n+1,n+1))

    epss = eps

    # Use the Chebyshev-Gauss-Lobatto nodes as the first guess
    for i in range(n+1):
        x[i] = -cos(pi*i / n)

    # Compute P using the recursion relation
    # Compute its first and second derivatives and
    # update x using the Newton-Raphson method.

    xold = 2.0

    for i in range(100):
        xold = x

        P[:,0] = 1.0
        P[:,1] = x

        for k in range(2,n+1):
            P[:,k] = ( (2*k-1)*x*P[:,k-1] - (k-1)*P[:,k-2] ) / k

        x = xold - ( x*P[:,n] - P[:,n-1] )/( (n+1)*P[:,n])

        if (max(abs(x - xold).flatten()) < epss ):
            break

    w = 2.0 / ( (n*(n+1))*(P[:,n]**2))

    return x, w


#!/usr/bin/python

# This is based on Andrew Ng's course on Coursera
# The dataset has been taken from there and I have
# attempted the implement the linear regression of
# one variable using gradient descent alogrithm in
# python.
# Additionally, to provide a view of how the algorithm is
# working, a graph continuously updates itself while
# algorithm calculateas the best fitting regression line
# in each iteration.
# Note: that makes the program run slowly, so the graphing
# capabilities have been commented out. uncomment lines below
# followin comment:
#	# XXXXXXXXXXXX  For Graph XXXXXXXXXXX
import numpy as np
import matplotlib.pyplot as plt

num_iter = 1500
# Change the learning rate and see the effect
# The fastest convergence is observed when set to
# 0.01. Visually, best convergence is on 0.01.
learning_rate = 0.01

def computeCost(X, Y, theta):
    #print X.shape,"*", theta.shape," - ", Y.shape
    #print "X*theta - Y\n", X, "*", theta, "-", Y
    sumsq = np.sum(np.square(np.dot(X,theta) - Y))
    J = 1.0/(2 * len(Y)) * sumsq
    return J

def plot_graph(X, theta, Y):
	y = []
	x = np.array([0, max(X)])
	plt.cla()
	y.append(theta[0] + x[0] * theta[1])
	y.append(theta[0] + x[1] * theta[1])
	m, c = np.polyfit(x, y, 1)
	ax.scatter(X, Y)
	ax.plot(x,x*m + c, 'r')
	fig.canvas.draw()

def gradient_descent(X, Y, theta, learning_rate, num_iter):
    m = len(Y)
    delta_theta = np.array([0,0])
    for iter in xrange(num_iter):
	error_vector = np.dot(X, theta) - Y
	#print "Error_vector", error_vector, error_vector.shape
	sum_ev = np.dot(error_vector, X)
	delta_theta = (learning_rate/m) * sum_ev
	#print "sum_ev:", sum_ev, sum_ev.shape
	theta = theta - delta_theta
	# see how this affects cost
	#print "Cost:", computeCost(X, Y, theta)
	# XXXXXXXXXXXX  For Graph XXXXXXXXXXX
	#print "Iter: ", iter, "graph:" , plot_graph(X[:,1],theta, Y)

    return theta

#XXXXXXXXXXXX  For Graph XXXXXXXXXXX
#plt.ion()
#fig, ax = plt.subplots()

X = np.genfromtxt('ex1data1.txt', delimiter=",", usecols=(0), comments='#')
Y = np.genfromtxt('ex1data1.txt', delimiter=",", usecols=(1), comments='#')
theta = np.array([0,0])
X = np.c_[ np.ones(len(Y)),X ]

#computeCost(X, Y, theta)

grad_theta = gradient_descent(X, Y, theta, learning_rate, num_iter)
print "Geadient'd theta:", grad_theta

# Caution:
# note how the 'X[i]' values in dataset are normalised. Look at ex1.pdf fig 1,
# the X axis represents the city population in 10000s. If we use 35000 directly
# in matrix multiplication the result differs.
# Nutshell, be aware of the intitial trteatment to be done to the X. We need to
# feed the asking value treated in the same way as rest of the sample values in
# the data set.  Rather if we directly do np.dot(np.array([1,35000]), grad_theta).
# Result is somewhat different. Its easy to every thing correct and still get a
# wrong answer.
print "For population of 35000, profit predicted:", np.dot(np.array([1,3.5]), grad_theta) * 10000
print "For population of 70000, profit predicted:", np.dot(np.array([1,7.0]), grad_theta) * 10000

# Last but not the leaset, may thanks to Prof. Andrew Ng.
# Kapil


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Mon Feb  7 18:52:58 2022

@author: Analabha Roy
"""
import numpy as np

def GEPP(A, b, doPP=True):
    '''
    Gaussian elimination with partial pivoting.

    input: A is an n x n numpy matrix
           b is an n x 1 numpy array
    output: x is the solution of Ax=b
            with the entries permuted in
            accordance with the pivoting
            done by the algorithm
    post-condition: A and b have been modified.
    '''
    n = len(A)
    if b.size != n:
        raise ValueError("Invalid argument: incompatible sizes between" +
                         "A & b.", b.size, n)
    # k represents the current pivot row. Since GE traverses the matrix in the
    # upper right triangle, we also use k for indicating the k-th diagonal
    # column index.

    # Elimination
    for k in range(n-1):
        if doPP:
            # Pivot
            maxindex = abs(A[k:, k]).argmax() + k
            if A[maxindex, k] == 0:
                raise ValueError("Matrix is singular.")
            # Swap
            if maxindex != k:
                A[[k, maxindex]] = A[[maxindex, k]]
                b[[k, maxindex]] = b[[maxindex, k]]
        else:
            if A[k, k] == 0:
                raise ValueError("Pivot element is zero. Try setting doPP to True.")
        # Eliminate
        for row in range(k+1, n):
            multiplier = A[row, k]/A[k, k]
            A[row, k:] = A[row, k:] - multiplier * A[k, k:]
            b[row] = b[row] - multiplier*b[k]
    # Back Substitution
    x = np.zeros(n)
    for k in range(n-1, -1, -1):
        x[k] = (b[k] - np.dot(A[k, k+1:], x[k+1:]))/A[k, k]
    return x

def det(A):
    _ = GEPP(A, np.ones(A.shape[0]), doPP=True)
    return np.prod(np.diagonal(A))

if __name__ == '__main__':
    A = np.array([[25., 5., 1.],
                  [64., 8., 1.],
                  [144., 12., 1.]])

    b = np.array([106.8,
                  177.2,
                  279.2])

    x = GEPP(np.copy(A), np.copy(b), doPP=False)
    print("First solution is given by x =", x)
    print("Error is ", np.linalg.norm(A@x - b) * 100/np.linalg.norm(b), "%")

    print("Determinant of first matrix is ", det(np.copy(A)))

    A = np.array([[12., 10., -7.],
                  [6., 5., 3.],
                  [5., -1., 5.]])
    b = np.array([15.,
                  4.,
                  9.])

    try:
        x = GEPP(np.copy(A), np.copy(b), doPP=False)
    except ValueError:
        x = GEPP(np.copy(A), np.copy(b))

    print("Second solution is given by x =", x)
    print("Error is ", np.linalg.norm(A @ x - b) * 100/np.linalg.norm(b), "%")


# import modules
import numpy as np

'''
# Description.
Given an array of two columns (n x 2), the function extracts unique row
values that are not necessarily and strictly equal among other rows; the
desition is governed by a tolerance value.

# Input(s).
Array of n x 2 of dimension (array).

Number that will define the tolerance of the selection criterium (tolerance).

# Output(s).
New array with unique rows (uniqueArray).

# Example:
Given the following array:
array = np.array([\
   [ 9.   ,  2.   ],
   [ 7.   ,  4.123],
   [ 5.   ,  6.129],
   [ 7.   ,  8.12 ],
   [ 0.   ,  1.1  ],
   [ 9.001,  2.   ]])
it's obtained:
array([[ 0.   ,  1.1  ],
       [ 5.   ,  6.129],
       [ 7.   ,  4.123],
       [ 7.   ,  8.12 ],
       [ 9.   ,  2.   ]])
---
uniqueArray = uniquewithtolerance(array, tolerance = 0.001)
'''
def uniquewithtolerance( array, tolerance = 0.001):

    ## Doing the sort
    indexes = np.argsort(array, 0)
    sortedArray = array[indexes[:,0], :]

    ## Applying the tolerance criterium
    uniMaskX = np.diff(np.hstack((np.zeros(1), sortedArray[:,0]))) > tolerance
    uniMaskY = np.diff(np.hstack((np.zeros(1), sortedArray[:,1]))) > tolerance

    uniMask = np.logical_or(uniMaskX, uniMaskY)
    uniqueArray = sortedArray[uniMask,:]

    return uniqueArray
'''
BSD 2 license.

Copyright (c) 2016, Universidad Nacional de Colombia, Ludger O.
   Suarez-Burgoa and Exneyder Andrés Montoya Araque.
All rights reserved.

Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:

1. Redistributions of source code must retain the above copyright notice,
this list of conditions and the following disclaimer.

2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
'''


import numpy as np
from numpy import matmul

class FunctionProvider:
    def __init__(self, f, g, h):
        self.f = f
        self.grad = g
        self.hessian = h

    def __add__(self, other):
        f = lambda x: self.f(x) + other.f(x)
        g = lambda x: self.grad(x) + other.grad(x)
        h = lambda x: self.hessian(x) + other.hessian(x)

        return FunctionProvider(f, g, h)

    def __mul__(self, other):
        f = lambda x: self.f(x) * other
        g = lambda x: self.grad(x) * other
        h = lambda x: self.hessian(x) * other

        return FunctionProvider(f, g, h)

    def __rmul__(self, other):
        return self.__mul__(other)

class QuadraticFunctionProvider(FunctionProvider):
    """
    1/2 x^𝑇 A x - b^𝑇 x
    """
    def __init__(self, A, b):
        b = np.reshape(b, [-1, 1])
        assert np.shape(b)[0] == np.shape(A)[0] == np.shape(A)[1]

        f = lambda x: 0.5 * matmul(matmul(x.T, A), x) - matmul(b.T, x)
        g = lambda x: matmul(A, x) - b
        h = lambda x: A

        FunctionProvider.__init__(self, f, g, h)


import matplotlib.pyplot as plt
import numpy as np
import csv
import time

def compute_polynoms(points):
    a = points[1]
    n = a.size

    h = np.zeros(n - 1)
    for i in range(n - 1):
        h[i] = points[0][i + 1] - points[0][i]

    u = np.zeros((n, n))
    u[0][0] = 1
    u[n - 1][n - 1] = 1
    for i in range(1, n - 1):
        u[i][i - 1] = h[i - 1]
        u[i][i] = 2 * (h[i - 1] + h[i])
        u[i][i + 1] = h[i]

    v = np.zeros(n)
    for i in range(1, n - 1):
        v[i] = (3 * (a[i + 1] - a[i]) / h[i]) - \
            (3 * (a[i] - a[i - 1]) / h[i - 1])

    c = np.linalg.inv(u).dot(v)

    d = np.zeros(n - 1)
    b = np.zeros(n - 1)

    for i in range(n - 1):
        d[i] = (c[i + 1] - c[i]) / (3 * h[i])
        b[i] = (a[i + 1] - a[i]) / h[i] - ((2 * c[i] + c[i + 1]) * h[i]) / 3

    return a, b, c, d

def spline_interpolation(points, x, a, b, c, d):
    k = -1
    for i in range(points[0].size - 1):
        if points[0][i] <= x and x <= points[0][i + 1]:
            k = i

    return a[k] + (b[k] * (x - points[0][k])) + (c[k] * (x - points[0][k]) ** 2) + (d[k] * (x - points[0][k]) ** 3)

def read_csv(filename):
    with open(filename, 'rt') as csv_file:
        points = []
        buffer = csv.reader(csv_file)
        for row in buffer:
            points.append([float(x) for x in row])
    return points

def linear_interpolation(points, x):
    for i in range(points[0].size - 1):
        if points[0][i] <= x and x < points[0][i + 1]:
            return points[1][i] + (points[1][i + 1] - points[1][i]) \
                * (x - points[0][i]) / (points[0][i + 1] - points[0][i])

def lagrange_multiplicator(points, x, k):
    numerator = 1
    denominator = 1
    for i in range(points[0].size):
        if i != k:
            numerator *= (x - points[0][i])
            denominator *= (points[0][k] - points[0][i])
    return numerator / denominator

def lagrange_interpolation(points, x):
    y = 0
    for i in range(points[0].size):
        y += lagrange_multiplicator(points, x, i)*points[1][i]
    return y

if(__name__ == "__main__"):
    points = np.array(read_csv("points.csv"))
    interval = np.linspace(min(points[0]), max(points[0]), 5000)
    lin_time = time.time()
    linear_interpolation = [linear_interpolation(points, x) for x in interval]
    print("Linear Interpolation Time: ", time.time() - lin_time)

    spline_time = time.time()
    a, b, c, d = compute_polynoms(points)
    spline_interpolation = [spline_interpolation(points, x, a, b, c, d) for x in interval]
    print("Spline Interpolation Time: ", time.time() - spline_time)

    poly_time = time.time()
    poly_interpolation = [lagrange_interpolation(points, x) for x in interval]
    print("Lagrange Poly Interpolation Time: ", time.time() - poly_time)

    plt.plot(interval, linear_interpolation, label='Linear')
    plt.plot(interval, spline_interpolation, label='Spline')
    plt.plot(interval, poly_interpolation, label='Poly')
    plt.plot(points[0], points[1], 'o', label='Points')
    plt.legend()
    plt.show()


import numpy as np

def gradient(A, b, eps, maxIterations):
  m = len(A)
  xk = np.zeros(m)
  rk = b - np.dot(A, xk)
  zk = rk

  for k in range(maxIterations):
    a = np.dot(rk, rk) / np.dot(np.dot(A, zk), zk)
    x = xk + a * zk
    r = rk - a * np.dot(A, zk)
    beta = np.dot(r, r) / np.dot(rk, rk)
    z = r + zk * beta
    print(f"{k}-th iteration:")
    print(xk)
    if np.linalg.norm(np.dot(A, xk) - b) <= eps * np.linalg.norm(b):
      print("^^^^^^^^^^^^^^")
      print(">> Малость невязки << Solution found")
      return
    if np.linalg.norm(x - xk) <= eps * np.linalg.norm(xk):
      print("^^^^^^^^^^^^^^")
      print(">> Малость нормы приближений << Solution found")
      return
    xk, rk, zk = x, r, z

  print("Number of iterations exceeded")


#from math import sqrt, cos
from numpy import sqrt, cos

def objective_function(params):
    x1 = params[0]
    x2 = params[1]

    return ( 1.0 + 1.0/4000.0*(x1**2 + x2**2) - ( cos(x1/sqrt(1.0))*cos(x2/sqrt(2.0)) ) )


import numpy as np
from scipy.special import erf

class GELU:
    def __call__(self, x: np.ndarray, approximate: bool = True) -> np.ndarray:
        if approximate:
            return x * 0.5 * (1 + np.tanh(np.sqrt(2 / np.pi) * (x + 0.044715 * np.power(x, 3))))
        return x * 0.5 * (1.0 + erf(x / np.sqrt(2.0)))

    def gradient(self, x: np.ndarray, approximate: bool = True) -> np.ndarray:
        if approximate:
            return 0.5 * np.tanh(0.0356774 * np.power(x, 3) + 0.797885 * x) + (0.0535161 * np.power(x, 3) + 0.398942 * x) * np.power(1 / np.cosh(x), 2) * (0.0356774 * np.power(x, 3) + 0.797885 * x) + 0.5
        return 0.5 * (1.0 + erf(x / np.sqrt(2.0))) + x * 1 / np.sqrt(2 * np.pi) * np.exp(-0.5 * np.power(x, 2))


from sympy import *
from sympy.printing import print_ccode
from sympy.physics.vector import ReferenceFrame, gradient, divergence
from sympy.vector import CoordSysCartesian

R = ReferenceFrame('R')
x = R[0]
y = R[1]

a=-0.5
b=1.5
visc=1e-1
lambda_=(1/(2*visc)-sqrt(1/(4*visc**2)+4*pi**2))
print(" visc=%f" % visc)

u=[0,0]
u[0]=1-exp(lambda_*x)*cos(2*pi*y)
u[1]=lambda_/(2*pi)*exp(lambda_*x)*sin(2*pi*y)
p=(exp(3*lambda_)-exp(-lambda_))/(8*lambda_)-exp(2*lambda_*x)/2
p=p - integrate(p, (x,a,b))

grad_p = gradient(p, R).to_matrix(R)
f0 = -divergence(visc*gradient(u[0], R), R) + grad_p[0]
f1 = -divergence(visc*gradient(u[1], R), R) + grad_p[1]
f2 = divergence(u[0]*R.x + u[1]*R.y, R)

print("\n * RHS:")
print(ccode(f0, assign_to = "values[0]"))
print(ccode(f1, assign_to = "values[1]"))
print(ccode(f2, assign_to = "values[2]"))

print("\n * ExactSolution:")
print(ccode(u[0], assign_to = "values[0]"))
print(ccode(u[1], assign_to = "values[1]"))
print(ccode(p, assign_to = "values[2]"))

print("")
print("pressure mean:", N(integrate(p,(x,a,b))))


import numpy as np
import matplotlib.pyplot as plt

A=np.array([[0.7,0.2]
          ,[0.3,0.8]])

B=np.array([[8000]
          ,[2000]])

year=10
married=[B[0,0]]
single=[B[1,0]]
for i in range(year):
    temp=np.matmul(A,B)
    B=temp
    married.append(B[0,0])
    single.append(B[1,0])

#visualize
x=np.arange(year+1)
plt.plot(x,married,color='blue',lw=1.0,ls='-',label="married")
plt.plot(x,single,color='red',lw=1.0,ls='-',label="single")
plt.xlabel("year")
plt.ylabel("number of people")
plt.title("martial population")
plt.legend()
plt.show()


import numpy as np

def cross_entropy_error(y, t):
    delta = 1e-7
    return -np.sum(t * np.log(y + delta))

t = np.array([0, 0, 1, 0, 0, 0, 0, 0, 0, 0])
y = np.array([0.1, 0.05, 0.6, 0.0, 0.05, 0.1, 0.0, 0.1, 0.0, 0.0])
print(cross_entropy_error(y, t))

y = np.array([0.1, 0.05, 0.1, 0.0, 0.05, 0.1, 0.0, 0.6, 0.0, 0.0])
print(cross_entropy_error(y, t))


import numpy as np
from optfunction import OptFunction

def rotation_matrix(theta):
    return np.array([
        [np.cos(theta), -np.sin(theta)],
        [np.sin(theta), np.cos(theta)]
    ])

def ex_f1(x):
    """
    Example function 1: the norm of a 2D vector.
    """
    return x.T.dot(x)

def ex_gradf1(x):
    """
    Gradient function 1
    """
    return 2 * x

ex1 = OptFunction(ex_f1, ex_gradf1)

def conditioned_quadratic(cond=50):
    """
    Produces a quadratic form which has a specified condition number. The
    quadratic form has some random variation and a random rotation.
    """
    A = np.array([[cond, 0], [0, 1]])
    theta = np.random.rand() * 2 * np.pi
    forward_rot = rotation_matrix(theta)
    backward_rot = rotation_matrix(-theta)
    A = backward_rot.dot(A.dot(forward_rot))

    def f(x):
        return x.T.dot(A.dot(x))

    def gradf(x):
        return (A + A.T).dot(x)

    def hessf(x, y):
        return A + A.T

    return OptFunction(f, gradf, hessf)

def ex_f2(x):
    return np.log(x[0]) * np.log(x[0]) + x[1] * x[1]

def ex_gradf2(x):
    dx = 2 * np.log(x[0]) / x[0]
    dy = 2 * x[0]
    return np.array([dx, dy])

def ex_hessf2(x):
    dxdx = (2. - 2 * np.log(x[0])) / x[0] / x[0]
    dydy = 2.
    return np.array([[dxdx, 0.], [0., dydy]])

ex2 = OptFunction(ex_f2, ex_gradf2, ex_hessf2)


from __future__ import unicode_literals

import numpy as np

def sample_linear_density(nsamp, x0, w, y0, y1):
    '''
        Input:
            nsamp   -   number of samples
            x0      -   left boundary
            w       -   interval length
            y0      -   density at left boundary
            y1      -   density at right boundary
    '''
    m = y0
    k = y1-y0
    u = np.random.rand(nsamp)
    if k != 0:
        q = m/k
        return (-q + np.sign(q)*np.sqrt(q**2+(1+2*q)*u))*w + x0
    return u*w + x0

def evenly_spaced_linear_density(nsamp, x0, w, y0, y1):
    '''
        Input:
            nsamp   -   number of samples
            x0      -   left boundary
            w       -   interval length
            y0      -   density at left boundary
            y1      -   density at right boundary
    '''
    m = y0
    k = y1-y0
    u = np.linspace(0, 1, nsamp, endpoint=False)
    if k != 0:
        q = m/k
        return (-q + np.sign(q)*np.sqrt(q**2+(1+2*q)*u))*w + x0
    return u*w + x0

def fp_blurring(data, w, even_spaced=False):
    '''
        Blurs data using the frequency polygon. Data is assumed to
        be binned with bin width w. Purpose of blurring is to counter
        effect of truncation. When even_spaced is True, the blurred data
        is put in a deterministic way according to the density, when
        False it is sampled from the density.

        Ref: Minnotte (1997): Nonparametric Testing of Existence of Modes.

        Input:
            data    -   data set (one-dimensional)
            w       -   bin width

    '''
    y, x = np.histogram(data, bins=np.arange(min(data)-0.5*w, max(data)+1.5*w, w))
    y_count = np.hstack([[0], y, [0]])
    x_fp = np.zeros(2*len(x)-1)
    x_fp[0::2] = x
    x_fp[1::2] = (x[1:]+x[:-1])/2
    y_fp = np.zeros(2*len(x)-1)
    y_fp[1::2] = y
    y_fp[::2] = (y_count[1:]+y_count[:-1])*1./2

    n_fp = np.zeros(2*len(y), dtype=np.int)
    p_left = (y_count[:-2] + 3*y_count[1:-1])*1./(y_count[:-2] + 6*y_count[1:-1] + y_count[2:])
    p_left[np.isnan(p_left)] = 0
    if not even_spaced:
        n_fp[0::2] = np.random.binomial(y, p_left)
    else:
        n_fp[0::2] = np.round(y*p_left)
    n_fp[1::2] = y - n_fp[0::2]
    data_fp = []
    for n, x0, y0, y1 in zip(n_fp, x_fp[:-1], y_fp[:-1], y_fp[1:]):
        if not even_spaced:
            data_fp.append(sample_linear_density(n, x0, w*0.5, y0, y1))
        else:
            data_fp.append(evenly_spaced_linear_density(n, x0, w*0.5, y0, y1))
    data_blurred = data.copy().astype(np.float)
    for i, (x0, x1) in enumerate(zip(x[:-1], x[1:])):
        ind = (data >= x0)*(data < x1)
        if len(ind) > 0:
            data_blurred[ind] = np.hstack(data_fp[(2*i):(2*i+2)])
    return data_blurred


#!/usr/bin/env python3

import argparse
import matplotlib.pyplot as plt
import numpy as np
from BoundingBox import BoundingBox
from fibonacci_sphere import fibonacci_sphere_section

# parse command line arguments
parser = argparse.ArgumentParser(description = 'fibonacci sphere')
parser.add_argument('numpts', type=int, help='number of points/2 to distribute across spherical cap')
parser.add_argument('latmin', type=float, help='minimum latitude in deg')
parser.add_argument('latmax', type=float, help='maximum latitude in deg')
parser.add_argument('lonmin', type=float, help='minimum longitude in deg')
parser.add_argument('lonmax', type=float, help='maximum longitude in deg')
args = parser.parse_args()

# initialize bounding box
bbox = np.array([args.latmin, args.latmax, args.lonmin, args.lonmax])
bbox = np.deg2rad(bbox) # convert from degrees to radians
bbox = BoundingBox(bbox[0], bbox[1], bbox[2], bbox[3])

# fibonacci sphere
x, y, z = fibonacci_sphere_section(args.numpts, bbox)

# Display points in a scatter plot
fig = plt.figure()
ax = fig.add_subplot(111, projection='3d')
ax.set_title("min lat: {:.3f}°, max lat: {:.3f}°, min lon: {:.3f}°, min lon: {:.3f}°\n {} points; the whole sphere has {}".format(args.latmin, args.latmax, args.lonmin, args.lonmax, x.size, args.numpts))
ax.scatter(x, y, z, color="red")

# uncomment if you want to fix limits of the axis
#ax.set_xlim(-1, 1)
#ax.set_ylim(-1, 1)
#ax.set_zlim(-1, 1)

ax.set_box_aspect([1,1,1])

ax.set_xlabel('X')
ax.set_ylabel('Y')
ax.set_zlabel('Z')

plt.tight_layout()

plt.show()


import numpy as np

def estsimt(X1,X2):
    # from estsimt.m in MultiCameSelfCal

    # ESTimate SIMilarity Transformation
    #
    # [s,R,T] = estsimt(X1,X2)
    #
    # X1,X2 ... 3xN matrices with corresponding 3D points
    #
    # X2 = s*R*X1 + T
    # s ... scalar scale
    # R ... 3x3 rotation matrix
    # T ... 3x1 translation vector
    #
    # This is done according to the paper:
    # "Least-Squares Fitting of Two 3-D Point Sets"
    # by K.S. Arun, T. S. Huang and S. D. Blostein

    N = X1.shape[1]
    if N != X2.shape[1]:
        raise ValueError('both X1 and X2 must have same number of points')

    X1cent = np.mean(X1,axis=1)
    X2cent = np.mean(X2,axis=1)
    # normalize coordinate systems for both set of points
    x1 = X1 - X1cent[:,np.newaxis]
    x2 = X2 - X2cent[:,np.newaxis]

    # mutual distances
    d1 = x1[:,1:]-x1[:,:-1]
    d2 = x2[:,1:]-x2[:,:-1]
    ds1 = np.sqrt( np.sum( d1**2, axis=0) )
    ds2 = np.sqrt( np.sum( d2**2, axis=0) )

    scales = ds2/ds1
    s = np.median( scales )

    # undo scale
    x1s = s*x1

    # finding rotation
    H = np.zeros((3,3))
    for i in range(N):
        tmp1 = x1s[:,i,np.newaxis]
        tmp2 = x2[np.newaxis,:,i]
        tmp = np.dot(tmp1,tmp2)
        H += tmp

    U,S,Vt = np.linalg.svd(H)
    V = Vt.T
    X = np.dot(V,U.T)
    R=X

    T = X2cent - s*np.dot(R,X1cent)
    return s,R,T

def build_xform(s,R,t):
    T = np.zeros((4,4),dtype=np.float)
    T[:3,:3] = R
    T = s*T
    T[:3,3] = t
    T[3,3]=1.0
    return T

def align_points( s,R,T, X ):
    T = build_xform(s,R,T)
    if X.shape[0]==3:
        # make homogeneous
        Xnew = np.ndarray((4,X.shape[1]),dtype=X.dtype)
        Xnew[3,:].fill(1)
        Xnew[:3,:] = X
        X = Xnew
    X = np.dot(T,X)
    return X

def align_M( s,R,T, P ):
    T = build_xform(s,R,T)
    P = np.dot(P,np.linalg.inv(T))
    return P

def align_M2( M, P ):
    P = np.dot(P,np.linalg.inv(M))
    return P


import numpy as np
import scipy.interpolate as spip
"""path3d is a given file/library from my Uni (UCL) and is not Ralph Goffart Winkin's own work."""

def path_points(points, steps=None):
    """
    Calcule un chemin courbe à partir de points de passage.
    Les courbes sont des splines cubiques.

    Paramètres:
        points: array[3,N]
            N points de passage en 3 dimensions
        steps: int
            nombre de points à générer, par défaut 10 * N
    Retourne:
        XPath: array[3,steps]
            coordonnées des points sur le chemin en 3 dimensions
    """

    # paramètre: racine carrée de la corde accumulée:
    deltaTau = np.sqrt(np.sum(np.diff(points)**2, axis=0))
    tauPoints = np.hstack(((0), np.cumsum(deltaTau)))
    tauEnd = tauPoints[-1]

    # Interpoler avec des spline cubiques:
    spline = spip.splprep(points, u=tauPoints, s=0)[0]

    # Echantillonner à intervalles réguliers:
    if not steps: steps = 10*points.shape[1]
    tau = np.linspace(0, tauEnd, steps)
    XPath = np.array(spip.splev(tau, spline))

    return XPath

def path_vectors(XPath):
    """
    Calcule la distance curvilinéaire, le vecteur tangent et
    le vecteur de courbure le long du chemin.

    Paramètres:
        XPath: array[3,N]
            coordonnées de N points en 3 dimensions
    Retourne: (sPath, TPath, CPath)
        sPath: array[N]
            distance curvilinéaire des points
        TPath: array[3,N]
            vecteur tangent aux points (|T| = 1)
        CPath: array[3,N]
            vecteur de courbure aux points (|C| = courbure = 1/rayon)
    """
    ds = np.sqrt(np.sum(np.diff(XPath)**2, axis=0))
    s = np.hstack(((0), np.cumsum(ds)))

    dX_ds = np.gradient(XPath, s, axis=1, edge_order=2)
    d2X_ds = np.gradient(dX_ds, s, axis=1, edge_order=2)

    return s, dX_ds, d2X_ds

def path(points, steps=None):
    """
    Calcule les éléments d'un chemin courbe à partir de points de passage.

    Paramètres:
        points: array[3,N]
            N points de passage en 3 dimensions
        steps: int
            nombre de points à générer, par défaut 10 * N
    Retourne: (sPath, TPath, CPath)
        XPath: array[3,steps]
            coordonnées des points sur le chemin en 3 dimensions
        sPath: array[N]
            distance curvilinéaire des points
        TPath: array[3,N]
            vecteur tangent aux points (|T| = 1)
        CPath: array[3,N]
            vecteur de courbure aux points (|C| = courbure = 1/rayon)
    """
    XPath = path_points(points, steps)
    sPath, TPath, CPath = path_vectors(XPath)
    return sPath, XPath, TPath, CPath

def ainterp(x, xp, ap, **kwargs):
    """
    Interpolation en x sur plusieurs fonctions xp -> ap[k].

    Paramètres:
        x: float ou array[M]
            abscisses x_m où les fonctions sont évaluées
        xp: array[N]
            abscisses x_n des points des fonctions
        ap: array[K, N]
            ordonnées f_k(x_n) des points des fonctions

    Résultat:
        a: array[K] ou array[K, M]
            ordonnées évaluées f_k(x_m)
    """
    a = np.array([np.interp(x, xp, fp, **kwargs) for fp in ap])
    return a

def path_at(s, path, **kwargs):
    """
    Retourne les éléments en un point donné d'un chemin,
    par interpolation des données du chemin.

    Paramètres:
        s: float
            distance curviligne du point
        path: (sPath, XPath, TPath, CPath)
            éléments du chemin, tels que retournés par path().

    Retourne: X, T, C
        X: array[3]
            coordonnées du point
        T: array[3]
            vecteur tangent au point
        C: array[3]
            vecteur de courbure au point
    """
    sPath, XPath, TPath, CPath = path
    X = ainterp(s, sPath, XPath, **kwargs)
    T = ainterp(s, sPath, TPath, **kwargs)
    C = ainterp(s, sPath, CPath, **kwargs)
    return X, T, C

def tilt_vectors(T, beta=0.):
    """
    Retourne les vecteurs d'inclinaison le long d'un chemin.

    Paramètres:
        T: array[3]
            vecteur tangent au chemin
        beta: float, défaut=0.
            angle d'inclinaison par rapport à l'horizontale
            (> 0 = penche à droite dans le sens de T, en radians)

    Retourne: B, N
        B: array[3]
            vecteur unitaire normal parallèle à l'inclinaison
            (à gauche dans le sens de T)
        B: array[3]
            vecteur unitaire normal perpendiculaire à l'inclinaison
            (vers z > 0)
    """
    T /= np.sqrt(np.sum(T**2, axis=0)) # unit
    Z = np.array((0, 0, 1))

    D = np.cross(Z, T, axis=0)
    D /= np.sqrt(np.sum(D**2, axis=0)) # unit normale horizontale

    U = np.cross(T, D, axis=0)  # unit normale verticale

    B = D*np.cos(beta) + U*np.sin(beta) # unit normale parallèle
    N = np.cross(T, B, axis=0)  # unit normale perpendiculaire
    return B, N

def looping_points(steps=21):
    """
    Calcule les points de passage pour un chemin simple avec
    un looping.  Le chemin a pour équations:

        x = t*(t-1)**2 + 0.5*t
        y = 0.5*t*(t-1)**2
        z = 2*t**2*(t-0.9)**2

    pour -1 ≤ t ≤ 1

    Paramètres:
        steps: int (défaut 21)
            le nombre de points
    Retourne: array[3, steps]
        coordonnées des point de passage
    """
    tPoints = np.linspace(-1, 1, 21)
    Xpoints = np.vstack((
        0.5*tPoints + tPoints*(tPoints-1)*(tPoints+1),
        0.5*tPoints*(tPoints-1)*(tPoints+1),
        2*tPoints*tPoints*(tPoints-0.9)*(tPoints+0.9)
    ))
    return Xpoints

#### Code de test ####

if __name__ == '__main__':
    import matplotlib.pyplot as plt
    from mpl_toolkits.mplot3d import Axes3D

    #points de passage
    xyzPoints = looping_points()

    # chemin et vecteurs
    xyzPath = path_points(xyzPoints)
    sPath, TPath, CPath = path_vectors(xyzPath)

    # points jalons à afficher sur le graphique
    length = sPath[-1]
    sMarks = np.linspace(0, length, 30)
    xyzMarks = np.array([np.interp(sMarks, sPath, uPath) for uPath in xyzPath])
    TMarks = np.array([np.interp(sMarks, sPath, uPath) for uPath in TPath])
    CMarks = np.array([np.interp(sMarks, sPath, uPath) for uPath in CPath])

    # graphique 3D : chemin et vecteurs
    fig = plt.figure()
    ax = fig.add_subplot(projection='3d')
    ax.set_box_aspect(np.ptp(xyzPath, axis=1))
    ax.plot(xyzPoints[0],xyzPoints[1],xyzPoints[2],'bo', label='points')
    ax.plot(xyzPath[0],xyzPath[1],xyzPath[2],'k.', ms=0.5)
    ax.plot(xyzMarks[0],xyzMarks[1],xyzMarks[2],'r.', ms=2)
    scale = 0.1
    ax.quiver(xyzMarks[0],xyzMarks[1],xyzMarks[2],
              scale*TMarks[0],scale*TMarks[1],scale*TMarks[2],
              color='r', linewidth=0.5, label='T')
    ax.quiver(xyzMarks[0],xyzMarks[1],xyzMarks[2],
              scale*CMarks[0],scale*CMarks[1],scale*CMarks[2],
              color='g', linewidth=0.5, label='C')
    ax.legend()
    plt.show()

    # Vecteurs normaux
    #tilt = 2*xyzMarks[1]
    tilt = 0.
    BMarks, NMarks = tilt_vectors(TMarks, tilt)

    # graphique 3D : vecteurs normaux
    fig = plt.figure()
    ax = fig.add_subplot(projection='3d')
    ax.set_box_aspect(np.ptp(xyzPath, axis=1))
    ax.plot(xyzPath[0],xyzPath[1],xyzPath[2],'k.', ms=0.5)
    #ax.plot(xyzMarks[0],xyzMarks[1],xyzMarks[2],'r.', ms=2)
    scale = 0.1
    ax.quiver(xyzMarks[0],xyzMarks[1],xyzMarks[2],
              scale*TMarks[0],scale*TMarks[1],scale*TMarks[2],
              color='r', linewidth=0.5, label='T')
    ax.quiver(xyzMarks[0],xyzMarks[1],xyzMarks[2],
              scale*BMarks[0],scale*BMarks[1],scale*BMarks[2],
              color='b', linewidth=0.5, label='B')
    ax.quiver(xyzMarks[0],xyzMarks[1],xyzMarks[2],
              scale*NMarks[0],scale*NMarks[1],scale*NMarks[2],
              color='g', linewidth=0.5, label='N')
    plt.legend()
    plt.show()


import numpy as np

def multiply_matrix(A, B):
    if A.shape[1] != B.shape[0]:
        raise NameError('Not possible')

    C = np.zeros(shape=(A.shape[0], B.shape[1]))

    for i in range(C.shape[0]):
        for j in range(C.shape[1]):
            for k in range(B.shape[0]):
                C[i, j] += A[i, k] * B[k, j]

    return C

def sum_matrix(A, B):
    if A.shape != B.shape:
        raise NameError('Matrix of different dimension')

    lines = A.shape[0]
    columns = A.shape[1]

    C = np.zeros(shape=(lines, columns))

    for i in range(lines):
        for j in range(columns):
            C[i, j] = A[i, j]+B[i, j]

    return C

def generate_matrix_c(A):
    dimension = A.shape[0]

    C = np.zeros(shape=(dimension, dimension))

    for i in range(dimension):
        for j in range(dimension):
            C[i, j] = 0 if i == j else -A[i, j] / A[i, i]

    return C

def generate_matrix_g(A, b):
    dimension = A.shape[0]

    g = np.zeros(shape=(dimension, 1))

    for i in range(dimension):
        g[i, 0] = b[i, 0] / A[i, i]

    return g

A = np.matrix('10. 2. 3.; 1. 5. 1.; 2. 3. 10.')
b = np.matrix('7.; -8.; 6.')

x = np.matrix('0.; 0.; 0.')
previous_x = np.matrix('1.; 1.; 1.')

C = generate_matrix_c(A)
g = generate_matrix_g(A, b)

e = 0.001

while abs(x[0, 0] - previous_x[0, 0]) > e:
    previous_x = x
    x = sum_matrix(multiply_matrix(C, previous_x), g)

print x


import numpy as np

def distancePointPoint(P1,P2):
    """Calculate the distance between to points P1 and P2"""
    distVec=np.atleast_2d(P2)-np.atleast_2d(P1)
    dist=np.linalg.norm(distVec,axis=1)

    return dist, distVec

def distancePointLine(P,A,B,finiteLine=True):
    """Calculate the distance between a point P and a line defined by points
    A and B. If A and B are the starting and ending points of the line, i.e.,
    the line is finite, set finiteLine flag to True."""

    distAB, AB = distancePointPoint(A,B)
    PA = np.atleast_2d(P)-np.atleast_2d(A)
    PB = np.atleast_2d(P)-np.atleast_2d(B)

    distVec=np.cross(PA,PB)/(distAB)
    if finiteLine:
        clostestPointIsA=np.where(np.dot(AB,-PA.T) < 0)
        clostestPointIsB=np.where(np.dot(AB,PB.T) < 0)
        distVec[clostestPointIsA]=PA[clostestPointIsA]
        distVec[clostestPointIsB]=PA[clostestPointIsB]

    return np.linalg.norm(distVec,axis=1), distVec

def distanceLineLine(A1,B1,A2,B2,finiteLine=True):
    """Calculate the distance between two lines each defined by points A_i and
    B_i. If A_i and B_i are the starting and end points of the lines, i.e., the
    lines are finite, set finiteLine flag to True"""
    pass


import p2_data as data
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import time

# *****************************************************************
# ====================helper functions=========================
def sigmoid(x):
	'''
		The sigmoid function.
	'''
	return 1. / (1. + np.exp(-x))

def grad_logreg(X, y, W, reg = 0.0):
	'''
		Return the gradient of W for logistic regression.
	'''
	# YOUR CODE BELOW
	grad = X.T @ (sigmoid(X @ W) - y) + reg * W

	return grad

def newton_step(X, y, W, reg = 0.0):
	'''
	Return the change of W according to Newton's method.
	'''
	# YOUR CODE BELOW
	mu = sigmoid(X @ W)
	g = grad_logreg(X, y, W, reg = reg)
	diag = np.diag(np.squeeze(np.asarray(np.multiply(mu, 1. - mu))))
	H = X.T @ diag @ X + reg * np.eye(X.shape[1])
	d = np.linalg.solve(H, g)
	return d

def NLL(X, y, W, reg = 0.0):
	'''
		Calculate negative log likelihood.
	'''
	# YOUR CODE GOES BELOW
	mu = sigmoid(X @ W)
	temp = np.multiply(y, np.log(mu)) + np.multiply((1. - y), np.log(1. - mu))
	nll = -sum(temp) + reg / 2 * np.linalg.norm(W) ** 2
	return nll.item(0)

def grad_descent(X, y, reg = 0.0, lr = 1e-4, eps = 1e-6, \
	max_iter = 500, print_freq = 20):
	'''
		X is matrix with dimension m x (n + 1).
		y is label with dimension m x 1.
		reg is the parameter for regularization.
		lr is the learning rate.
		eps is the threshold of the norm for the gradients.
		max_iter is the maximum number of iterations.
		print_freq is the frequency of printing the report.

		Return the optimal weight by gradient descent and
		the corresponding learning objectives.
	'''
	m, n = X.shape
	nll_list = []
	# initialize the weight and its gradient
	W = np.zeros((n, 1))
	W_grad = np.ones_like(W)
	print('==> Running gradient descent...')
	iter_num = 0
	t_start = time.time()
	# Running the gradient descent algorithm
	# Update W
	# Calculate learning objectives
	# YOUR CODE GOES BELOW
	while iter_num < max_iter and np.linalg.norm(W_grad) > eps:
		# calculate NLL
		nll = NLL(X, y, W, reg = reg)
		if np.isnan(nll):
			break
		nll_list.append(nll)
		# calculate gradients and update W
		W_grad = grad_logreg(X, y, W, reg = reg)
		W -= lr * W_grad
		if (iter_num + 1) % print_freq == 0:
			print('-- Iteration {} - \
				negative log likelihood {: 4.4f}'.format(iter_num + 1, nll))
		iter_num += 1
	# benchmark
	t_end = time.time()
	print('-- Time elapsed for running gradient descent: {t:2.2f} \
		seconds'.format(t = t_end - t_start))

	return W, nll_list

def newton_method(X, y, reg = 0.0, eps = 1e-6, \
	max_iter = 20, print_freq = 5):
	'''
		X is matrix with dimension m x (n + 1).
		y is label with dimension m x 1.
		reg is the parameter for regularization.
		eps is the threshold of the norm for the gradients.
		max_iter is the maximum number of iterations.
		print_freq is the frequency of printing the report.

		Return the optimal weight by Netwon's method and the corresponding
		learning objectives.
	'''
	m, n = X.shape
	nll_list = []
	# initialize the weight and its gradient
	W = np.zeros((n, 1))
	step = np.ones_like(W)
	print('==> Running Newton\'s method...')
	iter_num = 0
	t_start = time.time()
	# Running the Newton's method
	# Update W
	# Calculate learning objectives
	# YOUR CODE GOES BELOW
	while iter_num < max_iter and np.linalg.norm(step) > eps:
		# calculate NLL
		nll = NLL(X, y, W, reg = reg)
		if np.isnan(nll):
			break
		nll_list.append(nll)
		# calculate gradients and update W
		step = newton_step(X, y, W, reg = reg)
		W -= step
		if (iter_num + 1) % print_freq == 0:
			print('-- Iteration {} - \
				negative log likelihood {: 4.4f}'.format(iter_num + 1, nll))
		iter_num += 1
	# benchmark
	t_end = time.time()
	print('-- Time elapsed for running Newton\'s method: {t:2.2f} \
		seconds'.format(t = t_end - t_start))

	return W, nll_list

def predict(X, W):
	'''
		Return the predicted labels.
	'''
	mu = sigmoid(X @ W)
	return (mu >= 0.5).astype(int)

def get_description(X, y, W):
	'''
		X is matrix with dimension m x (n + 1).
		y is label with dimension m x 1.
		W is the weight with dimension (n + 1) x 1.

		Return the accuracy, precision, recall and F-1 score of the prediction.
	'''
	# YOUR CODE GOES BELOW
	m, n = X.shape
	y_pred = predict(X, W)
	count_a, count_p, count_r = 0, 0, 0
	total_p, total_r = 0, 0
	for index in range(m):
		actual, pred = y.item(index), y_pred.item(index)
		if actual == pred:
			count_a += 1
		if actual == 1:
			total_r += 1
			if pred == 1:
				count_r += 1
		if pred == 1:
			total_p += 1
			if actual == 1:
				count_p += 1
	accuracy = 1. * count_a / m
	precision = 1. * count_p / total_p
	recall = 1. * count_r / total_r
	f1 = 2. * precision * recall / (precision + recall)
	return accuracy, precision, recall, f1

def plot_description(X_train, y_train, X_test, y_test):
	'''
		X is matrix with dimension m x (n + 1).
		y is label with dimension m x 1.

		Plot accuracy/precision/recall/F-1 score versus lambda.
		Return the lambda that maximizes accuracy.
	'''
	# YOUR CODE GOES BELOW
	reg_list = [0., 0.01, 0.1, 0.5, 1.0, 2.0, 5.0, 10.0, 20.0, 30.0]
	reg_list.sort()
	a_list = []
	p_list = []
	r_list = []
	f1_list = []
	# Run Newton's method or gradient descent
	for index in range(len(reg_list)):
		reg = reg_list[index]
		W_opt, obj = grad_descent(X_train, y_train, reg = reg, \
			lr = 2e-4, print_freq = 100)
		accuracy, precision, recall, f1 = get_description(X_test, y_test, W_opt)
		a_list.append(accuracy)
		p_list.append(precision)
		r_list.append(recall)
		f1_list.append(f1)

	# Generate plots
	a_vs_lambda_plot, = plt.plot(reg_list, a_list)
	plt.setp(a_vs_lambda_plot, color = 'red')
	p_vs_lambda_plot, = plt.plot(reg_list, p_list)
	plt.setp(p_vs_lambda_plot, color = 'green')
	r_vs_lambda_plot, = plt.plot(reg_list, r_list)
	plt.setp(r_vs_lambda_plot, color = 'blue')
	f1_vs_lambda_plot, = plt.plot(reg_list, f1_list)
	plt.setp(f1_vs_lambda_plot, color = 'yellow')
	plt.legend((a_vs_lambda_plot, p_vs_lambda_plot, r_vs_lambda_plot, \
		f1_vs_lambda_plot), ('accuracy', 'precision', 'recall', 'F-1'),\
		 loc = 'best')
	plt.title('Testing descriptions')
	plt.xlabel('regularization parameter')
	plt.ylabel('Metric')
	plt.savefig('DESCRIPTION_2a.png', format = 'png')
	plt.close()
	print('==> Plotting completed.')

	# Find the param that maximizes accuracy
	opt_reg_index = np.argmax(a_list)
	reg_opt = reg_list[opt_reg_index]
	return reg_opt

# *****************************************************************
# ====================main driver function=========================
if __name__ == '__main__':
	# STEP 0: LOADING DATA
	df_train = data.df_train
	df_test = data.df_test
	X_train = data.X_train
	y_train = data.y_train
	X_test = data.X_test
	y_test = data.y_test

	# STEP 1: Logistic regression
	print('==> Step 1: Running logistic regression...')
	# splitting data for logistic regression
	df_train_logreg = df_train[df_train.label <= 1]
	X_train_logreg = np.array(df_train_logreg[:][[col for \
		col in df_train_logreg.columns if col != 'label']]) / 256.
	y_train_logreg = np.array(df_train_logreg[:][['label']])
	df_test_logreg = df_test[df_test.label <= 1]
	X_test_logreg = np.array(df_test_logreg[:][[col for \
		col in df_test_logreg.columns if col != 'label']]) / 256.
	y_test_logreg = np.array(df_test_logreg[:][['label']])
	# stacking a column of 1's
	X_train_logreg = np.hstack((np.ones_like(y_train_logreg), X_train_logreg))
	X_test_logreg = np.hstack((np.ones_like(y_test_logreg), X_test_logreg))
	# STEP 1a: Gradient descent
	print('==> Step 1a: Running gradient descent...')
	# Fill in the code in NLL and grad_descent
	W_gd, nll_list_gd = grad_descent(X_train_logreg, y_train_logreg, reg = 1e-6)
	# STEP 1b: Newton's method
	# Fill in the code in newton_step and newton_method
	print('==> Step 1b: Running Newton\'s method...')
	W_newton, nll_list_newton = newton_method(X_train_logreg, y_train_logreg, \
		reg = 1e-6)
	# STEP 2: Generate convergence plot
	print('==> Plotting convergence plot...')
	plt.style.use('ggplot')
	nll_gd_plot, = plt.plot(range(len(nll_list_gd)), nll_list_gd)
	plt.setp(nll_gd_plot, color = 'red')
	nll_newton_plot, = plt.plot(range(len(nll_list_newton)), nll_list_newton)
	plt.setp(nll_newton_plot, color = 'green')
	plt.legend((nll_gd_plot, nll_newton_plot), \
		('Gradient descent', 'Newton\'s method'), loc = 'best')
	plt.title('Convergence Plot on Binary MNIST Classification')
	plt.xlabel('Iteration')
	plt.ylabel('NLL')
	plt.savefig('CONVERGENCE_2a.png', format = 'png')
	plt.close()
	print('==> Plotting completed.')
	# STEP 3: Generate accuracy/precision plot=
	print('Step 3: ==> Generating plots for accuracy, precision, recall, and F-1 score...')
	reg_opt = plot_description(X_train_logreg, y_train_logreg, \
		X_test_logreg, y_test_logreg)
	print('==> Optimal regularization parameter is {:4.4f}'.format(reg_opt))


import numpy

def cgSolver(A, b, lam, Tol=1e-16, MaxIter=1000):
    '''
    Solve (A^T * A + lam * I) * w = b.
    '''
    d = A.shape[1]
    b = b.reshape(d, 1)
    tol = Tol * numpy.linalg.norm(b)
    w = numpy.zeros((d, 1))
    r = b - lam * w - numpy.dot(A.T, numpy.dot(A, w))
    p = r
    rsold = numpy.sum(r ** 2)

    for i in range(MaxIter):
        Ap = lam * p + numpy.dot(A.T, numpy.dot(A, p))
        alpha = rsold / numpy.dot(p.T, Ap)
        w += alpha * p
        r -= alpha * Ap
        rsnew = numpy.sum(r ** 2)
        if numpy.sqrt(rsnew) < tol:
            print('Converged! res = ' + str(rsnew))
            break
        p = r + (rsnew / rsold) * p
        rsold = rsnew

    #print('res = ' + str(rsnew) + ',   iter = ' + str(i))
    #if i == MaxIter-1:
        #print('Warn: CG does not converge! Res = '  + str(rsnew))
    return w

def cgSolver2(AA, b, lam, Tol=1e-16, MaxIter=1000):
    '''
    Solve (A^T * A + lam * I) * w = b.
    '''
    d = AA.shape[1]
    b = b.reshape(d, 1)
    tol = Tol * numpy.linalg.norm(b)
    w = numpy.zeros((d, 1))
    r = b - lam * w - numpy.dot(AA, w)
    p = r
    rsold = numpy.sum(r ** 2)

    for i in range(MaxIter):
        Ap = lam * p + numpy.dot(AA, p)
        alpha = rsold / numpy.dot(p.T, Ap)
        w += alpha * p
        r -= alpha * Ap
        rsnew = numpy.sum(r ** 2)
        if numpy.sqrt(rsnew) < tol:
            print('Converged! res = ' + str(rsnew))
            break
        p = r + (rsnew / rsold) * p
        rsold = rsnew

    #if i == MaxIter-1:
    #    print('Warn: CG does not converge! Res = '  + str(rsnew))
    return w

def svrgSolver(A, b, lam, alpha=0.01, Tol=1e-16, MaxIter=1000, BatchSize=100):
    '''
    Solve (A^T * A + lam * I) * w = b.
    '''
    s, d = A.shape
    b = b.reshape(d, 1)

    # parameter
    Scaling = s / BatchSize
    NumInnerLoop = int(numpy.ceil(Scaling))

    # initialize
    w = numpy.zeros((d, 1))

    for q in range(MaxIter):
        wtilde = numpy.copy(w)

        # compute full gradient at wtilde
        aw = numpy.dot(A, wtilde)
        gradFull = numpy.dot(A.T, aw) - b + lam * wtilde

        # mini-batch stochastic gradient
        for j in range(NumInnerLoop):
            idx = numpy.random.choice(s, BatchSize)
            Arand = A[idx, :]

            # the stochastic gradient at w
            aw = numpy.dot(Arand, w)
            grad1 = numpy.dot(Arand.T, aw) * Scaling - b + lam * w

            # the stochastic gradient at wtilde
            aw = numpy.dot(Arand, wtilde)
            grad2 = numpy.dot(Arand.T, aw) * Scaling - b + lam * wtilde

            gradRand = grad1 - grad2 + gradFull
            w -= alpha * gradRand

    return w