import numpy as np
from numpy.core.umath import sin, cos, e, absolute, sqrt, pi, power

def fun1(x, y):
    return sin(x) * cos(y)

def fun2(x, y):
    return (e**(cos(y) * sin(x) * 2)  +  x + y + sin(x) * 5)**3

def rosenbrock(x, y):
    Z = (power((1 - x), 2) + 100 * power((y - x * x), 2))
    return Z

def rastrigin(x,y):
    return 20 + (x * x - 10 * cos(2 * np.pi * x)) + (y * y - 10 * cos(2 * np.pi * y))

def himmelblau(x,y):
    return (x * x + y -11)**2 + (x + y * y - 7)**2

def htable(x,y):
    return - absolute(sin(x) * cos(x) * power(e, (absolute(1 - (sqrt(x * x + y * y)/pi)))))


from SequenceType.Base import SequenceType
from scipy.special import comb
from scipy.optimize import fsolve
from sympy import factorial

class CatalanNumberSequence(SequenceType):

    def __init__(self, seq):
        super(CatalanNumberSequence, self).__init__(seq)
        self.n = 0  # nth catalan number.  Number in the 0,1,2,3,etc catalan sequence

    def __bool__(self):
        """ Check whether sequence is Catalan Numbers """

        # solve for the catalan term position of the first number
        # arbitrarily start with initial guess of 5
        # todo think about this because it won't work with much larger numbers
        self.n = round(fsolve(lambda x: (comb(2 * x, x) * 1 / (x + 1)) - self.seq.ls[0], 5)[0])

        if self.n == 1 and self.seq.ls[1] == 1:
            self.n = 0

        for i in range(1, self.seq.size):
            if self.seq.ls[i] != comb(2 * (self.n + i), self.n + i) * 1 / (self.n + i + 1):
                return False

        return True

    @staticmethod
    def nth_catalan_number(n):
        """ Get the nth term catalan number """
        return comb(2 * n, n) * 1 / (n + 1)

    def term_number(self, index):
        """ The ith number in the sequence """
        n = self.n + index - 1

        return CatalanNumberSequence.nth_catalan_number(n)

    def seq_str(self, n):
        n_repr = self.n - 1
        expression = factorial(2 * (n + n_repr)) / (factorial(n + n_repr + 1) * factorial(n + n_repr))
        return expression


# coding: utf-8

# In[1]:

import numpy as np
import numpy.linalg as la
import matplotlib.pyplot as plt
import pandas as pd

# In[5]:

ks=pd.read_csv('kstable.csv', sep=',',header=[0]) #Soultion to the Kuramoto-Sivashinsky equation

# In[10]:

plt.imshow(ks, cmap='Set1', interpolation='none') #cmap: magma, inferno, Pastel1...
                                                  #https://matplotlib.org/examples/color/colormaps_reference.html
plt.show()

# In[11]:

U, sigma, V = np.linalg.svd(ks);

# In[13]:

reconstimg = np.matrix(U[:, :1]) * np.diag(sigma[:1]) * np.matrix(V[:1, :])
plt.imshow(reconstimg, cmap='Set1', interpolation='none');
plt.show()

# In[26]:

#Even with just the second and third vectors, the shape of the stiffness mtx begins to appear.
nmax=3;
for i in range(1,nmax):
    reconstimg = np.matrix(U[:, :i]) * np.diag(sigma[:i]) * np.matrix(V[:i, :])
    plt.imshow(reconstimg, cmap='Set1', interpolation='none')
    title = "n = %s" % i
    plt.title(title)
    plt.show()

# In[62]:

fig=plt.figure()
fig.subplots_adjust(wspace=2, hspace=0.6)

plt.subplot(2,2,1)
plt.imshow(np.matrix(U[:, :1]) * np.diag(sigma[:1]) * np.matrix(V[:1, :]), cmap='Set1', interpolation='none')
plt.title('n=1, \ncompressed representation')
plt.subplot(2,2,2)
plt.imshow(np.matrix(U[:, :2]) * np.diag(sigma[:2]) * np.matrix(V[:2, :]), cmap='Set1', interpolation='none')
plt.title('n=2')
plt.subplot(2,2,3)
plt.imshow(np.matrix(U[:, :3]) * np.diag(sigma[:3]) * np.matrix(V[:3, :]), cmap='Set1', interpolation='none')
plt.title('n=3')
plt.subplot(2,2,4)
plt.imshow(ks, cmap='Set1', interpolation='none')
plt.title('full representation')
plt.grid(True)
fig.suptitle('Singular Value Decomposition of Kuramoto-Sivashinsky chaotic PDE', size=14, y=1.08) # or plt.suptitle('Main title')
plt.show()
fig.savefig("ks_SVD.pdf", bbox_inches='tight')
fig.savefig("ks_SVD.png", bbox_inches='tight')

# In[ ]:


from Exercise5 import polynomial
from Exercise5 import splines
import numpy as np
import matplotlib.pyplot as plt
from math import sqrt
from Exercise5 import leastSquares

def plot_two_functions(x_values, y_values, z_values, plot_text):
    plt.figure(figsize=(10, 10))
    plt.grid(True, which='both')
    plt.xlim([min(x_values) - 0.2, max(x_values) + 0.2])
    plt.ylim([-1.1, 1.1])
    plt.xlabel("x", fontsize="xx-large")
    plt.ylabel("f(x)", fontsize="xx-large")
    plt.title("Approximation of " + r'$sinx$' + plot_text, fontsize="xx-large")
    plt.axhline(y=0, color='k')
    plt.axvline(x=0, color='k')
    plt.legend(["f(x)"], loc='upper left', fontsize="xx-large")
    plt.plot(x_values, y_values)
    plt.plot(x_values, z_values)
    plt.show()

def main():
    step = 0.0315

    # Polynomial Interpolation
    x_values = np.arange(-np.pi, np.pi, step)
    print("len x newton = " + str(len(x_values)))
    y_values = []
    for value in x_values:
        y_values.append(polynomial.custom_sin(value))
    real_sin_values = np.sin(x_values)
    plot_two_functions(x_values, y_values, real_sin_values, " using Newton method")
    print("Newton Root Mean Squared Error of prediction = " + str(sqrt(sum((real_sin_values - y_values) ** 2) /
                                                                       len(x_values))))
    # Splines Interpolation
    x_values = np.arange(-np.pi, np.pi, step)
    print("len x splines = " + str(len(x_values)))
    y_values = []
    for value in x_values:
        y_values.append(splines.custom_sin(value))
    real_sin_values = np.sin(x_values)
    plot_two_functions(x_values, y_values, real_sin_values, " using natural cubic Splines")
    print(
        "Splines Root Mean Squared Error of prediction = " + str(sqrt(sum((real_sin_values - y_values) ** 2) /
                                                                      len(x_values))))
    # Least Squares Interpolation
    degree = 10
    x_values = np.arange(-np.pi, np.pi, step)
    print("len x least squares = " + str(len(x_values)))
    y_values = []
    for value in x_values:
        y_values.append(leastSquares.custom_sin(value, degree))
    real_sin_values = np.sin(x_values)
    plot_two_functions(x_values, y_values, real_sin_values, " using Least Squares with " + str(degree) +
                       " degree polynomial")
    rmse = sqrt(sum((real_sin_values - y_values) ** 2) / len(x_values))
    print(
        "Least Squares using " + str(degree) + " degree polynomial model Root Mean Squared Error of prediction = " +
        str(rmse))

    # Comparison on [-π,π]

    # Newton Method
    x_values = np.arange(-np.pi, np.pi, step)
    y_values = []
    for value in x_values:
        y_values.append(polynomial.custom_sin(value))
    real_sin_values = np.sin(x_values)
    error = abs(real_sin_values - y_values)
    for value in error:
        if value > 0.5 * (10 ** (-3)):
            print("newton false assumption of correct digits")
            print(value)
            break

    # error figure for newton method
    plt.figure(figsize=(10, 10))
    plt.grid(True, which='both')
    plt.xlim([min(x_values) - 0.1, max(x_values) + 0.1])
    plt.ylim([-0.00001, 0.0001])
    plt.xlabel("x", fontsize="xx-large")
    plt.ylabel("Error", fontsize="xx-large")
    plt.title("Error of Newton method for 200 points on [-π,π]", fontsize="xx-large")
    plt.axhline(y=0, color='k')
    plt.axvline(x=0, color='k')
    plt.legend(["Error Function"], loc='upper left', fontsize="xx-large")
    plt.plot(x_values, error)
    plt.show()

    print("newton max error = " + str(max(error)))
    print("newton min error = " + str(min(error)))
    for i in range(len(error)):
        error[i] *= 100000
    hist, bin_edges = np.histogram(error)
    bin_edges = np.round(bin_edges, 0)
    plt.style.use('ggplot')
    plt.figure(figsize=[11, 11])
    plt.bar(bin_edges[:-1], hist, width=0.8, color='#2b2b2b', alpha=0.9)
    plt.xlim(min(bin_edges - 1), max(bin_edges + 1))
    plt.grid(axis='y', alpha=0.75)
    plt.xlabel('Error multiplied by 100000', fontsize='xx-large')
    plt.ylabel('Frequency', fontsize='xx-large')
    plt.xticks(fontsize='xx-large')
    plt.yticks(fontsize='xx-large')
    plt.ylabel('Frequency', fontsize='xx-large')
    plt.title("Histogram for error using Newton method", fontsize='xx-large')
    plt.show()

    # Splines method
    x_values = np.arange(-np.pi, np.pi, step)
    y_values = []
    for value in x_values:
        y_values.append(splines.custom_sin(value))
    real_sin_values = np.sin(x_values)
    error = abs(real_sin_values - y_values)
    for value in error:
        if value > 0.5 * (10 ** (-2)):
            print("splines false assumption of correct digits")
            print(value)
            break
    # error figure for splines method
    plt.figure(figsize=(10, 10))
    plt.grid(True, which='both')
    plt.xlim([min(x_values) - 0.1, max(x_values) + 0.1])
    plt.ylim([-0.0001, 0.0045])
    plt.xlabel("x", fontsize="xx-large")
    plt.ylabel("Error", fontsize="xx-large")
    plt.title("Error of Splines method for 200 points on [-π,π]", fontsize="xx-large")
    plt.axhline(y=0, color='k')
    plt.axvline(x=0, color='k')
    plt.legend(["Error Function"], loc='upper left', fontsize="xx-large")
    plt.plot(x_values, error)
    plt.show()
    print("splines max error = " + str(max(error)))
    print("splines min error = " + str(min(error)))
    multiplier = 1000
    for i in range(len(error)):
        error[i] *= multiplier
    hist, bin_edges = np.histogram(error)
    bin_edges = np.round(bin_edges, 0)
    plt.style.use('ggplot')
    plt.figure(figsize=[11, 11])
    plt.bar(bin_edges[:-1], hist, width=0.8, color='#2b2b2b', alpha=0.9)
    plt.xlim(min(bin_edges - 1), max(bin_edges + 1))
    plt.grid(axis='y', alpha=0.75)
    plt.xlabel('Error multiplied by ' + str(multiplier), fontsize='xx-large')
    plt.ylabel('Frequency', fontsize='xx-large')
    plt.xticks(fontsize='xx-large')
    plt.yticks(fontsize='xx-large')
    plt.ylabel('Frequency', fontsize='xx-large')
    plt.title("Histogram for error using natural cubic Splines", fontsize='xx-large')
    plt.show()

    # Least Squares
    x_values = np.arange(-np.pi, np.pi, step)
    y_values = []
    for value in x_values:
        y_values.append(leastSquares.custom_sin(value))
    real_sin_values = np.sin(x_values)
    error = abs(real_sin_values - y_values)
    for value in error:
        if value > 0.5 * (10 ** (-3)):
            print("least squares false assumption of correct digits")
            print(value)
            break
    # error figure for least squares method
    plt.figure(figsize=(10, 10))
    plt.grid(True, which='both')
    plt.xlim([min(x_values) - 0.1, max(x_values) + 0.1])
    plt.ylim([-0.00001, 0.0001])
    plt.xlabel("x", fontsize="xx-large")
    plt.ylabel("Error", fontsize="xx-large")
    plt.title("Error of Least Squares method for 200 points on [-π,π]", fontsize="xx-large")
    plt.axhline(y=0, color='k')
    plt.axvline(x=0, color='k')
    plt.legend(["Error Function"], loc='upper left', fontsize="xx-large")
    plt.plot(x_values, error)
    plt.show()
    print("least squares max error = " + str(max(error)))
    print("least squares min error = " + str(min(error)))
    multiplier = 100000
    for i in range(len(error)):
        error[i] *= multiplier
    hist, bin_edges = np.histogram(error)
    bin_edges = np.round(bin_edges, 0)
    plt.style.use('ggplot')
    plt.figure(figsize=[11, 11])
    plt.bar(bin_edges[:-1], hist, width=0.8, color='#2b2b2b', alpha=0.9)
    plt.xlim(min(bin_edges - 1), max(bin_edges + 1))
    plt.grid(axis='y', alpha=0.75)
    plt.xlabel('Error multiplied by ' + str(multiplier), fontsize='xx-large')
    plt.ylabel('Frequency', fontsize='xx-large')
    plt.xticks(fontsize='xx-large')
    plt.yticks(fontsize='xx-large')
    plt.ylabel('Frequency', fontsize='xx-large')
    plt.title("Histogram for error using least squares with 10th degree polynomial model", fontsize='xx-large')
    plt.show()

main()


import numpy as np
import math
from random import shuffle
from matplotlib import pyplot as plt
import matplotlib.pyplot as plt
import matplotlib.pyplot as pause
from mpl_toolkits.mplot3d import Axes3D
from time import sleep
import matplotlib.animation as animation
from matplotlib import cm
import sys

# DEFAULT PARAMETERS
threshold = 0.0000000000001

def plotPoints(plotOutFunction, fy=[]):
	x = []
	y = []

	for i in range(len(xyPoints)):
		x.append(xyPoints[i][0])
		y.append(xyPoints[i][len(xyPoints[0])-1])

	plt.title("XY points and predicted function")
	plt.scatter(x,y,color='blue',label='Actual Data')
	if (plotOutFunction == 1):
		plt.plot(x,fy,color='red',label='Curve Fitted')

	plt.xlabel('Input Data')
	plt.ylabel('Output Data')
	plt.legend()
	plt.show()

def plot3D(xyPoints,plotThetaX, plotThetaY, plotJo, timeGap):
	theta0, theta1 = np.mgrid[0:2:50j, -1:1:50j]
	mesh = np.c_[theta0.flatten(), theta1.flatten()]

	JVal = (np.array([calculateError(xyPoints,point) for point in mesh]).reshape(50,50))

	plt.ion()
	fig = plt.figure()
	ax = fig.add_subplot(111, projection='3d')

	ax.plot_surface(theta0, theta1, JVal, cmap=cm.RdBu_r)
	ax.set_xlabel(r'$\theta_0$', labelpad=10)
	ax.set_ylabel(r'$\theta_1$', labelpad=10)
	ax.set_zlabel(r'$J(\theta)$', labelpad=10)

	plt.show()

	for i in range(len(plotJo)):
		if i % 100 == 0:
			ax.plot([plotThetaX[i]], [plotThetaY[i]], [plotJo[i]], linestyle='-',color='r', marker='o', markersize=3)
			plt.pause(timeGap)

	plt.savefig("Assign1Q1-3DPlot.png")

def plotContours(xyPoints, plotThetaX, plotThetaY, plotJo,title, doMod100 = True):
	theta0, theta1 = np.mgrid[-2:2:50j, -1:2:50j]
	mesh = np.c_[theta0.flatten(), theta1.flatten()]

	JVal = (np.array([calculateError(xyPoints,point) for point in mesh]).reshape(theta0.shape))

	plt.ion()
	plt.title(title)
	plt.contour(theta0, theta1, JVal, 25, colors="k")
	plt.xlabel(r'$\theta_0$', labelpad=10)
	plt.ylabel(r'$\theta_1$', labelpad=10)

	plt.show()

	for i in range(len(plotJo)):
		if doMod100 and i % 100 == 0:
			plt.plot([plotThetaX[i]], [plotThetaY[i]], linestyle='-',color='r', marker='o', markersize=3)
			plt.pause(0.02)
		elif doMod100 == False:
			plt.plot([plotThetaX[i]], [plotThetaY[i]], linestyle='-',color='r', marker='o', markersize=3)
			plt.pause(0.02)

	plt.savefig("Assign1Q1-Contours.png")
	plt.close()

def readPoints(fileX, fileY):
	noOfPoints = 0
	xyPoints = []

	f1 = open(fileX, "r")
	f2 = open(fileY, "r")

	xPoint = f1.read().splitlines()
	yPoint = f2.read().splitlines()

	for i in range(len(xPoint)):
		xy1Point = []
		xy1Point.append(float(xPoint[i]))
		xy1Point.append(1.0)			# This is the value corresponding to theta0 (since theta0 is the intercept)
		xy1Point.append(float(yPoint[i]))

		xyPoints.append(xy1Point)
		noOfPoints = noOfPoints + 1

	f1.close()
	f2.close()

	return xyPoints, noOfPoints

def normalize(xyPoints):
	mean = []
	var = []

	for j in range(len(xyPoints[0])):
		if j == len(xyPoints[0])-2:
			break

		x = []
		for i in range(len(xyPoints)):
			x.append(xyPoints[i][j])

		mean.append(np.mean(x))
		var.append(np.std(x))

		for i in range(len(xyPoints)):
			xyPoints[i][j] = (xyPoints[i][j] - mean[j]) / var[j]

	return xyPoints, mean, var

def unormalize(xyPoints, mean, var, fx=[], isFxAvailable=False):
	for i in range(len(xyPoints[0])):
		if (i == len(xyPoints[0])-2):
			break

		for j in range(len(xyPoints)):
			xyPoints[j][i] = (xyPoints[j][i] * var[i]) + mean[i]

			if isFxAvailable == True:
				fx[j] = (fx[j] * var[i]) + mean[i]

	return xyPoints, fx

def getNewFunction(xyPoints, thetaVec):
	fy = []

	for k in range(len(xyPoints)):
		fy1 = 0.0
		for i in range(len(thetaVec)):
			fy1 = fy1 + thetaVec[i] * xyPoints[k][i]

		fy.append(fy1)

	return fy

def calculateError(xyPoints, thetaVec):
	diffSq = 0.0
	for i in range(len(xyPoints)):
		htheta = 0.0
		for j in range(len(xyPoints[i])-1):
			htheta = htheta + thetaVec[j] * xyPoints[i][j]

		diff = htheta - xyPoints[i][len(xyPoints[0])-1]
		diffSq = diffSq + diff * diff

	Jo = diffSq / float(2 * len(xyPoints))
	return Jo

def derivateOfError(thetaVec, xyPoints, featureNo):
	derivJo = 0.0
	for i in range(len(xyPoints)):
		htheta = 0.0
		for j in range(len(xyPoints[i])-1):
			htheta = htheta + thetaVec[j] * xyPoints[i][j]

		derivJo = derivJo + (htheta - xyPoints[i][len(xyPoints[0])-1]) * xyPoints[i][featureNo]

	derivJo = derivJo / float(len(xyPoints))
	return derivJo

def updateTheta(xyPoints, thetaVec, alpha):
	tempThetaVec = [0] * (len(xyPoints[0]) - 1)

	# First loop is just to accumulate changed theta. DONOT modify thetaVec in this loop
	for i in range(len(thetaVec)):
		tempThetaVec[i] = thetaVec[i] - alpha * derivateOfError(thetaVec, xyPoints, i)

	# Once all newTheta is calculated, update the thetaVec in this loop
	for i in range(len(thetaVec)):
		thetaVec[i] = tempThetaVec[i]

	return thetaVec

def gradientDescent(xyPoints, noOfPoints, alpha):
	JPvs = 0
	thetaVec = [0] * (len(xyPoints[0]) - 1)		# Last col is y so we will subtract that col in thetaVec and last theta is O0 which we will add a column in theta Vec
	mean = [0] * len(xyPoints[0])
	var = [0] * len(xyPoints[0])

	plotThetaX = []
	plotThetaY = []
	plotJo = []

	while 1:
		thetaVec = updateTheta(xyPoints, thetaVec, alpha)
		JCur = calculateError(xyPoints, thetaVec)

		if JCur == float("inf"):
			break

		if JCur < 1000:
			plotThetaX.append(thetaVec[0])
			plotThetaY.append(thetaVec[1])
			plotJo.append(JCur)

		if abs(JCur-JPvs) <= threshold:
			break							# Converged

		JPvs = JCur

	return thetaVec, plotThetaX, plotThetaY, plotJo

fileX = sys.argv[1] # prints python_script.py
fileY = sys.argv[2] # prints var1
learningRate = float(sys.argv[3]) # prints var2
timeGap = float(sys.argv[4]) # prints var2

xyPoints,noOfPoints = readPoints(fileX, fileY)
xyPoints, mean, var = normalize(xyPoints)

############ Part (a)
alpha = learningRate
thetaVec, plotThetaX, plotThetaY, plotJo = gradientDescent(xyPoints, noOfPoints, alpha)

print ("Learning Rate = ", alpha)
print ("Stopping Criteria = J0 - JPvs <= ", threshold)
print ("Theta Vector = ", thetaVec)

############# Part (b)
fx = getNewFunction(xyPoints, thetaVec)
plotPoints(True, fx)

############# Part (c)
plot3D(xyPoints,plotThetaX, plotThetaY, plotJo, timeGap)

############# Part (d)
plotContours(xyPoints,plotThetaX, plotThetaY, plotJo, "Contours for learning rate = " + str(alpha))

############# Part (e)
alphaL = [0.001, 0.002, 0.003, 0.02, 2.1]
for i in range(len(alphaL)):
	alpha = alphaL[i]
	thetaVec, plotThetaX, plotThetaY, plotJo = gradientDescent(xyPoints, noOfPoints, alpha)
	if i == 4:
		plotContours(xyPoints,plotThetaX, plotThetaY, plotJo, "Contours for learning rate = " + str(alpha),False)
	else:
		plotContours(xyPoints,plotThetaX, plotThetaY, plotJo, "Contours for learning rate = " + str(alpha))


from IMLearn.learners import UnivariateGaussian, MultivariateGaussian, gaussian_estimators
import numpy as np
import plotly.graph_objects as go
import plotly.io as pio
pio.templates.default = "simple_white"

def test_univariate_gaussian():
    # Question 1 - Draw samples and print fitted model
    n = 1000
    mu = 10
    var = 1
    X = np.random.normal(mu, var, size=n)
    estimator = gaussian_estimators.UnivariateGaussian().fit(X)
    print("(", estimator.mu_, estimator.var_, ")")

    # Question 2 - Empirically showing sample mean is consistent
    ms = np.linspace(10, 1000, 100).astype(np.int_)
    estimated_mean = []
    for m in ms:
        _X = X[:m]
        estimated_mean.append(abs(gaussian_estimators.UnivariateGaussian().fit(_X).mu_ - mu))  # emprical error
    go.Figure([go.Scatter(x=ms, y=estimated_mean, mode='markers+lines', name=r'$\widehat\mu$')],
              layout=go.Layout(
                  title=r"$\text{(3.1.2) Absolute distance between the estimated and empirical expectation against sample size}$",
                  xaxis_title="$\\text{number of samples}$",
                  yaxis_title="r$|\hat\mu - \mu|$",
                  height=300)).show()

    # Question 3 - Plotting Empirical PDF of fitted model
    go.Figure([go.Scatter(x=X, y=estimator.pdf(X), mode='markers', name='$\\text{estimated PDF}$'),
               go.Scatter(x=X, y=np.norm.pdf(X, mu, var), mode='markers', name='$\\text{empirical PDF}$')],
              layout=go.Layout(title="$\\text{(3.1.3) Estimated vs Empirical PDF }$",
                               xaxis_title="$\\text{ sample values}$",
                               yaxis_title="$\\text{PDF}$",
                               height=300)).show()

def test_multivariate_gaussian():
    # Question 4 - Draw samples and print fitted model
    mu_mult = [0, 0, 4, 0]
    sigma_mult = \
        [[1, 0.2, 0, 0.5],
         [0.2, 2, 0, 0],
         [0, 0, 1, 0],
         [0.5, 0, 0, 1]]
    X_mult = np.random.multivariate_normal(mu_mult, sigma_mult, n)
    estimator_mult = gaussian_estimators.MultivariateGaussian().fit(X_mult)
    print(estimator_mult.mu_)
    print(estimator_mult.cov_)

    # Question 5 - Likelihood evaluation
    f1 = np.linspace(-10, 10, 200)
    f3 = np.linspace(-10, 10, 200)
    L = [estimator_mult.log_likelihood(np.array([i, 0, j, 0]), sigma_mult, X_mult) for i in f1 for j in f3]
    L = np.array(L).reshape((200, 200))
    fig = go.Figure(data=go.Heatmap(z=L.tolist(), x=f1, y=f3))
    fig.show()

    # Question 6 - Maximum likelihood
    raise NotImplementedError()

if __name__ == '__main__':
    np.random.seed(0)
    test_univariate_gaussian()
    test_multivariate_gaussian()


import plotly.figure_factory as pff
from sklearn.datasets import load_boston
import pandas as pd
import numpy as np

# Loading the dataset
boston = load_boston()
X = pd.DataFrame(boston.data, columns=boston.feature_names)  # Feature Matrix
Y = pd.DataFrame(boston.target, columns=["MEDV"])  # Target Variable

# Filtering: Pearson correlation: https://en.wikipedia.org/wiki/Pearson_correlation_coefficient

Data = pd.concat([X, Y], axis=1)
CoRel = Data.corr()

# Plotting the heat map of Pearson Correlation for all features in relation to each other
# fig_1 = pff.create_annotated_heatmap(np.array(CoRel).round(decimals=2), colorscale='RdBu', x=list(Data.columns.values), y=list(Data.columns.values))
# fig_1.show()

# We are looking for correlation above 0.5 or below -0.5
CoRel_Abs = abs(CoRel["MEDV"])
SelFeat = CoRel_Abs[CoRel_Abs > 0.5]

# We need to make sure that Selected Features are independant of each other
SelCoRel = Data[SelFeat.index.values].corr()

# Plotting the heat map of Selected Features Correlation
# fig_2 = pff.create_annotated_heatmap(np.array(SelCoRel).round(decimals=2), colorscale='RdBu', x=list(SelCoRel.columns.values), y=list(SelCoRel.columns.values))
# fig_2.show()

# Checking for correlation of Selected feature beaing above 0.5 or below -0.5
mask = np.array(0.5 < abs(SelCoRel.iloc[:-1, :-1]))  # droping target (MEDV)
# droping the diagonal elements (since they are meaningless) and upper triangle (since it is repetative)
mask *= np.tri(mask.shape[0], mask.shape[0], -1, dtype="bool")
# finding the location of coorelated selected features
SelCoRelInd = np.argwhere(mask)

# checking the correlation of them (correlated selected features) with target
TargetSelCoRel = np.array(abs(SelCoRel.iloc[-1]))

# find the index of minimum correlation in each row
MinInd = np.argmin(TargetSelCoRel[SelCoRelInd], axis=1)
# find the index of non independant features
NonIndFeat = SelCoRelInd[np.arange(MinInd.size), MinInd]

# set dependand features to NaN and drop them
SelFeat.iloc[NonIndFeat] = None
SelFeat = SelFeat.dropna()
print(SelFeat)


import numpy as np

a = np.zeros((2,2))   # Create an array of all zeros
print(a)              # Prints "[[ 0.  0.]
                      #          [ 0.  0.]]"

b = np.ones((1,2))    # Create an array of all ones
print(b)              # Prints "[[ 1.  1.]]"

c = np.full((2,2), 7)  # Create a constant array
print(c)               # Prints "[[ 7.  7.]
                       #          [ 7.  7.]]"

d = np.eye(2)         # Create a 2x2 identity matrix 单位矩阵
print(d)              # Prints "[[ 1.  0.]
                      #          [ 0.  1.]]"

e = np.random.random((2,2))  # Create an array filled with random values
print(e)                     # Might print "[[ 0.91940167  0.08143941]
                             #               [ 0.68744134  0.87236687]]"
print(e.data[1,1])
print(e.size)
print(a.size)


"""
revised_simplex_algorithm.py

Returns the minimizer of the function
"""

import numpy as np
np.set_printoptions(formatter={'float': lambda x: "{0:0.3f}".format(x)});

def pivoting(tabl, p, q):
    # pivoting about (p,q)-th element
    tabl_new = tabl - tabl[p] * (tabl[:, [q]] / tabl[p, [q]]);
    tabl_new[p] = tabl[p] / tabl[p, [q]];

    return tabl_new;

def revised_simplex_algorithm(A, b, C, indx_cols, options):

    b_print = options['b_print'];
    b_blands_rule = options['b_blands_rule'];
    epsilon = -10e-6; # workaround for -0 issues in while loop

    m, n = A.shape;
    Binv = np.linalg.inv(A[:, indx_cols]); # inverse matrix of basic columns of A
    A = Binv @ A;
    b = Binv @ b;
    cB = C[indx_cols]; # vector of cost coefficients for basic cols of A
    r = C - A.T @ cB; # vector of relative cost coefficients
    cost = cB.T @ b; # cost function
    y0 = b;

    if (b_print):
        # Forming initial revised canonical tableau
        tabl = np.concatenate((np.array(indx_cols).reshape((m, 1)), Binv, y0), axis = 1);
        print(' ');
        print('Initial revised canonical Tableau [indx_cols inv(B) y0]');
        print(tabl);

    while (np.sum(r >= epsilon * np.ones((n, 1))) != n):

        if (b_blands_rule):
            q = 0;
            while(r[q] >= 0):
                q += 1;
        else:
            q = np.argmin(r); # col #q comes into basis

        yq = Binv @ A[:, [q]];

        if (np.sum((yq > 0)) == 0):
            print('Problem unbounded, exit..');
            break;

        a = yq / y0;
        p = np.argmax(a); # col #p leaves basis

        # pivoting about (p,q)-th element
        tabl_aug = np.concatenate((Binv, y0, yq), axis = 1);
        tabl_aug = pivoting(tabl_aug, p, m + 1);
        Binv = tabl_aug[:, 0:m];
        y0 = tabl_aug[:, [m]];
        indx_cols[p] = q;
        cB = C[indx_cols];
        r = C - ((cB.T @ Binv) @ A).T; # vector of relative cost coefficients
        cost = cB.T @ b; # cost function

        if (b_print):
            # Forming updated revised canonical tableau
            tabl = np.concatenate((np.array(indx_cols).reshape((m, 1)), Binv, y0), axis = 1);
            print(' ');
            print('Pivoting about (%d, %d)-th element' % (p, q));
            print(' ');
            print('Updated revised canonical Tableau [indx_cols inv(B) y0]');
            print(tabl);

    X = np.zeros((n, 1));
    X[indx_cols] = y0;
    cost = C.T @ X; # cost function

    if (b_print):
        print(' ');
        print('f(X) = %0.3f' % (cost));
        print('Solution:');
        print(X);

    return (X, cost);


import numpy as np
from utils import load_data
from img_scale import img_to_matrix , plot_img
import matplotlib.pyplot as plt
path_to_save_img = '../report/img/'

## function to convolve the image  with an operator matrix in 2d dimension
def conv_2d(img, mat):
    c = int(mat.shape[0] / 2)
    conv_img = np.array(img, dtype=complex)

    ## Loop over each pixel of the image [py,px] \in 32 * 32 to do the convolution
    for py in range(img.shape[0]):
        for px in range(img.shape[1]):

            mat_sub = mat[max(0, c - py):mat.shape[0] - max(0, (py + c + 1) - img.shape[0]),
                                    max(0, c - px):mat.shape[1] - max(0, (px + c + 1) - img.shape[1])]
            img_sub = img[max(0, py - c):min(img.shape[0], py + c + 1),
                                    max(0, px - c):min(img.shape[1], px + c + 1)]
            conv_img[py, px] = np.sum(np.multiply(mat_sub, img_sub))

    return conv_img

## extract hog features for one channel
def extract_hog_feat_channel(channel):
    # Convolution matrix Gx + j Gy
    # Sobel Operator for computing the gradient 3* 3 in complex form
    Gx  = np.array([ [ -1 ,   0 , 1 ] , [-2 , 0 , 2] , [-1, 0 , 1 ]]) # For Horizontal derivative approximation
    Gy  = np.array([ [ -1 , - 2 , -1 ], [ 0 , 0 , 0] ,[ 1, 2  , 1  ]])# For Vertical Derivative approximation
    # Define the convolution matrix
    G = np.zeros((3,3), dtype=complex)
    G.real = Gx
    G.imag = Gy

    grad = conv_2d(channel, G)
    hist = np.zeros((8, 8, 9))
    bsize = 4 # Block Size in the detection window
    nbins = 9 # Number of bins in the HOG histogram
    # Computing histogram per block for 8*8 blocks
    for i in range(8):
        for j in range(8):
            for grad_pixel in np.nditer(grad[i * bsize:(i + 1) * bsize, j * bsize:(j + 1) * bsize]):
                # Add the magnitude to the correspondant bins in the histogram
                hist[i, j, int( abs(nbins * ( (360 + np.angle(grad_pixel, deg=True)) % 360) / 360))] += np.absolute(grad_pixel)

    return hist, np.absolute(grad)

## Compute hog features for an image by taking the histogram of the channel with the highest gradient intensity in each block
def extract_hog_feat_img(img ) :
    img_mat = img_to_matrix(img)
    hist_rgb = np.zeros((8,8,9,3))
    hist = np.zeros((8,8,9))
    grad_rgb = np.zeros((32, 32, 3))
    grad = np.zeros((32,32))
    bsize = 4
    for i in range(3) :
        hist_rgb[:,:,:,i], grad_rgb[:,:,i] = extract_hog_feat_channel(img_mat[:,:,i])
    for j in range(8):
        for k in range(8):
            m = np.argmax(np.sum(grad_rgb[j*bsize:(j+1)*bsize ,k*bsize:(k+1)*bsize,:],axis=(0,1) ))
            hist[j,k,:] = hist_rgb[ j , k , : , m]
            grad[j * bsize:(j + 1) * bsize,k * bsize:(k + 1) * bsize ] =\
                grad_rgb[j * bsize:(j + 1)* bsize, k * bsize:(k + 1) * bsize, m]
    return hist,grad

# Extract HOG features in dimension 8*8*9
def hog_feat_img(img) :
    hist,_ = extract_hog_feat_img(img)
    hist_ = np.array(hist)
    return hist_.flatten()

## Function to plot gradient magnitude
def plot_grad_magn(grad,save = False , name = None):
    print "Plot Gradient Magnitude"
    plt.imshow(grad)
    if save :
        print "Saving Gradient Magnitude .. "
        assert name != None
        plt.savefig(path_to_save_img+name+'.png')
        print "Gradient Magnitude saved !"
    plt.show()

if __name__ == '__main__':
    ## Load data and print info :
    x_train, x_test,y_train = load_data()
    hist,grad = extract_hog_feat_img(x_train[789])
    plot_img(x_train[789],save = True, name = 'hog_train1')
    plot_grad_magn(grad,save = True, name = 'grad_train1')
    hist1, grad1 = extract_hog_feat_img(x_train[2384])
    plot_img(x_train[2384], save=True, name='hog_train2')
    plot_grad_magn(grad1, save=True, name='grad_train2')


"""Collection of activation functions"""
import numpy as np

from .ops import UnaryOp, ElementWiseUnaryOp
from .static_shape import StaticShape
from .tensor import Tensor

__all__ = ["ReLu", "LeakyReLu", "TanH", "Sigmoid", "SoftMax"]

class ReLu(ElementWiseUnaryOp):
    """Implements the rectified linear unit operation"""

    def compute(self, x: Tensor):
        super().compute(x)
        self.output = np.fmax(x, 0)

    def simple_derivative(self):
        return np.sign(self.output)

class LeakyReLu(ElementWiseUnaryOp):
    """Implements the leaky rectified linear unit operation"""

    def __init__(self, leak: float = 0.01):
        """Creates the op
        :param leak: A float representing the leak (Derivative on negatives)"""
        super().__init__()
        self.leak = leak

    def compute(self, x: Tensor):
        super().compute(x)
        self.output = np.fmax(self.leak * x, x)

    def simple_derivative(self):
        return np.fmax(np.sign(self.x), self.leak)

    def compute_out_dtype(self, dtype):
        return np.result_type(dtype, self.leak)

class TanH(ElementWiseUnaryOp):
    """Hyperbolic tangent activation function"""

    def compute(self, x: Tensor):
        super().compute(x)
        self.output = np.tanh(x)

    def simple_derivative(self):
        sq = self.output ** 2
        return 1 - sq

class Sigmoid(ElementWiseUnaryOp):
    """Sigmoid activation function. Used for binary classification"""

    @staticmethod
    def sigmoid(x: Tensor):
        return np.exp(np.fmin(x, 0)) / (1 + np.exp(-np.abs(x)))

    def compute(self, x: Tensor):
        super().compute(x)
        self.output = self.sigmoid(x)

    def simple_derivative(self):
        return self.output * (1 - self.output)

class SoftMax(UnaryOp):
    """Softmax activation function"""

    def __init__(self, class_axis=0):
        """:param class_axis: The axis of the input on which to find the values
        for the different classes
        """
        super().__init__()
        self.class_axis = class_axis

    @staticmethod
    def reduce_max(x: Tensor, class_axis):
        # dim(x) =  dim(out) = (x,y,z,t)
        # dims(max_1) =  (x,y,z,1)
        max_1 = np.max(x, axis=class_axis, keepdims=True)
        return np.subtract(x, max_1)

    @staticmethod
    def softmax(x: Tensor, class_axis):
        # dim(x) =  dim(e) =  dim(out) = (x,y,z,t)
        # dim(s) = (x,y,z,1)
        e = np.exp(SoftMax.reduce_max(x, class_axis))
        s = np.sum(e, axis=class_axis, keepdims=True)
        return np.divide(e, s)

    def compute(self, x: Tensor):
        super().compute(x)
        self.output = SoftMax.softmax(x, self.class_axis)

    def check_incoming_shapes(self, x: StaticShape):
        pass

    def compute_out_shape(self, x_shape: StaticShape) -> StaticShape:
        return x_shape

    def partials(self, d_output):
        raise NotImplementedError("todo")


import matplotlib.pyplot as plt
from statsmodels.tsa.stattools import adfuller

# Testing the Stationarity of the series

def test_stationarity(x, window):
    # Determine rolling statistics
    rollmean = x.rolling(window=window, center=False).mean()
    rollstd = x.rolling(window=window, center=False).std()

    # Plot rolling statistics:
    plt.plot(x, color='blue', label='Original')
    plt.plot(rollmean, color='red', label='Rolling Mean')
    plt.plot(rollstd, color='black', label='Rolling Std deviation')
    plt.legend(loc='best')
    plt.title('Rolling Mean & Standard Deviation')
    plt.show(block=False)

    # drop NaN values as it causes the test to crash
    x2 = x.dropna()

    # Perform Dickey Fuller test and log results
    # result = adfuller(x2['Closing Value'], autolag='AIC')
    result = adfuller(x2, autolag='AIC')
    print('Dickey-Fuller Statistic: %f' % result[0])
    print('p-value: %f' % result[1])
    pvalue = result[1]
    for key, value in result[4].items():
        print('Critical value %s: %.3f ' % (key, value))
    for key, value in result[4].items():
        if result[0] > value:
            print('The dataset is non-stationary')
            break
        else:
            print('The dataset is stationary (confidence level %s)' % key)
            break

# Calcluate the residual sum of squares
def rss(fitted, actual):
    series = (fitted-actual)**2
    series.dropna(inplace=True)
    return sum(series)


import numpy as np

"""
Spheroid
https://en.wikipedia.org/wiki/Spheroid
Spheroid equation
(x ** 2 / rx ** 2) +
(y ** 2 / ry ** 2) +
(z ** 2 / rz ** 2)
= 1
or
(x / rx) ** 2 +
(y / ry) ** 2 +
(z / rz) ** 2
= 1

width = 2*rx
height = 2*ry
depth = 2*rz

Rotation matrix
RX =[[1, 0, 0],
     [0, np.cos(rx), -np.sin(rx)],
     [0, np.sin(rx), np.cos(rx)]]
RY =[[np.cos(ry), 0, np.sin(ry)],
     [0, 1, 0],
     [-np.sin(ry), 0, np.cos(ry)]]
RZ =[[np.cos(rz), -np.sin(rz), 0],
     [np.sin(rz), np.cos(rz), 0],
     [0, 0, 1]]

RX * RY * RZ * [x, y, z]
"""

def spheroid_coordinate(shape, center, radius, rotation=(None, None, None)):
    """
    return coordinates of point in the spheroid.
    rotation on the xy plan around the z axis.
    warning: rotation is not a perfectly verified feature, use at risk.
    """
    x_lim, y_lim, z_lim = np.ogrid[0:float(shape[0]), 0:float(shape[1]), 0:float(shape[2])]
    x_org, y_org, z_org = center
    x_rad, y_rad, z_rad = radius
    x, y, z = (x_lim - x_org), (y_lim - y_org), (z_lim - z_org)

    if rotation == (None, None, None):
        d = (x / x_rad) ** 2 + (y / y_rad) ** 2 + (z / z_rad) ** 2
    else:
        r1, r2, r3 = rotation
        d = ((x * np.cos(r2) * np.cos(r3) + z * np.sin(r2) - y * np.cos(r2) * np.sin(r3)) / x_rad) ** 2 \
            + ((-z * np.cos(r2) * np.sin(r1) + x * (np.cos(r3) * np.sin(r1) * np.sin(r2) + np.cos(r1) * np.sin(r3)) + y * (np.cos(r1) * np.cos(r3) - np.sin(r1) * np.sin(r2) * np.sin(r3))) / y_rad) ** 2\
            + ((z * np.cos(r1) * np.cos(r2) + x * (-np.cos(r1) * np.cos(r3) * np.sin(r2) + np.sin(r1) * np.sin(r3)) + y * (np.cos(r3) * np.sin(r1) + np.cos(r1) * np.sin(r2) * np.sin(r3))) / z_rad) ** 2

    return np.nonzero(d < 1)

def spheroid(array, fill, center, radius, rotation=(None, None, None)):
    """
    return the given array with a filled spheroid.
    warning: using the returned value is not mandatory.
             the array is passed by reference, if you want to avoid modification on the original array, use np.copy
    """
    array[spheroid_coordinate(array.shape, center, radius, rotation)] = fill
    return array

def spheroid_array(radius):
    """
    return a numpy array containing a spheroid filled with 1.0.
    not to be used, but easiest way to understand the equation.
    """

    rx, ry, rz = radius

    shape = np.zeros((2*rx, 2*ry, 2*rz), dtype=np.float32)

    for x_ in range(shape.shape[0]):
        for y_ in range(shape.shape[1]):
            for z_ in range(shape.shape[2]):
                x = x_ - rx
                y = y_ - ry
                z = z_ - rz

                if (x / rx) ** 2 + (y / ry) ** 2 + (z / rz) ** 2 <= 1:
                    shape[x_, y_, z_] = 1.0

    return shape


# -*- coding: utf-8 -*-
"""
Created on Tue Oct 23 17:36:35 2018

@author: ark4d
"""
import numpy as np

class Probability():
    '''
    This class contains methods to work with probability
    '''
    @staticmethod
    def get_distr(data, un_values=[]):
        '''
        Creates array with probability of each values that to occur
        Parameters
        ----------
        data : numpy-array(int)
            data probability of element which will be tested
        un_values : array-like(int), optional
            unique values probability of which will be created
        Returns
        ----------
        numpy-array(float)
            chace of occurence
        '''
        if len(un_values)==0:
            un_values=np.unique(data)
        values, count = [],[]
        for value in un_values:
            values.append(value)
            count.append(len(data[data==value]))
        dist = np.array([values, np.array(count)/data.shape[0]])
        return dist

    @staticmethod
    def dist_table(data1, data2):
        '''
        Create table that represent joint probability of two distributions

        Parameters
        ----------
        data1 : numpy-array (int)
            data
        data2 : numpy-array (int)
            data

        Returns
        ----------
        list
            unique values of data1
        list
            unique values of data2
        numpy matrix
            joint probability of two distribution

        Raises
        ----------
        ValueError
            if `data1` shape is not equal to `data2` shape
        '''
        if data1.shape == data2.shape:
            size = data1.shape[0]
            un_x_values = np.unique(data1)
            un_y_values = np.unique(data2)
            table = np.zeros((un_x_values.shape[0], un_y_values.shape[0]))
            for row in np.transpose(np.array([data1,data2])):
                table[
                    np.where(un_x_values==row[0]),
                    np.where(un_y_values==row[1])
                ]+=1
            return un_x_values, un_y_values, table/size
        else:
            raise ValueError('{} != {}'.format(data1.shape, data2.shape))

class Itheory():
    @staticmethod
#    note: if you were to check this function with the function in scipy.stats
#    you will get different values# because scipy calculates entropy with base
#    e instead of 2. The values is different, but they showing same thing. you
#    can say that the function in scipy return decibels of entropy whereas
#    the function beneath returns bits (shannons)
#    of entropy.
    def entropy(pmf):
        '''
        Calculates entropy

        Parameters
        ----------
        pmf : numpy-array (float)
            Probabilities of each element to occur

        Retuns
        ----------
        float
            entropy
        '''
        entrp = -(pmf*np.nan_to_num(np.log2(pmf),0)).sum()
        return entrp

    @staticmethod
    def joint_entropy(table):
        '''
        Calculates joint entropy of 2 distributions

        Parameters
        ----------
        table : numpy matrix
            Joint probability distribution matrix (can be generated
            by `dist_table` function)

        Returns
        ----------
        float
            joint entropy
        '''
        j_ent = np.nan_to_num(table*np.log2(table),0).sum()
        return -j_ent

    @staticmethod
    def conditional_entropy(table, axis_marginal):
        '''
        Calculates conditional entropy of 2 distributions

        Parameters
        ----------
        table : numpy matrix
            Joint probability distribution matrix (can be generated
            by `dist_table` function)

        axis_marginal : int, 1 or 0
            Determines given which data entropy will be calculater
            0 - H(`data1`|`data2`)
            1 - H(`data2`|`data1`)
            look in `dist_table` for definition of `data1` and `data2`
        Returns
        ----------
        float
            conditional entropy
        '''
        #informal: relatively hard function to comprehend.
        marg = np.apply_along_axis(lambda x: x.sum(),axis_marginal,table)
        table=np.apply_along_axis(lambda x: x*marg**-1,1-axis_marginal,table)
        table=np.apply_along_axis(lambda x: Itheory.entropy(x),axis_marginal,table)
        table = np.nan_to_num(table,0)
        c_ent = np.matmul(marg, table)
        return c_ent

    @staticmethod
    def rel_entropy(prob_dist1, prob_dist2):
        '''
        Calculates relative entropy

        Parameters
        ----------
        prob_dist1 : numpy-array
            marginal probability of a distribution
        prob_dist2 : numpy-array
            marginal probability of a distribution

        Returns
        ----------
        float
            relative entropy
        '''
        div = prob_dist1/prob_dist2
        if np.isinf(div).any():
            return np.inf
        else:
            log = np.log2(div)
            log[np.isinf(log)]=0
            log[np.isnan(log)]=0
            rel_entropy = np.sum(prob_dist1*log)
            return rel_entropy

    @staticmethod
    def mutual_information(prob_dist1, prob_dist2, table):
        '''
        Calculates mutual information between 2 distributions

        Parameters
        ----------
        prob_dist1 : numpy-array
            marginal probability of a distribution
        prob_dist2 : numpy-array
            marginal probability of a distribtuion
        table: numpy-matrix
            Joint probability distribution matrix (can be generated
            by `dist_table` function)

        Returns
        ----------
        float
            Mutual information
        '''
        div = table/(prob_dist1.reshape(-1,1)* prob_dist2)
        if np.isinf(div).any():
            return np.inf
        else:
            log = np.log2(div)
            log[np.isinf(log)]=0
            log[np.isnan(log)]=0
            mutual_inf = np.sum(table*log)
            return mutual_inf

# TO-DO: Add docs to utility.
class Utility():

    @staticmethod
    def make_ax_look_good(ax):
        '''
        Makes axes look fancy :)

        Parameters
        ----------
        ax : matplotlib axes
        '''
        ax.grid(alpha=0.2)
        for spine in ax.spines:
                ax.spines[spine].set_visible(False)

    @staticmethod
    def less_ent_d(s, step_m = 2, low_boundary = 4, times =2 ):
        '''
        Changes distribution such that the entropy descends.
        '''
        mult = 2
        s_m = s.mean().round().astype('int')
        vals = np.array(np.unique(s,return_counts=True))
        vals_gtlw = (vals[0,vals[1,:]>low_boundary])
        vals_gtlw = vals_gtlw[(vals_gtlw<s_m) | (vals_gtlw>s_m)]
        np.set_printoptions(threshold=np.nan)
        for i in range(0,times):
            for indx, val in enumerate(vals_gtlw[vals_gtlw<s_m][::-1]):
                step_arr=np.zeros(shape=(len(s[s==val]),),dtype='int')
                step_arr[:int(len(s[s==val])/3)]=+step_m
                s[s==val]+=step_arr
                mult+=1
            mult=2
            for indx, val in enumerate(vals_gtlw[vals_gtlw>s_m]):
                step_arr=np.zeros(shape=(len(s[s==val]),),dtype='int')
                step_arr[:int(len(s[s==val])/3)]=-step_m
                s[s==val]+=step_arr
                mult+=1

        return s

    @staticmethod
    def less_ent(dist, multiplier=0.01,steps=2, low_border=0.004):
        '''
        Chages data such that the entropy descends.
        '''
        size = len(dist)
        s_indx = np.argmax(dist!=0)
        e_indx = np.argmax(np.flip(dist)!=0)
        m_indx_d = np.round(((size-(s_indx+e_indx))/2)).astype('int')
        m_indx = m_indx_d+s_indx
#        print(s_indx, -e_indx, m_indx)
        for step in range(0,steps):
            for i in range(s_indx,m_indx):
                distance = np.abs(m_indx - i)
                distance_coef = distance/m_indx_d
                buff = dist[i]*distance_coef* multiplier
                if (dist[i]-buff)<(low_border) and dist[i]>=low_border:
                    buff = (dist[i]-low_border)
                dist[i], dist[i+1] = dist[i]-buff, dist[i+1]+buff

            for i in range(size-1-e_indx, m_indx,-1):
                distance = np.abs(m_indx - i)
                distance_coef = distance/m_indx_d
                buff = dist[i]*distance_coef* multiplier
                if (dist[i]-buff)<(low_border) and dist[i]>=low_border:
                    buff = (dist[i]-low_border)
                dist[i], dist[i-1] = dist[i]-buff, dist[i-1]+buff
        return dist

    #this function shuffles values of !probability distribution!
    #dist: a probability distribution
    #times: greater - more shufflier
    #multiplier: greater - more shufflier
    @staticmethod
    def shuffle_dist(dist, times=10, multiplier=0.4):
        '''
        Changes values of distribtuions. Depends on variables that are passed.

        Parameters
        ----------
        dist : nparray
            distribution of a random variable
        times : int
            amount of times for the process to repeat
        multiplier : float
            determines how much the function will change the distribution on a iteration

        Returns
        ----------
        dist : nparray
            changed distribtuion
        '''
        deck = dist[dist!=0]
        start_index = np.argmax(dist!=0)
        end_index = start_index+deck.shape[0]
        buff=0
        for i in range(0, times):
            for i in range(start_index, end_index):
                if np.random.choice([True,False]):
                    buff, dist[i] = dist[i]*multiplier, (dist[i]-dist[i]*multiplier)+buff
            dist[np.random.randint(start_index, end_index)]+=buff
            buff=0
        return dist

    @staticmethod
    def _decide(chance):
        if np.random.rand()<chance:
            return True
        else:
            return False

    @staticmethod
    def shuffle_data(data, chance = 0.5, times=10, count_low_boundary=1, count_max_boundary=40):
        '''
        Changes data by changing a value to another value. Another value can be picked from range of
        unique values that are presented in the data.
        '''
        uni,vals = np.unique(data,return_counts = True)
        for j in range(0, times):
            for i in range(data.shape[0]):
                if (vals[np.where(uni==data[i])]>count_low_boundary)and(vals[np.where(uni==data[i])]<count_max_boundary):
                    if Utility._decide(chance):
                        n_i = np.random.randint(low=0,high=data.shape[0])
                        vals[np.where(uni==data[i])]-=1
                        data[i]=data[n_i]
                        vals[np.where(uni==data[n_i])]+=1
                else:
                    pass
        return data


# import math
# import numpy as np

def lagrange(f, xs, x):
    ys = [f(i) for i in xs]
    n = len(xs)
    y = 0
    for k in range(0, n):
        t = 1
        for j in range(0, n):
            if j != k:
                s = (x - xs[j]) / (xs[k] - xs[j])
                t = t * s
        y = y + t * ys[k]

    return y

def difference_quotient(f, xs):
    res = 0
    n = len(xs)
    for k in range(n):
        t = 1
        for j in range(n):
            if j != k:
                t *= (xs[k] - xs[j])
        res += f(xs[k]) / t

    return res

def newtown(f, xs, x):
    n = len(xs)
    y = f(xs[0])
    for k in range(1, n):
        t = difference_quotient(f, xs[:k + 1])
        for j in range(k):
            t *= (x - xs[j])
        y += t

    return y

def piecewise_linear(f, xs, x):
    interval = [0, 1]
    if x < xs[0]:
        interval = [xs[0], xs[1]]
    elif x > xs[-1]:
        interval = [xs[-2], xs[-1]]
    else:
        for i in range(len(xs) - 1):
            if x >= xs[i] and x <= xs[i + 1]:
                interval = [xs[i], xs[i + 1]]
                break

    return lagrange(f, interval, x)

# print(lagrange(lambda x: 1 / (1 + x * x), np.linspace(-5, 5, 11), 0.5))
# print(lagrange(lambda x: 1 / (1 + x * x), np.linspace(-5, 5, 11), 4.5))
# print(piecewise_linear(lambda x: 1 / (1 + x * x), np.linspace(-5, 5, 11), 0.5))
# print(piecewise_linear(lambda x: 1 / (1 + x * x), np.linspace(-5, 5, 11), 4.5))

# print(
#     difference_quotient(lambda x: x * x * x - 4 * x, (np.arange(5))[1:]),
#     difference_quotient(lambda x: x * x * x - 4 * x, [1, 2]),
#     difference_quotient(lambda x: x * x * x - 4 * x, [1, 2, 3])
# )

# print(newtown(lambda x: x * x * x - 4 * x, (np.arange(5))[1:], 5))
# print(newtown(lambda x: math.sqrt(x), [1, 4, 9], 5))


import sys

import pandas as pd
from sklearn.decomposition import PCA
import matplotlib.pyplot as plt
from numpy import mat, mean, cov, linalg, argsort, shape, nonzero, isnan

def load_data(csv_file):
    data = pd.read_csv(csv_file)
    # x = data[list(range(401))]

    print('data: {x}'.format(x=data))

    return mat(data)

def pca(data_mat, max_feature=105):
    mean_value = mean(data_mat, axis=0)

    data_rem_mat = data_mat - mean_value
    cov_mat = cov(data_rem_mat, rowvar=0)
    print('cov mat: {cm}'.format(cm=cov_mat))

    feature_value, feature_vect = linalg.eig(mat(cov_mat))
    print('feature_value: {fv}'.format(fv=feature_value))
    print('feature_vect: {fv}'.format(fv=feature_vect))

    fea_value_sort = argsort(feature_value)
    fea_value_topN = fea_value_sort[: -(max_feature + 1): -1]
    red_eig_vects = feature_vect[:, fea_value_topN]
    print('topN: {t}'.format(t=red_eig_vects))
    print(shape(red_eig_vects))

    low_data_mat = data_rem_mat * red_eig_vects
    reco_mat = low_data_mat * red_eig_vects.T + mean_value
    print('low_data_mat: {ldm}'.format(ldm=low_data_mat))

    return low_data_mat, reco_mat

def plot_w(low_data_mat, reco_mat):
    fig = plt.figure()
    ax = fig.add_subplot(111)
    ax.scatter(low_data_mat[:, 0], low_data_mat[:, 1], marker='*', s=90)
    ax.scatter(reco_mat[:, 0], reco_mat[:, 1], marker='*', s=50, c='red')
    plt.show()

def replace_nan_with_mean(file_path):
    dat_mat = load_data(csv_file=file_path)
    num_feat = shape(dat_mat)[1]
    for i in range(num_feat):
        mean_value = mean(dat_mat[nonzero(~isnan(dat_mat[:, i].A))[0], i])
        # set NaN values to mean
        dat_mat[nonzero(isnan(dat_mat[:, i].A))[0], i] = mean_value

    print('dat_mat: {dm}'.format(dm=dat_mat))

    return dat_mat

def test():
    csv_file = sys.argv[1]
    df = pd.read_csv(csv_file)
    data = df.iloc[:, :401]
    # type_array = df.iloc[:, 401]

    n_components = 400
    pca = PCA(n_components=n_components)
    pca.fit(data)

    plt.figure(1, figsize=(4, 3))
    plt.clf()
    plt.axes([.2, .2, .7, .7])
    plt.plot(pca.explained_variance_, linewidth=2)
    plt.axis('tight')
    plt.xlabel('n_components')
    plt.ylabel('explained_variance_')
    plt.show()

if __name__ == '__main__':
    file_path = sys.argv[1]
    data_mat = replace_nan_with_mean(file_path=file_path)
    low_data_mat, reco_mat = pca(data_mat=data_mat, max_feature=2)
    plot_w(low_data_mat=low_data_mat, reco_mat=reco_mat)


import copy
import numpy as np
from itertools import combinations

class Node:

    def __init__(self, k):
        self.key = k
        self.parent = None
        self.left = None
        self.right = None

def insert(root, n):
    # n is new node
    # return root
    y = None
    x = root
    while x is not None:
        y = x
        if n.key < x.key:
            x = x.left
        else:
            x = x.right
    n.parent = y
    if y is None:
        return n  # root is None
    elif n.key < y.key:
        y.left = n
    else:
        y.right = n
    return root

def gen_tree(numbers):
    root = None
    for n in numbers:
        root = insert(root, Node(n))
    return root

answer1 = []  # save the answer of solution1

def solution1(numbers):
    # ルートから、選択できるノードを順番に選択する
    # 選択されたノードの子ノードを選択肢に追加する

    # building a tree
    root = gen_tree(numbers)
    choices = [root]
    record = []
    choose(choices, record, None)

def choose(choices, record, chosen):
    if chosen is not None:
        # print(f'{chosen.key} is selected.')
        record.append(chosen.key)
        if chosen.right is not None:
            choices.append(chosen.right)
        if chosen.left is not None:
            choices.append(chosen.left)

    for choice in choices:
        choose([c for c in choices if c is not choice], copy.deepcopy(record), choice)
    if len(choices) == 0:
        answer1.append(record)

def solution2(numbers):
    # solve recursively

    # building a tree
    root = gen_tree(numbers)
    answer = solve2(root)
    return answer

def solve2(root):
    # return the answer of the partial tree
    if root is None:
        return []
    l_arrays = solve2(root.left)
    r_arrays = solve2(root.right)
    if len(l_arrays) == 0 and len(r_arrays) == 0:
        return [[root.key]]
    if len(l_arrays) == 0:
        for arr in r_arrays:
            arr.insert(0, root.key)
        return r_arrays
    if len(r_arrays) == 0:
        for arr in l_arrays:
            arr.insert(0, root.key)
        return l_arrays

    # combine both r and l answers. the first element is this node
    answers = []
    for l_array in l_arrays:
        for r_array in r_arrays:
            answers.extend(combine_arrays(l_array, r_array, root.key))
    return answers

# def combine_arrays(l_array, l_index, r_array, r_index, current, answers):
def combine_arrays(a, b, r):
    result = []
    n = len(a) + len(b)
    for indA in combinations(range(n), len(a)):
        x = np.zeros(n, dtype=int)
        mask = np.zeros(n, dtype=bool)
        mask[indA,] = True    # The ',' is important here for indexing
        x[mask] = a
        x[~mask] = b
        x = list(x)
        x.insert(0, r)
        result.append(x)
    return result

if __name__ == '__main__':
    # input_numbers = [4, 2, 1, 3, 5]
    input_numbers = [4, 2, 1, 3, 6, 5, 7]
    solution1(input_numbers)
    print(answer1)
    print(len(answer1))
    ans2 = solution2(input_numbers)
    print(ans2)
    print(len(ans2))
    if len(answer1) != len(ans2):
        print("incorrect")
    for a1 in answer1:
        if a1 not in ans2:
            print("incorrect")


import numpy as np
import math
from lmfit import Model

def calculate_HHCF(image , image_width):
    """
     Calculates the height-height correlation function for an image. The HHCF is calculated along the rows of the image.

     An Open Access reference for calculating HHCF from an image:
        "Height-Height Correlation Function to Determine Grain Size in Iron Phthalocyanine Thin Films", Gredig, T; Silverstein, E; Byrne, M. Journal of Physics:Conference Series. (2013). DOI: 10.1088/1742-6596/417/1/012069

     Inputs:
        image: numpy array. Contains the height image.
        image_width: float. Specifies the physical width corresponding to the image frame expressed in meters. (E.g. a 200 nm wide scan frame should be inputted as 1e-9.)

    Outputs:
        distances: numpy array. Contains the distances values from the reference pixel corresponding to each value in the HHCF array. This is the X data for plotting in a HHCF plot.
        HHCF: numpy array. Contains the calculated height-height correlation function data.
    """
    # Initialize parameters and an array for data storage.
    num_columns = np.shape(image)[0]
    HHCF = np.zeros(num_columns)

    # Perform calculation of HHCF. This works by finding the difference in value between pairs of pixels separated by a set number of pixels (corresponding to a physical distance).
    # This is a semi-vectorized implementation of the calculation. The idea is to effectively have two copies of the image stacked atop one another. Then the top image is shifted to the left relative to the bottom image by some number of pixels (corresponding to a certain physical difference).
    # Then the two copies of the image are 'cropped' to contain only the portions of the images which overlap with their shifted counterparts, discarding any pixels which do not have a pair.
    # Loop over all allowable (non-zero) pixel-wise separation distances.
    for separation_distance in range(1,num_columns):
        # Slice the image to generate the 'bottom' copy of the image as described above, discarding pixels on the right side, since they won't have pairs in the 'top' copy of the image.
        array1 = image[: , 0:-separation_distance]
        # Slice the image to generate the 'top' copy of the image as described above, discarding pixels on the left side, since they won't have pairs in the 'bottom' copy of the image.
        array2 = image[: , separation_distance:]
        # Calculate the HHCF value for each pixel pair in the overlapping regions of the 'image stack', then take the average to represent the HHCF for the current pixel-wise separation distance.
        HHCF[separation_distance] = np.average(np.square(array2-array1))

    # Determing the physical length corresponding to a single pixel.
    pixel_width = image_width/(num_columns - 1)

    # Generate an array of physical separation distances corresponding to the pixel-wise separation distances used to calculate the HHCF.
    distances = np.arange(num_columns) * pixel_width

    return [distances , HHCF]

def HHCF(r, sigma, xi , alpha):
    """
    Model for height-height correlation function. This model is suitable for fitting when data is present for both above and below the correlation length.

    Inputs:
        r: Numpy array. This contains the distance data which would be used for the x coordinate on a HHCF plot.
        sigma: float. This is related to the RMS roughness of the sample.
        xi: float. This is the correlation length of the sample.
        alpha: float. This is the Hurst parameter.

    Outputs:
        The output of this function is the height-height correlation function for the given distances and parameters.
    """
    return 2 * sigma**2 * (1 - np.exp(- (r/xi)**(2 * alpha)))

def fit_HHCF(HHCF_data , parameter_guesses = None):
    """
    Fit an analytic model for height-height correlation function to experimental HHCF data. This function assumes the HHCF data includes length scales both above and below the correlation length.

    The analytic model is mentioned in the following Open Source article:
        "Height-Height Correlation Function to Determine Grain Size in Iron Phthalocyanine Thin Films", Gredig, T; Silverstein, E; Byrne, M. Journal of Physics:Conference Series. (2013). DOI: 10.1088/1742-6596/417/1/012069

    Inputs:
        HHCF_data: Numpy array. Contains two columns, with the first column being the distance data and the second being the HHCF calculated from an experimental image.
        parameter_guesses Dict. This is an optional input containing guesses for the model parameters. If it is provided, it must have key:value pairs for the following parameters: sigma, xi, alpha.
    """
    # Generate initial guesses of model parameters for fitting if none are provided.
    if parameter_guesses == None:
        parameter_guesses = {'sigma':math.sqrt(np.max(HHCF_data[1])/2)}
        index_of_max = np.argmax(HHCF_data[1])
        parameter_guesses['xi'] = HHCF_data[0][int(index_of_max/4)]
        parameter_guesses['alpha'] = 0.5

    # Create a model object to fit to the HHCF data and assign initial guesses for the fitting parameters.
    fitting_model = Model(HHCF)
    params = fitting_model.make_params(sigma = parameter_guesses['sigma'], xi = parameter_guesses['xi'], alpha = parameter_guesses['alpha'])

    # Try to fit the model to the data.
    try:
        result = fitting_model.fit(HHCF_data[1], params, r = HHCF_data[0])
        return result
    except:
        print('There was an issue processing the data.')

def HHCF_short_range(r, A, alpha):
    """
    Model for height-height correlation function. This model is suitable for fitting when data is present only below the correlation length.

    Inputs:
        r: Numpy array. This contains the distance data which would be used for the x coordinate on a HHCF plot.
        A: float. Scaling constant used in the fitting process.
        alpha: float. This is the Hurst parameter.

    Outputs:
        The output of this function is the height-height correlation function for the given distances and parameters.
    """
    return A * np.power(r , 2 * alpha)

    ####### The function below still needs work!
def fit_HHCF_short_range(HHCF_data , parameter_guesses = None):
    """
    Fit an analytic model for height-height correlation function to experimental HHCF data. This function assumes the HHCF data includes only length scales below the correlation length.

    The analytic model is mentioned in the following Open Source article:
        "Height-Height Correlation Function to Determine Grain Size in Iron Phthalocyanine Thin Films", Gredig, T; Silverstein, E; Byrne, M. Journal of Physics:Conference Series. (2013). DOI: 10.1088/1742-6596/417/1/012069

    Inputs:
        HHCF_data: Numpy array. Contains two columns, with the first column being the distance data and the second being the HHCF calculated from an experimental image.
        parameter_guesses: Dict. This is an optional input containing guesses for the model parameters. If it is provided, it must have key:value pairs for the following parameters: A, alpha.
    """
    # Generate initial guesses of model parameters for fitting if none are provided.
    if parameter_guesses == None:
        # Guess the parameter values by assuming that the data will be linear on a log-log plot and using a linear fit.
        mid = int( len(HHCF_data) / 2)
        # Calculate the log of two data points near the middle of the data array.
        x = np.log( HHCF_data[0][mid : mid + 5])
        y = np.log( HHCF_data[1][mid : mid + 5] )
        # Perform a 'linear regression' according to: log(HHCF) = log(A) + 2 * alpha * log(r)
        alpha_guess = 0.5 * (y[1] - y[0]) / (x[1] - x[0]) # slope
        A_guess = np.exp((y[0]*x[1] - y[1]*x[0]) / (x[1] - x[0])) # y-intercept
        # Store parameter guesses in a dictionary.
        parameter_guesses = {'A': A_guess , 'alpha':alpha_guess}

    # Create a model object to fit to the HHCF data and assign initial guesses for the fitting parameters.
    fitting_model = Model(HHCF_short_range)
    params = fitting_model.make_params(A = parameter_guesses['A'], alpha = parameter_guesses['alpha'])

    # Try to fit the model to the data.
    try:
        result = fitting_model.fit(HHCF_data[1], params, r = HHCF_data[0])
        return result
    except:
        print('There was an issue processing the data.')

def make_histogram(image, num_bins = 100):
    """
    Bin a collection of numeric data for use in histogram plotting or analysis.

    Inputs:
        image: Numpy array. Contains the data to be binned.
        num_bins: int. Specifies the number of bins to use.

    Outputs:
        bin_centers: Float. The center coordinate for the histogram bins.
        counts: Int. The number of data points contained in each bin.
        bin_width: Float. The width of the bins.
    """
    # Group data into desired number of bins.
    counts , bin_edges = np.histogram(image , bins = num_bins)

    # calculate the center values for the bins
    bin_centers = 0.5*(bin_edges[1:] + bin_edges[:-1])

    # calculate the width of the bins.
    bin_width = (bin_edges[-1]-bin_edges[0])/len(counts)
    return bin_centers, counts, bin_width


import Power_Iteration as pi
import numpy as np
import networkx as nx

from matplotlib import pyplot as plt

def power_iteration_application_test():
    A = np.array([[1.5, 0.5], [0.5, 1.5]])
    print(A)
    norm_indicator = False
    dom_eigenvector, dom_eigenvalue = pi.PowerMethod(A, norm_indicator, 0.0001)
    print(dom_eigenvalue)
    print(dom_eigenvector)

    norm_indicator = True
    dom_eigenvector, dom_eigenvalue = pi.PowerMethod(A, norm_indicator, 0.0001)
    print(dom_eigenvalue)
    print(dom_eigenvector)

def page_rank_application_test():
    G = nx.DiGraph()
    G.add_edges_from([(1,2),(1,3),(1,4),(2,3),(2,4), (3, 1),(4,1), (4, 3)])
    nx.draw(G, with_labels=True)
    plt.savefig("page_rank_application_sample.png")
    plt.show()
    print("Limiting Distribution: ")
    print(PageRank(G, 0.15))

def stochastic_transition_matrix_from_G(G, adaptive, weight=0.15):
    Aj = nx.to_numpy_matrix(G).A

    N = len(G.nodes)
    v = np.empty(shape=(N,1))
    v.fill(1/N)

    P = np.zeros(shape=(N, N))
    dangling_notes = set()
    for j, node in enumerate(G.nodes()):
        out_deg = G.out_degree(node)
        if out_deg == 0:
            dangling_notes.add(j)
            for i in range(N):
                P[i][j] = 0
        else:
            for i in range(N):
                P[i][j] = Aj[j][i] / out_deg

    if adaptive:
        return P
    else:
        #test dangling nodes
        d = np.empty(shape=(1,N))
        d.fill(0)
        for i in range(N):
            if i in dangling_notes:
                d[0][i] = 1
        D = v.dot(d)

    P_prime = P + D
    S = np.ones(shape=(N, N))
    S = np.multiply(S, 1 / N)

    Aw = np.multiply(P_prime, 1 - weight)
    Sw = np.multiply(S, weight)
    M = Aw + Sw
    return M

def multiplication_with_P(P, c, x):
    N = A.shape[0]
    v = np.empty(N)
    v.fill(1/N)
    y = np.multiply(P, c).dot(x)
    w = np.linalg.norm(x,1) - np.linalg.norm(y,1)
    y = y + np.multiply(v,w)
    return y

def PageRank(G, weight):

    M = stochastic_transition_matrix(G, weight)

    return pi.PowerMethod(M,True, 0.01)[0]


import matplotlib.pyplot as plt
import numpy as np
from sklearn import datasets, linear_model
from sklearn.metrics import mean_squared_error, r2_score
import matplotlib.pyplot as plt
from math import fabs

def main():
	data_set = datasets.load_boston()

	features, target = split_features_target(data_set)

	split = 0.15
	train_features, train_target, test_features, test_target = split_train_test(features, target, split)

	train_mse = []
	test_mse = []
	x_axis =  range(1, 2500)

	min_train_mse, min_test_mse, min_difference, min_alpha = 1, 1, 1, 1

	for a in range(1, 2500):
		regr = linear_model.Ridge(alpha=a)
		regr.fit(train_features, train_target)

		train_predictions = regr.predict(train_features)
		test_predictions = regr.predict(test_features)

		train = mean_squared_error(train_target, train_predictions) / 100
		test = mean_squared_error(test_target, test_predictions) / 100

		train_mse.append(train)
		test_mse.append(test)

		difference = fabs(train-test)
		if (difference < min_difference):
			min_train_mse = train
			min_test_mse = test
			min_difference = difference
			min_alpha = a

	print('Minimal difference: ', min_difference)
	print('Alpha: ', min_alpha)
	print('MSE on training data', min_train_mse)
	print('MSE on test data', min_test_mse)

	plt.plot(x_axis, train_mse, label='train data')
	plt.plot(x_axis, test_mse, label='test_data')

	plt.xlabel('alpha')
	plt.ylabel('Mean squared error')

	plt.legend()

	plt.show()

def split_train_test(features, target, test_size):
	total_test_size = int(len(features) * test_size)
	np.random.seed(0)
	indices = np.random.permutation(len(features))
	train_features = features[indices[:-total_test_size]]
	train_target = target[indices[:-total_test_size]]
	test_features  = features[indices[-total_test_size:]]
	test_target  = target[indices[-total_test_size:]]
	return train_features, train_target, test_features, test_target

def split_features_target(data_set):
	features = data_set.data
	target = data_set.target
	return features, target

if __name__ == "__main__":
	main()


"""
@author: Abhirup Mishra
@Description: The class contains the methods for pricing options using Black-Scholes Formula
"""

from math import sqrt, pi
import scipy as sp
import scipy.special
import numpy as np

def mass_broadcast(*args):
	'''Cast some list of vectors of shape(a_i, 1) or (a_i,) to the same uniform (l,1)
	'''
	l = 0
	xs = []
	for i, x in enumerate(args):
		xs.append( np.asarray(x).reshape(-1,1) )
		l = max(l, xs[i].shape[0])

	for i, x in enumerate(xs):
		xs[i] = np.broadcast_to(x, (l, 1))

	return xs

class IncorrectInputs(Exception): pass

class TimeoutException(Exception): pass

class BlackScholes:

	#the constructor for the class
	def __init__(self, spot_price, strike_price, time_to_maturity, interest_rate, sigma, dividend_yield=0):

		''' initializtion parameters '''

		[
			self.spot_price,
			self.strike_price,
			self.time_to_maturity,
			self.interest_rate,
			self.sigma,
			self.dividend_yield
		] = mass_broadcast(
			spot_price, strike_price, time_to_maturity, interest_rate, sigma, dividend_yield
		)

		# self.spot_price = np.asarray(spot_price).astype(float)
		#
		# #checking if strike price is an array or not
		# if(not(hasattr(strike_price, "__len__"))):
		# 	self.strike_price = np.ones(len(spot_price))*strike_price
		# else:
		# 	self.strike_price = strike_price

		# #checking if time to maturity is an array or not
		# if(not(hasattr(time_to_maturity, "__len__"))):
		# 	self.time_to_maturity = np.ones(len(spot_price))*time_to_maturity
		# else:
		# 	self.time_to_maturity = time_to_maturity

		# #checking if interest rate is an array or not
		# if(not(hasattr(interest_rate, "__len__"))):
		# 	self.interest_rate = np.ones(len(spot_price))*interest_rate
		# else:
		# 	self.interest_rate = interest_rate

		# #checking if volatility is an array or not
		# if(not(hasattr(sigma, "__len__"))):
		# 	self.sigma = np.ones(len(spot_price))*sigma
		# else:
		# 	self.sigma = sigma
		#
		# #checking if dividend yield is an array or not
		# if(not(hasattr(dividend_yield, "__len__"))):
		# 	self.dividend_yield = np.ones(len(spot_price))*dividend_yield
		# else:
		# 	self.dividend_yield = dividend_yield

	#private method for erf function
	def bls_erf_value(self,input_number):
		erf_out = 0.5*(1 + sp.special.erf(input_number/sqrt(2.0)))
		return erf_out

	#vectorized method to price call option
	def european_option_price(self):

		"Price of the call option"
		"the vectorized method can compute price of multiple options in array"
		numerator = sp.add(
			sp.log(
				sp.divide(
					self.spot_price,
					self.strike_price,
				)
			),
			sp.multiply(
				(
					self.interest_rate - self.dividend_yield +
					0.5*sp.power(self.sigma,2)
				),
				self.time_to_maturity)
		)
		d1 = sp.divide(
			numerator,
			sp.prod(
				[
					self.sigma,
					sp.sqrt(self.time_to_maturity)
				],
				axis=0,
			)
		)
		d2 = sp.add(
			d1,
			-sp.multiply(
				self.sigma,
				sp.sqrt(self.time_to_maturity)
			)
		)

		ecall = sp.product(
			[
				self.spot_price,
				self.bls_erf_value(d1),
				sp.exp(sp.multiply(-self.dividend_yield,self.time_to_maturity))
			],
			axis=0
		) - sp.product(
			[
				self.strike_price,
				self.bls_erf_value(d2),
				sp.exp(
					-sp.multiply(
						self.interest_rate,
						self.time_to_maturity,
					)
				)
			],
			axis=0,
		)

		eput = sp.product(
			[
				-self.spot_price,
				self.bls_erf_value(-d1),
				sp.exp(
					sp.multiply(
						-self.dividend_yield,
						self.time_to_maturity
					)
				)
			],
			axis=0,
		) + sp.product(
			[
				self.strike_price,
				self.bls_erf_value(-d2),
				sp.exp(
					-sp.multiply(
						self.interest_rate,
						self.time_to_maturity
					)
				)
			],
			axis=0,
		)
		return ecall, eput

	#delta of the option
	def european_option_delta(self):
		numerator = sp.add(
			sp.log(
				sp.divide(
					self.spot_price,
					self.strike_price
				)
			),
			sp.multiply(
				( self.interest_rate - self.dividend_yield + 0.5*sp.power(self.sigma,2)),
				self.time_to_maturity
			)
		)
		d1 = sp.divide(
			numerator,
			sp.prod(
				[
					self.sigma,
					sp.sqrt(self.time_to_maturity)
				],
				axis=0,
			)
		)
		call_delta = self.bls_erf_value(d1)
		put_delta = call_delta - 1

		return call_delta, put_delta

	#gamma of the option (under construction)
	def european_option_gamma(self):
		pass

	#vega of the option
	def european_option_vega(self):
		numerator = sp.add(
			sp.log(
				sp.divide(
					self.spot_price,
					self.strike_price
				)
			),
			sp.multiply(
				( self.interest_rate - self.dividend_yield + 0.5*sp.power(self.sigma,2)),
				self.time_to_maturity
			)
		)
		d1 = sp.divide(
			numerator,
			sp.prod(
				[
					self.sigma,
					sp.sqrt(self.time_to_maturity)
				],
				axis=0,
			)
		)

		val = sp.multiply(
			sp.multiply(
				self.spot_price,
				sp.exp(
					-sp.multiply(
						self.dividend_yield,
						self.time_to_maturity
					)
				)
			),
			sp.exp(-sp.square(d1)*0.5)
		)
		val = sp.multiply(
			val,
			sp.sqrt(self.time_to_maturity)
		)
		vega = (1/sqrt(2*pi))*val

		return vega

	#theta of the option
	def european_option_theta(self):
		pass

	#rho of the option
	def european_option_rho(self):
		"Price of the call option"
		"the vectorized method can compute price of multiple options in array"
		numerator = sp.add(
			sp.log(
				sp.divide(
					self.spot_price,
					self.strike_price,
				)
			),
			sp.multiply(
				(
					self.interest_rate - self.dividend_yield +
					0.5*sp.power(self.sigma,2)
				),
				self.time_to_maturity)
		)
		d1 = sp.divide(
			numerator,
			sp.prod(
				[
					self.sigma,
					sp.sqrt(self.time_to_maturity)
				],
				axis=0,
			)
		)
		d2 = sp.add(
			d1,
			-sp.multiply(
				self.sigma,
				sp.sqrt(self.time_to_maturity)
			)
		)

		j = sp.product(
			[
				self.spot_price,
				self.time_to_maturity,
				sp.exp(
					sp.multiply(
						-self.interest_rate,
						self.time_to_maturity
					)
				),
			],
			axis=0
		)

		c_rho = j * self.bls_erf_value(d2)
		p_rho = -j * self.bls_erf_value(-d2)

		return c_rho, p_rho


"""
Libary of linear regressions.
"""
import numpy as np
from scipy.stats import linregress
from astroML.linear_model import TLS_logL
from scipy.odr import Model, Data, RealData, ODR
import scipy.optimize as op
import emcee

def mcmc_linear_model(x, y, xerr, yerr, nwalkers=100, nruns=2000,
                      cut=100):
    """
    This was built using the following references:

    * http://dan.iel.fm/emcee/current/user/line/
    * http://www.astroml.org/book_figures/chapter8/fig_total_least_squares.html
    * https://github.com/astroML/astroML/blob/master/astroML/linear_model/TLS.py

    The linear models is $y=a*x+b$

    Returns (a,b), ((a_lower, a_upper), (b_lower, b_upper))
    """

    # translate between typical slope-intercept representation,
    # and the normal vector representation

    def get_a_b(theta):
        b = np.dot(theta, theta) / theta[1]
        a = -theta[0] / theta[1]

        return a, b

    def get_beta(a, b):
        denom = (1 + a * a)
        return np.array([-b * a / denom, b / denom])

    # Define the log-maximum likelihood
    def lnlike(theta, x, y, xerr, yerr):
        arr = np.column_stack((x, y))
        arr_err = np.column_stack((xerr, yerr))

        return TLS_logL(theta, arr, arr_err**2)

    # Get a first extimative of the parameters
    # based on the maximum likelihood
    nll = lambda *args: -lnlike(*args)
    x0 = get_beta(*linregress(x, y)[:2]) # Initial guesses from
                                         # Ordinary Least Squares
    result = op.minimize(nll, x0=x0, args=(x, y, xerr, yerr),
                         method='Nelder-Mead')
    theta = result["x"]

    # Define the log-prior
    def lnprior(theta):
        a, b = get_a_b(theta)
        if -5.0 < a < 0.5 and 0.0 < b < 10.0:
            return 0.0
        return -np.inf

    # Define the full log-probability function
    def lnprob(theta, x, y, xerr, yerr):
        lp = lnprior(theta)
        if not np.isfinite(lp):
            return -np.inf
        return lp + lnlike(theta, x, y, xerr, yerr)

    # Set the MCMC walkers
    ndim = 2
    pos = [theta + 1e-4*np.random.randn(ndim) for i in range(nwalkers)]

    # Set and Run
    sampler = emcee.EnsembleSampler(nwalkers, ndim, lnprob, args=(x, y, xerr, yerr))
    sampler.run_mcmc(pos, nruns)

    # Get the samples. Do not get the first `cut` ones.
    samples = sampler.chain[:, cut:, :].reshape((-1, ndim))
    coeffs = np.array([get_a_b(bt) for bt in samples])

    # Get the a, b and their errors
    # The value of a and are the median of the
    # sampling distribution, while the lower and
    # upper values are the 16% and 84% percentiles.

    a_mcmc, b_mcmc = map(lambda v: (v[1], v[2]-v[1], v[1]-v[0]),
                         zip(*np.percentile(coeffs, [16, 50, 84], axis=0)))

    return (a_mcmc[0], b_mcmc[0]), ((a_mcmc[1], a_mcmc[2]),
                                    (b_mcmc[1], b_mcmc[2]))

def odr_linear_fit(x, y, xerr, yerr):
    """
    Obtained from Scipy ODR webpage with small modifications

    https://docs.scipy.org/doc/scipy/reference/odr.html
    """
    def f(B, x):
        '''Linear function y = m*x + b'''
        # B is a vector of the parameters.
        # x is an array of the current x values.
        # x is in the same format as the x passed to Data or RealData.
        #
        # Return an array in the same format as y passed to Data or RealData.
        return B[0]*x + B[1]

    # Create a model
    linear = Model(f)

    # Create a Data instance

    mydata = Data(x, y, wd=1./np.power(xerr,2), we=1./np.power(yerr,2))

    #Instantiate ODR with your data, model and initial parameter estimate.:
    beta0 = linregress(x, y)[:2]
    myodr = ODR(mydata, linear, beta0=beta0)

    #Run the fit.:
    myoutput = myodr.run()

    a, b = myoutput.beta
    a_error, b_error = myoutput.sd_beta

    return a, b, a_error, b_error


import matplotlib
matplotlib.use("TKagg")
import matplotlib.pyplot as plt
import seaborn as sns; sns.set()
import numpy as np

#Simple Linear Regression
#y =  ax + b

rng = np.random.RandomState(1)
x = 10 * rng.rand(50)
y = 2 * x - 5 + rng.randn(50)
plt.scatter(x, y);
plt.show()
plt.clf()

from sklearn.linear_model import LinearRegression
model = LinearRegression(fit_intercept=True)

model.fit(x[:, np.newaxis], y)

xfit = np.linspace(0,10,1000)
yfit = model.predict(xfit[:, np.newaxis])

plt.scatter(x,y)
plt.plot(xfit,yfit)
plt.show()
plt.clf()

#relvent info for the line
print("Model slope:    ", model.coef_[0])
print("Model intercept:", model.intercept_)

#we can do much more

rng = np.random.RandomState(1)
X = 10 * rng.rand(100, 3)
y = 0.5 + np.dot(X, [1.5, -2., 1.])

model.fit(X,y)
print(model.intercept_)
print(model.coef_)

#Basic Function Regression
##Polynomial basis functions

from sklearn.preprocessing import PolynomialFeatures
x = np.array([2,3,4])
poly = PolynomialFeatures(3, include_bias=False)
print(poly.fit_transform(x[:, None]))

from sklearn.pipeline import make_pipeline
poly_model = make_pipeline(PolynomialFeatures(7), LinearRegression())
#make a 7degree polynomial

#create a noisy sine wave
rng = np.random.RandomState(1)
x = 10 * rng.rand(50)
y = np.sin(x) + 0.1 * rng.randn(50)

poly_model.fit(x[:, np.newaxis], y)
yfit = poly_model.predict(xfit[:, np.newaxis])

plt.scatter(x, y)
plt.plot(xfit, yfit);
plt.show()
plt.clf()

#Example: predicting bicycle traffic

import pandas as pd
counts = pd.read_csv('data/FremontBridge.csv', index_col='Date', parse_dates=True)
weather = pd.read_csv('data/BicycleWeather.csv', index_col='DATE',
                      parse_dates=True)
#compute total daily traffic
daily = counts.resample('d').sum()
daily['Total'] = daily.sum(axis=1)
daily = daily[['Total']] # cut other columns
#add binary columns for days of week
days = ['Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun']
for i in range(7):
    daily[days[i]] = (daily.index.dayofweek == i).astype(float)

#account for holidays
from pandas.tseries.holiday import USFederalHolidayCalendar
cal = USFederalHolidayCalendar()
holidays = cal.holidays('2012', '2016')
daily = daily.join(pd.Series(1, index=holidays, name='holiday'))
daily['holiday'].fillna(0, inplace=True)

# add hours of daylight
def hours_of_daylight(date, axis=23.44, latitude=47.61):
    """Compute the hours of daylight for the given date"""
    days = (date - pd.datetime(2000, 12, 21)).days
    m = (1. - np.tan(np.radians(latitude))
         * np.tan(np.radians(axis) * np.cos(days * 2 * np.pi / 365.25)))
    return 24. * np.degrees(np.arccos(1 - np.clip(m, 0, 2))) / 180.

daily['daylight_hrs'] = list(map(hours_of_daylight, daily.index))
daily[['daylight_hrs']].plot()
plt.ylim(8, 17)
plt.show()
plt.clf()
#add temp and precip

# temperatures are in 1/10 deg C; convert to C
weather['TMIN'] /= 10
weather['TMAX'] /= 10
weather['Temp (C)'] = 0.5 * (weather['TMIN'] + weather['TMAX'])

# precip is in 1/10 mm; convert to inches
weather['PRCP'] /= 254
weather['dry day'] = (weather['PRCP'] == 0).astype(int)

daily = daily.join(weather[['PRCP', 'Temp (C)', 'dry day']])

#add year counter
daily['annual'] = (daily.index - daily.index[0]).days / 365.

print(daily.head())

#drop null rows
daily.dropna(axis=0, how='any', inplace=True)

column_names = ['Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun', 'holiday',
                'daylight_hrs', 'PRCP', 'dry day', 'Temp (C)', 'annual']
X = daily[column_names]
y = daily['Total']

model = LinearRegression(fit_intercept=False)
model.fit(X,y)
daily['predicted'] = model.predict(X)
daily[['Total', 'predicted']].plot(alpha=0.5);
plt.show()
plt.clf()

params = pd.Series(model.coef_, index=X.columns)

print(params)

from sklearn.utils import resample
np.random.seed(1)
err = np.std([model.fit(*resample(X,y)).coef_
              for i in range(1000)], 0)
print(pd.DataFrame({'effect': params.round(0),
                    'error': err.round(0)}))


import numpy as np
import matplotlib.pyplot as plt
from scipy.interpolate import UnivariateSpline

if __name__ == '__main__':
    N = 200
    step = 10
    NPulses = 6

    #Step 1: Generate a random 1D function
    np.random.seed(9)
    x = np.arange(0, N, step)
    y = np.random.randn(x.size)
    s = UnivariateSpline(x, y, s=1)
    xs = np.arange(N)
    y = s(xs)

    #Step 2: Generate a random set of impulses
    PIdx = np.random.permutation(N)[0:NPulses]
    idx = np.argsort(PIdx)
    PIdx = np.sort(PIdx)
    PMag = np.abs(np.random.randn(NPulses))
    PMag = PMag[idx]

    #Step 3: Plot Original function and impulse response

    #Step 4: Plot adding one impulse at a time
    R = np.max(np.abs(y))*np.max(np.abs(PMag))
    out = np.zeros(2*N)
    for p in range(len(PMag)):
        out[PIdx[p]:PIdx[p]+N] += PMag[p]*y
        plt.figure(figsize=[16, 10])
        plt.subplot(311)
        plt.plot(y)
        plt.xlim(0, 2*N)
        plt.ylim(-R, R)
        plt.xlabel('Sample Number')
        plt.ylabel('Amplitude')
        plt.title('Original Function')

        plt.subplot(312)
        plt.stem(PIdx, PMag, 'k')
        plt.plot([PIdx[p], PIdx[p]], [-R, R])
        plt.xlim(0, 2*N)
        plt.ylim(-R, R)
        plt.ylabel('Amplitude')
        plt.xlabel('Sample Number')

        plt.subplot(313)
        x = np.arange(len(y))
        for i in range(p+1):
            if (i == p):
                plt.plot(x+PIdx[i], y*PMag[i], 'b', linewidth=4.0)
            else:
                plt.plot(x+PIdx[i], y*PMag[i], 'k', linewidth=1.0)
        plt.plot([PIdx[p], PIdx[p]], [-R, R])
        plt.xlim(0, 2*N)
        plt.ylim(-R, R)
        plt.ylabel('Amplitude')
        plt.xlabel('Sample Number')

        plt.savefig("Conv%i.svg"%p, bbox_inches='tight')

    #Now plot the result
    plt.figure(figsize=[16, 10])
    plt.subplot(311)
    plt.plot(y)
    plt.xlim(0, 2*N)
    plt.ylim(-R, R)
    plt.xlabel('Sample Number')
    plt.ylabel('Amplitude')
    plt.title('Original Function')

    plt.subplot(312)
    plt.stem(PIdx, PMag, 'k')
    plt.xlim(0, 2*N)
    plt.ylim(-R, R)
    plt.ylabel('Amplitude')
    plt.xlabel('Sample Number')

    plt.subplot(313)
    plt.plot(out)
    plt.ylabel('Amplitude')
    plt.xlabel('Sample Number')
    plt.savefig("ConvResult.svg", bbox_inches='tight')

    #Convolution ground truth
#    plt.figure()
#    P = np.zeros(2*N)
#    P[PIdx] = PMag
#    plt.plot(np.convolve(y, P))
#    plt.show()


import seaborn as sns
from scipy.stats import uniform
from matplotlib import pyplot as plt

def uniformfig(size=1000, start=10, width=20, fig_w=8, fig_l=8, grid=True, color='skyblue', linewidth=10,
               alpha=1, hist=True, xlabel='Uniform Distribution', ylabel='Frequency',
               title='Uniform Distribution', legend_size=12, title_size=20, label_size=16,
               tick_size=12):
    """

    Uniform Distribution

    parameters
    ----------

    size: The Number of random variables. The default value is 1000.
    start: The distribution is uniform on [start, start+width]. The default value is 10.
    width: The distribution is uniform on [start, start+width]. The default value is 20.
    fig_w: Matplotlib `figsize` width, the default value is 8.
    fig_l: Matplotlib `figsize` length, the default value is 8.
    grid: Use 'True' or 'False' to show the grid. The default value is True.
    hist: To show Histogram, use 'True' or 'False'. The default value is True.
    color: The color to fill. The default value is 'skyblue'.
    linewidth: The line width. The default value is 10.
    alpha: The alpha(transparency) value for filling color. The default value is 1.
    title: The figure's title. The default value is 'Chi-Square Distribution'
    xlabel: The x-axis label. The default value is 'Value'.
    ylabel: The y-axis label. The default value is 'Frequency'.
    legend_size: The legend font size. The default value is 12.
    title_size: Title font size. The default value is 20.
    label_size: Label font size. The default value is 16.
    tick_size: x and y axis tick font size. The default value is 12.

    Example
    -------
    import statsfig as sf

    sf.uniformfig()

    sf.uniformfig(color='g', grid=False, fig_w=5, fig_l=4)

    """

    plt.figure(figsize=(fig_w, fig_l))
    data_uniform = uniform.rvs(size=size, loc=start, scale=width)

    label = 'size={}, start={}, width={}'.format(size, start, width)
    ax = sns.distplot(data_uniform,
                      bins=100,
                      kde=True,
                      color=color,
                      hist_kws={"linewidth": linewidth, 'alpha': alpha},
                      hist=hist,
                      label=label
                      )

    ax.set_title(title)
    ax.set(xlabel=xlabel, ylabel=ylabel)

    ax.legend(fontsize=legend_size)

    plt.rc('axes', titlesize=title_size)    # fontsize of the axes title
    plt.rc('axes', labelsize=label_size)    # fontsize of the x and y labels
    plt.rc('xtick', labelsize=tick_size)    # fontsize of the tick labels
    plt.rc('ytick', labelsize=tick_size)    # fontsize of the tick labels

    ax.grid(grid)

    plt.show()


import numpy as np

def mean_square_error(y_true: np.ndarray, y_pred: np.ndarray) -> float:
    """
    Calculate MSE loss

    Parameters
    ----------
    y_true: ndarray of shape (n_samples, )
        True response values
    y_pred: ndarray of shape (n_samples, )
        Predicted response values

    Returns
    -------
    MSE of given predictions
    """

    mse = np.mean(np.square(y_pred - y_true))
    return mse

def misclassification_error(y_true: np.ndarray, y_pred: np.ndarray, normalize: bool = True) -> float:
    """
    Calculate misclassification loss

    Parameters
    ----------
    y_true: ndarray of shape (n_samples, )
        True response values
    y_pred: ndarray of shape (n_samples, )
        Predicted response values
    normalize: bool, default = True
        Normalize by number of samples or not

    Returns
    -------
    Misclassification of given predictions
    """
    wrong_count = (y_pred != y_true).astype(int).sum()
    if normalize:
        return wrong_count/y_true.size
    return wrong_count

def accuracy(y_true: np.ndarray, y_pred: np.ndarray) -> float:
    """
    Calculate accuracy of given predictions

    Parameters
    ----------
    y_true: ndarray of shape (n_samples, )
        True response values
    y_pred: ndarray of shape (n_samples, )
        Predicted response values

    Returns
    -------
    Accuracy of given predictions
    """
    return (y_pred == y_true).astype(int).sum() / y_true.size

def cross_entropy(y_true: np.ndarray, y_pred: np.ndarray) -> float:
    """
    Calculate the cross entropy of given predictions

    Parameters
    ----------
    y_true: ndarray of shape (n_samples, )
        True response values
    y_pred: ndarray of shape (n_samples, )
        Predicted response values

    Returns
    -------
    Cross entropy of given predictions
    """
    raise NotImplementedError()

if __name__ == '__main__':
    pass


# SVM for regression in 1d
# Code is based on
# https://github.com/ageron/handson-ml2/blob/master/05_support_vector_machines.ipynb

import numpy as np
import matplotlib.pyplot as plt

from sklearn.pipeline import Pipeline
from sklearn.preprocessing import PolynomialFeatures
from sklearn.preprocessing import StandardScaler

from sklearn.svm import LinearSVR
from sklearn.svm import SVR

def plot_svm_regression(svm_reg, X, y, axes):
    x1s = np.linspace(axes[0], axes[1], 100).reshape(100, 1)
    y_pred = svm_reg.predict(x1s)
    plt.plot(x1s, y_pred, "k-", linewidth=2, label=r"$\hat{y}$")
    plt.plot(x1s, y_pred + svm_reg.epsilon, "k--")
    plt.plot(x1s, y_pred - svm_reg.epsilon, "k--")
    plt.scatter(X[svm_reg.support_], y[svm_reg.support_], s=180, facecolors='#FFAAAA')
    plt.plot(X, y, "bo")
    plt.xlabel(r"$x_1$", fontsize=18)
    #plt.legend(loc="upper left", fontsize=18)
    plt.axis(axes)

np.random.seed(42)
m = 100
X = 2 * np.random.rand(m, 1) - 1
#y = (0.2 + 0.1 * X + 0.5 * X**2 + np.random.randn(m, 1)/10).ravel()
y = (0.2 + 0.1 * X + 0.5 * X**2 + 0.1 *  X**3 + np.random.randn(m, 1)/10).ravel()

epsilons = [0.1, 0.05]
eps_names = ['0p1', '0p05']
for i, eps in enumerate(epsilons):
    #svm_poly_reg1 = SVR(kernel="poly", degree=5, C=1e3, epsilon=eps, gamma="scale")
    #svm_poly_reg2 = SVR(kernel="poly", degree=5, C=1e-3, epsilon=eps, gamma="scale")
    svm_reg1 = SVR(kernel="rbf", gamma=1, C=100, epsilon=eps)
    svm_reg2 = SVR(kernel="rbf", gamma=1, C=0.01, epsilon=eps)

    svm_reg1.fit(X, y)
    svm_reg2.fit(X, y)

    fig, axes = plt.subplots(ncols=2, figsize=(9, 4), sharey=True)
    plt.sca(axes[0])
    plot_svm_regression(svm_reg1, X, y, [-1, 1, 0, 1])
    plt.title(r"$C={}, \epsilon = {}$".format(svm_reg1.C, svm_reg1.epsilon), fontsize=18)
    plt.ylabel(r"$y$", fontsize=18, rotation=0)
    plt.sca(axes[1])
    plot_svm_regression(svm_reg2, X, y, [-1, 1, 0, 1])
    plt.title(r"$C={}, \epsilon = {}$".format(svm_reg2.C, svm_reg2.epsilon), fontsize=18)
    fname = '../figures/svm_regression_e{}.pdf'.format(eps_names[i])
    plt.savefig(fname, dpi=300)
    plt.show()


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Thu Sep 26 07:25:54 2019

@author: kenneth
"""
from __future__ import absolute_import
import numpy as np
from Utils.utils import EvalR

class Ridge(EvalR):
    '''Docstring
    Ridge regression via Closed form
    \beta := (XtX + \lamda * I)XtY
    --------------------------------------
    Regularization via Ridge regression.
    l2-norm reduces the risk of overfitting
    here but does not induce sparsity
    '''
    def __init__(self, lamda = None):
        super().__init__()
        self.lamda = lamda
        if not lamda:
            lamda = 0.01
            self.lamda = lamda
        else:
            self.lamda = lamda
        return

    def fit(self, X, Y):
        self.X = X
        self.Y = Y
        #--Closed form
        self.beta = np.linalg.solve(self.lamda*np.eye(self.X.shape[1]) + self.X.T.dot(self.X), self.X.T.dot(self.Y))
        return self

    def predict(self, X):
        Y_hat = X.dot(self.beta)
        return Y_hat

class RidgeGD(EvalR):
    '''
    Ridge regression via Gradient Descent
    '''
    def __init__(self, lamda = None):
        super().__init__()
        self.lamda = lamda
        if not lamda:
            lamda = 0.01
            self.lamda = lamda
        else:
            self.lamda = lamda
        return

    def cost(self, X, Y, beta):
        '''
        param: X = training examples/data. column vector <x1, x2, ...., xn | x E R^D>
        param: Y = target. vector  <y | y E R^DX1>
        param: beta = coefficients, e.g b0, b1, ..., bn
        Return: cost
        '''
        return (1/2*len(Y)) * (np.sum(np.square(X.dot(beta) - Y)) + (self.lamda*np.sum(np.square(beta))))

    def RGD(self, X, Y, beta, alpha, iterations):
        '''
        param: X = NxD feature matrix
        param: Y = Dx1 column vector
        param: beta = Dx1 beta vector coefficients
        param: alpha = learning rate. Default 1e-2

        Return type; final beta/coefficients, cost and bata iterations
        '''
        self.beta = beta
        self.cost_rec = np.zeros(iterations)
        self.beta_rec = np.zeros((iterations, X.shape[1]))
        for self.iter in range(iterations):
            #compute gradient
            self.beta = self.beta - (1/len(Y)) *(alpha) * ((np.dot(X.T, (np.dot(X, self.beta) - Y))) + (self.lamda*self.beta))
            self.beta_rec[self.iter, :] = self.beta.T
            self.cost_rec[self.iter] = self.cost(X, Y, self.beta)
            print('*'*40)
            print('%s iteratiion, cost = %s'%(self.iter, self.cost_rec[self.iter]))
        print('*'*40)
        return self

    def predict(self, X):
        '''
        param: X_test = NxD feature matrix
        '''
        return X.dot(self.beta)

    def plot_cost(self, cost, iter_):
        import matplotlib.pyplot as plt
        plt.plot(np.arange(iter_), self.cost)
        plt.title('Cost vs Number of iteration')
        plt.xlabel('Iteration')
        plt.ylabel('Cost')
        plt.show()

#%% Testing

from sklearn.datasets import load_boston
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import Normalizer

X, y = load_boston().data, load_boston().target
X = Normalizer().fit_transform(X)
X_train, X_test, Y_train, Y_test = train_test_split(X, y, test_size = .3)
ridge = Ridge().fit(X_train, Y_train)
ridge.summary(X, Y_test, ridge.predict(X_test))

#%% visualize coefficients in no particular order
import matplotlib.pyplot as plt
import pandas as pd
pd.Series(ridge.beta).plot(kind = 'bar')


# -*- coding: utf-8 -*-
"""
Created on Mon Feb  7 12:43:29 2022

@author: Akshatha
"""
import numpy as np
import time

def time_it(func):
    def wrapper(*args, **kwargs):
        start = time.time()
        result = func(*args,**kwargs)
        end = time.time()
        print(func.__name__ +" took " + str((end-start)*1000) + " mil sec")
        return result
    return wrapper

@time_it
def linear_search(arr, num):
    for idx,element in enumerate(arr):
        if num == element:
            return idx
    return -1

@time_it
def binary_search(arr, num):
    low_idx = 0
    high_idx = len(arr)-1
    while low_idx <= high_idx:
        mid_idx = (high_idx+low_idx)//2
        if arr[mid_idx] == num:
            return mid_idx
        if num > arr[mid_idx]:
            low_idx = mid_idx + 1
        else:
            high_idx = mid_idx - 1
    return -1

# try 1 - doesn't handle invalid input
def binary_search_recursion(arr, num):
    low_idx = 0
    high_idx = len(arr)-1
    mid_idx = (high_idx+low_idx)//2
    # if high_idx <= low_idx:
    #     return -1
    if arr[mid_idx] == num:
        return mid_idx
    elif num > arr[mid_idx]:
        return mid_idx + 1+binary_search_recursion(arr[mid_idx+1:], num)
    else:
        return binary_search_recursion(arr[:mid_idx], num)

def binary_search_recursive(arr,num,low_idx,high_idx):
    mid_idx = (high_idx+low_idx)//2
    if arr[mid_idx] == num:
        return mid_idx
    if high_idx <= low_idx:
        return -1
    if num > arr[mid_idx]:
        low_idx = mid_idx + 1
    else:
        high_idx = mid_idx - 1
    return binary_search_recursive(arr,num,low_idx,high_idx)

def find_all_occurences(arr, num):
    index = binary_search(arr, num)
    list_idx = [index]

    i = index-1
    while i >=0:
        if arr[i] == num:
            list_idx.append(i)
        else:
            break
        i = i-1

    i = index +1
    while i < len(arr):
        if arr[i] == num:
            list_idx.append(i)
        else:
            break
        i = i+1

    return sorted(list_idx)

if __name__ == '__main__':
    sorted_arr = range(100001)
    num_to_find = 100000
    index = linear_search(sorted_arr, num_to_find)
    print('linear search output:',index)
    index = binary_search(sorted_arr, num_to_find)
    print('binary search output:',index)
    index = binary_search_recursive(sorted_arr,num_to_find,0,len(sorted_arr)-1)
    print('binary search through recursion output:',index)
    # doesn't handle invalid input
    index = binary_search_recursion(sorted_arr, num_to_find)
    print('binary search through recursion output:',index)

    arr = [1,3,4,5,8,10,12,14,16,16,16,17,18,20,21,24]
    print('all occurences:',find_all_occurences(arr,16))