#############################################################################
### Python ODE solver, using the Runge-Kutta 4th order method
### Solves ordinary differential equations of the form (dy/dx) = f(x,y)
###
### User must input the following parameters:
### 1. Via command line arguments: time step, total time, output interval
### 2. The right-hand side of the differential equation
###    inside the rhs function, in the rungekutta.py file.
### 3. Initial conditions x_0, y_0 (example: y(x_0) = y_0),
###    at the beginning of the code, right below the imports
###
### Example: python ODE.py 0.1 100 10
### This will solve the ODE with a timestep of dt = 0.1, for 100 time units,
### writing output every 10 time steps.
###
### Output: File "RK4_output.txt", also plots results.
###
### Author: Nikos Tryfonidis, November 2015
### The MIT License (MIT)
### Copyright (c) 2015 Nikos Tryfonidis
### See LICENSE.txt
#############################################################################

# Import necessary packages
import numpy as np
import sys

# Import runge kutta and right-hand-side function
# from rungekutta.py
from rungekutta import RK4, rhs
from plot1D import plot1D

##### Set Initial Conditions Here: #####
x_0 = 0
y_0 = 1
########################################

#################################################################################
### Command line arguments:
#################################################################################

# Check number of command line arguments
if len(sys.argv) != 4:
	print "Usage: python ODE.py <timestep> <total time> <output interval>"
	print "Please run again following the command line input format above."
	print "Exiting..."
	sys.exit(1)

# Get command line arguments
h = float(sys.argv[1])
totalTime = float(sys.argv[2])
interval = int(sys.argv[3])

#################################################################################

# Calculate number of time steps
totalSteps = totalTime/h
# Find size of output
size = int(totalSteps/interval) + 1

# Print a summary of parameters
print "Running Runge-Kutta with the following parameters:"
print "Time step: %f\tTotal time: %f\t" %(h, totalTime)
print "Output interval: %d" %interval

# Initialize variable arrays:
x = np.linspace(x_0, x_0 + totalTime, size)
y = np.zeros(size)
# Initial y value:
y[0] = y_0

# Call Runge Kutta
xOld = x[0]
yOld = y[0]
for t in xrange(1, size):

    for i in xrange(0, interval):
        xOld = xOld + h
        yOld = RK4(xOld, yOld, h, rhs)

    y[t] = yOld

# Save x and y into output file (each array is a column)
np.savetxt('RK4_output.txt', np.c_[x, y], fmt='%.10f')

# Plot x, y (plot1d function in plot1D.py)
plot1D(size, "RK4_output.txt")


import numpy as np

def naive_relu(x):
    # X must be 2D Numpy tensor
    assert len(x.shape) == 2
    # Avoid overwriting the input tensor
    x = x.copy()

    for row in range(x.shape[0]):
        for col in range(x.shape[1]):
            x[row, col] = max(x[row, col], 0)

    return x

def naive_add(x, y):
    # X must be 2D Numpy tensor
    assert len(x.shape) == 2
    assert x.shape == y.shape
    # Avoid overwriting the input tensor
    x = x.copy()

    for row in range(x.shape[0]):
        for col in range(x.shape[1]):
            x[row, col] += y[row, col]

    return x

def naive_add_matrix_and_vector(x, y):
    # X must be 2D Numpy tensor
    assert len(x.shape) == 2
    # Y must be Numpy vector
    assert len(y.shape) == 1
    assert x.shape[1] == y.shape[0]
    # Avoid overwriting the input tensor
    x = x.copy()

    for row in range(x.shape[0]):
        for col in range(x.shape[1]):
            x[row, col] = y[col]

    return x

def naive_vector_dot(x, y):
    assert len(x.shape) == 1
    assert len(y.shape) == 1
    assert x.shape[0] == y.shape[0]

    z = 0.

    for row in range(x.shape[0]):
        z += x[row] * y[row]

    return z

def naive_matrix_vector_dot(x, y):
    assert len(x.shape) == 2
    assert len(y.shape) == 1
    assert x.shape[1] == y.shape[0]

    z = np.zeros(x.shape[0])

    for row in range(x.shape[0]):
        for col in range(x.shape[1]):
            z[row] += x[row, col] * y[col]

    return z

def naive_matrix_dot(x, y):
    assert len(x.shape) == 2
    assert len(y.shape) == 2
    assert x.shape[1] == y.shape[0]

    z = np.zeros((x.shape[0], y.shape[1]))

    for row in range(x.shape[0]):
        for col in range(y.shape[1]):
            row_x = x[row, :]
            column_y = y[:, col]
            z[row, col] = naive_vector_dot(row_x, column_y)

    return z

# 2D Numpy tensor
print('*** Naive ReLU ***')
input_x_2d_tensor = np.array(
    [
        [5, 78, 2, 34, 0],
        [6, 79, 3, 35, 1],
        [7, 80, 4, 36, 2]
    ]
)
print('Input 2D tensor shape: ', input_x_2d_tensor.shape)
print('Input 2D tensor ndim: ', input_x_2d_tensor.ndim)
output_2d_tensor = naive_relu(input_x_2d_tensor)
print('ReLU output: ', output_2d_tensor)
print('*** ---------- ***')
print()
print('*** Naive ADD operation ***')
input_y_2d_tensor = np.array(
    [
        [1, 7, 22, 31, 0],
        [3, 9, 33, 35, 1],
        [4, 8, 44, 36, 2]
    ]
)
output_2d_tensor = naive_add(input_x_2d_tensor, input_y_2d_tensor)
print('Naive ADD output: ', output_2d_tensor)
print('*** ---------- ***')
print()
print('*** Naive ADD operation - Matrix and Vector ***')
input_x_2d_tensor = np.array(
    [
        [1, 2, 3, 4],
        [5, 6, 7, 8]
    ]
)
input_y_2d_tensor = np.array([9, 10, 11, 12])
output_2d_tensor = naive_add_matrix_and_vector(input_x_2d_tensor, input_y_2d_tensor)
print('Naive ADD operation - Matrix and Vector output: ', output_2d_tensor)
print('*** ---------- ***')
print()
print('*** Naive DOT operation - Vector ***')
x_vector = np.array([1, 2, 3])
y_vector = np.array([4, 5, 6])
dot_product = naive_vector_dot(x_vector, y_vector)
print('Naive DOT output: ', dot_product)
print('*** ---------- ***')
print()
print('*** Naive DOT operation - Matrix and Vector ***')
x_matrix = np.array(
    [
        [1, 2, 3, 4],
        [5, 6, 7, 8]
    ]
)
y_vector = np.array([1, 2, 1, 2])
dot_product = naive_matrix_vector_dot(x_matrix, y_vector)
print('Naive DOT operation - Matrix and Vector output: ', dot_product)
print('*** ---------- ***')
print()
print('*** Naive DOT operation - Matrix ***')
x_matrix = np.array(
    [
        [1, 2, 3, 4],
        [5, 6, 7, 8]
    ]
)
y_matrix = np.array(
    [
        [1, 1, 1, 1],
        [0, 0, 0, 0],
        [1, 1, 1, 1],
        [0, 0, 0, 0]
    ]
)
dot_product = naive_matrix_dot(x_matrix, y_matrix)
print('Naive DOT operation - Matrix: ', dot_product)
print('*** ---------- ***')
print()


from PIL import Image as I
import numpy as np

from skimage.util import random_noise as imnoise
from scipy.misc import imresize

def whiten(img):
    """
	Function that takes as input one grayscale image array and returns a whitenend image. It applies
	the following transform:

	.. math::

		I = \frac{I - mean(I)}{var(I)} + min(\frac{I - mean(I)}{var(I)})

	Args:
		img: ``numpy ndarray`` of a two-dimensional image  of the form
									< height, width >

	Returns:
		numpy ndarray: gray
	"""
    img = (img - img.mean()) / img.std()
    img = img + img.min()
    return img

def normalize(img):
    """
	Function that takes as input one grayscale image array and returns a normalized image. It applies
	the following transform:

	.. math::

		I = \frac{I}{max(I)}

	Args:
		img: ``numpy ndarray`` of a two-dimensional image  of the form
									< height, width >

	Returns:
		numpy ndarray: gray
	"""
    return img / img.max()

def imread(infile):
    """
    Takes a string as input and outputs a image.

    Args:
        infile: input file string

    Returns:
        numpy.ndarray: image
    """
    img = I.open(infile)
    img.load()
    data = np.asarray(img, dtype="uint8")
    return data

def imwrite(img, outfile):
    """
    Takes an image and string as input and and writes the image down.

    Args:
        img: image numpy.ndarray
        outfile: output file string
    """
    if not img.dtype == 'uint8':
        np.clip(img, a_min=0, a_max=255, out=img)
        img = np.asarray(img, dtype='uint8')
    img = I.fromarray(img)
    img.save(outfile)

def imshow(img, window='image'):
    """
    Takes an image as input and displays it.

    Args:
        img: image numpy.ndarray
        window: (optional) title to name the window by.
    """
    I.fromarray(img).show(title=window)

def rgb2gray(rgb):
    """
	Function that takes as input one rgb image array and returns a grayscale image. It applies
	the following transform:

	.. math::

		I_{gray} = 0.2989I_r + 0.5870I_g + 0.1140I_b

	Args:
		rgb: ``numpy ndarray`` of a four-dimensional image batch of the form
									< (number of images??), height, width, channels>

	Returns:
		numpy ndarray: gray
	"""
    if len(rgb.shape) == 4:
        r, g, b = rgb[:, :, :, 0], rgb[:, :, :, 1], rgb[:, :, :, 2]
    elif len(rgb.shape) == 3:
        r, g, b = rgb[:, :, 0], rgb[:, :, 1], rgb[:, :, 2]
    gray = 0.2989 * r + 0.5870 * g + 0.1140 * b
    return gray

if __name__ == '__main__':
    pass


# %% [markdown]
# # Linear regression with non-linear link between data and target
#
# In the previous exercise, you were asked to train a linear regression model
# on a dataset where the matrix `data` and the vector `target` do not have a
# linear link.
#
# In this notebook, we show that even if the parametrization of linear models
# is not natively adapated to data with non-linearity, it is still possible
# to make linear model more flexible and expressive.
#
# To illustrate these concepts, we will reuse the same dataset generated in the
# previous exercise.

# %%
import numpy as np

rng = np.random.RandomState(0)

n_sample = 100
data_max, data_min = 1.4, -1.4
len_data = (data_max - data_min)
# sort the data to make plotting easier later
data = np.sort(rng.rand(n_sample) * len_data - len_data / 2)
noise = rng.randn(n_sample) * .3
target = data ** 3 - 0.5 * data ** 2 + noise

# %% [markdown]
# ```{note}
# To ease the plotting, we will create a Pandas dataframe containing the data
# and target
# ```

# %%
import pandas as pd
full_data = pd.DataFrame({"data": data, "target": target})

# %%
import seaborn as sns

_ = sns.scatterplot(data=full_data, x="data", y="target")
# %% [markdown]
# We will highlight the limitations of fitting a linear regression model as
# done in the previous exercise.
#
# ```{warning}
# In scikit-learn, by convention `data` (also called `X` in the scikit-learn
# documentation) should be a 2D matrix of shape `(n_samples, n_features)`.
# If `data` is a 1D vector, you need to reshape it into a matrix with a
# single column if the vector represents a feature or a single row if the
# vector represents a sample.
# ```

# %%
from sklearn.linear_model import LinearRegression
from sklearn.metrics import mean_squared_error

linear_regression = LinearRegression()
# X should be 2D for sklearn
data_2d = data.reshape((-1, 1))
linear_regression.fit(data_2d, target)
target_predicted = linear_regression.predict(data_2d)

# %%
mse = mean_squared_error(target, target_predicted)

# %%
ax = sns.scatterplot(data=full_data, x="data", y="target")
ax.plot(data, target_predicted, color="tab:orange")
_ = ax.set_title(f"Mean squared error = {mse:.2f}")

# %% [markdown]
# Here the coefficients learnt by `LinearRegression` is the best "straight
# line" that fits the data. We can inspect the coefficients using the
# attributes of the model learnt as follows:

# %%
print(f"weight: {linear_regression.coef_[0]:.2f}, "
      f"intercept: {linear_regression.intercept_:.2f}")

# %% [markdown]
# It is important to note that the model learnt will not be able to handle the
# non-linear relationship between `data` and `target` since linear models
# assume the relationship between `data` and `target` to be linear.
#
# Indeed, there are 3 possibilities to solve this issue:
#
# 1. choose a model that natively can deal with non-linearity,
# 2. "augment" features by including expert knowledge which can be used by
#    the model, or
# 3. use a "kernel" to have a locally-based decision function instead of a
#    global linear decision function.
#
# Let's illustrate quickly the first point by using a decision tree regressor
# which can natively handle non-linearity.

# %%
from sklearn.tree import DecisionTreeRegressor

tree = DecisionTreeRegressor(max_depth=3).fit(data_2d, target)
target_predicted = tree.predict(data_2d)
mse = mean_squared_error(target, target_predicted)

# %%
ax = sns.scatterplot(data=full_data, x="data", y="target")
ax.plot(data, target_predicted, color="tab:orange")
_ = ax.set_title(f"Mean squared error = {mse:.2f}")

# %% [markdown]
# Instead of having a model which can natively deal with non-linearity, we
# could also modify our data: we could create new features, derived from the
# original features, using some expert knowledge. In this example, we know that
# we have a cubic and squared relationship between `data` and `target` (because
# we generated the data). Indeed, we could create two new features (`data ** 2`
# and `data ** 3`) using this information.

# %%
data_augmented = np.concatenate([data_2d, data_2d ** 2, data_2d ** 3], axis=1)

linear_regression.fit(data_augmented, target)
target_predicted = linear_regression.predict(data_augmented)
mse = mean_squared_error(target, target_predicted)

# %%
ax = sns.scatterplot(data=full_data, x="data", y="target")
ax.plot(data, target_predicted, color="tab:orange")
_ = ax.set_title(f"Mean squared error = {mse:.2f}")

# %% [markdown]
# We can see that even with a linear model, we can overcome the linearity
# limitation of the model by adding the non-linear component into the design of
# additional features. Here, we created new features by knowing the way the
# target was generated. In practice, this is usually not the case.
#
# Instead, one is usually creating interaction between features (e.g. $x_1
# \times x_2$) with different orders (e.g. $x_1, x_1^2, x_1^3$), at the risk of
# creating a model with too much flexibility where the polynomial terms allows
# to fit noise in the dataset and thus lead overfit. In scikit-learn, the
# `PolynomialFeatures` is a transformer to create such feature interactions
# which we could have used instead of manually creating new features.
#
# To demonstrate `PolynomialFeatures`, we are going to use a scikit-learn
# pipeline which will first create the new features and then fit the model.
# We come back to scikit-learn pipelines and discuss them in more detail later.

# %%
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import PolynomialFeatures

model = make_pipeline(PolynomialFeatures(degree=3),
                      LinearRegression())
model.fit(data_2d, target)
target_predicted = model.predict(data_2d)
mse = mean_squared_error(target, target_predicted)

# %%
ax = sns.scatterplot(data=full_data, x="data", y="target")
ax.plot(data, target_predicted, color="tab:orange")
_ = ax.set_title(f"Mean squared error = {mse:.2f}")

# %% [markdown]
# Thus, we saw that `PolynomialFeatures` is actually doing the same
# operation that we did manually above.
#
# The last possibility is to make a linear model more expressive is to use a
# "kernel". Instead of learning a weight per feature as we previously
# emphasized, a weight will be assign by sample instead. However, not all
# samples will be used. This is the base of the support vector machine
# algorithm.

# %%
from sklearn.svm import SVR

svr = SVR(kernel="linear")
svr.fit(data_2d, target)
target_predicted = svr.predict(data_2d)
mse = mean_squared_error(target, target_predicted)

# %%
ax = sns.scatterplot(data=full_data, x="data", y="target")
ax.plot(data, target_predicted, color="tab:orange")
_ = ax.set_title(f"Mean squared error = {mse:.2f}")

# %% [markdown]
# The algorithm can be modified so that it can use non-linear kernel. Then,
# it will compute interaction between samples using this non-linear
# interaction.

# %%
svr = SVR(kernel="poly", degree=3)
svr.fit(data_2d, target)
target_predicted = svr.predict(data_2d)
mse = mean_squared_error(target, target_predicted)

# %%
ax = sns.scatterplot(data=full_data, x="data", y="target")
ax.plot(data, target_predicted, color="tab:orange")
_ = ax.set_title(f"Mean squared error = {mse:.2f}")

# %% [markdown]
# A method supporting kernel, as SVM, allows to efficiently create a non-linear
# model.


import numpy as np
import matplotlib.pyplot as plt

def feval(funcName, *args):
    return eval(funcName)(*args)

def RungeKutta4thOrder(func, yinit, xspan, h):
    m = len(yinit)
    n = int((xspan[-1] - xspan[0]) / h)

    x = xspan[0]
    y = yinit

    xsol = np.empty((0))
    xsol = np.append(xsol, x)

    ysol = np.empty((0))
    ysol = np.append(ysol, y)

    for i in range(n):
        k1 = feval(func, x, y)

        yp2 = y + k1*(h/2)

        k2 = feval(func, x+h/2, yp2)

        yp3 = y + k2*(h/2)

        k3 = feval(func, x+h/2, yp3)

        yp4 = y + k3*h

        k4 = feval(func, x+h, yp4)

        for j in range(m):
            y[j] = y[j] + (h/6)*(k1[j] + 2*k2[j] + 2*k3[j] + k4[j])

        x = x + h
        xsol = np.append(xsol, x)

        for r in range(len(y)):
            ysol = np.append(ysol, y[r])  # Save all new y's

    return [xsol, ysol]

def ABM4thOrder(func, yinit, xspan, h):

    m = len(yinit)

    dx = int((xspan[-1] - xspan[0]) / h)

    xrk = [xspan[0] + k * h for k in range(dx + 1)]

    [xx, yy] = RungeKutta4thOrder(func, yinit, (xrk[0], xrk[3]), h)

    x = xx
    xsol = np.empty(0)
    xsol = np.append(xsol, x)

    y = yy
    yn = np.array([yy[0]])
    ysol = np.empty(0)
    ysol = np.append(ysol, y)

    for i in range(3, dx):
        x00 = x[i]; x11 = x[i-1]; x22 = x[i-2]; x33 = x[i-3]; xpp = x[i]+h

        y00 = np.array([y[i]])
        y11 = np.array([y[i - 1]])
        y22 = np.array([y[i - 2]])
        y33 = np.array([y[i - 3]])

        y0prime = feval(func, x00, y00)
        y1prime = feval(func, x11, y11)
        y2prime = feval(func, x22, y22)
        y3prime = feval(func, x33, y33)

        ypredictor = y00 + (h/24)*(55*y0prime - 59*y1prime + 37*y2prime - 9*y3prime)
        ypp = feval(func, xpp, ypredictor)

        for j in range(m):
            yn[j] = y00[j] + (h/24)*(9*ypp[j] + 19*y0prime[j] - 5*y1prime[j] + y2prime[j])

        xs = x[i] + h
        xsol = np.append(xsol, xs)

        x = xsol

        for r in range(len(yn)):
            ysol = np.append(ysol, yn)

        y = ysol

    return [xsol, ysol]

def myFunc(x, y):
    dy = np.zeros((len(y)))
    dy[0] = (1/2)*(x - y[0])
    return dy

h = 0.2
xspan = np.array([0.0, 3.0])
yinit = np.array([1.0])

[ts, ys] = ABM4thOrder('myFunc', yinit, xspan, h)

dt = int((xspan[-1]-xspan[0])/h)
t = [xspan[0]+i*h for i in range(dt+1)]
yexact = []
for i in range(dt+1):
    ye = t[i] - 2 + 3*np.exp(-0.5*t[i])
    yexact.append(ye)

diff = ys - yexact
print("Maximum difference =", np.max(abs(diff)))
plt.plot(ts, ys, 'r')
plt.plot(t, yexact, 'b')
plt.xlim(xspan[0], xspan[1])
plt.legend(["ABM 4th Order ", "Exact solution"], loc=2)
plt.xlabel('x', fontsize=17)
plt.ylabel('y', fontsize=17)
plt.tight_layout()
plt.show()


from __future__ import annotations

import math

import numpy
import numpy as np
from numpy.linalg import inv, det, slogdet

class UnivariateGaussian:
    """
    Class for univariate Gaussian Distribution Estimator
    """
    def __init__(self, biased_var: bool = False) -> UnivariateGaussian:
        """
        Estimator for univariate Gaussian mean and variance parameters

        Parameters
        ----------
        biased_var : bool, default=False
            Should fitted estimator of variance be a biased or unbiased estimator

        Attributes
        ----------
        fitted_ : bool
            Initialized as false indicating current estimator instance has not been fitted.
            To be set as True in `UnivariateGaussian.fit` function.

        mu_: float
            Estimated expectation initialized as None. To be set in `UnivariateGaussian.fit`
            function.

        var_: float
            Estimated variance initialized as None. To be set in `UnivariateGaussian.fit`
            function.
        """
        self.biased_ = biased_var
        self.fitted_, self.mu_, self.var_ = False, None, None

    def fit(self, X: np.ndarray) -> UnivariateGaussian:
        """
        Estimate Gaussian expectation and variance from given samples

        Parameters
        ----------
        X: ndarray of shape (n_samples, )
            Training data

        Returns
        -------
        self : returns an instance of self.

        Notes
        -----
        Sets `self.mu_`, `self.var_` attributes according to calculated estimation (where
        estimator is either biased or unbiased). Then sets `self.fitted_` attribute to `True`
        """
        self.mu_ = np.mean(X)

        # Now calculate the expected variance based on whether the sample is biased or not
        the_divisor = X.size if self.biased_ == True else X.size - 1
        val_minus_mu_squared = lambda t: (t - self.mu_)**2
        sum_of_values_minus_mu_squared = 0
        for val in X:
            sum_of_values_minus_mu_squared += val_minus_mu_squared(val)
        self.var_ = (1/the_divisor)*sum_of_values_minus_mu_squared

        self.fitted_ = True
        return self

    def pdf(self, X: np.ndarray) -> np.ndarray:
        """
        Calculate PDF of observations under Gaussian model with fitted estimators

        Parameters
        ----------
        X: ndarray of shape (n_samples, )
            Samples to calculate PDF for

        Returns
        -------
        pdfs: ndarray of shape (n_samples, )
            Calculated values of given samples for PDF function of N(mu_, var_)

        Raises
        ------
        ValueError: In case function was called prior fitting the model
        """
        if not self.fitted_:
            raise ValueError("Estimator must first be fitted before calling `pdf` function")

        calc_pdf = lambda t: (1/math.sqrt(2*math.pi*self.var_))*math.exp((-1/2*self.var_)*((t - self.mu_)**2))

        # Apply the lambda function over all the sample values
        map_pdf = np.vectorize(calc_pdf)
        return map_pdf(X)

    @staticmethod
    def log_likelihood(mu: float, sigma: float, X: np.ndarray) -> float:
        """
        Calculate the log-likelihood of the data under a specified Gaussian model

        Parameters
        ----------
        mu : float
            Expectation of Gaussian
        sigma : float
            Variance of Gaussian
        X : ndarray of shape (n_samples, )
            Samples to calculate log-likelihood with

        Returns
        -------
        log_likelihood: float
            log-likelihood calculated
        """
        sum_of_vars_minus_mu_squared = 0
        for var in X:
            sum_of_vars_minus_mu_squared += (var-mu)**2

        return ((-X.size)/2)*math.log(2*math.pi*sigma)-(1/2)*sum_of_vars_minus_mu_squared

class MultivariateGaussian:
    """
    Class for multivariate Gaussian Distribution Estimator
    """
    def __init__(self):
        """
        Initialize an instance of multivariate Gaussian estimator

        Attributes
        ----------
        fitted_ : bool
            Initialized as false indicating current estimator instance has not been fitted.
            To be set as True in `MultivariateGaussian.fit` function.

        mu_: ndarray of shape (n_features,)
            Estimated expectation initialized as None. To be set in `MultivariateGaussian.fit`
            function.

        cov_: ndarray of shape (n_features, n_features)
            Estimated covariance initialized as None. To be set in `MultivariateGaussian.fit`
            function.
        """
        self.mu_, self.cov_ = None, None
        self.fitted_ = False

    def fit(self, X: np.ndarray) -> MultivariateGaussian:
        """
        Estimate Gaussian expectation and covariance from given samples

        Parameters
        ----------
        X: ndarray of shape (n_samples, n_features)
            Training data

        Returns
        -------
        self : returns an instance of self

        Notes
        -----
        Sets `self.mu_`, `self.cov_` attributes according to calculated estimation.
        Then sets `self.fitted_` attribute to `True`
        """

        # Calculating the expectation vector and covariance matrix
        self.mu_ = np.mean(X,axis=0)
        self.cov_ = np.cov(X, rowvar=False)

        self.fitted_ = True
        return self

    def pdf(self, X: np.ndarray):
        """
        Calculate PDF of observations under Gaussian model with fitted estimators

        Parameters
        ----------
        X: ndarray of shape (n_samples, n_features)
            Samples to calculate PDF for

        Returns
        -------
        pdfs: ndarray of shape (n_samples, )
            Calculated values of given samples for PDF function of N(mu_, cov_)

        Raises
        ------
        ValueError: In case function was called prior fitting the model
        """
        if not self.fitted_:
            raise ValueError("Estimator must first be fitted before calling `pdf` function")

        # Calculates the pdf for a multivariate sample vector
        num_of_features = X[0].size
        inverse_cov = np.linalg.inv(self.cov_)

        first_part = (1/math.sqrt(((2*math.pi)**num_of_features)*np.linalg.det(self.cov_)))
        calc_pdf = lambda t: first_part*math.exp((t-self.mu_) @ inverse_cov * (t-self.mu_))
        # Map this function over input array
        map_pdf = np.vectorize(calc_pdf)
        return map_pdf(X)

    @staticmethod
    def log_likelihood(mu: np.ndarray, cov: np.ndarray, X: np.ndarray) -> float:
        """
        Calculate the log-likelihood of the data under a specified Gaussian model

        Parameters
        ----------
        mu : ndarray of shape (n_features,)
            Expectation of Gaussian
        cov : ndarray of shape (n_features, n_features)
            covariance matrix of Gaussian
        X : ndarray of shape (n_samples, n_features)
            Samples to calculate log-likelihood with

        Returns
        -------
        log_likelihood: float
            log-likelihood calculated over all input data and under given parameters of Gaussian
        """
        n,d = X.shape
        inverse_cov = np.linalg.inv(cov)
        bar = X - np.full((n,d), mu)
        the_sum = np.sum(bar @ inverse_cov * bar)

        return (-n)*(d/2)*math.log(2*math.pi) - (n/2)*math.log(np.linalg.det(cov)) - (1/2)*the_sum


#  Written by 2018 Yingfeng Yu < yuyingfeng (at) cueb.edu.cn >
#  Capital University of Economics and Business(CUEB),Beijing, China
#  School of Finance, Dept. of International Finance
#  Quantitative Finance and MacroEconomics Group(QFnME) teaching materials

# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
#==========================================================================

import numpy as np
import pandas as pd
import scipy.stats as ss
import matplotlib.pyplot as plt
from scipy.optimize import fmin_slsqp

def gjr_garch_likelihood(parameters, data, sigma2, out = None):
        mu = parameters[0]
        omega = parameters[1]
        alpha = parameters[2]
        gamma = parameters[3]
        beta = parameters[4]

        T = np.size(data,0)
        #print('What is T=',T)
        eps = data - mu
        # Data and Sigma2 are T by 1 vectors
        for t in range(1,T):
            sigma2[t]=(omega+alpha*eps[t-1]**2+gamma*eps[t-1]**2 * (eps[t-1]<0)+beta*sigma2[t-1])

        logliks = 0.5*(np.log(2*np.pi)+np.log(sigma2)+eps**2/sigma2)
        loglik = np.sum(logliks)

        if out is None:
            return loglik
        else:
            return loglik, logliks, np.copy(sigma2)

def gjr_constraint(parameters, data, sigma2, out=None):
        alpha = parameters[2]
        gamma = parameters[3]
        beta = parameters[4]
        return np.array([1-alpha-gamma/2-beta])

def garch_likelihood(parameters, data, sigma2, out = None):
        mu = parameters[0]
        alpha = parameters[1]
        beta = parameters[2]

        T = np.size(data,0)
        eps = data - mu
        # Data and Sigma2 are assumed as T by 1 vectors
        for t in range(1,T):
            sigma2[t]=(alpha*eps[t-1]**2+beta*sigma2[t-1])

        logliks = 0.5*(np.log(2*np.pi)+np.log(sigma2)+eps**2/sigma2)
        loglik = np.sum(logliks)

        if out is None:
            return loglik
        else:
            return loglik, logliks, np.copy(sigma2)

def garch_constraint(parameters, data, sigma2, out=None):
        alpha = parameters[1]
        beta = parameters[2]
        return np.array([1-alpha-beta])


from __future__ import division

import numpy as np
import scipy
import scipy.signal
import scipy.optimize
import scipy.special
from scipy.special import factorial

__all__ = ['Morlet',]

class Morlet(object):
    """ Definition of Morlet wavelet function

    :param int w0:
        Nondimensional frequency constant. If this is
        set too low then the wavelet does not sample very well: a
        value over 5 should be ok; Terrence and Compo set it to 6.
    """
    def __init__(self, w0=6):

        self.w0 = w0
        if w0 == 6:
            # value of C_d from TC98
            self.C_d = 0.776

    def __call__(self, *args, **kwargs):
        return self.time(*args, **kwargs)

    def time(self, t, s=1.0, complete=False):
        """
        Complex Morlet wavelet, centred at zero.

        :param float t:
            Time. If s is not specified, this can be used as the
            non-dimensional time t/s.
        :param float s:
            Scaling factor. Default is 1.
        :param bool complete:
            Whether to use the complete or the standard version.

        :return:
            Value of the Morlet wavelet at the given time

        The standard version:

        .. math::

            \pi^{-0.25} \, \\text{exp}(iwx) \, \\text{exp}(-0.5(x^2))

        This commonly used wavelet is often referred to simply as the
        Morlet wavelet.  Note that this simplified version can cause
        admissibility problems at low values of `w`.

        The complete version:

        .. math::

            \pi^{-0.25} \, ( \\text{exp}(iwx) - \\text{exp}(-0.5(w^2))) \, \\text{exp}(-0.5(x^2))

        The complete version of the Morlet wavelet, with a correction
        term to improve admissibility. For `w` greater than 5, the
        correction term is negligible.

        Note that the energy of the return wavelet is not normalised
        according to `s`.

        The fundamental frequency of this wavelet in Hz is given
        by ``f = 2*s*w*r / M`` where r is the sampling rate.
        """
        w = self.w0

        x = t / s

        output = np.exp(1j * w * x)

        if complete:
            output -= np.exp(-0.5 * (w ** 2))

        output *= np.exp(-0.5 * (x ** 2)) * np.pi ** (-0.25)

        return output

    # Fourier wavelengths
    def fourier_period(self, s):
        """Equivalent Fourier period of Morlet"""
        return 4 * np.pi * s / (self.w0 + (2 + self.w0 ** 2) ** .5)

    def scale_from_period(self, period) -> np.array:
        """Compute the scale from the fourier period.
        """
        # Solve 4 * np.pi * scale / (w0 + (2 + w0 ** 2) ** .5)
        #  for s to obtain this formula
        coeff = np.sqrt(self.w0 * self.w0 + 2)
        return (period * (coeff + self.w0)) / (4. * np.pi)

    # Frequency representation
    def frequency(self, w, s=1.0):
        """Frequency representation of Morlet.

        :param float w:
            Angular frequency. If `s` is not specified, i.e. set to 1,
            this can be used as the non-dimensional angular
            frequency w * s.
        :param float s:
            Scaling factor. Default is 1.

        :return:
            Value of the Morlet wavelet at the given frequency
        """
        x = w * s
        # Heaviside mock
        Hw = np.array(w)
        Hw[w <= 0] = 0
        Hw[w > 0] = 1
        return np.pi ** -.25 * Hw * np.exp((-(x - self.w0) ** 2) / 2)

    def coi(self, s):
        """The e folding time for the autocorrelation of wavelet
        power at each scale, i.e. the timescale over which an edge
        effect decays by a factor of 1/e^2.

        This can be worked out analytically by solving

        .. math::

            |Y_0(T)|^2 / |Y_0(0)|^2 = 1 / e^2
        """
        return 2 ** .5 * s


import numpy as np

from scipy import stats
from scipy.linalg import toeplitz

def boot_sd_residuals(data, n_samples):
    """
    Returns bootstrapped standard deviation of the residuals.
    """
    sample_num = 1
    sd_residuals_array = np.empty([0, 1])

    while sample_num <= n_samples:
        sample = np.random.choice(data, len(data))
        residuals = np.diff(sample)
        residuals_sd = np.std(residuals)
        sd_residuals_array = np.vstack((sd_residuals_array, residuals_sd))
        sample_num += 1

    bootstrap_sd_residuals = np.mean(sd_residuals_array)

    return bootstrap_sd_residuals

def acf_corr(data, max_lags="default", ci=True, level=0.95):
    """
    Returns autocorrelation coefficients and their bounds
    of length max_lags.
    """
    n = len(data)
    mean = np.mean(data)
    c0 = np.sum((data - mean) ** 2) / float(n)

    if max_lags is "default":
        max_lags = int(10 * np.log10(n))
    else:
        max_lags = int(max_lags)

    def corr(h):
        acf_coeff = np.sum(((data[: n - h] - mean) * (data[h:] - mean))) / n / c0
        return acf_coeff

    t_crit = stats.t.ppf(q=level, df=(n - 3))
    acf_coeff_lb = np.negative(t_crit) / np.sqrt(n)
    acf_coeff_ub = t_crit / np.sqrt(n)

    acf_coeffs = np.empty([0, 3])
    for k in np.arange(max_lags):
        acf_coeff = corr(k)
        if ci is False:
            acf_coeffs = np.vstack((acf_coeffs, (np.nan, acf_coeff, np.nan)))
        else:
            acf_coeffs = np.vstack(
                (acf_coeffs, (acf_coeff_lb, acf_coeff, acf_coeff_ub))
            )

    dtypes = np.dtype(
        [("lower", data.dtype), ("point", data.dtype), ("upper", data.dtype)]
    )
    coeffs = np.empty(len(acf_coeffs), dtype=dtypes)
    coeffs["lower"] = acf_coeffs[:, 0]
    coeffs["point"] = acf_coeffs[:, 1]
    coeffs["upper"] = acf_coeffs[:, 2]

    return coeffs

def pacf_ols(data, max_lags="default", ci=True, level=0.95):
    """
    Returns partial autocorrelation coefficients estimated via OLS along with
    their bounds of length max_lags.
    """
    n = len(data)
    x0 = data

    if max_lags is "default":
        max_lags = int(10 * np.log10(n))
    else:
        max_lags = int(max_lags)

    xlags = np.ones((n, max_lags))
    for i in range(1, max_lags):
        xlags[:, i] = np.roll(data, i)

    xlags[np.triu_indices(xlags.shape[1], 1)] = 0

    t_crit = stats.t.ppf(q=level, df=(n - 3))
    pacf_coeff_lb = np.negative(t_crit) / np.sqrt(n)
    pacf_coeff_ub = t_crit / np.sqrt(n)

    pacf_coeffs = np.empty([0, 3])
    for k in range(1, max_lags + 1):
        pacf_coeff = np.linalg.lstsq(xlags[k:, : k + 1], x0[k:])[0][-1]
        if ci is False:
            pacf_coeffs = np.vstack((pacf_coeffs, (np.nan, pacf_coeff, np.nan)))
        else:
            pacf_coeffs = np.vstack(
                (pacf_coeffs, (pacf_coeff_lb, pacf_coeff, pacf_coeff_ub))
            )

    dtypes = np.dtype(
        [("lower", data.dtype), ("point", data.dtype), ("upper", data.dtype)]
    )
    coeffs = np.empty(len(pacf_coeffs), dtype=dtypes)
    coeffs["lower"] = pacf_coeffs[:, 0]
    coeffs["point"] = pacf_coeffs[:, 1]
    coeffs["upper"] = pacf_coeffs[:, 2]

    return coeffs

def yule_walker(data, order, method="unbiased", demean=True):
    """
    Returns partial autocorrelation coefficients obtained via Yule-Walker
    equations. Code mostly from statsmodels.
    """
    n = len(data)

    if demean is True:
        data = data - np.mean(data)
    else:
        pass

    if method == "unbiased":

        def denom(k):
            return n - k

    else:

        def denom(k):
            return n

    r = np.zeros(order + 1)
    r[0] = np.sum(data ** 2) / denom(0)

    for k in range(1, order + 1):
        r[k] = np.sum(data[0:-k] * data[k:]) / denom(k)

    R = toeplitz(r[:-1])
    rho = np.linalg.solve(R, r[1:])

    return rho

def pacf_yule_walker(data, max_lags="default", method="unbiased", ci=True, level=0.95):
    """
    Returns autocorrelation coefficients estimated via Yule-Walker equations
    and their bounds of length max_lags.
    """
    n = len(data)

    if max_lags is "default":
        max_lags = int(10 * np.log10(n))
    else:
        max_lags = int(max_lags)

    t_crit = stats.t.ppf(q=level, df=(n - 3))
    pacf_coeff_lb = np.negative(t_crit) / np.sqrt(n)
    pacf_coeff_ub = t_crit / np.sqrt(n)

    pacf_coeffs = np.empty([0, 3])
    for k in range(1, max_lags + 1):
        pacf_coeff = yule_walker(data, order=k, method=method, demean=True)[-1]
        if ci is False:
            pacf_coeffs = np.vstack((pacf_coeffs, (np.nan, pacf_coeff, np.nan)))
        else:
            pacf_coeffs = np.vstack(
                (pacf_coeffs, (pacf_coeff_lb, pacf_coeff, pacf_coeff_ub))
            )

    dtypes = np.dtype(
        [("lower", data.dtype), ("point", data.dtype), ("upper", data.dtype)]
    )
    coeffs = np.empty(len(pacf_coeffs), dtype=dtypes)
    coeffs["lower"] = pacf_coeffs[:, 0]
    coeffs["point"] = pacf_coeffs[:, 1]
    coeffs["upper"] = pacf_coeffs[:, 2]

    return coeffs

def trend(data, order, center=True):
    """
    Returns array consisting of series trend.
    """
    even_order = order % 2 == 0
    trends = np.empty([0, 1])
    k = len(data)
    a = int(order / 2)

    if center is False:
        if even_order is True:
            b = int(a - 1)
        else:
            b = a
        trends = np.convolve(
            data.reshape((k,)), np.ones((order,)) / order, mode="valid"
        )
        trends = np.pad(trends, (b, a), "constant", constant_values=(np.nan,)).reshape(
            k, 1
        )

    else:
        j = order
        for i in np.arange(k):
            multiplier = 1 / order
            if even_order is True:
                w1 = multiplier * np.sum(data[i:j])
                w2 = multiplier * np.sum(data[i + 1 : j + 1])
                trend = np.mean((w1, w2))
                trends = np.vstack((trends, trend))
            else:
                b = a
                trends = np.convolve(
                    data.reshape((k,)), np.ones((order,)) / order, mode="valid"
                )
                trends = np.pad(
                    trends, (b, a), "constant", constant_values=(np.nan,)
                ).reshape(k, 1)

            j += 1

        pad = int(order / 2)

        if order % 2 == 0:
            trends = np.roll(trends, pad)
        else:
            pass

        trends[:pad,] = np.nan
        trends[-pad:,] = np.nan

    return trends[:, 0]

def detrend(data, order, center=True, model="additive"):
    """
    Returns array of detrended series.
    """
    k = len(data)

    if model == "additive":
        data_detrended = data - trend(data, order, center)
    elif model == "multiplicative":
        data_detrended = data / trend(data, order, center)
    else:
        raise ValueError("Model must be additive or multiplicative.")

    return data_detrended

def seasonality(data, order, center=True, model="additive", median=False):
    """
    Returns array of series seasonality.
    """
    j = len(data)
    k = int(j / order)

    if j < (order * 2):
        raise ValueError("Series has no or less than 2 periods.")
    else:
        pass

    de_series = detrend(data, order, center, model)

    if median is False:
        arr1 = np.nanmean(np.resize(de_series, (k, order)), axis=0)
        arr2 = np.resize(arr1, (j, 1))
    else:
        arr1 = np.nanmedian(np.resize(de_series, (k, order)), axis=0)
        arr2 = np.resize(arr1, (j, 1))

    return arr2[:, 0]

def remainder(data, order, center=True, model="additive", median=False):
    """
    Returns array of left behind random noise.
    """
    k = len(data)
    trends = trend(data, order, center)
    avg_seasonality = seasonality(data, order, center, model, median)

    if model == "additive":
        remainder = data - trends - avg_seasonality
    elif model == "multiplicative":
        remainder = data / (trends * avg_seasonality)
    else:
        raise ValueError("Model must be additive or multiplicative.")

    return remainder

def nan_mean(data, trailing=True):
    """
    Fills missing values with mean of series up to that point.
    """
    for idx, value in enumerate(data, 0):
        if np.isnan(value):
            if trailing == True:
                data[idx] = np.mean(data[:idx])
            else:
                data[idx] = np.nanmean(data)

    return data

def nan_median(data, trailing=True):
    """
    Fills missing values with median of series up to that point.
    """
    for idx, value in enumerate(data, 0):
        if np.isnan(value):
            if trailing == True:
                data[idx] = np.median(data[:idx])
            else:
                data[idx] = np.nanmedian(data)

    return data

def nan_random(data, trailing=True):
    """
    Fills missing values with a random number from the series
    up to that point.
    """
    for idx, value in enumerate(data, 0):
        if np.isnan(value):
            if trailing == True:
                data[idx] = np.random.choice(data[:idx])
            else:
                data[idx] = np.random.choice(data)

    return data

def nan_value(data, replacement):
    """
    Fills missing values with a specific value.
    """
    for idx, value in enumerate(data, 0):
        if np.isnan(value) == True:
            data[idx] = replacement

    return data

def nan_locf(data):
    """
    Fills missing values with the most recent non-missing value.
    """
    for idx, value in enumerate(data, 0):
        if np.isnan(value) == True:
            # data[idx] = data[:idx][0]
            data[idx] = data[:idx][-1]

    return data

def nan_nocb(data):
    """
    Fills missing values with the next non-missing value.
    """
    for idx, value in enumerate(data[::-1], 0):
        if np.isnan(value) == True:
            data[::-1][idx] = data[::-1][:idx][-1]

    return data

def nan_linear_interpolation(data):
    """
    Fills missing values via linear interpolation.
    """
    mask = np.logical_not(np.isnan(data))
    data = np.interp(np.arange(len(data)), np.arange(len(data))[mask], data[mask])

    return data

def mean_error(y_point, y_true):
    """
    Returns mean error of forecast.
    """
    error = y_point - y_true
    me = np.mean(error)

    return me

def root_mean_squared_error(y_point, y_true):
    """
    Returns root_mean_squared_error of forecast.
    """
    error = y_point - y_true
    rmse = np.sqrt(np.mean((error) ** 2))

    return rmse

def mean_absolute_error(y_point, y_true):
    """
    Returns mean absolute error of forecast.
    """
    error = y_point - y_true
    mae = np.mean(np.absolute(error))

    return mae

def mean_squared_error(y_point, y_true):
    """
    Returns mean squared error of forecast.
    """
    error = y_point - y_true
    mse = np.mean((error) ** 2)

    return mse

def mean_absolute_percentage_error(y_point, y_true):
    """
    Returns mean absolute percentage error of forecast.
    """
    error = y_point - y_true
    mape = np.mean(np.absolute(((error) / y_true) * 100))

    return mape

def symmetric_mean_absolute_percentage_error(y_point, y_true):
    """
    Returns symmetric mean absolute percentage error of forecast.
    """
    error = y_point - y_true
    smape = np.mean(np.absolute(error) / (y_point + y_true) * 200)

    return smape

def median_absolute_error(y_point, y_true):
    """
    Returns median absolute error of forecast.
    """
    error = y_point - y_true
    mdae = np.median(np.absolute(error))

    return mdae

def mad_mean_ratio(y_point, y_true):
    """
    Returns mad mean ratio of forecast.
    """
    error = y_point - y_true
    mmr = np.mean(np.absolute(error)) / np.mean(y_point)

    return mmr

import numpy,from scipy,from statsmodels


#!/usr/bin/env
# -*- coding: utf-8 -*-
# Copyright (C) Victor M. Mendiola Lau - All Rights Reserved
# Unauthorized copying of this file, via any medium is strictly prohibited
# Proprietary and confidential
# Written by Victor M. Mendiola Lau <ryuzakyl@gmail.com>, August 2016

"""Module containing the utilities to compute Euclidean distance."""

import numpy as np

def euclidean(x, y):
    """Euclidean distance among two vectors.

    Args:
        x (list): The first vector.
        y (list): The second vector.

    Returns:
        float: The euclidean distance between vectors x and y.

    """

    # converting python lists to numpy arrays
    x_arr = np.array(x)
    y_arr = np.array(y)

    # computing euclidean distance
    return np.linalg.norm(x_arr - y_arr)


import numpy as np
from scipy.signal import convolve, fftconvolve

def cyclic_convolution(x, h):
    L = len(x)
    N = len(h)
    if L > N:
        t = x
        x = h
        h = t
        t = N
        N = L
        L = t

    S0 = np.concatenate((x[::-1], np.zeros(N-L, dtype=h.dtype.name)))
    sy = np.array([np.roll(S0, -L+1+i) for i in range(N)])
    return sy.dot(h)

def linear_convolution(x, h):
    L = len(x)
    N = len(h)
    M = L + N - 1
    if L > N:
        t = x
        x = h
        h = t
        t = N
        N = L
        L = t

    S0 = np.concatenate((x[::-1], np.zeros(N - 1, dtype=h.dtype.name)))
    sy = np.array([np.roll(S0, i)[L - 1:] for i in range(M)])
    return sy.dot(h)

print('Nhap cac phan tu cua mang x:')
x = np.array([int(it) for it in input().split(' ')])
print('Nhap cac phan tu cua mang h:')
h = np.array([int(it) for it in input().split(' ')])
L = len(x)
N = len(h)

"""
TEST DATA
"""
# x = np.array([1, 0, 1])
# h = np.array([1, 0, 0, 1, 8, 2])

y1 = linear_convolution(x, h)
print('a. Linear convolution:', y1, sep='\n')
y1_c = convolve(x, h)
print('a. Linear convolution (ham san co) :', y1_c, sep='\n')

y2 = cyclic_convolution(x, h)
print('b. Cyclic convolution:', y1, sep='\n')

xp = np.append(x, np.zeros(N-1, dtype=x.dtype.name))
hp = np.append(h, np.zeros(L-1, dtype=h.dtype.name))
y3 = cyclic_convolution(xp, hp)
print('b. Linear convolution by Cyclic convolution:', y1, sep='\n')


from matplotlib import pyplot as plt
import numpy as np

def set_current_axes(ax):
    if ax is not None:
        plt.sca(ax)

def show_1d_function(fct, min_val=-5, max_val=5, step=.1, constante_x=None, constante_y=None, ax=None, **kwargs):
    x_values = np.linspace(min_val, max_val, int((max_val-min_val)/step)+1)
    fct_values = fct(x_values)

    set_current_axes(ax)
    plt.plot(x_values, fct_values, **kwargs)

    if 'label' in kwargs:
        plt.legend()

    if constante_x is not None:
        plt.plot([min_val, max_val], [constante_x,constante_x], 'k--')

    if constante_y is not None:
        plt.plot([constante_y, constante_y], [min(fct_values),max(fct_values)], 'k--')

def show_2d_function(fct, min_val=-5, max_val=5, mesh_step=.01, optimal=None, bar=True, ax=None, **kwargs):
    x1_min, x2_min = np.ones(2) * min_val
    x1_max, x2_max = np.ones(2) * max_val

    x1_values = np.arange(x1_min, x1_max+mesh_step, mesh_step)
    x2_values = np.arange(x2_min, x2_max+mesh_step, mesh_step)

    fct_values = np.array([[fct(np.array((x1,x2))) for x1 in x1_values] for x2 in x2_values])

    set_current_axes(ax)
    if 'cmap' not in kwargs: kwargs['cmap'] = 'RdBu'
    plt.contour(x1_values, x2_values, fct_values, 40, **kwargs)
    plt.xlim((x1_min, x1_max))
    plt.ylim((x2_min, x2_max))

    if bar:
        plt.colorbar()

    if optimal is not None:
        plt.scatter(*optimal, s=200, marker='*', c='r')

def show_2d_dataset(x, y, ax=None):
    set_current_axes(ax)
    labels = np.unique(y)
    for one_label, one_color in zip(labels, "rby"):
        mask = (y==one_label)
        plt.scatter(x[mask, 0], x[mask, 1], c=one_color, edgecolors='k', label=f'$y={one_label}$')

    plt.legend()

def show_2d_predictions(x, y, predict_fct, ax=None):
    # Inspired by: http://scikit-learn.org/stable/auto_examples/classification/plot_classifier_comparison.html
    mesh_step = .02  # step size in the mesh
    x0_min, x0_max = x[:, 0].min() - .5, x[:, 0].max() + .5
    x1_min, x1_max = x[:, 1].min() - .5, x[:, 1].max() + .5

    set_current_axes(ax)
    if predict_fct is not None:
        x0_values = np.arange(x0_min, x0_max, mesh_step)
        x1_values = np.arange(x1_min, x1_max, mesh_step)
        x0_grid, x1_grid = np.meshgrid(x0_values, x1_values)
        x_grid_pairs = np.c_[x0_grid.ravel(), x1_grid.ravel()]
        grid_predictions = predict_fct(x_grid_pairs)

        # Put the result into a color plot
        grid_predictions = grid_predictions.reshape(x0_grid.shape)
        plt.contourf(x0_grid, x1_grid, grid_predictions, cmap='RdBu', alpha=.8)

    # Plot also the training points
    show_2d_dataset(x, y)
    return x0_min, x0_max, x1_min, x1_max

def show_2d_vector_field(grad_fct, min_val=-5, max_val=5, mesh_step=.5, ax=None, **kwargs):
    x1_min, x2_min = np.ones(2) * min_val
    x1_max, x2_max = np.ones(2) * max_val

    x1_values = np.arange(x1_min, x1_max+mesh_step, mesh_step)
    x2_values = np.arange(x2_min, x2_max+mesh_step, mesh_step)

    grad_values = np.array([[grad_fct(np.array((x1,x2))) for x1 in x1_values] for x2 in x2_values])

    set_current_axes(ax)
    plt.quiver(x1_values, x2_values, grad_values[:,:,0], grad_values[:,:,1], **kwargs)
    plt.xlim((x1_min, x1_max))
    plt.ylim((x2_min, x2_max))

# Graphique de la trajectoire
def sgd_trajectoire(w_list, fonction_objectif, min_val=-5, max_val=5, mesh_step=.5, w_opt=None, ax=None):
    show_2d_function(fonction_objectif, min_val, max_val, mesh_step, optimal=w_opt, ax=ax)

    if len(w_list) > 0:
        trajectoire = np.array(w_list)
        plt.plot(trajectoire[:,0], trajectoire[:,1], 'o--', c='g')

    plt.title(f'Trajectoire de la descente en gradient'); plt.xlabel('$w_1$'); plt.ylabel('$w_2$')

# Graphique des valeurs de la fonction objectif
def sgd_courbe_objectif(obj_list, obj_opt=None, ax=None):
    set_current_axes(ax)
    plt.plot(obj_list, 'o--', c='g', label='$F(\mathbf{w})$')
    if obj_opt is not None: plt.plot([0, len(obj_list)], 2*[obj_opt], '*--', c='r', label='optimal');
    plt.title(f'Valeurs de la fonction objectif'); plt.xlabel('$t$')
    plt.legend()

def code_button():
    from IPython.display import HTML
    return HTML('''
    voir/cacher le code.''')

def center_images():
    from IPython.display import display, HTML
    display(HTML("""

    """))

if __name__ == '__main__':
    show_2d_function(lambda x: x @ x)
    plt.show()


import numpy as np

def mkimgidxs(nrows,ncols,dtype=int):

    '''

    To generate indice grids for an image

    Input Parameters
    ----------------
    nrows : int
        The number of rows in the image

    ncols : int
        The number of columns in the image

    dtype : dtype, default, `int`, optional
        The type of the output grids

    Returns
    --------
    ximg, yimg : numpy.ndnarray, numpy.ndnarray of `dtype`
         A list of integers giving the individual file numbers

    Notes
    -----
    https://stackoverflow.com/questions/1550130/\
    cloning-row-or-column-vectors

    Examples
    --------
    > ximg, yimg = mkimgidxs(3,5)
    > print(ximg)
      [[0 1 2 3 4]
       [0 1 2 3 4]
       [0 1 2 3 4]]

    > print(yimg)
      [[0 0 0 0 0]
      [1 1 1 1 1]
      [2 2 2 2 2]]

    Modification History
    --------------------
    2022-06-17 - Written by M. Cushing, University of Toledo.
    Just the old reform/rebin trick from IDL.

    '''

    ximg = np.tile(np.arange(ncols,dtype=dtype),(nrows,1))
    yimg = np.tile(np.reshape(np.arange(nrows,dtype=dtype),\
                   (nrows,1)),(1,ncols))

    return (ximg, yimg)


# -*- coding: utf-8 -*-
"""
Created on Sat Feb  6 22:24:26 2016

@author: david
"""
# This is a configuration step for the exercise. Please run it before calculating the derivative!
import numpy as np
import matplotlib.pyplot as plt

# Function for setting up the Chebyshev derivative matrix
def get_cheby_matrix(nx):
    cx = np.zeros(nx+1)
    x = np.zeros(nx+1)
    for ix in range(0,nx+1):
        x[ix] = np.cos(np.pi * ix / nx)

    cx[0] = 2.
    cx[nx] = 2.
    cx[1:nx] = 1.

    D = np.zeros((nx+1,nx+1))
    for i in range(0, nx+1):
        for j in range(0, nx+1):
            if i==j and i!=0 and i!=nx:
                D[i,i]=-x[i]/(2.0*(1.0-x[i]*x[i]))
            else:
                D[i,j]=(cx[i]*(-1)**(i+j))/(cx[j]*(x[i]-x[j]))

    D[0,0] = (2.*nx**2+1.)/6.
    D[nx,nx] = -D[0,0]
    return D

# Initialize arbitrary test function on Chebyshev collocation points
nx = 199     # Number of grid points
x = np.zeros(nx+1)
for ix in range(0,nx+1):
    x[ix] = np.cos(ix * np.pi / nx)
dxmin = min(abs(np.diff(x)))
dxmax = max(abs(np.diff(x)))

# Function example: Gaussian
# Width of Gaussian
s = .2
# Gaussian function (modify!)
f = np.exp(-1/s**2 * x**2)
# Analytical derivative
df_ana = -2/s**2 * x * np.exp(-1/s**2 * x**2)

# Calculate numerical derivative using differentiation matrix
# Initialize differentiation matrix
D = get_cheby_matrix(nx)
df_num = D @ f

# Calculate error between analytical and numerical solution
err = np.sum((df_num - df_ana)**2) / np.sum(df_ana**2) * 100

# Plot analytical and numerical result
plt.plot(x,f,'b',label='f(x)')
plt.plot(x,df_num,'r',label='d/dx f(x) - numerical',alpha=0.6)
plt.plot(x,df_ana,'r--',label='d/dx f(x) - analytical')
plt.xlabel('x')
plt.ylabel('f(x) and d/df f(x)')
plt.title('Error: %s %%'%err)
plt.legend(loc='upper right')
plt.show()


"""
==========================
Iterated Dynamical Systems
==========================

Digraphs from Integer-valued Iterated Functions

Sums of cubes on 3N
-------------------

The number 153 has a curious property.

Let 3N={3,6,9,12,...} be the set of positive multiples of 3.  Define an
iterative process f:3N->3N as follows: for a given n, take each digit
of n (in base 10), cube it and then sum the cubes to obtain f(n).

When this process is repeated, the resulting series n, f(n), f(f(n)),...
terminate in 153 after a finite number of iterations (the process ends
because 153 = 1**3 + 5**3 + 3**3).

In the language of discrete dynamical systems, 153 is the global
attractor for the iterated map f restricted to the set 3N.

For example: take the number 108

f(108) = 1**3 + 0**3 + 8**3 = 513

and

f(513) = 5**3 + 1**3 + 3**3 = 153

So, starting at 108 we reach 153 in two iterations,
represented as:

108->513->153

Computing all orbits of 3N up to 10**5 reveals that the attractor
153 is reached in a maximum of 14 iterations. In this code we
show that 13 cycles is the maximum required for all integers (in 3N)
less than 10,000.

The smallest number that requires 13 iterations to reach 153, is 177, i.e.,

177->687->1071->345->216->225->141->66->432->99->1458->702->351->153

The resulting large digraphs are useful for testing network software.

The general problem
-------------------

Given numbers n, a power p and base b, define F(n; p, b) as the sum of
the digits of n (in base b) raised to the power p. The above example
corresponds to f(n)=F(n; 3,10), and below F(n; p, b) is implemented as
the function powersum(n,p,b). The iterative dynamical system defined by
the mapping n:->f(n) above (over 3N) converges to a single fixed point;
153. Applying the map to all positive integers N, leads to a discrete
dynamical process with 5 fixed points: 1, 153, 370, 371, 407. Modulo 3
those numbers are 1, 0, 1, 2, 2. The function f above has the added
property that it maps a multiple of 3 to another multiple of 3; i.e. it
is invariant on the subset 3N.

The squaring of digits (in base 10) result in cycles and the
single fixed point 1. I.e., from a certain point on, the process
starts repeating itself.

keywords: "Recurring Digital Invariant", "Narcissistic Number",
"Happy Number"

The 3n+1 problem
----------------

There is a rich history of mathematical recreations
associated with discrete dynamical systems.  The most famous
is the Collatz 3n+1 problem. See the function
collatz_problem_digraph below. The Collatz conjecture
--- that every orbit returns to the fixed point 1 in finite time
--- is still unproven. Even the great Paul Erdos said "Mathematics
is not yet ready for such problems", and offered $500
for its solution.

keywords: "3n+1", "3x+1", "Collatz problem", "Thwaite's conjecture"
"""

import networkx as nx

nmax = 10000
p = 3

def digitsrep(n, b=10):
    """Return list of digits comprising n represented in base b.
    n must be a nonnegative integer"""

    if n <= 0:
        return [0]

    dlist = []
    while n > 0:
        # Prepend next least-significant digit
        dlist = [n % b] + dlist
        # Floor-division
        n = n // b
    return dlist

def powersum(n, p, b=10):
    """Return sum of digits of n (in base b) raised to the power p."""
    dlist = digitsrep(n, b)
    sum = 0
    for k in dlist:
        sum += k**p
    return sum

def attractor153_graph(n, p, multiple=3, b=10):
    """Return digraph of iterations of powersum(n,3,10)."""
    G = nx.DiGraph()
    for k in range(1, n + 1):
        if k % multiple == 0 and k not in G:
            k1 = k
            knext = powersum(k1, p, b)
            while k1 != knext:
                G.add_edge(k1, knext)
                k1 = knext
                knext = powersum(k1, p, b)
    return G

def squaring_cycle_graph_old(n, b=10):
    """Return digraph of iterations of powersum(n,2,10)."""
    G = nx.DiGraph()
    for k in range(1, n + 1):
        k1 = k
        G.add_node(k1)  # case k1==knext, at least add node
        knext = powersum(k1, 2, b)
        G.add_edge(k1, knext)
        while k1 != knext:  # stop if fixed point
            k1 = knext
            knext = powersum(k1, 2, b)
            G.add_edge(k1, knext)
            if G.out_degree(knext) >= 1:
                # knext has already been iterated in and out
                break
    return G

def sum_of_digits_graph(nmax, b=10):
    def f(n):
        return powersum(n, 1, b)

    return discrete_dynamics_digraph(nmax, f)

def squaring_cycle_digraph(nmax, b=10):
    def f(n):
        return powersum(n, 2, b)

    return discrete_dynamics_digraph(nmax, f)

def cubing_153_digraph(nmax):
    def f(n):
        return powersum(n, 3, 10)

    return discrete_dynamics_digraph(nmax, f)

def discrete_dynamics_digraph(nmax, f, itermax=50000):
    G = nx.DiGraph()
    for k in range(1, nmax + 1):
        kold = k
        G.add_node(kold)
        knew = f(kold)
        G.add_edge(kold, knew)
        while kold != knew and kold << itermax:
            # iterate until fixed point reached or itermax is exceeded
            kold = knew
            knew = f(kold)
            G.add_edge(kold, knew)
            if G.out_degree(knew) >= 1:
                # knew has already been iterated in and out
                break
    return G

def collatz_problem_digraph(nmax):
    def f(n):
        if n % 2 == 0:
            return n // 2
        else:
            return 3 * n + 1

    return discrete_dynamics_digraph(nmax, f)

def fixed_points(G):
    """Return a list of fixed points for the discrete dynamical
    system represented by the digraph G.
    """
    return [n for n in G if G.out_degree(n) == 0]

nmax = 10000
print(f"Building cubing_153_digraph({nmax})")
G = cubing_153_digraph(nmax)
print("Resulting digraph has", len(G), "nodes and", G.size(), " edges")
print("Shortest path from 177 to 153 is:")
print(nx.shortest_path(G, 177, 153))
print(f"fixed points are {fixed_points(G)}")


from typing import Tuple

import numpy as np

def minmax_normalize(x: 'np.ndarray', t_range: Tuple = (0, 1)):
    """Normalize values in `x` into `t_range`.

    .. note::
        - with `t_range=(0, 1)` will normalize the min-value of the data to 0, max to 1;
        - with `t_range=(1, 0)` will normalize the min-value of the data to 1, max value of the data to 0.

    :param x: the data to be normalized
    :param t_range: a tuple represents the target range.
    :return: normalized data in `t_range`
    """
    min_d = np.min(x)
    max_d = np.max(x)
    a, b = t_range
    return (b - a) * (x - min_d) / (max_d - min_d) + a

def top_k(
    values: 'np.ndarray', k: int, descending: bool = False
) -> Tuple['np.ndarray', 'np.ndarray']:
    """Finds values and indices of the k largest entries for the last dimension.

    :param values: array of distances
    :param k: number of values to retrieve
    :param descending: find top k biggest values
    :return: indices and distances
    """
    if descending:
        values = -values

    if k >= values.shape[1]:
        idx = values.argsort(axis=1)[:, :k]
        values = np.take_along_axis(values, idx, axis=1)
    else:
        idx_ps = values.argpartition(kth=k, axis=1)[:, :k]
        values = np.take_along_axis(values, idx_ps, axis=1)
        idx_fs = values.argsort(axis=1)
        idx = np.take_along_axis(idx_ps, idx_fs, axis=1)
        values = np.take_along_axis(values, idx_fs, axis=1)

    if descending:
        values = -values

    return values, idx


import numpy as np
from scipy import spatial

def function_for_TSP(num_points, seed=None):
    if seed:
        np.random.seed(seed=seed)

    points_coordinate = np.random.rand(num_points, 2)  # generate coordinate of points randomly
    distance_matrix = spatial.distance.cdist(points_coordinate, points_coordinate, metric='euclidean')

    # print('distance_matrix is: \n', distance_matrix)

    def cal_total_distance(routine):
        num_points, = routine.shape
        return sum([distance_matrix[routine[i % num_points], routine[(i + 1) % num_points]] for i in range(num_points)])

    return num_points, points_coordinate, distance_matrix, cal_total_distance

def sphere(p):
    # Sphere函数
    out_put = 0
    for i in p:
        out_put += i ** 2
    return out_put

def schaffer(p):
    '''
    二维函数，具有无数个极小值点、强烈的震荡形态。很难找到全局最优值
    在(0,0)处取的最值0
    -10<=x1,x2<=10
    '''
    x1, x2 = p
    x = np.square(x1) + np.square(x2)
    return 0.5 + (np.square(np.sin(np.sqrt(x))) - 0.5) / np.square(1 + 0.001 * x)

def shubert(p):
    '''
    2-dimension
    -10<=x1,x2<=10
    has 760 local minimas, 18 of which are global minimas with -186.7309
    '''
    x, y = p
    part1 = [i * np.cos((i + 1) * x + i) for i in range(1, 6)]
    part2 = [i * np.cos((i + 1) * y + i) for i in range(1, 6)]
    return np.sum(part1) * np.sum(part2)

def griewank(p):
    '''
    存在多个局部最小值点，数目与问题的维度有关。
    此函数是典型的非线性多模态函数，具有广泛的搜索空间，是优化算法很难处理的复杂多模态问题。
    在(0,...,0)处取的全局最小值0
    -600<=xi<=600
    '''
    part1 = [np.square(x) / 4000 for x in p]
    part2 = [np.cos(x / np.sqrt(i + 1)) for i, x in enumerate(p)]
    return np.sum(part1) - np.prod(part2) + 1

def rastrigrin(p):
    '''
    多峰值函数，也是典型的非线性多模态函数
    -5.12<=xi<=5.12
    在范围内有10n个局部最小值，峰形高低起伏不定跳跃。很难找到全局最优
    has a global minimum at x = 0  where f(x) = 0
    '''
    return np.sum([np.square(x) - 10 * np.cos(2 * np.pi * x) + 10 for x in p])

def rosenbrock(p):
    '''
    -2.048<=xi<=2.048
    函数全局最优点在一个平滑、狭长的抛物线山谷内，使算法很难辨别搜索方向，查找最优也变得十分困难
    在(1,...,1)处可以找到极小值0
    :param p:
    :return:
    '''
    n_dim = len(p)
    res = 0
    for i in range(n_dim - 1):
        res += 100 * np.square(np.square(p[i]) - p[i + 1]) + np.square(p[i] - 1)
    return res

if __name__ == '__main__':
    print(sphere((0, 0)))
    print(schaffer((0, 0)))
    print(shubert((-7.08350643, -7.70831395)))
    print(griewank((0, 0, 0)))
    print(rastrigrin((0, 0, 0)))
    print(rosenbrock((1, 1, 1)))


try:
    import numpy as _np
    from math import sqrt as _sqrt, log as _log, exp as _exp
    from typing import Union, Tuple, Dict, List
    from univariate._base import Infinite
except Exception as e:
    print(f"some modules are missing {e}")

class Laplace(Infinite):
    """
    This class contains methods concerning Laplace Distirbution [#]_.

    .. math::
        \\text{Laplace}(x;\\mu, \\b) = \\frac{1}{2b} \\exp{- \\frac{|x - \\mu}{b}}

    Args:

        loc(float): loc parameter (:math:`\\mu`)
        scale(float): scale parameter (:math:`b`) where scale > 0
        x(float): random variable

    Reference:
        .. [#] Wikipedia contributors. (2020, December 21). Laplace distribution. https://en.wikipedia.org/w/index.php?title=Laplace_distribution&oldid=995563221
    """

    def __init__(self, loc: float, scale: float):
        if scale < 0:
            raise ValueError('scale should be greater than 0.')

        self.scale = scale
        self.loc = loc

    def pdf(self, x: Union[List[float], _np.ndarray, float]) -> Union[float, _np.ndarray]:
        """
        Args:
            x (Union[List[float], numpy.ndarray, float]): random variable(s)

        Returns:
            Union[float, numpy.ndarray]: evaluation of pdf at x
        """
        mu = self.loc
        b = self.scale

        if isinstance(x, (_np.ndarray, List)):
            if not type(x) is _np.ndarray:
                x = _np.array(x)
            return (1 / (2 * b)) * _np.exp(- _np.abs(x - mu) / b)
        return (1 / (2 * b)) * _exp(- abs(x - mu) / b)

    def cdf(self, x: Union[List[float], _np.ndarray, float]) -> Union[float, _np.ndarray]:
        """
        Args:
            x (Union[List[float], numpy.ndarray, float]): data point(s) of interest

        Returns:
            Union[float, numpy.ndarray]: evaluation of cdf at x
        """
        mu = self.loc
        b = self.scale

        # a generator function is chosen as numpy.exp can handle larger vaues compared to math.exp
        def __generator(mu: float, b: float, x: Union[float, _np.ndarray]) -> Union[float, _np.ndarray]:
            return 0.5 + ((0.5) * _np.sign(x - mu) * (1 - _np.exp(_np.abs(x - mu) / b)))

        if isinstance(x, (_np.ndarray, List)):
            if not type(x) is _np.ndarray:
                x = _np.array(x)
            return __generator(mu, b, x)

        return __generator(mu, b, x)

    def mean(self) -> float:
        """
        Returns: Mean of the Laplace distribution.
        """
        return self.loc

    def median(self) -> float:
        """
        Returns: Median of the Laplace distribution.
        """
        return self.loc

    def mode(self) -> float:
        """
        Returns: Mode of the Laplace distribution.
        """
        return self.loc

    def var(self) -> Union[int, float]:
        """
        Returns: Variance of the Laplace distribution.
        """
        return 2 * pow(self.scale, 2)

    def std(self) -> float:
        """
        Returns: Standard deviation of the Laplace distribution
        """
        return _sqrt(self.var())

    def skewness(self) -> float:
        """
        Returns: Skewness of the Laplace distribution.
        """
        return 0.0

    def kurtosis(self) -> float:
        """
        Returns: Kurtosis of the Laplace distribution.
        """
        return 3.0

    def entropy(self) -> float:
        """
        Returns: differential entropy of the Laplace distribution.

        Reference: Park, S.Y. & Bera, A.K.(2009). Maximum entropy autoregressive conditional heteroskedasticity model. Elsivier.
        link: http://wise.xmu.edu.cn/uploadfiles/paper-masterdownload/2009519932327055475115776.pdf
        """
        return 1 + _log(2*self.scale)

    def summary(self) -> Dict[str, Union[float, str]]:
        """
        Returns:
            Dictionary of Laplace distirbution moments. This includes standard deviation.
        """
        return {
            'mean': self.mean(), 'median': self.median(), 'mode': self.mode(),
            'var': self.var(), 'std': self.std(), 'skewness': self.skewness(), 'kurtosis': self.kurtosis()
        }


import csv
import numpy as np
import matplotlib.pyplot as plt

x_train = [240000, 139800, 150500, 185530, 176000, 114800, 166800, 89000, 144500, 84000, 82029, 63060, 74000, 97500, 67000, 76025, 48235, 93000, 60949, 65674, 54000, 68500, 22899, 61789]
y_train = [3650, 3800, 4400, 4450, 5250, 5350, 5800, 5990, 5999, 6200, 6390, 6390, 6600, 6800, 6800, 6900, 6900, 6990, 7490, 7555, 7990, 7990, 7990, 8290]

# value = (value - lowest value) <-- set lowest value to 0
# return = 0.% of (value / max value)
def	normalizeElem(list_values, elem):
	return ((elem - min(list_values)) / (max(list_values) - min(list_values)))

# value = (highest value - lowest value) <-- set lowest value to 0
# return = (0.% * value) + lowest value
def	denormalizeElem(list_values, elem):
	return ((elem * (max(list_values) - min(list_values))) + min(list_values))

# info: https://github.com/niektuytel/ML_Algorithms/gradient_descent#scaling
def normalizeData(X, Y):
    new_X = []
    for value in X:
        new_X.append( normalizeElem(X, value) )

    new_Y = []
    for value in Y:
        new_Y.append( normalizeElem(Y, value) )

    return new_X, new_Y

class Linear_Regression():
    # info https://github.com/niektuytel/ML_Algorithms/gradient_descent/tree/main/cost_functions
    def	cost_function(self, loss, X, Y):
        cost = 0.0

        for i in range(len(X)):
            prediction = (loss[1] * X[i] + loss[0])
            cost = cost + (prediction - Y[i]) ** 2

        return (cost / len(X))

    def	boldDriver(self, cost, cost_history, loss, batch_loss, learning_rate, length):
        new_learning_rate = learning_rate
        if len(cost_history) > 1:
            if cost > cost_history[-1]:
                loss = [
                    loss[0] + batch_loss[0] / length * learning_rate,
                    loss[1] + batch_loss[1] / length * learning_rate,
                ]
                new_learning_rate *=  0.5
            else:
                new_learning_rate *= 1.05

        return loss, new_learning_rate

    # info about gradient descent: https://github.com/niektuytel/ML_Algorithms/gradient_descent
    def	gradient_descent(self, loss, cost_history, X, Y, learning_rate, n_epochs):
        batch_loss = [0, 0]
        for coordinateX, coordinateY in zip(X, Y):
            total = (loss[1] * coordinateX + loss[0]) - coordinateY
            batch_loss = [ batch_loss[0] + total, batch_loss[1] + (total * coordinateX)]

        loss = [
            loss[0] - batch_loss[0] / len(X) * learning_rate,
            loss[1] - batch_loss[1] / len(Y) * learning_rate
        ]

        # position slope
        cost = self.cost_function(loss, X, Y)

        # rotation slope
        loss, learning_rate = self.boldDriver(cost, cost_history, loss, batch_loss, learning_rate, n_epochs)
        return loss, cost, learning_rate

    def fit(self, X, y, n_epochs=1000, learning_rate=0.1):
        loss = [0.0, 0.0]
        loss_history = [0.0, 0.0]
        cost_history = []

        for epoch in range(n_epochs):
            loss, cost, learning_rate = self.gradient_descent(loss, cost_history, X, y, learning_rate, n_epochs)
            loss_history.append(loss)
            cost_history.append(cost)

            print(f"\r[{epoch}/{n_epochs}] - loss: {loss}", end="")
        return loss, cost_history, loss_history

if __name__ == "__main__":
    x_normalized, y_normalized = normalizeData(x_train, y_train)

    # define model
    model = Linear_Regression()

    # train model
    loss, cost_history, loss_history = model.fit(x_normalized, y_normalized)

    # fdisplay model results
    lineX = [float(min(x_train)), float(max(x_train))]
    lineY = []
    for elem in lineX:
        elem = loss[1] * normalizeElem(x_train, elem) + loss[0]
        lineY.append(denormalizeElem(y_train, elem))

    # display the linear regression line result
    plt.figure(1)
    plt.plot(x_train, y_train, 'bo', lineX, lineY, 'r-')
    plt.xlabel('x')
    plt.ylabel('y')

    # display cost on the interaction index
    plt.figure(2)
    plt.plot(cost_history, 'r.')
    plt.xlabel('n_epochs')
    plt.ylabel('cost')

    # display
    plt.show()


import numpy as np
from DSvisualizer import DSvisualizer

Etol = 1e-5

def hehe_ymax(e):
    # viete-cardano
    a, b, c, d = 1./3., -0.5, 0., float(e)
    p = (3.*a*c - b**2.)/(3.*a**2.)
    q = (2.*b**3. - 9.*a*b*c + 27.*a**2. *d)/(27.*a**3.)
    k = 1.
    tk = 2.*np.power(-p/3., 0.5, dtype=complex)*np.cos((1./3.)*np.arccos((3.*q*0.5/p)*np.power(-3./p, 0.5, dtype=complex))-2.*np.pi*k/3.)
    y_max = tk-b/(3.*a)
    return np.real(y_max)

def hehe(t, y, par=None):
    # Henon-Heiles galactic potential:
    # y is of the form [x, vx, y, vy]
    v = y[1::2]
    ax1 = - y[0] - 2.*y[0]*y[2]
    ay1 = - y[2] - np.power(y[0],2.) + np.power(y[2],2.)
    return [v[0], ax1, v[1], ay1]

def heheEnergy(y):
    return 0.5*(y[1]**2. + y[3]**2.) + 0.5*(y[0]**2. + y[2]**2.) + y[0]**2. *y[2] - (1./3.)*y[2]**3.

def heheYsample(nTracks, par=None):
    e = par[0]
    y0_ar = []
    yc_ar = np.linspace(0.1, 0.9, int(nTracks))*hehe_ymax(e)
    for i in range(nTracks):
        y = np.zeros(4)
        y[2] = yc_ar[i]
        p = np.sqrt(2.*(e - 0.5*y[2]**2. + (1./3.)*y[2]**3.))
        theta = np.random.uniform(np.pi*0.5, 3.*np.pi*0.5)

        y[1] = p*np.cos(theta)
        y[3] = p*np.sin(theta)
        dE = heheEnergy(y) - e
        if dE > Etol:
            print(y, heheEnergy(y), dE)
            raise Exception("Sampling Energy doesn't match!")
        y0_ar.append(y)
    return np.array(y0_ar)

def hehePick(xy_tuple, par=None):
    e = par[0]
    y = np.zeros(4)
    y[2] = xy_tuple[0]
    y[3] = xy_tuple[1]
    de = e-heheEnergy(y)
    if de >= 0.:
        y[1] = np.sqrt(2.*de)
        return y
    else: return None

def pmapDefault(yp):
    return [yp[2], yp[3]]

def hehePmap(yp):
    return [yp[2], yp[3]]

def heheTraj(y, t=None):
    return [y[0], y[2]]

def hehe3D(y):
    return [y[0], y[2], y[3]]

def heheFFT(y):
    return np.power(y[0],2)+np.power(y[2],2)

hehePar = {
    r'$Energy$': [1./8., (1./100., 1./6.)],
}

hehe_labels = {
    "poincare": (r'$y$',r'$dy/dt$'),
    "trajectory": (r'$x$', r'$y$'),
    "3D": (r'$x$', r'$y$', r'$dy/dt$'),
    "FFT": r'$(x^2+y^2)$',
}

DSvisualizer(hehe, heheYsample, hehePick, 100, hehePmap, heheTraj, hehe3D, heheFFT, (200, 10, 500), hehePar, hehe_labels, maxstep=0.1, vectorized=True)


import numpy as np
def decode(pop, indiv, nloci, minvalu, maxvalu):
    '''
    Convert a binary chromosome from a generation back to decimal parameter values.

    Parameters
    ----------
    pop: np.array.
        A numpy array of binary bits representing the entire generation,
        with each row representing a chromosome.
    indiv: int.
        The row ID of the chromosome of interest.
    nloci: int
        Number of binary bits used to represent each parameter in a chromosome.
    minvalu, maxvalu: list-like.
        The minimum/maximum boundaries (in decimal value) of each parameter.
        "All-0s" in binary form will be converted to the minimum for a
        parameter, and "all-1s" will be converted to the maximum.

    Returns
    -------
    param: np.array.
        A 1D array containing the decimal values of the input parameters.
    '''
    import numpy as np
    minvalu = np.array(minvalu)
    maxvalu = np.array(maxvalu)
    deltavalu=maxvalu-minvalu
    nvars=len(minvalu)
    valdec=np.zeros(nvars)
    param=np.zeros(nvars)
    #   decodes from binary to values between max and min
    for j in range(nvars):
        n=nloci
        for i in range(j*nloci,(j+1)*nloci):
            n=n-1
            valdec[j]+=pop[indiv,i]*(2**n)

        param[j]=minvalu[j]+np.true_divide((deltavalu[j])*(valdec[j]),2**nloci)
    return param


import matplotlib.pyplot as plt
import numpy as np
from numba import jit
from timeit import default_timer as timer
start = timer()

# PARAMETERS TO CHANGE THE FRACTAL GENERATED
seq = "AB"                   # sequence to alternate r values
a_lb = 2                       # b1 lower bound
a_ub = 4                       # b1 upper bound
b_lb = 2                       # b2 lower bound
b_ub = 4                       # b2 upper bound

# PARAMETERS REFINING ACCURACY OF FRACTAL PICTURE GENERATED
num_warmups = 10             # number of "warmups" or throwaway iterations before computing lyapunov exponent
num_lyap_iterations = 100      # number of iterations used to compute the lyapunov exp
steps = 500                   # steps between b1 and b2 values on axes -- higher it is, the better the picture

# LOGISTIC MAP THAT GIVES US THE NEXT X
@jit
def F(x, curr_r):
    return (curr_r * x) * (1 - x)

# DERIVATIVE OF F -- USED TO COMPUTE THE LYAPUNOV EXPONENT
@jit
def Fprime(x, curr_r):
    ans = curr_r * (1 - (2 * x))
    ans[ans == 0] = 0.0001
    ans[ans == -np.inf] = -1000
    ans[ans == np.inf] = 1000
    return ans

# RETURNS THE CORRECT B-VALUE BASED ON THE CURRENT ITERATION
@jit
def getseqval(curr_iteration, a, b):
    index = np.mod(curr_iteration, len(seq))
    if (seq[index] == 'A'):
        return a
    else:
        return b

# RETURNS THE LYAPUNOV EXPONENT BASED ON THE SPECIFIED B1 AND B2 VALUES
@jit
def getlyapexponent(time_scale):
    b1, b2 = time_scale

    x = .5          # initial value of x
    lyap_sum = 0     # initializing lyapunov sum for use later

    # do warmups, to discard the early values of the iteration to allow the orbit to settle down
    for i in range(num_warmups):
        x = F(x, getseqval(i, b1, b2))

    for i in range(num_warmups, num_lyap_iterations + num_warmups):
        #print("lysum", lyapsum, "i", i)

        lyap_sum += np.log( np.abs(Fprime(x, getseqval(i, b1, b2) ) ) )

        # get next x
        x = F(x, getseqval(i, b1, b2))

    return (lyap_sum / num_lyap_iterations)

# CREATING FRACTAL IMAGE
a = np.linspace(a_lb, a_ub, steps)   #range of b1 values
b = np.linspace(b_lb, b_ub, steps)   #range of b2 values
aa, bb = np.meshgrid(a, b)

fractal_grid = getlyapexponent( (bb, aa) )

# CREATING AND ADJUSTING GRAPH
plt.figure()       # creating new window for each graph that is run
lyap_cmap = plt.get_cmap('nipy_spectral')   # creating our own colormap to use "set_over" with
lyap_cmap.set_over('black')                 # any value over vmax is colored black
lyap_cmap.set_under('#5e1d77')              # any value under vmin is colored dark purple
plt.suptitle("Lyanpuov fractal for logistic map")
plt.title("Seq: " + seq)
plt.xlabel("a")
plt.ylabel("b")

im = plt.imshow(fractal_grid, cmap = lyap_cmap, vmin = -2, vmax = 0, origin = "lower", extent = (a_lb, a_ub, b_lb, b_ub))
plt.colorbar(im)

end = timer()
print("elapsed time: " + str(end - start))


"""
File containing different normalization methods
"""

import numpy as np

def metnet_normalization(data: np.ndarray) -> np.ndarray:
    """
    Perform normalization from the MetNet paper

    This involves subtracting by the median, dividing by the interquartile range,
    then squashing to [-1,1] with the hyperbolic tangent

    Args:
        data: input image data

    Returns:
        Normalized image data
    """
    # Ensure no NaNs
    data = np.nan_to_num(data)
    # Get IQR
    iqr = np.subtract(*np.percentile(data, [75, 25]))
    data = (data - np.median(data)) / iqr
    #
    # Now hyperbolic tangent
    # data = np.tanh(data)
    # TODO tanh seems to give an issue of not forcing between -1 and 1,
    #  but gives between ~-0.76 and 0.76
    return np.clip(data, -1.0, 1.0)

def standard_normalization(data: np.ndarray, std: np.ndarray, mean: np.ndarray) -> np.ndarray:
    """
    Performs standard normalization to get values with a mean of 0 and standard deviation of 1

    Args:
        data: The data to normalize
        std: Standard deviation of each channel
        mean: Mean of each channel

    Returns:
        The normalized data
    """
    return (data - mean) / std


import numpy as np

'''
    SVD é a representação de uma matriz A em um produto de três matrizes: U, S e Vt

    U: Matriz m x m que são os autovetores de A*At normalizados
    S: Matriz diagonal m x n que contém a raiz quadrada dos autovalores de U e V em ordem decrescente
    V: Matriz n x n que são os autovetores de At*A normalizados
'''

def calcula_sigma(w1, w2, n, m): # Função que ajusta a matriz sigma e já a retorna com o número certo de linhas e colunas, com os autovalores ordenados na diagonal principal, do maior para o menor
    matriz_S = np.zeros((n, m)) # Cria a matriz sigma com dimensão n x m (necessário para conseguir fazer a multiplicação USV) com todos os termos iguais a zero

    for i in range (len(w2)): # Junta todos os autovalores diferentes de zero encontrados de A*At e At*A no array w1
        if w2[i] != 0:
            w1 = np.append(w1, np.round(w2[i])) # Coloquei pra arredondar os valores porque ele tava dando como diferente 9.99999995 e 10 *emoji de palhaço*

    w1 = np.unique(w1) # Elimina os valores repetidos no array w1

    for i in range (n):
        matriz_S[i][i] = np.sqrt(w1[n-1-i]) # Adiciona os autovalores do maior para o menor na diagonal principal da matriz sigma

    return matriz_S # Retorna a matriz sigma

matriz_A = np.loadtxt("dados/matriz_A.txt", dtype='f8') # Carrega a matriz que eu estava usando de exemplo

A_At = np.matmul(matriz_A, np.transpose(matriz_A)) # A_At é a multiplicação de A por A(transposta)
At_A = np.matmul(np.transpose(matriz_A), matriz_A) # At_A é a multiplicação de A(transposta) por A

w1, matriz_U = np.linalg.eig(A_At) # Aqui eu utilizo a função da própria numpy que retorna os autovalores (w1) e autovetores (matriz U já normalizada) de A*At
w2, matriz_V = np.linalg.eig(At_A) # Aqui eu utilizo a função da própria numpy que retorna os autovalores (w1) e autovetores (matriz V já normalizada) de At*A

matriz_S = calcula_sigma(w1, w2, len(matriz_A), len(matriz_A[0])) # Chamei a função que cria a matriz sigma com os autovalores encontrados em w1 e w2

# Mostra os resultados encontrados pelo que eu fiz acima
print("Matriz U: \n", matriz_U, "\n")
print("Matriz S: \n", matriz_S, "\n")
print("Matriz Vt: \n", np.transpose(matriz_V))
print("\n", "-"*60, "\n")

# Aqui eu já utilizo diretamente a função SVD da numpy
U, s, V = np.linalg.svd(matriz_A)
S = np.diag(s)
Vt = np.transpose(V)

# Mostra os resultados da função SVD da numpy
print("Matriz U (numpy): \n", U, "\n")
print("Matriz S (numpy): \n", S, "\n")
print("Matriz Vt (numpy): \n",Vt, "\n")
print("\n", "-"*60, "\n")

numpy_US = np.matmul(U, matriz_S) # Multiplica U por S
numpy_USVt = np.matmul(numpy_US, V) # Multiplica (U*S) por Vt

matriz_US = np.matmul(matriz_U, matriz_S)
matriz_USVt = np.matmul(matriz_US, np.transpose(matriz_V))

print("Matriz_A: \n", matriz_A)
print("Matriz USVt = A(pedro): \n", matriz_USVt)
print("Matriz USVt = A(numpy): \n", numpy_USVt) # O resultado deveria ser a matriz A, mas tá diferente (porém próximo)


import numpy as np
import matplotlib.pyplot as plt
import scipy.fft

def sinWav(amp, freq, time, phase=0):
    return amp * np.sin(2 * np.pi * (freq * time - phase))

def plotFFT(f, speriod, time):
    """Plots a fast fourier transform

    Args:
        f (np.arr): A signal wave
        speriod (int): Number of samples per second
        time ([type]): total seconds in wave
    """

    N = speriod * time
    # sample spacing
    T = 1.0 / 800.0
    x = np.linspace(0.0, N*T, N, endpoint=False)

    yf = scipy.fft.fft(f)
    xf = scipy.fft.fftfreq(N, T)[:N//2]

    amplitudes = 1/speriod* np.abs(yf[:N//2])

    plt.plot(xf, amplitudes)
    plt.grid()
    plt.xlim([1,3])
    plt.show()

def main():
    srate = 800
    time  = {
        0: np.arange(0, 4, 1/srate),
        1: np.arange(4, 8, 1/srate),
        2: np.arange(8, 12, 1/srate)
    }

    signal = np.concatenate([
        sinWav(amp=0.25, freq=2, time=time[0]),
        sinWav(amp=1, freq=2, time=time[1]),
        sinWav(amp=0.5, freq=2, time=time[2])
    ])   # generate signal

    plotFFT(signal, speriod, 12)

main()


import jax.numpy as jnp
from jax import grad, jit, vmap
from jax import jacfwd, jacrev
from jax import random
import numpy as np
import sympy as sym
import matplotlib.pyplot as plt
from visualizaiton import Visualization

class UnconstrainedOptimization:
    def __init__(self, cost, direction_type, stepsize_type, x0,
                 termination_params, algo_params=None):
        self.cost = cost
        self.x0 = jnp.array(x0)
        self.direction_type = direction_type
        self.stepsize_type = stepsize_type
        self.algo_params = algo_params
        self.termination_params = termination_params

    def armijo_condition(self, alpha, sigma, x_current, x_next, direction):
        LHS = self.cost(x_current) - self.cost(x_next)
        RHS = - sigma * alpha * jnp.matmul(grad(self.cost)(x_current), direction)
        condition = True if (LHS >= RHS) else False
        return condition

    def get_step_size(self, x_current, direction):

        if self.stepsize_type == 'minimization_rule':
            # create symbolic step size: alpha
            alpha = sym.Symbol('alpha')
            # convert jax device array to numpy list
            x_current = x_current.tolist()
            direction = direction.tolist()
            sym_vec = []
            for d in range(len(direction)):
                mm = x_current[d] + alpha * direction[d]
                sym_vec.append(mm)
            sym_cost = self.cost(sym_vec)
            sym_derivative = sym.diff(sym_cost, alpha)
            alpha_sym = sym.solveset(sym_derivative, alpha)
            alpha_sym = list(alpha_sym)
            print(alpha_sym)

            # only keep the real number by checking data type
            i = 0
            while True:
                condition = type(alpha_sym[i] ** 2) is sym.numbers.Float
                if condition is True:
                    val = alpha_sym[i]
                    break
                else:
                    i += 1

            alpha = float(val)

        elif self.stepsize_type == 'constant':
            alpha = self.algo_params[2]

        elif self.stepsize_type == 'armijo_rule':
            beta, s, sigma = self.algo_params
            m = 0
            condition = False
            while True:
                alpha = pow(beta, m) * s
                x_next = x_current + alpha * direction
                condition = self.armijo_condition(alpha, sigma, x_current, x_next, direction)

                if condition is True:
                    break
                m += 1
        else:
            raise Exception(
                'Unknown step size type, please input one of the following: minimization_tule, constant, armijo rule')

        return alpha

    def get_gradient_direction(self, x_val):
        # get derivative of the cost function
        derivative = grad(self.cost)(x_val)

        if self.direction_type == 'steepest_descent':
            # D in case of steepest descent is identity matrix
            D = jnp.identity(len(x_val))
            direction = - jnp.matmul(D, derivative)

        elif self.direction_type == 'newton_method':
            hessian = jacfwd(jacrev(self.cost))(x_val)
            D = jnp.linalg.inv(hessian)
            direction = - jnp.matmul(D, derivative)

        else:
            raise Exception(
                'Unknown gradient based method, please input one of the following: '
                'steepest_descent, newton_method')

        return direction

    def update(self, x_current, X_opt, error_val, error_cache):

        # update states
        X_opt.append(np.asarray(x_current, dtype=float))
        direction = self.get_gradient_direction(x_current)
        step_size = self.get_step_size(x_current, direction)
        x_next = x_current + step_size * direction
        error_val = x_current - x_next
        error_norm = jnp.linalg.norm(error_val)
        error_cache.append(error_norm)

        return x_next, X_opt, error_norm, error_cache

    def run_algorithm(self):
        """
        Runs the algorithm provided the user with starting and termination conditions,
        If the termination condition is "convergence" then the algorithm runs until convergence
        is achieved upto the desired value, the other termination option is to run the algorithm for
        a fixed number of timesteps.
        :return: X_opt = a matrix with co-ordinates of all the states obtained from running the algorithm
        """

        # get termination conditions from algo_params
        termination_type, termination_condition = self.termination_params

        x_current = self.x0
        X_opt = []
        error_norm = np.inf
        error_cache = []
        fig = plt.figure()

        if termination_type == "convergence":
            while error_norm > termination_condition:
                x_next, X_opt, error_norm, error_cache = self.update(
                    x_current, X_opt, error_norm, error_cache)
                # update state vector
                x_current = x_next

        elif termination_type == "fixed_steps":
            for t in range(termination_condition):
                x_next, X_opt, error_norm, error_cache = self.update(
                    x_current, X_opt, error_norm, error_cache)
                # update state vector
                x_current = x_next

        return X_opt, error_cache

Gradient based methods/Unconstrained optimization/unconstrained_optimization.py

import numpy,import sympy,import jax,from jax


#!/usr/bin/env python3

import numpy as np
from rigid_transform_3D import rigid_transform_3D

# Test with random data

# Random rotation and translation
R = np.random.rand(3,3)
t = np.random.rand(3,1)

# make R a proper rotation matrix, force orthonormal
U, S, Vt = np.linalg.svd(R)
R = U@Vt

# remove reflection
if np.linalg.det(R) < 0:
   Vt[2,:] *= -1
   R = U@Vt

# number of points
n = 10

A = np.random.rand(3, n)
B = R@A + t

# Recover R and t
ret_R, ret_t = rigid_transform_3D(A, B)

# Compare the recovered R and t with the original
B2 = (ret_R@A) + ret_t

# Find the root mean squared error
err = B2 - B
err = err * err
err = np.sum(err)
rmse = np.sqrt(err/n)

print("Points A")
print(A)
print("")

print("Points B")
print(B)
print("")

print("Ground truth rotation")
print(R)

print("Recovered rotation")
print(ret_R)
print("")

print("Ground truth translation")
print(t)

print("Recovered translation")
print(ret_t)
print("")

print("RMSE:", rmse)

if rmse < 1e-5:
    print("Everything looks good!")
else:
    print("Hmm something doesn't look right ...")


import numpy as np
import matplotlib as mpl
import matplotlib.pyplot as plt

# Consider two functions,
#	f(x) = x
#   g(x) = x²
#
# Write a program that, by trial and error, finds approximately for which values of x
# the two graphs cross, i.e., f(x) = g(x).
# Do this by considering N equally distributed points in the interval, at each point checking whether:
# |f(x) - g(x)| < ɛ, where ɛ is some small number.
# Let N and ɛ  be user input to the program and let the result be printed to the screen.

# The base case is N = 400 and ɛ = 0.01.

g_xlim = [ -4, 4 ]
g_iter = 1000

def plot_fun( fun, name, col ):
	x_vals = np.linspace( g_xlim[0], g_xlim[1], g_iter, endpoint=True )
	y_vals = fun( x_vals )
	plt.plot( x_vals, y_vals, label = name, color = col )

def f(x):
	return x

def g(x):
	return x**2

plot_fun( f, 'x', 'B' )
plot_fun( g, 'x²', 'G' )

epsilon = 0.0001
N = 40000

v = []

for x in np.linspace( g_xlim[ 0 ], g_xlim[ 1 ], N ):
	delta = abs( f( x ) - g( x ) )

	if delta <= epsilon:
		v.append( x )
		plt.scatter( x, f(x) )

plt.legend()
plt.show()


import random
import numpy as np
import pandas as pd
from scipy import optimize

def seed_everything(seed=42):
    random.seed(seed)
    np.random.seed(seed)

def random_weights(num_assets):
    weights = np.random.rand(num_assets)
    return weights / sum(weights)

def stock_returns(data, log_returns=False, annual_days = 252):
    if log_returns:
       daily_returns = data.pct_change().apply(lambda x: np.log(1+x))
       returns = daily_returns.dropna(axis=0, how='all')
       covariance_matrix = returns.cov() * annual_days
    else:
       daily_returns = data.pct_change()
       returns = daily_returns.dropna(axis=0, how='all')
       covariance_matrix = returns.cov() * annual_days
    return (returns * annual_days).mean(), covariance_matrix

def portfolio_performance(weights, mean_returns, covariance_matrix):
    returns = mean_returns @ weights
    portfolio_variance = weights.T @ covariance_matrix @ weights
    portfolio_std = np.sqrt(portfolio_variance)
    return portfolio_std, returns

def neg_sharpe_ratio(weights, mean_returns, covariance_matrix, risk_free_rate=0.02):
    p_var, p_ret = portfolio_performance(weights, mean_returns, covariance_matrix)
    return -(p_ret - risk_free_rate) / p_var

def max_sharpe_ratio(mean_returns, covariance_matrix, risk_free_rate=0.02):
    num_assets = len(mean_returns)
    args = (mean_returns, covariance_matrix, risk_free_rate)
    constraints = ({'type': 'eq', 'fun': lambda x: np.sum(x) - 1})
    bound = (0.0, 1.0)
    bounds = tuple(bound for asset in range(num_assets))
    result = optimize.minimize(neg_sharpe_ratio, num_assets*[1./num_assets,], args=args,
                        method='SLSQP', bounds=bounds, constraints=constraints)
    return [round(it, 3) for it in result.x]

def portfolio_volatility(weights, mean_returns, covariance_matrix):
    return portfolio_performance(weights, mean_returns, covariance_matrix)[0]

def min_volatility(mean_returns, covariance_matrix):
    num_assets = len(mean_returns)
    args = (mean_returns, covariance_matrix)
    constraints = ({'type': 'eq', 'fun': lambda x: np.sum(x) - 1})
    bound = (0.0,1.0)
    bounds = tuple(bound for asset in range(num_assets))
    result = optimize.minimize(portfolio_volatility, num_assets*[1./num_assets,], args=args,
                        method='SLSQP', bounds=bounds, constraints=constraints)
    # summarize the result
    #print('Status : %s' % result['message'])
    return [round(it, 3) for it in result.x]

def efficient_return(mean_returns, covariance_matrix, target_return=0.055):
    """
    Efficient return (minimize risk given a target return)
    """
    num_assets = len(mean_returns)
    args = (mean_returns, covariance_matrix)

    def portfolio_return(weights):
        return portfolio_performance(weights, mean_returns, covariance_matrix)[1]
    constraints = ({'type': 'eq', 'fun': lambda x: portfolio_return(x) - target_return},
                   {'type': 'eq', 'fun': lambda x: np.sum(x) - 1})
    bounds = tuple((0,1) for asset in range(num_assets))
    result = optimize.minimize(portfolio_volatility, num_assets*[1./num_assets,], args=args, method='SLSQP', bounds=bounds, constraints=constraints)
    return [round(it, 3) for it in result.x]

def efficient_frontier(mean_returns, cov_matrix, returns_range):
    efficients = []
    for ret in returns_range:
        efficients.append(efficient_return(mean_returns, covariance_matrix, ret))
    return efficients

def diversification_ratio(weights, mean_returns, covariance_matrix):
    """
    The weighted average of volatility divided by the portfolio volatility
    """
    portfolio_std, returns = portfolio_performance(weights, mean_returns, covariance_matrix)
    weights_std = np.dot(np.sqrt(np.diag(covariance_matrix)), weights.T)
    diversification_ratio = weights_std / portfolio_std
    # return negative for minimization problem (maximize = minimize -)
    return -diversification_ratio

def max_diversification(mean_returns, covariance_matrix, long_only=True):
    """
    Maximum Diversification
    Reference Choueifaty, Yves, and Yves Coignard. 2008. “Toward Maximum Diversification.”
    paper https://www.tobam.fr/wp-content/uploads/2014/12/TOBAM-JoPM-Maximum-Div-2008.pdf
    """
    # long only: long only constraint
    args = (mean_returns, covariance_matrix)
    num_assets = len(mean_returns)
    bounds = tuple((0,1) for asset in range(num_assets))
    cons = ({'type': 'eq', 'fun': lambda x: np.sum(x) - 1})
    if long_only: # add in long only constraint
        cons = ({'type': 'ineq', 'fun':  lambda x: np.sum(x) - 1})
    result = optimize.minimize(diversification_ratio, num_assets*[1./num_assets,],  args=args, bounds=bounds, method='SLSQP', constraints=cons)
    return [round(it, 3) for it in result.x]

def monte_carlo_optimisation(mean_returns, covariance_matrix, num_portfolios, risk_free_rate=0.02):
    """
    Monte Carlo Simulation
    """
    portfolio_returns = [] # Define an empty array for portfolio returns
    portfolio_volatility = [] # Define an empty array for portfolio volatility
    portfolio_weights = [] # Define an empty array for asset weights

    for portfolio in range(num_portfolios):
        weights = random_weights(len(mean_returns))
        volatility, returns = portfolio_performance(weights, mean_returns, covariance_matrix)
        portfolio_volatility.append(volatility)
        portfolio_weights.append(weights)
        portfolio_returns.append(returns)

    data = {'Returns': portfolio_returns, 'Volatility': portfolio_volatility}
    for counter, symbol in enumerate(mean_returns.index.tolist()):
        data[symbol+' weight'] = [w[counter] for w in portfolio_weights]
    # Dataframe of the 10000 portfolios created
    portfolios = pd.DataFrame(data)

    assert int(portfolios.iloc[0,2:].values.sum()) <= 1, "Weights sum should be equal 1"
    portfolios.head()

    minimum_volatility_portfolio = portfolios.iloc[portfolios['Volatility'].idxmin()] # idxmin() gives us the minimum value in the column 'Volatility'.
    max_sharpe_ratio = portfolios.iloc[((portfolios['Returns']- risk_free_rate) / portfolios['Volatility']).idxmax()]

    return minimum_volatility_portfolio, max_sharpe_ratio