r"""
Linear Regression
=================

This example shows how to use the :py:class:`pylops.LinearRegression` operator
to perform *Linear regression analysis*.

In short, linear regression is the problem of finding the best fitting
coefficients, namely intercept :math:`\mathbf{x_0}` and gradient
:math:`\mathbf{x_1}`, for this equation:

    .. math::
        y_i = x_0 + x_1 t_i   \qquad \forall i=1,2,...,N

As we can express this problem in a matrix form:

    .. math::
        \mathbf{y}=  \mathbf{A} \mathbf{x}

our solution can be obtained by solving the following optimization problem:

    .. math::
        J= ||\mathbf{y} - \mathbf{A} \mathbf{x}||_2

See documentation of :py:class:`pylops.LinearRegression` for more detailed
definition of the forward problem.
"""
import numpy as np
import matplotlib.pyplot as plt

import pylops

plt.close('all')
np.random.seed(10)

###############################################################################
# Define the input parameters: number of samples along the t-axis (``N``),
# linear regression coefficients (``x``), and standard deviation of noise
# to be added to data (``sigma``).
N = 30
x = np.array([1., 2.])
sigma = 1

###############################################################################
# Let's create the time axis and initialize the
# :py:class:`pylops.LinearRegression` operator
t = np.arange(N, dtype='float64')
LRop = pylops.LinearRegression(t, dtype='float64')

###############################################################################
# We can then apply the operator in forward mode to compute our data points
# along the x-axis (``y``). We will also generate some random gaussian noise
# and create a noisy version of the data (``yn``).
y = LRop*x
yn = y + np.random.normal(0, sigma, N)

###############################################################################
# We are now ready to solve our problem. As we are using an operator from the
# :py:class:`pylops.LinearOperator` family, we can simply use ``/``,
# which in this case will solve the system by means of an iterative solver
# (i.e., :py:func:`scipy.sparse.linalg.lsqr`).
xest = LRop / y
xnest = LRop / yn

###############################################################################
# Let's plot the best fitting line for the case of noise free and noisy data
plt.figure(figsize=(5, 7))
plt.plot(np.array([t.min(), t.max()]),
         np.array([t.min(), t.max()]) * x[1] + x[0], 'k', lw=4,
         label=r'true: $x_0$ = %.2f, $x_1$ = %.2f' % (x[0], x[1]))
plt.plot(np.array([t.min(), t.max()]),
         np.array([t.min(), t.max()]) * xest[1] + xest[0], '--r', lw=4,
         label=r'est noise-free: $x_0$ = %.2f, $x_1$ = %.2f'
         %(xest[0], xest[1]))
plt.plot(np.array([t.min(), t.max()]),
         np.array([t.min(), t.max()]) * xnest[1] + xnest[0], '--g', lw=4,
         label=r'est noisy: $x_0$ = %.2f, $x_1$ = %.2f'
         %(xnest[0], xnest[1]))
plt.scatter(t, y, c='r', s=70)
plt.scatter(t, yn, c='g', s=70)
plt.legend()

###############################################################################
# Once that we have estimated the best fitting coefficients :math:`\mathbf{x}`
# we can now use them to compute the *y values* for a different set of values
# along the *t-axis*.
t1 = np.linspace(-N, N, 2*N, dtype='float64')
y1 = LRop.apply(t1, xest)

plt.figure(figsize=(5, 7))
plt.plot(t, y, 'k', label='Original axis')
plt.plot(t1, y1, 'r', label='New axis')
plt.scatter(t, y, c='k', s=70)
plt.scatter(t1, y1, c='r', s=40)
plt.legend()

###############################################################################
# We consider now the case where some of the observations have large errors.
# Such elements are generally referred to as *outliers* and can affect the
# quality of the least-squares solution if not treated with care. In this
# example we will see how using a L1 solver such as
# :py:func:`pylops.optimization.sparsity.IRLS` can drammatically improve the
# quality of the estimation of intercept and gradient.

# Add outliers
yn[1] += 40
yn[N-2] -= 20

# IRLS
nouter = 20
epsR = 1e-2
epsI = 0
tolIRLS = 1e-2

xnest = LRop / yn
xirls, nouter, xirls_hist, rw_hist = \
    pylops.optimization.sparsity.IRLS(LRop, yn, nouter, threshR=False,
                                      epsR=epsR, epsI=epsI,
                                      tolIRLS=tolIRLS, returnhistory=True)
print('IRLS converged at %d iterations...' % nouter)

plt.figure(figsize=(5, 7))
plt.plot(np.array([t.min(), t.max()]),
         np.array([t.min(), t.max()]) * x[1] + x[0], 'k', lw=4,
         label=r'true: $x_0$ = %.2f, $x_1$ = %.2f' % (x[0], x[1]))
plt.plot(np.array([t.min(), t.max()]),
         np.array([t.min(), t.max()]) * xnest[1] + xnest[0], '--r', lw=4,
         label=r'L2: $x_0$ = %.2f, $x_1$ = %.2f' %
         (xnest[0], xnest[1]))
plt.plot(np.array([t.min(), t.max()]),
         np.array([t.min(), t.max()]) * xirls[1] + xirls[0], '--g', lw=4,
         label=r'L1 - IRSL: $x_0$ = %.2f, $x_1$ = %.2f' %
         (xirls[0], xirls[1]))
plt.scatter(t, y, c='r', s=70)
plt.scatter(t, yn, c='g', s=70)
plt.legend()

###############################################################################
# Let's finally take a look at the convergence of IRLS. First we visualize
# the evolution of intercept and gradient

fig, axs = plt.subplots(2, 1, figsize=(8, 10))
fig.suptitle('IRLS evolution', fontsize=14,
             fontweight='bold', y=0.95)
axs[0].plot(xirls_hist[:, 0], xirls_hist[:, 1], '.-k', lw=2, ms=20)
axs[0].scatter(x[0], x[1], c='r', s=70)
axs[0].set_title('Intercept and gradient')
axs[0].grid()
for iiter in range(nouter):
    axs[1].semilogy(rw_hist[iiter],
                    color=(iiter/nouter, iiter/nouter, iiter/nouter),
                    label='iter%d' %iiter)
axs[1].set_title('Weights')
axs[1].legend(loc=5, fontsize='small')
plt.tight_layout()
plt.subplots_adjust(top=0.8)


from sys import argv
from matplotlib.pylab import figure, plot, show, subplot, title, xlabel
from scipy import array, linspace, ndarray
from scipy.interpolate import CubicSpline
from sympy import sympify, SympifyError
from integral_equation import integral_equation
from utils import print_error

def get_limits():
    try:
        a, b = map(float, input('Insert the limits with the format "a, b": ').split(','))
    except ValueError:
        print_error('Error: Incorrect format, the limits should be a numbers and the separator should be a comma ",".')
    return a, b

def get_tabular_form_input():
    try:
        n = int(input('Insert the number of points of the function: '))
    except ValueError:
        print_error('Error: The number of points should be a integer.')
    print('Insert the points with the format "x, y":')
    x_points, y_points = [], []
    for _ in range(n):
        try:
            x, y = map(float, input().split(','))
        except ValueError:
            print_error('Error: Incorrect format, the points should be a numbers and the separator should be a comma ",".')
        x_points.append(x)
        y_points.append(y)
    return x_points, y_points

def get_tabular_form():
    x_points, y_points = get_tabular_form_input()
    function = CubicSpline(x_points, y_points)
    return function

def get_analytical_form(dim2 = False):
    try:
        function = sympify(input('Insert the analytical form of the function: '))
    except SympifyError:
        print_error('Error: It is not a recognizable function')
    if dim2:
        return lambda x, y: function.evalf(subs={'x': x, 'y': y})
    return lambda x: function.evalf(subs={'x': x})

def sample1():
    number_sample = 1
    print(f'Sample {number_sample}:')
    K = lambda x, y: sympify('1 + x * y').evalf(subs={'x': x, 'y': y})
    f = lambda x: sympify('1').evalf(subs={'x': x})
    a = -1
    b = 1
    print(f'K(x, y) = {K}')
    print(f'f(x) = {f}')
    print(f'Interval = [{a}, {b}]')
    solve1, cond1 = integral_equation(K, f, a, b, use_python_libs=True)
    solve2, cond2 = integral_equation(K, f, a, b, use_python_libs=False)
    x = linspace(a, b)
    y1 = solve1(x)
    y2 = solve2(x)
    figure(f'Sample {number_sample}')
    subplot(1, 2, 1)
    plot(x, y1)
    title(f'Sample {number_sample}: Use Python Libraries')
    xlabel(f'Matrix condition: {cond1}')
    subplot(1, 2, 2)
    plot(x, y2)
    title(f'Sample {number_sample}: NO use Python Libraries')
    xlabel(f'Matrix condition: {cond2}')
    show()

def sample2():
    number_sample = 2
    print(f'Sample {number_sample}:')
    K = lambda x, y: sympify('1 + x * y').evalf(subs={'x': x, 'y': y})
    f = lambda x: sympify('2 * x / 3').evalf(subs={'x': x})
    a = -1
    b = 1
    print(f'K(x, y) = {K}')
    print(f'f(x) = {f}')
    print(f'Interval = [{a}, {b}]')
    solve1, cond1 = integral_equation(K, f, a, b, use_python_libs=True)
    solve2, cond2 = integral_equation(K, f, a, b, use_python_libs=False)
    x = linspace(a, b)
    y1 = solve1(x)
    y2 = solve2(x)
    figure(f'Sample {number_sample}')
    subplot(1, 2, 1)
    plot(x, y1)
    title(f'Sample {number_sample}: Use Python Libraries')
    xlabel(f'Matrix condition: {cond1}')
    subplot(1, 2, 2)
    plot(x, y2)
    title(f'Sample {number_sample}: NO use Python Libraries')
    xlabel(f'Matrix condition: {cond2}')
    show()

if __name__ == "__main__":
    try:
        if len(argv) > 1 and argv[1] == 'samples':
            sample1()
            sample2()
            exit()
        option = int(input('Insert 1 if use Python libraries or 2 if no: '))
        if option != 1 and option != 2:
            print_error('Error: The number of option should be 1 or 2.')
        use_python_libs = option == 1
        option = int(input('Insert 1 if the function K is in the form of a table or 2 if it is in analytical form: '))
        if option != 1 and option != 2:
            print_error('Error: The number of option should be 1 or 2.')
        K = get_tabular_form() if option == 1 else get_analytical_form(True)
        option = int(input('Insert 1 if the function f is in the form of a table or 2 if it is in analytical form: '))
        if option != 1 and option != 2:
            print_error('Error: The number of option should be 1 or 2.')
        f = get_tabular_form() if option == 1 else get_analytical_form()
        a, b = get_limits()
        solve, cond = integral_equation(K, f, a, b, use_python_libs=use_python_libs)
        x = linspace(a, b)
        y = solve(x)
        plot(x, y)
        title(f'Matrix condition: {cond}')
        show()
    except ValueError:
        print_error('Error: The number of option should be a integer.')


from sympy import symbols, poly, zeros, simplify
from anmi.T2 import factorizacion_QR

def matriz_krylov(A, x, n_iters=None):
    """Genera una matriz de krylov dada una matriz A y un vector x. Cada columna de la matriz es la iteración i de A^i*x.

    Args:
        A (matriz): Matriz de aplicación
        x (vector): Vector base
        n_iters (int, optional): Número de iteraciones. Por defecto es el número de filas de A + 1 (garantiza que la matriz de krylov tiene una combinación lineal).

    Returns:
        m_krylov: Matriz con las aplicaciones de krylov por columna.
    """
    if n_iters is None:
        n_iters = A.shape[0] + 1

    m_krylov = zeros(A.shape[0], n_iters)
    m_krylov[:, 0] = x

    for i in range(1, n_iters):
        m_krylov[:, i] = A * m_krylov[:, i - 1]

    return simplify(m_krylov)

def sucesion_krylov(A, x):
    """
    La sucesión de krylov viene dada como la expresión (ejemplo para 3x3)
    p(l) = -l³ + a2 * l² + a1 * l + a0
    donde a = [a0, a1, a2] y es la solución del sistema de ecuaciones
    [x.T, Ax.T, Ax².T] * a.T = Ax³.T

    Args:
        A (matriz): Matriz para aplicar Krylov
        x (vector): Vector base

    Returns:
        poli_krylov (poly): Polinomio para la matriz de Krylov.
        a (vector): vector de coeficientes.
    """
    m_krylov = matriz_krylov(A, x)

    a = (m_krylov[:, : A.shape[0]] ** (-1)) * (
        ((-1) ** (A.shape[0] + 1)) * m_krylov[:, -1]
    )
    a = simplify(a)

    lambda_ = symbols("lambda")
    poli_krylov = poly(((-1) ** A.shape[0]) * (lambda_ ** A.shape[0]) + a[0])

    for i in range(1, A.shape[0]):
        poli_krylov += poly(a[i] * (lambda_ ** i))

    return poli_krylov, a

def potencia_iterada(A, x, n_iters, devolver_ultimo=True):
    """Aplica el método de la potencia iterada para calcular el mayor autovalor de la matriz, usando el método de Krylov.

    Args:
        A (matriz): Matriz aplicación
        x (vector): Vector base para el método. Si el vector es autovector dará fallo.
        n_iters (int): Número de iteraciones
        devolver_ultimo (bool, optional): Si True, devuelve el vector de la última iteración. Si False, devuelve todas las iteraciones.

    Returns:
        m_cocientes (matriz, vector): matriz/vector con el número de filas igual al de A, con los cocientes. Los números deberían tender al mayor autovalor de A.
    """

    m_krylov = matriz_krylov(A, x, n_iters=n_iters)

    if devolver_ultimo:
        m_cocientes = zeros(m_krylov.shape[0], 1)
        for row in range(m_cocientes.shape[0]):
            m_cocientes[row, 0] = (
                m_krylov[row, m_krylov.shape[1] - 1]
                / m_krylov[row, m_krylov.shape[1] - 2]
            )

    else:
        m_cocientes = zeros(m_krylov.shape[0], m_krylov.shape[1] - 1)
        for col in range(m_cocientes.shape[1]):
            for row in range(
                m_cocientes.shape[0]
            ):  # No hay element-wise division en sympy
                m_cocientes[row, col] = m_krylov[row, col + 1] / m_krylov[row, col]

    m_cocientes = simplify(m_cocientes)
    return m_cocientes

def metodo_autovals_QR(A, n_iters=3, verbose=True):
    """Aplica el método QR para el cálculo de autovalores de una matriz.

    Args:
        A (matriz): Matriz para el metodo
        n_iters (int, optional): Número de iteraciones. Defaults to 3.
        verbose (bool, optional): Imprime información intermedia.

    Returns:
        dict: "A": lista de valores de A = R*Q en cada iteración, "R" Y "Q": matrices Q y R del método.
    """
    list_A, list_Q, list_R = [], [], []

    # La lista de A tiene los A^{(2)} hasta A^{(k+1)}, porque A^{(1)} = A, y ya lo conocemos
    for i in range(n_iters):
        dict_QR = factorizacion_QR(A, verbose=verbose)
        Q, R = simplify(dict_QR["Q"]), simplify(dict_QR["R"])
        list_Q.append(Q)
        list_R.append(R)

        A = simplify(R * Q).copy()
        list_A.append(A)

    return {"A": list_A, "Q": list_Q, "R": list_R}


#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Time     : 2018/10/23 14:02
# @Author   : Iydon
# @File     : 5.3.4.py

import numpy as np

def linspace(start, end, num=None, step=None):
    """
    Args:
        num:  numpy.linspace(start, end, num)
        step: start:step:end
    """
    if num:
        sepera = (end-start) / (num-1)
        return [start+i*sepera for i in range(num)]
    elif step:
        length = int((end-start)/step)
        return [start+i*step for i in range(length+1)]
    else:
        return [start, end]

def taylor_method(dy0, start, end, step, **args):
    """
    Approximate the solution of the initial-value
    problem.
    ----------------------------
    Args:
        args: "d%dy", like "d1y", "d2y".
    """
    xs = linspace(start, end, step=step)
    ys = [dy0 for i in xs]
    for i in range(len(xs)-1):
        tmp,pro = 0,1
        for j in range(len(args)):
            tmp += pro * args["d%dy"%(j+1)](xs[i],ys[i])
            pro *= step / (j+2)
        ys[i+1] = ys[i] + step*tmp
    return ys

def modified_euler_method(dy, dy0, start, end, step):
    """
    Approximate the solution of the initial-value
    problem.
    | y' = f(t,y), a<=t<=b
    | y(a) = /alpha
    ----------------------------
    Args:
        dy:    y'.
        dy0:   /alpha.
        start: a.
        end:   b.
        step:  h.
    """
    xs = linspace(start, end, step=step)
    ys = [dy0 for i in xs]
    for i in range(len(xs)-1):
        tmp = ys[i] + step*dy(xs[i],ys[i])
        ys[i+1] = ys[i] + step*(dy(xs[i],ys[i])+dy(xs[i+1],tmp))/2
    return ys

def runge_kutta(dy, dy0, start, end, step, order:int=2):
    """
    Runge-Kutta Methods of Order Two.
    """
    if order not in [1,2,3,4]:
        raise ValueError
    xs = linspace(start, end, step=step)
    ys = [dy0 for i in xs]
    for i in range(len(xs)-1):
        k1 = step * dy(xs[i],ys[i])
        if order<2:
            ys[i+1] = ys[i] + k1
        elif order<3:
            k2 = step * dy(xs[i]+step/2,ys[i]+k1/2)
            ys[i+1] = ys[i] + k2
        elif order<4:
            k2 = step * dy(xs[i]+step/2,ys[i]+k1/2)
            k3 = step * dy(xs[i]+step,ys[i]-k1+2*k2)
            ys[i+1] = ys[i] + (k1+4*k2+k3)/6
        elif order<5:
            k2 = step * dy(xs[i]+step/2,ys[i]+k1/2)
            k3 = step * dy(xs[i]+step/2,ys[i]+k2/2)
            k4 = step * dy(xs[i]+step,ys[i]+k3)
            ys[i+1] = ys[i] + (k1+2*k2+2*k3+k4)/6
    return ys

d1y = lambda t,y: t*np.exp(3*t)-2*y
d2y = lambda t,y: (t+1)*np.exp(3*t)-4*y
result = taylor_method(0, 0, 1, 0.1, d1y=d1y, d2y=d2y)
print(result)

print("*"*ord("*"))

y = lambda t: t*np.exp(3*t)/5 - np.exp(3*t)/25 + np.exp(-2*t)/25
true_val = [y(i) for i in [0, 0.5, 1]]
print(true_val)
true_val = np.array(true_val)

dy = lambda t,y: t*np.exp(3*t) - 2*y
result = modified_euler_method(dy, 0, 0, 1, 0.5)
print(result, sum(abs(true_val-result)))

result = runge_kutta(dy, 0, 0, 1, 0.5, 2)
print(result, sum(abs(true_val-result)))

result = runge_kutta(dy, 0, 0, 1, 0.5, 4)
print(result, sum(abs(true_val-result)))


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Wed Dec  2 14:29:37 2020

Copyright 2020 by Hadrien Montanelli.
"""
# %% Imports.

# Standard library imports:
import numpy as np

# Chebpy imports:
from chebpy.cheb import chebpts, quadwts

# %% Integrate f(x) = exp(x) on [-1,1].

# Function:
f = lambda x: np.exp(x)

# Grid and weights:
n = 12
x = chebpts(n)
w = quadwts(n)

# Compute integral with quadrature:
I = w @ f(x)

# Compute error:
Iex = np.exp(1) - np.exp(-1)
error = np.abs(I - Iex)
print(f'Error: {error:.2e}')

# %% Integrate f(x) = exp(x) on [0,2].

# Function:
f = lambda x: np.exp(x)

# Grid and weights:
n = 12
x = chebpts(n, [0, 2])
w = quadwts(n, [0, 2])

# Compute integral with quadrature:
I = w @ f(x)

# Compute error:
Iex = np.exp(2) - np.exp(0)
error = np.abs(I - Iex)
print(f'Error: {error:.2e}')


import math
import numpy as np

def get_basis_dual(a1, a2):
    '''
    For the lattice
    :param a1, a2: basis vectors
    :return: dual basis vectors; https://en.wikipedia.org/wiki/Reciprocal_lattice
    '''
    R = np.array([[0, 1], [-1, 0]])  # rotation by 90deg
    b1 = 2 * np.pi * (R @ a2) / (a1 @ (R @ a2))
    b2 = 2 * np.pi * (R @ a1) / (a2 @ (R @ a1))
    return b1, b2

def mod2pi(x):
    '''
    fmod(x,y) is not equivalent to (x % y): https://docs.python.org/3/library/math.html and
    is preferred when working with floats
    :return: a value in interval from 0 to 2pi
    '''
    x = math.fmod(x, 2 * np.pi)
    if x < 0:
        x += 2 * np.pi
    return x


# %matplotlib inline
import numpy as np
import matplotlib.pyplot as plt
plt.style.use('seaborn-white')

data = np.random.randn(1000)
plt.hist(data)

# Pass additional args to plt.hist() to customize output
plt.hist(data, bins=30, density=True, alpha=0.5, histtype='stepfilled',
         color='steelblue', edgecolor='none')

# This approach is more granular, and capable of multiple distributions
x1 = np.random.normal(0, 0.8, 1000)
x2 = np.random.normal(-2, 1, 1000)
x3 = np.random.normal(3, 2, 1000)

kwargs = dict(histtype='stepfilled', alpha=0.3, density=True, bins=40)

plt.hist(x1, **kwargs)
plt.hist(x2, **kwargs)
plt.hist(x3, **kwargs)

# To compute histogram without displaying it (count elements in given bin),
# use np.histogram().
counts, bin_edges = np.histogram(data, bins=5)
print(counts)

# Displaying & customizing 2D histogram plots:
mean = [0, 0]
cov = [[1, 1], [1, 2]]
x, y = np.random.multivariate_normal(mean, cov, 10000).T

# 1 - plt.hist2d()
plt.hist2d(x, y, bins=30, cmap='Blues')
cb = plt.colorbar()
cb.set_label('counts in bin')
# as with plt.hist() above, can customize bins & plot output.

# 2 - np.histogram2d()
# plt.hist() is to np.histogram(),
# as plt.hist2d is to np.histogram2d
counts, xedges, yedges = np.histogram2d(x, y, bins=30)

# for dimensionality > 2, use np.histogramdd()

# Hexagonal binnings with plt.hexbin():

# By default, the 2d histogram creates a tesselation of squares across its axes
# Can specify hexagons instead if desired.
plt.hexbin(x, y, gridsize=30, cmap='Blues')
cb = plt.colorbar(label='count in bin')

# Kernel Density Estimation:

# KDE is a method of evaluating probability density function of a random var
# Said random variable can be multi-dimensional.
# KDE gets a dedicated chapter later.

# For now, demo a simple KDE implementation from scipy.stats
from scipy.stats import gaussian_kde

# Fit data to an array of size [Ndim, Nsamples].
data = np.vstack([x, y])
kde = gaussian_kde(data)

# Evaluate on a regular grid
xgrid = np.linspace(-3.5, 3.5, 40)
ygrid = np.linspace(-6, 6, 40)
Xgrid, Ygrid = np.meshgrid(xgrid, ygrid)
Z = kde.evaluate(np.vstack([Xgrid.ravel(), Ygrid.ravel()]))

# Plot the result as img
plt.imshow(Z.reshape(Xgrid.shape), origin='lower', aspect='auto',
           extent=[-3.5, 3.5, -6, 6], cmap='Blues')
cb = plt.colorbar()
cb.set_label("density")

# The output demonstrates the smoothing effect in action.
# There's a loss of fidelity, but gaussian_kde autofinds a near-optimal
# smoothing length on the input data.

# other KDE implementations in SciPy include:

# sklearn.neighbors.KernelDensity
# statsmodels.nonparametric.kernel_density.KDEMultivariate


from skimage import io, filters
import numpy as np
from matplotlib import pyplot as plt
from scipy import ndimage

im = io.imread('me.jpeg')
im = im[:,:,0]
im = im.astype('float')

lin, col = im.shape

im2 = np.zeros((lin,col))

filtro = np.array([[1, 0, -1],
                   [1, 0, -1],
                   [1, 0, -1]]) / 6

for i in range(1, lin-1):
    for j in range(1, col-1):

        sub_im = im[i-1:i+2, j-1:j+2]

        mult = sub_im * filtro
        media = mult.sum()

        im2[i,j] = media

# Filtro passa baixa, onde temos o realce das bordas da imagem
plt.figure()
plt.subplot(121)
plt.imshow(im.astype('uint8'), cmap='gray')
plt.subplot(122)
plt.imshow(im2.astype('uint8'), cmap='gray')

# ------------------------------------

# filtro mediana, deixamos a imagem mais clara e nítida

im2 = np.zeros((lin,col))

for i in range(1, (lin - 1)):
  for j in range(1, (col - 1)):
    subPic = im[i-1 : i+2, j-1 : j+2]
    median = np.median(subPic)
    if(not((im[i][j] < (median-5)) or (im[i][j] > (median+5)))):
      im2[i][j] = median
    else:
      im2[i][j] = im[i][j]

plt.figure()
plt.subplot(121)
plt.imshow(im.astype('uint8'), cmap='gray')
plt.subplot(122)
plt.imshow(im2.astype('uint8'), cmap='gray')

# ---------------------
# Recorte de 10 pixels da imagem inicial na área central

nim = im[800:810,440:450].astype('uint8')
plt.figure()
plt.imshow(nim)

# ---------------------------------------
# reflect, constant, nearest, mirror, wrap passa baixa no recorte da imagem 10x10

shape = nim.shape

lin = shape[0]
col = shape[1]
print(im.shape)

k = np.array([[0,1,0], [0,1,0], [0,1,0]])

reflect = ndimage.filters.convolve(
  nim, k,
  mode='reflect'
)

constant = ndimage.filters.convolve(
  nim, k,
  mode='constant'
)
nearest = ndimage.filters.convolve(
  nim, k,
  mode='nearest'
)
mirror = ndimage.filters.convolve(
  nim, k,
  mode='mirror'
)
wrap = ndimage.filters.convolve(
  nim, k,
  mode='wrap'
)

plt.figure()
plt.subplot(231)
plt.imshow(reflect, cmap='gray')
plt.subplot(232)
plt.imshow(constant, cmap='gray')
plt.subplot(233)
plt.imshow(nearest, cmap='gray')
plt.subplot(234)
plt.imshow(mirror, cmap='gray')
plt.subplot(235)
plt.imshow(wrap, cmap='gray')

# ------------------------
# filtro passa baixa scharr_h
im2 = filters.scharr_h(im)

plt.figure()
plt.subplot(121)
plt.imshow(im.astype('uint8'), cmap='gray')
plt.subplot(122)
plt.imshow(im2, cmap='gray')

#
def plot(data, title):
    plot.i += 1
    plt.subplot(2,2,plot.i)
    plt.imshow(data)
    plt.gray()
    plt.title(title)

plot.i = 0

im = io.imread('me.jpeg')
plot(im, 'Original')

data = np.array(im, dtype=float)

# Outra maneira de fazer um filtro passa-alto é simplesmente subtrair um filtro passa-baixo
# imagem filtrada do original. Aqui, usaremos um filtro gaussiano simples
# para "desfocar" (ou seja, um filtro passa-baixa) o original.

lowpass = ndimage.gaussian_filter(data, 3)
gauss_highpass = data - lowpass
plot(gauss_highpass, r'Gaussian Highpass, $\sigma = 3 pixels$')

plt.show()


<<<<<<< HEAD
import numpy as np

def cross_entropy_error(y,t):
    if y.ndim==1:
        t = t.reshape(1,t.size)
        y = y.reshape(1,y.size)
    batch_size = y.shape[0]
    return -np.sum(np.log(y[np.arange(batch_size),t]+1e-7))/batch_size

def sum_squares_error(y,t):
    return 0.5*np.sum((y-t)**2)

def numerical_diff(f,x):
    h = 1e-4
    return (f(x+h)-f(x-h))/(2*h)

def func_1(x):
    return 0.01*x**2 + 0.1*x

def numerical_gradient(f,x):
    h = 1e-4
    grad = np.zeros_like(x)

    for idx in range(x.size):
        tmp_val = x[idx]
        x[idx] = tmp_val+h
        fxh1 = f(x)

        x[idx] = tmp_val -h
        fxh2 = f(x)

        grad[idx]= (fxh1-fxh2) / (2*h)
        x[idx] = tmp_val
    return grad

def gradient_descent(f,init_x, lr = 0.01, step_num = 100):
    x = init_x

    for i in range(step_num):
        grad = numerical_gradient(f,x)
        x -=lr*grad
    return x
=======
import numpy as np

def cross_entropy_error(y,t):
    if y.ndim==1:
        t = t.reshape(1,t.size)
        y = y.reshape(1,y.size)
    batch_size = y.shape[0]
    return -np.sum(np.log(y[np.arange(batch_size),t]+1e-7))/batch_size

def sum_squares_error(y,t):
    return 0.5*np.sum((y-t)**2)

def numerical_diff(f,x):
    h = 1e-4
    return (f(x+h)-f(x-h))/(2*h)

def func_1(x):
    return 0.01*x**2 + 0.1*x

def numerical_gradient(f,x):
    h = 1e-4
    grad = np.zeros_like(x)

    for idx in range(x.size):
        tmp_val = x[idx]
        x[idx] = tmp_val+h
        fxh1 = f(x)

        x[idx] = tmp_val -h
        fxh2 = f(x)

        grad[idx]= (fxh1-fxh2) / (2*h)
        x[idx] = tmp_val
    return grad

def gradient_descent(f,init_x, lr = 0.01, step_num = 100):
    x = init_x

    for i in range(step_num):
        grad = numerical_gradient(f,x)
        x -=lr*grad
    return x
>>>>>>> e8dc90c85b9519d31ca84678bd32c29f74538635


#-*- coding:utf-8 -*-
'''
Normal Distribution, also called Gaussian Distribution
'''

import numpy as np
import matplotlib.pyplot as plt

def simple_plot():
    x = np.linspace(0, 10, 10000)
    y = np.random.normal(0, x)
    z = np.cos(x**2)

    plt.figure(figsize = (8, 4))
    plt.plot(x, y, label = "sin(x)", color = "red", linewidth = 2)
    plt.plot(x, z, "b--", label = "cos(x^2)")
    plt.xlabel("Time(s)")
    plt.ylabel("Volt")
    plt.title("PyPlot First Example")
    plt.ylim(-1.2, 1.2)
    plt.legend()
    plt.show()

def normal_plot(mu = 0, sigma = 1, num = 1000):
    y = np.random.normal(mu, sigma, num)

    plt.figure(figsize = (8, 4))
    plt.plot(y, label = "norm", color = "red", linewidth = 2)
    plt.xlabel("X")
    plt.ylabel("Distribution")
    plt.title("Normal-Distribution")
    plt.legend()
    plt.show()

def normal_distribution(mu = 0, sigma = 1, start = None, end = None, num = 1000):
    if start and end:
        x = np.linspace(start, end, num)
    else:
        x = np.linspace(mu - 5 * sigma, mu + 5 * sigma, num)
    # gaussian density function
    y = np.e ** ((x - mu)**2 / (-2 * sigma ** 2)) / np.sqrt(2 * np.pi * sigma)

    plt.figure(figsize = (8, 4))
    plt.plot(x, y, color = "red", linewidth = 2)
    plt.xlabel("X")
    plt.ylabel("density")
    plt.title("Normal-Distribution")
    plt.show()

#simple_plot()
#normal_plot()
normal_distribution()


# -*- coding: utf-8 -*-
"""
Created on Fri Jul  2 12:09:14 2021

@author: vohuynhq
"""

import numpy as np
import pandas as pd
from sympy import symbols, init_printing, pi, sqrt, diff, sin, cos, exp

def example(A):
    A_true = np.array([[3, 4, 5], [3, 4, 5]])
    np.testing.assert_equal(A, A_true)
    return None

def derivative(eval_1,eval_2,eval_3,eval_4,*args):
    init_printing(use_unicode= True)
    x = symbols('x')

    f_1 = x **(3/2) + pi * x ** 2 + sqrt(2)
    diff_1 = diff(f_1,x)
    eval_1_true = diff_1.subs(x,2)

    f_2 = sin(x)*exp(cos(x))
    diff_2 = diff(f_2,x)
    eval_2_true= diff_2.subs(x,pi)

    f_3 = exp((x + 1) ** 2)
    diff_3 = diff(f_3,x)
    eval_3_true = diff_3.subs(x,1)

    f_4 = x ** 2 * (cos(x) ** 3)
    diff_4 = diff(f_4,x)
    eval_4_true = diff_4.subs(x,pi)

    assert eval_1 == eval_1_true, "Wrong answer!"
    assert eval_2 == eval_2_true, "Wrong answer!"
    assert eval_3 == eval_3_true, "Wrong answer!"
    assert eval_4 == eval_4_true, "Wrong answer!"

    return None

def jacobians_hessians(eval_1,eval_2,eval_3,eval_4):
    init_printing(use_unicode= True)
    x, y, z = symbols('x y z')

    f_1 = x ** 2 * cos(y) + exp(z)*sin(y)
    J_1 = [diff(f_1,x), diff(f_1,y), diff(f_1,z)]
    eval_1_true = [J_1[0].subs([(x,pi), (y,pi),(z,1)]),
              J_1[1].subs([(x,pi), (y,pi),(z,1)]),
              J_1[2].subs([(x,pi), (y,pi),(z,1)])]

    u = x ** 2 *y - cos(x) * sin(y)
    v = exp(x + y)
    J_u = [diff(u,x), diff(u,y)]
    J_v = [diff(v,x), diff(v,y)]
    eval_u = [J_u[0].subs([(x,0),(y,pi)]),
              J_u[1].subs([(x,0),(y,pi)])]
    eval_v = [J_v[0].subs([(x,0),(y,pi)]),
              J_v[1].subs([(x,0),(y,pi)])]
    eval_2_true = [eval_u,eval_v]

    f_3 = x ** 3 * cos(y) - x * sin(y)
    H_3 = [[diff(diff(f_3,x),x), diff(diff(f_3,x),y)],
           [diff(diff(f_3,y),x), diff(diff(f_3,y),y)]]
    eval_3_true = [[H_3[0][0].subs([(x,pi), (y,pi)]), H_3[0][1].subs([(x,pi),(y,pi)])],
                   [H_3[1][0].subs([(x,pi), (y,pi)]), H_3[1][1].subs([(x,pi),(y,pi)])]]

    f_4 = x * y * cos(z) - sin(x) * exp(y) * z**3
    H_4 = [[diff(diff(f_4,x),x), diff(diff(f_4,x),y), diff(diff(f_4,x),z)],
           [diff(diff(f_4,y),x), diff(diff(f_4,y),y), diff(diff(f_4,y),z)],
           [diff(diff(f_4,z),x), diff(diff(f_4,z),y), diff(diff(f_4,z),z)]]

    eval_4_true = [
        [H_4[0][0].subs([(x,pi),(y,pi),(z,pi)]), H_4[0][1].subs([(x,pi),(y,pi),(z,pi)]), H_4[0][2].subs([(x,pi),(y,pi),(z,pi)])],
        [H_4[1][0].subs([(x,pi),(y,pi),(z,pi)]), H_4[1][1].subs([(x,pi),(y,pi),(z,pi)]), H_4[1][2].subs([(x,pi),(y,pi),(z,pi)])],
        [H_4[2][0].subs([(x,pi),(y,pi),(z,pi)]), H_4[2][1].subs([(x,pi),(y,pi),(z,pi)]), H_4[2][2].subs([(x,pi),(y,pi),(z,pi)])]
        ]

    assert set(eval_1) == set(eval_1_true), 'Wrong answer!'
    assert set(eval_2) == set(eval_2_true), 'Wrong answer!'
    assert set(eval_3) == set(eval_3_true), 'Wrong answer!'
    assert set(eval_4) == set(eval_4_true), 'Wrong answer!'

    return None

def question1(m,b,X,y,*args):
    ##
    # Load dataset:
    #
    data = pd.read_csv("./dataset/Xy_dataset.csv")
    X = np.array(data['X'])
    y = np.array(data['y'])
    X = X.reshape(X.shape[0],1)
    y = y.reshape(y.shape[0],1)

    ##
    # Step 1: Initialize m and b:
    #
    m_true = 0
    b_true = 0

    for i in range(50):

        ##
        # Step 2: Find y_pred = mx + b:
        #
        y_pred = m*X + b

        ##
        # Step 3: Update m and b using the Gradient Descent algorithm:
        #
        dm = np.mean((y_pred - y) * X)
        db = np.mean(y_pred - y)
        m_true = m_true - 0.1*dm
        b_true = b_true - 0.1*db

    np.testing.assert_equal(m,m_true)
    np.testing.assert_equal(b,b_true)
    return None

def question2(m,b,X,y,*args):

    ##
    # Step 1: Initialize m and b:
    #
    m_true = 0
    b_true = 0

    for i in range(50):

        ##
        # Step 2: Find y_pred = mx + b:
        #
        y_pred = m*X + b

        ##
        # Step 3: Update m and b using the Gradient Descent algorithm:
        #
        dm = np.mean((y_pred - y) * X)
        db = np.mean(y_pred - y)
        m_true = m_true - 0.1*dm
        b_true = b_true - 0.1*db

    np.testing.assert_equal(m,m_true)
    np.testing.assert_equal(b,b_true)
    return None

def question3(m,b,X,y,*args):

    ##
    # Step 1: Initialize m and b:
    #
    m_true = 0
    b_true = 0

    for i in range(50):
        ##
        # Step 2: Find y_pred = mx + b:
        #
        y_pred = m*X + b

        ##
        # Step 3: Update m and b using the Gradient Descent algorithm:
        #
        dm = np.mean((y_pred - y) * X)
        db = np.mean(y_pred - y)
        m_true = m_true - 0.2*dm
        b_true = b_true - 0.2*db

    np.testing.assert_equal(m,m_true)
    np.testing.assert_equal(b,b_true)

    return None

def question4(m,b,X,y,*args):

    ##
    # Step 1: Initialize m and b:
    #
    m_true = 0
    b_true = 0

    for i in range(100):
        ##
        # Step 2: Find y_pred = mx + b:
        #
        y_pred = m*X + b

        ##
        # Step 3: Update m and b using the Gradient Descent algorithm:
        #
        dm = np.mean((y_pred - y) * X)
        db = np.mean(y_pred - y)
        m_true = m_true - 0.1*dm
        b_true = b_true - 0.1*db

    np.testing.assert_equal(m,m_true)
    np.testing.assert_equal(b,b_true)

    return None

1. Mathematics for Machine Learning/1.2 Multivariate Calculus/utils/autograder_multivariate_calculus.py


#######################################################    De la clase S3C1 #############################################################################
import numpy as np
import matplotlib.pylab as plt

# Escriba aca su funcion GENERAL de eliminacion Gaussiana para resolver el sistema Ax=B.

def GJ(A,B):
    NA=len(A)
    for c in range(NA):
        for r in range(NA):
            if(r==c):
                B[r]=B[r]/A[r,c]
                A[r,:]=A[r,:]/A[r,c]
            if(r>c):
                B[r]=B[r]-(B[c]*A[r,c])
                A[r,:]=A[r,:]-(A[c,:]*A[r,c])

    for c in range(NA):
        for r in range(NA):
            if(r<c):
                B[r]=B[r]-(B[c]*A[r,c])
                A[r,:]=A[r,:]-(A[c,:]*A[r,c])
    return B

########################################################################################################################################################
#######################################################   S3C2 minimos cuadrados     ###################################################################
########################################################################################################################################################
# Ejercicio: Minimos cuadrados (https://github.com/ComputoCienciasUniandes/MetodosComputacionales/blob/master/secciones/05.SistemasEcuaciones/lin_algebra.pdf)

# Almacene los datos de parabolico.dat
datos=np.genfromtxt('parabolico.dat')

# Grafique los datos. Guarde la grafica sin mostrarla en plotdatos.pdf
plt.figure()
plt.scatter(datos[:,0],datos[:,1],s=3,label='Datos')
plt.xlabel('X')
plt.ylabel('Y')
plt.title('Datos')
plt.legend()
plt.grid()
plt.savefig('plotdatos.pdf')
plt.close()

# teniendo en cuenta que el modelo que usted queiere ajustar a los datos es de la forma: xt=m1+m2*t+m3*0.5*t*t+ruido
# use los datos t y xt, implemente un esquema de minimos cuadrados para encontrar los mejores valores de los parametros m1, m2, m3. (ver explicación en el tablero)
t=datos[:,0]
xt=datos[:,1]
unos=np.ones(len(t))

AT=np.matrix([unos,t,(t**2)/2])   # La matriz creada de por sí es la Matriz transpuesta
A=AT.transpose()                # Se transpone la transpuesta para crear la matriz normal de tamaño N x 3
B=np.copy(xt)

Anew=np.dot(AT,A)
Bnew=np.dot(AT,B)
Bnew=Bnew.transpose()           #se voltea para que quede una matriz 3x1 y pueda entrar en la eliminación gaussiana

Bsol=GJ(Anew,Bnew)

sol=np.dot(A,Bsol)

# Grafique los datos y el ajuste con los parametros obtenidos. Guarde la grafica sin mostrarla en plotAjustedatos.pdf

plt.figure()
plt.scatter(datos[:,0],datos[:,1],s=3,c='b',label='Datos')
plt.plot(t,sol,c='r',label='Función')
plt.xlabel('X')
plt.ylabel('Y')
plt.title('Función Ajustada')
plt.legend()
plt.grid()
plt.savefig('plotAjustados.pdf')
plt.close()
########################################################################################################################################################
#######################################################   S3C3 Preparacion PCA       ###################################################################
########################################################################################################################################################
## Ejercicio casa:
# Los arrays `u` y `v` representan dos series en funcion del tiempo `t`.
# 1) Grafique las dos series de datos en una misma imagen y guarde dicha grafica sn mostrarla en 'serie.pdf'
# 2) Calcule la covarianza entre `u` y `v` e imprima su valor.
# 3) Calcule la varianza de `u` e imprima su valor.
# 4) Imprima un mensaje donde explique que puede inferir del valor de covarianza obtenido.
import numpy as np
t = np.array([0.,0.1,0.2,0.3,0.4,0.5,0.6, 0.8, 0.9])
u = np.array([-12.,-45.,-6.,-78.,-34.,-22.,10.,-31.,27.])
v = np.array([3.,11.,1.3,37.,11.,6.,-23.,7.,7.])

# si alcanzan:
# 5) Calcule la matriz de covarianza de los datos anteriores.
# Lectura preparatorias de PCA: https://github.com/ComputoCienciasUniandes/MetodosComputacionales/blob/master/secciones/05.SistemasEcuaciones/lin_algebra.pdf
# https://towardsdatascience.com/a-one-stop-shop-for-principal-component-analysis-5582fb7e0a9c


"""
Sample from posterior distribution using Markov Chain Monte Carlo.

Summary:

    Begin with parameter guess θ.

    1. Generate proposal θ′ with distribution Q(θ′ | θ)
    2. Accept this proposal with probability A(θ′, θ)
    3. Record current parameter

    Records are samples from posterior.
"""

import numpy as np
from matplotlib import pyplot as plt
from scipy import stats

def proposal(x, sigma=0.5):
    """
    Propose new parameters.
    """

    x_next = stats.norm(x, sigma).rvs(size=x.shape)

    return x_next

def log_likelihood(data, mu, sigma=1):
    """
    Calculate the log likelihood of some data assuming a normal distribution.
    """

    log_prob = np.log(stats.norm(mu, sigma).pdf(data)).sum()

    return log_prob

def log_prior(parameter, mu=0, sigma=1):
    """
    Calculate the log probability of some parameter assuming a normal prior distribution on that parameter.
    """

    log_probs = np.log(stats.norm(mu, sigma).pdf(parameter))

    return log_probs

def metropolis_hastings(data):
    """
    Calculate the posterior distribution for parameters given some data.
    """

    x_current = proposal(np.ones(1))
    ll_current = log_likelihood(data, x_current)
    log_prior_current = log_prior(x_current)

    posteriors = [x_current]

    for i in range(1100):

        x_proposal = proposal(x_current)
        ll_proposal = log_likelihood(data, x_proposal)
        log_prior_proposal = log_prior(x_proposal)

        accept = np.log(np.random.rand()) < ll_proposal + log_prior_proposal - ll_current - log_prior_current

        if accept:
            x_current, ll_current = x_proposal, ll_proposal

        posteriors.append(x_current)

    return posteriors

if __name__ == '__main__':

    np.random.seed(0)

    data = np.random.randn(1000) + 0.5

    print(data.mean(), data.std())

    posteriors = metropolis_hastings(data)

    plt.hist([x[0] for x in posteriors[100:]])
    plt.show()


import numpy as np
from sigmoid import sigmoid_
from vec_log_gradient import vec_log_gradient_
# Test n.1
x = np.array([1, 4.2])
# x[0] represent the intercept
y_true = 1
theta = np.array([0.5, -0.5])
y_pred = sigmoid_(np.dot(x, theta))
print(vec_log_gradient_(x, y_pred, y_true))
# [0.83201839 3.49447722]# Test n.2x = np.array([1, -0.5, 2.3, -1.5, 3.2]) # x[0] represent the intercepty_true = 0theta = np.array([0.5, -0.5, 1.2, -1.2, 2.3])y_pred = sigmoid_(np.dot(x, theta))print(vec_log_gradient_(x, y_true, y_pred))# [ 0.99999686 -0.49999843 2.29999277 -1.49999528 3.19998994]# Test n.3x_new = np.arange(2, 14).reshape((3, 4))x_new = np.insert(x_new, 0, 1, axis=1)# first column of x_new are now intercept values initialized to 1y_true = np.array([1, 0, 1])theta = np.array([0.5, -0.5, 1.2, -1.2, 2.3])y_pred = sigmoid_(np.dot(x_new, theta))print(vec_log_gradient_(x_new, y_true, y_pred))# [0.99994451 5.99988885 6.99983336 7.99977787 8.99972238]
x = np.array([1, -0.5, 2.3, -1.5, 3.2])
 # x[0] represent the intercept
y_true = 0
theta = np.array([0.5, -0.5, 1.2, -1.2, 2.3])
y_pred = sigmoid_(np.dot(x, theta))
print(vec_log_gradient_(x, y_true, y_pred))
# [ 0.99999686 -0.49999843 2.29999277 -1.49999528 3.19998994]# Test n.3x_new = np.arange(2, 14).reshape((3, 4))x_new = np.insert(x_new, 0, 1, axis=1)# first column of x_new are now intercept values initialized to 1y_true = np.array([1, 0, 1])theta = np.array([0.5, -0.5, 1.2, -1.2, 2.3])y_pred = sigmoid_(np.dot(x_new, theta))print(vec_log_gradient_(x_new, y_true, y_pred))# [0.99994451 5.99988885 6.99983336 7.99977787 8.99972238]

x_new = np.arange(2, 14).reshape((3, 4))
x_new = np.insert(x_new, 0, 1, axis=1)
# first column of x_new are now intercept values initialized to 1
y_true = np.array([1, 0, 1])
theta = np.array([0.5, -0.5, 1.2, -1.2, 2.3])
y_pred = sigmoid_(np.dot(x_new, theta))
print(vec_log_gradient_(x_new, y_true, y_pred))
# [0.99994451 5.99988885 6.99983336 7.99977787


import math
import numpy as np

### Flat Dimension
def flat_dim( shape ):

    return np.prod( shape[1:] )

### Flatten
def flatten( x ):

    return x.reshape( [ -1 , flat_dim( x ) ] )

### 2D Side Dimension
def side2D( x , d = 1 ):

    if isinstance( x , list ) : x = x[1]
    return int( round( math.pow( x / d , 1.0 / 2.0 ) ) )

### 3D Side Dimension
def side3D( x , d = 1 ):

    if isinstance( x , list ) : x = x[1]
    return int( round( math.pow( x / d , 1.0 / 3.0 ) ) )

### Shape 1D
def shape1D( x ):

    return [ None , x.shape[1] ]

### Shape 2D
def shape2D( x , channels = 1 ):

    side = side2D( x.shape[1] , channels )
    return [ None , side , side , channels ]

### Shape 3D
def shape3D( x , channels = 1 ):

    side = side3D( x.shape[1] , channels )
    return [ None , side , side , side , channels ]

### Spread
def spread( x , n ):

    return x if isinstance( x , list ) else [ x ] * n

### Flatten List
def flatten_list( list1 ):

    list2 = []

    for item1 in list1:
        if isinstance( item1 , list ):
            for item2 in item1:
                list2.append( item2 )
        else:
            list2.append( item1 )

    return list2


# x-axis represents the hours of the day
# y-axis represents the speed

# scatter plot

import numpy as np
import matplotlib.pyplot as plt

x = np.array([1,2,3,5,6,7,8,9,10,12,13,14,15,16,18,19,21,22])
y = np.array([100,90,80,60,60,55,60,65,70,70,75,76,78,79,90,99,99,100])

plt.scatter(x, y)
plt.show()

# linear-regression
from scipy import stats

slope, intercept, r, p, std_err = stats.linregress(x, y)

def myfunc(x):
	return slope * x + intercept

mymodel = list(map(myfunc, x))

plt.scatter(x, y)
plt.plot(x, mymodel)
plt.show()

# polynomial_regression

# polynomial model
mymodel = np.poly1d(np.polyfit(x, y, 3))

# start at position 1 and end at position 22
myline = np.linspace(1, 22, 100)

plt.scatter(x, y)
plt.plot(myline, mymodel(myline))

plt.show()

# r-squared

# the relationship to predict

# pip install numpy scipy scikit-learn
from sklearn.metrics import r2_score

mymodel = np.poly1d(np.polyfit(x, y, 3))

print(r2_score(y, mymodel(x)))

# predict the car passing at 17 P.M\

# mymodel = np.poly1d(np.polyfit(x, y, 3))

speed = mymodel(17)
print(speed)

# bad fit
x = np.array([89,43,36,36,95,10,66,34,38,20,26,29,48,64,6,5,36,66,72,40])
y = np.array([21,46,3,35,67,95,53,72,58,10,26,34,90,33,38,20,56,2,47,15])

mymodel = np.poly1d(np.polyfit(x, y, 3))

myline = np.linspace(2, 95, 100)

plt.scatter(x, y)
plt.plot(myline, mymodel(myline))
plt.show()

# r-squared
# from sklearn.metrics import r2_score

print(r2_score(y, mymodel(x)))


import numpy as np
from sklearn.base import BaseEstimator, TransformerMixin

# A good way to understand this is this Stack Exchange: Cross Validated answer:
# https://stats.stackexchange.com/a/120126
# Also good thing to know that this is also known as Gaussian Radial Basis Function
class GaussianFeatures(BaseEstimator, TransformerMixin):

    def __init__(self, N, width_factor=2.0):
        self.N = N
        self.width_factor = width_factor

    @staticmethod
    def _gauss_basis(x, y, width, axis=None):
        arg = (x - y) / width
        return np.exp(-1. / 2. * np.sum(arg ** 2, axis))

    def fit(self, x, y=None):
        self.centers_ = np.linspace(x.min(), x.max(), self.N)
        self.width_ = self.width_factor * (self.centers_[1] - self.centers_[0])
        return self

    def transform(self, x):
        return self._gauss_basis(x[:, :, np.newaxis],
                                 self.centers_,
                                 self.width_,
                                 axis=1)


import numpy as np
import matplotlib.pyplot as plt

n = 10
domain = np.linspace(0,1,n+1)
h = 1/n
# Build L
L = np.diag((n-1)*[2],0)
L += np.diag((n-2)*[-1],1)
L += np.diag((n-2)*[-1],-1)
L = L/(h**2)

# Printign for Sanitycheck
print(L[0,:])
print(L[1,:])
print(L[-1,:])

# Show the matrix structure
plt.spy(L, marker="o", color='r')
plt.show()

#Calculate the eingenvalues
numerical_eigenvalues = [x for x in np.linalg.eigh(L)[0]]
analyticl_eigenvalues_dc = [(2/h)**2*np.sin(np.pi*i/2/n)**2 for i in range(1,n)]
analyticl_eigenvalues_op = [(np.pi * i)**2 for i in range(1,n)]

#show eigenvalues
plt.scatter([x.real for x in numerical_eigenvalues], [x.imag for x in numerical_eigenvalues], label="numerical discrete")
plt.scatter(analyticl_eigenvalues_dc, (n-1)*[0], label="analytical discrete", marker='x')
plt.scatter(analyticl_eigenvalues_op, (n-1)*[0], label="analytical operator")
plt.grid()
plt.legend()
plt.xlabel("Re")
plt.ylabel("Im")
print(analyticl_eigenvalues_dc)
print(analyticl_eigenvalues_op)
plt.show()

# Plotting the eigenvectors
for i in range(n-1):
    l = ([np.sin((i+1)*np.pi*j) for j in domain])
    plt.plot(domain,l, "+", markersize=20)

# Plotting the eigenfunctions
domain_f = np.linspace(0,1,10*n)
for i in range(1,n):
    print(i)
    plt.plot(domain_f, [np.sin(np.pi*i*x) for x in domain_f])
plt.show()

# Plot the 10th eigenvalue
plt.plot(domain_f, [np.sin(np.pi*10*x) for x in domain_f])
plt.plot(domain, [np.sin(np.pi*10*x) for x in domain], "+", markersize=20)
plt.show()


import numpy as np
from numpy.typing import ArrayLike

# There also is
# <https://docs.scipy.org/doc/scipy/reference/generated/scipy.special.logsumexp.html>,
# but implementation is easy enough
def _logsumexp(x: ArrayLike):
    c = np.max(x)
    return c + np.log(np.sum(np.exp(x - c)))

def mean(x: ArrayLike, p: float = 1) -> np.ndarray:
    """Generalized mean.

    See <https://github.com/numpy/numpy/issues/19341> for the numpy issue.
    """
    x = np.asarray(x)

    n = len(x)
    if p == 1:
        return np.mean(x)
    if p == -np.inf:
        return np.min(np.abs(x))
    elif p == 0:
        # first compute the root, then the product, to avoid numerical difficulties with
        # too small values of prod(x)
        if np.any(x < 0.0):
            raise ValueError("p=0 only works with nonnegative x.")
        return np.prod(np.power(x, 1 / n))
        # alternative:
        # return np.exp(np.mean(np.log(x)))
    elif p == np.inf:
        return np.max(np.abs(x))

    if np.all(x > 0.0):
        # logsumexp trick to avoid overflow for large p
        # only works for positive x though
        return np.exp((_logsumexp(p * np.log(x)) - np.log(n)) / p)

    if not isinstance(p, (int, np.integer)):
        raise ValueError(f"Non-integer p (={p}) only work with nonnegative x.")

    return (np.sum(x**p) / n) ** (1.0 / p)


# -*- coding: utf-8 -*-

import copy
import numpy as np

class Objective(object):

    def forward(self, outputs, targets):

        raise NotImplementedError()

    def backward(self, outputs, targets):
        raise NotImplementedError()

    def __str__(self):
        return self.__class__.__name__

class MeanSquaredError(Objective):

    def forward(self, outputs, targets):
        return 0.5 * np.mean(np.sum(np.power(outputs - targets, 2), axis=1))

    def backward(self, outputs, targets):
        return outputs - targets

MSE = MeanSquaredError

class HellingerDistance(Objective):

    def forward(self, outputs, targets):
        root_difference = np.sqrt(outputs) - np.sqrt(targets)
        return np.mean(np.sum(np.power(root_difference, 2), axis=1) / np.sqrt(2))

    def backward(self, outputs, targets):
        root_difference = np.sqrt(outputs) - np.sqrt(targets)
        return root_difference / (np.sqrt(2) * np.sqrt(outputs))

HeD = HellingerDistance

class BinaryCrossEntropy(Objective):

    def __init__(self, epsilon=1e-11):
        self.epsilon = epsilon

    def forward(self, outputs, targets):
        outputs = np.clip(outputs, self.epsilon, 1 - self.epsilon)
        return np.mean(-np.sum(targets * np.log(outputs) + (1 - targets) * np.log(1 - outputs), axis=1))

    def backward(self, outputs, targets):
        outputs = np.clip(outputs, self.epsilon, 1 - self.epsilon)
        divisor = np.maximum(outputs * (1 - outputs), self.epsilon)
        return (outputs - targets) / divisor

BCE = BinaryCrossEntropy

class SoftmaxCategoricalCrossEntropy(Objective):

    def __init__(self, epsilon=1e-11):
        self.epsilon = epsilon

    def forward(self, outputs, targets):
        outputs = np.clip(outputs, self.epsilon, 1 - self.epsilon)
        return np.mean(-np.sum(targets * np.log(outputs), axis=1))

    def backward(self, outputs, targets):
        outputs = np.clip(outputs, self.epsilon, 1 - self.epsilon)
        return outputs - targets

SCCE = SoftmaxCategoricalCrossEntropy

def get(objective):
    if objective.__class__.__name__ == 'str':
        if objective in ['mse', 'MSE']:
            return MSE()
        if objective in ['mean_squared_error', 'MeanSquaredError']:
            return MeanSquaredError()
        if objective in ['hellinger_distance', 'HellingerDistance']:
            return HellingerDistance()
        if objective in ['hed', 'HeD']:
            return HeD()
        if objective in ['binary_cross_entropy', 'BinaryCrossEntropy']:
            return BinaryCrossEntropy()
        if objective in ['bce', 'BCE']:
            return BCE()
        if objective in ['softmax_categorical_cross_entropy', 'SoftmaxCategoricalCrossEntropy']:
            return SoftmaxCategoricalCrossEntropy()
        if objective in ['scce', 'SCCE']:
            return SCCE()
        raise ValueError('Unknown objective name: {}.'.format(objective))

    elif isinstance(objective, Objective):
        return copy.deepcopy(objective)

    else:
        raise ValueError("Unknown type: {}.".format(objective.__class__.__name__))

Machine Learning Projects/Xeno-Deep Learning library from scratch/xeno/objectives.py


#/usr/bin/python
#coding:utf8

import numpy as np
def Cosine(x, y):
  return 1.0*np.dot(x, y)/np.sqrt(np.dot(x,x))/np.sqrt(np.dot(y,y))
def Sigmoid(x):
  return 1.0 / (1.0 + np.exp(-1.0 * x))
def SigmoidGradient(x):
  return Sigmoid(x) * (1 - Sigmoid(x))

#平方误差,2范式的正则化项
def SquareErrorFunction2F(target_matrix_y, output_matrix_y, weights, lamda):
  m = len(target_matrix_y)
  err = target_matrix_y - output_matrix_y
  errors = 1.0/(2.0*m) * np.sum(err * err) + lamda*1.0/(2.0*m)*np.sum(weights * weights)
  return errors

def LogisticErrorFunction2F(target_matrix_y, output_matrix_y, weights, lamda):
  m = len(target_matrix_y)
  vector_j = target_matrix_y * np.log(output_matrix_y) + (1.0 - target_matrix_y) * np.log(1.0 - output_matrix_y)
  errors = -1.0/m*np.sum(vector_j) + lamda/(2.0*m) * np.sum(weights * weights)
  return errors

def RandomSample(size, a, b):
  epsilon_init = 0.12;
  arr = (b - a) * np.random.random_sample(size) + a
  return arr * epsilon_init * 2 - epsilon_init

import math
def GaussFunc(x, dimension, mu, sigma):
  inv = np.linalg.inv(sigma)
  e = -0.5 * np.dot(x - mu, inv)
  e = np.dot(e, (x-mu).T)
  re = 1./np.sqrt((2.*math.pi)**dimension) * 1./np.sqrt(np.linalg.det(sigma)) * np.exp(e)
  if x.ndim <= 1:return re
  else:return np.diag(re)

if __name__ == '__main__':
  print( Cosine(np.array([-1,-2]), np.array([1,2])))
  print( SigmoidGradient(np.array([-1,-2])))
  print( SquareErrorFunction2F(np.array([[0,1], [1,0]]), np.array([[0.5,0.5], [0.5,0.5]]), np.array([1,2,3]), 1))
  print( LogisticErrorFunction2F(np.array([[0,1], [1,0]]), np.array([[0.5,0.5], [0.5,0.5]]), np.array([1,2,3]), 1))
  print( RandomSample((2,3), 0, 1))
  print( GaussFunc(np.array([0,0]), 2, [0, 0], np.array([[1.,0],[0.,1]])))
  print( GaussFunc(np.array([[0,0.],[1,2.]]), 2, [0, 0], np.array([[1.,0],[0.,1]])))


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Mon May 20 14:40:10 2019

@author: kenneth
"""

import numpy as np
import pandas as pd

class Regression(object):
    def __init__(self):
        return

    def fit_predict(self, X, Y):
        self.X = X
        self.Y = Y
        #--either beta syntax below would work
        #beta = np.linalg.solve(np.dot(self.X.T, self.X), np.dot(self.X.T, self.Y))
        beta = np.linalg.inv(self.X.T.dot(self.X)).dot(self.X.T).dot(self.Y)
        Y_hat = self.X.dot(beta)
        return Y_hat

    #-Mean Square Error
    def RMSE(self, yh, y):
        return np.sqrt(np.square(yh - y).mean())
    #-Mean Square Error
    def MSE(self, yh, y):
        return np.square(yh - y).mean()
    #-Mean Absolute Error
    def MAE(self, yh, y):
        return np.abs(yh - y).mean()
    #-R-squared Error
    def R_squared(self, yh, y):
        #-- R_square = 1 - (SS[reg]/SS[total])
        # 1 - (y-yh).dot(y-yh)/(y - y.mean()).dot(y - y.mean()) OR
        return (1 -(np.sum(np.square(y - yh))/np.sum(np.square(y - y.mean()))))

    def summary(self, X, y, y_hat):
        #y_hat = self.fit_predict(self.X, self.Y)
        print('*'*40)
        print('\t\tSummary')
        print('*'*40)
        print('RMSE: %s'%(self.RMSE(y_hat,  Y)))
        print('*'*40)
        print('MSE: %s'%(self.MSE(y_hat,  Y)))
        print('*'*40)
        print('MAE: %s'%(self.MAE(y_hat,  Y)))
        print('*'*40)
        print('R_squared = %s'%(self.R_squared(y_hat,  Y)))
        print('*'*40)

    def plot(self, X, Y, y_hat):
        import matplotlib.pyplot as plt
        plt.plot(np.arange(X.shape[0]), Y)
        plt.plot(np.arange(X.shape[0]), y_hat)
        plt.legend(loc = 2)
        plt.title('True vlaue vs Predicted value')
        plt.xlabel('Data point')
        plt.ylabel('True vlaue vs Predicted value')

class GradientDescent(Regression):
    '''
    Inherits Regression class
    '''
    def __init__(self):
        super().__init__()
        return

    def cost(self, X, Y, beta):
        '''
        param: X = training examples/data. column vector <x1, x2, ...., xn | x E R^D>
        param: Y = target. vector  <y | y E R^DX1>
        param: beta = coefficients, e.g b0, b1
        Return: cost
        '''
        return (1/2*len(Y)) * np.sum(np.square(X.dot(beta) - Y))

    def GD(self, X, Y, beta, alpha, iterations, early_stopping = None):
        '''
        param: X = NxD feature matrix
        param: Y = Dx1 column vector
        param: beta = Dx1 beta vector coefficients
        param: alpha = learning rate. Default 1e-2
        param: iterations = Number of times to run. Default 1000

        Return type; final beta/coefficients, cost and bata iterations
        '''
        cost_rec = np.zeros(iterations)
        beta_rec = np.zeros((iterations, X.shape[1]))
        if early_stopping:
            for ii in range(iterations):
                #compute gradient
                beta = beta - (1/len(Y)) *(alpha) * (np.dot(X.T, (np.dot(X,beta) - Y)))
                beta_rec[ii, :] = beta.T
                cost_rec[ii] = self.cost(X, Y, beta)
                print('*'*40)
                print('%s iteratiion, cost = %s'%(ii, cost_rec[ii]))
                #--compare last and previous value. stop if they are the same
                if not cost_rec[ii] == cost_rec[ii -1]:
                    continue
                else:
                    break
            y_hat = X.dot(beta)
            return beta, cost_rec[:ii], beta_rec, y_hat, ii
        else:
            for ii in range(iterations):
                #compute gradient
                beta = beta - (1/len(Y)) *(alpha) * (np.dot(X.T, (np.dot(X,beta) - Y)))
                beta_rec[ii, :] = beta.T
                cost_rec[ii] = self.cost(X, Y, beta)
                print('*'*40)
                print('%s iteratiion, cost = %s'%(ii, cost_rec[ii]))
            print('*'*40)
            y_hat = X.dot(beta)
            return beta, cost_rec, beta_rec, y_hat, ii

    def plot_cost(self, cost, iter_):
        import matplotlib.pyplot as plt
        plt.plot(np.arange(iter_), cost)
        plt.title('Cost vs Number of iteration')
        plt.xlabel('Iteration')
        plt.ylabel('Cost')
        plt.show()

class StochasticGradientDescent(Regression):
    '''
    Inherits Regression class
    '''
    def __init__(self):
        super().__init__()
        return

    def cost(self, X, Y, beta):
        '''
        param: X = training examples/data. column vector <x1, x2, ...., xn | x E R^D>
        param: Y = target. vector  <y | y E R^DX1>
        param: beta = coefficients, e.g b0, b1
        Return: cost
        '''
        return (1/2*len(Y)) * np.sum(np.square(X.dot(beta) - Y))

    def StochGD(self, X, Y, beta, alpha, iterations, early_stopping = None):
        '''
        param: X = NxD feature matrix
        param: Y = Dx1 column vector
        param: beta = Dx1 beta vector coefficients
        param: alpha = learning rate. Default 1e-2
        param: iterations = Number of times to run. Default 1000

        Return type; final beta/coefficients, cost and bata iterations
        '''
        cost_rec = np.zeros(iterations)
        len_y = len(Y)
        if early_stopping:
            for ii in range(iterations):
                #compute gradient
                cost_val = []
                for ij in range(len_y):
                    random_samples = np.random.randint(1, len_y)
                    X_samp = X[:random_samples]
                    Y_samp = Y[:random_samples]
                    beta = beta - (1/len(Y_samp)) *(alpha) * (np.dot(X_samp.T, (np.dot(X_samp,beta) - Y_samp)))
                    cost_val.append(self.cost(X_samp, Y_samp, beta))
                    if cost_val[ij] == cost_val[ij -1]:
                        break
                    else:
                        continue
                cost_rec[ii] = np.average(cost_val)
                print('*'*40)
                print('%s iteratiion, cost = %s'%(ii, cost_rec[ii]))
                #--compare last and previous value. stop if they are the same
                if not cost_rec[ii] == cost_rec[ii -1]:
                    continue
                else:
                    break
            print('*'*40)
            y_hat = X.dot(beta)
            return beta, cost_rec[:ii], y_hat, ii
        else:
            for ii in range(iterations):
                #compute gradient
                cost_val = 0.0
                for ij in range(len_y):
                    random_samples = np.random.randint(1, len_y)
                    X_samp = X[:random_samples]
                    Y_samp = Y[:random_samples]
                    beta = beta - (1/len(Y_samp)) *(alpha) * (np.dot(X_samp.T, (np.dot(X_samp,beta) - Y_samp)))
                    cost_val += self.cost(X_samp, Y_samp, beta)
                cost_rec[ii] = cost_val
                print('*'*40)
                print('%s iteratiion, cost = %s'%(ii, cost_rec[ii]))
            print('*'*40)
            y_hat = X.dot(beta)
            return beta, cost_rec, y_hat, ii

    def plot_cost(self, cost, iter_):
        import matplotlib.pyplot as plt
        plt.plot(np.arange(iter_), cost)
        plt.title('Cost vs Number of iteration')
        plt.xlabel('Iteration')
        plt.ylabel('Cost')
        plt.show()

class MinibatchGradientDescent(Regression):
    '''
    Inherits Regression class
    '''
    def __init__(self):
        super().__init__()
        return

    def cost(self, X, Y, beta):
        '''
        param: X = training examples/data. column vector <x1, x2, ...., xn | x E R^D>
        param: Y = target. vector  <y | y E R^DX1>
        param: beta = coefficients, e.g b0, b1
        Return: cost
        '''
        return (1/2*len(Y)) * np.sum(np.square(X.dot(beta) - Y))

    def minbatchGD(self, X, Y, beta, alpha, iterations, batch_size = None, early_stopping = None):
        '''
        param: X = NxD feature matrix
        param: Y = Dx1 column vector
        param: beta = Dx1 beta vector coefficients
        param: alpha = learning rate. Default 1e-2
        param: iterations = Number of times to run. Default 1000

        Return type; final beta/coefficients, cost and bata iterations
        '''
        cost_rec = np.zeros(iterations)
        len_y = len(Y)
        number_batches = int(len_y/batch_size)
        if early_stopping:
            for ii in range(iterations):
                cost_val = 0
                #randomize dataset using permutation
                random_samples = np.random.permutation(len_y)
                X_random = X[random_samples]
                Y_random = Y[random_samples]
                for ij in range(0, len_y, number_batches):
                    #split into batches
                    X_samp = X_random[ij:ij+batch_size]
                    Y_samp = Y_random[ij:ij+batch_size]
                    beta = beta - (1/len(Y_samp)) *(alpha) * (np.dot(X_samp.T, (np.dot(X_samp,beta) - Y_samp)))
                    cost_val += self.cost(X_samp, Y_samp, beta)
#                    if cost_val[ij] == cost_val[ij -1]:
#                        break
#                    else:
#                        continue
                cost_rec[ii] = cost_val #np.average(cost_val)
                print('*'*40)
                print('%s iteratiion, cost = %s'%(ii, cost_rec[ii]))
                #--compare last and previous value. stop if they are the same
                if not cost_rec[ii] == cost_rec[ii -1]:
                    continue
                else:
                    break
            print('*'*40)
            y_hat = X.dot(beta)
            return beta, cost_rec[:ii], y_hat, ii
        else:
            for ii in range(iterations):
                cost_val = 0
                #randomize dataset using permutation
                random_samples = np.random.permutation(len_y)
                X_random = X[random_samples]
                Y_random = Y[random_samples]
                for ij in range(0, len_y, number_batches):
                    #split into batches
                    X_samp = X_random[ij:ij+batch_size]
                    Y_samp = Y_random[ij:ij+batch_size]
                    beta = beta - (1/len(Y_samp)) *(alpha) * (np.dot(X_samp.T, (np.dot(X_samp,beta) - Y_samp)))
                    cost_val += self.cost(X_samp, Y_samp, beta)
                cost_rec[ii] = cost_val
                print('*'*40)
                print('%s iteratiion, cost = %s'%(ii, cost_rec[ii]))
            print('*'*40)
            y_hat = X.dot(beta)
            return beta, cost_rec, y_hat, ii

    def plot_cost(self, cost, iter_):
        import matplotlib.pyplot as plt
        plt.plot(np.arange(iter_), cost)
        plt.title('Cost vs Number of iteration')
        plt.xlabel('Iteration')
        plt.ylabel('Cost')
        plt.show()

#%%

from collections import Counter
def extractFeatures(model, k_features, fscore = None):
    features = Counter(model.get_booster().get_score())
    features = features.most_common(k_features)
    features = [x[0] for x in features if x[1] >= fscore]
    return features

features = extractFeatures(estimator, 20, 1)
features = ['Area m2',
             'District_id',
             'Street Width',
             'Driver Room',
             'Extra Unit',
             'Apartments',
             'Bed Rooms',
             'WC',
             'With Stairs',
             'Living Rooms',
             'Servant Room']

import statsmodels.api as sm
X = df_standard_no_out[features]
X = np.c_[np.ones((X.shape[0], 1)), X]
Y = df_standard_no_out[['Price']].values

#--Multivariant regression
lm = Regression()
yhat = lm.fit_predict(X, Y)
lm.summary(X, Y, yhat)
gd.plot(X[:200], Y[:200], y_hat[:200])

#--Gradient descent
iterations = 1000
gd = GradientDescent()
beta,cost_rec,theta_rec, yhat, stopping = gd.GD(X, Y, beta = np.zeros(X.shape[1]).reshape(-1, 1), alpha = 0.1, iterations = iterations, early_stopping=True)
gd.summary(X, Y, yhat)
gd.plot_cost(cost_rec, stopping)

#--stochastic gradient descent
stgrad = StochasticGradientDescent()
beta,cost_rec, yhat, stopping = stgrad.StochGD(X, Y, beta = np.zeros(X.shape[1]).reshape(-1, 1), alpha = 0.8, iterations = iterations, early_stopping=True)
stgrad.summary(X, Y, yhat)
stgrad.plot_cost(cost_rec, stopping)

#--minibatch gradient descent
minibatch = MinibatchGradientDescent()
beta,cost_rec, yhat, stopping = minibatch.minbatchGD(X, Y, beta = np.zeros(X.shape[1]).reshape(-1, 1), alpha = 0.01, iterations = iterations, batch_size = 20, early_stopping=True)
minibatch.summary(X, Y, yhat)
minibatch.plot_cost(cost_rec, stopping)

#%% ADDISTIONAL FEATURES FOR REGRESSION


from __future__ import division
import numpy as np
import scipy as sp
import matplotlib.pyplot as plt
import numpy.linalg as npla
import scipy.linalg as spla
import scipy.special
import pandas as pd

# # Part (a)

# Code might send out RunTime warnings. These are benign, caused due to Python 2.7.
# Can verify here: https://stackoverflow.com/questions/40845304/runtimewarning-numpy-dtype-size-changed-may-indicate-binary-incompatibility

def get_approx_jv(n, z):

    # returns approximate value of J_{n+1}

    if (n < 1).any() == True:
        return ValueError

    return ( ( ( ( 2 * n ) / z ) * scipy.special.jv(n, z) ) - scipy.special.jv(n - 1, z) )

z = 20

true_jv = scipy.special.jv(np.arange(2, 51), z)
approx_jv = get_approx_jv(np.arange(1, 50), z)

# I have printed the magnitude of the Relative Error

relative_err = (approx_jv - true_jv) / true_jv
relative_err = np.abs(relative_err)

df = pd.DataFrame({"n" : [i for i in range(2, 51)], "LHS" : true_jv, "RHS" : approx_jv, "Relative Error" : relative_err}, columns=["n", "LHS", "RHS", "Relative Error"])
print(df.to_string(index_names=False))

plt.xlabel("Value of n")
plt.ylabel("Absolute Value of Relative Error")
plt.title("Problem 3(a)")
plt.plot( np.arange(2, 51), relative_err)
plt.savefig("problem_3a.png")

# Uncomment to see figures
plt.show()


import numpy as np
from matplotlib import pyplot as plt

from numpy import sin, cos, pi
from scipy.optimize import fsolve

def _Kronig_Penney(K, k, a, U0b):
    """
    Kronig-Penney model.
    """
    return U0b / (2*K) * sin(K * a) + cos(K * a) - cos(k * a)

@np.vectorize
def E(k, n=1, a=1, U0b=4, **kwargs):
    """
    Get energy from k in Kronig-Penney model.
    ---
    Arguments:
        k: wavenumber, a number or an numpy.array.
        n: the n-th band, starting from 0.
        a: distance between atoms.
        U0b: the area of the potential wall.
        **kwargs: something else to pass to scipy.optimize.fsolve.
    Returns:
        The energies corresponding to k, in the unit of hbar^2/2m.
    """
    K = fsolve(_Kronig_Penney, (n + 1/2)*pi, args=(k, a, U0b))
    return K**2

def getBands(n=3, a=1, U0b=4, **kwargs):
    """
    Get n bands in Kronig-Penney model.
    ---
    Arguments:
        k: wavenumber, a number or an numpy.array.
        n: the n-th band, starting from 1.
        a: distance between atoms.
        U0b: the area of the potential wall.
        **kwargs: something else to pass to scipy.optimize.fsolve.
    Returns:
        bands: An array of the energies, in the unit of hbar^2/2m.
            `bands[i]` is the i-th band.
    """
    k = np.linspace(-pi/a, pi/a, 100)
    bands = [E(k, i, a, U0b=4, **kwargs) for i in range(n)]
    return bands

def plotBands(n=3, a=1, U0b=4,
    title=None, **kwargs):
    """
    Plot n bands in Kronig-Penney model.
    ---
    Arguments:
        k: wavenumber, a number or an numpy.array.
        n: the n-th band, starting from 1.
        a: distance between atoms.
        U0b: the area of the potential wall.
        title: the title of the ax.
            If title == None then it is "Plotting of n bands".
        **kwargs: something else to pass to scipy.optimize.fsolve.
    Returns:
        (fig, ax)
    """
    if title == None:
        title = "Plotting of {} bands".format(n)
    k = np.linspace(-pi/a, pi/a, 100)
    bands = getBands(n, a, U0b, **kwargs)

    fig, ax = plt.subplots(figsize=(5, 15), dpi=200)

    for i, band in enumerate(bands):
        ax.plot(k, band, label="${}$-th band".format(i))

    ax.set_xlabel("$k$")
    ax.set_ylabel("$2m E/\hbar^2$")

    ax.set_title(title)
    ax.minorticks_on()
    ax.grid(True)

    ax.legend()

    return fig, ax

if __name__ == "__main__":
    plotBands()


"""Contains the quadratic cost function."""

from typing import cast

import numpy as np

def quadratic_loss(x: np.ndarray, y: np.ndarray) -> np.ndarray:
    """
    Calculate the vectorized quadratic loss for a single example.

    Parameters
    ----------
    x : np.ndarray
        A vector of the predicted values.
    y : np.ndarray
        A vector of the expected values. Must match x's dimensions.

    """
    if x.ndim != 1:
        raise ValueError("Can only compute the quadratic cost for vectors")

    if x.shape != y.shape:
        raise ValueError(
            "Can only compute the quadratic cost of two vectors of equal shape"
        )

    result = 0.5 * ((y - x) ** 2)
    return cast(np.ndarray, result)

def delta_quadratic_loss(x: np.ndarray, y: np.ndarray) -> np.ndarray:
    """
    Calculate the vectorization of the derivative of quadratic loss.

    Partial derivative taken with respect to x.
    """
    if x.ndim != 1:
        raise ValueError("Can only compute the quadratic cost for vectors")

    if x.shape != y.shape:
        raise ValueError(
            "Can only compute the quadratic cost of two vectors of equal shape"
        )

    result = x - y
    return cast(np.ndarray, result)


import numpy as np
import matplotlib
import matplotlib.pyplot as plt
from sklearn import linear_model
import matplotlib.animation as animation
import os
import subprocess

# apply gradient descent algorithm to optimiza cost function of several variables

def cost(x):
	"""Cost function of linear regression (LR) model with coefficient x"""
	m = A.shape[0]
	return .5/m *np.linalg.norm(A.dot(x)-b,2)**2

def grad(x):
	"""Gradient function of cost function"""
	m = A.shape[0]
	return 1/m *A.T.dot(A.dot(x)-b)

def numerical_grad(x, cost):
	"""Calculating numerical gradient with coefficient x to check whether our grad function is correct or not"""
	eps = 1e-4
	g = np.zeros_like(x)
	for i in range(len(x)):
		x_p = x.copy()
		x_n = x.copy()
		x_p[i] += eps
		x_n[i] -= eps
		g[i] = (cost(x_p) - cost(x_n))/(2*eps)
	return g

def check_grad(x, cost, grad):
	"""Check grad function by comparing actual gradient value and numerical gradient (estimation)"""
	x = np.random.rand(x.shape[0], x.shape[1])
	grad1 = grad(x)
	grad2 = numerical_grad(x, cost)
	return True if np.linalg.norm(grad1 - grad2) < 1e-6 else False

def gradient_descent(x_init, grad, alpha):
	"""
	Running gradient descent with initial value x_init, grad function, learning rate alpha
	This function return x: list of solution after each iteration and iteration: last iteration it has gone through
	"""
	m = len(x_init)
	x = [x_init]
	for iteration in range(itr):
		x_new = x[-1] - alpha*grad(x[-1])
		if np.linalg.norm(grad(x_new))/m < 1e-3: #Stop gradient descent when grad value is too small.
			break
		x.append(x_new)
	return (x, iteration)

############### Main ####################

# create random data
A = np.array([[2,9,7,9,11,16,25,23,22,29,29,35,37,40,46]]).T
b = np.array([[2,3,4,5,6,7,8,9,10,11,12,13,14,15,16]]).T

fig1 = plt.figure('GD for Linear Regression')
ax = plt.axes(xlim=(-10, 60), ylim=(-1, 20)) #restrict the figure showing only values in specified range
plt.plot(A,b,'ro', label='_nolegend_') # plot data points
line, = ax.plot([], [], color = 'blue') # plot function used in animation

# plot solution found by scikit learn
lr = linear_model.LinearRegression()
lr.fit(A,b)
x0_gd = np.linspace(1,46,2)
y0_sklearn = lr.intercept_[0] + lr.coef_[0][0]*x0_gd
plt.plot(x0_gd, y0_sklearn, color='green')

# apply gradient descent
itr = 90
learning_rate = .0001 #.0001
x_init = np.array([[1], [2]])

ones = np.ones((A.shape[0],1), dtype=np.int8) #Append bias to A
A = np.concatenate((ones,A), axis=1)

print('Checking gradient...', check_grad(np.random.rand(2, 1), cost, grad))

x_init = np.array([[1], [2]])

# running gradient descent
myGD = gradient_descent(x_init, grad, learning_rate)

# plot x_init (black line)
x0_gd = np.linspace(1,46,2)
y0_init = x_init[0][0] + x_init[1][0]*x0_gd
plt.plot(x0_gd, y0_init, color = 'black')

# plot lines in each iteration
def update(i):
	x_gd = myGD[0][i]
	y0_gd = x_gd[0][0] + x_gd[1][0]*x0_gd
	line.set_data(x0_gd,y0_gd)
	plt.xlabel('Iteration: {}/{}, learning rate: {}'.format(i+1, myGD[1], learning_rate))
	return line,

# legend for graph
plt.legend(('Value in each GD iteration', 'Solution by formular', 'Inital value for GD'), loc=(0.52, 0.01))
ltext = plt.gca().get_legend().get_texts()
plt.setp(ltext[0], fontsize=10)
plt.setp(ltext[1], fontsize=10)
plt.setp(ltext[2], fontsize=10)

# save animation to mp4 file using ffmpeg
Writer = animation.writers['ffmpeg']
writer = Writer(fps=15, metadata=dict(artist='Me'), bitrate=1800)

line_ani = animation.FuncAnimation(fig1, update, myGD[1]+1, interval=50, blit=True)
# # Save mp4 file for animation
# line_ani.save('lines.mp4', writer='ffmpeg')

# # Convert mp4 to gif
# # https://stackoverflow.com/questions/11269575/how-to-hide-output-of-subprocess-in-python-2-7
# FNULL = open(os.devnull, 'w')
# subprocess.call(['ffmpeg', '-i', 'lines.mp4', 'lines.gif'], stdout=FNULL, stderr=subprocess.STDOUT) # Execute command line to convert .mp4 to .gif using ffmpeg and hide output of command line to terminal
plt.show()

plt.figure('Iter and cost function')

cost_list = []
iter_list = []

for i in range(itr):
	iter_list.append(i+1)

for i in range(itr):
	cost_list.append(cost(myGD[0][i]))

plt.plot(iter_list, cost_list)
plt.xlabel('Iteration')
plt.ylabel('Cost value')

plt.show()


import numpy as np

def get_row(bp):
    return int(bp[:7].replace("F", "0").replace("B", "1"), 2)

def get_col(bp):
    return int(bp[7:].replace("L", "0").replace("R", "1"), 2)

def get_id(bp):
    return 8 * get_row(bp) + get_col(bp)

def get_missing_seat(bp_list):
    seats = [
        int(
            i.replace("F", "0").replace("B", "1").replace("L", "0").replace("R", "1"), 2
        )
        for i in bp_list
    ]

    seats.sort()
    return seats[np.diff(seats).argmax()] + 1

if __name__ == "__main__":

    assert get_row("FBFBBFFRLR") == 44
    assert get_row("BFFFBBFRRR") == 70
    assert get_row("FFFBBBFRRR") == 14
    assert get_row("BBFFBBFRLL") == 102

    assert get_col("FBFBBFFRLR") == 5
    assert get_col("BFFFBBFRRR") == 7
    assert get_col("FFFBBBFRRR") == 7
    assert get_col("BBFFBBFRLL") == 4

    assert get_id("FBFBBFFRLR") == 357
    assert get_id("BFFFBBFRRR") == 567
    assert get_id("FFFBBBFRRR") == 119
    assert get_id("BBFFBBFRLL") == 820

    with open("day5.input", "r") as f:
        bp_list = [i.rstrip() for i in f.readlines()]

    print("Part 1", np.max([get_id(i) for i in bp_list]))

    print("Part 2", get_missing_seat(bp_list))


#!/usr/bin/env python
from __future__ import division
import numpy as np
import matplotlib.pyplot as plt
from scipy import optimize
from numpy import newaxis, r_, c_, mat, e
from numpy.linalg import *

def plotData(X, y):
    #pos = (y.ravel() == 1).nonzero()
    #neg = (y.ravel() == 0).nonzero()
    pos = (y == 1).nonzero()[:1]
    neg = (y == 0).nonzero()[:1]

    plt.plot(X[pos, 0].T, X[pos, 1].T, 'k+', markeredgewidth=2, markersize=7)
    plt.plot(X[neg, 0].T, X[neg, 1].T, 'ko', markerfacecolor='r', markersize=7)

def sigmoid(z):
    g = 1. / (1 + e**(-z.A))
    return g

def costFunction(theta, X, y):
    m = X.shape[0]

    predictions = sigmoid(X * c_[theta])
    J = 1./m * (-y.T.dot(np.log(predictions)) - (1-y).T.dot(np.log(1 - predictions)))

    #grad = 1./m * X.T * (predictions - y)
    return J[0][0]##, grad.A

def predict(theta, X):
    p = sigmoid(X * c_[theta]) >= 0.5
    return p

def plotDecisionBoundary(theta, X, y):
    plotData(X[:, 1:3], y)

    if X.shape[1] <= 3:
        plot_x = r_[X[:,2].min()-2,  X[:,2].max()+2]
        plot_y = (-1./theta[2]) * (theta[1]*plot_x + theta[0])

        plt.plot(plot_x, plot_y)
        plt.legend(['Admitted', 'Not admitted', 'Decision Boundary'])
        plt.axis([30, 100, 30, 100])
    else:
        pass

if __name__ == '__main__':
    data = np.loadtxt('ex2data1.txt', delimiter=',')
    X = mat(c_[data[:, :2]])
    y = c_[data[:, 2]]

    # ============= Part 1: Plotting

    print 'Plotting data with + indicating (y = 1) examples and o ' \
          'indicating (y = 0) examples.'

    plotData(X, y)
    plt.ylabel('Exam 1 score')
    plt.xlabel('Exam 2 score')
    plt.legend(['Admitted', 'Not admitted'])
    plt.show()

    raw_input('Press any key to continue\n')

    # ============= Part 2: Compute cost and gradient

    m, n = X.shape

    X = c_[np.ones(m), X]

    initial_theta = np.zeros(n+1)

    cost, grad = costFunction(initial_theta, X, y), None

    print 'Cost at initial theta (zeros): %f' % cost
    print 'Gradient at initial theta (zeros):\n%s' % grad

    raw_input('Press any key to continue\n')

    # ============= Part 3: Optimizing using fminunc

    options = {'full_output': True, 'maxiter': 400}

    theta, cost, _, _, _ = \
        optimize.fmin(lambda t: costFunction(t, X, y), initial_theta, **options)

    print 'Cost at theta found by fminunc: %f' % cost
    print 'theta: %s' % theta

    plotDecisionBoundary(theta, X, y)
    plt.show()

    raw_input('Press any key to continue\n')

    # ============== Part 4: Predict and Accuracies

    prob = sigmoid(mat('1 45 85') * c_[theta])
    print 'For a student with scores 45 and 85, we predict an admission ' \
          'probability of %f' % prob

    p = predict(theta, X)
    print 'Train Accuracy:', (p == y).mean() * 100

    raw_input('Press any key to continue\n')


import numpy as np
import matplotlib.pyplot as plt

class DescidaGradienteRosenbrock():

    MAX_PASSOS = 20000
    TOLERANCIA_MINIMA = 1.0 * (10 ** -4)

    def __init__(self, learning_rate):
        self.learning_rate = learning_rate

    def executar(self):
        passo = 1
        ponto = np.array([0, 0, 0])
        ponto_anterior = None
        gradiente = None
        tolerancia = 1000000
        valor = self.func_rosenbrock(ponto[0], ponto[1], ponto[2])
        valor_anterior = None
        self.todos_valores = [valor]

        while passo <= self.MAX_PASSOS and tolerancia >= self.TOLERANCIA_MINIMA:
            # Calculando gradiente
            gradiente = self.calc_gradiente(ponto[0], ponto[1], ponto[2])

            # Calculando novo ponto
            ponto_anterior = ponto
            ponto = ponto - (self.learning_rate * gradiente)

            # Calculando novo valor
            valor_anterior = valor
            valor = self.func_rosenbrock(ponto[0], ponto[1], ponto[2])
            self.todos_valores.append(valor)

            # Calculando tolerância
            tolerancia = self.calc_tolerancia(ponto_anterior, ponto)

            print("(%d) x1=%.16f x2=%.16f x3=%.16f tolerancia=%.16f valor=%.16f" % (passo, ponto[0], ponto[1], ponto[2], tolerancia, valor))
            passo += 1

        print("=============================================================")
        print("Passos = %d" % (passo - 1))
        print("x1 = %.16f" % (ponto[0]))
        print("x2 = %.16f" % (ponto[1]))
        print("x3 = %.16f" % (ponto[2]))
        print("valor = %.16f" % (valor))

    def plotar_grafico(self, titulo):
        # Mostrando gráfico
        plt.plot(self.todos_valores)
        plt.title(titulo)
        plt.xlabel("Número de atualizações de $x$")
        plt.ylabel("Valor da função $f(x)$")
        plt.show()

    def func_rosenbrock(self, x1, x2, x3):
        resp = 100 * ((x2 - (x1 ** 2)) ** 2)
        resp += (1 - x1) ** 2
        resp += 100 * ((x3 - (x2 ** 2)) ** 2)
        resp += (1 - x2) ** 2

        return resp

    def calc_gradiente(self, x1, x2, x3):
        grad_x1 = 2 * ((200 * (x1 ** 3)) - (200 * x1 * x2) + x1 - 1)

        grad_x2 = (-200 * (x1 ** 2)) + (400 * (x2 ** 3)) + (x2 * (202 - (400 * x3))) -2

        grad_x3 = 200 * (x3 - (x2 ** 2))

        return np.array([grad_x1, grad_x2, grad_x3])

    def calc_tolerancia(self, valor_anterior, valor):
        return np.linalg.norm(valor - valor_anterior) / np.linalg.norm(valor_anterior)

# CÓDIGO PRINCIPAL
if __name__ == "__main__":
    descida = DescidaGradienteRosenbrock(10 ** -3)
    try:
        descida.executar()
    except:
        pass

    descida.plotar_grafico("Evolução do valor da função $f(x)$ com $lr = 10^{-3}$")


from __future__ import division
import numpy as np

from activation_functions import Sigmoid

def accuracy_score(y_true, y_pred):
    accuracy = np.sum(y_true == y_pred) /len(y_true)
    return accuracy

class Loss(object):
    def loss(self, y_true, y_pred):
        return NotImplementedError()

    def gradient(self, y, y_pred):
        raise NotImplementedError()

    def acc(self, y, y_pred):
        return NotImplementedError()

class SquareLoss(Loss):
    def __init__(self):
        self.loss_name = 'SquareLoss'

    def loss(self, y, y_pred):
        return 0.5 * np.power((y - y_pred), 2)

    def gradient(self, y, y_pred):
        return -(y - y_pred)

    def acc(self, y, y_pred):
        return 0.5 * np.mean(np.power((y - y_pred), 2))

class CrossEntropy(Loss):
    def __init__(self):
        self.loss_name = 'CrossEntropy'

    def loss(self, y, AL):
        # Avoid division by zero
        AL = np.clip(AL, 1e-15, 1 - 1e-15)
        return (- y * np.log(AL) - (1 - y) * np.log(1 - AL)) #/(y.shape[-1])#(1)normalize in main

    def acc(self, y, AL):
        return accuracy_score(y[0], AL[0]>=0.5)

    def gradient(self, y, AL):
        # Avoid division by zero
        AL = np.clip(AL, 1e-15, 1 - 1e-15)
        #print  ((- (y / AL) + (1 - y) / (1 - AL) ).shape,'cross-function output dA')
        return (- (y / AL) + (1 - y) / (1 - AL)) #/(y.shape[-1])#(2)normalize when updating grad

class MultiClassCrossEntropy(Loss):
    def __init__(self):
        self.loss_name = 'MultiClassCrossEntropy'

    def loss(self, y, AL):
        return -np.sum(y * np.log(AL) ,axis= 0, keepdims = True) #/(y.shape[-1])#(1)normalize in main

    def gradient(self, y, AL):
        # Avoid division by zero
        AL = np.clip(AL, 1e-15, 1 - 1e-15)

        assert(AL.shape == (-(y / AL)).shape)

        return - (y / AL) # /(y.shape[-1])#(2)normalize when updating grad

class SoftmaxCrossEntropy(Loss):
    def __init__(self):
        self.loss_name = 'SoftmaxCrossEntropy'

    def loss(self, y, Z):
        # Avoid division by zero
        Z_aux = Z.shape
        Z = Z - np.max(Z, axis = 0, keepdims=True)
        assert Z_aux ==Z.shape

        log_e_Z = Z- np.log(np.sum( np.exp(Z), axis=0, keepdims=True))
        return (-np.sum( y * log_e_Z ,axis= 0))  # (1)normalize in main

    def gradient(self, y, Z):
        Z -= np.max(Z, axis = 0, keepdims=True)
        p = np.exp(Z)/ np.sum( np.exp(Z), axis=0, keepdims=True)

        # Avoid division by zero
        return  (-y + p)  #(2)Normalize when updating grad


"""
Activation functions and their derivatives
"""
import numpy as np

def sigmoidp(x):
    return 1.0 / (1.0 + np.exp(-x))

def sigmoidn(x):
    return np.exp(x) / (1.0 + np.exp(x))

# The sigmoid functions for positive and negative valus
# are done differently to avoid overvflow erros while
# calculating the exponent
def sigmoid(x):
    xp = np.multiply(x, (x>0))
    xn = np.multiply(x, (x<0))
    return sigmoidp(xp) + sigmoidn(xn) - 0.5

def sigmoid_prime(x):
    return sigmoid(x) * (1 - sigmoid(x))

# softmax neumerator and denominators are both devided by the largest component of
# the denominator to avoid overflow errors
def softmax(x):
    return np.exp(x-np.max(x, axis=0)) / np.sum(np.exp(x-np.max(x, axis=0)), axis=0)

def tanh(x):
    return np.tanh(x)

def tanh_prime(x):
    return 1 - tanh(x) * tanh(x)

def relu(x):
    return np.maximum(x, 0)

def relu_prime(x):
    return (x > 0).astype(float)


# 5.2 Learning is just parameter estimation
import torch
import numpy as np
import csv
import matplotlib.pyplot as plt

# data
t_c = [0.5, 14.0, 15.0, 28.0, 11.0, 8.0, 3.0, -4.0, 6.0, 13.0, 21.0]
t_u = [35.7, 55.9, 58.2, 81.9, 56.3, 48.9, 33.9, 21.8, 48.4, 60.4, 68.4]
t_c = torch.tensor(t_c)
t_u = torch.tensor(t_u)

# Visualizing the data
plt.figure(figsize=(12, 6))
# plt.plot(np.arange(1, 12, step=1), t_c.numpy())
plt.scatter(t_u.numpy(), t_c.numpy(), s=120, c="Orange")
plt.xlabel("MEASUREMENT")
plt.ylabel("TEMPERATURE (Celsius)")
plt.grid()
plt.show()

# Define model
def model(t_u, w, b):
    out = w * t_u + b
    return out

# Define loss function
def loss_fn(t_p, t_c):
    squared_diffs = (t_p - t_c)**2
    return squared_diffs.mean()

# Initializer the parameters
w = torch.ones(())
b = torch.zeros(())

t_p = model(t_u, w, b)
t_p

# check the value of the loss
loss = loss_fn(t_p, t_c)
loss

delta = 0.1
loss_rate_of_change_w = \
(loss_fn(model(t_u, w + delta, b), t_c) -
loss_fn(model(t_u, w - delta, b), t_c)) / (2.0 * delta)

learning_rate = 1e-2
w = w - learning_rate * loss_rate_of_change_w

loss_rate_of_change_b = \
(loss_fn(model(t_u, w, b + delta), t_c) -
loss_fn(model(t_u, w, b - delta), t_c)) / (2.0 * delta)

b = b - learning_rate * loss_rate_of_change_b

# d loss_fn / d w = (d loss_fn / d t_p) * (d t_p / d w)

def loss_fn(t_p, t_c):
    squared_diffs = (t_p - t_c)**2
    return squared_diffs.mean()

def dloss_fn(t_p, t_c):
    dsq_diffs = 2 * (t_p - t_c) / t_p.size(0)
    return dsq_diffs

def model(t_u, w, b):
    return w * t_u + b

def dmodel_dw(t_u, w, b):
    return t_u

def dmodel_db(t_u, w, b):
    return 1.0

def grad_fn(t_u, t_c, t_p, w, b):
    dloss_dtp = dloss_fn(t_p, t_c)
    dloss_dw = dloss_dtp * dmodel_dw(t_u, w, b)
    dloss_db = dloss_dtp * dmodel_db(t_u, w, b)
    return torch.stack([dloss_dw.sum(), dloss_db.sum()])

def training_loop(n_epochs, learning_rate, params, t_u, t_c):
    for epoch in range(1, n_epochs + 1):
        w, b = params
        t_p = model(t_u, w, b)
        loss = loss_fn(t_p, t_c)
        grad = grad_fn(t_u, t_c, t_p, w, b)
        params = params - learning_rate * grad
        print('Epoch %d, Loss %f' % (epoch, float(loss)))
    return params

training_loop(
    n_epochs = 100,
    learning_rate = 1e-2,
    params = torch.tensor([1.0, 0.0]),
    t_u = t_u,
    t_c = t_c)

training_loop(
    n_epochs = 100,
    learning_rate = 1e-4,
    params = torch.tensor([1.0, 0.0]),
    t_u = t_u,
    t_c = t_c)

t_un = 0.1 * t_u

training_loop(
    n_epochs = 100,
    learning_rate = 1e-2,
    params = torch.tensor([1.0, 0.0]),
    t_u = t_un,
    t_c = t_c)

params = training_loop(
    n_epochs = 5000,
    learning_rate = 1e-2,
    params = torch.tensor([1.0, 0.0]),
    t_u = t_un,
    t_c = t_c,
    )

t_p = model(t_un, *params)
fig = plt.figure(dpi=600)
plt.xlabel("Temperature (°Fahrenheit)")
plt.ylabel("Temperature (°Celsius)")
plt.plot(t_u.numpy(), t_p.detach().numpy())
plt.plot(t_u.numpy(), t_c.numpy(), 'o')
plt.grid()
plt.show()

5_The mechanics of learning/5.2 Learning is just parameter estimation.py


import numpy as np
import copy

testing_matrix = np.matrix('1 7 -1 5; 4 -1 1 1; 5 -3 1 -1')

def gauss(matrix: np.matrix, show=False) -> tuple:
    '''
    Gaussian elimination can solve linear equation systems of any size.
    Input should be a numpy matrix. There is a variable called
    'testing_matrix' already defined for demo purpose :)
    '''
    matrix_copy = copy.deepcopy(matrix)
    variables_amount = len(np.asarray(matrix_copy[0]).reshape(-1)) - 1

    if (variables_amount > len(matrix_copy)):
        raise Exception('Too much variables!')
        return None

    if show:
        print('starting matrix:')
        print(matrix_copy)

    for subtraction_index in range(variables_amount - 1):

        subtraction_matrix = matrix_copy[subtraction_index]

        for line in range(1 + subtraction_index, variables_amount):
            matrix_to_subtract = matrix_copy[line]
            matrix_copy[line] = matrix_to_subtract * subtraction_matrix[0, subtraction_index] - subtraction_matrix * matrix_to_subtract[0, subtraction_index]

        if show:
            print('iteration {}:'.format(subtraction_index))
            print(matrix_copy)

    solutions = np.array([])

    for equation in range(variables_amount - 1, -1, -1):
        addition = 0
        for variable, solution in zip(range(variables_amount - 1, equation, -1), solutions):
            addition += matrix_copy[equation, variable] * solution

        solutions = np.append(solutions, (matrix_copy[equation, variables_amount] - addition) / matrix_copy[equation, equation])

    solutions = np.flip(solutions)

    if show:
        print('\nsolutions set:\n{}'.format(solutions))

        print('\nsolutions:')
        for solution, index in zip(solutions, range(len(solutions))):
            print('x{i}: {s}'.format(s = solution, i = index))

    return tuple(solutions)


def price_options(S=100.0, K=100.0, sigma=0.25, r=0.05, days=260, paths=10000):
    """
    Price European and Asian options using a Monte Carlo method.

    Parameters
    ----------
    S : float
        The initial price of the stock.
    K : float
        The strike price of the option.
    sigma : float
        The volatility of the stock.
    r : float
        The risk free interest rate.
    days : int
        The number of days until the option expires.
    paths : int
        The number of Monte Carlo paths used to price the option.

    Returns
    -------
    A tuple of (E. call, E. put, A. call, A. put) option prices.
    """
    import numpy as np
    from math import exp,sqrt

    h = 1.0/days
    const1 = exp((r-0.5*sigma**2)*h)
    const2 = sigma*sqrt(h)
    stock_price = S*np.ones(paths, dtype='float64')
    stock_price_sum = np.zeros(paths, dtype='float64')
    for j in range(days):
        growth_factor = const1*np.exp(const2*np.random.standard_normal(paths))
        stock_price = stock_price*growth_factor
        stock_price_sum = stock_price_sum + stock_price
    stock_price_avg = stock_price_sum/days
    zeros = np.zeros(paths, dtype='float64')
    r_factor = exp(-r*h*days)
    euro_put = r_factor*np.mean(np.maximum(zeros, K-stock_price))
    asian_put = r_factor*np.mean(np.maximum(zeros, K-stock_price_avg))
    euro_call = r_factor*np.mean(np.maximum(zeros, stock_price-K))
    asian_call = r_factor*np.mean(np.maximum(zeros, stock_price_avg-K))
    return (euro_call, euro_put, asian_call, asian_put)


import numpy as np
from .Model import Model
from src.si.data.dataset import Dataset
from src.si.util.metrics import mse

class LinearRegression(Model):
    def __init__(self, gd=False, epochs=1000, lr=0.001):
        super(LinearRegression, self).__init__()
        self.gd = gd
        self.theta = None
        self.num_iterations = epochs
        self.lr = lr

    def fit(self, dataset):
        X, Y = dataset.getXy()
        X = np.hstack((np.ones((X.shape[0], 1)), X))
        self.X = X
        self.Y = Y
        if self.gd:
            self.train_gd(X, Y)
        else:
            self.train_closed(X, Y)
        self.is_fited = True

    def train_gd(self, X, Y):
        m = X.shape[0]
        n = X.shape[1]
        self.history = {}
        self.theta = np.zeros(n)
        for epoch in range(self.num_iterations):
            grad = 1/m*(X.dot(self.theta)-Y).dot(X)
            self.theta -= self.lr*grad
            self.history[epoch] = [self.theta[:], self.cost()]

    def train_closed(self, X, Y):
        self.theta = np.linalg.inv(X.T.dot(X)).dot(X.T).dot(Y)

    def predict(self, x):
        assert self.is_fited
        _X = np.hstack(([1], x))
        return np.dot(self.theta, _X)

    def cost(self):
        y_pred = np.dot(self.X, self.theta)
        return mse(y_pred, self.Y)

class LinearRegressionReg(LinearRegression):
    def __init__(self, gd = False, epochs = 1000, lr=0.001,lbd = 1):
        super(LinearRegressionReg, self).__init__(gd=gd, epochs=epochs, lr=lr)
        self.lbd = lbd
        self.gd = gd
        self.num_iterations = epochs

    def train_closed(self, X, Y):
        n = X.shape[1]
        identity = np.eye(n)
        identity[0, 0] = 0
        self.theta = np.linalg.inv(X.T.dot(X)+self.lbd*identity).dot(X.T).dot(Y)
        self.is_fited = True

    def train_gd(self, X, Y):
        m = X.shape[0]
        n = X.shape[1]
        self.history = {}
        self.theta = np.zeros(n)
        lbds = np.full(m, self.lbd)
        lbds[0] = 0
        for epoch in range(self.num_iterations):
            grad = (X.dot(self.theta)-Y).dot(X)
            self.theta -= (self.lr/m)*(lbds+grad)
            self.history[epoch] = [self.theta[:], self.cost()]

    def predict(self, X):
        assert self.is_fited
        _x = np.hstack(([1], X))
        return np.dot(self.theta, _x)

    def cost(self):
        y_pred = np.dot(self.X, self.theta)
        return mse(self.Y, y_pred)/2


# gradient descent optimization with adam for a two-dimensional test function
from math import sqrt
from numpy import asarray
from numpy.random import rand
from numpy.random import seed

# objective function
def objective(x, y):
	return x**2.0 + y**2.0

# derivative of objective function
def derivative(x, y):
	return asarray([x * 2.0, y * 2.0])

# gradient descent algorithm with adam
def adam(objective, derivative, bounds, n_iter, alpha, beta1, beta2, eps=1e-8):
	# generate an initial point
	x = bounds[:, 0] + rand(len(bounds)) * (bounds[:, 1] - bounds[:, 0])
	score = objective(x[0], x[1])
	# initialize first and second moments
	m = [0.0 for _ in range(bounds.shape[0])]
	v = [0.0 for _ in range(bounds.shape[0])]
	# run the gradient descent updates
	for t in range(n_iter):
		# calculate gradient g(t)
		g = derivative(x[0], x[1])
		# build a solution one variable at a time
		for i in range(x.shape[0]):
			# m(t) = beta1 * m(t-1) + (1 - beta1) * g(t)
			m[i] = beta1 * m[i] + (1.0 - beta1) * g[i]
			# v(t) = beta2 * v(t-1) + (1 - beta2) * g(t)^2
			v[i] = beta2 * v[i] + (1.0 - beta2) * g[i]**2
			# mhat(t) = m(t) / (1 - beta1(t))
			mhat = m[i] / (1.0 - beta1**(t+1))
			# vhat(t) = v(t) / (1 - beta2(t))
			vhat = v[i] / (1.0 - beta2**(t+1))
			# x(t) = x(t-1) - alpha * mhat(t) / (sqrt(vhat(t)) + eps)
			x[i] = x[i] - alpha * mhat / (sqrt(vhat) + eps)
		# evaluate candidate point
		score = objective(x[0], x[1])
		# report progress
		print('>%d f(%s) = %.5f' % (t, x, score))
	return [x, score]

# seed the pseudo random number generator
seed(1)
# define range for input
bounds = asarray([[-1.0, 1.0], [-1.0, 1.0]])
# define the total iterations
n_iter = 60
# steps size
alpha = 0.02
# factor for average gradient
beta1 = 0.8
# factor for average squared gradient
beta2 = 0.999
# perform the gradient descent search with adam
best, score = adam(objective, derivative, bounds, n_iter, alpha, beta1, beta2)
print('Done!')
print('f(%s) = %f' % (best, score))


import matplotlib.pyplot as plt
import numpy as np
#
x_0 = int(input("Enter the initial value of x (try 0): "))
y_0 = int(input("Enter the initial value of y (try 0): "))
t=float(input("Enter the value of x at which we want the value of y(try 1.2): "))
h= float(input("Enter the value of interval h(0.1 or 0.01 or 0.001): "))
def euler_method(x_0, y_0, h, t):  ## t is final point where we want the value of function
 x=x_0
 y=y_0
 def fun(x,y):
     return x+2*y                                                   # return 3*np.exp(-x)-0.4*y
                                                        #   #return (x + y + x * y)

## make lists to store values
 lsty=[]  # y data points which we get numerically
 lstx=[]  # x data points which we get numerically
 while x < t:  # here t is final point
   y+=fun(x,y)*h
   lsty.append(y)
   x=x+h
   lstx.append(x)

 print("Approximate solution at x = ", int(x), " is ", "%.6f"% y)
 print("The number of iterations are n = ",int((x-x_0)/h+1))
   # print(list[y], end=" ")
 # print (lstx)
 # print (lsty)

 ## --------------Plotting------------------------
 ## ----------Numerical plot----------------------

 ## for interval h=0.1-->
 if h==0.1:
  # plt.subplot(1,3,1)
  plt.plot(lstx,lsty,'-r',label ="Numerical curve")
  ##---------- functional plot---------------------
  xp= np.linspace(0,1.2,12)
  f=0.25*np.exp(2*xp) - 0.5*xp-0.25

  plt.plot(xp,f, 'c', label="Functional curve")
  plt.savefig("Euler_Method_h_0.1.png")
  plt.grid()
  plt.legend()
  plt.title("h=0.1")
  plt.show()

 ### for interval h=0.01--->
 elif h==0.01:
  # plt.subplot(1,3,2)
  plt.plot(lstx,lsty,'-r',label ="Numerical curve")
  xp= np.linspace(0,1.2,121)
  f=0.25*np.exp(2*xp) - 0.5*xp-0.25
  plt.plot(xp,f, 'c', label="Functional curve")
  plt.savefig("Euler_Method_h_0.01.png")
  plt.grid()
  plt.legend()
  plt.title("h=0.01")
  plt.show()

 # for interval h=0.001--->
 elif h==0.001:
  # plt.subplot(1,3,3)
  plt.plot(lstx,lsty,'-r',label ="Numerical Euler method curve")
  xp= np.linspace(0,1.2,1201)
  f=0.25*np.exp(2*xp) - 0.5*xp-0.25
  plt.plot(xp,f, 'c', label="Functional curve")
  plt.title("h=0.001")
  plt.savefig("Euler_Method_h_0.001.png")
  plt.grid()
  plt.legend()
  plt.show()

euler_method(x_0,y_0,h,t)
#euler_method(0,1,0.025,.1)

##----------plotting -------------

def f(x):
    return 0.25*np.exp(2*x) - 0.5*x-0.25
print ("The exact value is ", f(1.2))

#x= np.linspace(0,1.2,121)
# f=0.25*np.exp(2*x) - 0.5*x-0.25
# plt.plot(x,f,lstx,lsty)
# plt.xlabel('x')  # naming the x axis
# plt.ylabel('f(x)')  # naming the y axis
# plt.show()