# -*- coding: utf-8 -*-
"""
Created on Wed Feb 28 08:13:49 2018

@author: adrian
"""

import math
import sympy as sy

# Define the variable and the function to approximate
x = sy.Symbol('x')
f = sy.exp(x)

# Taylor approximation at x0 of the function 'function'
def taylor(function,x0,n):
    i = 0
    p = 0
    while i <= n:
        p = p + (function.diff(x,i).subs(x,x0))/(math.factorial(i))*(x-x0)**i
        i += 1
    return p

#n es número de términos y x es valor a evaluar
def evaluarTaylor(n,valorEvaluacion):
    func = str(taylor(f,0,n))
    x = valorEvaluacion
    return eval(func)

def aitken(x0,x1,x2,valorEvaluacion,E):
    err = 1
    j = 3
    i = 1
    print("\t      Término \t     Iteración \t Error relativo")
    a0 = x2-((x2-x1)**2)/(x2-2*x1+x0) #fórmula
    print("\t",a0,"\t                ",i)
    while(err >= E):
        a1 = a0
        x0 = evaluarTaylor(j-2,valorEvaluacion)
        x1 = evaluarTaylor(j-1,valorEvaluacion)
        x2 = evaluarTaylor(j,valorEvaluacion)
        a0 = x2-((x2-x1)**2)/(x2-2*x1+x0) #fórmula
        err = (abs(a0-a1)/abs(a0))
        j += 1
        i += 1
        print("\t",a0,"\t",i,"\t",err)

#func = str(taylor(f,0,5))
#print("Términos de la serie tomados: ",func)

valorEvaluacion = 1

x0 = evaluarTaylor(0,valorEvaluacion)
x1 = evaluarTaylor(1,valorEvaluacion)
x2 = evaluarTaylor(2,valorEvaluacion)

E = 1.e-15

aitken(x0, x1, x2, valorEvaluacion,E)


#!/usr/bin/env python
# -*- coding: utf-8 -*-

import numpy as np
import matplotlib.pylab as plt
import matplotlib.image as mpimg
plt.rcParams['figure.figsize'] = 15, 6
#plt.rcParams['figure.dpi'] = 300

NUMBER = 10
IMG_SHAPE = (256, 256)
GREY_LEVEL = 128
MU = 0
SIGMA = 2

class Homework2(object):

    def __init__(self):
        pass

    @staticmethod
    def gen_one_image():
        """Generate one image."""
        img = GREY_LEVEL * np.ones(IMG_SHAPE)
        noise = np.random.normal(MU, SIGMA, IMG_SHAPE)
        image = img + noise

        return image

    def gen_image(self, nimage):
        """Generate all required images."""
        tmp = [0] * nimage
        for i in np.arange(nimage):
            tmp[i] = self.gen_one_image()

        return np.array(tmp)

    @staticmethod
    def EST_NOISE(images):
        """Implementation of EST_NOISE in Chapter 2 of Trucco and Verri."""
        num = images.shape[0]
        m_e_bar = sum(images)/num
        m_sigma = np.sqrt(sum((images - m_e_bar)**2)/(num - 1))

        return m_sigma

    def solve_p1(self):
        """Solve problem #1."""
        all_image = self.gen_image(NUMBER)
        result = self.EST_NOISE(all_image)
        print("\n\nSolution for problem #1.")
        self.plot_result(all_image, result)

        self.images = all_image

    @staticmethod
    def apply_2d_filter(bfilter, timage):
        """Apply given 2D filter onto an image.

        Parameters
        ----------
        bfilter: array-like
            The filter
        timage: array-like
            Targeted image
        """
        image_shape = timage.shape
        ovrlay = int(bfilter.shape[0] / 2)
        tmp_matrix = np.zeros(np.array(image_shape) + 2 * ovrlay)
        tmp_matrix[ovrlay:-ovrlay, ovrlay:-ovrlay] = timage
        res_matrix = np.zeros(timage.shape)
        for i in np.arange(image_shape[0]) + ovrlay:
            for j in np.arange(image_shape[1]) + ovrlay:
                local_matrix = tmp_matrix[i - ovrlay:i + ovrlay + 1,
                                          j - ovrlay:j + ovrlay + 1]
                res_matrix[i - ovrlay, j - ovrlay] = sum(sum(local_matrix * bfilter))
        return res_matrix

    def solve_p2(self):
        """Solve problem #2."""
        box_filter = np.ones((3, 3)) / 9
        all_image = self.images
        filted_image = [0] * NUMBER
        for i in np.arange(NUMBER):
            filted_image[i] = self.apply_2d_filter(box_filter, all_image[i])
        filted_image = np.array(filted_image)
        result = self.EST_NOISE(filted_image)
        print("\n\nSolution for problem #2.")
        self.plot_result(filted_image, result)

    @staticmethod
    def plot_result(images, res):
        """Plot results."""
        fig = plt.figure()
        for i in np.arange(NUMBER):
            ax = fig.add_subplot(2, 5, i + 1)
            ax.axis('off')
            ax.imshow(images[i], cmap=plt.cm.gray)
        fig.suptitle(f"Estimated Noise {res.mean():.4f}, Worst Case Noise {res.max():.4f}",
                     fontsize=12)
        fig.show()

    def solve_p3(self, sigma=1.4, show=True):
        """Solve problem #1."""
        n = int(2 * np.ceil(2 * sigma) + 1)
        # 2D Gaussian
        ovrlay = int(n / 2)
        inds = np.arange(-ovrlay, ovrlay + 1)
        x, y = np.meshgrid(inds, inds)
        mask = np.exp(-(x**2 + y**2)/(2*sigma**2))
        mask = mask/sum(sum(mask))

        # two 1D Gaussian
        gaussian_1d = np.exp(-inds**2/(2 * sigma**2))
        gaussian_1d = gaussian_1d /sum(gaussian_1d).reshape((-1, 1))
        mask2 = gaussian_1d * gaussian_1d.T

        print("\n\nSolution for problem #3.")

        if show:
            Fsize = 16
            test_img = mpimg.imread(r".\fig\test.png")
            output_img1 = self.apply_2d_filter(mask, test_img)
            output_img2 = self.apply_2d_filter(mask2, test_img)
            fig = plt.figure()
            ax1 = fig.add_subplot(131)
            ax1.axis('off')
            ax1.imshow(test_img, cmap=plt.cm.gray)
            ax1.set_title("Raw image", fontsize=Fsize)
            ax2 = fig.add_subplot(132, sharex=ax1, sharey=ax1)
            ax2.axis('off')
            ax2.imshow(output_img1, cmap=plt.cm.gray)
            ax2.set_title(f"2D {n}x{n} Gaussian", fontsize=Fsize)
            ax3 = fig.add_subplot(133, sharex=ax1, sharey=ax1)
            ax3.axis('off')
            ax3.imshow(output_img2, cmap=plt.cm.gray)
            ax3.set_title(f"Two 1D n={n} Gaussian", fontsize=Fsize)
            fig.show()

        return mask, gaussian_1d

    @staticmethod
    def apply_1d_filter(bfilter, timage):
        """Apply given 1D filter onto an image.

        Parameters
        ----------
        bfilter: array-like
            The filter
        timage: array-like
            Targeted image
        """
        image_length = len(timage)
        ovrlay = int(bfilter.shape[0] / 2)
        tmp_array = np.zeros(image_length + 2 * ovrlay)
        tmp_array[ovrlay:-ovrlay] = timage
        res_array = np.zeros(image_length )
        for i in np.arange(image_length) + ovrlay:
            local_matrix = tmp_array[i - ovrlay:i + ovrlay + 1]
            res_array[i - ovrlay] = sum(local_matrix * bfilter)
        return res_array

    @staticmethod
    def apply_1d_median_filter(n, timage):
        """Applying a 3 median flter on the image I assuming that the
        border pixels are not changed.

        Parameters
        ----------
        n: int
            Shape of median filter
        timage: array-like
            Targeted image
        """
        image_shape = timage.shape
        ovrlay = int(n / 2)
        res_matrix = np.copy(timage)
        for i in np.arange(image_shape[0])[1:-1]:
            local_matrix = timage[i - ovrlay:i + ovrlay + 1]
            median = np.median(local_matrix)
            res_matrix[i] = median
        return res_matrix

    def solve_p4(self):
        """Solve problem #4."""
        I = np.array([10] * 5 + [40] * 5)
        filter1 = np.ones(5)/5
        filter2 = np.array([1, 2, 4, 2, 1]) / 10

        O1 = self.apply_1d_filter(filter1, I).astype(int)
        O2 = self.apply_1d_filter(filter2, I).astype(int)
        print("\n\nSolution for problem #4.")
        print("Filter (a)")
        print(O1)
        print("Filter (b)")
        print(O2)
        return O1, O2

    @staticmethod
    def apply_2d_median_filter(n, timage):
        """Applying a nxn median filter on the image I assuming that the
        border pixels are not changed."""
        image_shape = timage.shape
        ovrlay = int(n / 2)
        res_matrix = np.copy(timage)
        for i in np.arange(image_shape[0])[1:-1]:
            for j in np.arange(image_shape[1])[1:-1]:
                local_matrix = timage[i - ovrlay:i + ovrlay + 1,
                                      j - ovrlay:j + ovrlay + 1]
                median = np.median(local_matrix)
                res_matrix[i, j] = median
        return res_matrix

    def solve_p5(self, show=True):
        """Solve problem #5."""
        print("\n\nSolution for problem #5.")
        if show:
            fig = plt.figure()
            ax1 = fig.add_subplot(121)
            ax1.stem([-100, 0, 100], np.array([49, 42, 9])/100)
            ax1.set_ylabel("Probability")
            ax1.set_xlabel("Output")
            ax1.set_title("On the line")
            ax2 = fig.add_subplot(122)
            ax2.stem([-150, -50, 50, 100], np.array([21, 9, 21, 49])/100)
            ax2.set_ylabel("Probability")
            ax2.set_xlabel("Output")
            ax2.set_title("On the adjacent line")
            fig.show()

    def solve_p6(self):
        """Solve problem #6."""
        I = np.zeros((8, 8)).astype(int)
        for i in np.arange(8):
            for j in np.arange(8):
                I[i, j] = np.abs(i - j)
        O = self.apply_2d_median_filter(3, I).astype(int)
        print("\n\nSolution for problem #6.")
        print(I)
        print(O)
        return I, O

    def solve_p7(self):
        """Solve problem #7."""
        I = np.array([4] * 4 + [8] * 4)
        O1 = self.apply_1d_median_filter(3, I)
        avgfilter = np.array([1, 2, 1]) / 4
        O2 = np.copy(I)
        O2[1:-1] = self.apply_1d_filter(avgfilter, I)[1:-1]

        print("\n\nSolution for problem #7.")
        print("Median Filter")
        print(O1)
        print("Average Mask")
        print(O2)
        return O1, O2

if __name__ == "__main__":
    hw2 = Homework2()
    hw2.solve_p1()
    hw2.solve_p2()
    hw2.solve_p3()
    hw2.solve_p4()
    hw2.solve_p5()
    hw2.solve_p6()
    hw2.solve_p7()
#


'''
This script demonstrate how to optimize the hyperparameters for a
Gaussian process based on the marginal likelihood. Optimization is
performed in two ways, first with a grid search method and then with a
downhill simplex method.
'''
import numpy as np
import matplotlib.pyplot as plt
from scipy.optimize import fmin
from rbf.gproc import gpiso
np.random.seed(3)

# True signal which we want to recover. This is a exponential function
# with mean=0.0, variance=2.0, and time-scale=0.1. For graphical
# purposes, we will only estimate the variance and time-scale.
eps = 0.1
var = 2.0
gp = gpiso('exp', eps=eps, var=var)

n = 500 # number of observations
time = np.linspace(-5.0, 5.0, n)[:,None] # observation points
data = gp.sample(time) # signal which we want to describe

# find the optimal hyperparameter with a brute force grid search
eps_search = 10**np.linspace(-2, 1, 30)
var_search = 10**np.linspace(-2, 2, 30)
log_likelihoods = np.zeros((30, 30))
for i, eps_test in enumerate(eps_search):
  for j, var_test in enumerate(var_search):
    gp = gpiso('exp', eps=eps_test, var=var_test)
    log_likelihoods[i, j] = gp.log_likelihood(time, data)

# find the optimal hyperparameters with a positively constrained
# downhill simplex method
def fmin_pos(func, x0, *args, **kwargs):
  '''fmin with positivity constraint'''
  def pos_func(x, *blargs):
    return func(np.exp(x), *blargs)

  out = fmin(pos_func, np.log(x0), *args, **kwargs)
  out = np.exp(out)
  return out

def objective(x, t, d):
  '''objective function to be minimized'''
  gp = gpiso('exp', eps=x[0], var=x[1])
  return -gp.log_likelihood(t,d)

# maximum likelihood estimate
eps_mle, var_mle = fmin_pos(objective, [1.0, 1.0], args=(time, data))

# plot the results
fig, axs = plt.subplots(2, 1, figsize=(6, 6))
ax = axs[0]
ax.grid(True)
ax.plot(time[:,0], data, 'k.', label='observations')
ax.set_xlim((-5.0, 5.0))
ax.set_ylim((-5.0, 7.0))
ax.set_xlabel('time', fontsize=10)
ax.set_ylabel('observations', fontsize=10)
ax.tick_params(labelsize=10)
ax.text(0.55, 0.85, r"$C_u(x,x') = \sigma^2\exp(-|x - x'|/\epsilon)$", transform=ax.transAxes)
ax.text(0.55, 0.775, r"$\sigma^2 = %s$, $\epsilon = %s$" % (var, eps), transform=ax.transAxes)

ax = axs[1]
ax.set_yscale('log')
ax.set_xscale('log')
ax.set_xlabel(r'$\epsilon$', fontsize=10)
ax.set_ylabel(r'$\sigma^2$', fontsize=10)
ax.tick_params(labelsize=10)
p = ax.contourf(eps_search, var_search, log_likelihoods.T, np.linspace(-5000.0, -200.0, 11), cmap='viridis')
cbar = plt.colorbar(p, ax=ax)
ax.plot([eps], [var], 'ro', markersize=10, mec='none', label='truth')
ax.plot([eps_mle], [var_mle], 'ko', markersize=10, mec='none', label='max likelihood')
ax.legend(fontsize=10, loc=2)
cbar.set_label('log likelihood', fontsize=10)
cbar.ax.tick_params(labelsize=10)
ax.grid(True)
plt.tight_layout()
plt.savefig('../figures/gproc.f.png')
plt.show()


# a simple cubic spline example.
#
# generate some random data in 10 intervals -- note the data changes
# each time this is run.
#
# Our form of the spline polynomial comes from Pang, Ch. 2
#
# in this version, we use an iterative method to solve the tridiagonal system
#
# plot the splines
#
# M. Zingale (2013-03-03)

import numpy
import pylab
import math
from scipy import linalg   # scipy modules need to be imported separately

# Jacobi tolerance
tol = 1.e-12

# plot a spline
def plot_spline(x0, x1, f0, f1, ppp0, ppp1):

    # lots of points for a smooth plot
    x = numpy.linspace(x0, x1, 100)

    dx = x1-x0

    alpha = ppp1/(6.0*dx)
    beta = -ppp0/(6.0*dx)

    gamma = (-ppp1*dx*dx/6.0 + f1)/dx
    eta = (ppp0*dx*dx/6.0 - f0)/dx

    p = alpha*(x-x0)**3 + beta*(x-x1)**3 + gamma*(x-x0) + eta*(x-x1)

    pylab.plot(x, p)

# number of intervals
n = 20

xmin = 0.0
xmax = 1.0

# coordinates of the data locations
x = numpy.linspace(xmin, xmax, n+1)
dx = x[1] - x[0]

# random data
f = numpy.random.rand(n+1)

# we are solving for n-1 unknowns

# setup the righthand side of our matrix equation
b = numpy.zeros(n+1)

# b_i = (6/dx) * (f_{i-1} - 2 f_i + f_{i+1})
# here we do this with slice notation to fill the
# inner n-1 slots of b
b[1:n] = (6.0/dx)*(f[0:n-1] - 2.0*f[1:n] + f[2:n+1])

# use Jacobi iteration to solve this system.  Note, we don't need to
# write down the matrix for this, we know that the form of the system
# we are solving is:
#
#   dx x_{i-1} + 4 dx x_i + dx_{i+1} = b_i
#
# we only solve for i = 1, ..., n-1, and we impose the boundary
# conditions: x[0] = x[n] = 0

# allocate space for the solution and old value
xold = numpy.zeros(n+1)
xsol = numpy.zeros(n+1)

# note that the BCs: x[0] = x[n] = 0 are already initialized

# iterate
err = 1.e30
iter = 0
while (err > tol):
    # xsol_i = (b_i - dx x_{i-1} - dx x_{i+1})/(4dx)
    xsol[1:n] = (b[1:n] - dx*xold[0:n-1] - dx*xold[2:n+1])/(4.0*dx)

    iter += 1
    err = numpy.max(numpy.abs((xsol[1:n] - xold[1:n])/xsol[1:n]))

    print iter, err
    xold = xsol.copy()

# for debugging, use the built-in banded solver from numpy
u = numpy.zeros(n-1)
d = numpy.zeros(n-1)
l = numpy.zeros(n-1)

d[:] = 4.0*dx
u[:] = dx; u[0] = 0.0
l[:] = dx; l[n-2] = 0.0

# create a banded matrix -- this doesn't store every element -- just
# the diagonal and one above and below and solve
A = numpy.matrix([u,d,l])
xsol_np = linalg.solve_banded((1,1), A, b[1:n])

# xsol_np now hold all the second derivatives for points 1 to n-1.
# Natural boundary conditions are imposed here
xsol_np = numpy.insert(xsol_np, 0, 0)
xsol_np = numpy.insert(xsol_np, n, 0) # insert at the end

# report error from our iterative method vs. direct solve
err = numpy.max(numpy.abs((xsol[1:n] - xsol_np[1:n])/xsol_np[1:n]))

print "relative error of iterative solution compared to direct: ", err

# go ahead with our iterative solution -- natural boundary conditions are
# already in place for this
ppp = xsol

# now plot -- data points first
pylab.scatter(x, f, marker="x", color="r")

# plot the splines
i = 0
while i < n:

    # working on interval [i,i+1]
    ppp_i = ppp[i]
    ppp_ip1 = ppp[i+1]

    f_i = f[i]
    f_ip1 = f[i+1]

    x_i = x[i]
    x_ip1 = x[i+1]

    plot_spline(x_i, x_ip1, f_i, f_ip1, ppp_i, ppp_ip1)

    i += 1

pylab.savefig("spline-iterative.png")

# note: we could have done this all through scipy -- here is their
# spline, but it doesn't seem to support natural boundary conditions

#s = interpolate.InterpolatedUnivariateSpline(x, f, k=3)
#xx = numpy.linspace(xmin, xmax, 1000)
#pylab.plot(xx, s(xx), color="k", ls=":")

# old way from scipy -- this raises a NotImplementedError for natural
#spl1 = interpolate.splmake(x, f, order=3, kind="natural")
#xx = numpy.linspace(xmin, xmax, 1000)
#yy = interpolate.spleval(spl1, xx)
#pylab.plot(xx, yy, color="k", ls=":")

#pylab.savefig("spline-scipy.png")


import numpy as np
import matplotlib.pyplot as plt

class regressor(object):
    """
    This is a sample class for lecture 1.

    Args:
        data: Is a tuple, ``(x,y)``
              ``x`` is a two or one dimensional ndarray ordered such that axis 0 is independent
              data and data is spread along axis 1. If the array had only one dimension, it implies
              that data is 1D.
              ``y`` is a 1D ndarray it will be of the same length as axis 0 or x.

    """
    def __init__(self, data):
        self.x, self.y = data
        # Here is where your training and all the other magic should happen.
        # Once trained you should have these parameters trained.
        x = np.concatenate((np.ones((self.x.shape[0],1)), self.x), axis = 1)
        w = np.dot(np.linalg.pinv(np.dot(x.T,x)), np.dot(x.T,self.y))
        self.w = w[1:]
        self.b = w[0]

    def get_params (self):
        """
        Method that should return the model parameters.

        Returns:
            tuple of numpy.ndarray: (w, b).

        Notes:
            This code will return a random numpy array for demonstration purposes.

        """
        return (self.w, self.b)

    def get_predictions (self, x):
        """
        Method should return the outputs given unseen data

        Args:
            x: array similar to ``x`` in ``data``. Might be of different size.

        Returns:
            numpy.ndarray: ``y`` which is a 1D array of predictions of the same length as axis 0 of
                            ``x``
        """
        predictions = np.add(np.dot(x, self.w), self.b)
        return predictions

    def plot(self, data = None, color = 'r'):
        """ Method will plot the line on an existing pyplot.
        If data is provided, it will plot with the data.

        Args:
            data: tuple of `(x,y)`.
            color: What color the line should be default is red.
        """
        if data is not None:
            x, y = data
            plt.plot(x[:], y, 'bo')
        plt.axis('equal')
        plt.title(' Analytical Solution for Least Squares Linear Regression')
        plt.title('Amazon Employee Compensation (Linear) Dataset')
        plt.xlabel('Years of experience of the employee.')
        plt.ylabel('Compensation in $100,000')
        grid = np.asarray([0, 10])[:,np.newaxis]
        predictions = self.get_predictions(grid)
        plt.plot(grid, predictions, color)
        plt.show()

class ridge_regressor(regressor):
    """
    This is a sample class for lecture 1.

    Args:
        data: Is a tuple, ``(x,y)``
              ``x`` is a two or one dimensional ndarray ordered such that axis 0 is independent
              data and data is spread along axis 1. If the array had only one dimension, it implies
              that data is 1D.
              ``y`` is a 1D ndarray it will be of the same length as axis 0 or x.
        alpha: Co-efficient for L2 regularizer.

    """
    def __init__(self, data, alpha = 0.0001):
        self.x, self.y = data
        # Here is where your training and all the other magic should happen.
        # Once trained you should have these parameters trained.
        x = np.concatenate((np.ones((self.x.shape[0],1)), self.x), axis = 1)
        w = np.dot(np.linalg.pinv(np.dot(x.T,x) + alpha*np.eye(x.shape[1])), np.dot(x.T,self.y))
        self.w = w[1:]
        self.b = w[0]

if __name__ == '__main__':
    pass


"""
.. module:: lin_rg
    :synopsis: Provides routines to construct a Linear Regression.

.. moduleauthor:: Benardi Nunes <benardinunes@gmail.com>
"""

from numpy import zeros, float64
from numpy.linalg import inv, LinAlgError

# model hypothesis
def h(X, theta):
    """Linear regression hypothesis.

    :param X: Features' dataset plus bias column.
    :type X: numpy.array

    :param theta: Column vector of model's parameters.
    :type theta: numpy.array

    :returns: The projected value for each line of the dataset.
    :rtype: numpy.array
    """
    return X.dot(theta)

def cost_func(X, y, theta):
    """Computes the cost function J for Linear Regression.

    :param X: Features' dataset plus bias column.
    :type X: numpy.array

    :param y: Column vector of expected values.
    :type y: numpy.array

    :param theta: Column vector of model's parameters.
    :type theta: numpy.array

    :returns: Computed cost.
    :rtype: float
    """
    m = len(y)  # number of training examples
    J = (1 / (2 * m)) * ((h(X, theta) - y).T).dot(h(X, theta) - y)
    return J

def reg_cost_func(X, y, theta, _lambda):
    """Computes the regularized cost function J for Linear Regression.

    :param X: Features' dataset plus bias column.
    :type X: numpy.array

    :param y: Column vector of expected values.
    :type y: numpy.array

    :param theta: Column vector of model's parameters.
    :type theta: numpy.array

    :param _lambda: The regularization hyperparameter.
    :type _lambda: float

    :returns: Computed cost with regularization.
    :rtype: float
    """
    m = len(y)  # number of training examples
    J = (1 / (2 * m)) * ((h(X, theta) - y).T).dot(h(X, theta) - y)
    J = J + (_lambda / (2 * m)) * (theta[1:, :].T).dot(theta[1:, :])
    return J

def grad(X, y, theta):
    """Computes the gradient for Linear Regression.

    :param X: Features' dataset plus bias column.
    :type X: numpy.array

    :param y: Column vector of expected values.
    :type y: numpy.array

    :param theta: Column vector of model's parameters.
    :type theta: numpy.array

    :returns: Gradient column vector.
    :rtype: numpy.array
    """
    m = len(y)
    grad = (1 / m) * (X.T).dot(h(X, theta) - y)
    return grad

def reg_grad(X, y, theta, _lambda):
    """Computes the regularized gradient for Linear Regression.

    :param X: Features' dataset plus bias column.
    :type X: numpy.array

    :param y: Column vector of expected values.
    :type y: numpy.array

    :param theta: Column vector of model's parameters.
    :type theta: numpy.array

    :param _lambda: The regularization hyperparameter.
    :type _lambda: float

    :returns: Regularized gradient column vector.
    :rtype: numpy.array
    """
    m = len(y)
    grad = (1 / m) * (X.T).dot(h(X, theta) - y)
    grad[1:, :] = grad[1:, :] + (_lambda / m) * theta[1:, :]
    return grad

def predict(X, theta):
    """Computes prediction vector.

    :param X: Features' dataset plus bias column.
    :type X: numpy.array

    :param theta: Column vector of model's parameters.
    :type theta: numpy.array

    :returns: vector with predictions for each input line.
    :rtype: numpy.array
    """
    return X.dot(theta)

def normal_eqn(X, y):
    """Produces optimal theta via normal equation.

    :param X: Features' dataset plus bias column.
    :type X: numpy.array

    :param y: Column vector of expected values.
    :type y: numpy.array

    :raises: LinAlgError

    :returns: Optimized model parameters theta.
    :rtype: numpy.array
    """
    n = X.shape[1]  # number of columns
    theta = zeros((n + 1, 1), dtype=float64)

    try:
        X_T = X.T
        theta = inv(X_T.dot(X)).dot(X_T).dot(y)

    except LinAlgError:
        pass

    return theta


from __future__ import print_function
from sympy import symbols, Matrix, Poly, zeros, eye, Indexed, simplify, IndexedBase, init_printing, pprint
from operator import mul
from functools import reduce
import numpy as np

def At(a,m,n):
    return Matrix(m, n, lambda i,j: a[i]**j)

def A(a,m,n):
    return At(a, m-1, n).row_insert(m-1, Matrix(1, n, lambda i,j: 1 if j==n-1 else 0))

def T(a,n):
    return Matrix(Matrix.eye(n).col_insert(n, Matrix(n, 1, lambda i,j: -a[i]**n)))

def Lx(a,n):
    x=symbols('x')
    return Matrix(n, 1, lambda i,j: Poly((reduce(mul, ((x-a[k] if k!=i else 1) for k in range(0,n)), 1)).expand(basic=True), x))

def F(a,n):
    return Matrix(n, 1, lambda i,j: reduce(mul, ((a[i]-a[k] if k!=i else 1) for k in range(0,n)), 1))

def Fdiag(a,n):
    f=F(a,n)
    return Matrix(n, n, lambda i,j: (f[i,0] if i==j else 0))

def FdiagPlus1(a,n):
    f = Fdiag(a,n-1)
    f = f.col_insert(n-1, zeros(n-1,1))
    f = f.row_insert(n-1, Matrix(1,n, lambda i,j: (1 if j==n-1 else 0)))
    return f

def L(a,n):
    lx = Lx(a,n)
    f = F(a, n)
    return Matrix(n, n, lambda i,j: lx[i, 0].nth(j)/f[i]).T

def Bt(a,n):
    return L(a,n)*T(a,n)

def B(a,n):
    return Bt(a,n-1).row_insert(n-1, Matrix(1, n, lambda i,j: 1 if j==n-1 else 0))

FractionsInG=0
FractionsInA=1
FractionsInB=2
FractionsInF=3

def cookToomFilter(a,n,r,fractionsIn=FractionsInG):
    alpha = n+r-1
    f = FdiagPlus1(a,alpha)
    if f[0,0] < 0:
        f[0,:] *= -1
    if fractionsIn == FractionsInG:
        AT = A(a,alpha,n).T
        G = (A(a,alpha,r).T/f).T
        BT = f * B(a,alpha).T
    elif fractionsIn == FractionsInA:
        BT = f * B(a,alpha).T
        G = A(a,alpha,r)
        AT = (A(a,alpha,n)).T/f
    elif fractionsIn == FractionsInB:
        AT = A(a,alpha,n).T
        G = A(a,alpha,r)
        BT = B(a,alpha).T
    else:
        AT = A(a,alpha,n).T
        G = A(a,alpha,r)
        BT = f * B(a,alpha).T
    return (AT,G,BT,f)

def filterVerify(n, r, AT, G, BT):

    alpha = n+r-1

    di = IndexedBase('d')
    gi = IndexedBase('g')
    d = Matrix(alpha, 1, lambda i,j: di[i])
    g = Matrix(r, 1, lambda i,j: gi[i])

    V = BT*d
    U = G*g
    M = U.multiply_elementwise(V)
    Y = simplify(AT*M)

    return Y

def convolutionVerify(n, r, B, G, A):

    di = IndexedBase('d')
    gi = IndexedBase('g')

    d = Matrix(n, 1, lambda i,j: di[i])
    g = Matrix(r, 1, lambda i,j: gi[i])

    V = A*d
    U = G*g
    M = U.multiply_elementwise(V)
    Y = simplify(B*M)

    return Y

def showCookToomFilter(a,n,r,fractionsIn=FractionsInG):

    AT,G,BT,f = cookToomFilter(a,n,r,fractionsIn)

    print ("AT = ")
    pprint(AT)
    print ("")

    print ("G = ")
    pprint(G)
    print ("")

    print ("BT = ")
    pprint(BT)
    print ("")

    if fractionsIn != FractionsInF:
        print ("FIR filter: AT*((G*g)(BT*d)) =")
        pprint(filterVerify(n,r,AT,G,BT))
        print ("")

    if fractionsIn == FractionsInF:
        print ("fractions = ")
        pprint(f)
        print ("")

def showCookToomConvolution(a,n,r,fractionsIn=FractionsInG):

    AT,G,BT,f = cookToomFilter(a,n,r,fractionsIn)

    B = BT.transpose()
    A = AT.transpose()

    calculateError(AT,G,BT,f)
    #print ("A = ")
    #pprint(A)
    #print ("")

    #print ("G = ")
    #pprint(G)
    #print ("")

    #print ("B = ")
    #pprint(B)
    #print ("")

    if fractionsIn != FractionsInF:
    	temp=2
        #print ("Linear Convolution: B*((G*g)(A*d)) =")
        #pprint(convolutionVerify(n,r,B,G,A))
        #print ("")

    if fractionsIn == FractionsInF:
    	temp=2
        #print ("fractions = ")
        #pprint(f)
        #print ("")

# testing method we found in the paper
H = np.array([1,2])
X = np.array([1, 1, 1])

AT,G,BT,f = cookToomFilter((0,1,-1), 2, 3)
print(AT," AT\n")
print(G," G\n")
print(BT," BT\n")
print(f," f\n")

# trying to apply method
# H = image
# X = kernel
# 1D convolution example
"""

(h ∗ x)1D = AT(Gh o BT x)

"""
tc_convolution_result = np.matmul(AT,np.multiply( np.matmul(G,H) , np.matmul(BT,X) ) )

print(tc_convolution_result)

def calculateError(point1, point2, point3):

	A,G,B,f=  cookToomFilter((0,point1, point2,point3,-1), 2, 3)

	A23 = np.asarray(A)
	B23 = np.asarray(B)
	G23 = np.asarray(G, dtype=np.int32)

	A23 = np.asarray(np.transpose(A23), dtype=np.int32)
	B23 = np.asarray(np.transpose(B23), dtype=np.int32)

	print("B23: ",B23,"\n")
	print("G23: ",G23,"\n")
	print("A23: ",A23,"\n")

	#A23 = np.asarray(np.transpose(A))
	#B23 = np.asarray(np.transpose(B))
	#G23 = np.asarray(G)

	#------------------------------------------

	g = np.random.random((4,4))
	f = np.random.random((3,3))

	direct = np.zeros((2,2))
	for i in range(2):
	    for j in range(2):
	        direct[i,j] = np.sum(f * g[i:i+3,j:j+3])

	inner = np.dot(G23, np.dot(f, G23.T)) * np.dot(B23.T, np.dot(g, B23))
	Y = np.dot(A23.T, np.dot(inner, A23))

	print("Error of one Winograd",la.norm(Y - direct)/la.norm(direct))

	convLib = scisig.convolve2d(f,g)
	conv2d = convolve2DToeplitz(f,g)
	g2 = revMatrix(g)
	g2 = padImage(g2,len(f))
	cWino = simpleWinogradAlg(f,g2,2,B23,G23,A23)[0]
	cWino = revMatrix(cWino)

	print("Standard Error:",la.norm(convLib - conv2d, ord=2)/la.norm(convLib, ord=2))
	print("Winograd/TC Error:",la.norm(convLib - cWino, ord=2)/la.norm(convLib, ord=2))

def runBruteForce():
	imageSize = int(input("Image Size?: "))
	kernalSize = int(input("Kernel Size?: "))
	amountOfPoints = (kernalSize + imageSize) -2

	if amountOfPoints == 3:
		for x in range(-1000, 1000, 50):
			point1 = x/1000
			for y in range(-1000, 1000, 100):
				point2 = y/1000
				for z in range(-1000, 1000, 100):
					point3 = z/1000
					if point1 == point2 or point1 == point3 or point2 == point3 or point3 ==0 or point2 ==0 or point1 ==0 or point3 ==-1 or point2 ==-1 or point1 ==-1:
						print("Skipped points ", point1)
					else:
						print(point1, point2, point3)
						calculateError(point1, point2, point3)
	else: print("Didn't seem to be a valid size. Sorry")


#########################
##                     ##
## Irving Gomez Mendez ##
##    April 01, 2021   ##
##                     ##
#########################

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import statsmodels.api as sm
import scipy.interpolate as interpolate
from scipy.stats import t

def m(x):
    return(3*x - 3*x**2 + x**3 - 0.1*x**4)

alpha = 0.05
nn = 30
xx = np.linspace(0,3,nn)
np.random.seed(111)
epsilon = np.random.normal(0, 0.15, nn)
yy = np.array([m(x) for x in xx])+epsilon
x_vect = np.linspace(0,3,100)
y_vect = np.array([m(x) for x in x_vect])

plt.figure(figsize=(10,7.5))
plt.scatter(xx, yy)
plt.plot(x_vect, y_vect,
    color = 'purple', linewidth = 2,
    label = "Regression function")
plt.legend()

XX = np.vstack([np.ones(nn),xx]).T
XX_vect =  np.stack([np.ones(len(x_vect)),x_vect]).T
results = sm.OLS(yy,XX).fit()

plt.figure(figsize=(10,7.5))
plt.scatter(xx, yy)
plt.plot(x_vect, y_vect,
    color = 'purple', linewidth = 2,
    label = "Regression function")
plt.plot(x_vect, results.predict(XX_vect),
    color = 'red', linewidth = 2,
    label = "Linear fit")
plt.legend()

XX = np.vstack([np.ones(nn),xx,xx**2]).T
XX_vect =  np.stack([np.ones(len(x_vect)),x_vect,x_vect**2]).T
results = sm.OLS(yy,XX).fit()

plt.figure(figsize=(10,7.5))
plt.scatter(xx, yy)
plt.plot(x_vect, y_vect,
    color = 'purple', linewidth = 2,
    label = "Regression function")
plt.plot(x_vect, results.predict(XX_vect),
    color = 'red', linewidth = 2,
    label = "Quadratic fit")
plt.legend()

XX = np.vstack([np.ones(nn),xx,xx**2,xx**3]).T
XX_vect =  np.stack([np.ones(len(x_vect)),x_vect,x_vect**2,x_vect**3]).T
results = sm.OLS(yy,XX).fit()

plt.figure(figsize=(10,7.5))
plt.scatter(xx, yy)
plt.plot(x_vect, y_vect,
    color = 'purple', linewidth = 2,
    label = "Regression function")
plt.plot(x_vect, results.predict(XX_vect),
    color = 'red', linewidth = 2,
    label = "Cubic fit")
plt.legend()

XX = np.vstack([np.ones(nn),xx,xx**2,xx**3,xx**4]).T
XX_vect =  np.stack([np.ones(len(x_vect)),x_vect,x_vect**2,x_vect**3,x_vect**4]).T
results = sm.OLS(yy,XX).fit()

plt.figure(figsize=(10,7.5))
plt.scatter(xx, yy)
plt.plot(x_vect, y_vect,
    color = 'purple', linewidth = 2,
    label = "Regression function")
plt.plot(x_vect, results.predict(XX_vect),
    color = 'red', linewidth = 2,
    label = "Quartic fit")
plt.legend()

A = np.vstack([np.ones(10),xx[:10]]).T
B = np.vstack([np.ones(10),xx[10:20]]).T
C = np.vstack([np.ones(10),xx[20:30]]).T
Z = np.zeros((10,2))

XX = np.block([[A,Z,Z],[Z,B,Z],[Z,Z,C]])
bets = np.linalg.solve(XX.T @ XX, XX.T @ yy)
s2 = sum((yy - XX @ bets)**2)/(nn-1)
p = len(bets)

x_grp_1 = x_vect[x_vect<=1]
XX_grp_1 = np.vstack([np.ones(len(x_grp_1)), x_grp_1]).T
x_grp_2 = x_vect[(1<=x_vect) & (x_vect<=2)]
XX_grp_2 = np.vstack([np.ones(len(x_grp_2)), x_grp_2]).T
x_grp_3 = x_vect[(2<=x_vect) & (x_vect<=3)]
XX_grp_3 = np.vstack([np.ones(len(x_grp_3)), x_grp_3]).T

XX_grp = np.block([
    [XX_grp_1[:-1,:], np.zeros((33,2)), np.zeros((33,2))],
    [np.zeros((33,2)), XX_grp_2[:-1,:], np.zeros((33,2))],
    [np.zeros((34,2)), np.zeros((34,2)), XX_grp_3]
])

aux_t_conf = np.sqrt(s2 * np.diag(XX_grp @ np.linalg.solve(XX.T @ XX, XX_grp.T)))
aux_t_pred = np.sqrt(s2 * (1+np.diag(XX_grp @ np.linalg.solve(XX.T @ XX, XX_grp.T))))

low_conf = XX_grp @ bets - t.ppf(1-alpha/2,nn-p)*aux_t_conf
upp_conf = XX_grp @ bets + t.ppf(1-alpha/2,nn-p)*aux_t_conf
low_pred = XX_grp @ bets - t.ppf(1-alpha/2,nn-p)*aux_t_pred
upp_pred = XX_grp @ bets + t.ppf(1-alpha/2,nn-p)*aux_t_pred

plt.figure(figsize=(10,7.5))
plt.fill_between(x_vect, low_pred, upp_pred, facecolor='green', alpha=0.5, label='Prediction interval')
plt.fill_between(x_vect, low_conf, upp_conf, facecolor='yellow', alpha=0.5, label='Confidence interval')
plt.scatter(xx, yy)
plt.plot(x_vect, y_vect,
    color = 'purple', linewidth = 2,
    label = "Regression function")
plt.plot(x_grp_1, XX_grp_1 @ bets[[0,1]],
    color = 'red', linewidth = 2,
    label = "Piecewise lineal fit")
plt.plot(x_grp_2, XX_grp_2 @ bets[[2,3]],
    color = 'red', linewidth = 2)
plt.plot(x_grp_3, XX_grp_3 @ bets[[4,5]],
    color = 'red', linewidth = 2)
plt.legend()

k1 = 1
k2 = 2
K = np.array([[1, k1, -1, -k1, 0, 0], [0, 0, 1, k2, -1, -k2]])

aux1 = np.linalg.solve(XX.T @ XX, K.T)
aux2 = np.linalg.solve(K @ aux1, K @ bets)
tilbets = bets - aux1 @ aux2
s2 = sum((yy - XX @ tilbets)**2)/(nn-1)
p = len(tilbets)

aux_t_conf = np.sqrt(s2 * np.diag(XX_grp @ np.linalg.solve(XX.T @ XX, XX_grp.T)))
aux_t_pred = np.sqrt(s2 * (1+np.diag(XX_grp @ np.linalg.solve(XX.T @ XX, XX_grp.T))))

low_conf = XX_grp @ tilbets - t.ppf(1-alpha/2,nn-p)*aux_t_conf
upp_conf = XX_grp @ tilbets + t.ppf(1-alpha/2,nn-p)*aux_t_conf
low_pred = XX_grp @ tilbets - t.ppf(1-alpha/2,nn-p)*aux_t_pred
upp_pred = XX_grp @ tilbets + t.ppf(1-alpha/2,nn-p)*aux_t_pred

plt.figure(figsize=(10,7.5))
plt.fill_between(x_vect, low_pred, upp_pred, facecolor='green', alpha=0.5, label='Prediction interval')
plt.fill_between(x_vect, low_conf, upp_conf, facecolor='yellow', alpha=0.5, label='Confidence interval')
plt.scatter(xx, yy)
plt.plot(x_vect, y_vect,
    color = 'purple', linewidth = 2,
    label = "Regression function")
plt.plot(x_grp_1, XX_grp_1 @ tilbets[[0,1]],
    color = 'red', linewidth = 2,
    label = "Piecewise lineal fit, with continuity")
plt.plot(x_grp_2, XX_grp_2 @ tilbets[[2,3]],
    color = 'red', linewidth = 2)
plt.plot(x_grp_3, XX_grp_3 @ tilbets[[4,5]],
    color = 'red', linewidth = 2)
plt.legend()

A = np.vstack([np.ones(10),xx[:10],xx[:10]**2,xx[:10]**3]).T
B = np.vstack([np.ones(10),xx[10:20],xx[10:20]**2,xx[10:20]**3]).T
C = np.vstack([np.ones(10),xx[20:30],xx[20:30]**2,xx[20:30]**3]).T
Z = np.zeros((10,4))

XX = np.block([[A,Z,Z],[Z,B,Z],[Z,Z,C]])
bets = np.linalg.solve(XX.T @ XX, XX.T @ yy)

K = np.array([[1, k1, k1**2, k1**3, -1, -k1, -k1**2, -k1**3, 0, 0, 0, 0],
    [0, 0, 0, 0, 1, k2, k2**2, k2**3, -1, -k2, -k2**2, -k2**3]])

aux1 = np.linalg.solve(XX.T @ XX, K.T)
aux2 = np.linalg.solve(K @ aux1, K @ bets)
tilbets = bets - aux1 @ aux2
s2 = sum((yy - XX @ tilbets)**2)/(nn-1)
p = len(tilbets)

x_grp_1 = x_vect[x_vect<=1]
XX_grp_1 = np.vstack([np.ones(len(x_grp_1)), x_grp_1, x_grp_1**2, x_grp_1**3]).T
x_grp_2 = x_vect[(1<=x_vect) & (x_vect<=2)]
XX_grp_2 = np.vstack([np.ones(len(x_grp_2)), x_grp_2, x_grp_2**2, x_grp_2**3]).T
x_grp_3 = x_vect[(2<=x_vect) & (x_vect<=3)]
XX_grp_3 = np.vstack([np.ones(len(x_grp_3)), x_grp_3, x_grp_3**2, x_grp_3**3]).T

XX_grp = np.block([
    [XX_grp_1[:-1,:], np.zeros((33,4)), np.zeros((33,4))],
    [np.zeros((33,4)), XX_grp_2[:-1,:], np.zeros((33,4))],
    [np.zeros((34,4)), np.zeros((34,4)), XX_grp_3]
])

aux_t_conf = np.sqrt(s2 * np.diag(XX_grp @ np.linalg.solve(XX.T @ XX, XX_grp.T)))
aux_t_pred = np.sqrt(s2 * (1+np.diag(XX_grp @ np.linalg.solve(XX.T @ XX, XX_grp.T))))

low_conf = XX_grp @ tilbets - t.ppf(1-alpha/2,nn-p)*aux_t_conf
upp_conf = XX_grp @ tilbets + t.ppf(1-alpha/2,nn-p)*aux_t_conf
low_pred = XX_grp @ tilbets - t.ppf(1-alpha/2,nn-p)*aux_t_pred
upp_pred = XX_grp @ tilbets + t.ppf(1-alpha/2,nn-p)*aux_t_pred

plt.figure(figsize=(10,7.5))
plt.fill_between(x_vect, low_pred, upp_pred, facecolor='green', alpha=0.5, label='Prediction interval')
plt.fill_between(x_vect, low_conf, upp_conf, facecolor='yellow', alpha=0.5, label='Confidence interval')
plt.scatter(xx, yy)
plt.plot(x_vect, y_vect,
    color = 'purple', linewidth = 2,
    label = "Regression function")
plt.plot(x_grp_1, XX_grp_1 @ tilbets[:4],
    color = 'red', linewidth = 2,
    label = "Piecewise cubic fit, with continuity")
plt.plot(x_grp_2, XX_grp_2 @ tilbets[4:8],
    color = 'red', linewidth = 2)
plt.plot(x_grp_3, XX_grp_3 @ tilbets[8:12],
    color = 'red', linewidth = 2)
plt.legend()

K = np.block([[K],
    [0, 1, 2*k1, 3*k1**2, 0, -1, -2*k1, -3*k1**2, 0, 0, 0, 0],
    [0, 0, 0, 0, 0, 1, 2*k2, 3*k2**2, 0, -1, -2*k2, -3*k2**2]])

aux1 = np.linalg.solve(XX.T @ XX, K.T)
aux2 = np.linalg.solve(K @ aux1, K @ bets)
tilbets = bets - aux1 @ aux2
s2 = sum((yy - XX @ tilbets)**2)/(nn-1)
p = len(tilbets)

aux_t_conf = np.sqrt(s2 * np.diag(XX_grp @ np.linalg.solve(XX.T @ XX, XX_grp.T)))
aux_t_pred = np.sqrt(s2 * (1+np.diag(XX_grp @ np.linalg.solve(XX.T @ XX, XX_grp.T))))

low_conf = XX_grp @ tilbets - t.ppf(1-alpha/2,nn-p)*aux_t_conf
upp_conf = XX_grp @ tilbets + t.ppf(1-alpha/2,nn-p)*aux_t_conf
low_pred = XX_grp @ tilbets - t.ppf(1-alpha/2,nn-p)*aux_t_pred
upp_pred = XX_grp @ tilbets + t.ppf(1-alpha/2,nn-p)*aux_t_pred

plt.figure(figsize=(10,7.5))
plt.fill_between(x_vect, low_pred, upp_pred, facecolor='green', alpha=0.5, label='Prediction interval')
plt.fill_between(x_vect, low_conf, upp_conf, facecolor='yellow', alpha=0.5, label='Confidence interval')
plt.scatter(xx, yy)
plt.plot(x_vect, y_vect,
    color = 'purple', linewidth = 2,
    label = "Regression function")
plt.plot(x_grp_1, XX_grp_1 @ tilbets[:4],
    color = 'red', linewidth = 2,
    label = "Piecewise cubic fit, with continuity in first derivative")
plt.plot(x_grp_2, XX_grp_2 @ tilbets[4:8],
    color = 'red', linewidth = 2)
plt.plot(x_grp_3, XX_grp_3 @ tilbets[8:12],
    color = 'red', linewidth = 2)
plt.legend()

K = np.block([[K],
    [0, 0, 2, 6*k1, 0, 0, -2, -6*k1, 0, 0, 0, 0],
    [0, 0, 0, 0, 0, 0, 2, 6*k2, 0, 0, -2, -6*k2]])

aux1 = np.linalg.solve(XX.T @ XX, K.T)
aux2 = np.linalg.solve(K @ aux1, K @ bets)
tilbets = bets - aux1 @ aux2
s2 = sum((yy - XX @ tilbets)**2)/(nn-1)
p = len(tilbets)

aux_t_conf = np.sqrt(s2 * np.diag(XX_grp @ np.linalg.solve(XX.T @ XX, XX_grp.T)))
aux_t_pred = np.sqrt(s2 * (1+np.diag(XX_grp @ np.linalg.solve(XX.T @ XX, XX_grp.T))))

low_conf = XX_grp @ tilbets - t.ppf(1-alpha/2,nn-p)*aux_t_conf
upp_conf = XX_grp @ tilbets + t.ppf(1-alpha/2,nn-p)*aux_t_conf
low_pred = XX_grp @ tilbets - t.ppf(1-alpha/2,nn-p)*aux_t_pred
upp_pred = XX_grp @ tilbets + t.ppf(1-alpha/2,nn-p)*aux_t_pred

plt.figure(figsize=(10,7.5))
plt.fill_between(x_vect, low_pred, upp_pred, facecolor='green', alpha=0.5, label='Prediction interval')
plt.fill_between(x_vect, low_conf, upp_conf, facecolor='yellow', alpha=0.5, label='Confidence interval')
plt.scatter(xx, yy)
plt.plot(x_vect, y_vect,
    color = 'purple', linewidth = 2,
    label = "Regression function")
plt.plot(x_grp_1, XX_grp_1 @ tilbets[:4],
    color = 'red', linewidth = 2,
    label = "Cubic spline with two nodes")
plt.plot(x_grp_2, XX_grp_2 @ tilbets[4:8],
    color = 'red', linewidth = 2)
plt.plot(x_grp_3, XX_grp_3 @ tilbets[8:12],
    color = 'red', linewidth = 2)
plt.legend()

###
tck = interpolate.splrep(xx, yy, t=[k1,k2], k=3)
y_vect_hat = interpolate.splev(x_vect, tck)
s2 = sum((yy - y_hat)**2)/(nn-1)
p = len(tck[1])

aux_t_conf = np.sqrt(s2 * np.diag(XX_grp @ np.linalg.solve(XX.T @ XX, XX_grp.T)))
aux_t_pred = np.sqrt(s2 * (1+np.diag(XX_grp @ np.linalg.solve(XX.T @ XX, XX_grp.T))))

low_conf = y_vect_hat - t.ppf(1-alpha/2,nn-p)*aux_t_conf
upp_conf = y_vect_hat + t.ppf(1-alpha/2,nn-p)*aux_t_conf
low_pred = y_vect_hat - t.ppf(1-alpha/2,nn-p)*aux_t_pred
upp_pred = y_vect_hat + t.ppf(1-alpha/2,nn-p)*aux_t_pred

plt.figure(figsize=(10,7.5))
plt.fill_between(x_vect, low_pred, upp_pred, facecolor='green', alpha=0.5, label='Prediction interval')
plt.fill_between(x_vect, low_conf, upp_conf, facecolor='yellow', alpha=0.5, label='Confidence interval')
plt.scatter(xx, yy)
plt.plot(x_vect, y_vect,
    color = 'purple', linewidth = 2,
    label = "Regression function")
plt.plot(x_vect, y_vect_hat,
    color = 'red', linewidth = 2,
    label = "Cubic spline with two nodes")
plt.legend()

import numpy,import scipy,from scipy,import statsmodels


import numpy as np

# coding: utf-8
import numpy as np

def identity_function(x):
    return x

def step_function(x):
    return np.array(x > 0, dtype=np.int)

def sigmoid(x):
    return 1 / (1 + np.exp(-x))

def sigmoid_grad(x):
    return (1.0 - sigmoid(x)) * sigmoid(x)

def relu(x):
    return np.maximum(0, x)

def relu_grad(x):
    grad = np.zeros(x)
    grad[x >= 0] = 1
    return grad

def softmax(x):
    if x.ndim == 2:
        x = x.T
        x = x - np.max(x, axis=0)
        y = np.exp(x) / np.sum(np.exp(x), axis=0)
        return y.T

    x = x - np.max(x)  # 오버플로 대책
    return np.exp(x) / np.sum(np.exp(x))

def mean_squared_error(y, t):
    return 0.5 * np.sum((y - t) ** 2)

def cross_entropy_error(y, t):
    if y.ndim == 1:
        t = t.reshape(1, t.size)
        y = y.reshape(1, y.size)

    # 훈련 데이터가 원-핫 벡터라면 정답 레이블의 인덱스로 반환
    if t.size == y.size:
        t = t.argmax(axis=1)

    batch_size = y.shape[0]
    return -np.sum(np.log(y[np.arange(batch_size), t])) / batch_size

def softmax_loss(X, t):
    y = softmax(X)
    return cross_entropy_error(y, t)

# ###################################################################################################
# ## 퍼셉트론(단층 & 다층)
# ###################################################################################################
# # AND Gate
# def AND(x1, x2):
#     w1, w2, theta = 0.5, 0.5, 0.7
#     tmp = x1*w1 + x2*w2
#
# # NAND Gate
# def NAND(x1, x2):
#     x = np.array([x1, x2])
#     w = np.array([-0.5, -0.5])
#     b = 0.7
#     return 1 if np.sum(x*w)+b > 0 else 0
#
# # OR Gate
# def OR(x1, x2):
#     x = np.array([x1, x2])
#     w = np.array([0.5, 0.5])
#     b = -0.4
#     return 1 if np.sum(x*w)+b > 0 else 0
#
# # XOR Gate
# def XOR(x1, x2):
#     return AND(OR(x1, x2), NAND(x1, x2))
#
#
# ###################################################################################################
# ## 활성화 함수
# ###################################################################################################
# # 계단 함수
# def step_function(x):
#     return np.array(x > 0, dtype=np.int)
#
# # 시그모이드 함수(출력층에서 2-Class 인 경우)
# def sigmoid(x):
#     return 1/(1+np.exp(-x))
#
# # 소프트맥스 함수(출력층에서 Multi-Class 인 경우)
# def softmax(a):
#     C = np.max(a)
#     exp_a = np.exp(a-C)
#     sum_exp_a = np.sum(exp_a)
#     return exp_a/sum_exp_a
#
# # 항등 함수(회귀 분석인 경우)
# def relu_function(x):
#     return np.maximum(0, x)
#
#
# ###################################################################################################
# ## 미분 관련 함수
# ###################################################################################################
# # 수치 미분
# def numerical_diff(f, x):
#     h = 1e-4  # 0.0001
#     return (f(x+h)-f(x-h))/(2*h)
#
# 편 미분
# def numerical_gradient(f, x):
#     h = 1e-4
#     grad = np.zeros_like(x)  # x와 형상이 같은 배열을 생성
#
#     it = np.nditer(x, flags=['multi_index'], op_flags=['readwrite'])
#
#     while not it.finished:
#         idx = it.multi_index
#
#         # f(x+h) 계산
#         tmp_val = x[idx]
#         x[idx] = tmp_val + h
#         fxh1 = f(x)
#
#         # f(x-h) 계산
#         x[idx] = tmp_val - h
#         fxh2 = f(x)
#
#         # f(x+h) - f(x-h)
#         grad[idx] = (fxh1 - fxh2) / (2*h)
#
#         x[idx] = tmp_val
#         it.iternext()
#
#     return grad
#
# # 경사 하강법
# def gradient_descent(f, init_x, lr=0.01, step_num=10000):
#     x = init_x
#
#     for i in range(step_num):
#         grad = numerical_gradient(f, x)
#         print(grad, x)
#         x -= lr * grad
#
#
# ###################################################################################################
# ## 손실 함수
# ###################################################################################################
# # 평균제곱오차 함수
# def mean_squared_error(y, t):
#     y = np.array(y)
#     t = np.array(t)
#     return np.sum(np.power(y-t, 2))/2
#
# # 교차 엔트로피 함수(배치용)
# def cross_entropy_error(y, t):
#     if y.ndim == 1:
#         t = t.reshape(1, t.size)
#         y = y.reshape(1, y.size)
#
#     delta = 1e-7
#     batch_size = y.shape[0]
#     return -np.sum(t * np.log(y+delta)) / batch_size


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
MIT License

Copyright (c) 2022 Jongrae.K

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
"""

from numpy import linspace
from scipy.integrate import solve_ivp

grv_const = 9.81 # [m/s^2]
init_pos = 0.0 # [m]
init_vel = 0.5 # [m/s]
init_mass = 5.0 #[kg]

init_cond = [init_pos, init_vel, init_mass]

init_time = 0 # [s]
final_time = 5.0 # [s]
num_data = 100
tout = linspace(init_time, final_time, num_data)

def free_falling_obj(time, state, grv_const):
     x1, x2, x3 = state
     dxdt = [x2,
          grv_const + (x3-2)*(x2/x3),
          -x3 + 2]
     return dxdt

sol = solve_ivp(free_falling_obj, (init_time, final_time), init_cond, t_eval=tout, args=(grv_const,))
xout = sol.y

import matplotlib.pyplot as plt
plt.figure(1)
plt.plot(tout,xout[0,:])
plt.ylabel('position [m]')
plt.xlabel('time [s]')

plt.figure(2);
plt.plot(tout,xout[1,:])
plt.ylabel('velocity [m/s]')
plt.xlabel('time [s]')

plt.figure(3)
plt.plot(tout,xout[2,:])
plt.ylabel('m(t) [kg]')
plt.xlabel('time [s]')

from matplotlib.gridspec import GridSpec
fig = plt.figure(2)
gs = GridSpec(2,2,figure=fig)

ax1 = fig.add_subplot(gs[0,0])
ax1.plot(tout,xout[0,:])
ax1.set_ylabel('position [m]')
ax1.set_xlabel('time [s]')

ax2 = fig.add_subplot(gs[0,1])
ax2.plot(tout,xout[1,:])
ax2.set_ylabel('velocity [m/s]')
ax2.set_xlabel('time [s]')

ax3 = fig.add_subplot(gs[1,:])
ax3.plot(tout,xout[2,:])
ax3.set_ylabel('m(t) [kg]')
ax3.set_xlabel('time [s]')


r"""
Polynomial Regression
=====================

This example shows how to use the :py:class:`pylops.Regression` operator
to perform *Polynomial regression analysis*.

In short, polynomial regression is the problem of finding the best fitting
coefficients for the following equation:

    .. math::
        y_i = \sum_{n=0}^{order} x_n t_i^n  \qquad \forall i=1,2,...,N

As we can express this problem in a matrix form:

    .. math::
        \mathbf{y}=  \mathbf{A} \mathbf{x}

our solution can be obtained by solving the following optimization problem:

    .. math::
        J= ||\mathbf{y} - \mathbf{A} \mathbf{x}||_2

See documentation of :py:class:`pylops.Regression` for more detailed
definition of the forward problem.
"""
import numpy as np
import matplotlib.pyplot as plt

import pylops

plt.close('all')
np.random.seed(10)

###############################################################################
# Define the input parameters: number of samples along the t-axis (``N``),
# order (``order``), regression coefficients (``x``), and standard deviation
# of noise to be added to data (``sigma``).
N = 30
order = 3
x = np.array([1., .05, 0., -.01])
sigma = 1

###############################################################################
# Let's create the time axis and initialize the
# :py:class:`pylops.Regression` operator
t = np.arange(N, dtype='float64') - N//2
PRop = pylops.Regression(t, order=order, dtype='float64')

###############################################################################
# We can then apply the operator in forward mode to compute our data points
# along the x-axis (``y``). We will also generate some random gaussian noise
# and create a noisy version of the data (``yn``).
y = PRop*x
yn = y + np.random.normal(0, sigma, N)

###############################################################################
# We are now ready to solve our problem. As we are using an operator from the
# :py:class:`pylops.LinearOperator` family, we can simply use ``/``,
# which in this case will solve the system by means of an iterative solver
# (i.e., :py:func:`scipy.sparse.linalg.lsqr`).
xest = PRop / y
xnest = PRop / yn

###############################################################################
# Let's plot the best fitting curve for the case of noise free and noisy data
plt.figure(figsize=(5, 7))
plt.plot(t, PRop*x, 'k', lw=4,
         label=r'true: $x_0$ = %.2f, $x_1$ = %.2f, '
               r'$x_2$ = %.2f, $x_3$ = %.2f' % (x[0], x[1], x[2], x[3]))
plt.plot(t, PRop*xest, '--r', lw=4,
         label='est noise-free: $x_0$ = %.2f, $x_1$ = %.2f, '
               r'$x_2$ = %.2f, $x_3$ = %.2f' %
               (xest[0], xest[1], xest[2], xest[3]))
plt.plot(t, PRop*xnest, '--g', lw=4,
         label='est noisy: $x_0$ = %.2f, $x_1$ = %.2f, '
               r'$x_2$ = %.2f, $x_3$ = %.2f' %
               (xnest[0], xnest[1], xnest[2], xnest[3]))
plt.scatter(t, y, c='r', s=70)
plt.scatter(t, yn, c='g', s=70)
plt.legend(fontsize='x-small')

###############################################################################
# We consider now the case where some of the observations have large errors.
# Such elements are generally referred to as *outliers* and can affect the
# quality of the least-squares solution if not treated with care. In this
# example we will see how using a L1 solver such as
# :py:func:`pylops.optimization.sparsity.IRLS` can drammatically improve the
# quality of the estimation of intercept and gradient.

# Add outliers
yn[1] += 40
yn[N-2] -= 20

# IRLS
nouter = 20
epsR = 1e-2
epsI = 0
tolIRLS = 1e-2

xnest = PRop / yn
xirls, nouter, xirls_hist, rw_hist = \
    pylops.optimization.sparsity.IRLS(PRop, yn, nouter, threshR=False,
                                      epsR=epsR, epsI=epsI,
                                      tolIRLS=tolIRLS, returnhistory=True)
print('IRLS converged at %d iterations...' % nouter)

plt.figure(figsize=(5, 7))
plt.plot(t, PRop*x, 'k', lw=4,
         label=r'true: $x_0$ = %.2f, $x_1$ = %.2f, '
               r'$x_2$ = %.2f, $x_3$ = %.2f' % (x[0], x[1], x[2], x[3]))
plt.plot(t, PRop*xnest, '--r', lw=4,
         label=r'L2: $x_0$ = %.2f, $x_1$ = %.2f, '
               r'$x_2$ = %.2f, $x_3$ = %.2f' % (xnest[0], xnest[1], xnest[2], xnest[3]))
plt.plot(t, PRop*xirls, '--g', lw=4,
         label=r'IRLS: $x_0$ = %.2f, $x_1$ = %.2f, '
               r'$x_2$ = %.2f, $x_3$ = %.2f' % (xirls[0], xirls[1], xirls[2], xirls[3]))
plt.scatter(t, y, c='r', s=70)
plt.scatter(t, yn, c='g', s=70)
plt.legend(fontsize='x-small')


"""
Agosto 08 de 2021
@author: Diana Sofía Carrillo, Nelson Alejandro Mosquera y Sergio Esteban Triana
Método de Bisección
Análisis Numérico

"""

#LIBRERIAS
import numpy as np
import math

#FUNCIONES
def funcionA(x):
    return (np.cos(x)*np.cos(x)) - x**2 #cos^2(x) - x^2

def funcionB(x):
    return (x*np.sin(x))-1 # xsen(x)-1, evaluar en [-1,2]

def funcionC(x):
    return (x**3)-(2*x**2)+((4/3)*x)-(8/27) #x^3-2x^2+4/3x-8/27

def funcionD(x):
    return ((68.1*9.81)/x)*(1-(math.e)**(-(10/68.1)*x))-40 # 0 = (mg/x)*(1-e^-((t/m) * x))-v(t)

def funcionE(x):
    return (x**3)-(2*x)-5 #x^3-2x-5=0

#METODO
def biseccion(funcion, a, b, eps):
    maxIteraciones = 5000 #Número máximo de iteraciones

    if funcion(a)*funcion(b) > 0 : #Se verifica que cumpla con el teorema del valor intermedio
        print('\t\tEl intervalo dado no contiene una raíz (cero) de la función')
        rta = [0]
        return rta

    for i in range (maxIteraciones):
        c = (a+b)/2

        if np.abs(funcion(c)) < eps:
            rta = [1, i+1, c]
            return rta

        if funcion(a)*funcion(c) < 0:
            b = c
        else:
            a = c

    if i == maxIteraciones - 1:
        rta = [2, i, c]
        return rta

#INVOCACIONES
print('\n----------------------------------------------------------------------------')
print('Funcion A:')
print('\tTorelancia 10E-8')
rta = biseccion(funcionA, 0, 1, 10E-8)
if rta[0] == 1:
    print('\t\tSe encontraron las raices en ' + str(rta[1]) + ' iteraciones. Su valor es: ' + "{:.8f}".format(rta[2]))
elif rta[0] == 2:
    print('\t\tSe ha llegado al número máximo de iteraciones. La raíz aproximada luego del máximo número de iteraciones es: ' + "{:.8f}".format(rta[2]))

print('\tTorelancia 10E-16')
rta = biseccion(funcionA, 0, 1, 10E-16)
if rta[0] == 1:
    print('\t\tSe encontraron las raices en ' + str(rta[1]) + ' iteraciones. Su valor es: ' + "{:.16f}".format(rta[2]))
elif rta[0] == 2:
    print('\t\tSe ha llegado al número máximo de iteraciones. La raíz aproximada luego del máximo número de iteraciones es: ' + "{:.16f}".format(rta[2]))

print('\tTorelancia 10E-32')
rta = biseccion(funcionA, 0, 1, 10E-32)
if rta[0] == 1:
    print('\t\tSe encontraron las raices en ' + str(rta[1]) + ' iteraciones. Su valor es: ' + "{:.32f}".format(rta[2]))
elif rta[0] == 2:
    print('\t\tSe ha llegado al número máximo de iteraciones. La raíz aproximada luego del máximo número de iteraciones es: ' + "{:.32f}".format(rta[2]))

print('\tTorelancia 10E-56')
rta = biseccion(funcionA, 0, 1, 10E-56)
if rta[0] == 1:
    print('\t\tSe encontraron las raices en ' + str(rta[1]) + ' iteraciones. Su valor es: ' + "{:.56f}".format(rta[2]))
elif rta[0] == 2:
    print('\t\tSe ha llegado al número máximo de iteraciones. La raíz aproximada luego del máximo número de iteraciones es: ' + "{:.56f}".format(rta[2]))

print('\n----------------------------------------------------------------------------')
print('Funcion B:')
print('\tTorelancia 10E-8')
rta = biseccion(funcionB, -1, 2, 10E-8)
if rta[0] == 1:
    print('\t\tSe encontraron las raices en ' + str(rta[1]) + ' iteraciones. Su valor es: ' + "{:.8f}".format(rta[2]))
elif rta[0] == 2:
    print('\t\tSe ha llegado al número máximo de iteraciones. La raíz aproximada luego del máximo número de iteraciones es: ' + "{:.8f}".format(rta[2]))

print('\tTorelancia 10E-16')
rta = biseccion(funcionB, -1, 2, 10E-16)
if rta[0] == 1:
    print('\t\tSe encontraron las raices en ' + str(rta[1]) + ' iteraciones. Su valor es: ' + "{:.16f}".format(rta[2]))
elif rta[0] == 2:
    print('\t\tSe ha llegado al número máximo de iteraciones. La raíz aproximada luego del máximo número de iteraciones es: ' + "{:.16f}".format(rta[2]))

print('\tTorelancia 10E-32')
rta = biseccion(funcionB, -1, 2, 10E-32)
if rta[0] == 1:
    print('\t\tSe encontraron las raices en ' + str(rta[1]) + ' iteraciones. Su valor es: ' + "{:.32f}".format(rta[2]))
elif rta[0] == 2:
    print('\t\tSe ha llegado al número máximo de iteraciones. La raíz aproximada luego del máximo número de iteraciones es: ' + "{:.32f}".format(rta[2]))

print('\tTorelancia 10E-56')
rta = biseccion(funcionB, -1, 2, 10E-56)
if rta[0] == 1:
    print('\t\tSe encontraron las raices en ' + str(rta[1]) + ' iteraciones. Su valor es: ' + "{:.56f}".format(rta[2]))
elif rta[0] == 2:
    print('\t\tSe ha llegado al número máximo de iteraciones. La raíz aproximada luego del máximo número de iteraciones es: ' + "{:.56f}".format(rta[2]))

print('\n----------------------------------------------------------------------------')
print('Funcion C:')
print('\tTorelancia 10E-8')
rta = biseccion(funcionC, 0, 1, 10E-8)
if rta[0] == 1:
    print('\t\tSe encontraron las raices en ' + str(rta[1]) + ' iteraciones. Su valor es: ' + "{:.8f}".format(rta[2]))
elif rta[0] == 2:
    print('\t\tSe ha llegado al número máximo de iteraciones. La raíz aproximada luego del máximo número de iteraciones es: ' + "{:.8f}".format(rta[2]))

print('\tTorelancia 10E-16')
rta = biseccion(funcionC, -1, 0.7, 10E-16)
if rta[0] == 1:
    print('\t\tSe encontraron las raices en ' + str(rta[1]) + ' iteraciones. Su valor es: ' + "{:.16f}".format(rta[2]))
elif rta[0] == 2:
    print('\t\tSe ha llegado al número máximo de iteraciones. La raíz aproximada luego del máximo número de iteraciones es: ' + "{:.16f}".format(rta[2]))

print('\tTorelancia 10E-32')
rta = biseccion(funcionC, -1, 0.7, 10E-32)
if rta[0] == 1:
    print('\t\tSe encontraron las raices en ' + str(rta[1]) + ' iteraciones. Su valor es: ' + "{:.32f}".format(rta[2]))
elif rta[0] == 2:
    print('\t\tSe ha llegado al número máximo de iteraciones. La raíz aproximada luego del máximo número de iteraciones es: ' + "{:.32f}".format(rta[2]))

print('\tTorelancia 10E-56')
rta = biseccion(funcionC, -1, 0.7, 10E-56)
if rta[0] == 1:
    print('\t\tSe encontraron las raices en ' + str(rta[1]) + ' iteraciones. Su valor es: ' + "{:.56f}".format(rta[2]))
elif rta[0] == 2:
    print('\t\tSe ha llegado al número máximo de iteraciones. La raíz aproximada luego del máximo número de iteraciones es: ' + "{:.56f}".format(rta[2]))

print('\n----------------------------------------------------------------------------')
print('Funcion D:')
print('\tTorelancia 10E-8')
rta = biseccion(funcionD, 14, 15, 10E-8)
if rta[0] == 1:
    print('\t\tSe encontraron las raices en ' + str(rta[1]) + ' iteraciones. Su valor es: ' + "{:.8f}".format(rta[2]))
elif rta[0] == 2:
    print('\t\tSe ha llegado al número máximo de iteraciones. La raíz aproximada luego del máximo número de iteraciones es: ' + "{:.8f}".format(rta[2]))

print('\tTorelancia 10E-16')
rta = biseccion(funcionD, 14, 15, 10E-16)
if rta[0] == 1:
    print('\t\tSe encontraron las raices en ' + str(rta[1]) + ' iteraciones. Su valor es: ' + "{:.16f}".format(rta[2]))
elif rta[0] == 2:
    print('\t\tSe ha llegado al número máximo de iteraciones. La raíz aproximada luego del máximo número de iteraciones es: ' + "{:.16f}".format(rta[2]))

print('\tTorelancia 10E-32')
rta = biseccion(funcionD, 14, 15, 10E-32)
if rta[0] == 1:
    print('\t\tSe encontraron las raices en ' + str(rta[1]) + ' iteraciones. Su valor es: ' + "{:.32f}".format(rta[2]))
elif rta[0] == 2:
    print('\t\tSe ha llegado al número máximo de iteraciones. La raíz aproximada luego del máximo número de iteraciones es: ' + "{:.32f}".format(rta[2]))

print('\tTorelancia 10E-56')
rta = biseccion(funcionD, 14, 15, 10E-56)
if rta[0] == 1:
    print('\t\tSe encontraron las raices en ' + str(rta[1]) + ' iteraciones. Su valor es: ' + "{:.56f}".format(rta[2]))
elif rta[0] == 2:
    print('\t\tSe ha llegado al número máximo de iteraciones. La raíz aproximada luego del máximo número de iteraciones es: ' + "{:.56f}".format(rta[2]))

print('\n----------------------------------------------------------------------------')
print('Funcion E:')
print('\tTorelancia 10E-8')
rta = biseccion(funcionE, 2, 3, 10E-8)
if rta[0] == 1:
    print('\t\tSe encontraron las raices en ' + str(rta[1]) + ' iteraciones. Su valor es: ' + "{:.8f}".format(rta[2]))
elif rta[0] == 2:
    print('\t\tSe ha llegado al número máximo de iteraciones. La raíz aproximada luego del máximo número de iteraciones es: ' + "{:.8f}".format(rta[2]))

print('\tTorelancia 10E-16')
rta = biseccion(funcionE, 2, 3, 10E-16)
if rta[0] == 1:
    print('\t\tSe encontraron las raices en ' + str(rta[1]) + ' iteraciones. Su valor es: ' + "{:.16f}".format(rta[2]))
elif rta[0] == 2:
    print('\t\tSe ha llegado al número máximo de iteraciones. La raíz aproximada luego del máximo número de iteraciones es: ' + "{:.16f}".format(rta[2]))

print('\tTorelancia 10E-32')
rta = biseccion(funcionE, 2, 3, 10E-32)
if rta[0] == 1:
    print('\t\tSe encontraron las raices en ' + str(rta[1]) + ' iteraciones. Su valor es: ' + "{:.32f}".format(rta[2]))
elif rta[0] == 2:
    print('\t\tSe ha llegado al número máximo de iteraciones. La raíz aproximada luego del máximo número de iteraciones es: ' + "{:.32f}".format(rta[2]))

print('\tTorelancia 10E-56')
rta = biseccion(funcionE, 2, 3, 10E-56)
if rta[0] == 1:
    print('\t\tSe encontraron las raices en ' + str(rta[1]) + ' iteraciones. Su valor es: ' + "{:.56f}".format(rta[2]))
elif rta[0] == 2:
    print('\t\tSe ha llegado al número máximo de iteraciones. La raíz aproximada luego del máximo número de iteraciones es: ' + "{:.56f}".format(rta[2]))


import sys
import numpy as np
from scipy.special import erfc, erfcinv, expm1

def trandn(l,u):
    ## truncated normal generator
    # * efficient generator of a vector of length(l)=length(u)
    # from the standard multivariate normal distribution,
    # truncated over the region [l,u];
    # infinite values for 'u' and 'l' are accepted;
    # * Remark:
    # If you wish to simulate a random variable
    # 'Z' from the non-standard Gaussian N(m,s^2)
    # conditional on l<Z<u, then first simulate
    # X=trandn((l-m)/s,(u-m)/s) and set Z=m+s*X;

    # Reference:
    # Botev, Z. I. (2016). "The normal law under linear restrictions:
    # simulation and estimation via minimax tilting". Journal of the
    # Royal Statistical Society: Series B (Statistical Methodology).
    # doi:10.1111/rssb.12162

    l = np.asarray(l)
    u = np.asarray(u)
    l = l.ravel()
    u = u.ravel() # make 'l' and 'u' column vectors

    if len(l) != len(u):
        print('Truncation limits have to be vectors of the same length')
        sys.exit()

    x = np.empty(len(l))
    a = .66 # treshold for switching between methods

    # three cases to consider:
    # case 1: a<l<u

    I = l>a
    if np.any(I):
        tl=l[I]
        tu=u[I]
        x[I]=ntail(tl,tu)

    # case 2: l<u<-a

    J = u<-a
    if np.any(J):
        tl=-u[J]
        tu=-l[J]
        x[J] = -ntail(tl,tu)

    # case 3: otherwise use inverse transform or accept-reject

    I=~(I|J);
    if np.any(I):
        tl=l[I]
        tu=u[I]
        x[I]=tn(tl,tu)

    return x

#################################################################

def ntail(l,u):

    # samples a column vector of length=length(l)=length(u)
    # from the standard multivariate normal distribution,
    # truncated over the region [l,u], where l>0 and
    # l and u are column vectors;
    # uses acceptance-rejection from Rayleigh distr.
    # similar to Marsaglia (1964);

    c = l**2/2
    n = len(l)
    f = expm1(c-u**2/2)

    x = c - np.log(1+np.random.uniform(size=n)*f); # sample using Rayleigh

    # keep list of rejected

    I = np.random.uniform(size=n)**2*x > c

    while np.any(I): # while there are rejections
        cy = c[I] # find the thresholds of rejected
        y = cy - np.log(1+np.random.uniform(size=len(cy))*f[I])
        idx = (np.random.uniform(size=len(cy))**2)*y < cy # accepted
        tmp = I.copy()
        I[tmp] = idx # make the list of elements in x to update
        x[I] = y[idx] # store the accepted
        I[tmp] = np.logical_not(idx) # remove accepted from list

#    while d>0: # while there are rejections
#        cy = c[I] # find the thresholds of rejected
#        y = cy - np.log(1+np.random.uniform(size=d)*f[I])
#        idx = (np.random.uniform(size=d)**2)*y < cy # accepted
#        x[I[idx]] = y[idx] # store the accepted
#        I = I[~idx] # remove accepted from list
#        d = len(I) # number of rejected

    x = np.sqrt(2*x); # this Rayleigh transform can be delayed till the end

    return x

##################################################################

def tn(l,u):

    # samples a column vector of length=length(l)=length(u)
    # from the standard multivariate normal distribution,
    # truncated over the region [l,u], where -a<l<u<a for some
    # 'a' and l and u are column vectors;
    # uses acceptance rejection and inverse-transform method;
    tol = 2 # controls switch between methods

    # threshold can be tuned for maximum speed for each platform
    # case: abs(u-l)>tol, uses accept-reject from randn

    I = np.abs(u-l)>tol
    x = l

    if np.any(I):
        tl=l[I]
        tu=u[I]
        x[I]=trnd(tl,tu)

    # case: abs(u-l)<tol, uses inverse-transform

    I=~I
    if np.any(I):

        tl=l[I]
        tu=u[I]
        pl = erfc(tl/np.sqrt(2))/2
        pu = erfc(tu/np.sqrt(2))/2

        x[I] = np.sqrt(2)*erfcinv(2*(pl-(pl-pu)
               *np.random.uniform(size=len(tl))))

    return x

#############################################################

def trnd(l,u):

    # uses acceptance rejection to simulate from truncated normal
    x=np.random.randn(len(l)) # sample normal

    # keep list of rejected
    I = np.logical_or(x<l ,x>u)
    while np.any(I): # while there are rejections
        ly = l[I] # find the thresholds of rejected
        uy = u[I]
        y = np.random.randn(len(ly))
        idx = np.logical_and(y>ly,y<uy) # accepted
        tmp = I.copy()
        I[tmp] = idx # make the list of elements in x to update
        x[I] = y[idx] # store the accepted
        I[tmp] = np.logical_not(idx) # remove accepted from list

#    d = len(I)
#    while d>0: # while there are rejections
#        ly = l[I] # find the thresholds of rejected
#        uy = u[I]
#        y = np.random.randn(len(ly))
#        idx = np.logical_and(y>ly,y<uy) # accepted
#        x[I[idx]] = y[idx] # store the accepted
#        I = I[~idx] # remove accepted from list
#        d = len(I) # number of rejected

    return x


# Plot 2d NLL loss surface for  binary logistic regression with 1 feature
# Loosely based on
# https://peterroelants.github.io/posts/neural-network-implementation-part02/

import numpy as np
import matplotlib.pyplot as plt
import pyprobml_utils as pml

from mpl_toolkits.mplot3d import axes3d, Axes3D

np.random.seed(0)

import sklearn
from sklearn.linear_model import LogisticRegression
from sklearn import datasets

iris = datasets.load_iris()

X = iris["data"][:, 3:]  # petal width
y = (iris["target"] == 2).astype(np.int)  # 1 if Iris-Virginica, else 0

log_reg = LogisticRegression(solver="lbfgs",  fit_intercept=True, penalty='none')
log_reg.fit(X, y)

w_mle = log_reg.coef_[0][0] # 12.947270212450366
b_mle = log_reg.intercept_[0]  # -21.125250539711022
ypred = log_reg.predict_proba(X)

# Add column of 1s to end of X to capture bias term
N = X.shape[0]
ones = np.ones((N,1))
X1 = np.hstack((X, ones))

log_reg1 = LogisticRegression(solver="lbfgs", fit_intercept=False,  penalty='none')
log_reg1.fit(X1, y)

w_mle1 = log_reg1.coef_[0][0]
b_mle1 = log_reg1.coef_[0][1]
ypred1 = log_reg1.predict_proba(X1)

assert np.isclose(w_mle, w_mle1)
assert np.isclose(b_mle, b_mle1)
assert np.isclose(ypred[0], ypred1[0]).all()

# Define the logistic function
def logistic(z):
    return 1. / (1 + np.exp(-z))

# Define the prediction function y = 1 / (1 + numpy.exp(-x*w))
def predict_prob(x, w):
    return logistic(x.dot(w.T))

# Define the NLL loss function (y=probability, t=binary target)
def loss(y, t):
    return - np.mean(
        np.multiply(t, np.log(y)) + np.multiply((1-t), np.log(1-y)))

params =np.asmatrix([[w_mle, b_mle]])
ypred2 = predict_prob(X1,params)
#assert np.isclose(ypred1[:,1], ypred2).all()

# We compute the loss on a grid of (w, b) values.
# We use for loops for simplicity.
ngrid = 50
sf = 0.5
ws = np.linspace(-sf*w_mle, +sf*w_mle, ngrid)
bs = np.linspace(-sf*b_mle, +sf*b_mle, ngrid)
grid_w, grid_b = np.meshgrid(ws, bs)
loss_grid = np.zeros((ngrid, ngrid))
for i in range(ngrid):
    for j in range(ngrid):
        params = np.asmatrix([grid_w[i,j], grid_b[i,j]])
        p = predict_prob(X1, params)
        loss_grid[i,j] = loss(p, y)

# Plot the loss function surface
plt.figure()
plt.contourf(grid_w, grid_b, loss_grid, 20)
cbar = plt.colorbar()
cbar.ax.set_ylabel('NLL', fontsize=12)
plt.xlabel('$w$', fontsize=12)
plt.ylabel('$b$', fontsize=12)
plt.title('Loss function surface')
pml.savefig('logregIrisLossHeatmap.pdf')
plt.show()

fig,ax = plt.subplots()
CS = plt.contour(grid_w, grid_b, loss_grid,  cmap='jet')
#plt.plot(b_mle, w_mle, 'x') # Plot centered at MLE
pml.savefig('logregIrisLossContours.pdf')
plt.show()

fig = plt.figure()
ax = fig.add_subplot(111, projection='3d')
surf = ax.plot_surface(grid_w, grid_b, loss_grid)
pml.savefig('logregIrisLossSurf.pdf')
plt.show()


#########################
##                     ##
## Irving Gomez Mendez ##
##   March 23, 2021    ##
##                     ##
#########################

import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
import seaborn as sns
from scipy.stats import beta, t, f

import statsmodels.api as sm
from statsmodels.graphics.gofplots import qqplot
from statsmodels.stats.outliers_influence import OLSInfluence

class slr:
    def __init__(self, dat, n0=100, alpha=0.05):
        self.p = 2
        self.n = dat.shape[0]
        self.n0 = n0
        self.alpha = alpha
        self.y = dat['y']
        self.x = dat['x']
        self.design_matrix = np.column_stack([np.ones(self.n),dat['x']])

    def OLS(self):
        results = sm.OLS(self.y,self.design_matrix).fit()
        self.results = results
        self.s2 = results.scale
        self.params = results.params

    def summary(self):
        results = self.results
        print(results.summary2())

    def plot_regression(self):
        XX = self.design_matrix
        alpha = self.alpha
        n = self.n
        p = self.p

        x0 = np.linspace(self.x.min(),self.x.max(),self.n0)
        XX0 = np.column_stack([np.ones(self.n0),x0])
        y0 = self.results.predict(XX0)

        aux_t_conf = np.sqrt(self.s2*(np.diag(XX0 @ np.linalg.solve((XX.T @ XX), XX0.T))))
        aux_t_pred = np.sqrt(self.s2*(1+np.diag(XX0 @ np.linalg.solve((XX.T @ XX), XX0.T))))
        upp_conf = y0 + t.ppf(1-alpha/2,n-p)*aux_t_conf
        low_conf = y0 - t.ppf(1-alpha/2,n-p)*aux_t_conf
        upp_pred = y0 + t.ppf(1-alpha/2,n-p)*aux_t_pred
        low_pred = y0 - t.ppf(1-alpha/2,n-p)*aux_t_pred

        plt.figure(figsize=(10,5))
        plt.plot(x0, y0, 'r', label='Fitted line')
        plt.fill_between(x0, low_pred, upp_pred, facecolor='green', alpha=0.5, label='Prediction interval')
        plt.fill_between(x0, low_conf, upp_conf, facecolor='yellow', alpha=0.5, label='Confidence interval')
        plt.scatter(self.x, self.y)
        plt.title("Estimated Regression Fucntion")
        plt.legend(loc='upper left')

    def conf_region(self,b0, b1):
        bb = np.array([b0, b1], dtype='object')
        XX = self.design_matrix
        return(1/(self.p*self.s2)*((bb-self.params).T @ XX.T @ XX @ (bb-self.params)))

    def plot_confidence_region(self):
        p = self.p
        n = self.n
        n0 = self.n0
        alpha = self.alpha
        beta_interval = self.results.conf_int(alpha=alpha)
        beta_0_interval = beta_interval.loc['const']
        beta_1_interval = beta_interval.loc['x1']

        start_beta_0 = beta_0_interval[0]-(beta_0_interval[1]-beta_0_interval[0])/6
        stop_beta_0 = beta_0_interval[1]+(beta_0_interval[1]-beta_0_interval[0])/6

        start_beta_1 = beta_1_interval[0]-(beta_1_interval[1]-beta_1_interval[0])/6
        stop_beta_1 = beta_1_interval[1]+(beta_1_interval[1]-beta_1_interval[0])/6

        beta_0_vect = np.linspace(start_beta_0,stop_beta_0,n0)
        beta_1_vect = np.linspace(start_beta_1,stop_beta_1,n0)

        z = self.conf_region(beta_0_vect[:,None], beta_1_vect[None,:])

        plt.figure(figsize=(10,5))
        plt.plot([beta_0_interval[0],beta_0_interval[1],beta_0_interval[1],beta_0_interval[0],beta_0_interval[0]],
            [beta_1_interval[0],beta_1_interval[0],beta_1_interval[1],beta_1_interval[1],beta_1_interval[0]])
        plt.contour(beta_0_vect, beta_1_vect, z, levels=[f.ppf(1-alpha, p, n-p)])
        plt.xlabel('beta_0')
        plt.ylabel('beta_1')
        plt.title("Confidence Regiona and Intervals for beta")

    def residual_analysis(self):
        self.residual_analysis = OLSInfluence(self.results).summary_frame()

    def qq_t_interval(self, k):
        n = self.n
        df = self.n-self.p
        alpha = self.alpha

        a1 = beta.ppf(alpha/2, k, n+1-k)
        a2 = beta.ppf(1-alpha/2, k, n+1-k)

        low_int = t.ppf(a1, df)
        upp_int = t.ppf(a2, df)

        return(low_int, upp_int)

    def resid_qq_plot(self):
        n = self.n
        p = self.p
        studentized_residuals = self.residual_analysis['student_resid']

        kk = np.linspace(1,n,n)
        qq_low_intervals, qq_upp_intervals = self.qq_t_interval(kk)

        fig, ax = plt.subplots(figsize=(10,5))
        fig = qqplot(studentized_residuals, line='45', ax=ax)
        plt.plot(t.ppf(kk/(n+1),df=n-1-p), qq_low_intervals, 'r--')
        plt.plot(t.ppf(kk/(n+1),df=n-1-p), qq_upp_intervals, 'r--')
        plt.title("QQ plot for Studentized Residuals")

    def resid_density_plot(self):
        n = self.n
        p = self.p
        n0 = self.n0
        studentized_residuals = self.residual_analysis['student_resid']
        ll = np.linspace(studentized_residuals.min(),studentized_residuals.max(),n0)
        t_density = t.pdf(ll,df=n-p)

        plt.figure(figsize=(10,5))
        sns.histplot(studentized_residuals, stat='density', kde=True, label='kernel estimator')
        plt.plot(ll, t_density, 'r--', linewidth=2, label='t_density')
        plt.title("Histogram and t-Density for Studentized Residuals")
        plt.legend(loc='upper left')

    def fitted_vs_resid_plot(self):
        plt.figure(figsize=(10,5))
        plt.scatter(self.results.fittedvalues,self.results.resid)
        plt.xlabel('Fitted Values')
        plt.ylabel('Residuals')
        plt.title("Residuals vs Fitted Values")

    def resid_vs_previous_resid_plot(self):
        residuals = self.results.resid
        plt.figure(figsize=(10,5))
        plt.scatter(residuals[:-1],residuals[1:])
        plt.xlabel('Residuals_i')
        plt.ylabel('Residuals_i-1')
        plt.title("Residuals vs Previous Residuals")

    def out_resid_vs_resid_plot(self):
        residuals = self.results.resid
        h_ii = self.residual_analysis['hat_diag']
        out_residuals = residuals/(1-h_ii)

        plt.figure(figsize=(10,5))
        plt.scatter(residuals,out_residuals)
        plt.plot(residuals,residuals, 'r--')
        plt.xlabel('Residuals')
        plt.ylabel('Outer Residuals')
        plt.title("Residuals vs Outer Residuals")

    def main(self):
        self.OLS()
        self.summary()
        self.plot_regression()
        self.plot_confidence_region()
        self.residual_analysis()
        self.resid_density_plot()
        self.resid_qq_plot()
        self.fitted_vs_resid_plot()
        self.resid_vs_previous_resid_plot()
        self.out_resid_vs_resid_plot()

dat = pd.DataFrame({
    'x': [100, 89, 89, 92, 78, 101, 90, 82, 95, 108, 111, 112,125, 123, 100, 102, 112, 112, 100, 107, 103, 129, 118, 149],
    'y':[709, 740, 586, 806, 694, 760, 664, 509, 619, 652, 772, 896,742, 831, 667, 772, 838, 679, 889, 690, 698, 860, 799, 882]
    })
dat = dat/10
slr(dat).main()

import numpy,from scipy,import statsmodels,from statsmodels


import numpy as np
from tabulate import tabulate
import matplotlib.pyplot as plt
import os
THIS_FOLDER = os.path.dirname(os.path.abspath(__file__))
info_list = []

def InverseMethod(A, converge_range=0.0001, file_path="."):

    r, c = A.shape

    if r != c:
        raise Exception("not a square matrix")

    #testing to determine if the matrix is invertible
    sing = 0
    det_A = np.linalg.det(A)

    if det_A != sing:
        print("matrix is invertible")
    else:
        raise Exception("not an invertible matrix")

    #initialize eigenvectors
    vec_list = []
    lambda_list = []
    diff_list = []

    idx = 0
    vec_init = np.zeros(r)
    vec_init[-1] = 1
    vec_list.append(vec_init)

    #initialize eigenvalues
    lambda_init = vec_init.dot(A.dot(vec_init))
    lambda_list.append(lambda_init)

    diff_init = float("inf")
    diff_list.append(diff_init)

    while diff_list[idx] > converge_range:
        #solve the Aw = v^(k-1) for w=vec_new
        vec_new = np.linalg.solve(A, vec_list[idx])
        #normalize the newly found vec_new=w
        vec_new = vec_new / np.linalg.norm(vec_new)
        #add this to the vector list
        vec_list.append(vec_new)
        lambda_new = vec_new.dot(A.dot(vec_new))
        lambda_list.append(lambda_new)
        diff = np.abs(lambda_new - lambda_list[idx])
        diff_list.append(diff)
        idx = idx + 1

    print_log(idx, vec_list, lambda_list, diff_list, file_path)

    #plot lambda_list
    x = [i for i in range(idx+1)]
    if len(x) > 20:
        x = x[:21]
        diff_list = diff_list[:21]

    plt.plot(x, diff_list)
    plt.title("Inverse Iteration Lambda Difference")
    plt.xlabel('iterations')
    plt.ylabel('lambda difference')
    plt.savefig(file_path+'/II_difference_list_plot.png')
    plt.show()

    eigenvector = vec_list[-1]
    eigenvalue = lambda_list[-1]
    return eigenvector, eigenvalue

def print_log(idx, vec_list, lambda_list, diff_list, file_path):
    info_list = [[i, vec_list[i], lambda_list[i], diff_list[i]] for i in range(idx)]
    print(file_path)
    with open(file_path + '/Inverse_Iteration_performance.txt', 'w') as outputfile:
        outputfile.write(tabulate(info_list, headers=["iteration","eigenvector", "eigenvalue","lambda_diff"]))


import numpy as np
import random

def ridgeHelper(y, X, lambda_vals):
    '''ridge shrinkage via svd with support for multiple lambdas'''

    # svd: X = U * Sigma * VT
    U, d, V = np.linalg.svd(X, full_matrices = False)
    col = X.shape[1]
    k = len(lambda_vals)
    d = d.reshape(-1,1)
    # initialize array
    betas = np.empty((k,col))

    # loop through values of lambda
    for i in range(k):
      l_val = np.repeat(lambda_vals[i], len(d)).reshape(-1,1)
      # Dplus = np.diag(d/(d ** 2 + l_val))
      Dplus = np.diagflat(np.divide(d, np.add(np.power(d, 2), l_val)))
      # w =  V * Dplus * UT * Y
      betas[i,:] = V.T.dot(Dplus).dot(U.T).dot(y).reshape(1,-1)

    return betas

def mse(y, yhat):
    '''calculate mean squared error'''

    mean = np.mean((y - yhat) ** 2)

    return mean

def folds(X, k):
    '''create indices for k-fold cross validation'''

    # shuffle indices in X
    shuffled = list(range(X.shape[0]))
    random.shuffle(shuffled)

    # initialize test and train fold indices
    test_inds = [[] for i in range(k)]
    train_inds = test_inds.copy()

    # initialize counters, get n and fold-size
    i, j, h = 0, 0, 0
    n = X.shape[0]
    m = int(n/k)

    # partition test indices with uneven chunksize handling
    for ind in shuffled:
        test_inds[i].append(ind)
        j += 1
        if n - j > m*k:
            i -= 1
        elif not j % m:
            i += 1

    # save train indices as remaining rows in X for each fold (K - 1)
    for ind in test_inds:
        train_inds[h] = [g for g in shuffled if g not in test_inds[h]]
        h += 1

    return test_inds, train_inds

def crossval(y, X, lambda_vals = np.exp(np.linspace(-2,6,100)), k = 10):
    '''k-fold cross validation of ridge regression'''

    # create folds of rough size n/k
    tests, trains = folds(X, k)

    # initialize array of errors and loop counter
    errors = [[] for i in range(k)]
    i = 0

    # loop through testing and training sets
    for test, train in zip(tests, trains):

        betas = ridgeHelper(y[train], X[train, ], lambda_vals)

        # get mse from coefficients from each lambda
        for row in betas:

            yhat = X[test,] @ row.reshape(-1,1)
            err = mse(y[test], yhat)
            errors[i].append(err)

        i += 1

    means = [sum(col) / float(len(col)) for col in zip(*errors)]

    index = np.argmin(means)

    return lambda_vals[index]


# 1d linear regression.
# Make the data and plot it.

import autograd.numpy as np  # Thinly-wrapped numpy
#import numpy as np
import matplotlib.pyplot as plt
from utils.linreg_model import LinregModel

# Sample (x,y) pairs from a straight line or a sine wave (plus noise).
def make_data_linreg_1d(N=21, linear=True):
    xtrain = np.linspace(0, 20, N)
    sigma2 = 2
    w_true = np.array([-1.5, 1/9.])
    if linear:
        fun = lambda x: w_true[0] + w_true[1]*x
    else:
        fun = lambda x: w_true[0] + w_true[1]*np.sin(x)
    noise = np.random.normal(0, 1, xtrain.shape) * np.sqrt(sigma2)
    ytrain = fun(xtrain) + noise
    return xtrain, ytrain, w_true

# Plot 2d error surface around model's parameter vaulues
def plot_error_surface(xtrain, ytrain, model, ax=None):
    params = model.params
    if ax is None:
        fig = plt.figure()
        ax = fig.add_subplot(111)
    w0s = np.linspace(-2*params[0], 2*params[0], 10)
    w1s = np.linspace(-2*params[1], 2*params[1], 10)
    w0_grid, w1_grid = np.meshgrid(w0s, w1s)
    def loss(w0, w1):
        return model.objective([w0, w1], xtrain, ytrain)
    lossvec = np.vectorize(loss)
    z = lossvec(w0_grid, w1_grid)
    cs = ax.contour(w0s, w1s, z)
    ax.clabel(cs)
    ax.plot(params[0], params[1], 'rx', markersize=14)

# Function to plot the observed data and predictions
def plot_data_and_pred(x, y, model, draw_verticals=True):
    x_range = np.linspace(np.min(x), np.max(x), 100)
    yhat_range = model.predict(x_range)
    fig = plt.figure()
    ax = fig.add_subplot(111)
    ax.plot(x, y, 'o', label='observed')
    ax.plot(x_range, yhat_range, 'r-', label='predicted')
    if draw_verticals: # from observed value to predicted true
        yhat_sparse = model.predict(x)
        for x0, y0, yhat0 in zip(x, y, yhat_sparse):
            ax.plot([x0, x0],[y0, yhat0],'k-')
    plt.legend() #[line_pred, line_true], ['predicted', 'true'])

def main():
    np.random.seed(1)
    xtrain, ytrain, w_true = make_data_linreg_1d()
    model = LinregModel(w_true)
    plot_data_and_pred(xtrain, ytrain, model)
    plot_error_surface(xtrain, ytrain, model)
    plt.show()

if __name__ == "__main__":
    main()


# ------------- Machine Learning - Topic 3: Multi-class Classification and Neural Networks

import numpy as np
from scipy.io import loadmat
import os, sys
sys.path.append(os.getcwd() + os.path.dirname('/ml/ex3/'))
from helpers import displayData, oneVsAll, predictOneVsAll

#  This file contains code that helps you get started on the
#  exercise. You will need to complete the following functions
#  in this exercise:
#
#     displayData
#     lrCostFunction (logistic regression cost function)
#     oneVsAll
#     predictOneVsAll
#     predict

## Setup the parameters you will use for this part of the exercise
input_layer_size  = 400  # 20x20 Input Images of Digits
num_labels = 10          # 10 labels, from 1 to 10
                          # (note that we have mapped "0" to label 10)

## =========== Part 1: Loading and Visualizing Data =============
#  We start the exercise by first loading and visualizing the dataset.
#  You will be working with a dataset that contains handwritten digits.
#

# Load Training Data
print('Loading and Visualizing Data ...')

mat = loadmat("ml/ex3/ex3data1.mat")

X = mat["X"]
y = mat["y"]

m = X.shape[0]

# crucial step in getting good performance!
# changes the dimension from (m,1) to (m,)
# otherwise the minimization isn't very effective...
y = y.flatten()

# Randomly select 100 data points to display
rand_indices = np.random.permutation(m)
sel = X[rand_indices[:100],:]

displayData(X=sel)

input('Program paused. Press enter to continue.\n')

## ============ Part 2: Vectorize Logistic Regression ============
#  In this part of the exercise, you will reuse your logistic regression
#  code from the last exercise. You task here is to make sure that your
#  regularized logistic regression implementation is vectorized. After
#  that, you will implement one-vs-all classification for the handwritten
#  digit dataset.
#

print('Training One-vs-All Logistic Regression...')

lambda_reg = 0.1
all_theta = oneVsAll(X, y, num_labels, lambda_reg)

input('Program paused. Press enter to continue.\n')

## ================ Part 3: Predict for One-Vs-All ================
#  After ...
pred = predictOneVsAll(all_theta, X)

print('Training Set Accuracy: {:f}'.format((np.mean(pred == y%10)*100)))
print('Training Set Accuracy for 1:  {:f}'.format(np.mean(pred[500:1000]  == y.flatten()[500:1000]%10)  * 100))
print('Training Set Accuracy for 2:  {:f}'.format(np.mean(pred[1000:1500] == y.flatten()[1000:1500]%10) * 100))
print('Training Set Accuracy for 3:  {:f}'.format(np.mean(pred[1500:2000] == y.flatten()[1500:2000]%10) * 100))
print('Training Set Accuracy for 4:  {:f}'.format(np.mean(pred[2000:2500] == y.flatten()[2000:2500]%10) * 100))
print('Training Set Accuracy for 5:  {:f}'.format(np.mean(pred[2500:3000] == y.flatten()[2500:3000]%10) * 100))
print('Training Set Accuracy for 6:  {:f}'.format(np.mean(pred[3000:3500] == y.flatten()[3000:3500]%10) * 100))
print('Training Set Accuracy for 7:  {:f}'.format(np.mean(pred[3500:4000] == y.flatten()[3500:4000]%10) * 100))
print('Training Set Accuracy for 8:  {:f}'.format(np.mean(pred[4000:4500] == y.flatten()[4000:4500]%10) * 100))
print('Training Set Accuracy for 9:  {:f}'.format(np.mean(pred[4500:5000] == y.flatten()[4500:5000]%10) * 100))
print('Training Set Accuracy for 10: {:f}'.format(np.mean(pred[0:500]     == y.flatten()[0:500]%10)     * 100))


"""
Created on May 12 22:49:49 2022
"""

import numpy as np

import error_utilities as e_utils

def translation_in_coordinates(change_in_coordinates, starting_point=(0, 0, 0),
                               deg_rad: str = 'rad'):
    """
    Calculates new coordinates of the point in spherical system given the translation
    in the object's coordinates.

    Parameters
    ----------
    change_in_coordinates:
        Change in the coordinates of the object given in a list in <r, theta, phi> order
    starting_point:
        Starting coordinates of the point in spherical coordinates.
    deg_rad:
        Whether the specified theta and phi arguments are in degree or radians

    Returns
    ----------
    float:
        New coordinates for the point in spherical coordinate system
    """

    # make sure the sum of each individual argument is > 0
    e_utils.ensure_positive([starting_point, change_in_coordinates])

    # the starting_point is taken in radians
    r1, theta1, phi1 = starting_point
    r2, theta2, phi2 = change_in_coordinates

    # convert theta and phi to radians if not already
    theta1, phi1 = np.radians([theta1, phi1]) if deg_rad == 'deg' else theta2, phi1
    theta2, phi2 = np.radians([theta2, phi2]) if deg_rad == 'deg' else theta2, phi2

    # make sure that theta does not exceed 360
    _theta1, _theta2 = e_utils.ensure_theta([theta1, theta2])

    # make sure that phi does not exceed 180
    _phi1, _phi2 = e_utils.ensure_phi([phi1, phi2])

    _changed = [sum(x) for x in zip((r1, theta1, phi1), (r2, theta2, phi2))]

    print(f'The starting coordinates of the point were {r1}, {np.degrees(_theta1)}, '
          f'{np.degrees(_phi1)} in degrees.\n'
          f'The new coordinates of the point are {_changed[0]}, '
          f'{np.degrees(_changed[1])}, {np.degrees(_changed[2])}')

    return _changed


"""Activation functions used in Deep Learning."""

import numpy as np

def softmax(x, derivative=False):
    """Softmax activation function.

    Activation function often used in the final layer. The probabilities will add up to 1.

    args:
        x (list): Vector with input values.
        derivative (bool): Whether the derivative should be returned instead. Defaults to False.
    """

    exps = np.exp(x - x.max())
    temp = exps / np.sum(exps, axis=0)
    if derivative:
        return temp * (1 - temp)
    return temp

def identity(x, derivative=False):
    """Identity activation function.

    Args:
        x (list): Vector with input values.
        derivative (bool, optional): Whether the derivative should be returned instead. Defaults to False.
    """

    if derivative: return np.full(x.shape, 1)
    else: return x

def sigmoid(x, derivative=False):
    """Non vectorized sigmoid activation function.

    The values are treated separately and will have a value between 0 and 1.

    Args:
        x (list): Vector with input values.
        derivative (bool, optional): Whether the derivative should be returned instead. Defaults to False.
    """
    if derivative:
        return (np.exp(-x))/((np.exp(-x)+1)**2)
    return 1/(1+np.exp(-x))

# _relu_not_vect
def _relu_not_vect(x, derivative=False):
    """Non vectorized ReLU activation function.

    Args:
        x (list): Vector with input values.
        derivative (bool, optional): Whether the derivative should be returned instead. Defaults to False.
    """

    if x > 0:
        if derivative:
            return 1
        else: return x
    else:
        return 0

def _leaky_relu_not_vect(x, derivative=False):
    """Non vectorized LeakyReLU activation function.

    Args:
        x (list): Vector with input values.
        derivative (bool, optional): Whether the derivative should be returned instead. Defaults to False.
    """
    a = 0.02
    if x > 0:
        if derivative:
            return 1
        else: return x
    else:
        if derivative:
            return a
        else: return a*x

# Vectorize activation functions
__relu = np.vectorize(_relu_not_vect)
__leaky_relu = np.vectorize(_leaky_relu_not_vect)
# __sigmoid = np.vectorize(_sigmoid_not_vect)

def relu(x, derivative=False):
    """Vectorized ReLU activation function.

    Args:
        x (list): Vector with input values.
        derivative (bool, optional): Whether the derivative should be returned instead. Defaults to False.
    """

    return __relu(x, derivative=derivative)

def leaky_relu(x, derivative=False):
    """Vectorized ReLU activation function.

    Args:
        x (list): Vector with input values.
        derivative (bool, optional): Whether the derivative should be returned instead. Defaults to False.
    """

    return __leaky_relu(x, derivative=derivative)


# sine function
if '__file__' in globals():
    import os, sys
    sys.path.append(os.path.join(os.path.dirname(__file__), '..'))

import numpy as np
from dezero import Variable, Function

class Sin(Function):
    def forward(self, x):
        y = np.sin(x)
        return y

    def backward(self, gy):
        x = self.inputs[0].data
        gx =  gy * np.cos(x)
        return gx

def sin(x):
    f = Sin()
    return f(x)

def mysin(x, threshold=0.0001):
    import math
    y = 0
    for i in range(100000):
        c = (-1) ** i / math.factorial(2 * i + 1)
        t = c * x ** (2 * i + 1)
        y = y + t
        if abs(t.data) < threshold:
            break
    return y

if __name__ == '__main__':
    x = Variable(np.array(np.pi/4))
    y = sin(x)
    y.backward()
    print(y.data)
    print(x.grad)

    x = Variable(np.array(np.pi/4))
    y = mysin(x)
    y.backward()
    print(y.data)
    print(x.grad)


from numpy import array, linalg

'''
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~ Taking user input ~~~~~~~~~~~~~~~~~~~~~~~~~~~
N = int(input("Enter the Number of Equations: "))

col = []
colB = []
for i in range(N):
    row = []
    for j in range(N):
        coeff = float(input("Enter Coefficient "+str(j+1)+\
                            " of Equation "+str(i+1)+": "))
        row.append(coeff)
    col.append(row)

    const = float(input("Enter the Constant of Equation "+str(i+1)+": "))
    colB.append(const)

A = array(col)
B = array(colB)
print()
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
'''
# System of Equations

A = array([[.3, .52, 1],
           [.5, 1, 1.9],
           [.1, .3, .5]], float)
B = array([-.01, .67, -.44], float)

N = len(B)

coeffMatrix = linalg.det(A)         # [D]

# Coincident or Parallel Equations
if abs(coeffMatrix) < 1e-12:        # Singuar Matrix, ZeroDivisionError
    print("Two or More Equations are Coincident or Parallel")
    raise SystemExit

detMatrix = []
for i in range(N):

    C = A.copy()                    # Making a copy to keep A unchanged
    C [:, i] = B                    # Replace i th coloumn of C with B

    Dx = linalg.det(C)              # |Dx|, |Dy|, |Dz|
    detMatrix.append(Dx)            # [Dx, Dy, Dz]

xMatrix = detMatrix / coeffMatrix   # [Dx/D, Dy/D, Dz/D]

print("The Solution of the System:")
for i in range(N):
    print('X[', i+1, '] = ', round(xMatrix[i], 6), sep='')

5. Systems of Linear Equations/1. a. Cramer's Rule using linalg det.py


# 27
# isPrime() takes reference from https://stackoverflow.com/a/15285588/8828580

import numpy as np

def isPrime(n):
  if n == 2 or n == 3: return True
  if abs(n) < 2 or n % 2 == 0 or n % 3 == 0: return False

  r = int(n ** 0.5)
  f = 5

  while f <= r:
    if n % f == 0: return False
    if n % (f+2) == 0: return False
    f += 6
  return True

def getPrimeList(n):
    p = [1] * n
    p[0] = 0
    p[1] = 0

    for val in range(1, n):
        if p[val] == 0: continue
        for i in range(2 * val, n, val): p[i] = 0

    return list(val for val in range(n) if p[val] == 1)

recLen = 0
recNums = [0, 0]

pList = getPrimeList(1000)
pList = list(np.negative(pList)) + pList

for a in range(-999, 1000):
    for b in pList:
        if abs(b) < recLen: continue
        n = 1
        while isPrime(abs(n**2 + a * n + b)):
            if n > recLen:
                recLen = n
                recNums = [a, b]
            n += 1

print(recLen, recNums)
print(recNums[0] * recNums[1])


import numpy as np
from chroma.transform import rotate

def uniform_sphere(size=None, dtype=np.double):
    """
    Generate random points isotropically distributed across the unit sphere.

    Args:
        - size: int, *optional*
            Number of points to generate. If no size is specified, a single
            point is returned.

    Source: Weisstein, Eric W. "Sphere Point Picking." Mathworld.
    """

    theta, u = np.random.uniform(0.0, 2*np.pi, size), \
        np.random.uniform(-1.0, 1.0, size)

    c = np.sqrt(1-u**2)

    if size is None:
        return np.array([c*np.cos(theta), c*np.sin(theta), u])

    points = np.empty((size, 3), dtype)

    points[:,0] = c*np.cos(theta)
    points[:,1] = c*np.sin(theta)
    points[:,2] = u

    return points

def flashlight(phi=np.pi/4, direction=(0,0,1), size=None, dtype=np.double):
    theta, u = np.random.uniform(0.0, 2*np.pi, size), \
        np.random.uniform(np.cos(phi), 1, size)

    c = np.sqrt(1-u**2)

    if np.equal(direction, (0,0,1)).all():
        rotation_axis = (0,0,1)
        rotation_angle = 0.0
    else:
        rotation_axis = np.cross((0,0,1), direction)
        rotation_angle = \
            -np.arccos(np.dot(direction, (0,0,1))/np.linalg.norm(direction))

    if size is None:
        return rotate(np.array([c*np.cos(theta), c*np.sin(theta), u]),
                      rotation_angle, rotation_axis)

    points = np.empty((size, 3), dtype)

    points[:,0] = c*np.cos(theta)
    points[:,1] = c*np.sin(theta)
    points[:,2] = u

    return rotate(points, rotation_angle, rotation_axis)