import numpy as np
import matplotlib.pyplot as plt

def lag(x, y, num):
    """
    'num' is a set of given points, use x and y to figure out lagrange
    polynomial and calculate all the corresponding values for num

    Implement as you wish but your 'total' numpy array
    has to return all the results
    """

    assert(len(x)==len(y))
    total  = np.zeros(len(num))

    #place your code here!!!!!!!!!!!!!!!!!!!!!!!!!
    for i in range(len(num)):
        total[i] = lagPol(num[i], x, y)

    return total

def lagPol(xi, x, y):
    res = 0
    for i in range(len(x)):
        res += y[i]*lagCoeff(xi, i, x)
    return res

def lagCoeff(xi, i, x):
    res = 1
    for j in range(len(x)):
        if(j != i):
            res *= (xi-x[j])/(x[i]-x[j])
    return res

data_x = np.array([-3.,-2.,-1.,0.,1.,3.,4.])
data_y = np.array([-60.,-80.,6.,1.,45.,30.,16.])
# data_x = np.array([10.,15.,20.,22.5])
# data_y = np.array([227.4,362.78,517.35,602.97])

#generating 50 points from -3 to 4 in order to create a smooth line
X = np.linspace(-3, 4, 50, endpoint=True)
F = lag(data_x, data_y, X)
print(F)
plt.plot(X,F)
plt.plot(data_x, data_y, 'ro')
plt.show()


from numpy import array, sqrt, zeros, set_printoptions

def cholesky(A,b):
    # Lower triangular matrix
    el = zeros((3,3))
    el[0,0] = sqrt(A[0,0])
    el[1,0] = A[1,0] / el[0,0]
    el[1,1] = sqrt(A[1,1] - el[1,0]**2)
    el[2,0] = A[2,0] / el[0,0]
    el[2,1] = (A[2,1] - el[2,0] * el[1,0])/el[1,1]
    el[2,2] = sqrt(A[2,2] - el[2,0]**2 - el[2,1]**2)
    # Forward substitution for y
    y = zeros((3))
    y[0] = b[0] / el[0,0]
    y[1] = (b[1] - el[1,0] * y[0]) / el[1,1]
    y[2] = (b[2] - el[2,0] * y[0] - el[2,1] * y[1]) / el[2,2]
    # Backward substitution for x
    x = zeros((3))
    elT = el.T
    x[2] = y[2] / elT[2,2]
    x[1] = (y[1] - elT[1,2] * x[2]) / elT[1,1]
    x[0] = (y[0] - elT[0,1] * x[1] - elT[0,2] * x[2]) / elT[0,0]
    return el, y, x

# Input parameters to choesky(A,b) function
A = array([[5,-1,1],[-1,3,-1],[1,-1,4]])
b = array([6,2,11])
el, y, xc = cholesky(A,b)

set_printoptions(precision = 4)
print("L = ", el)
print("y = ", y)
print("x1 = %8.4f " % xc[0])
print("x2 = %8.4f " % xc[1])
print("x3 = %8.4f " % xc[2])


import numpy as np
import time

#----- normal distribution

def model_prob(_x, _theta):
    #-- parameters
    _mu = _theta[0]
    _sigma_sq = _theta[1]
    #-- probability
    _p = np.exp(-(_x - _mu) ** 2 / (2 * _sigma_sq)) / np.sqrt(2 * np.pi * _sigma_sq)
    #print(_p)
    return _p

def max_likelihood_est(_x):
    _mu = np.mean(_x)
    _sigma_sq = np.var(_x)
    _theta = [_mu, _sigma_sq]
    #print(_theta)
    return _theta

#-----

def likelihood(_x, _theta):
    _l = np.sum(np.log(model_prob(_x, _theta)))
    #print(_l)
    return _l

def bootstrap_sample(_x):
    _n = _x.size
    _ids = np.random.randint(0, _n, _n)
    #print(_ids)
    _x_ast = _x[_ids]
    #print(_x_ast)
    return _x_ast

def EIC_bias(_x, _B):
    _D_ast = np.zeros(_B)
    for i in range(_B):
        _x_ast = bootstrap_sample(_x)
        _theta_ast = max_likelihood_est(_x_ast)
        _D_ast[i] = likelihood(_x_ast, _theta_ast) - likelihood(_x, _theta_ast)
        #print(_D_ast[i])
    #print(_D_ast)

    _b_b = np.mean(_D_ast)
    #print(_b_b)
    return _b_b

def EIC_bias2(_x, _B):
    _D_ast = np.zeros(_B)
    for i in range(_B):
        _x_ast = bootstrap_sample(_x)
        _theta_ast = max_likelihood_est(_x_ast)
        _eic = EIC_bias(_x_ast, _B)
        _D_ast[i] = likelihood(_x_ast, _theta_ast) - _eic - likelihood(_x, _theta_ast)
    _b_2nd = np.mean(_D_ast)
    return _b_2nd

#-----

if __name__ == '__main__':

    T = 100 #10000

    n = 25 #25, 100, 400
    B = 100 #1000

    t_eic = np.zeros(T)
    t_eic_2nd = np.zeros(T)

    prv_ut = time.time()
    for t in range(T):
        ut = time.time()
        if ut - prv_ut > 5.0:
            prv_ut = ut
            print("---", t)

        #-- samples from true distribution
        x = np.random.normal(0.0, 1.0, n)
        #print(x)

        t_eic[t] = EIC_bias(x, B)
        t_eic_2nd[t] = EIC_bias2(x, B) + t_eic[t]
        #print(t_eic[t])
        #print(t_eic_2nd[t])

    print("mean EIC bias:", np.mean(t_eic))
    print("mean EIC_2nd bias:", np.mean(t_eic_2nd))


#!/usr/bin/env python
# coding: utf-8

# In[1]:

import math
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns

# In[2]:

automobile_data = pd.read_csv('datasets/automobile_data_processed.csv')

automobile_data.sample(5)

# In[3]:

automobile_data.shape

# In[4]:

automobile_data.boxplot('mpg', figsize=(12, 8))

plt.title('Miles per gallon')

# In[5]:

automobile_data['mpg'].plot.kde(figsize=(12, 8))

plt.xlabel('mpg')

plt.title('Density plot for MPG')
plt.show()

# In[6]:

plt.figure(figsize=(12, 8))

sns.boxplot(x='cylinders', y='mpg', data=automobile_data)

plt.show()

# In[7]:

plt.figure(figsize=(12, 8))

sns.violinplot(x='cylinders', y='mpg', data=automobile_data, inner=None)
sns.swarmplot(x='cylinders', y='mpg', data=automobile_data, color='w')

plt.show()

# In[8]:

cylinder_stats = automobile_data.groupby(['cylinders'])['mpg'].agg(['mean', 'count', 'std'])

cylinder_stats

# In[9]:

ci95_high = []

ci95_low = []

# In[10]:

for i in cylinder_stats.index:

    mean, count, std = cylinder_stats.loc[i]

    ci95_high.append(mean + 1.96 * (std / math.sqrt(count)))
    ci95_low.append(mean - 1.96 * (std / math.sqrt(count)))

# In[11]:

cylinder_stats['ci95_HIGH'] = ci95_high
cylinder_stats['ci95_LOW'] = ci95_low

cylinder_stats

# In[12]:

cylinders = 4

cylinders4_df = automobile_data.loc[automobile_data['cylinders'] == cylinders]

cylinders4_df.sample(10)

# In[13]:

plt.figure(figsize=(12, 8))

sns.distplot(cylinders4_df['mpg'], rug=True, kde=True, hist=False)

plt.show()

# In[14]:

plt.figure(figsize=(12, 8))

sns.distplot(cylinders4_df['mpg'], rug=True, kde=True, hist=False)

plt.stem([cylinder_stats.loc[cylinders]['mean']],
         [0.07], linefmt = 'C1',
         markerfmt = 'C1', label = 'mean')

plt.stem([cylinder_stats.loc[cylinders]['ci95_LOW']],
         [0.07], linefmt = 'C2',
         markerfmt = 'C2', label = '95% CI High')

plt.stem([cylinder_stats.loc[cylinders]['ci95_HIGH']],
         [0.07], linefmt = 'C3',
         markerfmt = 'C3', label = '95% CI Low')

plt.xlabel('mpg')
plt.legend()
plt.show()

# In[15]:

cylinders = 6

cylinders6_df = automobile_data.loc[automobile_data['cylinders'] == cylinders]

cylinders6_df.sample(10)

# In[16]:

plt.figure(figsize=(12, 8))

sns.distplot(cylinders6_df['mpg'], rug=True, kde=True, hist=False)

plt.show()

# In[17]:

plt.figure(figsize=(12, 8))

sns.distplot(cylinders6_df['mpg'], rug=True, kde=True, hist=False)

plt.stem([cylinder_stats.loc[cylinders]['mean']],
         [0.16], linefmt = 'C1',
         markerfmt = 'C1', label = 'mean')

plt.stem([cylinder_stats.loc[cylinders]['ci95_LOW']],
         [0.16], linefmt = 'C2',
         markerfmt = 'C2', label = '95% CI High')

plt.stem([cylinder_stats.loc[cylinders]['ci95_HIGH']],
         [0.16], linefmt = 'C3',
         markerfmt = 'C3', label = '95% CI Low')

plt.xlabel('mpg')
plt.legend()
plt.show()

# In[ ]:

# In[ ]:

# In[ ]:

# In[ ]:

# In[ ]:

# In[ ]:

# In[ ]:

# In[ ]:

# In[ ]:

# In[ ]:

# In[ ]:

# In[ ]:

# In[ ]:

# In[ ]:

# In[ ]:

# In[ ]:

# In[ ]:

# In[ ]:

# In[ ]:

# In[ ]:

# In[ ]:

# In[ ]:

# In[ ]:

# In[ ]:

# In[ ]:

# In[ ]:

# In[ ]:

# In[ ]:

# In[ ]:

# In[ ]:

# In[ ]:

# In[ ]:

# In[ ]:

# In[ ]:

# In[ ]:

# In[ ]:

# In[ ]:

# In[ ]:

# In[ ]:

# In[ ]:

# In[ ]:


import numpy as np

l1 = np.array([4, 1, 9, 3])
l2 = np.array([1, 6, 3, 0])

#-------------------------------------------------------------

# Rounding Decimals
# There are primarily five ways of rounding off decimals in NumPy:

# # truncation
# arr = np.trunc([76.6754, -13.435])
# print(arr)

# # fix
# arr = np.fix([76.6754, -13.435])
# print(arr)

# # rounding
# arr = np.around(76.6754)
# print(arr)

# arr = np.around(76.6754, 2)
# print(arr)

# # floor
# arr = np.floor([76.265, -13.435])
# print(arr)

# # ceil
# arr = np.ceil([76.265, -13.435])
# print(arr)

#-------------------------------------------------------------
# LOGS

arr = np.arange(1, 10)
print(arr)
print(np.log2(arr))
print(np.log10(arr))
print(np.log(arr))  # log to the base e
#-------------------------------------------------------------

# Addition is done between two arguments whereas summation happens over n elements.
# Summations
a1 = np.sum([l1,l2])
print(a1)

# NumPy will sum the numbers in each array.
a2 = np.sum([l1,l2], axis = 1)
print(a2)

# cummulative summation
a3 = np.cumsum(l1)
print(a3)

# Products
a1 = np.prod([l1,l2])
print(a1)

# NumPy will return the product of each array.
a2 = np.prod([l1,l2], axis = 1)
print(a2)

# cummulative product
a3 = np.cumprod(l1)
print(a3)

#Difference

# subtracting two successive elements.
a1 = np.diff(l1)
print(a1)

# Compute discrete difference of the following array twice
a2 = np.diff(l1, n=2)
print(a2)

# Set Operations

a1 = np.array([2,3,5,2,1,2,3,1,5,5,3])

b = np.unique(a1)
print(b)

a2 = np.union1d(l1,l2)
print(a2)

a3 = np.intersect1d(l1,l2)
print(a3)

a4= np.setdiff1d(l1,l2)
a5= np.setdiff1d(l2,l1)
print(a5)
print(a4)

# LCM
a1 = np.lcm.reduce(l1)
print(a1)
arr = np.array([[3, 6, 9], [2,3,5]])
a2 = np.lcm.reduce(arr)
print(a2)

# GCD
a1 = np.gcd.reduce(l1)
print(a1)
arr = np.array([[3, 6, 9], [2,3,5]])
a2 = np.gcd.reduce(arr)
print(a2)


from statistics import mean
import numpy as np
import matplotlib.pyplot as plt
from matplotlib import style
import random

style.use('ggplot')

# Algoritmo para calcular regresion lineal
###########################
#     _   _   __
#     x . y - xy
# m = ------------------
#             __
#      _ 2     2
#     (x)   - x
###########################
#     _   __
# b = y - mx
#
###########################
#                 ^
#  2           SE y
# r  = 1 - --------------
#                 _
#              SE y
###########################

xs = np.array([1, 2, 3, 4, 5, 6], dtype=np.float64)
ys = np.array([4, 5, 6, 5, 6, 7], dtype=np.float64)

def create_dataset(hm, variance, step=2, correlation=False):
    val = 1
    ys = []
    for i in range(hm):
        y = val + random.randrange(-variance, variance)
        ys.append(y)
        if correlation and correlation == 'pos':
            val += step
        elif correlation and correlation == 'neg':
            val -= step
    xs = [i for i in range(len(ys))]

    return np.array(xs, dtype=np.float64), np.array(ys, dtype=np.float64)

def squared_error(ys_orig, ys_line):
    return sum((ys_line - ys_orig)**2)

def coeffiecient_of_determination(ys_orig, ys_line):
    y_mean_line = [mean(ys_orig) for y in ys_orig]
    squared_error_regr = squared_error(ys_orig, ys_line)
    squared_error_y_meean = squared_error(ys_orig, y_mean_line)
    return 1 - (squared_error_regr / squared_error_y_meean)

def best_fit_slope_and_intercept(xy, ys):
    m = (((mean(xs) * mean(ys)) - mean(xs * ys)) /
         ((mean(xs) * mean(xs)) - mean(xs * xs)))
    b = mean(ys) - (m * mean(xs))

    return m, b

xs, ys = create_dataset(40, 10, 2, correlation=False)

m, b = best_fit_slope_and_intercept(xs, ys)

regression_line = [(m * x) + b for x in xs]

predict_x = 8
predict_y = (m * predict_x) + b

r_squared = coeffiecient_of_determination(ys, regression_line)
print(r_squared)

plt.scatter(xs, ys)
plt.plot(xs, regression_line)
plt.scatter(predict_x, predict_y, s=100, color='g')
plt.show()
# plt.scatter(xs, ys)
# plt.show()


import numpy as np
import matplotlib.pyplot as plt
from numpy.random import randn
import seaborn as sns
sns.set()

from linear_classification.basic_logistic_unit import sigmoid
from e_commerce_example.logistic_training import cross_entropy

def generate_donut_data(number_of_points, dimensions, inner_radius, outer_radius):
    # The distance from origin = R + random normal
    # The theta angle =  Uniformly distributed from (0, 2*pi)

    R1 = np.random.randn(number_of_points//2) + inner_radius
    R2 = np.random.randn(number_of_points//2) + outer_radius

    theta1 = 2 * np.pi * np.random.random(number_of_points//2)
    theta2 = 2 * np.pi * np.random.random(number_of_points//2)

    X_inner = np.concatenate( [ [R1 * np.cos(theta1)], [R1 * np.sin(theta1)] ] ).T
    X_outer = np.concatenate( [ [R2 * np.cos(theta2)], [R2 * np.sin(theta2)] ] ).T

    X = np.concatenate([ X_inner, X_outer ])
    y = np.array([0]*(number_of_points//2) + [1]*(number_of_points//2))

    plt.figure()
    plt.scatter(X[:,0], X[:,1], c=y)
    plt.show()

    return X, y

def gradient_descent_l2_donut_problem(X, y, learning_rate=0.0001, epochs=5000, l2_norm=0.1):
    w = np.random.randn(X.shape[1])
    z = X.dot(w)
    y_hat = sigmoid(z)

    errors = []
    for epoch in range(epochs):
        error = cross_entropy(y, y_hat)
        errors.append(error)
        if epoch % 100 == 0:
            print("Epoch {}:{}".format(epoch, error))
        w += learning_rate * ( X.T.dot(y - y_hat) - l2_norm * w )

        y_hat = sigmoid(X.dot(w))

    plt.figure()
    plt.plot(errors)
    plt.title("Cross-entropy Loss/Iteration")
    plt.show()

    print("Final w:", w)
    print("Final classification rate:", 1 - np.abs(y - np.round(y_hat)).sum() / X.shape[0])

    return w

if __name__ == '__main__':
    number_of_points = 1000
    dimensions = 2
    inner_radius = 5
    outer_radius = 10
    learning_rate=0.0001
    epochs=5000
    l2_norm = 0.1

    X, y = generate_donut_data(number_of_points, dimensions, inner_radius, outer_radius)

    # Generate the ones + an additional column of r = sqrt(x^2 + y^2)
    # This is the radious of a point, and makes data points linearly separable

    ones = np.ones((number_of_points, 1))
    radiuses = np.sqrt( (X*X).sum(axis=1) ).reshape(-1,1)
    X = np.concatenate((ones, radiuses, X), axis=1)

    _ = gradient_descent_l2_donut_problem(X, y, learning_rate, epochs, l2_norm)

    # Classification does not appear to depend on the X and y term, but on the bias and the summed radiuses


import numpy as np

def swap_row(mat,r1,r2):
	temp = mat[r1]
	mat[r1] = mat[r2]
	mat[r2] = temp

def normalize(mat,i):
	for j in range(i+1,len(mat)):
		fact = (mat[j][i]/mat[i][i])
		for k in range(0,len(mat)+1):
			mat[j][k] -= fact*mat[i][k]

def fwd_elimination(mat):
	n = len(mat)
	for i in range(n-1):
		if mat[i][i] == 0:
			swap_row(mat,i,i+1)
		normalize(mat,i)
		m = np.array(mat)
		m = m[:,:n]
		if (np.linalg.det(m) == 0):
			print ("Singular Matrix!! No Unique solution Exists!!")
			return False
	return True

def bwd_elimination(mat):
	n = len(mat)
	x = [0]*n

	for i in range(n-1,-1,-1):
		x[i] = mat[i][n]

		for j in range(i+1,n):
			x[i] -= mat[i][j]*x[j]

		x[i] = x[i]/mat[i][i]

	print("Solutions are: \n\n")
	print("y(0.25): ",x[0],"\n\n")
	print("y(0.5): ",x[1],"\n\n")
	print("y(0.75): ",x[2],"\n\n")

def func(x):
	return np.exp(x**2)

def main():
	mat = []
	y0 = 0; y4 = 0;
	h = 0.25;
	x = [0.25,0.5,0.75]
	f = []
	for i in range(3):
		temp = func(x[i])*(h**2)
		f.append(temp)

	f[0] -= y0;
	f[2] -= y4;

	mat = [[-2,1,0,f[0]],[1,-2,1,f[1]],[0,1,-2,f[2]]]

	if(fwd_elimination(mat) == True):
		bwd_elimination(mat)

if __name__ == '__main__':
	main()


# -*- coding: utf-8 -*-
"""
Let d(n) be defined as the sum of proper divisors of n (numbers less than n which divide evenly into n).
If d(a) = b and d(b) = a, where a ≠ b, then a and b are an amicable pair and each of a and b are called amicable numbers.

For example, the proper divisors of 220 are 1, 2, 4, 5, 10, 11, 20, 22, 44, 55 and 110; therefore d(220) = 284. The proper divisors of 284 are 1, 2, 4, 71 and 142; so d(284) = 220.

Evaluate the sum of all the amicable numbers under 10000.

Solution commment:
I evaluate d(n) by first prime factorizing n, taking all the possible partitions of
the primes and summing the product of them. Hoping this is faster for large n because
prime factorization only needs to look at sqrt(n) numbers. Potential bottleneck is
the partitioning of the primes, a recursive method which is probably not very fast for
large numbers of primes. Find solution for 10 000 in under 2 sec, 100 000 in under 30 sec.
"""
from numpy import prod
from itertools import groupby

def partitions(arr):
    """Return a list of all the partitions of arr"""
    if len(arr) == 1:
        return [arr]
    without = partitions(arr[:-1])
    incl = [part + [arr[-1]] for part in without]
    return without + incl + [[arr[-1]]]

def prodComplex(listoflists):
    return [prod(arr) for arr in listoflists]

def primes(n):
    primefac = []
    d = 2
    while d*d <= n:
        while n % d == 0:
            primefac.append(d)
            n /= d
        d += 1
    if n > 1:
        primefac.append(n)
    return primefac

def unduplicate(listoflists):
    listoflists.sort()
    return list(k for k, _ in groupby(listoflists))

def d(n):
    if n <= 1:
        return 0
    primefac = primes(n)
    primeparts = unduplicate(partitions(primefac))
    primeprod = prodComplex(primeparts)
    return sum(primeprod) - n + 1

s = 0
seen = set()
num = 0
while num < 10000:
    num += 1
    if num in seen:
        continue

    amicable = d(num)
    if d(amicable) == num and num != amicable:
        print "Found pair:", num, amicable
        seen.add(num)
        seen.add(amicable)
        s += amicable + num
print 'sum of amicable numbers =', s


#########################################################################################
# This is a class that collects data science "distances methods".
# Distances measure similarities between features (both numeric and categorical).
# Distances are used in a lot of fields: in Machine Learning inside methods (such as the
# k-Means or the cosine similarity used inside the colloborative filtering tecnique).
#########################################################################################

import numpy as np

class Distances(object):

	def __init__(self):
		super().__init__()

	def Euclidean(self, points_1:list, points_2:list):

		'''
		Euclidean distance measures distance between points in a linear space. When the space we
		are considering is high dimensional, it suffers from the curse of dimensionality.

		param points_1: first point in form of [float, float]
		param points_2: second point in form of [float, float]
		'''

		assert len(points_1) == len(points_2)

		return ( sum( (x - y) ** 2 for x, y in zip(points_1, points_2) ) ) ** 0.5

	def Manhattan(self, points_1:list, points_2:list):

		'''
		Manhattan distance refers to the distance between two vectors if they could only move
		by following right angles. It takes its name from New York Manhattan hood where streets
		are forming almost always squre root angles against each other.

		points_1: first point in form of list
		points_2: second point in form of list
		'''

		assert len(points_1) == len(points_2)

		return sum( abs(x - y) for x, y in zip(points_1, points_2) )

	def Minkowski(self, p_order:float, x:list, y:list):

		'''
		Minkowski distance refers to the distance a metric used in Normed vector space
		(n-dimensional real space), which means that it can be used in a space where distances
		can be represented as a vector that has a length.
		When p_order is equal to 1, this distance is the Mahnattan one.
		When p_order is equal to 2, this distance is the Euclidean one.

		param x: first point in form of [float, float]
		param y: second point in form of [float, float]
		param p_order: order of distance magnitude
		'''

		assert len(x) == len(y)

		return ( sum( [abs(i - j) ** p_order for i, j in zip(x, y)] ) ) ** (1 / p_order)

	def Hamming(self, x:str, y:str):

		'''
		Hamming distance (used for categorical features such as strings) measures the distance in terms of different
		values between two vectors. It is tipically used to measure the difference between two strings.

		param x: first object (i.e. "Andrea")
		param y: second point object (i.e. "Annrea")
		'''

		assert len(x) == len(y)

		distance = 0
		length   = len(x)

		for i in range(length):
			if x[i] != y[i]:
				distance +=1

		return (length, distance)

	def CosineSimilarity(self, x:list, y:list):

		'''
		Cosine similarity measure the distance between two vectors getting the cosine among them.
		If the cosine is close to one, these two vectors are equal. If the cosine is almost equal
		to minus one, these two vectors are dissimilar.
		in order to be compared the vector must have same length.

		param x: is it the first vector in form [1,2,4,5]
		param y: is it the first vector in form [1,2,4,7]
		'''

		dot    = np.dot( x, y)
		norm_x = ( sum( [ k**2 for k in x ] ) ) ** 0.5
		norm_y = ( sum( [ k**2 for k in y ] ) ) ** 0.5

		return dot / (norm_x * norm_y)


'''
Simulate MA(1) Time Series

You will simulate and plot a few MA(1) time series, each with a different parameter, θ
θ
, using the arima_process module in statsmodels, just as you did in the last chapter for AR(1) models. You will look at an MA(1) model with a large positive θ
θ
 and a large negative θ
θ
.

As in the last chapter, when inputting the coefficients, you must include the zero-lag coefficient of 1, but unlike the last chapter on AR models, the sign of the MA coefficients is what we would expect. For example, for an MA(1) process with θ=−0.9
θ
=
−
0.9
, the array representing the MA parameters would be ma = np.array([1, -0.9])

INSTRUCTIONS
100XP
Import the class ArmaProcess in the arima_process module.
Plot the simulated MA(1) processes
Let ma1 represent an array of the AR parameters [1, θ
θ
] as explained above. The AR parmater array will contain just the lag-zero coeffienct of one.
With parameters ar1 and ma1, create an instance of the class ArmaProcess(ar,ma) called MA_object1.
Simulate 1000 data points from the object you just created, MA_object1, using the method .generate_sample(). Plot the simulated data in a subplot.
Repeat for the other MA parameter.
'''
# import the module for simulating data
from statsmodels.tsa.arima_process import ArmaProcess

# Plot 1: MA parameter = -0.9
plt.subplot(2,1,1)
ar1 = np.array([1])
ma1 = np.array([1, -0.9])
MA_object1 = ArmaProcess(ar1, ma1)
simulated_data_1 = MA_object1.generate_sample(nsample=1000)
plt.plot(simulated_data_1)

# Plot 2: MA parameter = +0.9
plt.subplot(2,1,2)
ar2 = np.array([1])
ma2 = np.array([1, 0.9])
MA_object2 = ArmaProcess(ar2, ma2)
simulated_data_2 = MA_object2.generate_sample(nsample=1000)
plt.plot(simulated_data_2)

plt.show()


def make_design_matrix(X, M):
    """ Return a design matrix.

    Args:
        X -- N -- float: Inputs for training or testing.
        M -- 1 -- int: # of features to use.

    Returns:
        Phi -- N x M -- float: Design matrix.
    """
    assert M > 0 and type(M) == int, "M should be a positive integer."

    from numpy import zeros

    N = len(X)
    Phi = zeros((N, M))
    for col in range(M):
        Phi[:, col] = X ** col

    return Phi

def CFS_LR(Phi, y_train):
    """ Return weights of linear regression using closed form solution.

    Args:
        Phi -- N x M -- float: Design matrix.
        y_train -- N -- float: Labels for training.

    Returns:
        w -- M -- float: Updated weights.
    """
    from numpy.linalg import pinv

    # Calculate closed form solution.
    w = pinv(Phi.T @ Phi) @ (Phi.T @ y_train)

    return w

def make_R(query, x_train, tau):
    """ Return R for locally weighted linear regression.

    Args:
        query -- 1 -- float: Input that I want to focus on.
        x_train -- N -- float:
            Entire train input to determine the local weight of
            each train input with respect to the current query.
        tau -- 1 -- float: Bandwidth parameter.

    Returns:
        R -- -- float: Local weights for weighted linear regression.
    """
    from numpy import zeros, exp, diag
    R = zeros((len(x_train), len(x_train)))

    # Calculate each element of R using give formula and vectorize them.
    R = [exp(- (query - x_train[idx]) ** 2 / (2 * tau ** 2)) for idx in range(len(x_train))]

    # Make a diagonal matrix using the vector.
    R = diag(R)

    return R

def CFS_LW(Phi, R, y_train):
    """ Return weights of locally weighted linear regression
        using closed form solution.

    Args:
        Phi -- N x M -- float: Design matrix.
        R: -- N x N -- float: Local weights for a given query.
        y_train -- N -- float: Labels for training.

    Returns:
        w -- M -- float: Updated weights.
    """
    from numpy.linalg import pinv

    # Calculated the closed form solution for locally weighted LR.
    w = pinv(Phi.T @ R @ Phi) @ (Phi.T @ (R @ y_train))

    return w

def main():
    """ Run linear regression with various local weight parameters.
    """
    from numpy import load, linspace
    from matplotlib.pyplot import subplots, savefig, show

    # load data.
    x_train = load("data/q2x.npy")
    y_train = load("data/q2y.npy")

    # Run unweighted linear regression.
    # Find closed form solution for unweighted linear regression.
    Phi = make_design_matrix(x_train, 2)
    w = CFS_LR(Phi, y_train)
    y_result = w @ Phi.T

    # Plot the data and the line, and save as file.
    fig, ax = subplots()
    ax.title.set_text("Linear Regression without Local Weighting\ny = {:.3f} + {:.3f}x".format(w[0], w[1]))
    ax.set_xlabel("x")
    ax.set_ylabel("y")
    ax.scatter(x_train, y_train, label="Data", color="black")
    ax.plot(x_train, y_result, label="Unweighted LR Line")
    ax.legend()
    savefig("q2-d-i.png")

    # Run locally weighted linear regression with tau = 0.8.
    # Prepare queries.
    xmin, xmax = min(x_train), max(x_train)
    xran = xmax - xmin
    queries = linspace(xmin, xmax, num=50)

    # Iterate over each query.
    y_result = []
    for query in queries:
        # Find closed for solution for locally weighted linear regression.
        R = make_R(query, x_train, 0.8)
        w = CFS_LW(Phi, R, y_train)
        y_result.append(w[0] + w[1] * query)

    # Plot the curve and save as file.
    ax.title.set_text("Local Weighting Linear Regression")
    ax.plot(queries, y_result, label="Locally Weighted LR Curve, tau=0.8")
    ax.legend()
    savefig("q2-d-ii.png")

    # Run locally weighted linear regression using various taus.
    # Nest another iteration over four tau = 0.1, 0.3, 2, 10.
    taus = [0.1, 0.3, 2, 10]
    for tau in taus:
        y_result = []
        for query in queries:
            # Find closed for solution for locally weighted linear regression.
            R = make_R(query, x_train, tau)
            w = CFS_LW(Phi, R, y_train)
            y_result.append(w[0] + w[1] * query)

        # Plot the curve and save as file.
        ax.plot(queries, y_result, label="Locally Weighted LR Curve, tau={}".format(tau))
    ax.title.set_text("Local Weighted Linear Regression")
    ax.legend()
    savefig("q2-d-iii.png")
    show()

if __name__ == "__main__":
    main()


"""
Example: Cafe Java Demand Distribution and Process Generator.

This example develops a discrete process generator for coffee demand data
observed in Cafe Java.

@author: Paul T. Grogan <pgrogan@stevens.edu>
"""

# import the python3 behavior for importing, division, and printing in python2
from __future__ import absolute_import, division, print_function

# import the numpy package and refer to it as `np`
# see http://docs.scipy.org/doc/numpy/reference/ for documentation
import numpy as np

# import the matplotlib pyplot package and refer to it as `plt`
# see http://matplotlib.org/api/pyplot_api.html for documentation
import matplotlib.pyplot as plt

# create a numpy array with the demand sizes
demands = np.array([0, 1, 2, 3])
labels = ["None", "Small", "Medium", "Large"]
print("demands = {}".format(demands))

# create a numpy array with the observed frequencies
frequency = np.array([8, 10, 22, 10])

# the probability mass function is the number of demands (frequency) divided
# by the total number of demands (sum of all frequency)
pmf = frequency/np.sum(frequency)
print("pmf = {}".format(pmf))

# create a new figure for a bar plot using the demand indices, pmf values,
# desired bar width, and color
plt.figure()
bar_width = 0.5
plt.bar(demands, pmf, bar_width, align='center', color='b')
plt.ylabel('p(x)')
plt.ylim([0,1])
plt.xlabel('Coffee Demand (x)')
plt.title('PMF for Cafe Java Demand')
plt.xticks(demands, labels)

# the cumulative distribution function is the cumulative sum of the PDF
cdf = np.cumsum(pmf)
print("cdf = {}".format(cdf))

# create a new figure for a step plot using the demand indices, cdf values,
# desired format (solid red line, -r), and option for post-steps
plt.figure()
plt.step(demands, cdf, '-r', where='post')
plt.ylabel('F(x)')
plt.ylim([0,1])
plt.xlabel('Coffee Demand (x)')
plt.title('CDF for Cafe Java Demand')
plt.xticks(demands, labels)

# define a function to generate demands following the inverse transform method
def generate_demand_ivt():
    """Generates a demand following the inverse transform method.

    Returns:
        demand (int): the size of coffee demanded
    """
    r = np.random.rand()

    # check the first cdf entry (index 0)
    if r <= cdf[0]:
        return demands[0]
    # check the second cdf entry (index 1)
    elif r <= cdf[1]:
        return demands[1]
    # check the third cdf entry (index 2)
    elif r <= cdf[2]:
        return demands[2]
    # otherwise no need to check, it's the final CDF entry (index 3)
    else:
        return demands[3]

    """
    note: the code above could be replaced with the following for
    loop which iterates over each array index i in demands

    for i in range(len(demands)):
        if r <= cdf[i]:
            return demands[i]
    """

# define number of samples
num_samples = 1000

# fill the samples arrays with samples from the generators
samples_ivt = [generate_demand_ivt() for i in range(num_samples)]

# count the number of each demand: use a generator expression to count the
# number of samples which match each demand level i
counts_ivt = np.array([sum(samples_ivt==i) for i in demands])
frequency_ivt = counts_ivt/np.sum(counts_ivt)

# create a new figure recreating the PMF bar plot
plt.figure()
bar_width = 0.3
plt.bar(demands, pmf, bar_width, align='center', color='k', label='Observed')
# also display the generated frequency (offset bars by bar_width)
plt.bar(demands+bar_width, frequency_ivt,
        bar_width, color='b', label='Generated (IVT)')
plt.ylabel('p(x)')
plt.ylim([0,1])
plt.xlabel('Coffee Demand (x)')
plt.title('Cafe Java Demand Process Generator Results (n={})'.format(num_samples))
plt.xticks(demands + bar_width, labels)
plt.legend()


import math

import numpy as np
import pandas as pd
from pandas import Timestamp
from scipy import interpolate

__all__ = [
    "compute_days_between",
    "treasury_bill_price",
    "bond_equivalent_yield",
    "discount_factor_from",
    "spot_rate_from",
    "forward_rate_from",
]

def compute_days_between(start_date, end_date):
    """Computes number of days between dates"""
    time_delta = Timestamp(end_date) - Timestamp(start_date)
    return time_delta.days

def treasury_bill_price(discount_yield, days_to_maturity):
    """Computes price ot treasury bill"""
    return 100 * (1 - days_to_maturity / 360 * discount_yield)

def bond_equivalent_yield(discount_yield, days_to_maturity):
    """Computes bond equivalent yield from treasury bill discount yield"""
    return 365 * discount_yield / (360 - discount_yield * days_to_maturity)

def is_valid_freq(freq):
    return math.isfinite(freq) and isinstance(freq, int) and freq > 0

def discount_factor_from(spot_rate, term, freq=math.inf):
    """Computes discount factor from spot rate

    The  discount factor from a spot rate given term and compounding frequency.

    Parameters
    ----------
    spot_rate : float
        The spot rate
    term : float
        The term or time to maturity
    freq : int or math.inf, optional
        The compounding frequency, default is math.inf which results in continuous compounding rates.

    Returns
    -------
    float
        The discount factor determined the spot rate.

    Raises
    ------
    ValueError
        If freq is not math.inf of positive int

    """
    if freq == math.inf:
        return math.exp(-spot_rate * term)
    elif is_valid_freq(freq):
        return math.pow(1 + spot_rate / freq, -freq * term)
    else:
        raise ValueError("Freq must be math.inf or positive int")

def spot_rate_from(discount_factor, term, freq=math.inf):
    """Computes spot rate from discount factor

    The spot rate from the discount factor given term and compounding frequency.

    Parameters
    ----------
    discount_factor : float or pandas.Series
        The discount factor to determine the spot rate.
    term : float
        The term or time to maturity
    freq : int or math.inf, optional
        The compounding frequency, default is math.inf which results in continuous compounding rates.

    Returns
    -------
    float or pandas.Series
        The spot rate

    Raises
    ------
    ValueError
        If freq is not math.inf of positive int

    """
    if freq == math.inf:
        package = math
        if isinstance(discount_factor, pd.Series):
            package = np
        return -1 / term * package.log(discount_factor)
    elif is_valid_freq(freq):
        return freq * (math.pow(discount_factor, -1 / (freq * term)) - 1)
    else:
        raise ValueError("Freq must be math.inf or positive int")

def forward_rate_from(rate_1, term_1, rate_2, term_2, freq=math.inf):
    """Computes forward rate from pair of spot rates

    The forward rate between pair of spot rates given terms and compounding frequency.

    Parameters
    ----------
    rate_1 : float
       The first spot rate
    term_1 : float
        The first term or time to maturity
    rate_2 : float
       The second spot rate
    term_2 : float
        The second term or time to maturity
    freq : int or math.inf, optional
        The compounding frequency, default is math.inf which results in continuous compounding rates.

    Returns
    -------
    float, float
        The term and rate of the forward

    Raises
    ------
    ValueError
        If freq is not math.inf of positive int

    """
    term = term_2 - term_1
    if freq == math.inf:
        return term, (rate_2 * term_2 - rate_1 * term_1) / term
    elif is_valid_freq(freq):
        df_1 = discount_factor_from(rate_1, term_1, freq)
        df_2 = discount_factor_from(rate_2, term_2, freq)
        forward_df = df_2 / df_1
        return term, spot_rate_from(discount_factor=forward_df, term=term, freq=freq)
    else:
        raise ValueError("Freq must be math.inf or positive int")

def interp_rates(rates, maturities=None, column="Rate"):
    assert "Maturity" in rates.columns
    assert column in rates.columns
    sorted_rates = rates.sort_values("Maturity")

    interpolator = interpolate.Akima1DInterpolator(
        sorted_rates["Maturity"], sorted_rates[column]
    )
    if maturities is None:
        maturities = np.arange(0.25, 7.25, 0.25)
    interpolated_rates = interpolator(maturities)

    index = pd.Series(data=maturities, name="Maturity")
    interpolated_rates = pd.Series(
        data=interpolated_rates, name=f"Interpolated {column}", index=index
    )
    interpolated_rates = pd.DataFrame(interpolated_rates).reset_index()

    return interpolated_rates

def add_libor_curve(rates, first_swap_maturity, delta=0.25):
    assert "Maturity" in rates.columns
    assert "Interpolated Rate" in rates.columns
    zeros = np.zeros((len(rates),))

    idx = rates.index < first_swap_maturity
    zeros[idx] = 1 / (
        1 + rates.loc[idx, "Maturity"] * rates.loc[idx, "Interpolated Rate"]
    )
    for i in range(sum(idx), len(zeros)):
        rate = rates["Interpolated Rate"][i]
        zeros[i] = (1 - rate * delta * np.sum(zeros[:i])) / (1 + rate * delta)

    rates["Zero"] = zeros
    rates["Spot Rate"] = -1 / rates["Maturity"] * np.log(zeros)

    return rates

def add_short_rates(rates, time_step):
    assert "Maturity" in rates.columns
    assert "Zero" in rates.columns
    rates["Short Rate"] = 1 / time_step * (1 / rates["Zero"] - 1)

def add_forward_discounts(rates, start_maturity=1):
    assert "Maturity" in rates.columns
    assert "Zero" in rates.columns
    rates["Forward Discount"] = rates["Zero"] / rates["Zero"].shift(start_maturity)
    return rates

def add_forward_rates(rates):
    assert "Maturity" in rates.columns
    assert "Forward Discount" in rates.columns
    rates["Forward Rate"] = (1 / rates["Forward Discount"] - 1) / rates[
        "Maturity"
    ].diff()
    return rates

def add_forward_swap_discounts(rates, start_period=1):
    assert "Maturity" in rates.columns
    assert "Zero" in rates.columns
    rates["Forward Swap Discount"] = (
        rates["Zero"] / rates["Zero"].values[start_period - 1]
    )
    rates.loc[rates.index < start_period, "Forward Swap Discount"] = np.nan
    return rates

def add_swap_rates(rates):
    assert "Zero" in rates.columns
    assert "Maturity" in rates.columns
    rates["Swap Rate"] = (
        (1 - rates["Zero"]) / rates["Zero"].cumsum() / rates["Maturity"].diff()
    )
    return rates


import numpy as np
import tensorflow as tf

def softmax(x):
  """
  Compute the softmax function in tensorflow.

  You might find the tensorflow functions tf.exp, tf.reduce_max,
  tf.reduce_sum, tf.expand_dims useful. (Many solutions are possible, so you may
  not need to use all of these functions). Recall also that many common
  tensorflow operations are sugared (e.g. x * y does a tensor multiplication
  if x and y are both tensors). Make sure to implement the numerical stability
  fixes as in the previous homework!

  Args:
    x:   tf.Tensor with shape (n_samples, n_features). Note feature vectors are
         represented by row-vectors. (For simplicity, no need to handle 1-d
         input as in the previous homework)
  Returns:
    out: tf.Tensor with shape (n_sample, n_features). You need to construct this
         tensor in this problem.
  """

  ### YOUR CODE HERE
  maxes = tf.expand_dims(tf.reduce_max(x, reduction_indices=[1]), 1)
  x_red = x - maxes
  x_exp = tf.exp(x_red)
  sums = tf.expand_dims(tf.reduce_sum(x_exp, reduction_indices=[1]), 1)
  out = x_exp / sums
  ### END YOUR CODE

  return out

def cross_entropy_loss(y, yhat):
  """
  Compute the cross entropy loss in tensorflow.

  y is a one-hot tensor of shape (n_samples, n_classes) and yhat is a tensor
  of shape (n_samples, n_classes). y should be of dtype tf.int32, and yhat should
  be of dtype tf.float32.

  The functions tf.to_float, tf.reduce_sum, and tf.log might prove useful. (Many
  solutions are possible, so you may not need to use all of these functions).

  Note: You are NOT allowed to use the tensorflow built-in cross-entropy
        functions.

  Args:
    y:    tf.Tensor with shape (n_samples, n_classes). One-hot encoded.
    yhat: tf.Tensorwith shape (n_sample, n_classes). Each row encodes a
          probability distribution and should sum to 1.
  Returns:
    out:  tf.Tensor with shape (1,) (Scalar output). You need to construct this
          tensor in the problem.
  """
  ### YOUR CODE HERE
  y = tf.to_float(y)
  out = -tf.reduce_sum(y * tf.log(yhat))
  ### END YOUR CODE
  return out

def test_softmax_basic():
  """
  Some simple tests to get you started.
  Warning: these are not exhaustive.
  """
  print "Running basic tests..."
  test1 = softmax(tf.convert_to_tensor(
      np.array([[1001,1002],[3,4]]), dtype=tf.float32))
  with tf.Session():
      test1 = test1.eval()
  assert np.amax(np.fabs(test1 - np.array(
      [0.26894142,  0.73105858]))) <= 1e-6

  test2 = softmax(tf.convert_to_tensor(
      np.array([[-1001,-1002]]), dtype=tf.float32))
  with tf.Session():
      test2 = test2.eval()
  assert np.amax(np.fabs(test2 - np.array(
      [0.73105858, 0.26894142]))) <= 1e-6

  print "Basic (non-exhaustive) softmax tests pass\n"

def test_cross_entropy_loss_basic():
  """
  Some simple tests to get you started.
  Warning: these are not exhaustive.
  """
  y = np.array([[0, 1], [1, 0], [1, 0]])
  yhat = np.array([[.5, .5], [.5, .5], [.5, .5]])

  test1 = cross_entropy_loss(
      tf.convert_to_tensor(y, dtype=tf.int32),
      tf.convert_to_tensor(yhat, dtype=tf.float32))
  with tf.Session():
    test1 = test1.eval()
  result = -3 * np.log(.5)
  assert np.amax(np.fabs(test1 - result)) <= 1e-6
  print "Basic (non-exhaustive) cross-entropy tests pass\n"

if __name__ == "__main__":
  test_softmax_basic()
  test_cross_entropy_loss_basic()


# Question 5 Lab 03
# AB Satyaprakash (180123062)

# imports ----------------------------------------------------------------------
import sympy as sp
# ------------------------------------------------------------------------------
# (i) To show that both P(x) and Q(x) interpolate the data:
# f(−2) = −1, f(−1) = 3, f(0) = 1, f(1) = −1, f(2) = 3
# Given:
# P(x) = 3 − 2(x + 1) + 0(x + 1)(x) + (x + 1)(x)(x − 1)
# Q(x) = −1 + 4(x + 2) − 3(x + 2)(x + 1) + (x + 2)(x + 1)(x)
x = sp.Symbol('x')
px = 3 - 2*(x+1) + 0*(x+1)*(x) + (x+1)*(x)*(x-1)
qx = -1 + 4*(x+2) - 3*(x+2)*(x+1) + (x+2)*(x+1)*(x)

X = [-2, -1, 0, 1, 2]
F = [-1, 3, 1, -1, 3]
ok = True  # will change this to false if either P or Q does not interpolate

for i in range(len(X)):
    pval = px.subs(x, X[i])
    print('The value of P({}) = {}'.format(X[i], pval))
    if pval != F[i]:
        ok = False
print('\n')
for i in range(len(X)):
    qval = qx.subs(x, X[i])
    print('The value of Q({}) = {}'.format(X[i], qval))
    if qval != F[i]:
        ok = False

if ok == True:
    print("\nThus, both cubic polynomails P(x) and Q(x) interpolate the given data")

# (ii) Why does part (i) not violate the uniqueness property of interpolating polynomials
px = sp.expand(px)
qx = sp.expand(qx)
print('Simplifying P(x) we get', px)
print('Simplifying Q(x) we get', qx)
print('Since we can clearly see that P(x) = Q(x), this ensures that the uniqueness property of interpolating polynmials is not violated')

# Question 5 ends --------------------------------------------------------------

Semester 6/MA 322 (Scientific Computing Theory)/Labs/Lab 3/Code/q5.py


#%%
"""
Created on July 05  2021
Incentive function as a function of a swap rate or the differential w.r.t. "old" mortgage rate

This code is purely educational and comes from "Financial Engineering" course by L.A. Grzelak
The course is based on the book “Mathematical Modeling and Computation
in Finance: With Exercises and Python and MATLAB Computer Codes”,
by C.W. Oosterlee and L.A. Grzelak, World Scientific Publishing Europe Ltd, 2019.
@author: Lech A. Grzelak and Emanuele Cassamassima
"""
import numpy as np
import matplotlib.pyplot as plt

def Annuity(rate,notional,periods,CPR):
    # it returns a matrix M such that
    # M = [t  notional(t)  prepayment(t)  notional_quote(t)  interest_quote(t)  installment(t)]
    # WARNING! here "rate" and "periods" are quite general, the choice of getting year/month/day.. steps, depends on the rate
    # that the function receives. So, it is necessary to pass the correct rate to the function
    M = np.zeros((periods + 1,6))
    M[:,0] = np.arange(periods + 1) # we define the times
    M[0,1] = notional
    for t in range(1,periods + 1):
        # we are computing the installment at time t knowing the oustanding at time (t-1)
        remaining_periods = periods - (t - 1)

        # Installment, C(t_i)
        M[t,5] = rate * M[t-1,1]/(1 - 1/(1 + rate)**remaining_periods)

        # Interest rate payment, I(t_i) = r * N(t_{i})
        M[t,4] = rate * M[t-1,1]

        # Notional payment, Q(t_i) = C(t_i) - I(t_i)
        M[t,3] = M[t,5] - M[t,4]

        # Prepayment, P(t_i)= Lambda * (N(t_i) -Q(t_i))
        M[t,2] = CPR * (M[t-1,1] - M[t,3])

        # notional, N(t_{i+1}) = N(t_{i}) - lambda * (Q(t_{i} + P(t_i)))
        M[t,1] = M[t-1,1] - M[t,3] - M[t,2]
    return M

def mainCode():

    IncentiveFunction = lambda x : 0.04 + 0.1/(1 + np.exp(115 * (0.02-x)))

    oldRate = 0.05
    newRate = np.linspace(-0.05,0.15,25)

    epsilon = oldRate-newRate
    incentive = IncentiveFunction(epsilon)

    plt.figure(1)
    plt.plot(newRate,incentive)
    plt.xlabel('S(t)')
    plt.ylabel('Incentive')
    plt.grid()

    plt.figure(2)
    plt.plot(epsilon,incentive)
    plt.xlabel('epsilon= K - S(t)')
    plt.ylabel('Incentive')
    plt.grid()

    return 0.0

mainCode()

Lecture 08-Mortgages and Prepayments/Materials/Incentives.py


import numpy as np

def mean_square_error(y_true: np.ndarray, y_pred: np.ndarray) -> float:
    """
    Calculate MSE loss

    Parameters
    ----------
    y_true: ndarray of shape (n_samples, )
        True response values
    y_pred: ndarray of shape (n_samples, )
        Predicted response values

    Returns
    -------
    MSE of given predictions
    """

    val = 0
    diffArr = y_true - y_pred
    for i in diffArr:
        val += np.power(i, 2)

    return val * (1 / diffArr.size)

def misclassification_error(y_true: np.ndarray, y_pred: np.ndarray,
                            normalize: bool = True) -> float:
    """
    Calculate misclassification loss

    Parameters
    ----------
    y_true: ndarray of shape (n_samples, )
        True response values
    y_pred: ndarray of shape (n_samples, )
        Predicted response values
    normalize: bool, default = True
        Normalize by number of samples or not

    Returns
    -------
    Misclassification of given predictions
    """
    counter = 0
    for i, j in zip(y_pred, y_true):
        if i != j:
            counter += 1
    if normalize: return counter / y_true.size
    return counter

def accuracy(y_true: np.ndarray, y_pred: np.ndarray) -> float:
    """
    Calculate accuracy of given predictions

    Parameters
    ----------
    y_true: ndarray of shape (n_samples, )
        True response values
    y_pred: ndarray of shape (n_samples, )
        Predicted response values

    Returns
    -------
    Accuracy of given predictions
    """
    counter = 0
    for i, j in zip(y_pred, y_true):
        if i == j:
            counter += 1

    return counter / y_true.size

def cross_entropy(y_true: np.ndarray, y_pred: np.ndarray) -> float:
    """
    Calculate the cross entropy of given predictions

    Parameters
    ----------
    y_true: ndarray of shape (n_samples, )
        True response values
    y_pred: ndarray of shape (n_samples, )
        Predicted response values

    Returns
    -------
    Cross entropy of given predictions
    """
    raise NotImplementedError()


import numpy as np

def hilb(n, m=0):
    """
    hilb   Hilbert matrix.
       hilb(n,m) is the n-by-m matrix with elements 1/(i+j-1).
       it is a famous example of a badly conditioned matrix.
       cond(hilb(n)) grows like exp(3.5*n).
       hilb(n) is symmetric positive definite, totally positive, and a
       Hankel matrix.

       References:
       M.-D. Choi, Tricks or treats with the Hilbert matrix, Amer. Math.
           Monthly, 90 (1983), pp. 301-312.
       N.J. Higham, Accuracy and Stability of Numerical Algorithms,
           Society for Industrial and Applied Mathematics, Philadelphia, PA,
           USA, 2002; sec. 28.1.
       M. Newman and J. Todd, The evaluation of matrix inversion
           programs, J. Soc. Indust. Appl. Math., 6 (1958), pp. 466-476.
       D.E. Knuth, The Art of Computer Programming,
           Volume 1, Fundamental Algorithms, second edition, Addison-Wesley,
           Reading, Massachusetts, 1973, p. 37.

       NOTE added in porting.  We do not use the function cauchy here to
       generate the Hilbert matrix.  That is done so we can unit test the
       the functions against each other.  Also, the function has been
       generalized to take by row and column sizes.  If only a row size
       is given, we assume a square matrix is desired.
    """
    if n < 1 or m < 0:
        raise ValueError("Matrix size must be one or greater")
    elif n == 1 and (m == 0 or m == 1):
        return np.array([[1]])
    elif m == 0:
        m = n

    return 1. / (np.arange(1, n + 1) + np.arange(0, m)[:, np.newaxis])


#
# Source Code  from https://people.duke.edu/~ccc14/sta-663/EMAlgorithm.html
# And here https://mk-minchul.github.io/EM/
#
import numpy as np

# This is our observed data
# Coin flips by two people, each 10 times, the first number is the number heads and scond is the number of tails
ys = np.array([(9,1), (8,2), (5,5), (4,6), (7,3)])

thetas = np.array([[0.6, 0.4], [0.5, 0.5]])  # initialize theta_1, 2
# Thetas can be the probabilities for the first and second persons for example.

pis =np.array([0.5, 0.5])  # prob of choosing coin 1 or coin2 is the same.

tolerance = 0.01
max_iter = 100

loglike_old = 0
for i in range(max_iter):
    E_c1 = []
    E_c2 = []
    EcY_1 = []
    EcY_2 = []

    loglike_new = 0
    # E-step: calculate probability distributions over possible completions
    for i in range(len(ys)):
        # multinomial (binomial) log likelihood
        log_k1 = np.sum([ys[i]*np.log(thetas[0])])  #  \log [\theta_k^{y_{oi}} (1-\theta_k)^{n - y_{oi}} ]
        log_k2 = np.sum([ys[i]*np.log(thetas[1])])  #  \log [\theta_k^{y_{oi}} (1-\theta_k)^{n - y_{oi}} ]

        # Getting the expectation of c_ik
        denom = np.exp(log_k1) * pis[0] + np.exp(log_k2) * pis[1]
        E_ci1 = np.exp(log_k1) * pis[0] / denom
        E_ci2 = np.exp(log_k2) * pis[1] / denom

        # update complete log likelihood
        # we need it only to check if it converged.
        # we dont need it for updating theta.
        loglike_new += E_ci1 * log_k1 + E_ci2 * log_k2
        E_c1.append(E_ci1)
        E_c2.append(E_ci2)

    # M-step: update values for parameters given current distribution
    for i in range(len(ys)):
        EcY_1.append(E_c1[i] * ys[i] )  # this is a scalar times a vector.
        EcY_2.append(E_c2[i] * ys[i] )

    thetas[0] = np.sum(EcY_1, 0)/np.sum(EcY_1)
    thetas[1] = np.sum(EcY_2, 0)/np.sum(EcY_2)

    print ("Iteration: %d" % (i+1))
    print ("theta_A = [%.2f, %.2f], theta_B = [%.2f, %.2f] , difference in loglike = %.2f" % (thetas[0,0], thetas[0,1], thetas[1,0], thetas[1,1], loglike_new - loglike_old))

    if np.abs(loglike_new - loglike_old) < tolerance:
        break

    loglike_old = loglike_new


#!/usr/bin/env python

''' Método de Newton: encontrar o mínimo dessa função '''

from sympy import lambdify, diff, cos, sin, exp
from sympy.plotting import plot, plot3d
from sympy.abc import x

from Error import *
from Log import *

MAX = 50
PATH = 'log/newton/'
TOLERANCE = 0.00000001 # 10**(-8)

def newton(fn, cx, tol, nmax) :
    l = Log()
    e = Error()
    previous = 0
    function = fn

    dx = fn.diff(x)
    dx = lambdify(x, dx)
    fn = lambdify(x, fn)

    for n in range(nmax) :
        if (dx(cx) == 0) :
            return "Error 25: derivative less than zero"
            breakpoint
        cx = cx - (fn(cx) / dx(cx) )
        e.absolute(cx, previous)
        e.relative(cx, previous)
        l.append([cx, e._absolute, e._relative, fn(cx), dx(cx)])
        if (e._absolute < tol) :
            l.set_header(['x', 'absolute_error', 'relative_error', 'function', 'derivative'])
            l.list2file((PATH+str(function)))
            return cx
            breakpoint
        previous = cx
    return False

def run_test(function, a, TOLERANCE, MAX):
    m = newton(function, a, TOLERANCE, MAX)
    print('f(x) =',f, '-> newton =', m)
    return m

if __name__ == "__main__":
    ''' Tests '''
    a = 0.5
    fx = [(cos(x) - x**3), ((3*(x**3))-2), (6*(x**2) - 400),
          (x**2 - 4*x -5), (x ** 2 - 10), (x**2 - 612),
          (x**3 - 1),(x ** 3 - 2 * x - 5), ((x**3)-(2*x)+2),
          (x**3-6*x**2+4*x+2), (x**3 - x**2 - 1),
          (x**6 - x - 1)]

    for f in fx:
        r = run_test(f, a, TOLERANCE, MAX)
        # graph = plot(f, show=False)
        # graph.save('view/function/'+str(f))

        # graph = plot3d(f, show=False)
        # graph.save('view/function/3D/'+str(f))


# coding: utf-8

# ## Matrix Object in Numpy
# Numpy, short for Numerical Python, is the fundamental package required for hight performance scientific computing and its best library to learn and apply on data science career.
#
# This is just little illustration.
#
# <img style="float: left;" src="https://www.safaribooksonline.com/library/view/python-for-data/9781449323592/httpatomoreillycomsourceoreillyimages1346880.png" width=400 height=200>

# In[1]:

import numpy as np

# In[2]:

a = np.array([[1,2,4],
              [2,5,3],
              [7,8,9]])
A = np.mat(a)
A

# In[3]:

A = np.mat('1,2,4;2,5,3;7,8,9')
A

# In[5]:

a = np.array([[ 1, 2],
              [ 3, 4]])
b = np.array([[10,20],
              [30,40]])

np.bmat('a,b;b,a')

# In[10]:

x = np.array([[4], [2], [3]])
x

# In[11]:

A * x

# In[12]:

print(A * A.I)

# In[13]:

print(A ** 3)


from numpy import array
from numpy.linalg import det

def poly_fit(x,y):
    n = len(x)
    x1sm = sum(x)
    y1sm = sum(y)
    x2sm = sum(x**2)
    x3sm = sum(x**3)
    x4sm = sum(x**4)
    xysm = sum(x*y)
    x2ysm = sum((x**2)*y)
    # Cramer's rule
    eq = array([[x2sm,x3sm,x4sm],[x1sm,x2sm,x3sm],[n,x1sm,x2sm]])
    rig = array([x2ysm, xysm, y1sm])
    detabc = det(eq)
    eqc = array([[x2ysm,x3sm,x4sm],[xysm,x2sm,x3sm],[y1sm,x1sm,x2sm]])
    eqb = array([[x2sm,x2ysm,x4sm],[x1sm,xysm,x3sm],[n,y1sm,x2sm]])
    eqa = array([[x2sm,x3sm,x2ysm],[x1sm,x2sm,xysm],[n,x1sm,y1sm]])
    detc = det(eqc)
    detb = det(eqb)
    deta = det(eqa)
    a = deta/detabc
    b = detb/detabc
    c = detc/detabc
    return a,b,c

x = array([0, 1, 2, 4])
y = array([2, 3, 9, 15])
a,b,c = poly_fit(x,y)

print("a = %8.4f" % a)
print("b = %8.4f" % b)
print("b = %8.4f" % c)
print("y = %8.4f x^2 + %8.4f x + %8.4f" % (a, b, c))


import numpy as np
import scipy as sp
import scipy.linalg as la
import scipy.spatial.distance as dist
from sklearn import manifold

def pca(X, n_components=2):
    centered = X - X.mean(axis=0)

    U, s, Vt = la.svd(centered, full_matrices = False)

    s2 = s ** 2

    U = U[:, :n_components]
    s = s[:n_components]
    Vt = Vt[:n_components, :]

    return U, s, Vt, s2

def t_SNE(X, n_components=2):
    return manifold.TSNE(n_components=n_components, init='pca').fit_transform(X)

def spectral_embedding(X, n_components=2, n_neighbors=10):
    return manifold.SpectralEmbedding(
            n_components=n_components, n_neighbors=n_neighbors).fit_transform(X)

def Linfty_centering(X, options, metric='euclidean'):
    return dist.squareform(dist.pdist(X, metric=metric)).max(axis=0)

def pca_projection(X, n_axis = 2):
    N = X.shape[0]
    U, s, Vt, s2 = pca(X, n_axis)

    explained_variance = s2 / N

    # U * diag(s) is a score matrix.
    return U.dot(np.diag(s)), explained_variance / explained_variance.sum()

def simple_axis_projection(X, axis = 0):
    return X[:, axis]

def gaussian_density(X, options, metric='euclidean'):
    eps = float(options['epsilon'])
    dist_mat = dist.squareform(dist.pdist(X, metric=metric))

    return np.exp(-(dist_mat ** 2) / eps).sum(axis=0)


import pandas as pd
import numpy as np
import numpy.linalg as ln
from matplotlib import pylab as plt

def out(filename, s):
	with open(filename, 'w') as f:
		f.write(s)

def mserror(y, y_pred):
	return (1 / float(y.size)) * np.sum((y - y_pred) ** 2)

def normal_equation(X, y):
	return ln.solve(X.T.dot(X), X.T.dot(y))

def linear_prediction(X, w):
	return X.dot(w)

def stochastic_gradient_step(X, y, w, train_ind, eta=0.01):
	mult = (2 * eta / float(X.shape[0])) * (y[train_ind] - X[train_ind, :].dot(w))
	w_new = np.zeros(X.shape[1])
	w_new[0] = (w[0] + mult)
	for i in xrange(1, X.shape[1]):
		w_new[i] = (w[i] + mult * X[train_ind, i])
	return w_new

def stochastic_gradient_descent(X, y, w_init, eta=1e-2, max_iter=1e4,
								min_weight_dist=1e-8, seed=42, verbose=False):
	weight_dist = np.inf
	w = w_init
	errors = []
	iter_num = 0
	np.random.seed(seed)

	while weight_dist > min_weight_dist and iter_num < max_iter:
		random_ind = np.random.randint(X.shape[0])
		w_new = stochastic_gradient_step(X, y, w, random_ind, eta)

		weight_dist = np.linalg.norm(w_new - w)
		w = w_new
		errors.append(mserror(y, linear_prediction(X, w)))
		iter_num += 1

	return w, errors

data = pd.read_csv('advertising.csv')

y = data['Sales'].values
X = data.drop('Sales', axis=1).values

means, stds = np.mean(X, axis=0), np.std(X, axis=0)
X = (X - means) / stds
X = np.hstack((np.ones((X.shape[0], 1)), X))

y_pred = np.full(y.shape, np.median(y))
err = mserror(y, y_pred)
res = '%.3f' % err
out('1_1.txt', res)

norm_eq_weights = normal_equation(X, y)
res = '%.3f' % norm_eq_weights.dot(np.mean(X, axis=0))
out('1_2.txt', res)

y_pred = linear_prediction(X, norm_eq_weights)
err = mserror(y, y_pred)
res = '%.3f' % err
out('1_3.txt', res)

stoch_grad_desc_weights, stoch_errors_by_iter = stochastic_gradient_descent(X, y, np.zeros(X.shape[1]), max_iter=1e5)
plt.plot(range(len(stoch_errors_by_iter)), stoch_errors_by_iter)
plt.xlabel('Iteration number')
plt.ylabel('MSE')
plt.show()

print stoch_grad_desc_weights
print stoch_errors_by_iter[-1]

y_pred = linear_prediction(X, stoch_grad_desc_weights)
err = mserror(y, y_pred)
res = '%.3f' % err
print res
out('1_4.txt', res)


#%%
import numpy as np
import matplotlib.pyplot as plt

#%%
u = np.linspace(-2,2,65)
v = np.linspace(-1,1,33)

#%%
X,Y = np.meshgrid(u,v)

#%%
Z = X**2 /25 + Y**2/4
# Z = np.sin(X) + np.cos(Y)

#%%
plt.pcolor(Z)
plt.colorbar()
plt.show()

#%%
plt.pcolor(X, Y, Z,  cmap='gray') # show the meshgrid to label
plt.colorbar()
plt.axis('tight')
plt.show()

#%%
a = [1,2,3,4]
b = [10,11,12]
A,B = np.meshgrid(a,b)

#%%
plt.contour(X, Y, Z, 30, cmap='autumn') # show the meshgrid to label
plt.colorbar()
plt.axis('tight')
plt.show()

#%%
plt.contourf(X, Y, Z, 30) # show the meshgrid to label
plt.colorbar()
plt.axis('tight')
plt.show()

#%%
plt.subplot(2,2,1)
plt.pcolor(Z)
plt.colorbar()

plt.subplot(2,2,2)
plt.pcolor(X, Y, Z,  cmap='gray') # show the meshgrid to label
plt.colorbar()

plt.subplot(2,2,3)
plt.contour(X, Y, Z, 30, cmap='autumn') # show the meshgrid to label
plt.colorbar()

plt.subplot(2,2,4)
plt.contourf(X, Y, Z, 30) # show the meshgrid to label
plt.colorbar()

# plt.axis('tight')
plt.tight_layout()
plt.show()

#%%


import numpy as np
import matplotlib.pyplot as plt
import sympy as sp
from numpy import pi
plt.rcParams['lines.linewidth'] = 3
plt.rcParams['font.size'] = 30
plt.rcParams['figure.figsize'] = [30,14]
path = '/home/rosinante/PDS/PDS_UFCG/python/questao3/'
#%% Function
t = sp.symbols('t')
f = sp.cos(2*np.pi*3200*t) + 0.5*sp.cos(2*np.pi*600*t) + 0.01*sp.cos(2*np.pi*300*t)
x = sp.lambdify(t,f,'numpy')
#%% Continuous
L = 10
Fs = int(6000/L)
Ts = 1/Fs
t_sampled = np.linspace(0,1,Fs)
x_sampled = x(t_sampled)

#%% Sampled
F = 20*6400
T = 1/F
t_continuous = np.linspace(0,1,F)
x_continuous = x(t_continuous)

# %% FFT continuous
y_continuous = np.fft.fft(x_continuous)
y_continuous = np.abs(y_continuous)/np.max(np.abs(y_continuous))
nc = x_continuous.size
freq_continuous = np.fft.fftfreq(nc,d=T)#*T*2*pi

# %% FFT sampled
y_sampled = np.fft.fft(x_sampled)
y_sampled = np.abs(y_sampled)/np.max(np.abs(y_sampled))
n = x_sampled.size
freq_sampled = np.fft.fftfreq(n,d=Ts)#*Ts*2*pi

#%% plot FFT continuous
fig = plt.figure()
labels = ['-pi','-pi/2','0','pi/2','pi']
plt.plot(freq_continuous,y_continuous)
#plt.xticks(np.arange(-pi,pi+1,pi/2),labels)
plt.xlim([-3200-100,3200+100])
plt.xlabel('frequency (Hz)')
plt.title('Continuous Fourier transformed function')
fig.savefig(path+'fft_continuous.png')

#%% plot FFT sampled
fig = plt.figure()
plt.stem(freq_sampled,y_sampled)
#plt.xticks(np.arange(-pi,pi+1,pi/2),labels)
#plt.xlim([-3200-100,3200+100])
plt.grid(True)
plt.xlabel('frequency (Hz)')
plt.title('Sampled Fourier transformed function')
plt.legend(['Fs = '+str(Fs)+' Hz'])
fig.savefig(path+'fft_sampled_Fs_'+str(Fs)+'Hz.png')
#%%
fig = plt.figure()
plt.plot(freq_continuous,y_continuous,'k-')
plt.stem(freq_sampled,y_sampled,'r--')
#plt.xticks(np.arange(-pi,pi+1,pi/2),labels)
plt.xlim([-3200-100,3200+100])
plt.xlabel('frequency (Hz)')
plt.title('Continuous and Sampled Fourier transformed functions')
plt.grid(True)
plt.legend(['Continuous','Sampled - Fs = '+str(Fs)+' Hz'])
fig.savefig(path+'fft_both_Fs_'+str(Fs)+'Hz.png')


import numpy as np
import pandas as pd

############################################################################
# DO NOT MODIFY CODES ABOVE
# DO NOT CHANGE THE INPUT AND OUTPUT FORMAT
############################################################################

# Part 1.1
def mean_square_error(w, X, y):
    """
    Compute the mean square error of a model parameter w on a test set X and y.
    Inputs:
    - X: A numpy array of shape (num_samples, D) containing test features
    - y: A numpy array of shape (num_samples, ) containing test labels
    - w: a numpy array of shape (D, )
    Returns:
    - err: the mean square error
    """
    n, y_predicted = len(y), np.dot(X, w)
    err = np.sum(np.power(y - y_predicted, 2), dtype=np.float) / n
    return err

# Part 1.2
def linear_regression_noreg(X, y):
    """
    Compute the weight parameter given X and y.
    Inputs:
    - X: A numpy array of shape (num_samples, D) containing features
    - y: A numpy array of shape (num_samples, ) containing labels
    Returns:
    - w: a numpy array of shape (D, )
    """
    # w* = (X^T X)^(-1) X^T y
    w = np.dot(np.dot(np.linalg.inv(np.dot(X.T, X)), X.T), y)
    return w

# Part 1.3
def regularized_linear_regression(X, y, lambd):
    """
    Compute the weight parameter given X, y and lambda.
    Inputs:
    - X: A numpy array of shape (num_samples, D) containing features
    - y: A numpy array of shape (num_samples, ) containing labels
    - lambd: a float number specifying the regularization parameter
    Returns:
    - w: a numpy array of shape (D, )
    """
    D = np.shape(X)[1]
    I = np.eye(D, k=0)
    w = np.dot(np.dot(np.linalg.inv(np.dot(X.T, X)+lambd*I), X.T), y)
    return w

# Part 1.4
def tune_lambda(Xtrain, ytrain, Xval, yval):
    """
    Find the best lambda value.
    Inputs:
    - Xtrain: A numpy array of shape (num_training_samples, D) containing training features
    - ytrain: A numpy array of shape (num_training_samples, ) containing training labels
    - Xval: A numpy array of shape (num_val_samples, D) containing validation features
    - yval: A numpy array of shape (num_val_samples, ) containing validation labels
    Returns:
    - bestlambda: the best lambda you find among 2^{-14}, 2^{-13}, ..., 2^{-1}, 1.
    """
    lambds, best_loss, best_lambda = [pow(2.0, -x) for x in range(15)], float('inf'), None
    for lambd in lambds:
        w = regularized_linear_regression(X=Xtrain, y=ytrain, lambd=lambd)
        mse = mean_square_error(w=w, X=Xval, y=yval)
        if mse < best_loss:
            best_lambda = lambd
            best_loss = mse

    return best_lambda

# Part 1.5
def mapping_data(X, P):
    """
    Augment the data to [X, X^2, ..., X^p]
    Inputs:
    - X: A numpy array of shape (num_training_samples, D) containing training features
    - P: An integer that indicates the degree of the polynomial regression
    Returns:
    - X: The augmented dataset. You might find np.insert useful.
    """
    raw = X
    for p in range(2, P+1):
        x_poly = np.power(raw, p)
        X = np.column_stack((X, x_poly))

    return X

"""
NO MODIFICATIONS below this line.
You should only write your code in the above functions.
"""


# -*- coding: utf-8 -*-
# Created on Thu Feb 27 14:58:01 2020
# @author: arthurd

import numpy as np

def softmax(X):
    """
    Compute and return the softmax of the input.

    Parameters
    ----------
    X : numpy.ndarray
        Inputs of floats with shape [n, m]

    Returns
    -------
    numpy.ndarray
        Outputs of floats with shape [n, m]
    """

    exp = np.exp(X)
    t = X - np.log(np.sum(exp, axis=0))
    S = np.exp(t)
    return S

sigmoid = lambda X: 1.0 / (1.0 + np.exp(-X))

tanh = lambda X: np.tanh(X)

relu = lambda X: np.maximum(0, X)

leaky_relu = lambda X: np.where(X > 0, X, X * 0.01)

linear = lambda X: X


# -*- coding: utf-8 -*-
"""
@author: salimt
"""
#Problem 1: Curve Fitting
#15/15 points (graded)
#Implement the generate_models function.
#
#x and y are two lists corresponding to the x-coordinates and y-coordinates of the data samples (or data points); for example, if you have N data points, x = [x1 , x2 , ..., xN ] and y = [y1 , y2 , ..., yN ], where x_i and y_i are the x and y coordinate of the i-th data points. In this problem set, each x coordinate is an integer and corresponds to the year of a sample (e.g., 1997); each corresponding y coordinate is a float and represents the temperature observation (will be computed in multiple ways) of that year in Celsius. This representation will be used throughout the entire problem set.
#degs is a list of integers indicating the degree of each regression model that we want to create. For each model, this function should fit the data (x,y) to a polynomial curve of that degree.
#This function should return a list of models. A model is the numpy 1d array of the coefficients of the fitting polynomial curve. Each returned model should be in the same order as their corresponding integer in degs.
#Example:
#
#print(generate_models([1961, 1962, 1963],[4.4,5.5,6.6],[1, 2]))
#Should print something close to:
#
#[array([ 1.10000000e+00, -2.15270000e+03]), array([ -8.86320195e-14, 1.10000000e+00, -2.15270000e+03])]
#The above example was generating a linear and a quadratic curve on data samples (xi, yi ) = (1961, 4.4), (1962, 5.5), and (1963, 6.6). The resulting models are in the same order as specified in degs. Note that it is fine you did not get the exact number because of numerical errors.
#
#Note: If you want to use numpy arrays, you should import numpy as np and use np.METHOD_NAME in your code. Unfortunately, pylab does not work with the grader
#
## Problem 1
#
def generate_models(x, y, degs):
    """
    Generate regression models by fitting a polynomial for each degree in degs
    to points (x, y).
    Args:
        x: a list with length N, representing the x-coords of N sample points
        y: a list with length N, representing the y-coords of N sample points
        degs: a list of degrees of the fitting polynomial
    Returns:
        a list of numpy arrays, where each array is a 1-d array of coefficients
        that minimizes the squared error of the fitting polynomial
    """
    import numpy as np

    models = []
    for d in degs:
        model = np.polyfit(x, y, d)
        models.append(model)
    return models

#print(generate_models([1961, 1962, 1963],[4.4,5.5,6.6],[1, 2]))
#[array([ 1.10000000e+00, -2.15270000e+03]), array([ -8.86320195e-14, 1.10000000e+00, -2.15270000e+03])]


import numpy as np
from math import factorial

def _sin(x):
    val = np.float64(0)
    terms = np.array([], dtype=np.float64)
    for n in range(41):
        term = (((-1) ** n) * (x ** (2*n+1)) / factorial(2*n + 1))
        terms = np.append(terms, term)
        val += term
    return val, terms

def debug_sin(x):
    val, terms = _sin(x)
    tmp = np.float64(0)
    for i, t in enumerate(terms):
        tmp += t
        print("for x_{0}, term is {1}, sin({2}) approximation is {3}".format(
            i, t, x, tmp))
    return val

def sin(x):
    v, _ = _sin(x)
    return v

def cos(x):
    val = np.float64(0)
    for n in range(41):
        term = (((-1) ** n) * (x ** (2*n)) / factorial(2*n))
        val += term
    return val

pi = np.float64(22./7.)
print("pi is initially {0}".format(pi))
for i in range(5):
    pi += sin(pi)
    print("improving pi iteration {0}, pi: {1}".format(i, pi))

if __name__ == "__main__":
    vals = np.array([0, pi / 4., pi / 2., pi * 3./4., pi], dtype=np.float64)
    for n in vals:
        print("sin({0}) = {1}".format(n, sin(n)))
        print("cos({0}) = {1}".format(n, cos(n)))


import numpy as np
import matplotlib.pyplot as plt
import time
import math

def fft_own(xt):
    """A vectorized, non-recursive version of the Cooley-Tukey FFT"""
    x = np.asarray(xt, dtype=float)
    N = x.shape[0]

    if np.log2(N) % 1 > 0:
        raise ValueError("size of x must be a power of 2")

    # N_min here is equivalent to the stopping condition above,
    # and should be a power of 2
    N_min = min(N, 32)

    # Perform an O[N^2] DFT on all length-N_min sub-problems at once
    n = np.arange(N_min)
    k = n[:, None]
    M = np.exp(-2j * np.pi * n * k / N_min)
    X = np.dot(M, x.reshape((N_min, -1)))

    # build-up each level of the recursive calculation all at once
    while X.shape[0] < N:
        X_even = X[:, :int(X.shape[1] / 2)]
        X_odd = X[:, int(X.shape[1] / 2):]
        factor = np.exp(-1j * np.pi * np.arange(X.shape[0])
                        / X.shape[0])[:, None]
        X = np.vstack([X_even + factor * X_odd,
                       X_even - factor * X_odd])

    return X.ravel()

def brev(inDat):
    N = len(inDat)
    outDat = []
    for k in range(N):
        revN = int('{:0{width}b}'.format(k, width=int(np.ceil(np.log2(N))))[::-1], 2)
        outDat.append(inDat[revN])
    return outDat

if __name__ == "__main__":
    fig, axes = plt.subplots(ncols=1, nrows=2)
    ax1, ax2 = axes.ravel()

    # generate an input signal
    Num = 1024
    fs = 1000
    t = np.linspace(0, Num / fs, Num)
    x_t = np.sin(2 * np.pi * 1 * t) + 0.5 * np.sin(2 * np.pi * 5 * t) + 0.2 * np.sin(2 * np.pi * 10 * t)

    # Generate the FFT twiddle factors

    # Compute the FFT with time measurement
    start_time = time.time()
    X_f = fft_own(x_t)
    stop_time = time.time()

    f = np.linspace(0, fs / 2, int(Num / 2))
    # Magnitude
    A_f = np.zeros(Num, dtype=float)
    for k in range(Num):
        A_f[k] = np.sqrt(np.power(X_f[k].imag, 2) + np.power(X_f[k].real, 2))

    elapsed_time = stop_time - start_time
    print("The time of execution of our DFT is: %.5f s." % elapsed_time)

    # plot input signal
    ax1.plot(t, x_t)
    ax1.margins(0)
    ax1.grid()
    # plot output signal
    ax2.plot(f, A_f[0:int(Num / 2)])
    plt.xscale("log")
    ax2.margins(0)
    ax2.grid()
    plt.show()


import numpy as np
import Matrix_2 as M2

"""
lin - Linear interpolation
Input: inp = a matrix containing all the x and y values (columns 0 and 1),
sorted in ascending order in x.
Output: array f whose elements i provide the y values for any x between x_i and
x_i+1 using a linear interpolation method.
"""

def lin(inp, stepsize = 0.01):
    x = inp[:, 0]
    y = inp[:, 1]
    f = np.array([])
    xsamp = np.array([])
    for i in np.arange(len(x)-1):
        xnew = np.arange(start = x[i], stop = x[i+1], step = stepsize)
        xsamp = np.append(xsamp, xnew)
        # Samples points at step size 0.01 between two adjacent data points
        # Note: last point of each section is not included, but it does not
        # matter because it is automatically the first point of the next,
        # EXCEPT the very last point
        for j in np.arange(len(xnew)):
            f = np.append(f, (((x[i+1] - xsamp[j + len(xsamp) - len(xnew)])*y[i]
            + (xsamp[j + len(xsamp) - len(xnew)] - x[i])*y[i+1]) / (x[i+1]-x[i])))
    xsamp = np.append(xsamp, x[-1]) # Adds in the last point because it is not
    # included in any interpolation
    f = np.append(f, y[-1])
    return xsamp, f

"""
cubspline - Cublic spline
Input: inp = a matrix containing all the x and y values (columns 0 and 1),
sorted in ascending order in x.
Output: array f whose elements i provide the y values for any x between x_i and
x_i+1 using a cubic spline method.
"""

def cubspline(inp, stepsize = 0.01, showcoeff = False, showb = False): # show the coefficient matrix
# or the array of constants
    x = inp[:, 0]
    y = inp[:, 1]
    fdashdash = np.array([0.]) # first second derivative = 0
    # Calculate the array contatining all the second derivatives of f at each
    # data point.
    coeff = np.zeros([len(x)-2, len(x)-2]) # Contains all the coefficients in
    # the fundamental equation in finding f". If we assume f"_0 = f"_n = 0
    # (natural spline), then this leaves us with n-1 unknowns in n-1 eqs.
    b = np.zeros(len(x)-2) # Contains the constant values
    for i in np.arange(len(x)-2):
        coeff[i, i] = (x[i+2] - x[i]) / 3 # Valid for all points
        if i > 0: # First point (i = 0) only has 2 coefficients as f"_0 = 0
            coeff[i, i-1] = (x[i+1] - x[i]) / 6
        if i < len(x)-3: # Last point (i = len(x) - 3) only has 2 coefficients as f"_n = 0
            coeff[i, i+1] = (x[i+2] - x[i+1]) / 6
        b[i] = (y[i+2] - y[i+1])/(x[i+2] - x[i+1]) - (y[i+1] -y[i])/(x[i+1] - x[i])
    if showcoeff == True:
        print(coeff)
    if showb == True:
        print(b)
    fdashdash = np.append(fdashdash, M2.solve(coeff, b))
    fdashdash = np.append(fdashdash, 0.) # last second derivative = 0

    # Now we solve for the cubic spline function f(x).
    f = np.array([])
    xsamp = np.array([])
    for i in np.arange(len(x)-1):
        xnew = np.arange(start = x[i], stop = x[i+1], step = stepsize)
        #xnew = np.linspace(x[i], x[i+1], 500)
        xsamp = np.append(xsamp, xnew)
        for j in np.arange(len(xnew)):
            A = (x[i+1] - xnew[j])/(x[i+1] - x[i])
            B = (xnew[j] - x[i])/(x[i+1] - x[i])
            C = 1./6.*(A**3-A)*((x[i+1] - x[i])**2)
            D = 1./6.*(B**3-B)*((x[i+1] - x[i])**2)
            f = np.append(f, A*y[i] + B*y[i+1] + C*fdashdash[i] + D*fdashdash[i+1])
    xsamp = np.append(xsamp, x[-1]) # Adds in the last point because it is not
    # included in any interpolation
    f = np.append(f, y[-1])
    return xsamp, f


import unittest
import pytest

import numpy as np

from yaonet.tensor import Tensor
from yaonet.basic_functions import *

class TestTensorBscFunstion(unittest.TestCase):
    def test_sin(self):
        # 90, 30, 150
        t1 = Tensor([np.pi/2., np.pi/6., np.pi*(5/6)], requires_grad=True)

        t2 = sin(t1)

        # assert
        np.testing.assert_array_almost_equal(t2.data, [1, 0.5, 0.5])

        t2.backward(Tensor([-1., -2., -3.]))

        assert t1.grad.tolist() == (np.array([-1., -2., -3]) * np.cos(t1.data)).tolist()

    def test_log(self):

        t1 = Tensor([1, 2, 4], requires_grad=True)

        t2 = log(2, t1)

        np.testing.assert_array_almost_equal(t2.data, [0, 1, 2])

        t2.backward(Tensor([-1., -2., -3.]))

        assert t1.grad.tolist() == (np.array([-1., -2., -3]) * (1 / (t1.data * np.log(2)))).tolist()


#!/usr/bin/env python

"""
Machine Learning Online Class Coursera
Exercise 1: Linear regression with multiple variables
Modified with few changes,Solved the issue
Modified by github.com/utkarshmani1997
"""

# Initialization
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D
from matplotlib import cm
from ex1_utils import *

## ================ Part 1: Feature Normalization ================

print('Loading data ...','\n')

## Load Data
print('Plotting Data ...','\n')

data = pd.read_csv("ex1data2.txt",names=["size1","bedrooms","price"])
s = np.array(data.size1)
b = np.array(data.bedrooms)
p = np.array(data.price)
m = len(b) # number of training examples

# Design Matrix
s = np.vstack(s)
b = np.vstack(b)
X = np.hstack((s,b))

# Print out some data points
print('First 10 examples from the dataset: \n')
print(" size = ", s[:10],"\n"," bedrooms = ", b[:10], "\n")

input('Program paused. Press enter to continue.\n')

# Scale features to zero mean and standard deviation of 1
print('Normalizing Features ...\n')

X = featureNormalize(X)

# Add intercept term to X
X = np.hstack((np.ones_like(s),X))

## ================ Part 2: Gradient Descent ================

print('Running gradient descent ...\n')

# Choose some alpha value
alpha = 0.05
num_iters = 400

# Init Theta and Run Gradient Descent
theta = np.zeros(3)

# Multiple Dimension Gradient Descent
theta, hist = gradientDescentMulti(X, p, theta, alpha, num_iters)

# Plot the convergence graph
fig = plt.figure()
ax = plt.subplot(111)
plt.plot(np.arange(len(hist)),hist ,'-b')
plt.xlabel('Number of iterations')
plt.ylabel('Cost J')
plt.show()

# Display gradient descent's result
print('Theta computed from gradient descent: \n')
print(theta,'\n')

# Estimating the price of a house on the given size in square feet and no of bedrooms
siz=int(input("Program paused. Enter the size of house in sq-ft:"))
br=int(input("Program paused. Enter the no of bedrooms:"))

# Recall that the first column of X is all-ones. Thus, it does
# not need to be normalized.
normalized_specs = np.array([1,((siz-s.mean())/s.std()),((br-b.mean())/b.std())])
price = np.dot(normalized_specs,theta)

print('Predicted price of house with size {} and {} no of bedrooms (using gradient descent):\n '.format(siz, br),
      price)

input('Program paused. Press enter to continue.\n')

## ================ Part 3: Normal Equations ================
print('Solving with normal equations...\n')

data = pd.read_csv("ex1data2.txt",names=["sz","bed","price"])
s = np.array(data.sz)
b = np.array(data.bed)
p = np.array(data.price)
m = len(b) # number of training examples

# Design Matrix
s = np.vstack(s)
b = np.vstack(b)
X = np.hstack((s,b))

# Add intercept term to X
X = np.hstack((np.ones_like(s),X))

# Calculate the parameters from the normal equation
theta = normalEqn(X, p)

# Display normal equation's result
print('Theta computed from the normal equations: \n')
print(theta)
print('\n')

# Estimate the price of a house, by Normal equations
price = np.dot([1,siz,br],theta) # You should change this

print('Predicted price of house with size = {} and {} no of bedrooms (Normalization):\n '.format(siz, br),
      price)

# vim: tabstop=8 expandtab shiftwidth=4 softtabstop=4


import numpy as np
#!----Relu activation function-----!

inputs = [1, 2, -2, 100, -130, 22, 0, 1, 4]
outputs =[]
for i in inputs:
    if i > 0:
        outputs.append(i)
    else:
        outputs.append(0)

print(outputs)

# print( np.exp(3))
# print(2.718**3)

#another way
inputs = [1, 2,-2, 100, -130,22,0,1,4]
outputs =[]
for i in inputs:
        outputs.append(max(0,i))
print(outputs)

#another way using numpy
outputs=np.maximum(0,inputs)
print(outputs)

#                 !-----soft_max activation function------!
outputs =[4.8, 1.21, 2.385]
E = 2.71828
exp_values = []
for values in outputs:
    exp_values.append(E**values)

print(exp_values)

norm_base = sum(exp_values)

norm_values = []

for n_value in exp_values:
    norm_values.append(n_value / norm_base)

print("CE", norm_values)

#Do it with numpy
outputs =([[1.8, 1.21, 3.385], [2, 1.21, 2.385],[3, 1.21, 2.385]])
exp_values = np.exp(outputs)
probabilities = exp_values / np.sum(exp_values, axis=1, keepdims=True)
print("Hello", probabilities)

np.exp(1000)  #doesn't take maximum number

# for this reason write code as

exp_values = np.exp(outputs-np.max(outputs, axis=1, keepdims=True))
print("Exponential values",exp_values)
probabilities = exp_values /np.sum(exp_values, axis =1, keepdims=True)
print("Hi", probabilities)


import numpy as np

# Longest Palindrome Subsequence

def lps(input_string):
    # The function should return one value: the LPS length for the given input string

    strlen = len(input_string)

    lps_matrix = np.zeros((strlen, strlen))
    np.fill_diagonal(lps_matrix, 1)  # main diagonal

    for row in range(strlen - 2, -1, -1):
        for column in range(row + 1, strlen):
            if input_string[row] == input_string[column]:
                lps_matrix[row, column] = lps_matrix[row + 1, column - 1] + 2
            else:
                lps_matrix[row, column] = max(lps_matrix[row + 1, column], lps_matrix[row, column - 1])

    return lps_matrix[0, -1]

def test_function(test_case):
    string = test_case[0]
    solution = test_case[1]
    output = lps(string)
    print(output)
    if output == solution:
        print("Pass")
    else:
        print("Fail")

string = 'BxAoNxAoNxA'
solution = 5  # ANANA
test_case = [string, solution]
test_function(test_case)

string = 'BANANO'
solution = 3  # NAN
test_case = [string, solution]
test_function(test_case)

string = "TACOCAT"
solution = 7  # TACOCAT
test_case = [string, solution]
test_function(test_case)

string = "COOL"
solution = 2  # OO
test_case = [string, solution]
test_function(test_case)


import numpy as np

# Write a function that takes as input a list of numbers, and returns
# the list of values given by the softmax function.
def softmax(L):
    return np.exp(L) / np.sum(np.exp(L))

res = softmax([1, 8, 16])
print(res, '\nsum =', sum(res))


import numpy as np
from grid_world import GridworldEnv

env = GridworldEnv()

def policy_eval(policy, env, discount_factor=1.0, theta=0.00001):
    """
    Evaluate a policy given an environment and a full description of the environment's dynamics.

    Args:
        policy: [S, A] shaped matrix representing the policy.
        env: OpenAI env. env.P represents the transition probabilities of the environment.
            env.P[s][a] is a list of transition tuples (prob, next_state, reward, done).
            env.nS is a number of states in the environment.
            env.nA is a number of actions in the environment.
        theta: We stop evaluation once our value function change is less than theta for all states.
        discount_factor: Gamma discount factor.

    Returns:
        Vector of length env.nS representing the value function.
    """
    # Start with a random (all 0) value function
    V = np.zeros(env.nS)
    while True:
        max_v_changed = 0
        for s in range(env.nS):
            v = 0
            for a, action_prob in enumerate(policy[s]):
                for prob, next_state, reward, done in env.P[s][a]:
                    v += action_prob * prob * \
                        (reward + discount_factor * V[next_state])

            max_v_changed = max(max_v_changed, np.abs(v - V[s]))
            V[s] = v
        # print(max_v_changed)
        if max_v_changed < theta:
            break

    return np.array(V)

if __name__ == '__main__':

    random_policy = np.ones([env.nS, env.nA]) / env.nA
    v = policy_eval(random_policy, env)

    print("Value Function:")
    print(v)
    print("")

    print("Reshaped Grid Value Function:")
    print(v.reshape(env.shape))
    print("")

    expected_v = np.array([0, -14, -20, -22, -14, -18, -20, -20, -20, -20, -18, -14, -22, -20, -14, 0])
    np.testing.assert_array_almost_equal(v, expected_v, decimal=2)


#   # -*- coding: UTF-8 -*-
#   trial on the : Satomi machine
#   Created by Ush on 2018/3/16
#   Project name :  Class3_FindingRoots
#   Please contact CHIH, HSIN-CHING/D0631008 when expect to refer this source code.
#   NOTE : no liability on any loss nor damage by using this source code. it is your own risk.

import numpy as np
import math

def FUNC(type, x):
    F_select = 0
    if (type == 1):  # function1 is f(x)=x^2-1
        F_select = x ** 2 - 2
    elif (type == 2):
        F_select = x ** 5 + x ** 3 + 3
    elif (type == 3):
        F_select = x ** 3 - 2 * math.sin(x) - 1
    elif (type == 4):
        F_select = 0.5 * math.tan(x) - math.tanh(x)
    else:
        print("invalid function")
    return (float(F_select))

def Bisection(f, left, right, eps):
    iteration = 0
    while (math.fabs(right - left) > eps * math.fabs(right)):
        middle = (left + right) / 2
        if (np.sign(FUNC(f, middle)) == np.sign(FUNC(f, right))):
            right = middle
        else:  # root is  founded, move left = middle value
            left = middle
        iteration = iteration + 1
        # print(iteration, "\t", right, "\t", FUNC(f, right))
    return (right, iteration)

def Secant(f, left, right, eps):
    iteration = 0
    while (math.fabs(right - left) > eps * math.fabs(right)):
        middle, left = left, right
        right = right + ((right - middle) / ((FUNC(f, middle)) / (FUNC(f, right)) - 1))
        iteration = iteration + 1
        # print(iteration, left, FUNC(f, left), right, FUNC(f, right))
    return (right, iteration)

# IQI : Inverse Quadratic Interpolation
# https://en.wikipedia.org/wiki/Inverse_quadratic_interpolation
# https://nickcdryan.com/2017/09/13/root-finding-algorithms-in-python-line-search-bisection-secant-newton-raphson-boydens-inverse-quadratic-interpolation-brents/
def PolyInterp(f, p0, p1, p2):
    a, b, c = FUNC(f, p0), FUNC(f, p1), FUNC(f, p2)
    L0 = (p0 * b * c) / ((a - b) * (a - c))
    L1 = (p1 * a * c) / ((b - a) * (b - c))
    L2 = (p2 * b * a) / ((c - a) * (c - b))
    return (L0 + L1 + L2)

def IQI(f, left, middle, right, eps):
    iteration = 0
    while (math.fabs(right - middle) > eps * math.fabs(right)):
        x = PolyInterp(f, left, middle, right)
        left, middle, right = middle, right, x
        iteration = iteration + 1
        # print(iteration, right, FUNC(f, right))
    return (right, iteration)

# https://en.wikipedia.org/wiki/Brent%27s_method#Dekker.27s_method
# it is Dekker's method in 1969
# https://blogs.mathworks.com/cleve/2015/10/12/zeroin-part-1-dekkers-algorithm/
# https://cocalc.com/share/ab93d447b6728ae561bf1f9e18f0b103316d715f/Efficiency%20of%20Standard%20and%20Hybrid%20Root%20Finding%20Methods.sagews?viewer=share
def Mixed_Scant_Bisection(f, left, right, eps):
    fa = FUNC(f, left)
    fb = FUNC(f, right)
    if (np.sign(fa) == np.sign(fb)):
        print(" Interval in ", left, " and ", right, " is in the same sign signal")
    #print("initial-value from :", right, "\t", fb)
    #print("N-Type-Step\t\tbn\t\t\t\t\t\tf(bn) ")
    iteration = 1
    # left is the previous value of right and [right, middle] always contains the zero.
    middle, fc = left, fa
    while (math.fabs(right - left) > eps):
        if (np.sign(fb) == np.sign(fc)):
            middle, fc = left, fa
        if (math.fabs(fc) < math.fabs(fb)):  # Swap to insure f(b) is the smallest value so far.
            left, fa = right, fb
            right, fb = middle, fc
            middle, fc = left, fa
        mid_point = float(right + middle) / 2  # BiSection ( Step1 )
        # secant_left/secant_right is the the secant step.
        secant_left = (right - left) * fb
        if (secant_left >= 0):
            secant_right = fa - fb  # swap like BiSection
        else:
            secant_right = -1 * (fa - fb)  # swap like BiSection
            secant_left = -1 * secant_left
        left, fa = right, fb  # prepare for the next iteration process points.
        if (secant_left <= ((mid_point - right) * secant_right)):
            right = right + (secant_left / secant_right)  # Secant
            fb = FUNC(f, right)
            #print(iteration, "Secant-step", right, "\t", fb)
        else:
            right = mid_point  # BiSection
            fb = FUNC(f, right)
            #print(iteration, "Bisect-step", right, "\t", fb)
        iteration = iteration + 1
    return (right, iteration)

# https://cocalc.com/share/ab93d447b6728ae561bf1f9e18f0b103316d715f/Efficiency%20of%20Standard%20and%20Hybrid%20Root%20Finding%20Methods.sagews?viewer=share
def brentsMethod(f, a, b, accuracy):
    '''
    Code inspired by:
    https://en.wikipedia.org/wiki/Brent's_method (The pseudocode was very helpful in translating this to Python)
    http://blogs.mathworks.com/cleve/2015/10/26/zeroin-part-2-brents-version/
    Function that computes an approximate root
    for a given function using Dekker's method.
    args:
        f: a function
        a, b: an initial value, most efficient when close to the root
        iterations: number of times to run the loop
    output:
        a list that contains an approximate root s and the number
        of iterations required
    '''
    fa = FUNC(f, a)
    fb = FUNC(f, b)
    # When this is set, we use the bisection method
    bisection = True
    if fa * fb >= 0:
        raise Exception("Invalid inputs for a and b, require sign change")
    c = a
    s = 0
    d = 0
    iteration = 0
    while abs(b - a) > accuracy:
        # quadratic interpolation
        if (FUNC(f, a) != FUNC(f, c)) and (FUNC(f, b) != FUNC(f, c)):
            s = PolyInterp(f, a, b, c)      # IQI core
            #print(iteration, "IQI-step", s, "\t", FUNC(f,s))
        else:
            s = ((a * FUNC(f, b)) - (b * FUNC(f, a))) / (FUNC(f, b) - FUNC(f, a))# secant
            #print(iteration, "Secant-step", s, "\t", FUNC(f, s))
        if brentConditional(s, a, b, c, d, bisection, accuracy):
            s = (a + b) / 2.0 # bisection
            bisection = True
        else:
            bisection = False
        d = c
        c = b
        if (FUNC(f, a) * FUNC(f, s) < 0):   #step (3a)
            b = s
            fb = FUNC(f, s)
        else:
            a = s
            fa = FUNC(f, s)
        if abs(fa) < abs(fb):
            temp = a
            a = b
            b = temp
            temp2 = fa
            fa = fb
            fb = temp2
        iteration = iteration + 1
    return (s, iteration)

def brentConditional(s, a, b, c, d, mflag, accuracy):
    if ((s < (3 * a + b) * 0.25) or
            (mflag and (abs(s - b) >= (abs(b - c) * 0.5)) or
                 (not mflag and (abs(s - b) >= (abs(c - d) * 0.5)) or
                      (mflag and (abs(b - c) < accuracy)) or
                      (not mflag and (abs(c - d) < accuracy))))):
        return True
    return False

import matplotlib.pyplot as plt
from scipy import optimize

def f1(x):
    return (x ** 2 - 2)

def main():
    print
    "let's make it happen!"
    # eps function in matlab
    # Python code expression : np.spacing(1)
    eps = np.spacing(1)
    eps = 1e-6
    print("Python eps : " + str(eps))

    # formula 0
    print("\nFormula 0 : x ** 2 - 2 \t\t\t (root, iteration times)")
    ITR, Root = Bisection(1, 1, 2, eps)
    print("BiSection Result:\t\t\t\t", Bisection(1, 1, 2, eps))
    print("Secant Result:\t\t\t\t\t", Secant(1, 1, 2, eps))
    print("IQI Result:\t\t\t\t\t\t", IQI(1, 1, 1.5, 2, eps))
    print("Mixed_Scant_Bisection Result:\t", Mixed_Scant_Bisection(1, 1, 2, eps))
    print("Brent Result:\t\t\t\t\t", brentsMethod(1, 1, 2, eps))

    # formula 1
    print("\nFormula 1 : x ** 5 + x ** 3 + 3  (root, iteration times)")
    print("BiSection Result:\t\t\t\t", Bisection(2, -2, 0, eps))
    print("Secant Result:\t\t\t\t\t", Secant(2, -2, 0, eps))
    print("IQI Result:\t\t\t\t\t\t", IQI(2, -2, -1, 0, eps))
    print("Mixed_Scant_Bisection Result:\t", Mixed_Scant_Bisection(2, -2, 0, eps))
    print("Brent Result:\t\t\t\t\t", brentsMethod(2, -2, 0, eps))

    # formula 2
    print("\nFormula 2 : x ** 3 - 2 * math.sin(x) - 1\t  (root, iteration times)")
    print("BiSection Result:\t\t\t\t", Bisection(3, 0, 2, eps))
    print("Secant Result:\t\t\t\t\t", Secant(3, 0, 2, eps))
    print("IQI Result:\t\t\t\t\t\t", IQI(3, 0, 1, 2, eps))
    print("Mixed_Scant_Bisection Result:\t", Mixed_Scant_Bisection(3, 0, 2, eps))
    print("Brent Result:\t\t\t\t\t", brentsMethod(3, 0, 2, eps))

    # formula 3
    print("\nFormula 3 : 0.5 * math.tan(x) - math.tanh(x)\t  (root, iteration times)")
    A = 1e-3
    B = math.pi / 2
    print("BiSection Result:\t\t\t\t", Bisection(4, A, B, eps))
    print("Secant Result:\t\t\t\t\t", Secant(4, A, B, eps))
    print("IQI Result:\t\t\t\t\t\t", IQI(4, A, B, B, eps))
    print("Mixed_Scant_Bisection Result:\t", Mixed_Scant_Bisection(4, A, B, eps))
    print("Brent Result:\t\t\t\t\t", brentsMethod(4, A, B, eps))

if __name__ == "__main__":
    main()


import numpy as np

class Secant(object):
    def __init__(self, f=None, **extras):
        """
        f : a function f(x) that returns 'y', the function applied to it
        """
        self.f = f

    def solve(self, init_a=None, init_b=None, tol=None, max_it=50, **extras):
        """fast implementation of the secant method
        init_a : bottom bound of the initial interval
        init_b : top bound of the initial interval
        tol : maximum absolute error after which the method should stop. (e.g. 1e-15)
        max_it : maximum number of iterations after which the method should stop
        """
        xo = init_a
        x = init_b
        xn = 0

        i = 0
        while i < max_it:
            xn = x - ( self.f(x) * (x - xo) ) / ( self.f(x) - self.f(xo) )
            if abs(xn - x) < tol:
                break
            xo = x
            x = xn
            i+=1
        return x

    def vsolve(self, init_a=None, init_b=None, tol=None, max_it=50, **extras):
        """verbose implementation of the secant method
        init_a : first iteration estimate
        init_b : second iteration estimate
        tol : maximum absolute error after which the method should stop. (e.g. 1e-15)
        max_it : maximum number of iterations after which the method should stop
        """

        xo = init_a
        x = init_b
        xn = 0
        i = 0
        iteration_list = []
        dist_list = []
        estimate_list = []

        while i < max_it:
            dist = abs(x-xo)
            iteration_list.append(i)
            dist_list.append(dist)
            estimate_list.append(x)
            xn = x - ( self.f(x) * (x - xo) ) / ( self.f(x) - self.f(xo) )

            if abs(xn - x) < tol:
                break

            xo = x
            x = xn
            i+=1

        return iteration_list, estimate_list, dist_list

    def __repr__(self):
        # return "Secant "
        raise NotImplementedError

    def __str__(self):
        return "Secant method"


import numpy as np
import time
import scipy.linalg

def GENP(A, b):

    n =  len(A)
    if b.size != n:
        raise ValueError("Invalid argument", b.size, n)
    for pivot_row in xrange(n-1):
        for row in xrange(pivot_row+1, n):
            multiplier = A[row][pivot_row]/A[pivot_row][pivot_row]
            A[row][pivot_row] = multiplier
            for col in xrange(pivot_row + 1, n):
                A[row][col] = A[row][col] - multiplier*A[pivot_row][col]

            b[row] = b[row] - multiplier*b[pivot_row]
    print A
    print b
    x = np.zeros(n)
    k = n-1
    x[k] = b[k]/A[k,k]
    while k >= 0:
        x[k] = (b[k] - np.dot(A[k,k+1:],x[k+1:]))/A[k,k]
        k = k-1
    return x

def GEPP(A, b):

    n =  len(A)
    if b.size != n:
        raise ValueError("Invalid argument", b.size, n)

    for k in xrange(n-1):
        maxindex = abs(A[k:,k]).argmax() + k
        if A[maxindex, k] == 0:
            raise ValueError("singular")
        #Swap rows
        if maxindex != k:
            A[[k,maxindex]] = A[[maxindex, k]]
            b[[k,maxindex]] = b[[maxindex, k]]
        for row in xrange(k+1, n):
            multiplier = A[row][k]/A[k][k]

            A[row][k] = multiplier
            for col in xrange(k + 1, n):
                A[row][col] = A[row][col] - multiplier*A[k][col]

            b[row] = b[row] - multiplier*b[k]
    print A
    print b
    x = np.zeros(n)
    k = n-1
    x[k] = b[k]/A[k,k]
    while k >= 0:
        x[k] = (b[k] - np.dot(A[k,k+1:],x[k+1:]))/A[k,k]
        k = k-1
    return x

if __name__ == "__main__":
    A = np.array([[1e-20,0.,1.],
                  [1,1e+20,1.],
                  [0,1,-1],
                  ])
    b =  np.array([[0.],[0.],[1.]])
    print GENP(np.copy(A), np.copy(b))
    print GEPP(A,b)
    print '\n\nresidual', scipy.linalg.norm(np.dot(A, GEPP(A,b)) - b)/scipy.linalg.norm(A)
    tic = time.clock()
    toc = time.clock()
    print "Processing time is : %f" %(toc - tic)


import numpy as np

def sigmoid(Z):
    return 1/(1+np.exp(-Z))

def hard_sigmoid(Z):
    y = 0.2 * Z + 0.5
    return np.clip(y, 0, 1)

def relu(Z):
    return (Z > 0) * Z

def softmax(Z):
    e_x = np.exp(Z - np.max(Z))
    return e_x / e_x.sum(axis = 1)

def backward_sigmoid(a):
    return a * (1 - a)

def backward_tanh(Z):
    return 1-(np.tanh(Z))**2

def backward_relu(Z):
    mul = np.ones(Z.shape)
    return (Z > 0) * mul

def backward_softmax(t_hat, i):
    t_hat[i] = t_hat[i] - 1
    return t_hat

# based on keras drop out
def dropout(input, level):
    noise_shape = input.shape # (Tx, n_a)
    noise = np.random.choice([0,1], noise_shape, replace = True, p=[level, 1-level])
    return input * noise / (1 - level), noise / (1 - level)


import torch
import numpy as np
import matplotlib.pylab as plt

def numerical_diff(f, x):
    h = 1e-4
    return (f(x + h) - f(x)) / h

def function_1(x):
    return 0.01 * x **2 + 0.1 * x

x = np.arange(0.0, 20.0, 0.1)
y = function_1(x)
plt.xlabel("x")
plt.ylabel("f(x)")
plt.plot(x, y)
plt.show()

print(numerical_diff(function_1, 5))
print(numerical_diff(function_1, 10))
print("---------------------------------------------------")

def function_2(x):
    return x[0] ** 2 + x[1] ** 2

# center numerical diff
def numerical_gradient(f, x):
    h = 1e-4
    grad = torch.zeros_like(x)

    for idx in range(x.size()[0]):
        tmp_val = x[idx].item()

        x[idx] = tmp_val + h
        fxh1 = f(x)

        x[idx] = tmp_val - h
        fxh2 = f(x)

        grad[idx] = (fxh1 - fxh2) / (2*h)
        x[idx] = tmp_val

    return grad
#
#
# print(numerical_gradient(function_2, torch.tensor([3.0, 4.0])))
# print(numerical_gradient(function_2, torch.tensor([0.0, 2.0])))
# print(numerical_gradient(function_2, torch.tensor([3.0, 0.0])))
# print("---------------------------------------------------")

def numpy_numerical_gradient(f, x):
    h = 1e-4
    grad = np.zeros_like(x)

    for idx in range(x.size):
        tmp_val = x[idx]

        x[idx] = tmp_val + h
        fxh1 = f(x)

        x[idx] = tmp_val - h
        fxh2 = f(x)

        grad[idx] = (fxh1 - fxh2) / (2*h)
        x[idx] = tmp_val

    return grad

print(numpy_numerical_gradient(function_2, np.array([3.0, 4.0])))
print(numpy_numerical_gradient(function_2, np.array([0.0, 2.0])))
print(numpy_numerical_gradient(function_2, np.array([3.0, 0.0])))

# gradient descent
def numpy_gradient_descent(f, init_x, lr=0.001, step_num=100):
    x = init_x

    for i in range(step_num):
        # grad = numerical_gradient(f, x)
        grad = numpy_numerical_gradient(f, x)
        x -= lr * grad

    return x

init_x = np.array([-3.0, 4.0])
print(numpy_gradient_descent(function_2, init_x, lr=10.0, step_num=100))

def gradient_descent(f, init_x, lr=0.001, step_num=100):
    x = init_x

    for i in range(step_num):
        grad = numerical_gradient(f, x)
        x -= lr * grad

    return x

init_x = torch.tensor([-3.0, 4.0], dtype=torch.float64)
print(gradient_descent(function_2, init_x, lr=0.1, step_num=100))
print("---------------------------------------------------")

print(gradient_descent(function_2, init_x, lr=10.0, step_num=100))
print(gradient_descent(function_2, init_x, lr=1e-10, step_num=100))
print("---------------------------------------------------")