"""
demo_constrained_optimization_nonlinear_equality_constraints.py

Minimize a (quadratic) function of two variables:
Minimize f(X), where X = [x0, x1].T;
X* is MIN;

m = number of equality constraints;
n = number of parameters;

Objective function:
    Minimize a (quadratic) function of two variables: f(X);
    X = [x1, ..., xn].T is vector of variables;

Box constraints:
    X_lower = [x1_lower, ..., xn_lower].T; lower bounds for X;
    X_upper = [x1_upper, ..., xn_upper].T; upper bounds for X;

(Non)linear equality constraints:
    Subject to (non)linear equality constraints: h(X);
    h(X) = [h1(X), ..., hn(X)].T is vector of constraints;
    m < n;

First order Lagrange algorithm

Lagrangian function:
l(X, Lambda) = f(X) + Lambda.T @ h(X);
Lambda = [lambda1, ..., lambdan].T is vector of langrange multipliers;

Update equation for X:
    X = X_old + alpha * d;
    alpha is step size (<<1);
    d is direction:
        d = - (Df(X_old) + Dh(X_old).T @ Lambda);

Update equation for Lambda:
    Lambda = Lambda_old + beta * d;
    beta is step size (<<1);
    d is direction:
        d = h(X_old);

If alpha and beta are small enough, the algorithm will converge to a fixed point, which satisfies the Lagrange condition.
"""

import numpy as np
import matplotlib.pyplot as plt
from matplotlib import cm
from matplotlib.ticker import LinearLocator, FormatStrFormatter
from mpl_toolkits.mplot3d import Axes3D
from first_order_lagrangian_algorithm import first_order_lagrangian_algorithm
from print_report import print_report
from plot_progress_y import plot_progress_y
from plot_progress_x import plot_progress_x
import time

reg_coeff =10; # regularization coefficient
# Minimize the following objective function:
# X = [x0, x1].T;
# f(X) = 1 / 2 * X.T @ Q @ X + X.T @ B + C + reg_coeff * X.T @ X;
Q = np.array([[8, 2 * np.sqrt(2)], [2 * np.sqrt(2), 10]]);
B = np.array([[3], [6]]);
C = 24;
func = lambda X : 1 / 2 * X.T @ Q @ X + X.T @ B + C + reg_coeff * X.T @ X;
Dy = lambda X : Q @ X + B + 2 * reg_coeff * X;

# Box constraints:
X_lower = np.array([[-20], [-20]]); # X lower bound
X_upper = np.array([[20], [20]]); # X upper bound

# (Non)linear equality constraints:
# h(X) = x1 * x2 - 20 = 0
h = lambda X : X[0] * X[1] - 20;
Dh = lambda X : np.array([X[1], X[0]]);
# Initial guess (subject to the equality constraints):
X0 = np.array([[-50], [-0.4]]);
Lambda0 = np.array([[0]]);

fig = plt.figure();
X_data = np.arange(-100, 100, 5);
Y_data = np.arange(-100, 100, 5);
Z_data = np.zeros(shape = (X_data.size,Y_data.size));
for iX in range (X_data.size):
    for iY in range (Y_data.size):
        Z_data[iX][iY] = func(np.array([X_data[iX], Y_data[iY]]));
X_data, Y_data = np.meshgrid(X_data, Y_data);

# Plot the surface
ax = fig.add_subplot(1, 2, 1, projection ='3d')
surf = ax.plot_surface(X_data, Y_data, Z_data, cmap = cm.coolwarm,
                       linewidth = 0, antialiased = False)
ax.set_xlabel('X')
ax.set_xlim(-100, 100)
ax.set_ylabel('Y')
ax.set_ylim(-100, 100)
plt.title('Objective function: surface plot')

# Plot the contour
ax = fig.add_subplot(1, 2, 2)
cset = ax.contour(X_data, Y_data, Z_data, 50, cmap = cm.coolwarm);
ax.set_xlabel('X')
ax.set_xlim(-100, 100)
ax.set_ylabel('Y')
ax.set_ylim(-100, 100)
plt.title('Objective function: contour plot');
plt.show();

# First order Lagrangian algorithm
print('***********************************************************************');
print('First order Lagrangian algorithm');
N_iter_max = 10000;
tolerance_x = 10e-6;
tolerance_y = 10e-6;
options = {'tolerance_x' : tolerance_x, 'tolerance_y' : tolerance_y, 'N_iter_max' : N_iter_max, 'x_lower' : X_lower, 'x_upper' : X_upper};
start = time.time();
X, report = first_order_lagrangian_algorithm(X0, Lambda0, func, Dy, Dh, h, options);
end = time.time();
print_report(func, report);
# Plot path to X* for Y
algorithm_name = 'First order Lagrangian algorithm';
plot_progress_y(algorithm_name, report);
# Plot path to X* for X
plot_progress_x(X_data, Y_data, Z_data, algorithm_name, report);
print('Elapsed time [s]: %0.5f' % (end - start));
print('***********************************************************************\n');


# -*- coding: utf-8 -*-
#
#  Activate Function Model
#
#  @auth whuang022ai
#

import math
import numpy as np
from abc import ABCMeta, abstractmethod
import matplotlib.pyplot as plt
from enum import Enum

class Activate_Function_Type(Enum):

    sigmoid = 'sigmoid'
    tanh = 'tanh'
    relu = 'relu'
    leaky_relu = 'leaky-relu'
    identity = 'identity'

class Activate_Function(metaclass=ABCMeta):

    @abstractmethod
    def f(self, x):
        pass

    @abstractmethod
    def df(self, y):
        pass

    def plot_fx_dfx(self, min, max, inter, title):

        x = np.arange(min, max, inter)
        f_vect = np.vectorize(self.f)
        df_vect = np.vectorize(self.df)
        plt.figure(title)
        plt.subplot(211)
        plt.ylabel('f(x)')
        plt.plot(x, f_vect(x))
        plt.grid(True)
        plt.subplot(212)
        plt.ylabel('d f(x)')
        plt.plot(x, df_vect(f_vect(x)), color='red',
                 linewidth=1.0, linestyle='--')
        plt.grid(True)
        plt.show()

    @abstractmethod
    def show_plot(self):
        pass

class Activate_Function_Sigmoid(Activate_Function):
    # ref: https://timvieira.github.io/blog/post/2014/02/11/exp-normalize-trick/
    def f(self,x):

        if x >= 0:
            z = np.exp(-x)
            return 1 / (1 + z)
        else:
            z = np.exp(x)
            return z / (1 + z)

    def df(self, y):
        return y*(1-y)

    def show_plot(self):
        self.plot_fx_dfx(-5.0, 5.0, 0.1, 'The Sigmoid Function')

class Activate_Function_Tanh(Activate_Function):

    def f(self, x):
        return np.tanh(x)

    def df(self, y):
        return 1-y**2

    def show_plot(self):
        self.plot_fx_dfx(-5.0, 5.0, 0.1, 'The Tanh Function')

class Activate_Function_Relu(Activate_Function):

    def __init__(self):
        self.lrelu = Activate_Function_LeakyRelu()
        self.lrelu.alpha = 0.0

    def f(self, x):
        return self.lrelu.f(x)

    def df(self, y):
        return self.lrelu.df(y)

    def show_plot(self):
        self.plot_fx_dfx(-5.0, 5.0, 0.01, 'The ReLU Function')

class Activate_Function_LeakyRelu(Activate_Function):

    def __init__(self):
        self.alpha = 0.1

    def f(self, x):
        if x > 0:
            return x
        else:
            return (self.alpha*x)

    def df(self, y):
        if y > 0:
            return 1.0
        else:
            return self.alpha

    def show_plot(self):
        self.plot_fx_dfx(-5.0, 5.0, 0.01, 'The Leaky ReLU Function')

class Activate_Function_Identity(Activate_Function):

    def f(self, x):
        return x

    def df(self, y):
        return 1.0

    def show_plot(self):
        self.plot_fx_dfx(-5.0, 5.0, 0.01, 'The Identity Function')

class Activate_Function_Generator():

    def __init__(self, name):

        if isinstance(name, Activate_Function_Type):
            name = str(name.value)
            self.__genfromname(name)
        else:
            self.__genfromname(name)

    def __genfromname(self, name):

        if name == 'sigmoid':
            self.get = Activate_Function_Sigmoid()
        elif name == 'tanh':
            self.get = Activate_Function_Tanh()
        elif name == 'relu':
            self.get = Activate_Function_Relu()
        elif name == 'leaky-relu':
            self.get = Activate_Function_LeakyRelu()
        elif name == 'identity':
            self.get = Activate_Function_Identity()


import numpy as np
import matplotlib.pyplot as plt
#
# test the fourier transform and inverse transform in the numpy.fft library
# define a simple function, compute and plot its fft,
# then compute and plot the inverse fft of the fft
# this is a good way to test the fft normalization
#
pathout1='/Users/riser/Desktop/ocean.569A/fft.test.out.jpg'
pathout2='/Users/riser/Desktop/ocean.569A/inverse.fft.test.out.jpg'
#
# set the number of points and number of frequencies
#
nn=1000
mm=int(nn/2+1)
#
# define the arrays
#
tt0=np.zeros(nn)
tt1=np.zeros(nn)
yy0=np.zeros(nn)
yy1=np.zeros(nn)
zz=np.zeros(mm)
freq_rad=np.zeros(mm)
freq=np.zeros(mm)
#
# define some constants
#
pi=np.pi
amp=1.0
n_fac=50.
omega0=2.*pi/nn
#
# set the time and amplitude coordinates
#
for i in range(0,nn):
    tt0[i]=i # t
    yy0[i]=amp*np.sin(2.*pi*tt0[i]/n_fac)    #y_sin
#
# set the frequency values
#
for i in range(0,mm):
    freq_rad[i]=i*omega0
    freq[i]=freq_rad[i]/(2.*pi)
#
# carry out the fourier transform
# convert from real-imaginary parts to amplitude and phase
#
zz=np.fft.rfft(yy0,n=nn)/nn
zz_real=zz.real
zz_imag=zz.imag
zz_mag=np.sqrt(zz_real**2+zz_imag**2)
zz_phase=np.arctan(zz_imag/zz_real)
#
# plot the magnitude of the transform as a function of frequency
#
sz=15
fig=plt.figure(figsize=(10,5))
plt.xlim([0,0.5])
plt.ylim([0,1])
plt.plot(freq_rad,zz_mag,color='firebrick')
plt.plot([0,pi],[0.5,0.5],'--k')
plt.plot([0.12566,0.12566],[0,1],'--k')
plt.xlabel('$\omega$ (cycles)  (continues to $\pi$)',fontsize=sz)
plt.ylabel('Fourier coefficient magnitude (FFT/$\it{n}$)',fontsize=0.85*sz)
plt.text(0.132,0.55,'$\omega$=0.12566, $\it{f}$=0.02')
plt.grid(color='blue')
plt.title('FFT of sin(2$\pi$$\it{t}$/50)')
#plt.savefig(pathout1)
plt.show()
#
# now test the inverse transform
#
yy1=nn*np.fft.irfft(zz,n=nn)
#
fig=plt.figure(figsize=(10,5))
plt.xlim([0,1000])
plt.ylim([-1,1])
plt.plot(yy1)
plt.plot([0,1000],[0,0],'--m')
per=50.
plt.plot([per,per],[-1,1],'--k')
plt.xlabel('Time',fontsize=sz)
plt.ylabel('Amplitude',fontsize=sz)
plt.text(35,-1.12,'50',fontsize=15)
plt.title('Inverse FFT of [FFT of sin(2$\pi$$\it{t}$/50)]')
#plt.savefig(pathout2)
plt.show()


import numpy as np
import matplotlib.pyplot as plt
import pandas as pd

# Exercise 1

A = np.array([[0.3,0.6,0.1],[0.5,0.2,0.3],[0.4,0.1,0.5]])
v = np.array([1/3,1/3,1/3])

v_dist = []
v_next = v
for i in range(1,25):
    v_current = v_next
    v_next = v_next*A
    v_dist.append(np.linalg.norm(v_next-v_current))

plt.plot(v_dist)
plt.show

# Exercise 2

N = 1000
Ys = np.random.randn(10000,N)
Ysum = np.sum(Ys, axis=1)
plt.hist(Ysum, bins=100)
plt.show()

Ysum.mean()
Ysum.var()

# Exercise 3

df = pd.read_csv("train.csv")

digit = 7
D = df[df['label'] == digit]
D_mean = D.mean().values
im = D_mean[1:]
im = im.reshape(28,28)
plt.imshow(im, cmap="gray")
plt.show()

# Exercise 4

im90 = np.rot90(im,3)
plt.imshow(im90, cmap="gray")
plt.show()

# Exercise 5

def is_symmetric(M):
    return np.abs(M-M.T).sum() == 0

M1 = np.array([[1,2],[3,4]])
M2 = np.array([[1,2],[2,1]])

is_symmetric(M1)
is_symmetric(M2)

# Exercise 6

XOR = np.random.random((5000,2))*2-1
z = (np.sign(XOR[:,0]*XOR[:,1])+1)/2
z = np.expand_dims(z, axis=0).T
XOR = np.append(XOR, z, axis=1)
plt.scatter(XOR[:,0], XOR[:,1], c=XOR[:,2], cmap=plt.cm.RdBu, alpha=0.5)
plt.axis('equal')
plt.show()

# Exercise 7

N = 2000
C = np.random.random((N,2))
s = np.sign(C[:,1]-0.5)
r0 = np.random.randn(N)
r = (s-3)/2*10+r0
x = np.cos(C[:,0]*2*np.pi)*r
y = np.sin(C[:,0]*2*np.pi)*r
plt.scatter(x, y, c=s, cmap=plt.cm.RdBu, alpha=0.5)
plt.axis('equal')
plt.show()

# Exercise 8

N = 2000
t6 = np.random.randint(6, size=N)
s = np.mod(t6,2)
t = 0.5+np.random.random(N)*2+np.random.randn(N)*0.1
r = t
a = (t+t6)*np.pi/3
r0 = r+np.random.randn(N)*0.1
x = np.cos(a)*r0
y = np.sin(a)*r0
plt.scatter(x, y, c=s, cmap=plt.cm.RdBu, alpha=0.5)
plt.axis('equal')
plt.show()

df = pd.DataFrame(data={'x1': x, 'x2': y, 'y': s})
df.to_csv('spiral.csv', index=False)


import math
import itertools
import numpy as np
import pandas as pd
from typing import Dict
from IPython.display import display

def cg_coefficient(
    j1, j2,
    m1, m2,
    j, m
):
    '''
    Compute the Clebsch-Gordan coefficient :math:`\langle j_1, j_2; m_1, m_2 | j_1, j_2; j, m \rangle`.
    References:
        - Section 3.8 of J. J. Sakurai and Jim Napolitano, "Quantum Mechanics", 2nd ed., Cambridge, 2017
        - https://en.wikipedia.org/wiki/Table_of_Clebsch–Gordan_coefficients
    '''

    if m != m1 + m2:
        return 0
    if j < abs(j1 - j2):
        return 0
    if j > j1 + j2:
        return 0

    # coefficient outside of the summation
    numerator = 2 * j + 1
    try:
        numerator *= math.factorial(j + j1 - j2)
        numerator *= math.factorial(j - j1 + j2)
        numerator *= math.factorial(j1 + j2 - j)
        numerator *= math.factorial(j + m)
        numerator *= math.factorial(j - m)
        numerator *= math.factorial(j1 - m1)
        numerator *= math.factorial(j1 + m1)
        numerator *= math.factorial(j2 - m2)
        numerator *= math.factorial(j2 + m2)
    except ValueError:  # invalid j1, j2, m1, m2, j, or m -> 0
        return 0

    denominator = math.factorial(j1 + j2 + j + 1)

    c = math.sqrt(numerator / denominator)

    # summation
    summation = 0
    # ranges of k
    k_min = max(0, -(j - j2 + m1), -(j - j1 - m2))
    k_max = min(j1 + j2 - j, j1 - m1, j2 + m2)
    k_max = max(0, k_max)  # avoid negative k's
    k_list = np.linspace(k_min, k_max, int(k_max - k_min + 1))

    for k in k_list:
        numerator = (-1) ** k

        denominator = math.factorial(k)
        denominator *= math.factorial(j1 + j2 - j - k)
        denominator *= math.factorial(j1 - m1 - k)
        denominator *= math.factorial(j2 + m2 - k)
        denominator *= math.factorial(j - j2 + m1 + k)
        denominator *= math.factorial(j - j1 - m2 + k)

        summation += numerator / denominator

    return c * summation

def cg_table(j1, j2, m) -> pd.DataFrame:
    '''
    Returns the Clebsch-Gordan table given :math:`j_1, j_2, m`.
    '''
    table = dict()
    m1_list = np.linspace(-j1, j1, int(2*j1 + 1))
    m2_list = np.linspace(-j2, j2, int(2*j2 + 1))
    j_list = list(set(
        abs(m1 + m2)
        for m1, m2 in list(itertools.product(m1_list, m2_list))
        if abs(m1 + m2) >= abs(m)  # constraints
    ))

    for m1, m2 in list(itertools.product(m1_list, m2_list)):
        if m1 + m2 != m:
            continue
        key = (m1, m2)
        value = []
        for j in j_list:
            coefficient = cg_coefficient(
                j1=j1, j2=j2,
                m=m1+m2,
                j=j,
                m1=m1, m2=m2,
            )
            value.append(coefficient)
        table[key] = value

    df = pd.DataFrame.from_dict(table, orient='index', columns=j_list)
    df.index.name = '(m1, m2)'
    df.columns.name = 'j'
    return df

def cg_matrix(j1, j2, m, return_indices=False):
    '''
    Returns the Clebsch-Gordan matrix given :math:`j_1, j_2, m` in matrix form.
    Returns row indices (`m1, m2`) and column indices (`j`) if `return_indices` is `True`.
    '''
    table = cg_table(j1, j2, m)

    if return_indices:
        return table.to_numpy(), table.index.to_numpy(), table.columns.to_numpy()

    return table.to_numpy()

def cg_tables_all_m(j1, j2, display_tables=False) -> Dict:
    '''
    Returns a dictionary of CClebsch-Gordan table given :math:`j_1, j_2`
    with all possible :math:`m`.
    '''
    tables = dict()
    m_max = j1 + j2
    m_min = - m_max
    m_list = np.linspace(m_min, m_max, int((m_max - m_min) + 1))

    for m in m_list:
        tables[m] = cg_table(j1, j2, m)

    if display_tables:
        print(f'{j1 = }, {j2 = }')
        for m, table in tables.items():
            print(f'{m = }:')
            display(table)

    return tables


# Problem: https://projecteuler.net/problem=323

import numpy as np

"""
    T: transistion matrix where,
        T[i][j] is the probability of (a state where i bits are set) transition to (a state where j bits are set).
        We have:
            T[i][j] = C(N-i, j-i) * 2^i / 2^N
        Why?
            Assume S' = S | y, where S has i bits set, while S' has j bits set.
            For a state S where i bits are set, there are (N-i) are not set.
            To transition to S' where 2^j bits are set, we need to pick (j-i) bits from (N-i) bits.
            There are C(N-i, j-i) ways to pick (j-i) bits from (N-i) bits.
            Also, y might have bits that are already set in S, and there are 2^i ways to choose those bits.
            Therefore, there are C(N-i, j-i) * 2^i ways to choose y.
            Also, there are 2^N different N-bit numbers.
            Therefore, T[i][j] = C(N-i, j-i) * 2^i / 2^N.
    S: state vector where,
        S[i] means the probability of the current number has i bits set.

    Apply Markov chain, we can figure out the probability of having 2^N bits set after applying bitwise-OR k times as follows,
        S_{k} = S * T^k
    To calculate the expected value of N, and due to linearity of expectations, we can follow the following formula to calculate E(N):
        E(N) = sum_{i = 1 to inf} (S[N] - prev_S[N]) * i
    We expect this series to converge, so we calculate the sum for each K as in sum_{i = 1 to K} (S[N] - prev_S[N]) * i until it converges.
"""

N = 32

factorials = [1]
for i in range(1, N+1):
    factorials.append(factorials[-1] * i)

def C(n, k):
    return factorials[n] / factorials[k] / factorials[n-k]

T = np.zeros((N+1, N+1), dtype = np.float)

for alpha in range(N+1):
    for beta in range(alpha, N+1):
        T[alpha][beta] = 1.0 * C(N - alpha, beta - alpha) / (2**(N - alpha))

S = np.zeros((N+1), dtype = np.float)
S[0] = 1.0

ans = 0
prev_ans = -1
stop_threshold = 10
stop_counter = 0
tol = 10**-12

prev_SN = 0.0
n_trials = 0
while True:
    n_trials = n_trials + 1
    S = S.dot(T)
    ans = ans + (S[N]-prev_SN) * n_trials
    prev_SN = S[N]
    if abs(ans - prev_ans) < tol:
        stop_counter = stop_counter + 1
        if stop_counter == stop_threshold:
            break
    else:
        stop_counter = 0
    prev_ans = ans

print(ans)


# import modules
import numpy as np

'''
# Description.
Obtains the unit vector of an input vector.

# Input(s).
Array nx1 with size different to unit (vector)

# Example
vector = np.random.rand(2)

---
uVector = unitvector(vector)
'''
def unitvector(vector):

    vecDist = np.sqrt(np.dot(vector,vector))
    uVector = vector/vecDist

    return uVector
'''
BSD 2 license.

Copyright (c) 2016, Universidad Nacional de Colombia, Ludger O.
   Suarez-Burgoa and Exneyder Andrés Montoya Araque.
All rights reserved.

Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:

1. Redistributions of source code must retain the above copyright notice,
this list of conditions and the following disclaimer.

2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
'''


#   A quick guide to sympy
from sympy import *

#   INITIALIZE
x,y,z = symbols('x y z')
f = x**3 + exp(x) + cos(3*x) + log(sin(x))
print('function: '); print(f)

    #initialize with string:
expression = "x**3 + exp(x) + cos(3*x) + log(sin(x))"
f = sympify(expression)

print()
#   VALUE
value = f.subs(x,3)
print('symbolic f(3): '); print(value)

value = N(value)    # or evalf()
print('numeric f(3): '); print(value)

value = value.evalf(100)    # up to 1e-100 precision ?!
print('numagic f(3): '); print(value)

print('1/6: ') # don't test 1/7
value = (x/6).subs(x,1); print(value.evalf(100))

    #multivariable
f1 = x*y*z
print('with x=2, y=4, z=3 then f1 = x*y*z = ')
value = f.subs( [(x,2),(y,4),(z,3)] )
print(value)

print()
#   DERIVATIVE
df = diff(f,x)
print('first order derivative: '); print(df)

df = diff(f,x,2)    # or diff(f, x, x)
print('second order derivative: '); print(df)

    #multivariable
df = diff(x*y*z + x**y, x, y)   #derivative with respect to x then y
print(df)

print(value)

print()
#   INTEGRAL

print()
#   MATRIX

print()
#   SOLVER  f(x) = 0
fx = x**2 - 5
solution = solveset(fx, x, domain=S.Reals)
print(solution)

fx = sin(x) - 1
solution = solveset(fx, x, domain=S.Reals)
print(solution)

#


import numpy as np
import matplotlib.pyplot as plt
import calculus.integ as numint
import scipy.integrate as spint
from interpolation.lagrange import lagrangepoly
#Real value 7.32473
def f(x):
	return -x**5 + 5*x**4 - 7*x**3 + x**2 + 4*x + 0.5

x = np.linspace(-5,5, 150)
y = f(x)
a = 0.5
b = 2.6
x2 = np.linspace(a, b)
fig, ax = plt.subplots()
ax.plot(x, y)
ax.set_xlim(-3, 5)
ax.set_ylim(-4, 7)
ax.axhline(0, color='black')
ax.legend(['f(x)'])
ax.fill_between(x2, 0, f(x2), color = 'g', alpha=0.3)
plt.title('Integral of f(x)')

#Trap rule
fig, ax = plt.subplots()
ax.plot(x, y)
ax.set_xlim(-3, 5)
ax.set_ylim(-4, 7)
ax.axhline(0, color='black')
ax.legend(['f(x)'])

h = b-a

y2 = f(a) + (f(b) - f(a))/(b - a) * (x2 - a)
ax.plot(x2, y2, '-g')
ax.fill_between(x2, 0, y2, color = 'g', alpha=0.3)
ax.grid(True)
textstr = str(numint.trap(h, f(a), f(b)))
props = dict(boxstyle='round', facecolor='white')
# place a text box in upper left in axes coords
ax.text(0.05, 0.95, textstr, transform=ax.transAxes, fontsize=12,
        verticalalignment='top', bbox=props)

plt.title('Trapezoid Rule')

#Trap rule with multiple segs
fig, ax = plt.subplots()
ax.plot(x, y)
numpts = 5
n = 5-1
ax.set_xlim(-3, 5)
ax.set_ylim(-4, 7)
ax.axhline(0, color='black')
ax.legend(['f(x)'])
x3 = np.linspace(a, b, numpts)
x4 = np.linspace(a, b, numpts)
x3 = x3[1:]
lastx = a
for i in x3:
	xn = np.linspace(lastx, i, 15)
	yn = f(lastx) + (f(i) - f(lastx))/(i - lastx) * (xn - lastx)
	ax.plot(xn, yn, '-g')
	ax.fill_between(xn, 0, yn, color='g', alpha=0.3)
	lastx = i

textstr = str(numint.trapm(h/n, f(x4)))
props = dict(boxstyle='round', facecolor='white')
# place a text box in upper left in axes coords
ax.text(0.05, 0.95, textstr, transform=ax.transAxes, fontsize=12,
        verticalalignment='top', bbox=props)
ax.grid(True)
plt.title('Trapezoid Rule w/ Multiple Segments')

#Simps 1/3 rule
fig, ax = plt.subplots()
ax.plot(x, y)
ax.set_xlim(-3, 5)
ax.set_ylim(-4, 7)
ax.axhline(0, color='black')
ax.legend(['f(x)'])
a = 0.5
b = 2.6
h = b-a
xx = np.linspace(a, b, 3)
x2 = np.linspace(a, b)
y2 = lagrangepoly(xx, f(xx), x2)
ax.plot(x2, y2, '-g')
ax.fill_between(x2, 0, y2, color = 'g', alpha=0.3)
ax.grid(True)
textstr = str(numint.simp13(h, f(xx[0]), f(xx[1]), f(xx[2])))
props = dict(boxstyle='round', facecolor='white')
# place a text box in upper left in axes coords
ax.text(0.05, 0.95, textstr, transform=ax.transAxes, fontsize=12,
        verticalalignment='top', bbox=props)

plt.title('Simpson 1/3 Rule')

#Simps 1/3 rule with multiple segs
fig, ax = plt.subplots()
ax.plot(x, y)
numpts = 5
n = 5-1
ax.set_xlim(-3, 5)
ax.set_ylim(-4, 7)
ax.axhline(0, color='black')
ax.legend(['f(x)'])
x3 = np.linspace(a, b, numpts)
x4 = np.linspace(a, b, numpts)
x3 = x3[1:]
lastx = a
for i in x3:
	xn = np.linspace(lastx, i, 15)
	xx2 = np.linspace(lastx, i, 3)
	yn = lagrangepoly(xx2, f(xx2), xn)
	ax.plot(xn, yn, '-g')
	ax.fill_between(xn, 0, yn, color='g', alpha=0.3)
	lastx = i

textstr = str(numint.simp13m(h/n, f(x4)))
props = dict(boxstyle='round', facecolor='white')
# place a text box in upper left in axes coords
ax.text(0.05, 0.95, textstr, transform=ax.transAxes, fontsize=12,
        verticalalignment='top', bbox=props)
ax.grid(True)
plt.title('Simpson Rule 1/3 w/ Multiple Segments')
plt.show()


from numba import njit
import numpy as np

def py_l2_diff(f1, f2):
    """
    Computes the l2-norm of the difference
    between a function f1 and a function f2

    Parameters
    ----------
    f1 : array of floats
        function 1
    f2 : array of floats
        function 2

    Returns
    -------
    diff : float
        The l2-norm of the difference.
    """
    l2_diff = np.sqrt(np.sum((f1 - f2)**2))/f1.size

    return l2_diff

@njit
def l2_diff(f1, f2):
    """
    Computes the l2-norm of the difference
    between a function f1 and a function f2

    Parameters
    ----------
    f1 : array of floats
        function 1
    f2 : array of floats
        function 2

    Returns
    -------
    diff : float
        The l2-norm of the difference.
    """
    l2_diff = np.sqrt(np.sum((f1 - f2)**2)) / f1.size

    return l2_diff


from __future__ import division
import numpy
import matplotlib.pyplot as pyplot

USER = 'Maciej Grabias'
USER_ID = 'njgh39'

# constants used
T_HALF = 20.8 #half-time
TAU = T_HALF / numpy.log(2) # average lifetime

def f(n):
    return -n / TAU

def analytic(N0, ts):
    return N0 * numpy.exp(-ts / TAU)

def solve_euler(N0, dt, n_panels):

    #initial parameters
    n, t = N0, 0

    # array for iteration
    n_t = numpy.zeros((n_panels,))

    for i in range(n_panels):
        n_t[i] = n
        t = i * dt
        n = n + f(n) * dt
    return n_t

def solve_heun(N0, dt, n_panels):

    # initial parameters
    n, t = N0, 0

    # array for iteration
    n_t = numpy.zeros((n_panels,))

    for i in range(n_panels):
        n_t[i] = n
        k0 = f(n)
        k1 = f(n + k0 * dt)
        n = n + (k0 + k1) * dt /2
    return n_t

# constants for plots
T1 = 60 # time range
N_PANELS = 15 # number of panels
N0 = 1500 # initial numbe of nuclei

dt = T1 / N_PANELS
ts = numpy.arange(0, T1, dt)

# functions for plots

n_analytic  = analytic(N0, ts)
n_euler     = solve_euler(N0, dt, N_PANELS)
n_heun      = solve_heun(N0, dt, N_PANELS)

#plot

pyplot.figure()
pyplot.subplot(211) # count VS time for methods
pyplot.plot(ts, n_euler, label='Euler Method', color ='red')
pyplot.plot(ts, n_heun, label='Heun Method', color = 'blue', linestyle = '--')
pyplot.plot(ts, n_analytic, label='Analytic', color = 'grey')
pyplot.xlabel("Time, h")
pyplot.ylabel("Number of nuclei")
pyplot.title("Number of nuclei for different methods used VS time")
pyplot.legend()

pyplot.subplot(212) # error VS time for numerics
pyplot.semilogy()
err_euler = abs(n_euler - n_analytic) / n_analytic
err_heun = abs(n_heun - n_analytic) / n_analytic
pyplot.plot(ts, err_euler, color="red", label =" Euler")
pyplot.plot(ts, err_heun, color ="blue", linestyle="--", label="Heun")
pyplot.xlabel("Time, h")
pyplot.ylabel("Error")
pyplot.title("Error between methods used and the analytically determined solution")
pyplot.legend()

pyplot.show()

ANSWER1 = """Heun's method is more accurate than Euler's method because it integrates a differential
equation over trapeziums rather than rectangles"""


import matplotlib.pyplot as plt
import numpy as np
from scipy.integrate import quad

def smooth_product(u, v, domain=(0, 1)):
    approx, _ = quad(lambda x: u(x)*v(x), *domain)  # Discard error
    return approx

def least_squares(func, basis, inner, **kwargs):
    # Take the inner product of each pair of basis
    lhs = np.array([
        [inner(p, q, **kwargs) for p in basis]
        for q in basis
    ])

    # The the inner product of the function with each basis
    rhs = np.array([
        inner(p, func, **kwargs) for p in basis
    ])

    # Create the approximation
    return np.linalg.solve(lhs, rhs)

def Q1():
    legendre = ('1', '2*x - 1', '6*x**2 - 6*x + 1')
    basis    = [  # Generate the functions based off strings
        np.vectorize(eval('lambda x:' + poly))
        for poly in legendre
    ]

    coeffs = least_squares(
        np.exp, basis, smooth_product, domain=(0, 1))

    domain = np.linspace(0, 1)
    approx = coeffs @ [p(domain) for p in basis]

    fig, ax = plt.subplots()
    ax.plot(domain, np.exp(domain), label='Exact')
    ax.plot(domain, approx, label='Approx')

def Q2():
    basis_funcs = [  # Generate the functions based off strings
        np.vectorize(eval('lambda x:' + poly))
        for poly in ('1', 'x', 'x**2')
    ]

    x_data, y_data = np.loadtxt('data_points.txt', unpack=True)
    basis_data = [p(x_data) for p in basis_funcs]

    coeffs = least_squares(
        y_data, basis_data, np.inner)

    domain = np.linspace(x_data.min(), x_data.max())
    approx = coeffs @ [p(domain) for p in basis_funcs]

    fig, ax = plt.subplots()
    ax.scatter(x_data, y_data, label='Exact')
    ax.plot(domain, approx, 'k-', label='Approx')

def Q3():
    basis = [  # Generate the functions based off strings
        eval('lambda x:' + func)
        for func in ('np.sin(x)', 'np.sin(2*x)', 'np.sin(3*x)')
    ]

    def function(x):
        return x*(np.pi-x)

    coeffs = least_squares(
        function, basis,
        smooth_product, domain=(0, np.pi)
    )

    for coeff, exact in zip(coeffs, (8/np.pi, 0, 8/(27*np.pi))):
        print(f'Coefficient error: {abs(coeff-exact):.3e}')

    domain = np.linspace(0, np.pi)
    approx = coeffs @ [p(domain) for p in basis]

    fig, ax = plt.subplots()
    ax.plot(domain, function(domain), label='Exact')
    ax.plot(domain, approx, label='Approx')

if __name__ == '__main__':
    questions = (Q1, Q2, Q3)
    for question in questions:
        input(f'Press `Enter` to run {question.__name__} ')

        plt.close('all')        # <- Close all existing figures
        question()
        plt.show(block=False)   # <- Allow code execution to continue

    input('Press `Enter` to quit the program.')


import pandas as pd
import matplotlib.pyplot as plt
import numpy as np
import time

'''
ComputeCost computes the cost function
'''
def computeCost(X, y, theta):
	#computeCost Compute cost for linear regression
	#   J = computeCost(X, y, theta) computes the cost of using theta as the
	#   parameter for linear regression to fit the data points in X and y

	# Initialize some useful values
	m = len(y); # number of training examples

	# You need to return the following variables correctly
	J = 0;

	# ====================== YOUR CODE HERE ======================
	# Instructions: Compute the cost of a particular choice of theta
	# =========================================================================
	# You should set J to the cost.

	X_product 	= np.matmul(X,theta) 		# X*theta
	X_diff	  	= np.subtract(X_product, y)	# X*theta - y
	X_square	= np.square(X_diff)			# Square each element of the matrix computed above
	X_sum		= np.sum(X_square)			# Sum all the elements
	J 			= (1.0/(2.0*m))*X_sum		# Cost Function

	return J

'''
gradientDescent function iterates till it finds a minima
'''
def gradientDescent(X, y, theta, alpha, num_iters):
	#function [theta, J_history] = gradientDescent(X, y, theta, alpha, num_iters)
	#GRADIENTDESCENT Performs gradient descent to learn theta
	#   theta = GRADIENTDESENT(X, y, theta, alpha, num_iters) updates theta by
	#   taking num_iters gradient steps with learning rate alpha

	# Initialize some useful values
	m = len(y); # number of training examples
	J_history = np.zeros((num_iters, 1));

	for iter in range(num_iters):

		# ====================== YOUR CODE HERE ======================
		# Instructions: Perform a single gradient step on the parameter vector
		#               theta.
		#
		# Hint: While debugging, it can be useful to print out the values
		#       of the cost function (computeCost) and gradient here.
		#
		X_0 			= X[:,0].reshape((m,1));
		X_1 			= X[:,1].reshape((m,1));
		X_0_tr			= np.transpose(X_0)
		X_1_tr			= np.transpose(X_1)
		X_theta_prod	= (np.matmul(X,theta)).reshape((m,1))
		X_theta_y_diff	= np.subtract(X_theta_prod,y)
		theta_0 = theta.item(0) - (float(alpha)/float(m))*(np.matmul(X_0_tr, X_theta_y_diff)).item(0)
		theta_1 = theta.item(1) - (float(alpha)/float(m))*(np.matmul(X_1_tr, X_theta_y_diff)).item(0)
		#print X_0.shape, X_0_tr.shape, theta.shape, X.shape, X_theta_prod.shape, y.shape, X_theta_y_diff.shape
		theta = np.array([theta_0, theta_1]).reshape((2,1))

		# Plot the linear fit
		if(iter%200==0):
			plt.scatter(X_data, y_data, marker='o',  color='g', label='orig')
			y_data_predicted = np.matmul(X,theta)
			plt.plot(X_data, y_data_predicted, marker='*', linestyle='-', color='b', label='pred')
			plt.legend(loc='lower right')
			plt.show(block=False)
			time.sleep(3)
			plt.close()

		# ============================================================

		# Save the cost J in every iteration
		J_history[iter] = computeCost(X, y, theta)
		print "Cost @ iteration: ",iter, " = ", J_history[iter]
	return theta

data 	= pd.read_csv('ex1data1.txt', header =  None, names = ['Population', 'Profits'])
y_data 	= data.iloc[:,1]
X_data 	= data.iloc[:,0]

m 		= len(y_data)						  #Number of training samples
y 		= np.array(y_data).reshape(m,1)
X 		= np.c_[np.ones(m), np.array(X_data)] # Add a column of ones to x

fig = plt.figure()
ax = fig.add_subplot(111)
ax.set_title('Population - Profit Scatter Plot')
ax.set_xlabel('Population in 10000s')
ax.set_ylabel('Profit in 10000$')

theta 	= np.zeros((2, 1)).reshape((2,1)) # initialize fitting parameters
theta 	= np.array([40,40]).reshape((2,1))# Try initializing from a different point. The convergence will be seen easily

print "Cost Function Value is:", computeCost(X, y, theta)

# Some gradient descent settings
iterations = 1500;
alpha = 0.01;

# run gradient descent
theta = gradientDescent(X, y, theta, alpha, iterations);

# print theta to screen
print 'Theta found by gradient descent: ', theta.item(0), theta.item(1)

# Plot the linear fit
plt.scatter(X_data, y_data, marker='o',  color='g', label='orig')
y_data_predicted = np.matmul(X,theta)
plt.plot(X_data, y_data_predicted, marker='*', linestyle='-', color='b', label='pred')
plt.legend(loc='lower right')

# Predict values for population sizes of 35,000 and 70,000
#predict1 = [1, 3.5] *theta;
#fprintf('For population = 35,000, we predict a profit of %f\n',...
#    predict1*10000);
#predict2 = [1, 7] * theta;
#fprintf('For population = 70,000, we predict a profit of %f\n',...
#   predict2*10000);


#!/usr/bin/python
""" Calculation of Pi via Monte-Carlo """

import random
import numpy as np
from time import process_time
MAX_STEPS = 1000000
REPEATS = 100

def method_1() -> float:
    """
    Generate two random numbers between 0 and 1 (as x and y coordinates).
    If x*x + y*y < 1, that is (x, y) inside the unit circle (segment).
    Pi is calculated by the fraction of points in the circle vs total number.
    """
    for i in range(REPEATS):
        step = 0
        sum = 0
        while step < MAX_STEPS:
            x = random.random()
            y = random.random()
            if x*x + y*y < 1.0:
                sum += 1
            step += 1
        estimate = 4.0 * sum / MAX_STEPS
    return estimate

def method_2() -> float:
    """
    Generates one random number [0, 1) as x and calculates the corresponding
    y value to put (x, y) on the unit circle.
    """
    for i in range(REPEATS):
        step = 0
        sum = 0.0
        while step < MAX_STEPS:
            x = random.random()
            y = (1.0 - x*x) ** 0.5
            sum = sum + y
            step += 1
        estimate = 4.0 * sum / MAX_STEPS
    return estimate

def method_3() -> float:
    """
    Generates a periodic x-grid and generates the corresponding y values to form a unit circle.
    """
    step = 1.0 / MAX_STEPS
    for i in range(REPEATS):
        sum = 0.0
        x = 0.0
        while x <= 1.0:
            y = (1.0 - x*x) ** 0.5
            sum = sum + y
            x += step
        estimate = 4.0 * sum / MAX_STEPS
    return estimate

def method_4() -> float:
    """
    Basically the same as method3, but utiliting the numpy module.
    """
    x_dom = np.linspace(0, 1, MAX_STEPS)
    for i in range(REPEATS):
        y     = np.sqrt(1-np.power(x_dom, 2))
        estimate    = 4.0 * y.sum() / MAX_STEPS
    return estimate

if __name__ == '__main__':
    lst_methods = [
        method_1,
        method_2,
        method_3,
        method_4,
        ]
    for method in lst_methods:
        t1 = process_time()
        pi = method()
        t2 = process_time()
        print("Pi was approximated to {} using {} steps and {} repeats in {} seconds.".format(pi, MAX_STEPS, REPEATS, round(t2-t1, 5)))


# Polynomial Regression

# Importing the libraries
import numpy as np
import matplotlib.pyplot as plt
import pandas as pd

# Importing the dataset
dataset = pd.read_csv('Position_Salaries.csv')
X = dataset.iloc[:, 1:-1].values
Y = dataset.iloc[:, -1].values

## Splitting the dataset into the training set and test set
# Only 10 observations, thus doesn't make sense to split dataset
"""from sklearn.cross_validation import train_test_split
X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size= 0.2, random_state= 0)"""

"""# Feature scaling
from sklearn.preprocessing import StandardScaler
sc_X = StandardScaler()
X_train = sc_X.fit_transform(X_train)
X_test = sc_X.transform(X_test)"""

# Fit linear regression to dataset
from sklearn.linear_model import LinearRegression
LinReg = LinearRegression()
LinReg.fit(X,Y)

# Fit polynomial regression to dataset
from sklearn.preprocessing import PolynomialFeatures
Degree = 3
PolyReg = PolynomialFeatures(degree = Degree)
X_poly = PolyReg.fit_transform(X)
LinReg2 = LinearRegression()
LinReg2.fit(X_poly, Y)

# Calculate R^2 and adjusted R^2
def calculateRSquared(y_orig, y_pred, X):
    Resid = sum((y_orig - y_pred)**2)
    Total = sum((y_orig - np.mean(y_orig))**2)
    R_squared = 1 - (float(Resid))/Total
    R_squared_adjusted = 1 - (1-R_squared)*(len(y_orig)-1)/(len(y_orig)-X.shape[1]-1)
    return R_squared, R_squared_adjusted

# Visualize regression fits
# Visualize linear fit
Y_Lin = LinReg.predict(X)
[R, R_adjust] = calculateRSquared(Y,Y_Lin,X)
plt.scatter(X, Y, color = 'red')
plt.plot(X,Y_Lin, color = 'blue')
plt.title('Salary vs Position (Linear Regression)')
plt.text(0.05, 0.85, '$R^2$ = {}'.format(round(R,2))+'\n$R^2$ adjusted = {}'.format(round(R_adjust,2)), transform=plt.gca().transAxes)
plt.xlabel('Position grade')
plt.ylabel('Salary ($)')

X_grid = np.arange(min(X), max(X), 0.1)
X_grid = X_grid.reshape(len(X_grid), 1)
Y_Poly = LinReg2.predict(PolyReg.fit_transform(X_grid))
[R, R_adjust] = calculateRSquared(Y,LinReg2.predict(PolyReg.fit_transform(X)),X)
plt.figure()
plt.scatter(X, Y, color = 'red')
plt.plot(X_grid, Y_Poly, color = 'blue')
plt.title('Salary vs Position (Polynomial Regression, D = ' + str(Degree) + ')')
plt.text(0.05, 0.85, '$R^2$ = {}'.format(round(R,2))+'\n$R^2$ adjusted = {}'.format(round(R_adjust,2)), transform=plt.gca().transAxes)
plt.xlabel('Position grade')
plt.ylabel('Salary ($)')
plt.show()

# Predicting a new result with linear regression
print(LinReg.predict(6.5))

# Predicting a new result with polynomial regression
print(LinReg2.predict(PolyReg.fit_transform(6.5)))


import numpy as np

#############################
# SINGLE DIMENSIONAL ARRAYS #
#############################

# This creates our array
np_array = np.array([5, 10, 15, 20, 25, 30])
print("--0--")

# Gets the unique values
print(np.unique(np_array))
print("--1--")

# Calculates the standard deviation
print(np.std(np_array))
print("--2--")

# Calculates the maximum
print(np_array.max())
print("--3--")

# Squares each value in the array
print(np_array ** 2)
print("--4--")

# Adds the arrays together element wise
print(np_array + np_array)
print("--5--")

# The sum of the squares of the elements
print(np.sum(np_array ** 2))
print("--6--")

# Gives you the shape: (rows, columns)
print(np_array.shape)

##########################
# TWO DIMENSIONAL ARRAYS #
##########################
# Create 2d array
print("--0--")
np_2d_array = np.array([[1,2,3],
                        [4,5,6]])
print(np_2d_array)

# Calculate the transpose, which is when you swap the columns and rows.
print("--1--")
np_2d_array_T = np_2d_array.T
print(np_2d_array_T)

# Print the shape of the array as (number of rows, number of columns)
print("--3--")
print(np_2d_array.shape)

# Access elements in the 2d array by index.
# First index is the row number
# Second index is the column number
# Index numbers start from 0
print("--4--")
print(np_2d_array[1,1])
print(np_2d_array[0,2])

###########################
# CALCULATING DOT PRODUCT #
###########################
np_array = np.array([5, 10, 15, 20, 25, 30])
dot_product = np.dot(np_array, np_array)
print(dot_product)

############################
# GENERATING RANDOM VALUES #
############################
# Generage a single random number in range [0,1)
print("--0--")
print(np.random.rand())

# Generate a matrix of random numbers in range [0,1) with shape (3,2)
print("--1--")
print(np.random.rand(3,2))

# Low=5, High=15, Size=2. Generate 2 values between 5 and 15 (exclusive)
print("--0--")
print(np.random.randint(5, 15, 2))

# Low=5, High=15, Size=(3,2). Generate a matrix of shape (3,2) with values between 5 and 15 (exclusive)
print("--1--")
print(np.random.randint(5, 15, (3,2)))

#####################
# SAMPLING THE DATA #
#####################
array = np.array([1,2,3,4,5])

# Sample 10 data points with replacement.
print("--0--")
print(np.random.choice(array, 10, replace=True))

# Sample 3 data points without replacement.
# Sampling without replacement means the same value can’t be sampled more than once
print("--1--")
print(np.random.choice(array, 3, replace=False))

#############################
# RANDOMLY SHUFFLING VALUES #
#############################
x = [1,2,3,4,5]  # Create a list of 5 elements
np.random.shuffle(x)  # Randomly shuffle the order of the elements in the list

print(x)

# if you want the random selections to be the same every time, you need to set a seed
np.random.seed(42)


# TheoIV
# 1. Exercise Sheet
#  -> 4. Shooting Pi

import numpy as np
import matplotlib.pyplot as plt
import time

# num_shoots is N from the exercise sheet.
def shoot_pi(num_shoots = 10, verbose = True):

    if verbose:
        print('Shooting PI with ' + str(num_shoots) + ' samples...')

    # shoot randomly (uniform) at 1x1 area and count hits on inner quarter circle.
    num_hits = 0
    for _ in range(num_shoots):
        point = np.random.rand(2)
        if (np.linalg.norm(point) < 1):
            num_hits += 1

    # compute pi as ratio of hits to all samples
    # (factor 4 necessary, since we're only shooting at a quarter circle)
    pi = (num_hits / num_shoots) * 4

    if verbose:
        print('PI is approximately ', str(pi), '!', sep='')

    return pi

# num_samples is M from the exercise sheet.
def sample_pi_shoots(num_samples = 500, num_shoots_per_sample = 10, verbose = True):

    # shoot pi M times and compute mean + std of results
    pi_samples = []
    for _ in range(num_samples):
        pi_samples.append(shoot_pi(num_shoots_per_sample, False))
    pi_mean = np.mean(pi_samples, axis=0)
    pi_std = np.std(pi_samples, axis=0)
    pi_var = pi_std**2

    if verbose:
        print('Mean:', round(pi_mean, 3))
        print('Std: ', round(pi_std, 3))
        print('Var: ', round(pi_var, 3))

    return (pi_mean, pi_std, pi_var)

# show proportionality between standard deviation and 1/sqrt(N)
def show_dependency():

    M = 500
    Ns = []
    stds = []

    t = time.time()

    for N in range(500, 8_001, 500):
        (_, std, _) = sample_pi_shoots(M, N, False)
        Ns.append(N)
        stds.append(std)

    print('Runtime:', time.time()-t, 'seconds')

    fig, axs = plt.subplots(3)
    fig.canvas.set_window_title('Homework 1')
    fig.suptitle(r'Showing: $\Delta x \sim \frac{1}{\sqrt{N}}$, M=' + str(M))
    axs[0].set_xlabel('N', fontsize=15)
    axs[0].set_ylabel(r'$\Delta x$', rotation=0, fontsize=15)
    axs[0].plot(Ns, np.array(stds))
    axs[1].set_xlabel(r'$log(N)$', fontsize=15)
    axs[1].set_ylabel(r'$log(\Delta x)$', rotation=0, fontsize=15)
    axs[1].plot(np.log(np.array(Ns)), np.log(np.array(stds)))
    axs[2].set_xlabel(r'$\frac{1}{\sqrt{N}}$', fontsize=15)
    axs[2].set_ylabel(r'$\Delta x$', rotation=0, fontsize=15)
    axs[2].plot(1/np.sqrt(np.array(Ns)), np.array(stds))
    plt.show()

if __name__ == '__main__':
    # 4a):
    # shoot_pi(10_000)

    # 4b):
    show_dependency()


#!/usr/bin/env python3
import sys
import numpy as np

np.random.seed(2017)

def inside_circle(total_count):

    x = np.float32(np.random.uniform(size=total_count))
    y = np.float32(np.random.uniform(size=total_count))

    radii = np.sqrt(x*x + y*y)

    count = len(radii[np.where(radii<=1.0)])

    return count

def estimate_pi(n_samples):

    return (4.0 * inside_circle(n_samples) / n_samples)

if __name__=='__main__':

    n_samples = 10000
    if len(sys.argv) > 1:
        n_samples = int(sys.argv[1])

    my_pi = estimate_pi(n_samples)
    sizeof = np.dtype(np.float32).itemsize

    print("[serial version] required memory %.3f MB" % (n_samples*sizeof*3/(1024*1024)))
    print("[serial version] pi is %f from %i samples" % (my_pi,n_samples))


"""
One of the central problems in statistics is to make estimations — and quantify
how good these estimations are — of the distribution of an entire population
given only a small (random) sample. A classic example is to estimate the average
height of all the people in a country when measuring the height of a randomly
selected sample of people. These kinds of problems are particularly interesting
when the true population distribution, by which we usually mean the mean of the
whole population, cannot feasibly be measured. In this case, we must rely on our
knowledge of statistics and a (usually much smaller) randomly selected sample to
estimate the true population mean and standard deviation, and also quantify how
good our estimations are. It is the latter that is the source of confusion,
misunderstanding, and misrepresentation of statistics in the wider world.

This module illustrates how to estimate the population mean and give a
confidence interval fo these estimates.
"""
import math
import pandas as pd

from scipy import stats

sample_data = pd.Series([
    172.3, 171.3, 164.7, 162.9, 172.5, 176.3, 174.8, 171.9,
    176.8, 167.8, 164.5, 179.7, 157.8, 170.6, 189.9, 185. ,
    172.7, 165.5, 174.5, 171.5])

sample_mean = sample_data.mean()
sample_std = sample_data.std()

print(f"Mean: {sample_mean}, st. dev: {sample_std}")
# Mean: 172.15, st. dev: 7.473778724383846

N = sample_data.count()
std_err = sample_std/math.sqrt(N)

cv_95, cv_99 = stats.t.ppf([0.975, 0.995], df=N-1)

pm_95 = cv_95 * std_err
pm_99 = cv_99 * std_err
conf_interval_95 = [sample_mean - pm_95, sample_mean + pm_95]
conf_interval_99 = [sample_mean - pm_99, sample_mean + pm_99]

print(f"95% confidence: {conf_interval_95}")
print(f"99% confidence: {conf_interval_99}")
# 95% confidence: [168.65216388659374, 175.64783611340627]
# 99% confidence: [167.36884119608774, 176.93115880391227]


import numpy as np

def softmax_numpy(x):
    """
    :param x:  numpy array of shape (n_samples, n_features)
    :return: numpy array of shape (n_samples, n_features)
    Things that are important in this code:
    1. delegate looping into numpy and don't use for loops in python
    2. Use broadcasting to to operations on vectors/matrices with different shapes
    a good article on broadcasting can be found here:
    http://eli.thegreenplace.net/2015/broadcasting-arrays-in-numpy/
    3. Direct computation of exp(x) can lead to overflow, use the property:
        softmax(x) = softmax(x+c) for any constant c
    """
    x_max = np.max(x, axis=1)
    x_exp = np.exp((x.transpose() - x_max).transpose())
    x_exp_sum = np.sum(x_exp, axis=1)
    return x_exp / x_exp_sum[:, None]

def test_softmax_numpy():
    """
     Warning: these are not exhaustive.
    """
    print "Running basic tests..."
    test1 = softmax_numpy(np.array([[1001, 1002], [3, 4]]))
    assert np.amax(np.fabs(test1 - np.array(
        [0.26894142, 0.73105858]))) <= 1e-6
    test2 = softmax_numpy(np.array([[-1001, -1002]]))
    assert np.amax(np.fabs(test2 - np.array(
        [0.73105858, 0.26894142]))) <= 1e-6
    print "softmax tests pass\n"

if __name__ == "__main__":
    test_softmax_numpy()


Vamos continuar nossa trilha em computação simbólica estendendo o conhecimento sobre o tipo `bool`, expressões e testes lógicos.

## Operadores lógicos

Vimos que `True` e `False` são os dois valores atribuíves a um objeto de tipo `bool`. Eles são úteis para testar condições, realizar verificações e comparar quantidades. Vamos estudar *operadores de comparação*, *operadores de pertencimento* e *operadores de identidade*.

### Operadores de comparação

A tabela abaixo resume os operadores de comparação utilizados em Python.

| operador | significado | símbolo matemático |
|---|---|---|
| `<` | menor do que | $<$ |
| `<=` | menor ou igual a | $\leq$ |
| `>` | maior do que | $>$ |
| `>=` | maior ou igual a | $\geq$ |
| `==` | igual a | $=$ |
| `!=` | diferente de | $\neq$ |

Podemos usá-los para comparar objetos.

**Nota:** `==` está relacionado à igualdade, ao passo que `=` é uma atribuição. São conceitos operadores com finalidade distinta.

2 < 3 # o resultado é um 'bool'

5 < 2 # isto é falso

2 <= 2 # isto é verdadeiro

4 >= 3 # isto é verdadeiro

6 != -2

4 == 4 # isto não é uma atribuição!

Podemos realizar comparações aninhadas:

x = 2
1 < x < 3

3 > x > 4

2 == x > 3

As comparações aninhadas acima são resolvidas da esquerda para a direita e em partes. Isso nos leva a introduzir os seguintes operadores.

| operador | símbolo matemático | significado | uso relacionado a |
|---|---|---|---|
| `or` | $\vee$ | "ou" booleano | união, disjunção |
| `and` | $\wedge$ | "e" booleano | interseção, conjunção |
| `not` | $\neg$ | "não" booleano | exclusão, negação |

# parênteses não são necessários aqui
(2 == x) and (x > 3) # 1a. comparação: 'True'; 2a.: 'False'. Portanto, ambas: 'False'

# parênteses não são necessários aqui
(x < 1) or (x < 2) # nenhuma das duas é True. Portanto,

not (x == 2) # nega o "valor-verdade" que é 'True'

not x + 1 > 3 # estude a precedência deste exemplo. Por que é 'True'?

not (x + 1 > 3) # estude a precedência deste exemplo. Por que também é 'True'?

### Operadores de pertencimento

A tabela abaixo resume os operadores de pertencimento.

| operador | significado | símbolo matemático
|---|---|---|
| `in` | pertence a | $\in$ |
| `not in` | não pertence a | $\notin$ |

Eles terão mais utilidade quando falarmos sobre sequências, listas. Neste momento, vejamos exemplos com objetos `str`.

'2' in '2 4 6 8 10' # o caracter '2' pertence à string

frase_teste = 'maior do que'
'maior' in frase_teste

'menor' in frase_teste # a palavra 'menor' está na frase

1 in 2 # 'in' e 'not in' não são aplicáveis aqui

### Operadores de identidade

A tabela abaixo resume os operadores de identidade.

| operador | significado
|---|---|
| `is` | "aponta para o mesmo objeto"
| `is not` | "não aponta para o mesmo objeto" |

Esses operadores são úteis para verificar se duas variáveis se referem ao mesmo objeto. Exemplo:

```python
a is b
a is not b
```

- `is` é `True` se `a` e `b` se referem ao mesmo objeto; `False`, caso contrário.
- `is not` é `False` se `a` e `b` se referem ao mesmo objeto; `True`, caso contrário.

a = 2
b = 3
a is b # valores distintos

a = 2
b = a
a is b # mesmos valores

a = 2
b = 3
a is not b # de fato, valores não são distintos

a = 2
b = a
a is not b # de fato, valores são distintos

## Equações simbólicas

Equações simbólicas podem ser formadas por meio de `Eq` e não com `=` ou `==`.

# importação
from sympy.abc import a,b
import sympy as sy
sy.init_printing(pretty_print=True)

sy.Eq(a,b) # equação simbólica

sy.Eq(sy.cos(a), b**3) # os objetos da equação são simbólicos

### Resolução de equações algébricas simbólicas

Podemos resolver equações algébricas da seguinte forma:

```python
solveset(equação,variável,domínio)
```

**Exemplo:** resolva $x^2 = 1$ no conjunto $\mathbb{R}$.

from sympy.abc import x
sy.solveset( sy.Eq( x**2, 1), x,domain=sy.Reals)

Podemos reescrever a equação como: $x^2 - 1 = 0$.

sy.solveset( sy.Eq( x**2 - 1, 0), x,domain=sy.Reals)

Com `solveset`, não precisamos de `Eq`. Logo, a equação é passada diretamente.

sy.solveset( x**2 - 1, x,domain=sy.Reals)

**Exemplo:** resolva $x^2 + 1 = 0$ no conjunto $\mathbb{R}$.

sy.solveset( x**2 + 1, x,domain=sy.Reals) # não possui solução real

**Exemplo:** resolva $x^2 + 1 = 0$ no conjunto $\mathbb{C}$.

sy.solveset( x**2 + 1, x,domain=sy.Complexes) # possui soluções complexas

**Exemplo:** resolva $\textrm{sen}(2x) = 3 + x$ no conjunto $\mathbb{R}$.

sy.solveset( sy.sin(2*x) - x - 3,x,sy.Reals) # a palavra 'domain' também pode ser omitida.

O conjunto acima indica que nenhuma solução foi encontrada.

**Exemplo:** resolva $\textrm{sen}(2x) = 1$ no conjunto $\mathbb{R}$.

sy.solveset( sy.sin(2*x) - 1,x,sy.Reals)

## Expansão, simplificação e fatoração de polinômios

Vejamos exemplos de polinômios em uma variável.

a0, a1, a2, a3 = sy.symbols('a0 a1 a2 a3') # coeficientes
P3x = a0 + a1*x + a2*x**2 + a3*x**3 # polinômio de 3o. grau em x
P3x

b0, b1, b2, b3 = sy.symbols('b0 b1 b2 b3') # coeficientes
Q3x = b0 + b1*x + b2*x**2 + b3*x**3 # polinômio de 3o. grau em x
Q3x

R3x = P3x*Q3x # produto polinomial
R3x

R3x_e = sy.expand(R3x) # expande o produto
R3x_e

sy.simplify(R3x_e) # simplify às vezes não funciona como esperado

sy.factor(R3x_e) # 'factor' pode funcionar melhor

# simplify funciona para casos mais gerais
ident_trig = sy.sin(x)**2 + sy.cos(x)**2
ident_trig

sy.simplify(ident_trig)

## Identidades trigonométricas

Podemos usar `expand_trig` para expandir funções trigonométricas.

sy.expand_trig( sy.sin(a + b) ) # sin(a+b)

sy.expand_trig( sy.cos(a + b) ) # cos(a+b)

sy.expand_trig( sy.sec(a - b) ) # sec(a-b)

## Propriedades de logaritmo

Com `expand_log`, podemos aplicar propriedades válidas de logaritmo.

sy.expand_log( sy.log(a*b) )

A identidade não foi validada pois `a` e `b` são símbolos irrestritos.

a,b = sy.symbols('a b',positive=True) # impomos que a,b > 0

sy.expand_log( sy.log(a*b) ) # identidade validada

sy.expand_log( sy.log(a/b) )

m = sy.symbols('m', real = True) # impomos que m seja um no. real
sy.expand_log( sy.log(a**m) )

Com `logcombine`, compactamos as propriedades.

sy.logcombine( sy.log(a) + sy.log(b) ) # identidade recombinada

## Fatorial

A função `factorial(n)` pode ser usada para calcular o fatorial de um número.

sy.factorial(m)

sy.factorial(m).subs(m,10) # 10!

sy.factorial(10) # diretamente

**Exemplo:** Sejam $m,n,x$ inteiros positivos. Se $f(m) = 2m!$, $g(n) = \frac{(n + 1)!}{n^2!}$ e $h(x) = f(x)g(x)$, qual é o valor de $h(2)$?

from sympy.abc import m,n,x

f = 2*sy.factorial(m)
g = sy.factorial(n + 1)/sy.factorial(n**2)

h = (f.subs(m,x)*g.subs(n,x)).subs(x,4)
h

## Funções anônimas

A terceira classe de funções que iremos aprender é a de *funções anônimas*. Uma **função anônima** em Python consiste em uma função cujo nome não é explicitamente definido e que pode ser criada em apenas uma linha de código para executar uma tarefa específica.

Funções anônimas são baseadas na palavra-chave `lambda`. Este nome tem inspiração em uma área da ciência da computação chamada de cálculo-$\lambda$.

Uma função anônima tem a seguinte forma:

```python
lambda lista_de_parâmetros: expressão
```

Funções anônimas podem são bastante úteis para tornar um código mais conciso.

Por exemplo, na aula anterior, definimos a função

```python
def repasse(V):
    return 0.0103*V
```

para calcular o repasse financeiro ao corretor imobiliário.

Com uma função anônima, a mesma função seria escrita como:

repasse = lambda V: 0.0103*V

Não necessariamente temos que atribui-la a uma variável. Neste caso, teríamos:

lambda V: 0.0103*V

Para usar a função, passamos um valor:

repasse(100000) # repasse sobre R$ 100.000,00

O modelo completo com "bonificação" seria escrito como:

r3 = lambda c,V,b: c*V + b # aqui há 3 parâmetros necessários

Redefinamos objetos simbólicos:

from sympy.abc import b,c,V
r3(b,c,V)

O resultado anterior continua sendo um objeto simbólico, mas obtido de uma maneira mais direta. Podemos usar funções anônimas para tarefas de menor complexidade.

## "Lambdificação" simbólica

Usando `lambdify`, podemos converter uma expressão simbólica do *sympy* para uma expressão que pode ser numericamente avaliada em outra biblioteca. Essa função desempenha papel similar a uma função *lambda* (anônima).

expressao = sy.sin(x) + sy.sqrt(x) # expressão simbólica
f = sy.lambdify(x,expressao,"math") # lambdificação para o módulo math
f(0.2) # avalia

Para avaliações simples como a anterior, podemos usar `evalf` e `subs`. A lambdificação será útil quando quisermos avaliar uma função em vários pontos, por exemplo. Na próxima aula, introduziremos sequencias e listas. Para mostrar um exemplo de lambdificação melhor veja o seguinte exemplo.

from numpy import arange # importação de função do módulo numpy

X = arange(40) # gera 40 valores de 0 a 39

X

f = sy.lambdify(x,expressao,"numpy")(X) # avalia 'expressao' em X
f


################################################################## clase ######################################################################################
# Ejercicio 1:

import numpy as np
from numpy.linalg import inv
from numpy.linalg import solve
# 1a.) Tiene un sistema de 3 ecuaciones lineales Ax=B, donde:

A=np.matrix([[8.,2.,1.],[1.,-2.,-3.],[-1.,1.,2,]])
print(A)

B=np.array([-8.,0.,3.])

# Implemente el algoritmo de eliminacion gaussiana para resolver este sistema de ecuaciones. IMPRIMA la matriz aumentada paso a paso. IMPRIMA su vector solucion
def GJ(A,B):
    NA=len(A)
    paso=0
    for c in range(NA):
        for r in range(NA):
            if(r==c):
                B[r]=B[r]/A[r,c]
                A[r,:]=A[r,:]/A[r,c]
                paso+=1
                print('Paso',paso)
                print(A)
            if(r>c):
                B[r]=B[r]-(B[c]*A[r,c])
                A[r,:]=A[r,:]-(A[c,:]*A[r,c])
                paso+=1
                print('Paso',paso)
                print(A)

    for c in range(NA):
        for r in range(NA):
            if(r<c):
                B[r]=B[r]-(B[c]*A[r,c])
                A[r,:]=A[r,:]-(A[c,:]*A[r,c])
                paso+=1
                print('Paso',paso)
                print(A)
    print('El vector solución es:',B)

GJ(A,B)

# IMPRIMA la solucion encontrada usando los paquetes de numpy
print('Usando los paquetes de numpy la solución da:',solve(A,B))

# 1b). Repita lo anterior para un sistema de ecuaciones mas general: Imprimiendo los pasos intermedios.
#Escriba aca un codigo GENERAL de eliminacion Gaussiana para resolver el sistema Ax=B.

print('')
print('PUNTO 1B')
print('')

N=np.random.randint(3 , 8)
print ('Matriz de tamaño:',N,'x',N)
Arreglo=(np.random.random((N,N))*10.0)-5.0
B=(np.random.random((N,1))*10.0)-5.0

print ('Matriz A =')
print (Arreglo)
print ('B =')
print (B)
print('')
print('Solución')
GJ(Arreglo,B)

# IMPRIMA la solucion encontrada usando los paquetes de numpy

print('Usando los paquetes de numpy la solución da:',solve(Arreglo,B))


import numpy as np
import matplotlib.pyplot as plt

N = 101
L = 2.0
mesh = np.linspace(-L,L,N)

# Plot quadratic loss for comparison
PLOT_QUADRATIC = True

# Plot sequence of hyperparameter values
PLOT_SEQ = True

normal_loss = lambda x, s:  np.power(x,2)/(2*np.power(s,2)) + 0.5*np.log(2*np.pi*np.power(s,2))
laplace_loss = lambda x, b:  np.abs(x)/b + np.log(2*b)
cauchy_loss = lambda x, g: np.log( np.pi*g*(1.0 + np.power(x,2)/np.power(g,2)) )
quadratic_loss = lambda x: x*x

theta = 0.5
normal_vals = np.array([normal_loss(x,theta) for x in mesh])
laplace_vals = np.array([laplace_loss(2*x,theta) for x in mesh])
cauchy_vals = np.array([cauchy_loss(2*x,theta) for x in mesh])
quadratic_vals = np.array([quadratic_loss(x) for x in mesh])

fig = plt.figure()
ax = fig.gca()
plt.plot(mesh, normal_vals, c='C0', label="Normal")
plt.plot(mesh, laplace_vals, c='C1', label="Laplace")
plt.plot(mesh, cauchy_vals, c='C2', label="Cauchy")
if PLOT_QUADRATIC:
    plt.plot(mesh, quadratic_vals, c='C3', label="Quadratic")
ax.legend(fontsize=24)
plt.show()

if PLOT_SEQ:

    normal_vals = np.array([normal_loss(x,theta) for x in mesh])
    laplace_vals = np.array([laplace_loss(2*x,theta) for x in mesh])
    cauchy_vals = np.array([cauchy_loss(2*x,theta) for x in mesh])

    for rate, title in [[0.9,"Decreasing"],[1.1,"Increasing"]]:
        for loss, label, color in [[normal_loss, "Normal", "C0"],[laplace_loss, "Laplace", "C1"],[cauchy_loss, "Cauchy", "C2"]]:
            if label == "Normal":
                theta = 1.0
            elif label == "Laplace":
                theta = 0.5
            else:
                theta = 0.5
            vals = np.array([loss(x,theta) for x in mesh])
            plt.plot(mesh, vals, c=color, label=label, linewidth=3.0)
            plt.plot(mesh, quadratic_vals, c='C3', label="Quadratic")

            for k in range(1,11):
                theta = theta*rate
                vals = np.array([loss(x,theta) for x in mesh])
                plt.plot(mesh, vals, c=color, linestyle="dashed", label=None)
            plt.title(title + " " + label + " Paramater", fontsize=24)
            plt.show()


import numpy as np
from scipy import signal

# source: https://stackoverflow.com/questions/40703751/using-fourier-transforms-to-do-convolution?utm_medium=organic&utm_source=google_rich_qa&utm_campaign=google_rich_qa

x = [[1, 0, 0, 0], [0, -1, 0, 0], [0, 0, 3, 0], [0, 0, 0, 1]]
x = np.array(x)
y = [[4, 5], [3, 4]]
y = np.array(y)

standard_conv = signal.convolve2d(x, y, 'full')

print("conv:", standard_conv)

s1 = np.array(x.shape)
s2 = np.array(y.shape)
print("s1: ", s1)
print("s2: ", s2)

size = s1 + s2 - 1
print("size: ", size)

fsize = 2 ** np.ceil(np.log2(size)).astype(int)
fslice = tuple([slice(0, int(sz)) for sz in size])
print("fslice: ", fslice)

# Along each axis, if the given shape (fsize) is smaller than that of the input, the input is cropped.
# If it is larger, the input is padded with zeros. if s is not given, the shape of the input along the axes
# specified by axes is used.
new_x = np.fft.fft2(x, fsize)

new_y = np.fft.fft2(y, fsize)
result = np.fft.ifft2(new_x * new_y)
print("first result: ", result)

result = np.fft.ifft2(new_x * new_y)[fslice].copy()
result_int = np.array(result.real, np.int32)

my_result = np.array(result, np.double)
print("my_result (doubles): ", my_result)

print("fft for my method (ints):", result_int)
print("is my method correct (for ints): ", np.array_equal(result_int, standard_conv))
print("fft for my method (doubles):", result)

print("fft with int32 output:", np.array(signal.fftconvolve(x, y), np.int32))
lib_result = np.array(signal.fftconvolve(x, y), np.double)
print("fft with double output:", np.allclose(my_result, lib_result, atol=1e-12))

# the correct way is to take the amplitude:  the abs of a complex number gives us its amplitude/mangnitude
lib_magnitude = np.abs(signal.fftconvolve(x, y))
print("lib_magnitude: ", lib_magnitude)
my_magnitude = np.abs(result)
print("is the magnitude correct: ", np.allclose(my_magnitude, lib_magnitude, atol=1e-12))


import numpy as np
from sympy import simplify, integrate, zeros, S, Matrix, symbols, pi, cos, sin
from .funcs_aproximacion import producto_asecas

def producto_escalar_trigono(f, g, var=symbols('x'), a=-pi, b=pi, I=None, numeric=False):
    """Aplica el producto escalar <f,g> = 1/(2pi) ∫_[-pi]^[pi] f.g

    Args:
        f (funcion): f
        g (funcion): g
        var (variable): variable de integración
        a (int, optional): limite inferior de integracion. Defaults to 0.
        b (int, optional): limite superior de integracion. Defaults to 1.
        I (list, optional): Si no es None, lista de valores sobre los que hacer un sumatorio discreto. Defaults to None.
        numeric (bool, optional): si True, realiza una aproximación numérica de la integral usando un método de sympy.

    Returns:
        funcion, float: Valor del producto escalar. Se devuelve como funcion si tiene variables.
    """
    prod = producto_asecas(f, g, var, a, b, I, numeric)
    return simplify(prod / (2 * pi))

def coefs_fourier(f, var=symbols('x'), I=[0, 1], n_coefs=2):
    """Genera los coeficientes de la serie de fourier. Esta es la versión continua, donde los coeficientes se calculan usando la expresión de la integral.

    Args:
        f (funcion): Función a aproximar
        var (variable, optional): Variable de la función. Defaults to symbols('x').
        I (list, optional): Intervalo de aproximación de la función. Defaults to [0, 1].
        n_coefs (int, optional): Número de coeficientes de la serie a generar. Defaults to 2.

    Returns:
        dict_coefs: {a_0, a_1, b_1, a_2, b_2, ...}
    """
    dict_coefs = {}
    dict_coefs['a0'] = simplify(1 / pi * integrate(f, (var, I[0], I[1])))
    for i in range(1, n_coefs):
        dict_coefs[f'a{i}'] = simplify(1 / pi * integrate(f * cos(i * var), (var, I[0], I[1])))
        dict_coefs[f'b{i}'] = simplify(1 / pi * integrate(f * sin(i * var), (var, I[0], I[1])))

    return dict_coefs

def coefs_fourier_discr(f, var=symbols('x'), I=[0, 1], n_coefs=2, m=10):
    """Genera los coeficientes de la serie de fourier. Esta es la versión donde la integral se aproxima como un sumatorio discreto de m términos sobre I.

    Args:
        f (funcion): Función a aproximar
        var (variable, optional): Variable de la función. Defaults to symbols('x').
        I (list, optional): Intervalo de aproximación de la función. Defaults to [0, 1].
        n_coefs (int, optional): Número de coeficientes de la serie a generar. Defaults to 2.
        m (int, optional): Número de elementos en los que dividir I para el sumatorio.

    Returns:
        dict_coefs: {a_0, a_1, b_1, a_2, b_2, ...}
    """
    dict_coefs = {}
    lista_xk = np.linspace(I[0], I[1], 2 * m)

    dict_coefs['a0'] = np.sum([f.subs(var, xk) * cos(0 * xk) for xk in lista_xk]) / m
    for i in range(1, n_coefs):
        dict_coefs[f'a{i}'] = np.sum([f.evalf(subs={var: S(xk)}) * cos(S(i) * xk) for xk in lista_xk]) / m
        dict_coefs[f'b{i}'] = np.sum([f.evalf(subs={var: S(xk)}) * sin(S(i) * xk) for xk in lista_xk]) / m

    return dict_coefs

def serie_fourier(f, var=symbols('x'), I=[0, 1], n_coefs=3, discreto=False, m=10):
    """Genera la serie de Fourier para la función f sobre un intervalo.

    Args:
        f (funcion): Función a aproximar
        var (variable, optional): Variable de la función. Defaults to symbols('x').
        I (list, optional): Intervalo de aproximación de la función. Defaults to [0, 1].
        n_coefs (int, optional): Número de coeficientes de la serie a generar. Defaults to 2.
        discreto (bool, optional): Si True, genera una aproximación discreta de los coeficientes empleando m términos.
        m (int, optional): Número de elementos en los que dividir I para el sumatorio.

    Returns:
        funcion: Función polinómica con la serie de Fourier.
    """
    if discreto:
        dict_coefs = coefs_fourier_discr(f, var, I, n_coefs, m)
    else:
        dict_coefs = coefs_fourier(f, var, I, n_coefs)

    serie_fourier = dict_coefs['a0'] / 2
    for i in range(1, n_coefs):
        serie_fourier += dict_coefs[f'a{i}'] * cos(i * var) + dict_coefs[f'b{i}'] * sin(i * var)

    return simplify(serie_fourier)


import matplotlib
matplotlib.use('TkAgg')
import matplotlib.pyplot as plt
import numpy as np
import random
import torch
import torch.optim as optim
import utils
import math

print('定义 adagrad')
eta = 0.4

def adagrad_2d(x1, x2, s1, s2):
    g1, g2, eps = 0.2 * x1, 4 * x2, 1e-6
    s1 += g1 ** 2
    s2 += g2 ** 2
    x1 -= eta / math.sqrt(s1 + eps) * g1
    x2 -= eta / math.sqrt(s2 + eps) * g2
    return x1, x2, s1, s2

def f_2d(x1, x2):
    return 0.1 * x1 ** 2 + 2 * x2 ** 2

print('lr= 0.4 的轨迹')
utils.show_trace_2d(f_2d, utils.train_2d(adagrad_2d))
print('lr= 2.0 的轨迹，更快逼近最优解')
eta = 2.0
utils.show_trace_2d(f_2d, utils.train_2d(adagrad_2d))

print('自行实现 Adagrad')

features, labels = utils.get_nasa_data()

def init_adagrad_states():
    s_w = torch.zeros((features.shape[1], 1), dtype=torch.float32)
    s_b = torch.zeros(1, dtype=torch.float32)
    return (s_w, s_b)

def adagrad(params, states, hyperparams):
    eps = 1e-6
    for p, s in zip(params, states):
        s.data += (p.grad.data ** 2)
        p.data -= hyperparams['lr'] * p.grad.data / torch.sqrt(s + eps)

print('Adagrad 进行优化')
utils.train_opt(adagrad, init_adagrad_states(), {'lr': 0.1}, features, labels)

print('简洁实现')
utils.train_opt_pytorch(optim.Adagrad, {'lr': 0.1}, features, labels)


"""
Evaluate the integral of x * exp(-a * x) with a = 2 on [0,1].
    Determine the relative error as a function of step size N and
    find N such that the approximate integral isaccurate to
    five significant digits

Solution to Problem Set 2, Problem 1

To run:
    python trap_rule_2_1.py

"""

import numpy as np
import matplotlib.pyplot as plt

#original trapezoidal rule; loop from 1 to n - 1
def trapezoidalRule(func, a, b, n, *P):
        h = (b - a) / n
        Int_part1 = func(a, *P) + func(b, *P)
        Int_part2 = 0
        for i in range(1, n):
            x = a + i * h
            Int_part2 += func(x, *P)
        I = (h / 2) * (Int_part1 + 2 * Int_part2)
        return I
#for part 4 of problem 1: modify step d to loop from 0 to n
def trapezoidalRule_modified(func, a, b, n, *P):
        h = (b - a) / n
        Int_part1 = func(a, *P) + func(b, *P)
        Int_part2 = 0
        for i in range(0, n+1):
            x = a + i * h
            Int_part2 += func(x, *P)
        I = (h / 2) * (Int_part1 + 2 * Int_part2)
        return I

#the function to be integrated
def integrand(x,alpha):
    return x*np.exp(-alpha*x)
#the indefinite integral
def int_indef(x,alpha):
    return ((-x/alpha) - (1./alpha**2)) * np.exp(-alpha*x)

alpha = 2.0
#integral boundaries
xmax = 1.0
xmin = 0.0

#analytic result
int_analytic = int_indef(xmax,alpha)-int_indef(xmin,alpha)
#build the n vs relErr arrays
N = np.array([50, 100, 200, 400])
rel_err = np.zeros(4)
for i in range(N.shape[0]):
    int_trap = trapezoidalRule(integrand, xmin, xmax, N[i], alpha)
    rel_err[i] = np.abs(int_trap - int_analytic) / int_analytic

#fit a line to log(N), log(rel_err) to get slope/convergence rate
m, b = np.polyfit(np.log(N), np.log(rel_err), 1)
print("convergence rate regular trapezoidal rule %.3f" % m)
plt.xlabel("log(number of steps)")
plt.ylabel("log(relative Error)")
plt.yscale('log')
plt.xscale('log')
#plot relative error
plt.plot(N, rel_err, marker ='x',label='Regular Trapezoidal Rule',color ='b')
# and best fit
plt.plot(N,np.exp(np.log(N)*m+b),linestyle = "--",color ='b')

#recalcuate the n and relErr arrays, but with the modified step d
rel_err_mod = np.zeros(4)
for i in range(N.shape[0]):
    int_trap = trapezoidalRule_modified(integrand, xmin, xmax, N[i], alpha)
    rel_err_mod[i] = np.abs(int_trap - int_analytic) / int_analytic

#fit a line to log(N), log(rel_err) to get slope/convergence rate
m, b = np.polyfit(np.log(N), np.log(rel_err_mod), 1)
print("convergence rate modified trapezoidal rule %.3f" % m)

#plot results
plt.plot(N, rel_err_mod, marker ='o',label='Modified Trapezoidal Rule',color ='r')
# and best fit
plt.plot(N,np.exp(np.log(N)*m+b),linestyle = "--",color ='r')
plt.legend()
plt.show()

#find an n accurate to at least five significant digits
N_5 = 10
int_trap = trapezoidalRule(integrand, xmin, xmax, N_5, alpha)
print("finding N_5...")
while np.abs(int_trap - int_analytic) / int_analytic >= 5.e-6:
    N_5 += 1
    int_trap = trapezoidalRule(integrand, xmin, xmax, N_5, alpha)
#    print("\tN=%d\t I=%.7e\taccuracy=%.6e"%(N_5,int_trap,np.abs(int_trap - int_analytic) / int_analytic))
#print the results
print("Integration with N = %d has relative error %e\naccurate to at least 5 digits!"%(N_5,np.abs(int_trap - int_analytic) / int_analytic))

Problem Sets/Problem Set 2/Solutions/trap1D_ps2_1.py


import numpy as np

def multivariate_gaussian_pdf(X : np.array, mu : np.array, cov : np.array) -> float:
    """
    Computes the likelihood of a vector w.r.t. the multivariate gaussian PDF.

    Parameters
    ==========
    X: numpy.array.
        The input array.
    mu: numpy.array.
        The mean array.
    cov: numpy.array.
        The covariance array.
    Returns
    ===========
    p: float.
        The likelihood of X given the mean mu and covariance cov.
    """
    diff = X.reshape(-1,1) - mu.reshape(-1,1)
    exp_factor = float( - 0.5 * np.dot( np.dot ( diff.T, np.linalg.inv(cov) ), diff ) )
    norm_factor = float ( np.linalg.det(2 * np.pi * cov) ** (-0.5) )
    p = norm_factor * np.exp( exp_factor )
    return p


# -*- coding: utf-8 -*-
"""
Created on Tue Oct 13 10:15:04 2020

#https://xavierbourretsicotte.github.io/Intro_optimization.html
"""

#matplotlib inline
import matplotlib.pyplot as plt
plt.style.use('seaborn-white')
import numpy as np
from mpl_toolkits import mplot3d

def Rosenbrock(x,y):
    return 100*(y-x**2)**2 + (1-x)**2

def Grad_Rosenbrock(x,y):
    g1 = 2*x + 2 - 400*x*y+400*x**3
    g2 = 200*y-200*x**2
    return np.array([g1,g2])

def Hessian_Rosenbrock(x,y):
    h11 = -400*y + 1200*x**2 + 2
    h12 = -400 * x
    h21 = -400 * x
    h22 = 200
    return np.array([[h11,h12],[h21,h22]])

def Gradient_Descent(Grad,x,y, gamma = 0.00125, epsilon=0.0001, nMax = 10000 ):
    #Initialization
    i = 0
    iter_x, iter_y, iter_count = np.empty(0),np.empty(0), np.empty(0)
    error = 10
    X = np.array([x,y])

    #Looping as long as error is greater than epsilon
    while np.linalg.norm(error) > epsilon and i < nMax:
        i +=1
        iter_x = np.append(iter_x,x)
        iter_y = np.append(iter_y,y)
        iter_count = np.append(iter_count ,i)
        #print(X)

        X_prev = X
        X = X - gamma * Grad(x,y)
        error = X - X_prev
        x,y = X[0], X[1]

    print(X)
    return X, iter_x,iter_y, iter_count

root,iter_x,iter_y, iter_count = Gradient_Descent(Grad_Rosenbrock,-2,2)

#PLOTTING THE SOLUTION

x = np.linspace(-2,2,250)
y = np.linspace(-1,3,250)
X, Y = np.meshgrid(x, y)
Z = Rosenbrock(X, Y)

#Angles needed for quiver plot
anglesx = iter_x[1:] - iter_x[:-1]
anglesy = iter_y[1:] - iter_y[:-1]

#matplotlib inline
fig = plt.figure(figsize = (16,8))

#Surface plot
ax = fig.add_subplot(1, 2, 1, projection='3d')
ax.plot_surface(X,Y,Z,rstride = 5, cstride = 5, cmap = 'jet', alpha = .4, edgecolor = 'none' )
ax.plot(iter_x,iter_y, Rosenbrock(iter_x,iter_y),color = 'r', marker = '*', alpha = .4)

ax.view_init(45, 280)
ax.set_xlabel('x')
ax.set_ylabel('y')

#Contour plot
ax = fig.add_subplot(1, 2, 2)
ax.contour(X,Y,Z, 50, cmap = 'jet')
#Plotting the iterations and intermediate values
ax.scatter(iter_x,iter_y,color = 'r', marker = '*')
ax.quiver(iter_x[:-1], iter_y[:-1], anglesx, anglesy, scale_units = 'xy', angles = 'xy', scale = 1, color = 'r', alpha = .3)
ax.set_title('Gradient Descent with {} iterations'.format(len(iter_count)))

plt.show()

#Newton's Method (Multi-Dimensional)

def Newton_Raphson_Optimize(Grad, Hess, x,y, epsilon=0.000001, nMax = 200):
    #Initialization
    i = 0
    iter_x, iter_y, iter_count = np.empty(0),np.empty(0), np.empty(0)
    error = 10
    X = np.array([x,y])

    #Looping as long as error is greater than epsilon
    while np.linalg.norm(error) > epsilon and i < nMax:
        i +=1
        iter_x = np.append(iter_x,x)
        iter_y = np.append(iter_y,y)
        iter_count = np.append(iter_count ,i)
        print(X)

        X_prev = X
        X = X - np.linalg.inv(Hess(x,y)) @ Grad(x,y)
        error = X - X_prev
        x,y = X[0], X[1]

    return X, iter_x,iter_y, iter_count

root,iter_x,iter_y, iter_count = Newton_Raphson_Optimize(Grad_Rosenbrock,Hessian_Rosenbrock,-2,2)

x = np.linspace(-3,3,250)
y = np.linspace(-9,8,350)
X, Y = np.meshgrid(x, y)
Z = Rosenbrock(X, Y)

#Angles needed for quiver plot
anglesx = iter_x[1:] - iter_x[:-1]
anglesy = iter_y[1:] - iter_y[:-1]

#matplotlib inline
fig = plt.figure(figsize = (16,8))

#Surface plot
ax = fig.add_subplot(1, 2, 1, projection='3d')
ax.plot_surface(X,Y,Z,rstride = 5, cstride = 5, cmap = 'jet', alpha = .4, edgecolor = 'none' )
ax.plot(iter_x,iter_y, Rosenbrock(iter_x,iter_y),color = 'r', marker = '*', alpha = .4)

#Rotate the initialization to help viewing the graph
ax.view_init(45, 280)
ax.set_xlabel('x')
ax.set_ylabel('y')

#Contour plot
ax = fig.add_subplot(1, 2, 2)
ax.contour(X,Y,Z, 60, cmap = 'jet')
#Plotting the iterations and intermediate values
ax.scatter(iter_x,iter_y,color = 'r', marker = '*')
ax.quiver(iter_x[:-1], iter_y[:-1], anglesx, anglesy, scale_units = 'xy', angles = 'xy', scale = 1, color = 'r', alpha = .3)
ax.set_title('Newton method with {} iterations'.format(len(iter_count)))

plt.show()

week 8 tutorial, Multivariate optimization.py


import numpy as np
from math import pi as PI

def estimate_pi(sims):
    """
    takes the number of simulations as input to estimate pi
    """

    # counter to hold points lying inside the circle
    in_circle = 0

    for s in range(0,sims):

        x = np.random.rand()
        y = np.random.rand()

        if (x**2 + y**2) <= 1:
            in_circle += 1

    # The ratio of pts. inside the circle and the total pts. will be same as the ratio
    # of the area of circle to the area of the square, inside which the circle is inscribed
    # Area of circle = PI * R * R
    # Area of square = (2R) * (2R)

    pi_estimated = 4.0 * in_circle / sims

    print("Simulations ran: ", sims)
    print("Estimated pi", pi_estimated)
    print("Error", PI - pi_estimated)

pow = 0
input_sims = 100
while pow <= 8:
    estimate_pi(sims=input_sims)
    pow += 1
    input_sims *= 10


# -*- coding: utf-8 -*-
"""
@author: Alex Muirhead
"""

import numpy as np
from numpy.core.numeric import ones_like

def fact(n):
    """Calculate the value of n factorial."""
    # n! = n*(n-1)*...*2*1
    product = 1
    for i in range(1, n+1):
        product *= i
    return product

def exptaylor(n, x):
    """Taylor series expansion of e^x

    NOTE:
        This will fail with x as a np.ndarray when
        n > 21, due to a quirk with division by
        large numbers (21! = 51090942171709440000).
        Just another reason to avoid this implemenation!
    """
    output = 0
    for i in range(n):
        output += x**i / fact(i)
    return output

def better_exptaylor(n, x):
    """
    Better Taylor series expansion of e^x

    term 0: 1
    term 1: x**1 / 1! == (term 0) * x
    term 2: x**2 / 2! == (term 1) * x / 2
    term 3: x**3 / 3! == (term 2) * x / 3
    """
    term = ones_like(x)
    output = term
    for i in range(1, n):
        term   *= x / i
        output += term
    return output

def lntaylor(n, x):
    """
    Efficient Taylor series expansion of ln(x+1) at x=1

    term 0: -1 {not included}
    term 1:   (x-1)**1 / (2**1 * 1)
    term 2: - (x-1)**2 / (2**2 * 2)
    term 3:   (x-1)**3 / (2**3 * 3)
    term 4: - (x-1)**4 / (2**4 * 4)
    """
    term = -1
    output = np.log(2.)  # <- 0th term
    for i in range(1, n):
        term *= (1-x) / 2
        output += term / i
    return output

if __name__ == '__main__':

    approx = better_exptaylor(100, 2)
    exact  = np.exp(2)
    error  = np.abs(exact - approx)
    print(f'The error for 100 terms was {error:.2e}')


"""
Jacobi Iteration Method - https://en.wikipedia.org/wiki/Jacobi_method
"""
from __future__ import annotations

import numpy as np

# Method to find solution of system of linear equations
def jacobi_iteration_method(
    coefficient_matrix: np.ndarray,
    constant_matrix: np.ndarray,
    init_val: list,
    iterations: int,
) -> list[float]:
    """
    Jacobi Iteration Method:
    An iterative algorithm to determine the solutions of strictly diagonally dominant
    system of linear equations

    4x1 +  x2 +  x3 =  2
     x1 + 5x2 + 2x3 = -6
     x1 + 2x2 + 4x3 = -4

    x_init = [0.5, -0.5 , -0.5]

    Examples:

    >>> coefficient = np.array([[4, 1, 1], [1, 5, 2], [1, 2, 4]])
    >>> constant = np.array([[2], [-6], [-4]])
    >>> init_val = [0.5, -0.5, -0.5]
    >>> iterations = 3
    >>> jacobi_iteration_method(coefficient, constant, init_val, iterations)
    [0.909375, -1.14375, -0.7484375]

    >>> coefficient = np.array([[4, 1, 1], [1, 5, 2]])
    >>> constant = np.array([[2], [-6], [-4]])
    >>> init_val = [0.5, -0.5, -0.5]
    >>> iterations = 3
    >>> jacobi_iteration_method(coefficient, constant, init_val, iterations)
    Traceback (most recent call last):
    ...
    ValueError: Coefficient matrix dimensions must be nxn but received 2x3

    >>> coefficient = np.array([[4, 1, 1], [1, 5, 2], [1, 2, 4]])
    >>> constant = np.array([[2], [-6]])
    >>> init_val = [0.5, -0.5, -0.5]
    >>> iterations = 3
    >>> jacobi_iteration_method(coefficient, constant, init_val, iterations)
    Traceback (most recent call last):
    ...
    ValueError: Coefficient and constant matrices dimensions must be nxn and nx1 but
                received 3x3 and 2x1

    >>> coefficient = np.array([[4, 1, 1], [1, 5, 2], [1, 2, 4]])
    >>> constant = np.array([[2], [-6], [-4]])
    >>> init_val = [0.5, -0.5]
    >>> iterations = 3
    >>> jacobi_iteration_method(coefficient, constant, init_val, iterations)
    Traceback (most recent call last):
    ...
    ValueError: Number of initial values must be equal to number of rows in coefficient
                matrix but received 2 and 3

    >>> coefficient = np.array([[4, 1, 1], [1, 5, 2], [1, 2, 4]])
    >>> constant = np.array([[2], [-6], [-4]])
    >>> init_val = [0.5, -0.5, -0.5]
    >>> iterations = 0
    >>> jacobi_iteration_method(coefficient, constant, init_val, iterations)
    Traceback (most recent call last):
    ...
    ValueError: Iterations must be at least 1
    """

    rows1, cols1 = coefficient_matrix.shape
    rows2, cols2 = constant_matrix.shape

    if rows1 != cols1:
        raise ValueError(
            f"Coefficient matrix dimensions must be nxn but received {rows1}x{cols1}"
        )

    if cols2 != 1:
        raise ValueError(f"Constant matrix must be nx1 but received {rows2}x{cols2}")

    if rows1 != rows2:
        raise ValueError(
            f"""Coefficient and constant matrices dimensions must be nxn and nx1 but
            received {rows1}x{cols1} and {rows2}x{cols2}"""
        )

    if len(init_val) != rows1:
        raise ValueError(
            f"""Number of initial values must be equal to number of rows in coefficient
            matrix but received {len(init_val)} and {rows1}"""
        )

    if iterations <= 0:
        raise ValueError("Iterations must be at least 1")

    table = np.concatenate((coefficient_matrix, constant_matrix), axis=1)

    rows, cols = table.shape

    strictly_diagonally_dominant(table)

    # Iterates the whole matrix for given number of times
    for i in range(iterations):
        new_val = []
        for row in range(rows):
            temp = 0
            for col in range(cols):
                if col == row:
                    denom = table[row][col]
                elif col == cols - 1:
                    val = table[row][col]
                else:
                    temp += (-1) * table[row][col] * init_val[col]
            temp = (temp + val) / denom
            new_val.append(temp)
        init_val = new_val

    return [float(i) for i in new_val]

# Checks if the given matrix is strictly diagonally dominant
def strictly_diagonally_dominant(table: np.ndarray) -> bool:
    """
    >>> table = np.array([[4, 1, 1, 2], [1, 5, 2, -6], [1, 2, 4, -4]])
    >>> strictly_diagonally_dominant(table)
    True

    >>> table = np.array([[4, 1, 1, 2], [1, 5, 2, -6], [1, 2, 3, -4]])
    >>> strictly_diagonally_dominant(table)
    Traceback (most recent call last):
    ...
    ValueError: Coefficient matrix is not strictly diagonally dominant
    """

    rows, cols = table.shape

    is_diagonally_dominant = True

    for i in range(0, rows):
        sum = 0
        for j in range(0, cols - 1):
            if i == j:
                continue
            else:
                sum += table[i][j]

        if table[i][i] <= sum:
            raise ValueError("Coefficient matrix is not strictly diagonally dominant")

    return is_diagonally_dominant

# Test Cases
if __name__ == "__main__":
    import doctest

    doctest.testmod()


import numpy as np

def BFGSAlgo(f,g,xd,epsilon=1e-6):
    """
       quasi-Newton method, use BFGS algorithm
       :param f:target function, it's a method having a float parameter.
       :param g:the derivative of the target function, it's a method having a float parameter.
       :param xd: the dimension of the vector
       :param epsilon: precision
       :return: the optimization point
    """
    #x=np.random.rand(xd)
    x=np.zeros(xd)
    B=np.eye(xd)
    gk=g(x)
    if np.linalg.norm(gk)<epsilon:
        return x
    while True:
        pk = np.linalg.solve(B, -1 * gk)
        #pk=solveEquation(B,-1*gk)

        abs=np.abs(gk)
        # print("grad",np.max(abs),np.argmax(abs))
        # print("function",f(x))
        # print("w",np.max(x),np.argmax(x),np.median(x),np.mean(x))
        # print()
        stepLength=calLambdaByArmijoRule(x,f(x),gk,pk,f)
        #stepLength=0.01
        xPre=x
        gkPre=gk

        x=x+stepLength*pk
        gk=g(x)
        if np.linalg.norm(gk)<epsilon:
            return x

        #update B
        yk=gk-gkPre
        deltaK=x-xPre
        deltaMatrix=np.matmul(deltaK.reshape(-1,1),deltaK.reshape(1,-1))
        B=B+(np.matmul(yk.reshape(-1,1),yk.reshape(1,-1))/np.sum(yk*deltaK))- \
          (np.matmul(np.matmul(B,deltaMatrix),B)/np.matmul(np.matmul(deltaK.reshape(1,-1),B),deltaK.reshape(-1,1)))

def calLambdaByArmijoRule(xCurr, fCurr, gCurr, pkCurr,f, c=1.e-4, v=0.5):
    """
    refer to https://www.cnblogs.com/xxhbdk/p/11785365.html
    to calculate lambda
    """
    i = 0
    alpha = v ** i
    xNext = xCurr + alpha * pkCurr
    fNext = f(xNext)

    while True:
        if fNext <= fCurr + c * alpha * np.sum(pkCurr*gCurr):
            break
        i += 1
        alpha = v ** i
        xNext = xCurr + alpha * pkCurr
        fNext = f(xNext)

    return alpha

#PALU decomposition
def PALU_Factorization(A: np.array):
    U = A.copy()
    P = np.eye(U.shape[0])
    L = np.zeros(U.shape)
    for index in range(U.shape[1]):
        maxIndex = index + np.argmax(U[index:, index])
        # exchange 2 rows
        #print(U[[index, maxIndex], :])
        P[[index, maxIndex], :] = P[[maxIndex, index], :]
        U[[index, maxIndex], :] = U[[maxIndex, index], :]
        L[[index, maxIndex], :] = L[[maxIndex, index], :]
        # eliminate non-zero elements
        for rIndex in range(index + 1, U.shape[0]):
            # try:
            #     assert U[index, index]!=0
            # except:
            #     print(index)
            #     print(U)
            multiFactor = U[rIndex, index] / U[index, index]
            U[rIndex, :] -= U[index, :] * multiFactor
            L[rIndex, index] = multiFactor

        # 给L加上对角线的1
    for i in range(U.shape[0]):
        L[i, i] = 1.0
    return P, L, U

# 使用PA=LU分解解方程
def solveEquation(A: np.array, b: np.array):
    P, L, U = PALU_Factorization(A)
    Pb = np.matmul(P, b.reshape(-1,1))
    # 此时方程为：LUx=Pb
    # 先解Lc=Pb
    c = np.zeros([L.shape[0], 1])
    for i in range(L.shape[0]):
        c[i, 0] = Pb[i, 0]
        for j in range(i):
            c[i, 0] -= L[i, j] * c[j, 0]

    # 再解Ux=c
    x = np.zeros([U.shape[0], 1])
    for i in range(U.shape[0] - 1, -1, -1):
        x[i, 0] = c[i, 0]
        for j in range(U.shape[1] - 1, i, -1):
            x[i, 0] -= U[i, j] * x[j, 0]
        x[i, 0] /= U[i, i]

    return x.reshape(-1,)

if __name__ == '__main__':
    f=lambda x:5*x[0]*x[0]+2*x[1]*x[1]+3*x[0]-10*x[1]+4
    g=lambda x:np.array([10*x[0]+3,4*x[1]-10])
    print(BFGSAlgo(f,g,2))
    #print(solveEquation(np.eye(50),np.array([i for i in range(50)])))


from itertools import ifilter
import numpy as np

def sieve(n):
    '''Recursive Sieve of Eratosthenes'''
    acc = []
    def aux(numList):
        try:
            k = numList[0]
            acc.append(k)
            aux(filter(lambda x: x % k, numList))
        except IndexError:
            pass
    aux(xrange(2,n))
    return acc

def sieve2(n):
    '''Iterative Sieve of Eratosthenes'''
    acc = []
    numList = xrange(2,n)
    while numList:
        k = numList[0]
        acc.append(k)
        numList = filter(lambda x: x % k, numList)
    return acc

def sieve3(n):
    '''Iterative Sieve of Eratosthenes using iterator'''
    acc = []
    numList = iter(xrange(2,n))
    try:
        while True:
            k = numList.next()
            acc.append(k)
            numList = ifilter(lambda x: # all(x % np.array(acc))
                reduce(lambda acc,a: acc and (x % a), acc, True)
                , numList)
    except StopIteration:
        pass
    return acc

def sieve4(n):
    '''Sieve of Eratosthenes in memory'''
    acc = []
    sieve_array = np.arange(2,n)
    next_index = 0
    for (i,k) in enumerate(sieve_array):
        if k != 0:
          sieve_array[sieve_array % k == 0] = 0
          sieve_array[i] = k # Store the prime back into the array
    return sieve_array[sieve_array != 0]

if __name__ == '__main__':
    import sys
    if len(sys.argv) > 1:
        n = int(sys.argv[1])
    else:
        n =100
    print 'There are {0} primes less than {1}'.format(len(sieve2(n)), n)


"""
File: yx_ODE.py
Copyright (c) 2016 Andrew Malfavon
License: MIT
Exercise C.3
Description: Solve an ODE using the Forward Euler method.
"""

import numpy as np
import sympy as sp
import matplotlib.pyplot as plt

#ode to be solved
def y_prime(y, x):
    return 1 / (2 * (y - 1))

#forward euler method based on code from book
def Forward_Euler(prime, a, b, dx, eps = 1e-3):
    n = int((b - a) / (dx))
    x = np.zeros(n + 1)
    y = np.zeros(n + 1)
    y[0] = 1 + np.sqrt(eps)
    x[0] = 0
    for i in range(n):
        x[i + 1] = x[i] + dx
        y[i + 1] = y[i] + dx * prime(y[i], x[i])
    return y, x

#solves the ode using sympy
#does not take into account initial condition
def sympy_solution():
    x = sp.Symbol('x')
    f = sp.Function('f')
    eq = 1 / (2 * (f(x) - 1))
    return sp.dsolve(sp.Eq(sp.diff(f(x)), eq))[1]#solves ode. first solution is negative so we use the second solution

#plot approximations with three step sizes and plots analytical solution
def plot(eps = 1e-3):
    x_exact = np.linspace(0, 4, 1001)
    y_exact = 1 + np.sqrt(x_exact + eps)
    x1 = Forward_Euler(y_prime, 0, 4, 1)[1]
    y1 = Forward_Euler(y_prime, 0, 4, 1)[0]
    x2 = Forward_Euler(y_prime, 0, 4, 0.25)[1]
    y2 = Forward_Euler(y_prime, 0, 4, 0.25)[0]
    x3 = Forward_Euler(y_prime, 0, 4, 0.01)[1]
    y3 = Forward_Euler(y_prime, 0, 4, 0.01)[0]
    plt.plot(x_exact, y_exact, label = 'Exact Solution')#labels are for the key in the graph
    plt.plot(x1, y1, label = 'step size = 1')
    plt.plot(x2, y2, label = 'step size = 0.25')
    plt.plot(x3, y3, label = 'step size = 0.01')
    plt.title('Approximations and exact solution of ODE')
    plt.xlabel('x')
    plt.ylabel('y')
    plt.legend(bbox_to_anchor=(1.05, 1), loc=2, borderaxespad=0.)#creates key for graph

#test the most accurate approximation with the analytical solution
def test():
    eps = 1e-3
    exact = 1 + np.sqrt(4 + eps)
    assert (Forward_Euler(y_prime, 0, 4, 0.01)[0][-1]) - (exact) < 0.01


# Importing required libraries

import numpy as np
import scipy.stats
import math
from scipy.stats import multivariate_normal

def normal_dist(x , mean , var):

    """
    Calculates a normal distribution in a variate x,
    with mean mu (mean) and variance sigma^2 (sd)
    as a statistic distribution with probability density function.
    source: https://mathworld.wolfram.com/NormalDistribution.html

    note that variance is std^2, e.g. a distribution with variance equal to 36, has a std dev of 6 (36 = 6^2).

    sigma^2 = sd, standard deviation
    sigma = variance
    """

    prob_density = (1/((2*np.pi*var)**(1/2))) * (np.exp(-0.5*((x-mean)**2/var)))

    return prob_density

def stackoverflow_normpdf(x, mean, sd):

    """
    Another alternative from: https://stackoverflow.com/a/12413491
    This uses the formula found here: http://en.wikipedia.org/wiki/Normal_distribution#Probability_density_function
    """
    var = float(sd)**2
    denom = (2*math.pi*var)**.5
    num = math.exp(-(float(x)-float(mean))**2/(2*var))
    return num/denom

def scipy_norm_pdf(x, mean, sd):

    #Using scipy library for calculating the normal probability distribution
    # loc being the mean, varies for probability of 'yes' vs 'no'
    # scale being the sd (not variance), varies for probability of 'yes' vs 'no'
    # use cdf to indicate the value at which we would like to calculate the probability; e.g. x = 4 in this particular scenario

    prob = scipy.stats.norm(mean, sd).pdf(x)
    prob_dist = scipy.stats.norm(mean, sd)

    return prob, prob_dist

x = 4
mean_yes = 10
mean_no = 0
var = 36
sd = var**(1/2)

#using the function we have created

our_prob_dist_yes = normal_dist(x, mean_yes, var)
our_prob_dist_no = normal_dist(x, mean_no, var)

print(f"Using the function we have created, for \n X = {x} \n mean_yes = {mean_yes} \n mean_no= {mean_no} n var = {var} \n the probability for Yes is {our_prob_dist_yes} \n and the probability for No is {our_prob_dist_no}")

#using the alternative method from stackoverflow and wikipedia formulation; note you are passing on here sd and not variance (e.g. passing on sd and not var = sd^(2)
alt_prob_dist_yes = stackoverflow_normpdf(x, mean_yes, sd)
alt_prob_dist_no = stackoverflow_normpdf(x, mean_no, sd)

print(f"Using the alternative method from stackoverflow and wikipedia formulation, for \n X = {x} \n mean_yes = {mean_yes} \n mean_no = {mean_no} \n var = {var} the probability for Yes is {alt_prob_dist_yes} \n and the probability for No is {alt_prob_dist_no}")

#using scipy library to compare/validate above function
scipy_prob_yes, scipy_prob_dist_yes = scipy_norm_pdf(x, mean_yes, sd)
scipy_prob_no, scipy_prob_dist_no = scipy_norm_pdf(x, mean_no, sd)

print(f"Using scipy library to compare/validate above function, for \n X = {x} \n mean_yes = {mean_yes} \n mean_no = {mean_no} \n var = {var} the probability for Yes is {scipy_prob_yes} \n and the probability for No is {scipy_prob_no}")


from sympy import *

t, a, Q, w = symbols('t a A w', positive=True, real=True)
u = Q*exp(-a*t)*sin(w*t)
dudt = diff(u, t)
dudt
factor(dudt)
simplify(dudt)
# Alternative, manually derived expression
phi = atan(-a/w)
A = Q*sqrt(a**2 + w**2)
dudt2 = exp(-a*t)*A*cos(w*t - phi)

simplify(expand_trig(dudt2))
simplify(expand_trig(dudt2 - dudt))  # are they equal?
s = solve(dudt2, t)
s


'''
Print the series:
x + x**2 + x**3 ... + x**n
'''

from sympy import Symbol, pprint, init_printing

def print_series( n, x_value ):

	# Initialize printing system with reverse order
	init_printing( order = 'rev-lex' )

	x = Symbol( 'x' )

	series = x
	for i in range( 2, n + 1 ):
		series = series + ( x ** i ) / i
	pprint( series )

	series_value = series.subs( { x: x_value } )

	print( 'Value of the series at {0}: {1}'.format( x_value, series_value ) )

if __name__ == '__main__':
	n = input( 'Enter the number of terms you want in the series: ' )
	x_value = input( 'Enter the value of x at which you want to evaluate the series: ' )

	print_series( int( n ), float( x_value ) )


#NAME:
#  K_ARR
#PURPOSE:
#  Generate an array of wavenumber values (by default, in units of cycles per
#  sample interval), arranged corresponding to the FFT of an N-element array.
#CALLING SEQUENCE:
#  k = k_arr(N [, dx=dx] [,/radians])
#OUTPUT:
#  k = N-element array of positive and negative wavenumbers, where the maximum
#     (Nyquist) frequency has a value of pi radians per sample interval. The
#     dx and cycles keywords can be used to modify the units of k.
#INPUTS:
#  N = Number of elements. The program has been verified for both odd and
#     even N.
#OPTIONAL KEYWORD INPUTS:
#  dx = sample interval. Default=1. If included, then k will be converted into units
#     of cycles (or radians, if that keyword is set) per unit distance (or time),
#     corresponding to the units of dx.
#  radians = if set, then let k be in radians (rather than cycles).
#HISTORY:
#  2013-Jun-25 C. Kankelborg
#  2019 Jun 21  JTE Translated into Python 3

import numpy as np

def k_arr(N,
          dx = 1.0,
          radians = False):

    dk = np.power(N * dx, -1.0)  #frequency interval equals the fundamental frequency
       #of one cycle per N samples, or one cycle per total distance N*dx.

    k = dk * [ np.arange(np.floor_divide(N, 2.0) + 1), -np.arange(np.ceil(0.5 * N - 1))[::-1] + 1  ]
    #        [  positive frequencies,        negative frequencies              ]

    if radians:

        k *= 2.0 * np.pi #convert from cycles to radians.

    return k


import numpy as np
itr_limit = 20
conv_crit = 0.0000001

def stop_cond(A, b, x):

	n = len(b)
	res = (A@x - b)
	magnitude = (res@res)**0.5

	if magnitude < conv_crit:
		return True
	else:
		return False

def jacobi(A, b, N = -1, x_old = None):

	n = len(b)
	itr = 0
	converged = False
	x_new = np.empty(n)
	if x_old == None:
		x_old = np.zeros(n)

	while not converged and itr < itr_limit and itr != N:

		for i in range (n):
			dot = 0
			for j in range(n):
				if(i!=j):
					dot += A[i,j]*x_old[j]

			x_new[i] = (b[i] - dot)/A[i,i]

		itr += 1
		converged = stop_cond(A,b,x_new)
		x_old = x_new.copy()

	if not converged:
		print("Iteration limit reached without convergence.")

	return x_new

M = np.array([[4,1],[1,4]],dtype=float)
x = np.array([5,5],dtype=float)
print(jacobi(M,x))