import numpy as np
from numpy import exp
from numpy import polyfit
from numpy import log
import matplotlib.pyplot as plt

RImageDistance,RObjectDistance,Magnification,ObjectDistance = np.loadtxt(r"2021.11.25 Lenses - Computing/lenses-experiment-a.csv",delimiter=",",skiprows=1,unpack=True)
font = {'fontname':'CMU Serif'}                                                                             # Assign font parameters
fontAxesTicks = {'size':7}

    ## 1/s against 1/s'
polyfitA,cov_polyfitA = np.polyfit(RObjectDistance,RImageDistance,1,cov=True)
plt.xlabel("Object Distance¯¹ / m¯¹", **font)                                                                        # Label axes, add titles and error bars
plt.ylabel("Image Distance¯¹ / m¯¹", **font)
plt.xticks(**font, **fontAxesTicks)
plt.yticks(**font, **fontAxesTicks)
plt.title("Experiment A: 1/s against 1/s'", **font)
plt.plot(RObjectDistance,RImageDistance,'x')
plt.plot(RObjectDistance, (polyfitA[0]*RObjectDistance+polyfitA[1]))
plt.show()
print(polyfitA)

polyfitB,cov_polyfitB = np.polyfit(RImageDistance,RObjectDistance,1,cov=True)
plt.xlabel("Image Distance¯¹ / m¯¹", **font)                                                                        # Label axes, add titles and error bars
plt.ylabel("Object Distance¯¹ / m¯¹", **font)
plt.xticks(**font, **fontAxesTicks)
plt.yticks(**font, **fontAxesTicks)
plt.title("Experiment A: 1/s against 1/s'", **font)
plt.plot(RImageDistance,RObjectDistance,'x')
plt.plot(RImageDistance, (polyfitB[0]*RImageDistance+polyfitB[1]))
plt.show()
print(polyfitB)

    ## M against s
#polyfitB,cov_polyfitB = np.polyfit(b,h,1,cov=True)
plt.xlabel("Object Distance / m", **font)                                                                        # Label axes, add titles and error bars
plt.ylabel("Magnification / no units", **font)
plt.xticks(**font, **fontAxesTicks)
plt.yticks(**font, **fontAxesTicks)
plt.title("Experiment A: M against s'", **font)
#plt.errorbar(f,e, yerr=((1/(f+0.1))-(1/(f-0.1))),xerr=(1/(e+0.1)-1/(e-0.1)),ls='',mew=1.5,ms=3,capsize=3)                         # Plots uncertainties in points
plt.plot(ObjectDistance,Magnification,'x')
#plt.plot(f, (polyfitA[0]*f+polyfitA[1]))
plt.show()

''' plt.plot(1/objectDistance, 1/imageDistance,'x')
polyfitA,cov_polyfitA = np.polyfit(1/objectDistance,1/imageDistance,2,cov=True)
#plt.plot(1/objectDistance, (polyfitA[0]*(1/objectDistance)**2+polyfitA[1]*(1/objectDistance)+polyfitA))
plt.xlabel("Time (t) / s", **font)                                                                          # Label axes, add titles and error bars
plt.ylabel("Natural Log of Voltage (ln V) / V", **font)
plt.xticks(**font, **fontAxesTicks)
plt.yticks(**font, **fontAxesTicks)
#plt.errorbar(distanceDS, grayValueDS,yerr=0,xerr=0,ls='',mew=1.5,ms=3,capsize=3)
plt.title("Small Capacitor, Discharging (Linear)", **font)
plt.show() '''


from scipy.integrate import odeint
class SIS:

	sets = ['S', 'I', 'N']
	params = ['beta', 'gamma']
	equations = {
		'S' : lambda S,I,N,_S,_I,_N,beta,gamma: f' -({beta} * {S} * {_I}) / ({_N}) + {gamma} * {I}',
		'I' : lambda S,I,N,_S,_I,_N,beta,gamma: f' ({beta} * {S} * {_I}) / ({_N}) - {gamma} * {I}',
		'N' : lambda S,I,N,_S,_I,_N,beta,gamma: f' 0',
	}

	@staticmethod
	def deriv(y, t, params):
		S, I, N = y
		beta, gamma = params
		dSdt = -(beta * S * I) / (N) + gamma * I
		dIdt = (beta * S * I) / (N) - gamma * I
		dNdt = 0
		return dSdt, dIdt, dNdt

	@staticmethod
	def solve(y, t, params):
		return odeint(SIS.deriv, y, t, args=(params,))


import numpy as np
import random

def inference (w,b,x):
    pred_y=w*x+b
    return pred_y

def gradient (pred_y,gt_y,x):
    dw=(pred_y-gt_y)*x
    db=pred_y-gt_y
    return  dw, db

def theta_step_update (batch_x_list, batch_gt_y_list, w, b, lr):
    avg_dw=0
    avg_db=0
    batch_size=len(batch_x_list)

    for i in range (batch_size):
        pred_y=inference(w,b,batch_x_list[i])
        dw,db=gradient(pred_y,batch_gt_y_list[i],batch_x_list[i])
        avg_dw+= dw
        avg_db+= db
    avg_dw/=batch_size
    avg_db/=batch_size

    #update simultanously
    w=w-lr*avg_dw
    b=b-lr*avg_db

    return w,b

def loss_function (batch_x_list, batch_gt_y_list, w, b):
    batch_size=len(batch_x_list)
    sum_loss=0
    for i in range (batch_size):
        single_loss=(w*batch_x_list[i]+b-batch_gt_y_list[i])**2
        sum_loss+=single_loss
    J=sum_loss/2*batch_size
    return J

def train(x_list, gt_y_list, batch_size,lr, maxIrr):
    #initial setup: pick up a starting point for (w,b);
    w=0
    b=0
    total_size=len(x_list)

    # randomly choose batch size of samples from the whole data
    # throw the batch into the iteration as training data for each round
    for i in range (maxIrr):
        batch_idx=np.random.choice(total_size,batch_size)
        batch_x_list=[x_list[j] for j in batch_idx]
        batch_gt_y_list=[gt_y_list[j] for j in batch_idx]
        w,b=theta_step_update(batch_x_list,batch_gt_y_list,w,b,lr)
        J=loss_function(batch_x_list,batch_gt_y_list,w,b)
        print('w:{0}, b:{1}'.format(w, b))
        print('loss: {0}'.format(J))
    return w,b,J

def generate_data(num_samples):
    w=random.randint(0,10)+random.random()
    b=random.randint(0,5)+random.random()
    x_list=[]
    gt_y_list=[]
    for i in range (num_samples):
        x = random.randint(0, 100) * random.random()
        y=w*x+b+random.random()* random.randint(-1, 1)
        x_list.append(x)
        gt_y_list.append(y)
    return w,b,x_list,gt_y_list

def run(num_samples, batch_size,lr,maxIrr):
    w0,b0,x_list,gt_y_list=generate_data(num_samples)
    w,b,J=train(x_list, gt_y_list, batch_size,lr, maxIrr)
    print ('original w: {0}, b:{1}'.format(w0,b0))
    print ('estimated w: {0}, b:{1}'.format(w,b))
    print ('final loss J: {0}'.format(J))

if __name__=='__main__':
    num_samples=100
    batch_size=50
    lr=0.001
    maxIrr=10000
    run(num_samples, batch_size,lr,maxIrr)


import numpy
import random
import time

def integral_function(x):
    return(1/(numpy.cos(x) + 2))
    # result: https://www.wolframalpha.com/input/?i=integrate+1%2F(cos(x)+%2B+2)+from+0+to+6

def monte_carlo(xmin, xmax, ymin, ymax, N):
    whole_area = (xmax - xmin) * (ymax - ymin)
    points = 0

    time_start = time.time()

    for i in range(N):
        x = xmin + (xmax - xmin) * random.random()
        y = ymin + (ymax - ymin) * random.random()

        if integral_function(x) > 0 and y > 0 and y <= integral_function(x):
            points += 1

    time_stop = time.time()
    total_time = time_stop - time_start
    result = whole_area * points / N

    return {'result': result, 'time': total_time, 'N': N}

# define limits
xmin = 0
xmax = 6
ymin = 0
x = numpy.linspace(xmin, xmax, 1000)
ymax = max(integral_function(x)) + 0.5 * max(integral_function(x))
Nmin = 100
Nmax = 10000

while Nmin <= Nmax:
    calc = monte_carlo(xmin, xmax, ymin, ymax, Nmin)
    print("Result: ", calc['result'], "Total time:", calc['time'], "N = ", calc['N'])
    Nmin = Nmin * 10


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Mon Jan 28 14:21:06 2019

@author: alankar
"""

import numpy as np
import matplotlib.pyplot as plt

def derivative(func,x,*args): #function to find the derivative of the polynomial
    h = 1e-4
    derivative = (func(x + h,*args) - func(x-h,*args)) / (2*h)
    return derivative

def NR(func,x0,tol,*args): #Newton Raphson
    x_new = 0
    x_old = x0
    while(np.abs(x_new-x_old)>=tol):
        x_new = x_old - func(x_old,*args)/derivative(func,x_old,*args)
        x_old = x_new
    return x_new

def f(x):
    return 924*x**6-2772*x**5+3150*x**4-1680*x**3+420*x**2-42*x+1

x = np.linspace(0,1,500)
plt.plot(x,f(x))
plt.xlabel(r'$x$',size=18)
plt.ylabel(r'$f(x)=924x^6-2772x^5+3150x^4-1680x^3+420x^2-42x+1$',size=18)
plt.grid()
#plt.savefig('poly.png')
plt.show()

print ('Roots are ')
x0 = [0.03,0.17,0.38,0.62,0.83,0.97]
for i in range(len(x0)):
    print( '%.10f'%NR(f,x0[i],1.e-10))

"""
Output

Roots are
0.0335761962
0.1693944272
0.3806901845
0.6193098155
0.8306055728
0.9664238038
"""


import numpy as np
import time
import pandas as pd

def main():
    # initi conditions---------------------------------------------------------
    A=2
    B=0.01
    t0=0.0
    t_end=5
    #--------------------------------------------------------------------------

    # Analytic solution
    analytic = lambda t_ini, y_ini, t, a, b: 1/(b/a+(1/y_ini-b/a)*np.exp(-a*(t-t_ini)))
    #--------------------------------------------------------------------------
    # ODE
    f = lambda y: A*y-B*y*y
    #--------------------------------------------------------------------------
    # initial valuse
    y0_list = np.array([8, 10, 12])
    #y0_list = np.array([100, 190, 250])
    #--------------------------------------------------------------------------

    timesteps = np.array([0.05, 0.025, 0.0125, 0.00625, 0.003125])

    # maps of 4th order for different time steps (A=2, B=0.01)
    # M(0,:) is map for dt=0.05     (N=100)
    # M(1,:) is map for dt=0.025    (N=200)
    # M(2,:) is map for dt=0.0125   (N=400)
    # M(3,:) is map for dt=0.00625  (N=800)
    # M(4,:) is map for dt=0.003125 (N=1600)

    Maps = np.array([[-9.74718510761581e-11,2.82830795154941e-7,-0.00057683855758959,1.1047522448889,0.0164019262867156],
                    [-1.379332373516e-11,6.75987803436825e-8,-0.00026920890283235,1.05124251835198,0.00113151812515],
                    [-1.83493448551926e-12,1.63316814751669e-8,-0.00012976106224663,1.0253132530535,7.43225401875509e-5],
                    [-2.36633829318494e-13,3.99916101441112e-9,-6.36821468173527e-5,1.01257833218163,4.76237954022738e-6],
                    [-3.00445936310731e-14,9.88470260129351e-10,-3.15443221011727e-5,1.00626956445964,3.0138672729979e-7]
                    ])

    #--------------------------------------------------------------------------
    # resulting table
    # result(:, 0) is N
    # result(:, 1) is err_RK4
    # result(:, 2) is err_TM4
    # result(:, 3) is time_RK4
    # result(:, 4) is time_TM4
    # result(:, 5) is time_ratio = time_RK4/time_TM4
    result = np.zeros((len(timesteps), 6))

    for k, dt in enumerate(timesteps):  # for each time stemp dt
        M = Maps[k, :]                 # get TM for this dt
        N = int((t_end-t0)/dt)
        t = np.arange(t0, t_end, dt)

        result[k, 0] = N

        for y0 in y0_list: # for each y0
            # analytic solution------------------------------------------------
            y_sol = analytic(t0, y0, t, A, B)
            #------------------------------------------------------------------

            # RK4 integration--------------------------------------------------
            y_rk4 = np.zeros(N)

            y_rk4[0]=y0
            tic = time.time()
            for i in range(N-1):
                y = y_rk4[i]
                k1 = f(y)
                k2 = f(y+dt*k1/2)
                k3 = f(y+dt*k2/2)
                k4 = f(y+dt*k3)
                y_rk4[i+1] = y + dt*(k1+2*k2+2*k3+k4)/6

            elapsed_time = time.time()-tic
            result[k, 3] += elapsed_time # time_RK4
            #------------------------------------------------------------------

            # Mapping----------------------------------------------------------
            y_map = np.zeros(N)
            y_map[0] = y0
            tic = time.time()
            for i in range(N-1):
                y=y_map[i]
                y2 = y*y
                y_map[i+1] = (y2*(M[0]*y2 + M[2]) +
                            y *(M[1]*y2 + M[3]) +
                            M[4])

            elapsed_time = time.time()-tic
            result[k, 4] += elapsed_time # time_RK4
            #------------------------------------------------------------------
            result[k, 1] += np.abs(y_rk4 - y_sol).max() # err_RK4
            result[k, 2] += np.abs(y_map - y_sol).max() # err_TM4

    result[:, 1:] /= len(y0_list) # get average results
    result[:, 5] = result[:, 3]/result[:, 4] # get time_ratio

    result = pd.DataFrame(data=result[:,1:], index=result[:,0], columns=np.array(['err_RK4', 'err_TM4', 'time_RK4', 'time_TM4', 'time_ratio']))
    return result

if __name__ == "__main__":
    print(main())


# Create by Packetsss
# Personal use is allowed
# Commercial use is prohibited

from timeit import default_timer as timer
import numpy as np

"""
How many possible ways a traveller can travel through a N x M grid?

1x1: [0]
1 way

1x2: [0, 0]
1 way

2x2: [0, 0]
     [0, 0]
2 ways

2x3: [0, 0, 0] == 2 + 1
     [0, 0, 0]
3 ways

3x3: [0, 0, 0]    [0, 0, 0]   [0, 0]
     [0, 0, 0] == [0, 0, 0] + [0, 0] == 2 + 1 + 2 + 1
     [0, 0, 0]                [0, 0]

6 ways

3x4: [0, 0, 0, 0]    [0, 0, 0]   [0, 0, 0, 0]
     [0, 0, 0, 0] == [0, 0, 0] + [0, 0, 0, 0] == 6 + 3 + 1
     [0, 0, 0, 0]    [0, 0, 0]

"""

def grid_traveller(n, m):
    if n == 0 or m == 0:
        return 0
    elif n < 2 and m < 2:
        return min(n, m)
    else:
        return grid_traveller(n - 1, m) + grid_traveller(n, m - 1)

start = timer()
print(grid_traveller(13, 10))
end = timer()
print(end - start)
# Slow naive recursive solution

### Memoized
d = {}
def grid_traveller_topdown(n, m):
    key = f"{n}{m}"
    if n == 0 or m == 0:
        return 0
    elif n < 2 and m < 2:
        return min(n, m)
    elif key in d:
        return d[key]
    else:
        d[key] = grid_traveller_topdown(n - 1, m) + grid_traveller_topdown(n, m - 1)
        return d[key]

start = timer()
print(grid_traveller_topdown(13, 100))
end = timer()
print(end - start)

### Tabulation
def grid_traveller_bottom_up(n, m):
    if n == 0 or m == 0:
        return 0

    lst = np.zeros((n + 1, m + 1))
    lst[1:, 1] = 1
    lst[1, 1:] = 1

    for i in range(2, n + 1):
        for j in range(2, m + 1):
            lst[i, j] = lst[i - 1, j] + lst[i, j - 1]
    return int(lst[-1, -1])

start = timer()
print(grid_traveller_bottom_up(13, 100))
end = timer()
print(end - start)
# 2x faster

# best to use 0 for counting problems
def grid_traveller_bottom_up_1(n, m):
    table = [[0 for _ in range(n + 1)] for _ in range(m + 1)]
    # correct way to create a 2d array

    table[1][1] = 1

    # add the current value of i to it's right and bottom
    for i in range(1, m + 1):
        for j in range(1, n + 1):
            cur = table[i][j]
            if i < m:
                table[i + 1][j] += cur
            if j < n:
                table[i][j + 1] += cur

    return table[m][n]

start = timer()
print(grid_traveller_bottom_up_1(13, 100))
end = timer()
print(end - start)

Python Tutorial Dynamic Programming/2_Grid_Traveller.py


# -*- coding: utf-8 -*-

import numpy as np, tensorflow as tf, seaborn as sns, arviz as az, tensorflow_probability as tfp
tf.config.optimizer.set_jit(True)
tfd, tfb = tfp.distributions, tfp.bijectors

def snsdistplot(data):
	g = sns.displot(data, bins = "sqrt", kde = True)
	g.fig.set_figwidth(8)
	g.fig.set_figheight(6)
	return g

data = np.loadtxt("data/mixture_data.csv")
snsdistplot(data)

var_name = ["prob", "centers", "sigmas"]

@tfd.JointDistributionCoroutineAutoBatched
def mdl_batch():
	prob = yield tfd.Uniform(low = 0., high = 1., name = var_name[0])
	centers = yield tfd.Normal(loc = [120., 190.], scale = [10.]*2, name = var_name[1])
	sigmas = yield tfd.Uniform(low = [0.]*2, high = [100.]*2, name = var_name[2])
	categories = tfd.Categorical(probs = [prob, 1. - prob]) # assignments to a group
	values = tfd.Normal(loc = centers, scale = sigmas) # group
	mixture = tfd.MixtureSameFamily(mixture_distribution = categories, components_distribution = values)
	yield tfd.Sample(mixture, sample_shape = len(data), name = "obs")

# a helper function in McMC chain
def trace_fn(current_state, kernel_results):
	mdr = kernel_results.inner_results.inner_results
	return mdr.target_log_prob, mdr.leapfrogs_taken, mdr.has_divergence, mdr.energy, mdr.log_accept_ratio

@tf.function(autograph = False, experimental_compile = True) # speed up a lot the McMC sampling
def run_mcmc( # pass numeric arguments as Tensors whenever possible
	init_state, unconstraining_bijectors,
	num_steps = 50000, burnin = 10000,
	num_leapfrog_steps = 3, step_size = .5
):
	kernel0 = tfp.mcmc.NoUTurnSampler(
		target_log_prob_fn = lambda *args: mdl_batch.log_prob(obs=data, *args),
		step_size = step_size
	)
	kernel1 = tfp.mcmc.TransformedTransitionKernel(
		inner_kernel= kernel0,
		bijector = unconstraining_bijectors
	)
	kernel2 = tfp.mcmc.DualAveragingStepSizeAdaptation( # pkr = previous kernel results
		inner_kernel = kernel1,
		num_adaptation_steps = int(0.8*burnin),
		step_size_setter_fn = lambda pkr, new_step_size: pkr._replace(inner_results = pkr.inner_results._replace(step_size=new_step_size)),
		step_size_getter_fn = lambda pkr: pkr.inner_results.step_size,
		log_accept_prob_getter_fn = lambda pkr: pkr.inner_results.log_accept_ratio
	)
	# tf.get_logger().setLevel("ERROR") # multiple chains
	return tfp.mcmc.sample_chain( # ATTENTION: 2 values to unpack
		num_results = num_steps,
		num_burnin_steps = burnin,
		current_state = init_state,
		kernel = kernel2,
		trace_fn = trace_fn
	)

nchain = 4
init_state = [mdl_batch.sample(nchain)._asdict()[_] for _ in var_name]
unconstraining_bijectors = [tfb.Identity()]*len(var_name)
samples, sampler_stat = run_mcmc(init_state, unconstraining_bijectors)

#%% using the pymc3 naming convention, with log_likelihood instead of lp so that ArviZ can compute loo and waic
sample_stats_name = ['log_likelihood', 'tree_size', 'diverging', 'energy', 'mean_tree_accept']

sample_stats = {k: v.numpy().T for k, v in zip(sample_stats_name, sampler_stat)}
posterior = {k:np.swapaxes(v.numpy(), 1, 0) for k, v in zip(var_name, samples)}
az_trace = az.from_dict(posterior = posterior, sample_stats = sample_stats)

snsdistplot(posterior["prob"])
snsdistplot(posterior["centers"][:, 0])

# put the data into a tensor
datatf = tf.constant(data, dtype = tf.float32)[:, tf.newaxis]

# This produces a cluster per MCMC chain
rv_clusters_1 = tfd.Normal(posterior["centers"][:, 0], posterior["sigmas"][:, 0])
rv_clusters_2 = tfd.Normal(posterior["centers"][:, 1], posterior["sigmas"][:, 1])

# Compute the un-normalized log probabilities for each cluster
cluster_1_log_prob = rv_clusters_1.log_prob(datatf) + tf.math.log(posterior["prob"])
cluster_2_log_prob = rv_clusters_2.log_prob(datatf) + tf.math.log(1. - posterior["prob"])

# Bayes rule to compute the assignment probability: P(cluster = 1 | data) ∝ P(data | cluster = 1) P(cluster = 1)
log_p_assign_1 = cluster_1_log_prob - tf.math.reduce_logsumexp(tf.stack([cluster_1_log_prob, cluster_2_log_prob], axis=-1), -1)

# Average across the MCMC chain
log_p_assign_1bis = tf.math.reduce_logsumexp(log_p_assign_1, -1) - tf.math.log(tf.cast(log_p_assign_1.shape[-1], tf.float32))

p_assign_1 = tf.exp(log_p_assign_1bis)
p_assign = tf.stack([p_assign_1, 1 - p_assign_1], axis=-1)

assign_trace = log_p_assign_1bis.numpy()[np.argsort(data)]
plt.scatter(data[np.argsort(data)], assign_trace, cmap = "RdBu",c = (1 - assign_trace), s = 50)
plt.title("Probability of data point belonging to cluster 0")
plt.ylabel("probability")
plt.xlabel("value of data point")


import numpy as np

# Helper functions
def _predict(theta_0, theta_1, x):
    return theta_1 * x + theta_0       # good old line equation from high school when we were all young and happy

def compute_mse(theta_0, theta_1, data):
    """
    Calcula o erro quadratico medio
    :param theta_0: float - intercepto da reta
    :param theta_1: float -inclinacao da reta
    :param data: np.array - matriz com o conjunto de dados, x na coluna 0 e y na coluna 1
    :return: float - o erro quadratico medio
    """
    n = len(data)

    predicted = [_predict(theta_0, theta_1, element[0]) for element in data]

    square_error = lambda x, y: (x - y)**2
    squared_errors = [square_error(predicted[i], data[i][1]) for i in range(n)]

    mse = sum(squared_errors) / n

    return mse

def step_gradient(theta_0, theta_1, data, alpha):
    """
    Executa uma atualização por descida do gradiente  e retorna os valores atualizados de theta_0 e theta_1.
    :param theta_0: float - intercepto da reta
    :param theta_1: float -inclinacao da reta
    :param data: np.array - matriz com o conjunto de dados, x na coluna 0 e y na coluna 1
    :param alpha: float - taxa de aprendizado (a.k.a. tamanho do passo)
    :return: float,float - os novos valores de theta_0 e theta_1, respectivamente
    """
    n = len(data)

    predicted = [_predict(theta_0, theta_1, element[0]) for element in data]

    simple_error = lambda x, y: x - y
    errors = [simple_error(predicted[i], data[i][1]) for i in range(n)]

    # Both expressions below can be checked on page 14 of the second set of slides on week 7 ("Otimizacao Continua.pdf")
    df_d0 = (2 * sum(errors)) / n
    df_d1 = (2 * sum([errors[i] * data[i][0] for i in range(n)])) / n

    new_theta_0 = theta_0 - alpha*df_d0
    new_theta_1 = theta_1 - alpha*df_d1

    return (new_theta_0 , new_theta_1)

def fit(data, theta_0, theta_1, alpha, num_iterations):
    """
    Para cada época/iteração, executa uma atualização por descida de
    gradiente e registra os valores atualizados de theta_0 e theta_1.
    Ao final, retorna duas listas, uma com os theta_0 e outra com os theta_1
    obtidos ao longo da execução (o último valor das listas deve
    corresponder à última época/iteração).

    :param data: np.array - matriz com o conjunto de dados, x na coluna 0 e y na coluna 1
    :param theta_0: float - intercepto da reta
    :param theta_1: float -inclinacao da reta
    :param alpha: float - taxa de aprendizado (a.k.a. tamanho do passo)
    :param num_iterations: int - numero de épocas/iterações para executar a descida de gradiente
    :return: list,list - uma lista com os theta_0 e outra com os theta_1 obtidos ao longo da execução
    """
    current_iteration = 1
    all_theta_zeroes = []
    all_theta_ones = []

    while current_iteration <= num_iterations:
        (cur_theta_zero, cur_theta_one) = step_gradient(theta_0, theta_1, data, alpha)

        all_theta_zeroes.append(cur_theta_zero)
        all_theta_ones.append(cur_theta_one)

        theta_0 = cur_theta_zero
        theta_1 = cur_theta_one

        current_iteration += 1

    return (all_theta_zeroes, all_theta_ones)


"""
Linear System (Polynomial Interpolation)
"""

import numpy as np

def swap_rows(Z, a, b):
    """
    Swap two rows (a, b) from Z matrix (np.array)

    Parameters
    ----------
        Z : np.array
            matrix
        a : int
            index row from Z
        b : int
            index row from Z
    """
    temp = np.copy(Z[a])
    Z[a] = Z[b]
    Z[b] = temp

def partial_pivoting(Z, row):
    """
    Partial pivoting of Z matrix, starting at row param

    Parameters
    ----------
        Z : np.array
            matrix
        row : int
            index row from Z

    Returns
    -------
        pivot : int
            pivot is the max value from Z(row:,row)
    """
    pivot_row = np.argmax(np.abs(Z[row:, row])) + row
    swap_rows(Z, row, pivot_row)
    pivot = Z[row, row]
    return pivot

def solve_sys(X, Y):
    """
    Solve a linear system using Gauss Elimination

    Parameters
    ----------
        X : list
            list of x values
        Y : list
            list of y values

    Returns
    -------
        list
            returns the roots of linear system (X, Y)
    """
    Z = np.copy(X)
    Z = np.hstack([Z, np.transpose(np.array([Y]))])

    for j in range(Z.shape[0] - 1):
        pivot = partial_pivoting(Z, j)
        for i in range(j + 1, Z.shape[0]):
            if Z[i, j] != 0 :
                m = pivot / Z[i, j]
                Z[i, j:] = Z[j, j:] - (m * Z[i, j:])

    A = np.zeros((X.shape[0], 1))
    for k in range(Z.shape[0] - 1, -1, -1):
        A[k] = (Z[k, Z.shape[1]-1] - (Z[k, Z.shape[1]-2:k:-1] @ A[A.shape[0]:k:-1])) / Z[k, k]

    return np.ndarray.tolist(np.transpose(A))[0]

def vandermond(X):
    """
    Create a vandermond matrix(nxn) by x values

    Parameters
    ----------
        X : list
            list of x values

    Returns
    -------
        np.array
            vandermond matrix
    """
    n = len(X)
    V = np.zeros((n, n))

    for i in range(n):
        V[i, :] = [X[i]**k for k in range(n)]

    return V

def linsys(X, Y):
    """
    Polynomial Interpolation using Gauss Elimination

    Parameters
    ----------
        X : list
            list of X values
        Y : list
            list of Y values

    Returns
    -------
        function
            function of polynomial interpolation (using linear system)
    """
    V = vandermond(X)
    A = solve_sys(V, Y)

    def f(x):
       return sum([a*(x**p) for p, a in enumerate(A)])

    return f


import numpy as np
import scipy.io
import matplotlib.pyplot as plt
import seaborn as sns

from featureNormalize import feature_normalize
from PCA_algo import pca
from projectData import project_data
from recoverData import recover_data

# In this exercise, we'll use principal component analysis (PCA) to perform dimensionality reduction.
########### Implement PCA on 2D dataset ####################
print(f"Plotting data example 1")
data1 = scipy.io.loadmat("data/ex7data1.mat")
sns.scatterplot(data1["X"][:, 0], data1["X"][:, 1])
plt.show()

input("Pause program, Press enter to continue")

# Implement PCA (Reduce from 2D to 1D -> k=1)
# (1st compute the covariance matrix of dataset. 2nd use svd function to compute eigenvectors U1, U2,...
# Un which are corresponding to principal component (main component) of variation (thành phần) dataset)
X_norm, mu, std = feature_normalize(data1["X"])
U, S = pca(X_norm)

# Visualize the dimensional reduction data resulted by PCA
# Visualize original data
print(f"\nVisualize dimensional reduction data")
data1 = scipy.io.loadmat("data/ex7data1.mat")
ax = sns.scatterplot(data1["X"][:, 0], data1["X"][:, 1])
# Compute & visualize compressed data (Z)
p1 = mu  # (1, 2)
p2 = mu + 1.5 * S[0] * U[:, 0].T  # (1, 2)
ax.plot((p1[0], p2[0]), (p1[1], p2[1]), "k")

p1 = mu
p2 = mu + 1.5 * S[1] * U[:, 1].T
ax.plot([p1[0], p2[0]], [p1[1], p2[1]], 'k')
plt.title('Computed eigenvectors of the dataset')
plt.show()
input("Pause program, Press enter to continue")

############ Dimensionality reduction with PCA ##############
print(f"\nReduce dimensionality of data from n-dim to K-dim")
# Checking correctness of reduction
K = 1
Z = project_data(X_norm, U, K)  # (m, K)
print(f"Projection of the 1st example: {Z[0]}")
print(f"This value should be 1.481274")

# Reconstructing compressed data approximately to original data
print(f"\nReconstruct compressed data to approximately original data")
X_rec = recover_data(Z, U, K)
print(f"Approximation of 1st example: {X_rec[0, 0]}, {X_rec[0, 1]}")
print(f"The result should be -1.047419 -1.047419")

# Visualize the projection
ax = sns.scatterplot(X_norm[:, 0], X_norm[:, 1], s=50)
ax = sns.scatterplot(X_rec[:, 0], X_rec[:, 1], s=50, color="r")
for i in range(X_norm.shape[0]):
    p1 = X_norm[i, :]
    p2 = X_rec[i, :]
    plt.plot([p1[0], p2[0]], [p1[1], p2[1]], "k--")
plt.title("Normalized and projected data after applying PCA")
plt.show()
input("Pause program, Press enter to continue")

################## Face image dataset ######################
# Loading faces data
faces_data = scipy.io.loadmat("data/ex7faces.mat")
# Visualize the first 100 pic of faces
print("\nVisualize the first 100 pictures of faces")
rows = 7
cols = 7
count = 0
fig = plt.figure(figsize=(5, 5))
for row in range(rows):
    for col in range(cols):
        ax = fig.add_subplot(rows, cols, count+1)
        ax.imshow(faces_data["X"][count].reshape(32, 32).T, cmap="gray")
        ax.axis("off")
        count += 1
plt.show()
input("Pause program, Press enter to continue")

# Normalize data X & compute svd() function
X_norm, mu, std = feature_normalize(faces_data["X"])  # X_norm: (5000, 1024) (5000 pic)
U, S = pca(X_norm)

# Apply PCA to data with K = 100
K = 100
Z = project_data(X_norm, U, K)  # (m, K)
# Visualize the first 100 dimensional reduction pic of face
print("\nDrawing 1st 100 pic of face with dimensional reduction to K = 100")
rows = 7
cols = 7
count = 0
fig = plt.figure(figsize=(5, 5))
for row in range(rows):
    for col in range(cols):
        ax = fig.add_subplot(rows, cols, count+1)
        ax.imshow(Z[count, :].reshape(10, 10).T, cmap="gray")
        ax.axis("off")
        count += 1
plt.show()
input("Pause program, Press enter to continue")

# Reconstructing from compressed image to original image
X_rec = recover_data(Z, U, K)
print("\nPlot the reconstructed image from compressed image (From 10*10 to 32*32)")
rows = 7
cols = 7
count = 0
fig = plt.figure(figsize=(5, 5))
for row in range(rows):
    for col in range(cols):
        ax = fig.add_subplot(rows, cols, count+1)
        ax.imshow(X_rec[count, :].reshape(32, 32).T, cmap="gray")
        ax.axis("off")
        count += 1
plt.show()


"""
golden_section_algorithm.py

Returns the reduced uncertainty interval containing the minimizer of the function
func - anonimous function
interval0 - initial uncertainty interval
N_iter - number of iterations
"""

import math
import numpy as np

def golden_section_algorithm_calc_N_iter(interval0, uncertainty_range_desired):

    N_iter = math.ceil(math.log(uncertainty_range_desired / (interval0[1] - interval0[0]), 0.618));

    return N_iter;

def golden_section_algorithm(func, interval0, N_iter):

    rho = (3 - np.sqrt(5)) / 2;
    left_limit = interval0[0];
    right_limit = interval0[1];

    smaller = 'a';
    a = left_limit + (1 - rho) * (right_limit - left_limit);
    f_at_a = func(a);

    for iter_no in range(N_iter):
        if (smaller == 'a'):
            c = a;
            f_at_c = f_at_a;
            a = left_limit + rho * (right_limit - left_limit);
            f_at_a = func(a);
        else:
            a = c;
            f_at_a = f_at_c;
            c = left_limit + (1 - rho) * (right_limit - left_limit);
            f_at_c = func(c);
        if (f_at_a < f_at_c):
            right_limit = c;
            smaller = 'a';
        else:
            left_limit = a;
            smaller = 'c';

    interval = (left_limit, right_limit);
    return interval;


from matplotlib.patches import Polygon
import matplotlib.pyplot as plt
import numpy as np

# points at 2D
def plot_2D_points(points, polyg=False):
    fig, ax = plt.subplots()
    ax.spines['right'].set_color('none')
    ax.spines['top'].set_color('none')
    ax.xaxis.set_ticks_position('bottom')
    ax.spines['bottom'].set_position(('data',0)) # set position of x spine to x=0
    ax.yaxis.set_ticks_position('left')
    ax.spines['left'].set_position(('data',0))   # set position of y spine to y=0

    for point in points:
        plt.scatter(point[0], point[1], color="g")
        plt.text(point[0]+0.2, point[1]+0.2, f'({point[0]},{point[1]})' , fontsize=11)

    if polyg == False:
        plt.show()

# draw triangle
def plot_triangle(points):
    plot_2D_points(points, polyg=True)

    triangle_points = np.array(points)
    triangle_plot = plt.Polygon(triangle_points, color='g')
    plt.gca().add_patch(triangle_plot)

    plt.show()

# calculate the area o a triangle
def calc_triangle_area(x0, y0, x1, y1, x2, y2):
    return abs((x0 * (y1 - y2) + x1 * (y2 - y0) + x2 * (y0 - y1))/2)

# check if point (0,0) is inside the triangle
def check_origin_in(x0, y0, x1, y1, x2, y2):
    # calculate triangle areas for:
    triangle_area1 = calc_triangle_area(x0, y0, x1, y1, 0, 0) # Point0, Point1, (0.0)
    triangle_area2 = calc_triangle_area(x0, y0, 0, 0, x2, y2) # Point0,(0.0), Point2
    triangle_area3 = calc_triangle_area(0, 0, x1, y1, x2, y2) # (0,0), Point1, Point2
    triangle_area = calc_triangle_area(x0, y0, x1, y1, x2, y2)

    if(triangle_area1 + triangle_area2 + triangle_area3 == triangle_area):
        print('The interior of the triangle contains the origin (0, 0)')
    else:
        print('The interior of the triangle does not contain the origin (0, 0)')

#check if 3 points form a triangle
def check_for_triangle(x0, y0, x1, y1, x2, y2):
    # area of triangle
    # if area is 0, the points are in the same straight line
    triangle_area = calc_triangle_area(x0, y0, x1, y1, x2, y2)

    if (triangle_area != 0):
        return True
    else:
        return False

if __name__ == "__main__":

    # read 3 points from user
    print("Give 3 points.")
    points = list(tuple(map(int,input('Give a point: ').split())) for r in range(3))

    #check if these points form a triangle
    if check_for_triangle(points[0][0], points[0][1],
                        points[1][0], points[1][1],
                        points[2][0], points[2][1]):
        print ('These points form a triangle')
        check_origin_in(points[0][0], points[0][1],
                        points[1][0], points[1][1],
                        points[2][0], points[2][1])
        plot_triangle(points)
    else:
        print ('These points don\'t form a triangle')
        plot_2D_points(points)


# import modules
import numpy as np

'''
# Description.
Given a 2x1 vector, this function obtains the angle of the azimuth from
x-axis in a counter clockwise sense.
Coordiante system is with x in the abscisas and y in the ordinates  (i.e.
x horizontal pointing to the right and y vertical pointing upwards).

# Input(s).
Array that representing a bi-dimensional vector (vector).

# Output(s).
Angle in radians representing the azimut of the vector (angleRad).

Example: giving next array
vector = np.array([3,-5]), is obtined  5.2528 radians
---
angleRad = azimuthangle(vector)
'''
def azimuthangle(vector):
    if vector[0] == 0:
        if vector[1] >= 0:
            angleRad = np.pi/2
        else:
            angleRad = 3*np.pi/2
    else:
        basicAngleRad = np.arctan(np.abs(vector[1])/np.abs(vector[0]))
        if vector[0] >= 0:
            if vector[1] >= 0: #case 1
                angleRad = basicAngleRad
            elif vector[1] < 0: #case 4
                angleRad = 2*np.pi-basicAngleRad
        elif vector[0] < 0:
            if vector[1] >= 0: #case 2
                angleRad = np.pi-basicAngleRad
            elif vector[1] < 0: #case 3
                angleRad = np.pi+basicAngleRad
        else:
            print("Error: bad number")

    return angleRad
'''
BSD 2 license.

Copyright (c) 2016, Universidad Nacional de Colombia, Ludger O.
   Suarez-Burgoa and Exneyder Andrés Montoya Araque.
All rights reserved.

Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:

1. Redistributions of source code must retain the above copyright notice,
this list of conditions and the following disclaimer.

2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
'''


"""
The purpose of this code is to explore the use of relaxation and over-
relaxation method on computing non-linear equations i.e. f(x) = x.
"""
import numpy as np
import matplotlib.pyplot as plt

#Part a
#Text Exercise 6.10 a
#we can modify the number of loops to acheive different levels of accuracy
#in our case we achieved 10^-6 by setting the number of loops to 15
xa=1.0 #starting guess for x
numOfLoops = 15
for k in range(numOfLoops):
    xa = 1 - np.exp(-2*xa) #given equation

print("part a answer")
print("value of x with {} iterations:".format(numOfLoops), xa)
print("value of 1 - exp(-2*x) with {} iterations:".format(numOfLoops), 1 - np.exp(-2*xa))

#Text Exercise 6.10 b
#similar as above, we first reset the variables and instead of a constant c
#we generated a range of values from 0 to 300 with a stepsize of 0.01
x=1.0 #Starting guess for value of x
numOfLoops = 15
c = np.arange(0, 3, 0.01) #different values for c
for k in range(numOfLoops):
    x = 1 - np.exp(-c*x)

# plotting the results
fig, graph = plt.subplots()
graph.plot(c, x)
graph.set(xlabel="c", ylabel="x",
       title="Solutions of x for a range of c=(0, 3, 0.01) (Q3a)")
graph.grid()
fig.savefig("q3_A.png")
plt.show()

#Part b
#Text Exercise 6.11 b
def F(c):
    """
    This function uses the similar method as above and find the iterations
    when it approaches the convergence.
    """
    sigdig = 10 ** -6 #Define the accuracy (number of significant digits) we want the solution to be accurate to
    iterations = 1 #Starting value for iterations
    def f(x):
        return 1 - np.exp(-c*x)
    # Define our error function given in textbook
    # We need this so that when this value gets below the sigdigs we want it cuts off the iteration count
    def error(x1, x2):
        return (x1-x2)/(1 - 1/(c*np.exp(-c * x1)))

    x1 = 1.0 # starting value
    x2 = f(x1)
    #We need a condition so that when the error is greater than the value we want
    #It keeps running more iterations
    while(abs(error(x1, x2)) > sigdig):
        #This adds another Iteration
        x1, x2 = x2, f(x2)
        #This increases the iteration count tally
        iterations += 1
    print('The minimum number of iterations for an accuracy of 10**-6 = ', iterations)
    print("value of x:", x2)
    print("value of 1 - np.exp(-c*x)  is:", 1 - np.exp(-2*x2))

#Text Exercise 6.11 c
def F_over(c, w):
    """
    This function uses the over-relaxation method to increases the rate of
    convergence by artifically increasing the gradient of f(x)
    """
    sigdig = 10 ** -6
    iterations = 1
    def f(x):
        return 1 - np.exp(-c*x)

    def derivf(x):#This is the derivative of our function which we need for the over relax method
        return c * np.exp(-c*x)

    def error(x1, x2):# Error function for overrelaxation from text
        return (x1 - x2)/(1 - 1/((1 + w)*derivf(x1) - w))

    x1 = 1.0  # starting value
    x2 = (1 + w) * f(x1) - w * x1
    while abs(error(x1, x2)) > sigdig:
        #when the error is above the desired value, we do another iterations as before
        x1, x2 = x2, (1 + w) * f(x2) - w * x2
        #We then add 1 to the iteration count
        iterations += 1
    print('When we set omega to be ', w,'the minimum number of iterations for an accuracy of 10**-6 is ', iterations)
    print("value of x:", x2)
    print("value of 1 - exp(-2*x):", 1 - np.exp(-2*x2))

print()
print("part b answer")
F(2)
print()
F_over(2, 0.5)

#Part c
#reset the variables to desired accuracy and values
sigfig = 10 ** -6
a = 1
b = 2
def f(x, y):
    """
    the derived equation from 6.12(b)
    """
    return y*(a+x**2)

def g(x, y):
    """
    the derived equation from 6.12(b)
    """
    return b/(a+x**2)

def F(x,y):
    """
    the rewritten equation from 6.12(c)
    """
    return np.sqrt(b/y-a)

def G(x,y):
    """
    the rewritten equation from 6.12(c)
    """
    return x/(a+x**2)

def solution(f, g):
    """
    This function uses the similar method as above and find the iterations
    when it approaches the convergence.
    """
    iterations = 1
    def delta(x1, x2):
        return (x1 - x2) / x1
    x1 = 0.5
    y1 = 0.5
    x2 = f(x1, y1)
    y2 = g(x1, y1)
    #While the errors of x or y are higher than the desired value we need to do more iterations
    while abs(delta(x1, x2)) > sigfig and abs(delta(y1, y2)) > sigfig:
        #This allows our method to "Fail gracefully" as required
        if iterations > 1000:
            return 'error'

        x1, x2, y1, y2 = x2, f(x2, y2), y2, g(x2, y2)
        iterations += 1
    print('The number of iterations = ', iterations, 'the solution for x ~', x2, 'the solution for y ~', y2)
    return [ x2, y2 ]

print()
print("part c answer")
print(solution(F,G))


# We use the process of normalization to modify the values in the feature vector so that we can measure them on a common scale.
# In machine learning, we use many different forms of normalization.
# Some of the most common forms of normalization aim to modify the values so that they sum up to 1 . L1 normalization ,
# which refers to Least A bsolute Deviations , works by making sure that the sum of absolute values is 1 in each row. L2 normalization ,
# which refers to least squares, works by making sure that the sum of squares is 1 .
# In general, L1 normalization technique is considered more robust than L2 normalization technique.
# L1 normalization technique is robust because it is resistant to outliers in the data.
# A lot of times, data tends to contain outliers and we cannot do anything about it.
# We want to use techniques that can safely and effectively ignore them during the calculations.
# If we are solving a problem where outliers are important, then maybe L2 normalization becomes a better choice.

import numpy as np
from sklearn import preprocessing

input_data = np.array(
    [[5.1, -2.9, 3.3],
    [-1.2, 7.8, -6.1],
    [3.9, 0.4, 2.1],
    [7.3, -9.9, -4.5]]
)

# Normalize data
data_normalized_l1 = preprocessing.normalize(input_data, norm='l1')
data_normalized_l2 = preprocessing.normalize(input_data, norm='l2')
print("\nL1 normalized data:\n", data_normalized_l1)
print("\nL2 normalized data:\n", data_normalized_l2)


"""
Baby steps in Theano library

"""
import numpy
from matplotlib import pyplot as plt
import theano.tensor as T
from theano import function
from theano import pp

'''
this is a brief insight of how the theano library works

everything we do, every operation, instead of a usual for example "x + y" is declared differently

variables are first declared as Theano based ones and then we assemble an expression as theano function

the reason for this is that theano creates a C code in the backend to make an operation

then the operation's code is optimized by compiler

following code shows a baseline on how to create simple equations, though it may seem intuitive, it
does things in quite unusual way
'''

x = T.dscalar('x')
y = T.dscalar('y')
z = x + y
f = function([x, y], z)
print(f(2, 3))
print(numpy.allclose(f(16.3, 12.1), 28.4))

'''
here we defined simple addition function in more steps than usual

it's because theano acts on symbols to perform creation of C code that runs in background

we declare 2 "scalars" (numbers), assigned them to x and y, respectively, then we
hinted the operation (addition) that gets stored in z variable.

a function declaration takes inputs and outputs, in our case it takes an array [x, y]
and returns z. Z is evaluated as addition
'''

print(pp(z))
'''
here we see how theano assigned the operation
'''

print(z.eval({x : 16.3, y : 12.1}))
'''
we could also invoke the "eval()" from a z variable that had addition operation
"eval()" takes a dictionary with names of variables and values to be assigned to them

eval() utimately imports a "function()", so we end up in the same situation, so it is
slower the first time we invoke this, subsequent invocations are faster since it saves
"function()" imported already

that way we don't need to import "function()", but importing and using it is more
flexible than relyin on "eval()" itself
'''

'''
Addition of two matrices is also very simple, the only difference is using
T.dmatrix instead of T.dscalar
'''

x = T.dmatrix('x')
y = T.dmatrix('y')
z = x + y
f2 = function([x, y], z)

'''
now we do not assign the matrices to x and y directly, but instead we pass matrices
to function as variables, invoking it like: "f2([matrix_nr_1], [matrix_nr_2])

here we add, elementwise, 2 2-dimensional matrices
'''

a1 = [[1, 2],
      [3, 4]]
a2 = [[10, 20],
      [30, 40]]

result = f2(a1, a2)
print(result)

'''
we could also utilize numpy.array
'''
a3 = numpy.array([[1, 2],
                  [3, 4]])
a4 = numpy.array([[10, 20],
                  [30, 40]])

print(f2(a3, a4))

'''
ex1. modify the code at the bottom,

import theano
a = theano.tensor.vector() # declare variable
out = a + a ** 10
# build symbolic expression
f = theano.function([a], out)
# compile function
print(f([0, 1, 2]))

to reflect the expression: a ** 2 + b ** 2 + 2 * a * b
'''

c = 16
d = 13

sample_result = [c**2 + d**2 + 2*c*d]

a = T.vector('a') # declare variable
b = T.vector('b')
add = a + b
out = T.power(add, 2)
# build symbolic expression
f3 = function([a, b], out)
# compile function
print(f3([0, 1, 2], [0, 1, 2]))

assert sample_result == f3([16], [13])
print(f3([16], [13]))

'''
calculation more elaborate functions is also possible
here we calculate a result of a "logistic" function, sometimes referred to as "sigmoid"

'''

matrix = T.dmatrix('matrix')
s = 1/ (1+ T.exp(-matrix))
sigmoid = function([matrix], s)
X = numpy.linspace(-6, 6, 100)
values = sigmoid([X])[0]

print(X)
print(values)

plt.plot(X, values)
plt.show()

'''
We could also produce gradients.
Theano provides efficient symbolic differentiaition, using T.grad as a macro

lets compute a gradient of logistic function

note that we compute it for SCALAR values here!!!
'''

x2 = T.scalar('x2')
sigm = 1/ (1+ T.exp(-x2))
sigm_grad = T.grad(sigm, x2)
# uncomment this if you copy code to another program, i already declared this above
# X = numpy.linspace(-6, 6, 100)
'''
ok we declared derivative (gradient) and a regular function, at the coordinate passed
to the function

theano "function()" can also accept multiple outputs, not only multiple inputs

here we calculate the regular function output, as well as the derivative at the same point

KNOWN PROBLEM: the result is a list of 0-dimensional arrays, which can't be accessed in regular way
use "list[index].sum()" to extract a desired value from it
'''

multiple_sigmoid_outputs = function([x2], [sigm, sigm_grad])

val_out = multiple_sigmoid_outputs(4)
print(val_out)
print(val_out[0].sum(), val_out[1].sum())

'''
ok so we defined a derivative of a function at a single point, but what about computing
whole lists of points?

you have to wrap the result of a regular with the "T.sum" to make that happen
'''

x3 = T.matrix('x3')
sigm2 = T.sum(1/ (1+ T.exp(-x3)))
sigm2_grad = T.grad(sigm2, x3)
logistics = function([x3], [sigm2, sigm2_grad])
values2 = logistics([X])

print(values2)
Y = values2[1][0]
plt.plot(X, Y, c='red')
plt.plot(X, values, c='blue')
plt.show()

print(logistics.maker.fgraph.outputs)
pp(logistics.maker.fgraph.outputs[0])

'''
Back to the basics

How to set a default value for a variable in theano function?

it isn't that hard as it looks, just use the "In" class

"In" from theano accepts a variable and a "value=" which initializes default variable
if not present

You can think of a "In(variable_name, value=default_value)" class as an "Input" to a function

We will also use a "T.dscalars" macro, which creates multiple variables in one line, unlike the
"T.dscalar"
'''

from theano import In

x5, y5 = T.dscalars('x5', 'y5')
z5 = x5 + y5
func5 = function([x5, In(y5, value=4)], z5)
print(func5(32))
print(func5(20, 56.7))

assert func5(25) == 29

'''
and how to share a value between theano functions?

you can allocate memory space that will be accessible from any theano function, even after
it finishes work

the additional "set_value()" and "get_value()" help show and modify shared varaible's contents without a need of creting
a function

functions use shared variables in slightly different way than regular variables; we have to pass the in "updates" list
declaring what we want to do with the variable
'''

from theano import shared

state = shared(0)
inc = T.iscalar('inc')
accumulator = function([inc], state, updates=[(state, state+inc)])

print(state.get_value())
accumulator(1)
print(state.get_value())
accumulator(300)
print(state.get_value())
state.set_value(-1)
accumulator(3)
print(state.get_value())

decrementor = function([inc], state, updates=[(state, state-inc)])
decrementor(2)
print(state.get_value())


import numpy as np
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
from matplotlib import cm
from mpl_toolkits.mplot3d import Axes3D
import math
import sys
import os

data_dir = sys.argv[1]
out_dir = sys.argv[2]
test_set = os.path.join(sys.argv[1], 'q2test.csv')
out = os.path.join(sys.argv[2], 'Q2c.txt')
outfile = open(out, "w")

# 2. Sampling and Stochastic Gradient
print("\n################ 2. Sampling and Stochastic Gradient ################", file=outfile)

# (a) Sampling

w = np.array([[3],[1],[2]])
X1 = np.random.normal(3, 2, (int)(1e6))
X2 = np.random.normal(-1, 2, (int)(1e6))
noise = np.random.normal(0, np.sqrt(2), int(1e6)).reshape(-1,1)

X = np.column_stack((X1, X2))
X = np.column_stack((np.ones(X.shape[0]), X))
Y = np.dot(X, w) + noise
m = len(Y)

# (b) Stochastic Gradient

def hw(theta, i, r):
	x = X[i*r:(i+1)*r,]
	return np.dot(x, theta)

def Jw(h, i, r):
	y = Y[i*r:(i+1)*r,]
	return (0.5/r)*np.sum((y - h)**2)

def dJw(h, i, r):
	x = X[i*r:(i+1)*r,]
	y = Y[i*r:(i+1)*r,]
	return (1.0/r)*np.dot(x.T, (y - h))

def stochasticGradientDescent(x, y, r):
	b = (int)(m/r)
	eta = 0.001
	theta = np.zeros((x.shape[1], 1))
	thetaL = [theta]
	prevCost = 0
	converged = False
	itr = 0
	while not converged:
		for i in range(b):
			h = hw(theta, i, r)
			cost = Jw(h, i, r)
			theta = theta + eta*dJw(h, i, r)
			thetaL.append(theta)
		itr += 1
		error = abs(cost - prevCost)
		prevCost = cost
		if error < 1e-7 or itr > 25000:
			converged = True

	return thetaL, itr, cost

thetaL1, maxit1, cost1 = stochasticGradientDescent(X, Y, 1)
thetaL2, maxit2, cost2 = stochasticGradientDescent(X, Y, 100)
thetaL3, maxit3, cost3 = stochasticGradientDescent(X, Y, 10000)
thetaL4, maxit4, cost4 = stochasticGradientDescent(X, Y, 1000000)

# (c) Test Data

data = np.loadtxt(test_set, delimiter=',', skiprows=1)
q2X1 = data[:,0]
q2X2 = data[:,1]
q2X = np.column_stack((q2X1, q2X2))
q2X = np.column_stack((np.ones(q2X.shape[0]), q2X))
q2Y = data[:,-1].reshape(-1,1)
q2m = len(q2Y)

def testError(theta):
	h = np.dot(q2X, theta)
	return (0.5/q2m)*np.sum((q2Y - h)**2)

error1 = testError(thetaL1[-1])
error2 = testError(thetaL2[-1])
error3 = testError(thetaL3[-1])
error4 = testError(thetaL4[-1])

print('\nFor r = 1 : Training Error = {}, Test Error = {}'.format(cost1, error1), file=outfile)
print('For r = 100 : Training Error = {}, Test Error = {}'.format(cost2, error2), file = outfile)
print('For r = 10000 : Training Error = {}, Test Error = {}'.format(cost3, error3), file=outfile)
print('For r = 1000000 : Training Error = {}, Test Error = {}'.format(cost4, error4), file=outfile)


## 2. Systems of equations as matrices ##

import numpy as np

# Set the dtype to float to do float math with the numbers.
matrix = np.asarray([
    [2, 1, 25],
    [3, 2, 40]
], dtype=np.float32)
matrix[0] = matrix[0] * 2
matrix[0] = matrix[0] - matrix[1]
matrix[1] = matrix[1] - (matrix[0] * 3)
matrix[1] /= 2
print(matrix)

## 4. Solving more complex equations ##

import numpy as np

matrix = np.asarray([
    [1, 2, 0, 7],
    [0, 3, 3, 11],
    [1, 2, 2, 11]
], dtype=np.float32)
matrix[2] = matrix[2] - matrix[0]
matrix[2] = matrix[2] / 2
matrix[1] = matrix[1] / 3
matrix[1] = matrix[1] - matrix[2]
matrix[0] = matrix[0] - 2 * matrix[1]

## 5. Echelon form ##

matrix = np.asarray([
    [0, 0, 0, 7],
    [0, 0, 1, 11],
    [1, 2, 2, 11],
    [0, 5, 5, 1]
], dtype=np.float32)

# Swap the first and the third rows - first swap
matrix[[0,2]] = matrix[[2,0]]
matrix[[1,3]] = matrix[[3,1]]
matrix[[2,3]] = matrix[[3,2]]

## 6. Reduced row echelon form ##

A = np.asarray([
        [0, 2, 1, 5],
        [1, 2, 1, 8],
        [3, 0, 1, 10],
        ], dtype=np.float32)

# First, we'll swap the second row with the first to get a non-zero coefficient in the first column
A[[0,1]] = A[[1,0]]

# The leading coefficient is already 1, so there's no need to divide
# Now, we need to make sure that our 1 coefficient is the only coefficient in its column
# We have to subtract three times the first row from the third row
A[2] -= 3 * A[0]

# Now, we move to row 2
# We divide by 2 to get a one as the leading coefficient
A[1] /= 2

# We subtract 2 times the second row from the first to get rid of
# the second column coefficient in the first row
A[0] -= 2 * A[1]

# And we'll add 6 times the second row to the third to eliminate the leading coefficient there
A[2] += 6 * A[1]

# Now, we can move to the third row where the leading coefficient is already 1
# We just need to subtract half of the third from the second
A[1] -= 0.5 * A[2]

# We're finished, and our system is solved!
print(A)

## 7. Inconsistency ##

A = np.asarray([
    [10, 5, 20, 60],
    [3, 1, 0, 11],
    [8, 2, 2, 30],
    [0, 4, 5, 13]
], dtype=np.float32)

B = np.asarray([
    [5, -1, 3, 14],
    [0, 1, 2, 8],
    [0, -2, 5, 1],
    [0, 0, 6, 6]
], dtype=np.float32)

A_consistent,B_consistent = True,False

## 8. Infinite solutions ##

A = np.asarray([
        [2, 4, 8, 20],
        [4, 8, 16, 40],
        [20, 5, 5, 10]
], dtype=np.float32)
A_infinite = True

B = np.asarray([
        [1, 1, 1, 4],
        [3, -2, 5, 8],
        [8, -4, 5, 10]
        ], dtype=np.float32)

B_infinite = False

Linear Algebra/Solving systems of equations with matrices-55.py


#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
@author: Ashutosh Vyas

A function to compute cost for multivariate linear regression.

This function is used for the Optional Part of the Exercise 1: Linear Regression.
cost_J = compute_cost(x_data, y_data, theta) computes the cost of using theta
as the parameter for linear regression to fit the data points in x_data, y_data.
"""

import numpy

def compute_cost_multi(x_data, y_data, theta, num_examples):
    """
    Compute cost for single iteration of multivariate linear regression.

    Given the multivariate hypothesis,
    h(x) = theta0 + theta1*x1 + theta2*x2 = theta^transpose.x,
    this function calculates the cost incurred for the given value of theta.
    Thus computes J(theta)

    Parameters
    ----------
    x_data : ndarray
        X 2D array dataset with appended theta0 initalized as 1.
    y_data : ndarray
        A vector of actual values of y from the dataset.
    theta : ndarray
        theta0 (y-intercept) and theta1 (slope) of the linear model.
    num_examples : scalar
        Number of training examples.

    Returns
    -------
    cost_J : scalar
        Cost incurred as difference between hypothesis and actual value.
        Represented as sum -of squared errors

    """

    x_dot_theta = numpy.reshape(numpy.dot(x_data, theta), y_data.shape)  # calculating hypothesis
    difference = x_dot_theta - y_data  # error = hypothesis - actual
    squared = numpy.power(difference, 2.0)  # squared error
    factor = (1.0/(2.0*num_examples))
    cost_J = factor*numpy.sum(squared)  # sum of squared error
    return cost_J


####################################################################################
## PROBLEM1: Gradient Descent
## Gradient descent is a popular optimization technique to solve many
## machine learning problems. In this case, we will explore the gradient
## descent algorithm to fit a line for the given set of 2-D points.
## ref: https://tinyurl.com/yc4jbjzs
## ref: https://spin.atomicobject.com/2014/06/24/gradient-descent-linear-regression/
##
##
## input: directory of faces in ./data/1_points.csv/
## function for reading points is provided
##
##
## your task: fill the following functions:
## evaluate_cost
## evaluate_gradient
## udpate_params
## NOTE: do NOT change values of 'init_params' and 'max_iterations' in optimizer
##
##
## output: cost after convergence (rmse, lower the better)
##
##
## NOTE: all required modules are imported. DO NOT import new modules.
## NOTE: references are given intline
## tested on Ubuntu14.04, 22Oct2017, Abhilash Srikantha
####################################################################################

import numpy as np
import matplotlib.pyplot as plt
import time

def load_data(fname):
    points = np.loadtxt(fname, delimiter=',')
    y_ = points[:,1]
    # append '1' to account for the intercept
    x_ = np.ones([len(y_),2])
    x_[:,0] = points[:,0]
    # display plot
    #plt.plot(x_[:,0], y_, 'ro')
    #plt.xlabel('x-axis')
    #plt.ylabel('y-axis')
    #plt.show()
    print('data loaded. x:{} y:{}'.format(x_.shape, y_.shape))
    return x_, y_

def evaluate_cost(x_,y_,params):
    tempcost = 0
    for i in range(len(y_)):
        tempcost += (y_[i] - ((params[0] * x_[i,0]) + params[1])) ** 2
    return tempcost / float(10000)

def evaluate_gradient(x_,y_,params):
    m_gradient = 0
    b_gradient = 0
    N = float(len(y_))
    for i in range(len(y_)):
        m_gradient += -(2/N) * (x_[i,0] * (y_[i] - ((params[0] * x_[i,0]) + params[1])))
        b_gradient += -(2/N) * (y_[i] - ((params[0] * x_[i,0]) + params[1]))
    return [m_gradient,b_gradient]

def update_params(old_params, grad, alpha):
    new_m = old_params[0] - (alpha * grad[0])
    new_b = old_params[1] - (alpha * grad[1])
    return [new_m,new_b]

# initialize the optimizer
optimizer = {'init_params':np.array([4.5,2.0]) ,
             'max_iterations':10000,
             'alpha':0.69908,
             'eps':0.0000001,
             'inf':1e10}

# load data
x_, y_ = load_data("./data/1_points.csv")

# time stamp
start = time.time()

try:
    # gradient descent
    params = optimizer['init_params']
    old_cost = 1e10
    for iter_ in range(optimizer['max_iterations']):
        # evaluate cost and gradient
        cost = evaluate_cost(x_,y_,params)
        grad = evaluate_gradient(x_,y_,params)
        # display
        if(iter_ % 10 == 0):
            print('iter: {} cost: {} params: {}'.format(iter_, cost, params))
        # check convergence
        if(abs(old_cost - cost) < optimizer['eps']):
            break
        # udpate parameters
        params = update_params(params,grad,optimizer['alpha'])
        old_cost = cost
except:
    cost = optimizer['inf']

# final output
print('time elapsed: {}'.format(time.time() - start))
print('cost at convergence: {} (lower the better)'.format(cost))


from numpy import sqrt
import matplotlib as pl
'''
@author: Arleigh Dickerson
@see: p17 and 18 of Sauer's Numerical Analysis, example 0.6
'''

# square root of the result of b squared minus the quantity of 4 times a times c
def sqrtTerm(a, b, c): return sqrt((b ** 2) - (4 * a * c))

# call to get x1 for a, b, c
def naiveX1(a, b, c): return (-b + sqrt((b ** 2) - (4 * a * c))) / (2 * a)

# call to get x2 for a, b, c
# copy and paste with - instead of +...
def naiveX2(a, b, c): return (-b - sqrt((b ** 2) - (4 * a * c))) / (2 * a)

# second root (- part of eq) can be problematic if b is large compared to a or c
def naiveQuadratic(a, b, c):
    # a tuple with both root values. Values will be identical if double root.
    return (naiveX1(), naiveX2())

# use if b and sqrt(bsquared minus 4ac) are nearly equal in magniture
# and b is positive
def impl0dot13(a, b, c):
    def x1(): return -(b + sqrtTerm(a, b, c)) / (2 * a)
    def x2(): return -((2 * c) / (b + sqrtTerm(a, b, c)))
    return (x1(), x2())

# use if b and sqrt(bsquared minus 4ac) are nearly equal in magniture
# and b is negative
def impl0dot14(a, b, c):
    def x1(): return naiveX1(a, b, c)
    def x2(): return ((2 * c) / (-b + sqrtTerm(a, b, c)))
    return (x1(), x2())

def nearlyEqual(term0, term1) :
    threshold = 0.001  # arbitrary (for now)
    difference = abs(term0 - term1)
    return difference < threshold


from __future__ import print_function
import numpy as np
import matplotlib.pyplot as plt
from scipy import stats

np.random.seed(3)
a = np.random.random(10)
b = np.random.normal(size=10)
two_samp = stats.ks_2samp(a,b)
# since the p value is less than 0.05 I reject that the
# two samples came from the same distribution
# with a 95% confidence level.

np.random.seed(121)
binom_mars = stats.binom(100,0.0925)
mars_rv = binom_mars.rvs(10)
print(np.mean(mars_rv))
print(binom_mars.cdf(10))

x = np.arange(0,50,1)
plt.figure()
plt.plot(x, binom_mars.sf(x))
plt.show()

np.random.seed(32332)
# generate 1000 samples from the
rv = stats.laplace.rvs(loc=75, scale=5, size=1000)

# save the random variables as numpy array
np.save('Aluminum_youngs_moduli',rv)

# fit a normal distribion
norm_param = stats.norm.fit(rv)
laplace_param = stats.laplace.fit(rv)
maxwell_param = stats.maxwell.fit(rv)
# vonmises_param = stats.vonmises.fit(rv)
logistic_param = stats.logistic.fit(rv)

x = np.linspace(10,150,200)

plt.figure()
plt.hist(rv, bins=21, normed=True)
plt.plot(x, stats.norm.pdf(x, *norm_param), label='norm')
plt.plot(x, stats.maxwell.pdf(x, *maxwell_param), label='maxwell_param')
# plt.plot(x, stats.vonmises.pdf(x, *vonmises_param), label='vonmises_param')
plt.plot(x, stats.logistic.pdf(x, *logistic_param), label='logistic_param')
plt.plot(x, stats.laplace.pdf(x, *laplace_param), label='laplace')
plt.legend()
plt.show()

plt.figure()
plt.hist(rv, cumulative=True, bins=21, normed=True)
plt.plot(x, stats.norm.cdf(x, *norm_param), label='norm')
plt.plot(x, stats.maxwell.cdf(x, *maxwell_param), label='maxwell_param')
# plt.plot(x, stats.vonmises.pdf(x, *vonmises_param), label='vonmises_param')
plt.plot(x, stats.logistic.cdf(x, *logistic_param), label='logistic_param')
plt.plot(x, stats.laplace.cdf(x, *laplace_param), label='laplace')
plt.legend()
plt.show()

# ks_statistic
ks_norm = stats.kstest(rv, 'norm', norm_param)
ks_laplace = stats.kstest(rv, 'laplace', laplace_param)
ks_maxwell = stats.kstest(rv, 'maxwell', maxwell_param)
ks_logistic = stats.kstest(rv, 'logistic', logistic_param)

# accept either the logistic or laplace!


import solver.algorithms as alg
import numpy as np

def problem5(t0, tf, NA0, NB0, tau, n, returnlist=False):
    """Uses Euler's method to model the solution to a radioactive decay problem where dNA/dt = NB/tau - NA/tau and dNB/dt = NA/tau - NB/tau.

    Args:
        t0 (float): Start time
        tf (float): End time
        NA0 (int): Initial number of NA nuclei
        NB0 (int): Initial number of NB nuclei
        tau (float): Decay time constant
        n (float): Number of points to sample at
        returnlist (bool) = Controls whether the function returns the list of points or not. Defaults to false

    Returns:
        solution (list): Points on the graph of the approximate solution. Each element in the list has the form (t, array([NA, NB]))

    In the graph, NA is green and NB is blue
    """
    print("Problem 5: ~Radioactive Decay~ dNA/dt = NB/tau - NA/tau & dNB/dt = NA/tau - NB/tau.")
    N0 = np.array([NA0, NB0])
    A = np.array([[-1/tau, 1/tau],[1/tau, -1/tau]])
    def dN_dt(t, N):
        return A @ N
    h = (tf-t0)/(n-1)
    print("Time step of %f seconds." % h)
    solution = alg.euler(t0, tf, n, N0, dN_dt)
    if returnlist:
        return solution


"""
Created on Sat Apr 18 21:11:50 2020
@author: Mohammad Asif Zaman
Differentiaion matrix operators in 1D and 2D
-   First derivatives d/dx, d/dy
-   Second derivatives d2/dx2, d2/dy2
Notes:
        - kron() is different in python compared to MATLAB. The matrix order is reveresed here.
          MATLAB version:
          Dx_2d = sp.kron(Dx_1d,Iy)
          Dy_2d = sp.kron(Ix,Dy_1d)
"""

import scipy.sparse as sp

def Diff_mat_1D(Nx):

    # First derivative
    D_1d = sp.diags([-1, 1], [-1, 1], shape = (Nx,Nx)) # A division by (2*dx) is required later.
    D_1d = sp.lil_matrix(D_1d)
    D_1d[0,[0,1,2]] = [-3, 4, -1]               # this is 2nd order forward difference (2*dx division is required)
    D_1d[Nx-1,[Nx-3, Nx-2, Nx-1]] = [1, -4, 3]  # this is 2nd order backward difference (2*dx division is required)

    # Second derivative
    D2_1d =  sp.diags([1, -2, 1], [-1,0,1], shape = (Nx, Nx)) # division by dx^2 required
    D2_1d = sp.lil_matrix(D2_1d)
    D2_1d[0,[0,1,2,3]] = [2, -5, 4, -1]                    # this is 2nd order forward difference. division by dx^2 required.
    D2_1d[Nx-1,[Nx-4, Nx-3, Nx-2, Nx-1]] = [-1, 4, -5, 2]  # this is 2nd order backward difference. division by dx^2 required.

    return D_1d, D2_1d

def Diff_mat_2D(Nx,Ny):
    # 1D differentiation matrices
    Dx_1d, D2x_1d = Diff_mat_1D(Nx)
    Dy_1d, D2y_1d = Diff_mat_1D(Ny)

    # Sparse identity matrices
    Ix = sp.eye(Nx)
    Iy = sp.eye(Ny)

    # 2D matrix operators from 1D operators using kronecker product
    # First partial derivatives
    Dx_2d = sp.kron(Iy,Dx_1d)
    Dy_2d = sp.kron(Dy_1d,Ix)

    # Second partial derivatives
    D2x_2d = sp.kron(Iy,D2x_1d)
    D2y_2d = sp.kron(D2y_1d,Ix)

    # Return compressed Sparse Row format of the sparse matrices
    return Dx_2d.tocsr(), Dy_2d.tocsr(), D2x_2d.tocsr(), D2y_2d.tocsr()


import numpy as np
import pylab as pl
import random as rd
import time

# Part a
# First we use the 'random' package
# Single seed used for generating a sequence of random numbers
seed = 1
rd.seed(seed)

num_sample = 10**5
# Count the number of numbers generated
count = 0

# Contains the numbers generated, starting the seed
rannumrd = np.array([seed])

while count < num_sample:
    rannumrd = np.append(rannumrd, rd.random())
    count = count + 1

pl.figure()
pl.hist(rannumrd, bins = np.linspace(0., 1., 100)) # 100 bins between 0 and 1
pl.xlabel("x")
pl.ylabel("Number of counts in bin")
pl.title("Distribution of 1e5 random numbers using 'random' package")

# Compare with 'numpy.random.uniform' for uniform deviation
# numpy.seed does not allow seed dtype = float, must be integers
# Also all numerical inputs below must be integers
np.random.seed(seed)

rannumnp = np.array([seed])
rannumnp = np.append(rannumnp, np.random.uniform(low = 0, high = 1, size = num_sample))

pl.figure()
pl.hist(rannumnp, bins = np.linspace(0., 1., 100)) # 100 bins
pl.xlabel("x")
pl.ylabel("Number of counts in bin")
pl.title("Distribution of 1e5 random numbers using 'numpy.random.uniform' package")

# Conclusion: random.random is a better method to use

# Part b - transformation method
# Transforming from uniform P(x) into the PDF given, integrating the PDF between
# 0 and y and equating to x,
# the relation between x and y was found to be y = cos^-1 (1 - 2*x)

# Using x values computed from 'random' package in part a, compute corresponding y
start_time_trans = time.clock() # Records the start time
y = np.arccos(1-2*rannumrd)
time_trans = time.clock() - start_time_trans # Records the end time and subtract from
# start to check running time for this method

# Plotting sine function to check
check = np.linspace(0., np.pi, 500) # Used for plotting comparison function and predicted PDF
pl.figure()
(count, bins, patches) = pl.hist(y, bins = np.linspace(0., np.pi, 100),
normed=True, label = "Histogram of sinusoidal distribution")
# 100 bins between 0 and pi
pl.plot(check, 0.5*np.sin(check), 'r-', label = "Predicted PDF")
pl.xlabel("y")
pl.ylabel("Normalised PDF")
pl.legend()
pl.title("Distribution of 1e5 random numbers using a sinusoidal distribution")

# Part c - rejection method
# PDF (y) takes the values of the sin function in part b and substitute into
# the new PDF given.
PDF = 2./np.pi*(np.sin(y))**2.
comp = np.linspace(0., np.pi, 10**5.+1.)
PDFcomp = 2./np.pi*(np.sin(y))

# Accept = array containing all accepted values
accept = np.array([])
# Counters to keep track of when to stop iterating
num_accept = 0.
num_trials = 0.

start_time_rej = time.clock() # Records the start time
while num_accept < num_sample:
    indexP = rd.randrange(PDF.size)
    compval = rd.uniform(0, PDFcomp[indexP])
    if PDF[indexP] >= compval:
        accept = np.append(accept, y[indexP])
        num_accept += 1.
    num_trials += 1.
time_rej = time.clock() - start_time_rej # Records the end time and subtract from
# start to check running time for this method

pl.figure()
(count2, bins2, patches2) = pl.hist(accept, bins = np.linspace(0., np.pi, 100),
normed=True, label = "Histogram of sine squared distribution") # 100 bins between 0 and pi
pl.plot(check, 2./np.pi*(np.sin(check))**2, 'r-', label = "Predicted PDF")
pl.plot(check, 2./np.pi*(np.sin(check)), 'k-', label = "Comparison function")
pl.legend()
pl.xlabel("y")
pl.ylabel("Normalised PDF")
pl.title("Distribution of 1e5 random numbers using a sine squared distribution")

# Check measuring times
print(time_trans, time_rej, time_trans/time_rej)


import numpy as np
from matplotlib import pyplot as plt
from scipy.misc import comb
from numpy.random import rand

# Decasteljau's algorithm problem
def decasteljau(p,t):
    """ Evaluates a Bezier curve with control points 'p' at time 't'.
    The points in 'p' are assumed to be stored in its rows.
    'p' is assumed to be 2-dimensional."""
    pass

# Bernstein polynomial problem
def bernstein(i, n):
    """ Returns the 'i'th Bernstein polynomial of degree 'n'."""
    pass

# Coordinate function problem.
def bernstein_pt_aprox(X):
    """ Returns the 'x' and 'y' coordinate functions for a 2-dimensional
    Bezier curve with control points 'X'."""
    pass

# plot demonstrating numerical instability in Bernstein polys.
def compare_plot(n, res=501):
    """ Produces a plot showing a Bezier curve evaluated via
    the Decasteljau algorithm and a Bezier curve evaluated using
    Bernstein polynomials. Control points are chosen randomly.
    Instability should be evident for moderately large values of 'n'."""


import numpy as np
import scipy

__all__ = ['slepian_filter','slepian_psd']

def slepian_filter(length, filter_band_length, order=None):
    '''
    Slepian filters bank.

    Parameters
    --------------------
    * length: int,
        the length pf signal.
    * filter_band_length: [int, int],
        filter bandwidth with respect to length
        (in points).
    * order: int,
        order of the filter.

    Returns
    ------------------
    * Matrix of filter windows: 2d ndarray,
        base_band_ratio x order with Slepian
        sequence in the colunms.

    Refernce
    ---------------
    [1a] P. Stoica, R.L. Moses,
        Spectral analysis of signals
        - New-York: Present-Hall, 2005.
    [1b] http://www2.ece.ohio-state.edu/~randy/SAtext/
            - Dr.Moses Spectral Analysis of Signals: Resource Page.

    Examples
    ---------------
        h = slepian_filter(256,4,4)
        plt.plot(h[:,1])

    '''
    N = length
    K = filter_band_length

    if(order is None): order = filter_band_length

    vect = K/N*np.sinc(K/N*np.arange(N))
    gamma = scipy.linalg.toeplitz(vect)
    D,V=np.linalg.eig(gamma)

    h=V[:,:order]

    if np.sum(h[:,1])<0:
        h[:,1]=-h[:,1]

    return h

#---------------------------------------
def slepian_psd(x, order, n_psd=None):
    '''
    Estimation of the pseudo-spectrum based on the
        Slepian - Refil algorithm.

    Parameters
    -------------------
    * x: 1d ndarray.
    * order: int,
        order of the model.
    * n_psd: int or None,
        number of samples in psd.

    Returns
    ------------------------
    * pseudo-spectrum: 1d ndarray.

    Refernce
   ------------------------
    [1a] P. Stoica, R.L. Moses,
        Spectral analysis of signals
        - New-York: Present-Hall, 2005.
    [1b] http://www2.ece.ohio-state.edu/~randy/SAtext/
         Dr.Moses Spectral Analysis of Signals: Resource Page.

    Examples
    ------------------------

    See also
    ------------------------
    capone

    '''
    x=np.array(x)
    N=x.shape[0]

    if(n_psd is None):
        n_psd=N

    h=slepian_filter(N,order,order)

    psd = np.zeros(n_psd)

    for i in np.arange(order):
        sp   = np.fft.fft(x*h[:,i],n_psd)
        psd += np.real(sp*np.conj(sp))# 0 imagenary part remain

    return psd


import numpy as np
from scipy.stats import logistic
import matplotlib.pyplot as plt

class LogisticRegressioner(object):
    def __init__(self, tol=1e-6):
        self.tol = tol

    def first_derivative(self, X, Y, w):
        """
        Calculate the 1st derivative of log-loss function
        """
        d, T = X.shape
        sigma = logistic.cdf(np.multiply(-Y, np.dot(w.T, X)))
        ret = np.multiply(sigma, Y)
        ret = np.multiply(np.repeat(ret[np.newaxis, :], d, axis=0), X)
        ret = np.sum(ret, axis=1)
        return ret

    def log_loss(self, X, Y, w):
        L = np.log(logistic.cdf(np.multiply(Y, np.dot(w.T, X))))
        L = np.sum(L)
        L = -L
        return L

    def classifier_w(self, X, Y,eta=0.05):
        """
        and returns a classification vector w \in Rp obtained by gradient
        descent on the logistic regression loss function
        """
        X = X.T
        d, T = X.shape
        X = np.vstack([np.ones((T, )), X])  # add bias 1
        d += 1
        w = np.zeros(d)
        t = 0
        while True:
            t += 1
            w_ = w + eta*self.first_derivative(X, Y, w)
            L_old, L_new = self.log_loss(X, Y, w_), self.log_loss(X, Y, w)
            if np.abs(L_old - L_new) < self.tol: break
            w = w_
            if t % 300 == 1: yield w  # yield for trace the w

        yield w

    def test_sample(self):
        X = np.array([
            [2, 1],
            [1, 20],
            [1, 5],
            [4, 1],
            [1, 40],
            [3, 30],
        ])

        Y = np.array([
            -1,
            -1,
            -1,
            1,
            1,
            1,
        ])
        ws = list(self.classifier_w(X, Y))
        print ws[-1]


import numpy as np
import numpy.linalg as npla

def qr_iteration(A, tol):
    # Your implementation goes here

    for i in range(A.shape[1] - 1, 0, -1):
        while npla.norm(A[i - 1, :i - 1], ord=2) > tol:
            sigma = A[i, i]
            Q, R = npla.qr(A - sigma * np.diag(np.diag(np.ones(A.shape))))
            A = np.dot(R, Q) + sigma * np.diag(np.diag(np.ones(A.shape)))

    return np.diag(A)

def main():

	tol = 10 ** (-16)
	A_1 = np.array([[2, 3, 2], [10, 3, 4], [3, 6, 1]])
	eigenvalues_1 = qr_iteration(A_1.copy(), tol)
	print("Matrix:\n", A_1)
	print("\n")
	print ("Computed eigenvalues: ", eigenvalues_1)
	print ("Actual eigenvalues: ", np.linalg.eigvals(A_1))

	print("\n\n")

	tol = 10 ** (-16)
	A_2 = np.array([[6, 2, 1], [2, 3, 1], [1, 1, 1]])
	eigenvalues_2 = qr_iteration(A_2.copy(), tol)
	print("Matrix:\n", A_2)
	print("\n")
	print ("Computed eigenvalues: ", eigenvalues_2)
	print ("Actual eigenvalues: ", np.linalg.eigvals(A_2))

if __name__ == '__main__':
	main()


"""
Ankit Khandelwal
15863
Exercise 10
"""

from math import exp

import numpy as np

i = 3 * 10 ** -9
Vt = 0.05
V = 5.
R1 = 1000.
R2 = 4000.
R3 = 3000.
R4 = 2000.

def f1(V1, V2):
    return R2 * (V1 - V) + R1 * V1 + R1 * R2 * i * (exp((V1 - V2) / Vt) - 1)

def f2(V1, V2):
    return R4 * (V2 - V) + R3 * V2 - R3 * R4 * i * (exp((V1 - V2) / Vt) - 1)

def df1_x(V1, V2):
    return R2 + R1 + R1 * R2 * i * exp((V1 - V2) / Vt) / Vt

def df1_y(V1, V2):
    return -R1 * R2 * i * exp((V1 - V2) / Vt) / Vt

def df2_x(V1, V2):
    return -R3 * R4 * i * exp((V1 - V2) / Vt) / Vt

def df2_y(V1, V2):
    return R4 + R3 + R3 * R4 * i * exp((V1 - V2) / Vt) / Vt

x1 = 4.
y1 = 2.
f = np.zeros([2, 1])
J = np.zeros([2, 2])
r1 = np.zeros([2, 1])
r1[0, 0] = x1
r1[1, 0] = y1
d = np.ones([2, 1])
while abs(d[0]) > 10 ** -7 and abs(d[1]) > 10 ** -7:
    V1 = r1[0, 0]
    V2 = r1[1, 0]
    f[0, 0] = f1(V1, V2)
    f[1, 0] = f2(V1, V2)
    f1x = df1_x(V1, V2)
    f1y = df1_y(V1, V2)
    f2x = df2_x(V1, V2)
    f2y = df2_y(V1, V2)
    J[0, 0] = f2y
    J[0, 1] = -f1y
    J[1, 0] = -f2x
    J[1, 1] = f1x
    J = (1 / (f1x * f2y - f1y * f2x)) * J
    d = np.dot(J, f)
    r1 = r1 - d

print("V1 = ", r1[0, 0], "V\nV2 = ", r1[1, 0], "V")
print("V = V1 - V2 = ", r1[0, 0] - r1[1, 0], "V")

Ankit Khandelwal_HW3/Exercise 10/exercise 10.py


import math
import numpy
from numpy import linalg

class HMM(object):

    def __init__(self,
        pi: numpy.ndarray,
        T: numpy.matrix,
        O: numpy.matrix,
        states=None,
        emissions=None):

        """
        pi: initial probability distribution
        T : transition probability matrix
        O : emission probability matrix
        states: list of state names
        emissions: list of emission names
        """

        self.pi = pi
        assert(len(pi) == T.shape[0])
        assert(sum(pi) == 1)

        self.T = T
        assert(T.shape[0] == T.shape[1])
        assert(int(sum(T.A1)) == T.shape[0])

        self.O = O
        assert(O.shape[0] == T.shape[0])
        assert(int(sum(O.A1)) == O.shape[0])

        self.states = states
        if self.states:
            assert(len(self.states) == T.shape[0])

        self.emissions = emissions
        if self.emissions:
            assert(len(emissions) == O.shape[1])

    def allclose(self, A: numpy.matrix, B: numpy.matrix, **kwargs):
        """
        Since `numpy.allclose` method doesn't work as expected
        """
        return numpy.all(list(map(
            lambda a,b: math.isclose(a, b, **kwargs), A.A1, B.A1
        )))

    def stationary_dist(self, type: str, **kwargs):
        """
        Given a transition probabilities matrix T, this function computes
        the stationary distribution pi. One can choose two different methods:
        - naive: power method; loop until T^k == T^(k+1), within a certain tolerance
        (look at the method `numpy.allclose`)
        - linear: solve a linear system pi' = (pi * T)', where pi is a row vector of
        indipendent variables
        It returns the array of stationary distribution and the power k+1 such that
        T^k = T^(k+1), None if type is linear

        **kwargs is the keyword argument accepted by `math.isclose` function
        """
        # Check that T is a square matrix
        # assert(reduce(operator.and_, [len(row) == len(T) for row in T]) == True)
        assert(self.T.shape[0] == self.T.shape[1])
        if type == 'naive':
            T_k = self.T
            T_k_1 = self.T ** 2
            power = 2
            while (not self.allclose(T_k, T_k_1, **kwargs)):
                T_k = T_k_1
                T_k_1 = T_k * self.T
                power += 1
            return numpy.array(T_k_1[0].flat), power
        elif type == 'linear':
            """
            The linear system that one might resolve is,
            with pi a row vector of indipendent variables:
            (pi * T)' = pi'
            T' * pi' - pi' = 0
            Since pi is the vector of indipendent variables, the operation (- pi')
            is the same as subtract 1 from every element in the diagonal of T' * pi'.
            In order to obtain a unique solution we must replace one of the equation
            with sum(pi_i) = 1 for all i from 0 to n-1. In this case I choose the first one
            """
            self.T = self.T.T
            self.T[numpy.diag_indices_from(self.T)] -= 1
            self.T[0,:] = numpy.ones(self.T.shape[0])
            b = numpy.zeros(self.T.shape[0])
            b[0] = 1
            return linalg.solve(self.T, b), None
        else:
            raise Exception('Type unspecified')

    def forward(self, observations):
        f = self.pi
        message_f = [f]
        for obs in observations:
            f = (f @ self.T @ numpy.diag(self.O[:, obs].T.A1)).A1
            message_f.append(f / sum(f))
        return message_f

    def backward(self, observations):
        b = [1, 1]
        message_b = [b]
        for i in range(len(observations)-1, -1, -1):
            b = (self.T @ numpy.diag(self.O[:, observations[i]].T.A1) @ b).A1
            message_b.append(b / sum(b))
        return message_b

    def smoothing(self, observations):
        def normalized_hadamard(v1, v2):
            res = numpy.multiply(v1, v2)
            return res / sum(res)

        return list(
            map(normalized_hadamard,
                self.forward(observations),
                list(reversed(self.backward(observations)))
            )
        )

    def viterbi(self, observations: list):
        """
        Viterbi algorithm in matrix form
        """
        # Compute initial probability given the first observation
        S = numpy.matrix(numpy.diag(self.pi) @ numpy.diag(self.O[:, observations[0]].T.A1))
        saved_max = []
        for obs in observations[1:]:
            # Get, for every cols, the row of the max value
            amax = numpy.argmax(S, axis=0).A1
            max_per_col = []
            # Get max values per columns
            for col in range(len(self.T)):
                max_per_col.append(S[amax[col], col])
            # 'Forward' pass
            S = numpy.diag(max_per_col) @ self.T @ numpy.diag(self.O[:, obs].T.A1)
            # Save where the maximum comes from (needed fro the 'backward' pass)
            saved_max.append(amax)
        # Get indices of the global max
        row, col = numpy.unravel_index(numpy.argmax(S, axis=None), S.shape)
        # Cols will be the state, row becomes col while running backward
        sequence = [self.states[col] if self.states else col]
        for i in range(len(saved_max)-1, -1, -1):
            sequence.append(self.states[row] if self.states else row)
            row = saved_max[i][row]
        return list(reversed(sequence))

if __name__ == "__main__":
    pi = numpy.array([0.52, 0.48])
    T = numpy.matrix([[0.6, 0.4], [0.17, 0.83]])
    O = numpy.matrix([[1/10, 1/10, 1/10, 1/10, 1/10, 1/2], [1/6]*6])

    # pi = numpy.array([0.98, 0.02])
    # T = numpy.matrix([[0.4, 0.6], [0.1, 0.9]])
    # O = numpy.matrix([[0.8, 0.2], [0.1, 0.9]])

    # pi = numpy.array([.5, .5])
    # T = numpy.matrix([[.7, .3], [.3, .7]])
    # O = numpy.matrix([[.9, .1], [.2, .8]])

    hmm = HMM(pi, T, O, ['Loaded', 'Fair'])

    print(hmm.viterbi([2,0,5,5,5,3]))
    # print(hmm.viterbi([1, 0, 1]))
    # print(hmm.forward([0, 0, 1, 0]))
    # print(list(reversed(hmm.backward([0, 0, 1, 0]))))
    print(hmm.smoothing([0, 0, 1, 0]))


"""
Module: libfmp.c2.c2_fourier
Author: Frank Zalkow, Meinard Müller
License: The MIT license, https://opensource.org/licenses/MIT

This file is part of the FMP Notebooks (https://www.audiolabs-erlangen.de/FMP)
"""
import numpy as np
from numba import jit
import librosa

@jit(nopython=True)
def generate_matrix_dft(N, K):
    """Generates a DFT (discrete Fourier transfrom) matrix

    Notebook: C2/C2_DFT-FFT.ipynb

    Args:
        N (int): Number of samples
        K (int): Number of frequency bins

    Returns:
        dft (np.ndarray): The DFT matrix
    """
    dft = np.zeros((K, N), dtype=np.complex128)
    for n in range(N):
        for k in range(K):
            dft[k, n] = np.exp(-2j * np.pi * k * n / N)
    return dft

@jit(nopython=True)
def generate_matrix_dft_inv(N, K):
    """Generates an IDFT (inverse discrete Fourier transfrom) matrix

    Notebook: C2/C2_STFT-Inverse.ipynb

    Args:
        N (int): Number of samples
        K (int): Number of frequency bins

    Returns:
        dft (np.ndarray): The IDFT matrix
    """
    dft = np.zeros((K, N), dtype=np.complex128)
    for n in range(N):
        for k in range(K):
            dft[k, n] = np.exp(2j * np.pi * k * n / N) / N
    return dft

@jit(nopython=True)
def dft(x):
    """Compute the disrcete Fourier transfrom (DFT)

    Notebook: C2/C2_DFT-FFT.ipynb

    Args:
        x (np.ndarray): Signal to be transformed

    Returns:
        X (np.ndarray): Fourier transform of x
    """
    x = x.astype(np.complex128)
    N = len(x)
    dft_mat = generate_matrix_dft(N, N)/N
    return np.dot(dft_mat, x)

@jit(nopython=True)
def idft(X):
    """Compute the inverse discrete Fourier transfrom (IDFT)

    Args:
        X (np.ndarray): Signal to be transformed

    Returns:
        x (np.ndarray): Inverse Fourier transform of X
    """
    X = X.astype(np.complex128)
    N = len(X)
    dft_mat = generate_matrix_dft_inv(N, N)
    return np.dot(dft_mat, X)

@jit(nopython=True)
def twiddle(N):
    """Generate the twiddle factors used in the computation of the fast Fourier transform (FFT)

    Notebook: C2/C2_DFT-FFT.ipynb

    Args:
        N (int): Number of samples

    Returns:
        sigma (np.ndarray): The twiddle factors
    """
    k = np.arange(N // 2)
    sigma = np.exp(-2j * np.pi * k / N)
    return sigma

@jit(nopython=True)
def twiddle_inv(N):
    """Generate the twiddle factors used in the computation of the Inverse fast Fourier transform (IFFT)

    Args:
        N (int): Number of samples

    Returns:
        sigma (np.ndarray): The twiddle factors
    """
    n = np.arange(N // 2)
    sigma = np.exp(2j * np.pi * n / N)
    return sigma

@jit(nopython=True)
def fft(x):
    """Compute the fast Fourier transform (FFT)

    Notebook: C2/C2_DFT-FFT.ipynb

    Args:
        x (np.ndarray): Signal to be transformed

    Returns:
        X (np.ndarray): Fourier transform of x
    """
    x = x.astype(np.complex128)
    N = len(x)
    log2N = np.log2(N)
    assert log2N == int(log2N), 'N must be a power of two!'
    X = np.zeros(N, dtype=np.complex128)

    if N == 1:
        return x
    else:
        this_range = np.arange(N)
        A = fft(x[this_range % 2 == 0])
        B = fft(x[this_range % 2 == 1])
        C = twiddle(N) * B
        X[:N//2] = A + C
        X[N//2:] = A - C
        return X

@jit(nopython=True)
def ifft_noscale(X):
    """Compute the inverse fast Fourier transform (IFFT) without the final scaling factor of 1/N

    Args:
        X (np.ndarray): Fourier transform of x

    Returns:
        x (np.ndarray): Inverse Fourier transform of X
    """
    X = X.astype(np.complex128)
    N = len(X)
    log2N = np.log2(N)
    assert log2N == int(log2N), 'N must be a power of two!'
    x = np.zeros(N, dtype=np.complex128)

    if N == 1:
        return X
    else:
        this_range = np.arange(N)
        A = ifft_noscale(X[this_range % 2 == 0])
        B = ifft_noscale(X[this_range % 2 == 1])
        C = twiddle_inv(N) * B
        x[:N//2] = A + C
        x[N//2:] = A - C
        return x

@jit(nopython=True)
def ifft(X):
    """Compute the inverse fast Fourier transform (IFFT)

    Args:
        X (np.ndarray): Fourier transform of x

    Returns:
        x (np.ndarray): Inverse Fourier transform of X
    """
    return ifft_noscale(X) / len(X)

def stft_basic(x, w, H=8, only_positive_frequencies=False):
    """Compute a basic version of the discrete short-time Fourier transform (STFT)

    Notebook: C2/C2_STFT-Basic.ipynb

    Args:
        x (np.ndarray): Signal to be transformed
        w (np.ndarray): Window function
        H (int): Hopsize (Default value = 8)
        only_positive_frequencies (bool): Return only positive frequency part of spectrum (non-invertible)
            (Default value = False)

    Returns:
        X (np.ndarray): The discrete short-time Fourier transform
    """
    N = len(w)
    L = len(x)
    M = np.floor((L - N) / H).astype(int) + 1
    X = np.zeros((N, M), dtype='complex')
    for m in range(M):
        x_win = x[m * H:m * H + N] * w
        X_win = np.fft.fft(x_win)
        X[:, m] = X_win

    if only_positive_frequencies:
        K = 1 + N // 2
        X = X[0:K, :]
    return X

def istft_basic(X, w, H, L):
    """Compute the inverse of the basic discrete short-time Fourier transform (ISTFT)

    Notebook: C2/C2_STFT-Inverse.ipynb

    Args:
        X (np.ndarray): The discrete short-time Fourier transform
        w (np.ndarray): Window function
        H (int): Hopsize
        L (int): Length of time signal

    Returns:
        x (np.ndarray): Time signal
    """
    N = len(w)
    M = X.shape[1]
    x_win_sum = np.zeros(L)
    w_sum = np.zeros(L)
    for m in range(M):
        x_win = np.fft.ifft(X[:, m])
        # Avoid imaginary values (due to floating point arithmetic)
        x_win = np.real(x_win)
        x_win_sum[m * H:m * H + N] = x_win_sum[m * H:m * H + N] + x_win
        w_shifted = np.zeros(L)
        w_shifted[m * H:m * H + N] = w
        w_sum = w_sum + w_shifted
    # Avoid division by zero
    w_sum[w_sum == 0] = np.finfo(np.float32).eps
    x_rec = x_win_sum / w_sum
    return x_rec, x_win_sum, w_sum

@jit(nopython=True)
def stft(x, w, H=512, zero_padding=0, only_positive_frequencies=False):
    """Compute the discrete short-time Fourier transform (STFT)

    Args:
        x (np.ndarray): Signal to be transformed
        w (np.ndarray): Window function
        H (int): Hopsize (Default value = 512)
        zero_padding (bool): Number of zeros to be padded after windowing and before the Fourier transform of a frame
            (Note: The purpose of this step is to increase the frequency sampling.) (Default value = 0)
        only_positive_frequencies (bool): Return only positive frequency part of spectrum (non-invertible)
            (Default value = False)

    Returns:
        X (np.ndarray): The discrete short-time Fourier transform
    """

    N = len(w)
    x = np.concatenate((np.zeros(N // 2), x, np.zeros(N // 2)))

    L = len(x)
    M = int(np.floor((L - N) / H)) + 1

    X = np.zeros((N + zero_padding, M), dtype=np.complex128)
    zero_padding_vector = np.zeros((zero_padding, ), dtype=x.dtype)

    for m in range(M):
        x_win = x[m * H:m * H + N] * w
        if zero_padding > 0:
            x_win = np.concatenate((x_win, zero_padding_vector))
        X_win = fft(x_win)
        # Note: X_win = np.fft.fft(x_win) does not work in combination with @jit
        X[:, m] = X_win

    if only_positive_frequencies:
        K = 1 + (N + zero_padding) // 2
        X = X[0:K, :]
    return X

@jit(nopython=True)
def istft(X, w, H, L, zero_padding=0):
    """Compute the inverse discrete short-time Fourier transform (ISTFT)

    Args:
        X (np.ndarray): The discrete short-time Fourier transform
        w (np.ndarray): Window function
        H (int): Hopsize
        L (int): Length of time signal
        zero_padding (bool): Number of zeros to be padded after windowing and before the Fourier transform of a frame
            (Default value = 0)

    Returns:
        x (np.ndarray): Reconstructed time signal
    """
    N = len(w)
    L = L + N
    M = X.shape[1]
    w_sum = np.zeros(L)
    x_win_sum = np.zeros(L)
    w_sum = np.zeros(L)
    for m in range(M):
        start_idx, end_idx = m * H, m * H + N + zero_padding
        if start_idx > L:
            break

        x_win = ifft(X[:, m])
        # Note: x_win = np.fft.ifft(X[:, m]) does not work in combination with @jit
        if end_idx > L:
            end_idx = L
            x_win = x_win[:end_idx-start_idx]
            cur_w = w[:end_idx-start_idx]
        else:
            cur_w = w

        # Avoid imaginary values (due to floating point arithmetic)
        x_win_real = np.real(x_win)
        x_win_sum[start_idx:end_idx] = x_win_sum[start_idx:end_idx] + x_win_real
        w_shifted = np.zeros(L)
        w_shifted[start_idx:start_idx + len(cur_w)] = cur_w
        w_sum = w_sum + w_shifted
    # Avoid division by zero
    w_sum[w_sum == 0] = np.finfo(np.float32).eps
    x_rec = x_win_sum / w_sum
    x_rec = x_rec[N // 2:-N // 2]
    return x_rec

def stft_convention_fmp(x, Fs, N, H, pad_mode='constant', center=True, mag=False, gamma=0):
    """Compute the discrete short-time Fourier transform (STFT)

    Notebook: C2/C2_STFT-FreqGridInterpol.ipynb

    Args:
        x (np.ndarray): Signal to be transformed
        Fs (scalar): Sampling rate
        N (int): Window size
        H (int): Hopsize
        pad_mode (str): Padding strategy is used in librosa (Default value = 'constant')
        center (bool): Centric view as used in librosa (Default value = True)
        mag (bool): Computes magnitude STFT if mag==True (Default value = False)
        gamma (float): Constant for logarithmic compression (only applied when mag==True) (Default value = 0)

    Returns:
        X (np.ndarray): Discrete (magnitude) short-time Fourier transform
    """
    X = librosa.stft(x, n_fft=N, hop_length=H, win_length=N,
                     window='hann', pad_mode=pad_mode, center=center)
    if mag:
        X = np.abs(X)**2
        if gamma > 0:
            X = np.log(1 + gamma * X)
    F_coef = librosa.fft_frequencies(sr=Fs, n_fft=N)
    T_coef = librosa.frames_to_time(np.arange(X.shape[1]), sr=Fs, hop_length=H)
    # T_coef = np.arange(X.shape[1]) * H/Fs
    # F_coef = np.arange(N//2+1) * Fs/N
    return X, T_coef, F_coef


from sklearn.datasets import make_regression
from matplotlib import pyplot as plt
import numpy as np
from sklearn.linear_model import LinearRegression

# Functions
def gram_schmidt(a):
    q = []
    for i in range(len(a)):
        #orthogonalization
        q_tilde = a[i]
        for j in range(len(q)):
            q_tilde = q_tilde - (q[j] @ a[i])*q[j]
        #Test for dependennce
        if np.sqrt(sum(q_tilde**2)) <= 1e-10:
            print('Vectors are linearly dependent.')
            print('GS algorithm terminates at iteration ', i+1)
            return q
        #Normalization
        else:
            q_tilde = q_tilde / np.sqrt(sum(q_tilde**2))
            q.append(q_tilde)
    print('Vectors are linearly independent.')
    return q

def QR_factorization(A):
    Q_transpose = np.array(gram_schmidt(A.T))
    R = Q_transpose @ A
    Q = Q_transpose.T
    return Q, R

def back_subst(R,b_tilde):
    n = R.shape[0]
    x = np.zeros(n)
    for i in reversed(range(n)):
        x[i] = b_tilde[i]
        for j in range(i+1,n):
            x[i] = x[i] - R[i,j]*x[j]
        x[i] = x[i]/R[i,i]
    return x

def solve_via_backsub(A,b):
    Q,R = QR_factorization(A)
    b_tilde = Q.T @ b
    x = back_subst(R,b_tilde)
    return x

#########

# # Least squares problem
# A = np.array([[2,0],[-1,1],[0,2]])
# b = np.array([1,0,-1])
# x_hat = np.array([1/3, -1/3])
# r_hat = A @ x_hat - b
# print(np.linalg.norm(r_hat))
# x = np.array([1/2, -1/2]) #other value of x
# r = A @ x - b
# print(np.linalg.norm(r))
# print(np.linalg.inv(A.T @ A) @ A.T @ b)
# print(np.linalg.pinv(A) @ b)
# print((A.T @ A) @ x_hat - A.T @ b) #Check that normal equations hold
# # Principio da ortogonalidade
# z = np.array([-1.1,2.3])
# print(A @ z).T @ r_hat)
# z = np.array([5.3, -1.2])
# print((A @ z).T @ r_hat)

# # Resolvendo problemas de quadrados mínimos
# A = np.random.normal(size = (100,20))
# b = np.random.normal(size = 100)
# x1 = solve_via_backsub(A,b)
# x2 = np.linalg.inv(A.T @ A) @ (A.T @ b)
# x3 = np.linalg.pinv(A) @ b
# print(np.linalg.norm(x1-x2))
# print(np.linalg.norm(x2-x3))
# print(np.linalg.norm(x3-x1))

# Exemplo página 234
n = 10 # número de lâmpadas
# posições (x,y) das lâmpadas e altura acima do chão
lamps = np.array([[4.1 ,20.4, 4],
    [14.1, 21.3, 3.5],
    [22.6, 17.1,6],
    [5.5 ,12.3, 4.0],
    [12.2, 9.7, 4.0],
    [15.3, 13.8, 6],
    [21.3, 10.5, 5.5],
    [3.9 ,3.3, 5.0],
    [13.1, 4.3, 5.0],
    [20.3,4.2, 4.5]])

N = 25 # grid size
m = N*N # Número de pixels
# construct m x 2 matrix with coordinates of pixel centers
pixels = np.hstack([np.outer(np.arange(0.5,N,1),np.ones(N)).reshape(m,1), np.outer(np.ones(N),np.arange(0.5,N,1)).reshape(m,1)])
# The m x n matrix A maps lamp powers to pixel intensities.
# A[i,j] is inversely proportional to the squared distance of
# lamp j to pixel i.
A = np.zeros((m,n))
for i in range(m):
    for j in range(n):
        A[i,j] = 1.0 / (np.linalg.norm(np.hstack([pixels[i,:], 0]) - lamps[j,:])**2)

A = (m / np.sum(A)) * A # scale elements of A
# Least squares solution
x = solve_via_backsub(A, np.ones(m))
rms_ls = (sum((A @ x - 1)**2)/m)**0.5
print(rms_ls)
import matplotlib.pyplot as plt
plt.ion()
plt.hist(A @ x, bins = 25)
plt.show()
plt.pause(10)
# Intensity if all lamp powers are one
rms_uniform = (sum((A @ np.ones(n) - 1)**2)/m)**0.5
print(rms_uniform)
plt.hist(A @ np.ones(n), bins = 25)
plt.show()
plt.pause(10)