import numpy as np
from fraction import Fraction as fr

def get_leading_one(row: np.array):
    for idx, val in enumerate(row):
        if val != fr(0,1):
            return idx
    return len(row)

def rref(matrix: np.array):
    n, m = matrix.shape

    # Iterate through the diagonal of the matrix
    for i in range(min(m, n)):
        leading_one_index = get_leading_one(matrix[i])

        if leading_one_index == m:
            continue

        leading_one = matrix[i][leading_one_index]
        if leading_one != fr(1, 1):
            matrix[i] /= leading_one

        current_row = matrix[i]

        for j in range(n):
            if j == i:
                continue

            factor = matrix[j][leading_one_index]  # /leading_one
            matrix[j] -= current_row * factor

    res = np.empty((n,m))
    for i in range(n):
        for j in range(m):
            res[i,j]=int(matrix[i,j])

    print(res)

    return matrix

if __name__ == "__main__":

    A = np.array([
        [2, 1, 7, -7,2],
        [-3, 4, -5, -6,3],
        [1, 1, 4, -5,2]
    ])

    n, m = A.shape

    B = np.empty((n,m), dtype="object")

    for i in range(n):
        for j in range(m):
            B[i, j] = fr(A[i, j], 1)

    rref(B)


import numpy as np
import pymc3 as pm
import pandas as pd
import arviz as az
from scipy import stats
from scipy.special import logsumexp
from bayesian_routines import standardize
import matplotlib.pyplot as plt

np.random.seed(20897234)
# 7H1
data = pd.read_csv('Laffer.csv', delimiter=';')

rate = standardize(np.array(data['tax_rate']))
revenue = standardize(np.array(data['tax_revenue']))

n_samples = 2000 // 4
n_tuning = 1000

with pm.Model() as linear:
	#a = pm.Normal('b0', mu=0., sigma=0.5)
	b = pm.Normal('b1', mu=0., sigma=0.5)

	s = pm.Exponential('s', lam=0.5)

	m = b * rate

	r = pm.Normal('r', mu=m, sigma=s, observed=revenue)

	traceLinear = pm.sample(n_samples, tune=n_tuning)

with pm.Model() as quadratic:
	a = pm.Normal('b0', mu=0., sigma=0.5)
	b = pm.Normal('b1', mu=0., sigma=0.5)
	c = pm.Normal('b2', mu=0., sigma=0.5)

	s = pm.Exponential('s', lam=0.5)

	m = a + b * rate + c * (rate ** 2)

	r = pm.Normal('r', mu=m, sigma=s, observed=revenue)

	traceQuadratic = pm.sample(n_samples, tune=n_tuning)

with pm.Model() as cubic:
	a = pm.Normal('b0', mu=0., sigma=0.5)
	b = pm.Normal('b1', mu=0., sigma=0.5)
	c = pm.Normal('b2', mu=0., sigma=0.5)
	d = pm.Normal('b3', mu=0., sigma=0.5)

	s = pm.Exponential('s', lam=0.5)

	m = a + b * rate + c * (rate ** 2) + d * (rate ** 3)

	r = pm.Normal('r', mu=m, sigma=s, observed=revenue)

	traceCubic = pm.sample(n_samples, tune=n_tuning)

r = az.compare({'L': traceLinear, 'Q':traceQuadratic, 'C':traceCubic}, 'WAIC')
print(r)

'''
#------------------------------------------------------------------------------------------------------------------
# 7H2
ww = az.waic(traceLinear, pointwise=True)

sInd = np.argmax(revenue)
sampleImportance = np.array(ww[6])

with pm.Model() as linearRobust:
	a = pm.Normal('b0', mu=0., sigma=0.5)
	b = pm.Normal('b1', mu=0., sigma=0.5)

	s = pm.Exponential('s', lam=0.5)

	m = a + b * rate

	r = pm.StudentT('r', nu=1., mu=m, sigma=s, observed=revenue)

	traceLinearRobust = pm.sample(n_samples, tune=n_tuning)

ww = az.waic(traceLinearRobust, pointwise=True)

sampleImportance2 = np.array(ww[6])

fig = plt.figure()
ax = fig.add_subplot(111)
ax.scatter(sampleImportance, sampleImportance2)
ax.set_xlabel('Normal')
ax.set_ylabel('Student t')
ax.set_xlim(0, 25)
ax.set_ylim(0, 25)

# effect is reduce ~by half, but still it is pretty big, much bigger than the other points

#------------------------------------------------------------------------------------------------------------------
# 7H3
def entropy(x):
	return - np.sum(x * np.log2(x))

def kldivergence(p,q):
	return np.sum(p * np.log2(p / q))

d = np.array([[0.2, 0.2, 0.2, 0.2, 0.2], [0.8, 0.1, 0.05, 0.025, 0.025], [0.05, 0.15, 0.7, 0.05, 0.05]])

e = np.zeros(d.shape[0])
for i, island in enumerate(d):
	e[i] = entropy(island)

kls = np.zeros((3,3))

for i in range(3):
	for j in range(3):
		if i != j:
			kls[i,j] = kldivergence(d[i], d[j])

# entropy decreases from 1, 3, 2. 1 is the most 'random'. It is very much expecting species A very often
# more entropy means less KL divergence to other distributions, because it's harder to be surprised by what you see

plt.show()

#------------------------------------------------------------------------------------------------------------------
# 7H5

data = pd.read_csv('../chapter-6/foxes.csv', delimiter=';')

area = standardize(np.array(data['area']))
avgfood = standardize(np.array(data['avgfood']))
weight = standardize(np.array(data['weight']))
groupsize = standardize(np.array(data['groupsize']))

mp = 0.
sp = 0.4
lp = 0.5

n_samples = 2000 // 4
n_tuning = 1000

with pm.Model() as fox1:
	w_f = pm.Normal('wf', mu=mp, sigma=sp)
	w_g = pm.Normal('wg', mu=mp, sigma=sp)
	w_a = pm.Normal('wa', mu=mp, sigma=sp)
	w_0 = pm.Normal('w0', mu=mp, sigma=sp)

	s = pm.Exponential('s', lam=lp)

	mu = w_0 + w_f * avgfood + w_g * groupsize + w_a * area

	W = pm.Normal('w', mu=mu, sigma=s, observed=weight)

	trace1 = pm.sample(n_samples, tune=n_tuning)

with pm.Model() as fox2:
	w_f = pm.Normal('wf', mu=mp, sigma=sp)
	w_g = pm.Normal('wg', mu=mp, sigma=sp)
	w_0 = pm.Normal('w0', mu=mp, sigma=sp)

	s = pm.Exponential('s', lam=lp)

	mu = w_0 + w_f * avgfood + w_g * groupsize
	W = pm.Normal('w', mu=mu, sigma=s, observed=weight)

	trace2 = pm.sample(n_samples, tune=n_tuning)

with pm.Model() as fox3:
	w_g = pm.Normal('wg', mu=mp, sigma=sp)
	w_a = pm.Normal('wa', mu=mp, sigma=sp)
	w_0 = pm.Normal('w0', mu=mp, sigma=sp)

	s = pm.Exponential('s', lam=lp)

	mu = w_0 + w_g * groupsize + w_a * area

	W = pm.Normal('w', mu=mu, sigma=s, observed=weight)

	trace3 = pm.sample(n_samples, tune=n_tuning)

with pm.Model() as fox4:
	w_f = pm.Normal('wf', mu=mp, sigma=sp)
	w_0 = pm.Normal('w0', mu=mp, sigma=sp)

	s = pm.Exponential('s', lam=lp)

	mu = w_0 + w_f * avgfood
	W = pm.Normal('w', mu=mu, sigma=s, observed=weight)

	trace4 = pm.sample(n_samples, tune=n_tuning)

with pm.Model() as fox5:
	w_a = pm.Normal('wa', mu=mp, sigma=sp)
	w_0 = pm.Normal('w0', mu=mp, sigma=sp)

	s = pm.Exponential('s', lam=lp)

	mu = w_0 + w_a * area

	W = pm.Normal('w', mu=mu, sigma=s, observed=weight)

	trace5 = pm.sample(n_samples, tune=n_tuning)

r = az.compare({'m1': trace1, 'm2':trace2, 'm3':trace3, 'm4': trace4, 'm5': trace5}, 'WAIC')
'''
# model with all variables is the best
# ignoring group size is really bad, because it has a negative effect from the other two variables
# using area instead of avgfood is slightly worse and the difference is more consistent (smaller std error)
# probably because area is an indirect cause of weight, while food is direct?


import numpy as np
import matplotlib.pyplot as plt

def numerical_diff(f, x):
    h = 1e-4 # 0.0001
    return (f(x+h) - f(x-h)) / (2*h)
def function_1(x):
    return 0.01*x**2 + 0.1*x

x = np.arange(0.0, 20.0, 0.1) # 0에서 ~ 20까지 0.1 단위
y = function_1(x)
#plt.plot(x, y)
#plt.show()

nd = numerical_diff(function_1, x)
plt.plot(x, nd)
plt.plot(x, y)
plt.show()

a = np.array([[1, 2, 3, 4],
          [4, 5, 6, 7],
          [8, 9, 3, 1]])

#print(np.linalg.pinv(a))

## 4.3.3 partial differential ##
def fuction_2(x):
    return x[0]**2 + x[1]**2

def function_tmp1(x):
    return x*x + 4.0**2.0

print(numerical_diff(function_tmp1, 3.0))

def numericalgradient(f, x):
    h = 1e-4 # 0.0001
    grad = np.zeros_like(x)
    for idx in range(x.size):
        tmp_val = x[idx]
        x[idx] = tmp_val + h
        fxh1 = f(x)
        x[idx] = tmp_val - h
        fxh2 = f(x)
        grad[idx] = (fxh1 - fxh2) / (2*h)
        x[idx] = tmp_val
    return grad


"""
Demonstrate Various Algorithms or prerequisites.

    - Householder reflections
    - Givens Rotations used in Jacobi-Transform and QR-Method.
"""

import numpy as np
import copy
from matplotlib import pyplot as plt

def hreflect1D(x):
    """
    Calculate Householder reflection: Q = I - 2*uu'.

    Parameters:
        X: numpy array.

    Returns:
        q: reflected vector.
        Q: Reflector (matrix).
    """
    # Construct v:
    v = copy.deepcopy(x)
    v[0] += np.linalg.norm(x)

    # Construct u: normalize v.
    vnorm = np.linalg.norm(v)
    if vnorm:
        u = v / np.linalg.norm(v)
    else:
        u = v

    # Construct Q:
    Q = np.eye(len(x)) - 2 * np.outer(u, u)

    return np.dot(Q, x), Q

def qr_factorize(X):
    """
    Compute QR factorization of X s.t. QR = X.

    Parameters:
        - X: square numpy ndarray.

    Returns:
        Q: square numpy ndarray, same shape as X. Rotation matrix.
        R: square numpy ndarray, same shape as X. upper triangular matrix.
    """
    assert type(X) == np.ndarray
    assert X.shape[0] == X.shape[1]

    A = copy.deepcopy(X)
    Q = np.eye(X.shape[0])

    for i in range(X.shape[0]):
        Pi = np.eye(A.shape[0])
        _, Qi = hreflect1D(A[i:, i])
        Pi[i:, i:] = Qi

        Q = Pi.dot(Q)
        A = Pi.dot(A)

    return Q, A

A = np.array([[3, -98/28, 1, 1, 1],
              [1, 122/28, 1, 3, 1],
              [2, -8/28, 89, 1, 1],
              [1, 66/28, 1, 3, 1],
              [1, 10/28, 1, 1, 2]])

Q, R = qr_factorize(A)

Q
R
np.round(R)

np.isclose(A, np.linalg.inv(Q).dot(R))

hreflect1D(np.array([5, 1, 3, 1]))

1/66 * np.array([[-55, -11, -33, -11],
                 [-11, 65, -3, -1],
                 [-33, -3, 57, -3],
                 [-11, -1, -3, 65]])


import numpy as np
import pypolyagamma as pypolyagamma

def calculate_D(S):
    N = S.shape[1]

    D = np.empty((N, N))

    for i in range(N):
        for j in range(N):
            D[i, j] = np.dot(S[1:, i].T, S[:-1, j])

    return D * 0.5

def calculate_C_w(S, w_i):
    w_mat = np.diag(w_i)

    return np.dot(S.T, np.dot(w_mat, S))

def sample_w_i(S, J_i):
    """

    :param S: observation matrix
    :param J_i: neuron i's couplings
    :return: samples for w_i from a polyagamma distribution
    """

    ppg = pypolyagamma.PyPolyaGamma(np.random.randint(2 ** 16))

    T = S.shape[0]
    A = np.ones(T)
    w_i = np.zeros(T)

    ppg.pgdrawv(A, np.dot(S, J_i), w_i)

    return w_i

def sample_J_i(S, C, D_i, w_i, sigma_J):
    N = S.shape[1]

    cov_mat = (1. / sigma_J) * np.identity(N)

    cov = np.linalg.inv(C + cov_mat)
    mean = np.dot(cov, D_i)

    J_i = np.random.multivariate_normal(mean, cov)

    return J_i

def sample_neuron(samp_num, burnin, sigma_J, S, D_i, ro, thin=0, save_all=True):
    """ This function uses the Gibbs sampler to sample from w, gamma and J

    :param samp_num: Number of samples to be drawn
    :param burnin: Number of samples to burn in
    :param sigma_J: variance of the J slab
    :param S: Neurons' activity matrix. Including S0. (T + 1) x N
    :param C: observation correlation matrix. N x N
    :param D_i: time delay correlations of neuron i. N
    :return: samp_num samples (each one of length K (time_steps)) from the posterior distribution for w,x,z.
    """

    T, N = S.shape

    # actual number of samples needed with thining and burin-in
    if (thin != 0):
        N_s = samp_num * thin + burnin
    else:
        N_s = samp_num + burnin

    samples_w_i = np.zeros((N_s, T), dtype=np.float32)
    samples_J_i = np.zeros((N_s, N), dtype=np.float32)

    J_i = np.random.normal(0, sigma_J, N)

    for i in xrange(N_s):
        w_i = sample_w_i(S, J_i)
        C_w_i = calculate_C_w(S, w_i)
        J_i = sample_J_i(S, C_w_i, D_i, w_i, sigma_J)
        samples_w_i[i, :] = w_i
        samples_J_i[i, :] = J_i

    if thin == 0:
        return samples_w_i[burnin:, :], samples_J_i[burnin:, :]
    else:
        return samples_w_i[burnin:N_s:thin, :], samples_J_i[burnin:N_s:thin, :]


# Python program showing Graphical
# representation of tanh() function
import numpy as np
import matplotlib.pyplot as plt

def sigmoid(x):
    return (1 / (1 + np.exp(-x)))

in_array = np.linspace(-8, 8, 120)
out_array = sigmoid(in_array)

print("in_array : ", in_array)
print("\nout_array : ", out_array)

# red for numpy.tanh()
plt.figure(figsize=(5, 4))
plt.plot(in_array, out_array, color='blue')
plt.title("Funkcja logistyczna")
plt.xlabel("X")
plt.ylabel("Y")
plt.show()
#
# def ReLU(x):
#     return x * (x > 0)
#
# in_array = np.linspace(-8, 8, 120)
# out_array = ReLU(in_array)
#
# print("in_array : ", in_array)
# print("\nout_array : ", out_array)
#
# # red for numpy.tanh()
# plt.figure(figsize=(5, 4))
# plt.plot(in_array, out_array, color='blue')
# plt.title("Funkcja progowa unipolarna")
# plt.xlabel("X")
# plt.ylabel("Y")
# plt.show()

# in_array = np.linspace(-8, 8, 120)
# out_array = np.tanh(in_array)
#
# print("in_array : ", in_array)
# print("\nout_array : ", out_array)
#
# # red for numpy.tanh()
# plt.figure(figsize=(5, 4))
# plt.plot(in_array, out_array, color='blue')
# plt.title("Funkcja tangensa hiperbolicznego")
# plt.xlabel("X")
# plt.ylabel("Y")
# plt.show()

# in_array = np.linspace(-8, 8, 120)
# out_array = in_array
#
# print("in_array : ", in_array)
# print("\nout_array : ", out_array)

# # red for numpy.tanh()
# plt.figure(figsize=(5, 4))
# plt.plot(in_array, out_array, color='blue')
# plt.title("Funkcja liniowa")
# plt.xlabel("X")
# plt.ylabel("Y")
# plt.show()


import numpy as np

# This file implements various activation functions that can be used in a layer in a well structured way
# In all of the activations, "forward" states the forward pass and "backward" states the backpropogation on that activation

def sigmoid(z, stage="forward"):
    forward = 1 / (1 + np.exp(-z))
    if stage == "forward":
        return forward
    elif stage == "backward":
        return forward * (1 - forward)
    else:
        raise ValueError("Invalid function stage, should be either \"forward\" or \"backward\"")

def relu(z, stage="forward"):
    if stage == "forward":
        return np.maximum(0.0, z)
    elif stage == "backward":
        return np.where(z >= 0, 1, 0)
    else:
        raise ValueError("Invalid function stage, should be either \"forward\" or \"backward\"")

def leaky_relu(z, stage="forward", negative_slope=0.01):
    if stage == "forward":
        return np.maximum(negative_slope * z, z)
    elif stage == "backward":
        return np.where(z >= 0, 1, negative_slope)
    else:
        raise ValueError("Invalid function stage, should be either \"forward\" or \"backward\"")

def tanh(z, stage="forward"):
    forward = (np.exp(z) - np.exp(-z)) / (np.exp(z) + np.exp(-z))
    if stage == "forward":
        return forward
    elif stage == "backward":
        return 1 - (forward ** 2)
    else:
        raise ValueError("Invalid function stage, should be either \"forward\" or \"backward\"")


from statistics import mean
import numpy as np
import matplotlib.pyplot as plt
from matplotlib import style
import random

style.use('fivethirtyeight')

#xs = np.array([1,2,3,4,5,6], dtype=np.float64)
#ys = np.array([5,4,6,5,6,7], dtype=np.float64)

def create_dataset(hm, variance, step=2, correlation=False):
    val = 1
    ys = []
    for i in range (hm):
        y = val + random.randrange(-variance, variance)
        ys.append(y)
        if correlation and correlation == 'pos':
            val += step
        elif correlation and correlation == 'neg':
            val -= step
    xs = [i for i in range(len(ys))]
    return np.array(xs, dtype=np.float64), np.array(ys, dtype=np.float64)

def best_fit_slope_and_intercept(xs,ys):
    m = ( ((mean(xs) * mean(ys)) - mean(xs*ys))/ (mean(xs)**2 - mean(xs**2)))
    b = mean(ys) - m*mean(xs)
    return m, b

def squared_error(ys_orig, ys_line):
    return sum((ys_line-ys_orig)**2)

def coefficient_of_determination(ys_orig,ys_line):
    y_mean_line = [mean(ys_orig) for y in ys_orig]
    squared_error_reg = squared_error(ys_orig, ys_line)
    squared_error_y_mean = squared_error(ys_orig, y_mean_line)
    return 1 - (squared_error_reg / squared_error_y_mean)

xs, ys = create_dataset (40,10,2, correlation='pos')
#this generates the random data in format (how many points, variance, step (can be pos or neg), and correlation
m, b = best_fit_slope_and_intercept(xs,ys)

regression_line = [(m*x)+b for x in xs]

predict_x = 8
predict_y = m*predict_x + b

r_squared = coefficient_of_determination(ys, regression_line)
print r_squared

plt.scatter(xs,ys)
#plt.scatter(predict_x, predict_y)
plt.plot(xs, regression_line)
plt.show()


"""
Compute the mean of a data set of 3D rotations.
Performs tangent PCA at the mean.
"""

import matplotlib.pyplot as plt
import numpy as np

import geomstats.visualization as visualization

from geomstats.special_orthogonal_group import SpecialOrthogonalGroup

SO3_GROUP = SpecialOrthogonalGroup(n=3)
METRIC = SO3_GROUP.bi_invariant_metric

N_SAMPLES = 20

def main():
    fig = plt.figure(figsize=(15, 5))

    data = SO3_GROUP.random_uniform(n_samples=N_SAMPLES)
    mean = METRIC.mean(data)

    eigenvalues, tangent_eigenvecs = METRIC.tangent_pca(data, base_point=mean)
    n_eigenvalues = min(N_SAMPLES, SO3_GROUP.dimension)

    explained_variances = np.cumsum(eigenvalues)
    explained_variances = explained_variances / explained_variances[-1]

    ax_eig = fig.add_subplot(131)
    xticks = np.arange(0, n_eigenvalues, 1)
    ax_eig.xaxis.set_ticks(xticks)
    ax_eig.set_title('Eigenvalues in decreasing order')
    ax_eig.plot(xticks, eigenvalues)

    ax_var = fig.add_subplot(132)
    xticks = np.arange(0, n_eigenvalues, 1)
    ax_var.xaxis.set_ticks(xticks)
    ax_var.set_title('Explained variance')
    ax_var.set_xlabel('Number of Principal Components')
    ax_var.plot(xticks, explained_variances)

    ax = fig.add_subplot(133, projection="3d")
    plt.setp(ax,
             xlim=(-1, 1), ylim=(-1, 1), zlim=(-1, 1),
             xlabel="X", ylabel="Y", zlabel="Z")

    visualization.plot(data, ax, space='SO3_GROUP', color='black')
    visualization.plot(mean, ax, space='SO3_GROUP', linewidth=3)
    plt.show()

if __name__ == "__main__":
    main()


'''
Generation of Sweep Signals
===========================

Routines to generate and save to wav linear and exponential sweep signals.

Author: 2018 (c) Robin Scheibler
License: MIT License
'''
# Experiment from 2016/08/31
from __future__ import division, print_function

import numpy as np
import sounddevice as sd
import time
from scipy.io import wavfile

def window(signal, n_win):
    ''' window the signal at beginning and end with window of size n_win/2 '''

    win = np.hanning(2*n_win)

    sig_copy = signal.copy()

    sig_copy[:n_win] *= win[:n_win]
    sig_copy[-n_win:] *= win[-n_win:]

    return sig_copy

def exponential_sweep(T, fs, f_low=50., f_high=22000.):
    '''
    Exponential Sweep

    Parameters
    ----------
    T: float
        Length of the sweep in seconds
    fs: float
        Sampling frequency
    f_low: float
        Lowest frequency in the sweep
    f_high: float
        Highest frequency in the sweep
    '''

    f1 = f_low     # Start frequency in [Hz]
    f2 = f_high  # End frequency in [Hz]
    Ts = 1./fs   # Sampling period in [s]
    N = np.floor(T/Ts)
    n  = np.arange(0, N, dtype='float64')  # Sample index

    om1 = 2*np.pi*f1
    om2 = 2*np.pi*f2

    x_exp = 0.95*np.sin(om1*N*Ts / np.log(om2/om1) * (np.exp(n/N*np.log(om2/om1)) - 1))
    x_exp = x_exp[::-1]

    return x_exp

def linear_sweep(T, fs, f_low=50., f_high=22000.):
    '''
    Linear Sweep

    Parameters
    ----------
    T: float
        Length of the sweep in seconds
    fs: float
        Sampling frequency
    f_low: float
        Lowest frequency in the sweep
    f_high: float
        Highest frequency in the sweep
    '''

    f1 = f_low     # Start frequency in [Hz]
    f2 = f_high  # End frequency in [Hz]
    Ts = 1./fs   # Sampling period in [s]
    N = np.floor(T/Ts)
    n  = np.arange(0, N, dtype='float64')  # Sample index

    om1 = 2*np.pi*f1
    om2 = 2*np.pi*f2

    x_exp = 0.95*np.sin(2 * np.pi * 0.5 * (f1 + (f2 - f1) * n / N) * n / fs)
    x_exp = x_exp[::-1]

    return x_exp

if __name__ == "__main__":

    # Signal parameters
    fs = 48000
    f_lo = 20
    f_hi = fs / 2 - 50.
    T = 3.

    # Setup device
    sd.default.device = 2
    sd.default.samplerate = fs
    sd.default.channels = (8,1)

    # create sweep signal
    sweep = np.zeros(int(T*fs))

    # save the sweep to deconvolve later
    sweep_exp = exponential_sweep(T, fs, f_low=f_lo, f_high=f_hi)
    sweep_exp_win = window(sweep_exp, int(fs*0.01))

    wavfile.write("sweep_exp.wav", fs, sweep_exp)
    wavfile.write("sweep_exp_win.wav", fs, sweep_exp_win)

    sweep_lin = linear_sweep(T, fs, f_low=f_lo, f_high=f_hi)
    sweep_lin_win = window(sweep_lin, int(fs*0.01))

    wavfile.write('sweep_lin.wav', fs, sweep_lin)
    wavfile.write('sweep_lin_win.wav', fs, sweep_lin_win)


#function HammingG
#input: a number r
#output: G, the generator matrix of the (2^r-1,2^r-r-1) Hamming code

def hammingGeneratorMatrix(r):
    n = 2**r-1

    #construct permutation pi
    pi = []
    for i in range(r):
        pi.append(2**(r-i-1))
    for j in range(1,r):
        for k in range(2**j+1,2**(j+1)):
            pi.append(k)

    #construct rho = pi^(-1)
    rho = []
    for i in range(n):
        rho.append(pi.index(i+1))

    #construct H'
    H = []
    for i in range(r,n):
        H.append(decimalToVector(pi[i],r))

    #construct G'
    GG = [list(i) for i in zip(*H)]
    for i in range(n-r):
        GG.append(decimalToVector(2**(n-r-i-1),n-r))

    #apply rho to get Gtranpose
    G = []
    for i in range(n):
        G.append(GG[rho[i]])

    #transpose
    G = [list(i) for i in zip(*G)]

    return G

#function decimalToVector
#input: numbers n and r (0 <= n<2**r)
#output: a string v of r bits representing n

def decimalToVector(n,r):
    v = []
    for s in range(r):
        v.insert(0,n%2)
        n //= 2
    return v

#Preliminary setup ########################################

import numpy as np

def int_to_bin(num,w):
    #input number to be converted into binary and width of list

    bin_lst = []

    #turns binary string into list of integers
    for j in np.binary_repr(num, width = w):
        bin_lst.append(int(j))
    return bin_lst

def bin_to_int(bin_lst):
    #input binary list to be converted into integer

    #reverse the list
    bin_lst = bin_lst[::-1]
    total = 0
    for i in range(len(bin_lst)):
        total += bin_lst[i]*2**i
    return total

#Q1 #######################################################

def message(a):
    l = len(a)
    r = 2
    k = 2**r - r - 1
    while l > k - r:
        r += 1
        k = 2**r - r -1
    m = int_to_bin(l,r)
    m.extend(a)
    m.extend([0]*(k-l-r))
    return m

#Q2 #######################################################

def hammingEncoder(m):
    len_m = len(m)
    r = 2
    len_vect = 2**r - r - 1
    while len_m != len_vect:
        r += 1
        len_vect = 2**r - r - 1
        if len_vect > len_m:
            return []
    G = hammingGeneratorMatrix(r)
    c = np.dot(m,G) % 2
    c = list(c)
    return c

#Q3 #######################################################

def hammingDecoder(v):
    len_v = len(v)
    r = 2
    len_vect = 2**r - 1
    while len_v != len_vect:
        r += 1
        len_vect = 2**r - 1
        if len_vect > len_v:
            return []

    #Apply Syndrome decoding with HT as the transpose of the matrix H
    HT = []
    for i in range(1,len_v+1):
	    lst = int_to_bin(i,r)
	    HT.append(lst)
    HT = np.array(HT)
    vHT = np.dot(v, HT) % 2
    if 1 not in vHT:
        return v
    i = bin_to_int(vHT)
    v[i-1] = (v[i-1] + 1) % 2
    return(v)

#Q4 #######################################################

def messageFromCodeword(c):
    len_c = len(c)
    r = 2
    len_vect = 2**r - 1
    while len_c != len_vect:
        r += 1
        len_vect = 2**r - 1
        if len_vect > len_c:
            return []
    c = list(c)
    for i in range(r):
        #replace data in indices of list to be deleted with a dummy variable, in this case the number "3"
        c[2**i-1] = 3
    while 3 in c:
        #remove all dummy variables from list to get message
        c.remove(3)
    m = c
    return m

#Q5 #######################################################

def dataFromMessage(m):
    len_m = len(m)
    r = 2
    len_vect = 2**r - r - 1
    while len_m != len_vect:
        r += 1
        len_vect = 2**r - r - 1
        if len_vect > len_m:
            return []
    bin_str = m[:r]
    l = bin_to_int(bin_str)
    if r+l > len_vect:
        return []
    a = m[r:r+l]
    return a

#Q6 #######################################################

def repetitionEncoder(m,n):
    c = m*n
    return c

#Q7 #######################################################

def repetitionDecoder(v):
    num_zeros = v.count(0)
    num_ones = v.count(1)
    if num_zeros == num_ones:
        return []
    elif num_zeros > num_ones:
        m = [0]
    else:
        m = [1]
    return m

#end ######################################################


from tqdm import tqdm
import numpy as np
import math

from AKDPRFramework.dl.activations import Sigmoid
from AKDPRFramework.utils.dataops import diag

class L1():
    def __init__(self, alpha):
        self.alpha = alpha

    def __call__(self, w):
        return self.alpha * np.linalg.norm(w)

    def grad(self, w):
        return self.alpha * np.sign(w)

class L2():
    def __init__(self, alpha):
        self.alpha = alpha

    def __call__(self, w):
        return self.alpha * 0.5 * w.T.dot(w)

    def grad(self, w):
        return self.alpha * w

class L1L2():
    def __init__(self, alpha, l1_ratio=0.5):
        self.alpha = alpha
        self.l1_ratio = l1_ratio

    def __call__(self, w):
        l1_contr = self.l1_ratio * np.linalg.norm(w)
        l2_contr = (1 - self.l1_ratio) * 0.5 * w.T.dot(w)
        return self.alpha * (l1_contr + l2_contr)

    def grad(self, w):
        l1_contr = self.l1_ratio * np.sign(w)
        l2_contr = (1 - self.l1_ratio) * w
        return self.alpha * (l1_contr + l2_contr)

class Regression(object):
    def __init__(self, iterations, learning_rate):
        self.iterations = iterations
        self.learning_rate = learning_rate
        self.regularization = lambda x: 0

    def random_initialization(self, number_of_features):
        '''
        Random initialization of weights.
        '''
        lim = 1 / math.sqrt(number_of_features)
        self.w = np.random.uniform(-lim, lim, (number_of_features,))

    def fit(self, X, y):
        # Insert constant ones for bias weights
        X = np.insert(X, 0, 1, axis=1)
        self.training_errors = []        # Record of MSE errors during training
        self.random_initialization(number_of_features=X.shape[1])

        for _ in tqdm(range(self.iterations)):
            y_pred = X.dot(self.w)
            # MSE
            mse = np.mean(0.5 * (y - y_pred) ** 2 + self.regularization(self.w))
            self.training_errors.append(mse)
            # Gradient Calculation
            grad_w = -(y - y_pred).dot(X) + self.regularization.grad(self.w)
            # Update the weights
            self.w -= self.learning_rate * grad_w

    def predict(self, X):
        # Insert constant ones for bias weights
        X = np.insert(X, 0, 1, axis=1)
        y_pred = X.dot(self.w)
        return y_pred

    def get_weights(self):
        """
        Returns weights at the end of training.
        Call like this
        >>> model = LinearRegression()
        >>> weights_before_training = model.get_weights()     # Random Initialization
        >>> model.fit(X, y)
        >>> weights = model.get_weights() # Updated weights after training
        """
        return self.w

class LinearRegression(Regression):
    """
    Linear Regression model

        Args:
            - ``iterations``: The number of training iterations the algorithm will tune the weights for.
            - ``learning_rate``: The step length that will be used when updating the weights.
            - ``gradient_descent``: True or False depending if gradient descent should be used when training. If False then we use batch optimization by least squares.
    """

    def __init__(self, iterations=1000, learning_rate=1e-3, gradient_descent=True):
        super(LinearRegression, self).__init__(iterations=iterations,
                                               learning_rate=learning_rate)
        self.gradient_descent = gradient_descent
        self.regularization = lambda x: 0
        self.regularization.grad = lambda x: 0

    def fit(self, X, y):
        # If not gradient descent => Least squares approximation of w
        if not self.gradient_descent:
            # Insert constant ones for bias weights
            X = np.insert(X, 0, 1, axis=1)
            # Calculate weights by least squares (using Moore-Penrose pseudoinverse)
            U, S, V = np.linalg.svd(X.T.dot(X))
            S = np.diag(S)
            X_sq_reg_inv = V.dot(np.linalg.pinv(S)).dot(U.T)
            self.w = X_sq_reg_inv.dot(X.T).dot(y)
        else:
            super(LinearRegression, self).fit(X, y)

class LogisticRegression:
    """
    Logistic Regression Class

        Args:
            - ``learning_rate``: Mention the learning Rate for the training. Defaults to 1e-3
            - ``gradient_descent``: True or False depending upon training with gradient or batch optimization by least squares.
    """

    def __init__(self, learning_rate=1e-3, gradient_descent=True):
        self.params = None
        self.learning_rate = learning_rate
        self.gradient_descent = gradient_descent
        self.sigmoid = Sigmoid()

    def _initialize_parameters(self, X):
        number_of_features = np.shape(X)[1]
        lim = 1 / math.sqrt(number_of_features)

        self.params = np.random.uniform(-lim, lim, (number_of_features,))

    def fit(self, X, y, n_iterations=1000):
        self._initialize_parameters(X)
        # Tune parameters for n iterations
        for _ in tqdm(range(n_iterations)):
            # Make a new prediction
            y_pred = self.sigmoid(X.dot(self.params))
            if self.gradient_descent:
                # Move against the gradient of the loss function with
                # respect to the parameters to minimize the loss
                self.params -= self.learning_rate * -(y - y_pred).dot(X)
            else:
                # Make a diagonal matrix of the sigmoid gradient column vector
                diag_gradient = diag(self.sigmoid.gradient(X.dot(self.params)))
                # Batch opt:
                self.param = np.linalg.pinv(X.T.dot(diag_gradient).dot(X)).dot(X.T).dot(diag_gradient.dot(X).dot(self.param) + y - y_pred)

    def predict(self, X):
        y_pred = np.round(self.sigmoid(X.dot(self.param))).astype(int)
        return y_pred


import math
import numpy as np
import random
import time
import pylab
import matplotlib.pyplot as plt

from matplotlib import cm
from mpl_toolkits.mplot3d import Axes3D
from itertools import accumulate
from pprint import pprint

from task1 import RandomVariableSensor, RandomVariableTester

def timer(func: object):
    def wrapper(*args, **kwargs):
        start_time = time.time()
        func()
        print(f'\n\nLead time: {time.time() - start_time}s')
    return wrapper

class GenerationRandomVariables:

    def __init__(self, number_amount: int) -> None:
        self.number_amount = number_amount

    def density(self, x: float, y: float) -> float:
        return math.exp(-x - y)

    def neumann_method(self, a: float, b: float) -> list:
        return [self.generate_xy(a, b) for i in range(self.number_amount)]

    def generate_xy(self, a: float, b: float) -> tuple:
        m, k = 2**31, 16807
        start = random.randint(10**8, 10**9)
        value = random.randint(10**8, 10**9)

        xy = RandomVariableSensor(2).mult_congruent_method(start, m, k)
        w = random.random() * 1
        xy = (a + xy[0]*(b - a), xy[1])

        return xy if w <= self.density(*xy) else self.generate_xy(a, b)

    def correlation(
            self, x: list, y: list,
            m_x: float, m_y: float,
            d_x: float, d_y: float):
        x, y = np.array(x), np.array(y)
        return (sum(x * y) / len(x) - m_x * m_y) / (d_x * d_y) ** (1/2)

    def generate_matrix(self, m: int, n: int) -> list:
        numbers = np.array([i for i in range(1, m * n + 1)])
        random.shuffle(numbers)
        return np.array_split(numbers / sum(numbers), m)

    def generate_vector(self, size: int) -> list:
        return [i for i in range(size)]

    def distribution_func(self, probabilities: list) -> list:
        return [0] + list(accumulate(probabilities))

    def generate_matrix_y(self, matrix: list, x_probabilities: list) -> list:
        return [row / x_probabilities[i] for i, row in enumerate(matrix)]

    def discrete_random_variables(
            self, v_x: list, v_y: list,
            f_x: list, f_y: list):
        x = [random.random() for i in range(self.number_amount)]
        y = [random.random() for i in range(self.number_amount)]

        new_x, new_y = [], []

        for i, j in zip(x, y):
            new_x.append(
                v_x[f_x.index(next(filter(lambda x: i < x, f_x))) - 1])

            z = new_x[0]

            new_y.append(
                v_y[f_y[z].index(next(filter(lambda y: j < y, f_y[z]))) - 1])

        drv = list(zip(new_x, new_y))

        matrix = np.zeros((len(v_x), len(v_y)))
        empir_matrix = np.copy(matrix)

        for pair in drv:
            matrix[pair[0]][pair[1]] += 1
            empir_matrix[pair[0]][pair[1]] += (1 / self.number_amount)

        return matrix, empir_matrix, drv

    def construction_distrib_func(self, vector: list, dist: list) -> tuple:
        x = sum([[i, j, np.nan] for i, j in zip(vector[:-1], vector[1:])], [])
        y = sum([[i, i, np.nan] for i in dist[1:]], [])
        return x + [x[-2], x[-2] + (x[-2] - x[-3]), np.nan], y

@timer
def main():
    start, quantity = 19941995, 100_000
    m, k = 2**31, 16807
    a, b = 0, 1
    intervals = 10
    s = 2
    m, n = 4, 4

    generator = GenerationRandomVariables(quantity)
    sensor = RandomVariableSensor(quantity)

    new_xy = generator.neumann_method(a, b)
    x, y = zip(*new_xy)

    hits_rate_x = RandomVariableTester(intervals, x).hits_rate()
    hits_rate_y = RandomVariableTester(intervals, y).hits_rate()

    m_x, d_x, h_x = RandomVariableTester(intervals, x).uniformity_testing()
    m_y, d_y, h_y = RandomVariableTester(intervals, y).uniformity_testing()

    print('\nRelative hit rates X:')
    pprint(hits_rate_x)
    print(f'Sum of probabilities: {round(sum(hits_rate_x.values()), 10)}')

    print('\nRelative hit rates Y:')
    pprint(hits_rate_y)
    print(f'Sum of probabilities: {round(sum(hits_rate_y.values()), 10)}')

    print('\nExpected value, dispersion, histogram border for X:')
    print(m_x, d_x, h_x)

    print('\nExpected value, dispersion, histogram border for Y:')
    print(m_y, d_y, h_y)

    m_t = -2 * math.exp(-1) + 1
    d_t = -5 * math.exp(-1) + 2 - m_t ** 2

    print('\nTheoretical expected value, dispersion for X:')
    print(f'M[x]={m_t}')
    print(f'D[x]={d_t}')

    print('\nTheoretical expected value, dispersion for Y:')
    print(f'M[y]={m_t}')
    print(f'D[y]={d_t}')

    print('\nCorrelation coefficient X:')
    print(RandomVariableTester(intervals, x).independence_testing(s))

    print('\nCorrelation coefficient Y:')
    print(RandomVariableTester(intervals, y).independence_testing(s))

    print('\nCorrelation coefficient XY:')
    print(generator.correlation(x, y, m_x, m_y, d_x, d_y))

    fig, axs = plt.subplots(1, 2, sharey=True, tight_layout=True)
    axs[0].set_yticks(np.arange(0, 1, 0.02), minor=False)

    axs[0].bar(
        sorted([i[0] for i in hits_rate_x.keys()]),
        hits_rate_x.values(),
        width=1/intervals, align='edge')

    axs[0].set_title('X')
    axs[1].bar(
        sorted([i[0] for i in hits_rate_y.keys()]),
        hits_rate_y.values(),
        width=1/intervals, align='edge')
    axs[1].set_title('Y')

    x, y = np.arange(0, 1, 0.1), np.arange(0, 1, 0.1)
    x, y = np.meshgrid(x, y)
    fig = pylab.figure()
    axes = Axes3D(fig)
    axes.plot_surface(x, y, np.exp(-x - y), rstride=1, cstride=1, cmap=cm.jet)

    matrix_xy = generator.generate_matrix(m, n)

    print('\nProbability matrix XY:')
    pprint(matrix_xy)
    print(f'Sum of probabilities XY: {sum(sum(matrix_xy))}')

    vector_x = generator.generate_vector(m)
    vector_y = generator.generate_vector(n)

    print(f'\nVector X:\n{vector_x}')
    print(f'\nVector Y:\n{vector_y}')

    x_probabilities = np.array([sum(i) for i in matrix_xy])
    print(f'\nProbability X:\n{x_probabilities}')

    distribution_x = generator.distribution_func(x_probabilities)
    print(f'\nDistribution function F(X):\n{distribution_x}')

    y_propabities = generator.generate_matrix_y(matrix_xy, x_probabilities)
    print('\nProbability matrix Y:')
    pprint(y_propabities)

    distribution_y = [generator.distribution_func(i) for i in y_propabities]
    print('\nDistribution function F(Y):')
    pprint(distribution_y)

    drv = generator.discrete_random_variables(
        vector_x, vector_y,
        distribution_x, distribution_y)

    print(f'\nDRV matrix:\n{drv[0]}')
    print(f'\nEmpirical matrix:\n{drv[1]}')

    xy = np.array(drv[2])
    x, y = xy[:, 0], xy[:, 1]
    m_x, d_x = np.mean(x), sum(x ** 2 - np.mean(x) ** 2) / len(x)
    m_y, d_y = np.mean(y), sum(y ** 2 - np.mean(y) ** 2) / len(y)

    print(f'\nExpected value, dispersion X:\n{m_x} {d_x}')
    print(f'\nExpected value, dispersion Y:\n{m_y} {d_y}')

    fig, axs = plt.subplots(1, 1, sharey=True, tight_layout=True)
    x, y = generator.construction_distrib_func(vector_x, distribution_x)
    axs.set_yticks(np.arange(0, 1.05, 0.05), minor=False)
    axs.plot(x, y, 'o-')
    axs.set_title('Distribution function F(X):')

    fig, axs = plt.subplots(
        1, len(distribution_y),
        sharey=True, tight_layout=True)

    for i, distribution in enumerate(distribution_y):
        x, y = generator.construction_distrib_func(vector_y, distribution)
        axs[i].set_yticks(np.arange(0, 1.05, 0.05), minor=False)
        axs[i].plot(x, y, 'o-')
        axs[i].set_title(f'Distribution function F(Y)[{i}]:')

    plt.show()
    pylab.show()

if __name__ == '__main__':
    main()


#!/usr/bin/env python
# coding=utf-8
"""
Script to forecast pricing with geometric brownian motion
"""
from __future__ import division

import random
import numpy as np
import matplotlib.pyplot as plt

def calc_price_geo_brown_motion(price_init, timespan, timestep, mu, sigma):
    """
    Forecast pricing with geometric brownian motion

    Parameters
    ----------
    price_init : float
        Initial price
    timespan : int
        Total timespan (e.g. in years)
    timestep : int
        Time discretization (e.g. in years)
    mu : float
        Drift / mean of price change per timestep
    sigma : float
        Shock / standard deviation of price change per timestep

    Returns
    -------
    brown_motion : array-like
        Array with new pricing data values
    """

    steps = int(round(timespan / timestep))
    time_arranged = np.linspace(0, timespan, steps)
    w = np.random.standard_normal(size=steps)
    w = np.cumsum(w) * np.sqrt(timestep)
    x = (mu - 0.5 * sigma ** 2) * time_arranged + sigma * w
    brown_motion = price_init * np.exp(x)

    return brown_motion

if __name__ == '__main__':

    timespan = 20

    mu_min = -0.05
    mu_max = 0.05

    sigma = 0.05
    price_init = 0.26
    timestep = 1

    nb_runs = 50

    for i in range(nb_runs):

        mu = random.uniform(mu_min, mu_max)

        prices = calc_price_geo_brown_motion(timestep=timestep,
                                             timespan=timespan,
                                             mu=mu, sigma=sigma,
                                             price_init=price_init)

        plt.plot(prices)

    plt.title('Brownian motion price change for ' + str(nb_runs) + ' runs')
    plt.xlabel('Time in years')
    plt.ylabel('Price in Euro')
    plt.show()


import scipy.integrate as spi
import numpy as np

print('Double integral computed by SciPy nquad')
print('Example 2-02 nquad')
print('Integral of 2xye^-xy from y=1 to x-->+inf and from x=1 to x-->+inf')

integrand = lambda x, y : 2 * x * y * np.exp(-x * y)
ya = 1.
yb = 5.

result, error = spi.nquad(integrand, [[1, np.inf],[1, np.inf]])
print('Result is ', result, 'with error ', error)


# -*- coding: utf-8 -*-
"""poisson-distribution-and-poisson-process-tutorial.ipynb

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/1u-t6oSxbMd2FrzaIMimXnPMR5dfV6Gel

# Diving Into the Poisson Distribution and Poisson Process

* Tutorial: https://news.towardsai.net/pd
* Github: https://github.com/towardsai/tutorials/tree/master/poisson-distribution-process

A TV Assembly unit performs a defects analysis task to understand the number of defects that could happen for a given defective TV. Using past quality and audit data is noted that 12 defects are marked on an average for a defective TV, calculate below:

* The probability that a defective laptop has exactly 5 defects.
* The probability that a defective laptop has less than 5 defects.
"""

import numpy as np
import scipy.stats as stats
import matplotlib.pyplot as plt

n = np.arange(0,30)
n

rate = 12
poisson = stats.poisson.pmf(n,rate)

poisson

poisson[5]

poisson[0] + poisson[1] + poisson[2] + poisson[3] + poisson[4]

plt.plot(n,poisson, 'o-')
plt.show()


"""Softmax model."""

import numpy as np

class Softmax:
    def __init__(self, n_class: int, lr: float, epochs: int, reg_const: float):
        """Initialize a new classifier.

        Parameters:
            n_class: the number of classes
            lr: the learning rate
            epochs: the number of epochs to train for
            reg_const: the regularization constant
        """
        self.w = None
        self.lr = lr
        self.epochs = epochs
        self.reg_const = reg_const
        self.n_class = n_class

    def calc_gradient(self, X_train: np.ndarray, y_train: np.ndarray) -> np.ndarray:
        """Calculate gradient of the softmax loss.

        Inputs have dimension D, there are C classes, and we operate on
        mini-batches of N examples.

        Parameters:
            X_train: a numpy array of shape (N, D) containing a mini-batch
                of data
            y_train: a numpy array of shape (N,) containing training labels;
                y[i] = c means that X[i] has label c, where 0 <= c < C

        Returns:
            gradient with respect to weights w; an array of same shape as w
        """
        dw = np.zeros((np.shape(X_train)[1], self.n_class))
        for i in range(X_train.shape[0]):
            pred = np.dot(X_train[i], self.w)
            pred -= np.max(pred)
            prob = np.divide(np.exp(pred), np.sum(np.exp(pred), axis=0))
            for j in range(self.n_class):
                if j == y_train[i]:
                    dw[:, j] = np.add(dw[:, j], (prob[j] - 1) * X_train[i])
                else:
                    dw[:, j] = np.add(dw[:, j], prob[j] * X_train[i])
        dw /= X_train.shape[0]
        return dw + self.w * self.reg_const

    def train(self, X_train: np.ndarray, y_train: np.ndarray):
        """Train the classifier.

        Hint: operate on mini-batches of data for SGD.

        Parameters:
            X_train: a numpy array of shape (N, D) containing training data;
                N examples with D dimensions
            y_train: a numpy array of shape (N,) containing training labels
        """
        self.w = np.random.normal(0, .01, size=(np.shape(X_train)[1], self.n_class))
        batch_size = 200
        for _ in range(self.epochs):
            idx = np.sort(np.random.choice(X_train.shape[0],size=batch_size,replace=False))
            dw = self.calc_gradient(X_train[idx], y_train[idx])
            self.w -= self.lr * dw

    def predict(self, X_test: np.ndarray) -> np.ndarray:
        """Use the trained weights to predict labels for test data points.

        Parameters:
            X_test: a numpy array of shape (N, D) containing testing data;
                N examples with D dimensions

        Returns:
            predicted labels for the data in X_test; a 1-dimensional array of
                length N, where each element is an integer giving the predicted
                class.
        """
        pred = np.argmax(np.dot(X_test, self.w), axis = 1)
        return pred


from __future__ import annotations
import numpy as np
from numpy.linalg import inv, det, slogdet

# Implemented in Ex1

class UnivariateGaussian:
    """
    Class for univariate Gaussian Distribution Estimator
    """
    def __init__(self, biased_var: bool = False) -> UnivariateGaussian:
        """
        Estimator for univariate Gaussian mean and variance parameters

        Parameters
        ----------
        biased_var : bool, default=False
            Should fitted estimator of variance be a biased or unbiased estimator

        Attributes
        ----------
        fitted_ : bool
            Initialized as false indicating current estimator instance has not been fitted.
            To be set as True in `UnivariateGaussian.fit` function.

        mu_: float
            Estimated expectation initialized as None. To be set in `UnivariateGaussian.fit`
            function.

        var_: float
            Estimated variance initialized as None. To be set in `UnivariateGaussian.fit`
            function.
        """
        self.biased_ = biased_var
        self.fitted_, self.mu_, self.var_ = False, None, None

    def fit(self, X: np.ndarray) -> UnivariateGaussian:
        """
        Estimate Gaussian expectation and variance from given samples

        Parameters
        ----------
        X: ndarray of shape (n_samples, )
            Training data

        Returns
        -------
        self : returns an instance of self.

        Notes
        -----
        Sets `self.mu_`, `self.var_` attributes according to calculated estimation (where
        estimator is either biased or unbiased). Then sets `self.fitted_` attribute to `True`
        """

        self.mu_ = 1/len(X) * np.sum(X)
        if self.biased_ :
            self.var_ = 1/len(X) * np.sum( (X - self.mu_) **2 )
        else:
            self.var_ = 1/(len(X)-1) * np.sum( (X - self.mu_) **2 )

        self.fitted_ = True
        return self

    def pdf(self, X: np.ndarray) -> np.ndarray:
        """
        Calculate PDF of observations under Gaussian model with fitted estimators

        Parameters
        ----------
        X: ndarray of shape (n_samples, )
            Samples to calculate PDF for

        Returns
        -------
        pdfs: ndarray of shape (n_samples, )
            Calculated values of given samples for PDF function of N(mu_, var_)

        Raises
        ------
        ValueError: In case function was called prior fitting the model
        """
        if not self.fitted_:
            raise ValueError("Estimator must first be fitted before calling `pdf` function")
        stddiv = np.sqrt(self.var_)
        pdf = 1 / ( np.sqrt(2 * np.pi) * stddiv ) * \
                    np.exp( - ( X - self.mu_) ** 2 / (2 * stddiv ** 2) )

        return pdf

        # The full graph of a PDF:
        #sigma = np.sqrt(self.var_)
        #bins = round(np.sqrt(len(X)))
        #range_tuple = (self.mu_ - 5*sigma , self.mu_ + 5*sigma )
        #hist, bins_edges = np.histogram(X, bins=bins, range=range_tuple )
        #pdf = hist / len(X) * bins / (range_tuple[1] - range_tuple[0])
        #print("integral pdf samples "+str(sum(pdf) * (bins_edges[1]-bins_edges[0]) ) )

    @staticmethod
    def log_likelihood(mu: float, sigma: float, X: np.ndarray) -> float:
        """
        Calculate the log-likelihood of the data under a specified Gaussian model

        Parameters
        ----------
        mu : float
            Expectation of Gaussian
        sigma : float
            Variance of Gaussian
        X : ndarray of shape (n_samples, )
            Samples to calculate log-likelihood with

        Returns
        -------
        log_likelihood: float
            log-likelihood calculated
        """
        Xmm = X - mu
        m = len(X)
        loglike = m * ( - 0.5*np.log(2*np.pi) - np.log(sigma) ) - np.sum( Xmm * Xmm ) / 2*sigma**2

        return loglike

class MultivariateGaussian:
    """
    Class for multivariate Gaussian Distribution Estimator
    """
    def __init__(self):
        """
        Initialize an instance of multivariate Gaussian estimator

        Attributes
        ----------
        fitted_ : bool
            Initialized as false indicating current estimator instance has not been fitted.
            To be set as True in `MultivariateGaussian.fit` function.

        mu_: ndarray of shape (n_features,)
            Estimated expectation initialized as None. To be set in `MultivariateGaussian.fit`
            function.

        cov_: ndarray of shape (n_features, n_features)
            Estimated covariance initialized as None. To be set in `MultivariateGaussian.fit`
            function.
        """
        self.mu_, self.cov_ = None, None
        self.fitted_ = False

    def fit(self, X: np.ndarray) -> MultivariateGaussian:
        """
        Estimate Gaussian expectation and covariance from given samples

        Parameters
        ----------
        X: ndarray of shape (n_samples, n_features)
            Training data

        Returns
        -------
        self : returns an instance of self

        Notes
        -----
        Sets `self.mu_`, `self.cov_` attributes according to calculated estimation.
        Then sets `self.fitted_` attribute to `True`
        """

        self.mu_ =  1/np.size(X, axis=0) * np.sum(X, axis=0)

        # make a 2D array of mu (of each feature) per sample
        mu2d = [self.mu_] * np.size(X, axis=0)
        # X minus mu (expected values) to have only offsets from expected estimated value
        Xmm = X - mu2d
        # make 3D array having the new axis (num 2) in size of axis 1 (also size of mu)
        X3mm = np.repeat( Xmm[:,:,np.newaxis], np.size(X,1), axis=2 )
        X3mmT = np.transpose(X3mm, axes=(0,2,1))

        # The "in-place" X3mm * X3mmT will result in M[i,j,smp] = f_i*f_j of the sample
        # so averging along the 0 axis will give the Covariance of x_i with x_j
        self.cov_ = 1/(np.size(X,0)-1) * np.sum( X3mm * X3mmT  ,axis=0)

        self.fitted_ = True
        return self

    def pdf(self, X: np.ndarray):
        """
        Calculate PDF of observations under Gaussian model with fitted estimators

        Parameters
        ----------
        X: ndarray of shape (n_samples, n_features)
            Samples to calculate PDF for

        Returns
        -------
        pdfs: ndarray of shape (n_samples, )
            Calculated values of given samples for PDF function of N(mu_, cov_)

        Raises
        ------
        ValueError: In case function was called prior fitting the model
        """
        if not self.fitted_:
            raise ValueError("Estimator must first be fitted before calling `pdf` function")

        # make a 2D array of mu (of each feature) per sample
        mu2d = [self.mu_] * np.size(X, axis=0)
        # X minus mu (expected values) to have only offsets from expected estimated value
        Xmm = X - mu2d

        samples_pdf = []
        for smp in Xmm:
            pdf = 1 / ( np.sqrt(2 * np.pi) * np.linalg.det(self.cov_) ) * \
                        np.exp( -0.5 * smp @ self.cov_ @ smp.T )
            samples_pdf.append(pdf)

        return np.array(samples_pdf)

    @staticmethod
    def log_likelihood(mu: np.ndarray, cov: np.ndarray, X: np.ndarray) -> float:
        """
        Calculate the log-likelihood of the data under a specified Gaussian model

        Parameters
        ----------
        mu : ndarray of shape (n_features,)
            Expectation of Gaussian
        cov : ndarray of shape (n_features, n_features)
            covariance matrix of Gaussian
        X : ndarray of shape (n_samples, n_features)
            Samples to calculate log-likelihood with

        Returns
        -------
        log_likelihood: float
            log-likelihood calculated over all input data and under given parameters of Gaussian
        """

        # make a 2D array of mu (of each feature) per sample
        mu2d = [mu] * np.size(X, axis=0)
        # X minus mu (expected values) to have only offsets from expected estimated value
        Xmm = X - mu2d

        _, logdet_cov = np.linalg.slogdet(cov)
        loglike_coeff = - 0.5*np.log(2*np.pi) - logdet_cov
        accumulate = 0.0
        if False : # The clear but heavy computation method
            for smp in Xmm:
                loglike = loglike_coeff - 0.5 * smp @ np.linalg.inv(cov) @ smp.T
                accumulate = accumulate + loglike
        else: # cumputaion effective calculation
            # per sample feature set we have the multiple by the cov-matrix and then
            # each vector (alone) is multipled by the features vector again.
            accumulate =  len(X) * loglike_coeff - 0.5 * np.sum( (Xmm @ cov) * Xmm ) # like diag(Xmm @ cov @ Xmm)
            # Note the sum is both to complete the last inner-product (on axis 1)
            # and also summing along all samples (on axis 0)

        return accumulate


#numpy dizileri üzerinde vektörel işlemler  :işleme girerken karşılıklı elemanlar işleme giriyor
#her elemana aynı skaler işlemleri yap ak istersem np array ierisinde işlem yaptırmalıyım

#numpy mantığı:veri setini al sütunlar feature olsun satırlar item olsun bu şekilde analiz et. ####

import numpy as np
import matplotlib.pyplot as plt
#her matris
a = np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]])
b = np.array([[10], [20], [30]])

c = a + b
print(c)

#matris çarpımı yapmak istersem dot fonksiyonunu kullanmalıyım
a = np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]])
b = np.array([[1], [2], [3]])
c = np.dot(a, b)
print(c)

#broadcasting islemi ile satır satır toplama satır ve sütun uyumuna bakmadan işlem yapılabilir.satırlar işlem yaptırılıyor
a = np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]])
b = np.array([10, 20, 30])
c = a + b
print(c)

#yukarıdaki örneğe benzemesine rağmen farklı bir output oluşturur:sütunlar işleme sokuluyor
a = np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]])
b = np.array([[10], [20], [30]])
c = a + b
print(c)

#linspace fonksiyonu ile grafik çizdirme
import numpy as np
import matplotlib.pyplot as plt
x = np.linspace(-6.28, 6.28, 100) #100'er tane x ve y noktası elde edip grafik çizdireceğim
y = np.sin(x)
plt.plot(x, y) #grafiği shown etme işlemi yapıyorum

#x**2 -4 parabolünü çizdirmek istersem
x = np.linspace(-10, 10, 100)
y = x ** 2 - 4
plt.plot(x, y)

#max fonksiyonu ile en büyük elemana erişmek istersem
import numpy as np
a = np.array([[23, 4, 5], [17, 7, 8], [9, 4, 4]])
result = np.max(a)
print(result) #23 ekran çıktısı oluşuyor

#bir nparrayde ve bir sutunda en büyük değeri bulmak istersem
a = np.array([[23, 4, 5], [17, 7, 8], [9, 15, 4]])
print(a)
print('-----------')
result = np.max(a, axis=0) #sütunlardaki en büyük değerleri buluyor
result2 = np.max(a, axis=1) #satırlardaki en büyük değeri bulmak istersem
print(result)
print(result2)

#sort fonksiyonu:view oluşturmaz
a = np.array([23, 4, 5, 17, 7, 8, 9, 4, 4])
b = np.sort(a) # a yı sort edip yeni bir b nesnesi oluşturuyor
print(b)
print('-' * 10)
a.sort() #a dizisini kendi icinde srt ediyor
print(a)

#satırları da kendi aralarında sort edebilirim
a = np.array([[23, 4, 5], [17, 7, 8], [9, 4, 4]])
b = np.sort(a, axis=1) #satır sorting islemi yaptım
print(b)

#argsort fonksiyonu:indisleri sıraya diziyor sortta ise elemanlar sıraya diziliyordu
a = np.array([23, 4, 5, 17])
b = np.argsort(a)
print(a)
print(b)

#argsort yardımıyla sorting yapma örneği:
a = np.array([23, 4, 5, 17])
b = np.argsort(a)
print(a)
print('-' * 10)
print(a[b])

#argmax fonksiyonu:
a = np.array([23, 4, 50, 17])
b = np.argmax(a)
print(a)
print(b)

#fromiter fonksiyonu:iterable bir nesneyi alıyor nparray elde ediyor np.array fonksiyonu ile  de bu işlem yapılabiliyordu fakat
#np.array fonksiyonuna sadece tuple,list gibi iterable nesneler veriliyor
b = np.fromiter((i * i for i in range(10)), dtype='float32')
print(b)

#asarray fonksiyonu:içeriği aynı kalsın dtype değişsin istersem kullanıyorum typecast işlemi yapmış gibi oluyorum
a = np.array([1, 2.9, 3.2, 4.3], dtype='float32')
b = np.asarray(a, dtype='int32')


import numpy as np
import math

import sys
from pathlib import Path

#############################################
root_path = Path(__file__).parents
sorting_  = str( root_path[1] ) + '/sorting/'

sys.path.append( sorting_ )
from quick_sort import *
#############################################

############################################################################################
# This is a class that collects statistical indexes such as mean, variance and permutations.
############################################################################################

class StatisticalIndexes(object):

	def __init__(self):
		super().__init__()

	def Ranker(self, x:list):

		x_ = quick_sort(x)
		n  = len(x)
		r  = []

		for i in range(n):
			for j in range(n):

				if x[i] == x_[j]:
					r.append(j + 1)
					break

		return r

	def Combination(self, n:int, k:int):

		num = math.factorial(n)
		den = math.factorial(k) * math.factorial(n - k)

		return num / den

	def Power(self, base:float, exp:int):

		'''
		...it would be easier to use the ** Python operator, but it would be less funny !
		'''

		if base == 0:
			return 0
		elif base == 1:
			return 1
		else:
			res = base
			for i in range(2, exp + 1):
				res *= base
			return res

	def Mean(self, x:list=None):

		return sum(x) / len(x)

	def SampleVariance(self, x:list=None):

		s_mean = self.Mean( x )
		num    = sum( [(j - s_mean)**2 for j in x] )
		den    = len(x) - 1

		return num / den

	def Variance(self, x:list=None):

		mean   = self.Mean( x )
		num    = sum( [(j - mean)**2 for j in x] )
		den    = len(x)

		return num / den

	def StandardDeviationSample(self, x:list):

		return self.SampleVariance(x) ** 0.5

	def StandardDeviation(self, x:list):

		return self.Variance(x) ** 0.5

	def Covariance(self, x:list, y:list):

		'''
		Attention: X and Y variables must have same array length.
		'''

		assert len(x) == len(y)

		mean_x = self.Mean(x)
		mean_y = self.Mean(y)

		den = len(x) - 1
		num = sum( [ (x[j] - mean_x) * (y[j] - mean_y) for j in range( den + 1 ) ] )

		return num / den


from numpy import argmax

# Works out multinomial coefficient nC(k1,k2,...,km)
def multiCoeff(n, kList):
	kMaxFirstPos = argmax(kList)
	kMax = kList[kMaxFirstPos]
	coefficient = 1
	i = kMax + 1
	# Perform repeated multiplication and division
	# for part of kList up to the maximum
	for k in kList[:kMaxFirstPos]:
		for j in range(1, k + 1):
			coefficient *= i
			coefficient //= j
			i += 1
	# Perform repeated multiplication and division
	# for part of kList after the maximum
	for k in kList[kMaxFirstPos + 1:]:
		for j in range(1, k + 1):
			coefficient *= i
			coefficient //= j
			i += 1
	return coefficient

# Works out coefficients of multinomial expansion
# given polynomial coefficents and power
def multiExpand(coeffsList, powersList, power):
	l = len(coeffsList)
	if l == 1:
		return [coeffsList[0] ** power], \
			[powersList[0] * power]
	if power == 0:
		return [1], [0]
	if power == 1:
		return coeffsList, powersList
	newCoeffs = []
	newPowers = []
	kList = [power]
	for i in range(l - 1):
		kList.append(0)
	newCoeff, newPower = multiCoeff(power, kList), 0
	for i in range(l):
		newCoeff *= (coeffsList[i] ** kList[i])
		newPower += (powersList[i] * kList[i])
	newCoeffs.append(newCoeff)
	newPowers.append(newPower)
	# kList should now look like [power, 0, ... 0]
	# (e.g. [5, 0, 0] for power = 5, l = 3)
	# Used to work out multinomial coefficients
	while kList[l-1] != power:
		# Update kList
		if kList[l-2] != 0:
			kList[l-2] -= 1
			kList[l-1] += 1
		else:
			i = l - 2
			while kList[i] == 0:
				i -= 1
			kList[i] -= 1
			kList[i+1] = kList[l-1] + 1
			kList[l-1] = 0
		# Update newCoeff, newPower
		newCoeff, newPower = \
			multiCoeff(power, kList), 0
		for i in range(l):
			newCoeff *= (coeffsList[i] ** kList[i])
			newPower += (powersList[i] * kList[i])
		ind = newPowers.index(newPower) \
			if newPower in newPowers else -1
		if ind > -1:
			newCoeffs[ind] += newCoeff
		else:
			newCoeffs.append(newCoeff)
			newPowers.append(newPower)
	# Remove terms with zero coefficients
	i = 0
	while i < len(newCoeffs):
		if newCoeffs[i] == 0:
			newCoeffs.pop(i)
			newPowers.pop(i)
		else:
			i += 1
	return newCoeffs, newPowers

# Works out Qsigma - Q^p
def QsMinQp(a, b, QpC, QpP, p):
	QpC.pop(0), QpP.pop(0)	   # First elements cancel
	minQpC = [-el for el in QpC]	# Take -Q^p coeffs
	# Merge middle term of Qsigma into QsigmaMinQp
	ind = QpP.index(p) if p in QpP else -1
	if ind > -1:
		minQpC[ind] += (a ** p)
	else:
		ind = 0
		while ind < len(QpP) and QpP[ind] > p:
			ind += 1
		if ind == len(QpP):
			minQpC.extend((a ** p, p))
		else:
			minQpC.insert(ind, a ** p)
			QpP.insert(ind, p)
	# Deal with last element of Qsigma
	ind = -1 if 0 in QpP else -2
	if ind > -2:
		minQpC[ind] += (b ** p)
	else:
		minQpC.extend((b ** p, 0))
	# Pop first terms if zero
	while minQpC[0] and minQpC[0] == 0:
		minQpC.pop(0), QpP.pop(0)
	# Pop last terms if zero
	while minQpC[-1] and QpP[-1] == 0:
		minQpC.pop(), QpP.pop()
	return minQpC, QpP

# Works out reduction of x^m in terms of powers of tau
# Coefficients containing powers of x capped at x^2
def xPowerExpand(a, b, power, rDict):
	p, r, tauCoeffs = power // 3, power % 3, []
	# No need to recalculate if this monomial
	# is already in the dictionary - just return
	if power in rDict:
		return rDict[power]
	# "Base case" of recursion
	elif p == 0:
		new = [0 for i in range(0, r)]
		new.append(1)
		tauCoeffs.append(new)
	else:
		# May be higher powers of x before reduction
		for i in range(p + 1):
			new = [0 for j in range(r)]
			tauCoeffs.append(new)
			for j in range(i + 1):
				# Coefficient of x^j on tau^(i-power)
				coeff = multiCoeff(p, [p-i, j, i-j]) \
					* ((-a)**j) * ((-b)**(i-j))
				tauCoeffs[i].append(coeff)
		# Working from coefficients on powers of tau
		# closest to zero (higher indices)
		i = p
		while len(tauCoeffs[i]) > 3 and i >= 0:
			# Working from coeffs on power of tau
			# that are higher powers of x
			j = i + r
			while j > 2:
				# Reduce this power of x by recursion
				nCoeffs = xPowerExpand(a, b, j, rDict)
				# Merge smaller expansion into main
				# expansion to progress reduction
				for k in range(len(nCoeffs)):
					for l in range(len(nCoeffs[k])):
						ind = i + k + 1 - len(nCoeffs)
						if len(tauCoeffs[ind]) > l:
							tauCoeffs[ind][l] += \
								tauCoeffs[i][j] * \
								nCoeffs[k][l]
						else:
							tauCoeffs[ind].append \
								(tauCoeffs[i][j] * \
								nCoeffs[k][l])
				# Remove high power of x from coeff
				# of tau term once reduction is done
				tauCoeffs[i].pop(j)
				j -= 1
			i -= 1
	rDict[power] = tauCoeffs
	return tauCoeffs

# Works out coefficients of polynomial in
# tau = y^-2 after reduction (x^3 = y^2 - ax - b)
def reducePoly(a, b, cList, pList, p, rDict):
	# Rewrite given polynomial in terms of tau
	# Reduce each power of x in turn and
	# combine into a single list of coefficients
	# Use first monomial's expansion as a base
	expn = xPowerExpand(a, b, pList[0], rDict)
	expnStr = expStr(expn, 0)
	print(f"Reduction of x^{pList[0]} = {expnStr}")
	# Make sure the correct multiple is used
	factor = cList[0]
	for i in range(0, len(expn)):
		expn[i] = [factor * el for el in expn[i]]
	# Then add multiples of other lists
	for i in range(1, len(pList)):
		factor = cList[i]
		tmp = xPowerExpand(a, b, pList[i], rDict)
		tmpStr = expStr(tmp, 0)
		print(f"Reduction of x^{pList[i]} = {tmpStr}")
		for j in range(len(tmp)):
			for k in range(len(tmp[j])):
				expn[j + len(expn) - len(tmp)][k] += \
					(tmp[j][k] * factor)
	return expn

def rXY(a, b, p):
	# reductionDict is a dictionary for tracking
	# monomials whose reductions are already computed
	reductionDict = {}
	# Arrays to represent the polynomial Q(x)
	coeffs, powers = [1, a, b], [3, 1, 0]
	# Step 1: find Qsigma - Q^p
	QpC, QpP = multiExpand(coeffs, powers, p)
	QsMinQpC, QsMinQpP = QsMinQp(a, b, QpC, QpP, p)
	QsMinQpStr = polyStr(QsMinQpC, QsMinQpP)
	print(f"(Qsigma - Q^p) = {QsMinQpStr}\n")
	# Step 2: rewrite Qsigma - Q^p in terms of tau
	expansion = reducePoly(a, b, \
		QsMinQpC, QsMinQpP, p, reductionDict)
	return expansion

# Coefficients on Q(x) = x^3 + ax + b; prime = p
a = int(input("Please enter a value for (a) in " + \
	"x^3 + ax + b. "))
b = int(input("Please also enter a value for (b). "))
p = int(input("Finally, please enter a prime p " + \
	"to determine the field. "))

# Set up arrays to represent Q as a polynomial
coeffs, powers = [1, a, b], [3, 1, 0]
print("\nThe polynomial entered was " + \
	f"{polyStr(coeffs, powers)}.\n")

# Output results
rXY_str = expStr(rXY(a, b, p), p)
print(f"\nR(x,y) = {rXY_str}")


# -*- coding: utf-8 -*-
"""
Created on Wed Mar 23 01:12:47 2016
@author: Wang Ziang
"""

"""
This function solves a system of first-order differential equations
(or A first-order differential equation, of course) of the form:

-----------------------------------------------

                X'[t] = F[t,X]

-----------------------------------------------

    where  X = (x1,x2,...,xn)T
    F:  X -> Y
        (x1,x2,...,xn)T -> (y1,y2,...,yn)T
        where:
        y1 = f1(t,x1,x2,...,xn),
        y1 = f1(t,x1,x2,...,xn),
        .
        .
        yn = f1(t,x1,x2,...,xn)
"""

'''
in the following method:
xi could be a scalar, or a vector
    if xi is a vector,
    it should be given by the data structure "array" defined in numpy package
    e.g. numpy.array([x1,x2,x3])
the f is the RHS-function in X'[t] == f(X,t)
    f: ti,xi -> xii
    f should be defined in the form
        define f(ti, Xi):
            return a vector with the same size as Xi
        or given by a equivalent lambda expression
with fixed step size h, and a given f,
the followingMethod is a iterative function
    followingMethod: state ->state
                     (ti,xi) -> (tii,xii)
'''
import numpy as np
array = np.array
from nest import nestList
from nest import nestUntilList
import math

"""
the constructor and selector of a State:(t,X)
"""
def makeState(t, x): return [t,x]
def getX(state): return state[1]
def getT(state): return state[0]

"""
four methed in Runge-Kutta's family
"""

def forwardEulerMethod(state, f, h):
    ti = getT(state)
    xi = getX(state)
    tii = ti + h  # tii = Subscibe[t,i+1]
    xii = xi + h * f(ti,xi)  # xii = Subscibe[x,i+1]
    return makeState(tii,xii)

def explicitTrapezoidalMethod(state, f, h):
    ti = getT(state)
    xi = getX(state)
    tii = ti + h
    xii = xi + h/2.0 *(f(ti,xi) + f(ti+h, xi+h*f(ti,xi)))
    return makeState(tii,xii)

def midPointMethod(state, f, h):
    ti = getT(state)
    xi = getX(state)
    h = float(h)
    tii = ti + h
    xii = xi + h * f(ti+h/2, xi+h/2*f(ti,xi))
    return makeState(tii,xii)

def rk4Method(state, f, h):
    '''
    Classic fourth-order Runge-Kutta method
    '''
    ti = getT(state)
    xi = getX(state)
    h = float(h)
    tii = ti + h
    s1 = f(ti,xi)
    s2 = f(ti+h/2, xi+h/2*s1)
    s3 = f(ti+h/2, xi+h/2*s2)
    s4 = f(ti+h, xi+h*s3)
    xii = xi + h/6 * (s1 + 2*s2 + 2*s3 +s4)
    return makeState(tii,xii)

__methodDictionary={\
    "ForwardEuler":forwardEulerMethod,\
    "ExplicitTrapezoidal":explicitTrapezoidalMethod,\
    "Midpoint":midPointMethod,\
    "RK4":rk4Method\
}

'''-------------------------------------------------------------------------------'''

def normalize(initX):
    """
    normalize X,
    if array([X__]) ,-> array([X])
    if X_Number ,-> array([X])
    used in dSolve

    """
    if type(initX) == np.ndarray:
        return initX
    else:
        return array([initX])

def dSolve(func, x0, tMax, stepSize, method = "RK4"):
    """
    dSolve solves a system of first-order differential equations
    (or A first-order differential equation, of course) of the form:

    -----------------------------------------------

                    X'[t] = F[t,X]

    -----------------------------------------------
    (

        where  X = (x1,x2,...,xn)T
        F:  X -> Y
            (x1,x2,...,xn)T -> (y1,y2,...,yn)T
            where:
            y1 = f1(t,x1,x2,...,xn),
            y1 = f1(t,x1,x2,...,xn),
            .
            .
            yn = f1(t,x1,x2,...,xn)
    )

    with initial value X[0] =x0,
    (
        !!! if xi is a vector,
        it should be given by the data structure "array" defined in numpy package
        e.g. numpy.array([x1,x2,x3])
    )

    from t = 0 to t = tMax,

    with fixed step size = stepSize,

    using method selected from __methodDictionary

    Output: [state0, state1,... ]
        where statei has the form [ti, xi] for a ODE,
            or [ti, array([x1i,x2i...])] for s system of ODEs

    """

    x0 = normalize(x0)
    times = int(math.ceil(float(tMax) / stepSize))
    methodFunc = __methodDictionary[method]
    toNext = lambda state: methodFunc(state, func, stepSize)
    return nestList(toNext, makeState(0, x0), times)

def dSolveUntil(func, x0, stepSize, stopQ, method = "RK4", tMax = 1000):
    '''
    stopQ is about a state
    '''
    x0 = normalize(x0)
    maxStep = int(math.ceil(float(tMax) / stepSize))
    methodFunc = __methodDictionary[method]
    toNext = lambda state: methodFunc(state, func, stepSize)
    return nestUntilList(toNext, makeState(0, x0), stopQ, maxStep)

'''-------------------------------------------------------------------------------'''

def getColumn(table,i):
    "get the ith column of the table\
    without security "
    return [elem[i] for elem in table]

def getTList(result):
    "get the time list T of the result given by function dSolve\
    without security "
    return getColumn(result, 0)

def getXiList(result, i = 1):
    "get the list of i-th component Xi of the result given by function dSolve\
    without security "

    return getColumn( getColumn(result,1), i-1)
    '''
    if type(result[0][1]) == np.ndarray:
        return getColumn( getColumn(result,2), i-1)
        #return [elem[1][i-1] for elem in result]
    else:
        return getColumn(result,2)
        #return [elem[1] for elem in result]
    '''
def reduceStateList(result, includeList):
    def reduceState(state):
       return makeState(getT(state), array([getX(state)[i] for i in includeList]))
    return map(reduceState, result)

'''-------------------------------------------------------------------------------'''


# Import the necessary modules
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
import pboc_utils as pboc

# In this script, we will look at the distribution of polymer ell_range under
# some different types of regulation. We will first think of some cytoskeletal
# polymer growing in an unlimited pool of available monomers (tubulin dimers)
# and the case in which the shortening of the polymer is dependent on the total
# length.

# To begin, we'll define a set of parameters. We'll assume that the production
# rate and the depoymerization rate are independent of length. Our master
# equation becomes
#
#  P(l, t + dt) = P(l, t) + r * dt * P(l-1, t) + y * dt * P(l+1, t) - r * dt * P(l, t + dt) - gamma * dt * P(l, t + dt)
#
# where l is the length of the polymer, t is the time, dt is the timestep, r
# is the growth rate of the polymer, and y is the depolymerization rate.
r = 5  # addition rate in monomers per time
gamma = 6  # Degradation rate in monomers per time
dt = 1 / 50 # Time step for spreading.
tot_length = 100  # Maximum polymer length
tot_time = 1500   # Time to integrate in time steps

# Now we are ready to perform integration as we typically do.

# Set up the two-dimensional matrix of probabilities.
prob = np.zeros((tot_length + 1, tot_time))

# Set our initial condition.
prob[0, 0] = 1.0

# Start the integration.
for t in range(1, tot_time):
    for ell in range(tot_length):
        # Evaluate the master equation.
        prob[ell, t] = prob[ell, t-1] - r * dt * prob[ell, t-1] +\
            gamma * dt * prob[ell+1, t-1]

        # Now we can work out the boundary condition when ell is 0.
        if ell > 0:
            prob[ell, t] = prob[ell, t] + r * dt * prob[ell-1, t-1] - \
                            gamma * dt * prob[ell, t-1]

# Now we'll show the plot in three dimensions.
pboc.bar3(prob, xlabel='time (steps)', ylabel='polymer length in monomers',
          zlabel='P(l, t)', bin_step=50)
plt.show()

# We can see that we are approaching steady state and the distribution appears
# to be exponential. What is the mean?  From our master equation, we can reach
# an equation that describes the mean polymer length at steady state,
#
#   <ell> = (r/y)  / (1 - (r/y))
#
# We can compute the mean length of our polymers and compare it to the
# prediction.
ell_range = np.arange(0, tot_length + 1, 1)
ell_probabilities = ell_range * prob[:, -1]
mean_length = np.sum(ell_probabilities)
predicted_length = (r / gamma) / (1 - (r/gamma))
print('Our predicted length is ' + str(predicted_length) + '. Our computed mean length is ' + str(mean_length) + '.')

# We are actually pretty close, but not exactly correct. This could be due to
# numerical precision or maybe we aren't quite at setady state yet. However,
# this is pretty close. Let's now see how well the entire distribution agrees
# with our numerical integration. Remember, the entire distribution can be
# described by
#
#  P(l) = (r / y)**l * (1 - (r / y)).
#
predicted_distribution = (r / gamma)**ell_range * (1 - (r / gamma))
plt.figure()
plt.bar(ell_range, prob[:,-1])
plt.plot(ell_range, predicted_distribution, 'r-')
plt.xlabel('length in momonmers')
plt.ylabel('$P(l)$')
plt.show()

# That agrees quite well!

# Now, let's look at the case in which gamma is dependent on the length of the
# polymer. In essence, this means that we now can redfine gamma as some
# constant times the length of the polymer.
gamma_ell = 1/2
r = 1

# We'll reset our probability matrix and redo the integration.
prob = np.zeros((tot_length + 1, tot_time))

# We'll set our initial condition this time with long filaments.
prob[70, 0] = 1.0

# Now we just integrate.
for t in range(1, tot_time):
    for ell in range(tot_length):
        # Evaluate the master equation.
        prob[ell, t] = prob[ell, t-1] - r * dt * prob[ell, t-1] +\
            gamma_ell * (ell + 1) * dt * prob[ell+1, t-1]

        # Now we can work out the boundary condition when ell is 0.
        if ell > 0:
            prob[ell, t] = prob[ell, t] + r * dt * prob[ell-1, t-1] - \
                            gamma_ell * ell * dt * prob[ell, t-1]

# Now we'll show the plot in three dimensions.
pboc.bar3(prob, xlabel='time (steps)', ylabel='polymer length in monomers',
          zlabel='P(l, t)', bin_step=50)
plt.show()

# We can see that it now decreases in length to teh steady state value. As
# we did before, we can compare our computed mean length to the predicted
# value. For this model, the predicted mean length is
#
# <ell> = (r / gamma_ell)
#
length_prob = ell_range * prob[:, -1]
mean_length = np.sum(length_prob)
predicted_mean =  r / gamma_ell
print('Our predicted length is ' + str(predicted_mean) + '. Our computed length is ' + str(mean_length) + '.')

# Again, that agrees quite well. We can solve for the steady state distribution
# of polymer ell_range as
#
# P(l) = (r / y')^l * (1 / l!) * e^(-(r/y'))
#
predicted_dist = []
for ell in ell_range:
    prediction = (r / gamma_ell)**ell * (1 / np.math.factorial(ell)) *\
                  np.exp(-r/gamma_ell)
    predicted_dist.append(prediction)

# Let's plot it.
plt.figure()
plt.bar(ell_range, prob[:, -1])
plt.plot(ell_range, predicted_dist, 'r-')
plt.xlabel('polymer length in monomers')
plt.ylabel('$P(\ell)$')
plt.show()

# Again, it looks like we've nailed it.


import numpy as np
import matplotlib.pyplot as plt
prop_cycle = plt.rcParams['axes.prop_cycle']
colors = prop_cycle.by_key()['color']

class ConsumerClass():

    def __init__(self,**kwargs):

        # a. baseline setup

        # utility
        self.utility = 'cobb_douglas'
        self.alpha = 1
        self.beta = 1

        # prices and income
        self.p1 = 1
        self.p2 = 1
        self.I = 5

        # figures
        self.x1label = '$x_1$'
        self.x2label = '$x_2$'
        self.N = 100

        # b. update
        for key,val in kwargs.items():
            setattr(self,key,val)

        # c. calculations
        self.calculations()

    def calculations(self):

        # a. utility function
        if self.utility == 'cobb_douglas':
            self.u = lambda x1,x2,alpha,beta: x1**alpha*x2**beta
            self.indiff_func_x1 = lambda u0,x1,alpha,beta: (u0/x1**alpha)**(1/beta)
            self.indiff_func_x2 = lambda u0,x2,alpha,beta: (u0/x2**beta)**(1/alpha)
        elif self.utility == 'quasi_log':
            self.u = lambda x1,x2,alpha,beta: alpha*np.log(x1)+x2*beta
            self.indiff_func_x1 = lambda u0,x1,alpha,beta: (u0-alpha*np.log(x1))/beta
            self.indiff_func_x2 = lambda u0,x2,alpha,beta: np.exp((u0-x2*beta)/alpha)
        elif self.utility == 'concave':
            self.u = lambda x1,x2,alpha,beta: alpha*x1**2+beta*x2**2
            self.indiff_func_x1 = lambda u0,x1,alpha,beta: np.sqrt((u0-alpha*x1**2)/beta)
            self.indiff_func_x2 = lambda u0,x2,alpha,beta: np.sqrt((u0-beta*x2**2)/alpha)
        else:
            raise ValueError('unknown utility function')

        # b. figures
        self.x1max = 1.2*self.I/self.p1
        self.x2max = 1.2*self.I/self.p2

    def figure(self):

        fig = plt.figure(frameon=False,figsize=(6,6),dpi=100)
        ax = fig.add_subplot(1,1,1)

        ax.set_xlim([0,self.x1max])
        ax.set_ylim([0,self.x2max])
        ax.set_xlabel(self.x1label)
        ax.set_ylabel(self.x2label)

        fig.tight_layout()

        return fig,ax

    def legend(self,ax,**kwargs):

        kwargs.setdefault('loc','upper right')
        kwargs.setdefault('frameon',True)

        legend = ax.legend(**kwargs)
        frame = legend.get_frame()
        frame.set_facecolor('white')

    ##########
    # choice #
    ##########

    def maximize_utility(self,**kwargs):

        # a. update
        for key,val in kwargs.items():
            setattr(self,key,val)

        # b. solve
        if self.utility == 'cobb_douglas':
            self.x1_ast = self.alpha/(self.alpha+self.beta)*self.I/self.p1
            self.x2_ast = self.alpha/(self.alpha+self.beta)*self.I/self.p2
        elif self.utility == 'quasi_log':
            self.x1_ast = (self.alpha*self.p2)/(self.beta*self.p1)
            self.x2_ast = self.I-self.p1*self.x1_ast
            if self.x2_ast < 0:
                self.x1_ast = self.I/self.p1
                self.x2_ast = 0
        elif self.utility == 'concave':
            self.x1_ast = self.I/self.p1
            self.x2_ast = self.I/self.p2
            if self.x1_ast**self.alpha < self.x2_ast**self.beta:
                self.x1_ast = 0
            else:
                self.x2_ast = 0
        else:
            raise ValueError('unknown utility function')

        # c. utility
        self.u_ast = self.u(self.x1_ast,self.x2_ast,self.alpha,self.beta)

        # d. return
        return np.array([self.x1_ast,self.x2_ast,self.u_ast])

    def plot_max(self,ax,**kwargs):

        kwargs.setdefault('ls','')
        kwargs.setdefault('marker','*')
        kwargs.setdefault('markersize',7)
        kwargs.setdefault('color','black')
        kwargs.setdefault('label',f'$u({self.x1_ast:.2f},{self.x2_ast:.2f}) = {self.u_ast:.2f}$')

        ax.plot(self.x1_ast,self.x2_ast,**kwargs)

    def minimize_cost(self,u,**kwargs):

        # a. update
        for key,val in kwargs.items():
            setattr(self,key,val)

        # b. solve
        if self.utility == 'cobb_douglas':
            self.h1_ast = (u*(self.p1/self.p2)**(-self.beta))**(1/(self.alpha+self.beta))
            self.h2_ast = (u*(self.p2/self.p1)**(-self.alpha))**(1/(self.alpha+self.beta))
        elif self.utility == 'quasi_log':
            self.h1_ast = (self.alpha*self.p2)/(self.beta*self.p1)
            self.h2_ast = (u-np.log(self.h1_ast))/self.beta
        else:
            raise ValueError('unknown utility function')

        # c. cost
        self.h_cost = self.p1*self.h1_ast+self.p2*self.h2_ast

        # d. return
        return np.array([self.h1_ast,self.h2_ast,self.h_cost])

    #############
    # budgetset #
    #############

    def plot_budgetline(self,ax,**kwargs):

        kwargs.setdefault('lw',2)

        x = [0,self.I/self.p1]
        y = [self.I/self.p2,0]

        ax.plot(x,y,**kwargs)

    #######################
    # indifference curves #
    #######################

    def find_indifference_curve(self,u0=None):

        if u0 == None:
            u0 = self.u_ast

        # a. fix x1
        x1_x1 = np.linspace(0,self.x1max,self.N)
        with np.errstate(divide='ignore',invalid='ignore'):
            x2_x1 = self.indiff_func_x1(u0,x1_x1,self.alpha,self.beta)

        # b. fix x2
        x2_x2 = np.linspace(0,self.x2max,self.N)
        with np.errstate(divide='ignore',invalid='ignore'):
            x1_x2 = self.indiff_func_x2(u0,x2_x2,self.alpha,self.beta)

        # c. combine
        x1 = np.hstack([x1_x1,x1_x2])
        x2 = np.hstack([x2_x1,x2_x2])

        # d. sort
        I = np.argsort(x1)
        x1 = x1[I]
        x2 = x2[I]

        return x1,x2

    def plot_indifference_curves(self,ax,u0s=[],do_label=True,**kwargs):

        # a. find utility levels
        if len(u0s ) == 0:
            self.maximize_utility()
            u0s = [self.u_ast,
                   self.u(0.8*self.x1_ast,0.8*self.x2_ast,self.alpha,self.beta),
                   self.u(1.2*self.x1_ast,1.2*self.x2_ast,self.alpha,self.beta)]

        # b. construct and plot indifference curves
        for u0 in u0s:

            x1,x2 = self.find_indifference_curve(u0)

            if do_label:
                ax.plot(x1,x2,label=f'$u = {u0:.2f}$',**kwargs)
            else:
                ax.plot(x1,x2,**kwargs)

    def plot_monotonicity_check(self,ax,x1=None,x2=None,text='north-east'):

        # a. value
        if x1 == None:
            x1 = self.x1_ast
        if x2 == None:
            x2 = self.x2_ast

        # b. plot
        polygon = plt.Polygon([[x1, x2],[self.x1max, x2],[self.x1max, self.x2max],[x1, self.x2max],[x1, x2]],
            color='black',alpha=0.10)
        ax.add_patch(polygon)
        ax.text(x1*1.05,0.95*self.x2max,text)

    def plot_convexity_check(self,ax,u=None,text='mix'):

        # a. value
        if u == None:
            u = self.u_ast

        # b. indifference curve
        x1, x2 = self.find_indifference_curve(u)

        # c. select
        I = (np.isnan(x1) == False) & (np.isnan(x2) == False)
        J = (x1[I] > 0) & (x2[I] > 0) & (x1[I] < self.x1max) & (x2[I] < self.x2max)

        x1 = x1[I][J]
        x2 = x2[I][J]
        N = x1.size
        i_low = np.int(N/3)
        i_high = np.int(N*2/3)

        x1_low = x1[i_low]
        x2_low = x2[i_low]

        x1_high = x1[i_high]
        x2_high = x2[i_high]

        # d. plot
        ax.plot([x1_low,x1_high],[x2_low,x2_high],ls='--',marker='o',
            markerfacecolor='none',color='black')

        x1 = 0.05*x1_low + 0.95*x1_high
        x2 = 0.05*x2_low + 0.95*x2_high
        ax.text(x1,x2*1.05,text)

    ###########
    # slutsky #
    ###########

    def plot_slutsky_endogenous(self,ax,p1_old,p1_new,p2,e1,e2,steps=4):

        I_old = p1_old*e1 + p2*e2
        I_new = p1_new*e1 + p2*e2

        # a. calculations
        A = self.maximize_utility(p1=p1_old,p2=p2,I=I_old)
        x1,x2,u_max = A

        C1 = self.maximize_utility(p1=p1_new,p2=p2,I=I_old)
        x1_fixI,x2_fixI,u_max_fixI = C1

        B = self.minimize_cost(u_max,p1=p1_new,p2=p2)
        h1, h2, h_cost = B

        C2 = self.maximize_utility(p1=p1_new,p2=p2,I=I_new)
        x1_new,x2_new,u_max_new = C2

        # b. plots
        self.p1,self.p2,self.I = p1_old,p2,I_old
        self.plot_budgetline(ax,ls='-',label='original',color=colors[0])
        if steps > 1:
            self.p1,self.p2,self.I = p1_new,p2,I_new
            self.plot_budgetline(ax,ls='-',label='final',color=colors[1])
        if steps > 2:
            self.p1,self.p2,self.I = p1_new,p2,h_cost
            self.plot_budgetline(ax,ls='-',alpha=0.50,label='compensated',color=colors[2])
        if steps > 3:
            self.p1,self.p2,self.I = p1_new,p2,I_old
            self.plot_budgetline(ax,ls='-',label='constant income',color=colors[3])
        ax.plot(e1,e2,'ro',color='black')
        ax.text(e1*1.03,e2*1.03,'$E$')

        # A
        ax.plot(x1,x2,'ro',color='black')
        ax.text(x1*1.03,x2*1.03,'$A$')
        self.plot_indifference_curves(ax,[u_max],do_label=False,ls='--',color=colors[0])

        # B
        if steps > 2:
            ax.plot(h1,h2,'ro',color='black')
            ax.text(h1*1.03,h2*1.03,'$B$')

        # C2
        if steps > 1:
            ax.plot(x1_new,x2_new,'ro',color='black')
            ax.text(x1_new*1.03,x2_new*1.03,'$C_2$')
            self.plot_indifference_curves(ax,[u_max_new],do_label=False,ls='--',color=colors[1])

        # C1
        if steps > 3:
            ax.plot(x1_fixI,x2_fixI,'ro',color='black')
            ax.text(x1_fixI*1.03,x2_fixI*1.03,f'$C_1$')
            self.plot_indifference_curves(ax,[u_max_fixI],do_label=False,ls='--',color=colors[3])

        if steps > 3:
            line = f'subtitution: $B-A$ = ({h1-x1:5.2f},{h2-x2:5.2f})\n'
            line += f'income: $C_1-B$ = ({x1_fixI-h1:5.2f},{x2_fixI-h2:5.2f})\n'
            line += f'wealth: $C_2-C_1$ = ({x1_new-x1_fixI:5.2f},{x2_new-x2_fixI:5.2f})'
            ax.text(0.55*self.x1max,0.87*self.x2max,line)


import numpy

def rand_walk(init_val, final_high, final_low, prob, N):

    nleft = 0
    nright = 0
    csum = 0

    left_end = final_low
    right_end = final_high
    p = prob
    n = N

    for i in range(0, n):

        run_val = init_val
        while(1):

            if((run_val == right_end) or (run_val == left_end)):
                break;

            u = numpy.random.uniform(0, 1)
            if(u < p):
                run_val = run_val + 1
            else:
                run_val = run_val - 1

        csum = csum + run_val

        if(run_val == right_end):
            nright = nright + 1
        else:
            nleft = nleft + 1

    num = float(n)
    nright = float(nright)
    a = nright / num

    nleft = float(nleft)
    b = nleft / num

    csum = float(csum)
    c = csum / num
    return a, b, c

[a, b, c] = rand_walk(100, 150, 0, 0.5, 10000)
print "TEST CASE 1 >> (answer for 6(a)) ", a, "\t(answer for 6(b)) ", b, "\t(answer for 6(c)) ", c
[a, b, c] = rand_walk(100, 200, 0, 0.52, 10000)
print "TEST CASE 2 >> (answer for 6(a)) ", a, "\t(answer for 6(b)) ", b, "\t(answer for 6(c)) ", c
[a, b, c] = rand_walk(200, 250, 0, 0.54, 10000)
print "TEST CASE 3 >> (answer for 6(a)) ", a, "\t(answer for 6(b)) ", b, "\t(answer for 6(c)) ", c


# Para visualizar gráficos en la terminal interactiva
from IPython import get_ipython
get_ipython().run_line_magic('matplotlib', 'ipympl')

"""
# Actividad en clases: Diseño de filtros FIR

Objetivos:

- Aprender a utilizar scipy.signal.firwin para diseñar filtros FIR
- Filtrar señales de audio con filtros FIR

"""

import numpy as np
import scipy.fft
import scipy.signal
import matplotlib.pylab as plt
from IPython.display import display
import librosa
from utils import Audio

def listen_signal(signal: np.ndarray, Fs: int=44100, normalize:bool=True) -> None:
    if normalize:
        signal = signal.copy()/np.amax(np.absolute(signal))
    display(Audio(signal, rate=Fs))

"""

Sea la siguiente señal:

"""

data, Fs = librosa.load("nomekop.ogg", sr=8000)

Nw = 512
freq, ttime, Sxx = scipy.signal.spectrogram(data, fs=Fs, window='hamming', nperseg=Nw, noverlap=Nw//1.5)

fig, ax = plt.subplots(1, figsize=(8, 3), tight_layout=True)
ax.pcolormesh(ttime, freq, np.log10(Sxx+1e-5), shading='auto', cmap=plt.cm.magma);
ax.set_xlabel('Tiempo [s]')
ax.set_ylabel('Frecuencia [Hz]')
listen_signal(data, Fs=Fs)

"""

El siguiente código de ejemplo busca atenuar el primer personaje preservando lo más posible del segundo personaje

- Estudien atentamente la rutina y el resultado
- ¿Puedes mejorar el resultado modificando las frecuencias de corte?

"""

h = scipy.signal.firwin(numtaps=1000+1, cutoff=[50, 700, 1900, 2100], pass_zero=False, window='hamming', fs=Fs)
freq_response, response = scipy.signal.freqz(h, fs=Fs)
data_filt = scipy.signal.convolve(data, h)
freq, ttime, Sxx = scipy.signal.spectrogram(data_filt, fs=Fs, window='hamming', nperseg=Nw, noverlap=Nw//1.5)

# Resultados:
fig, ax = plt.subplots(1, 2, figsize=(7, 2), tight_layout=True)
ax[0].plot(h);
ax[1].semilogy(freq_response, np.abs(response))
fig, ax = plt.subplots(1, figsize=(8, 3), tight_layout=True)
ax.pcolormesh(ttime, freq, np.log10(Sxx+1e-5), shading='auto', cmap=plt.cm.magma);
ax.set_xlabel('Tiempo [s]')
ax.set_ylabel('Frecuencia [Hz]')
listen_signal(data_filt, Fs=Fs)

"""

- Modifique L y el tipo de ventana (truncamiento). Discuta sobre como cambia el resultado en términos visuales y acústicos
- Diseñe y pruebe un filtro FIR que atenue al segundo personaje y preserve al primer personaje

"""


# This script trains a MLP to approximate x^2
# Model source: https://sonnet.dev/
# Number of trainable source:
# https://stackoverflow.com/questions/38160940/how-to-count-total-number-of-trainable-parameters-in-a-tensorflow-model
import numpy as np
import sonnet as snt
import tensorflow as tf
import matplotlib.pyplot as plt

# %% Parameters.
# All parameters to be tweaked.
mlp_dimensions = [5, 5, 1]
num_training_iterations = 1000
learning_rate = 1e-3
range_tr = (-1, 1)
range_plt = (-2, 2)
iteration_print = 100

# Parameters depending on the tweaked ones.
x_plot = np.linspace(*range_plt, num=100)

# %% Data generator.
def get_data(low, high):
    x_np = np.random.uniform(low=low, high=high, size=(1, 1))
    y_np = np.square(x_np)
    x_tf = tf.convert_to_tensor(x_np, dtype=tf.float32)
    y_tf = tf.convert_to_tensor(y_np, dtype=tf.float32)
    return x_tf, y_tf

# %% Restart Training.
# Model.
model = snt.nets.MLP(mlp_dimensions)

# Optimizer.
optimizer = snt.optimizers.Adam(learning_rate)

# Starting values for training.
last_iteration = 0
loss_list = []

# %% Training (can be run multiple times for added num_training_iterations iterations).
gradients = None
for iteration in range(last_iteration, last_iteration + num_training_iterations):
    last_iteration = iteration
    x, y = get_data(*range_tr)
    with tf.GradientTape() as tape:
        y_pred = model(x)
        loss = tf.compat.v1.losses.mean_squared_error(y, y_pred)
    gradients = tape.gradient(loss, model.trainable_variables)
    loss_list.append(loss.numpy())
    if (iteration + 1) % iteration_print == 0:
        optimizer.apply(gradients, model.trainable_variables)
        print(f"Iteration: {iteration + 1}, Mean Loss: {np.mean(loss_list[-iteration_print:-1])}")

#%% Plot results.
num_trainable_variables = np.sum([np.prod(v.shape) for v in model.trainable_variables])
y_plot = np.array([model(tf.convert_to_tensor([[x_i]], dtype=tf.float32)).numpy().squeeze() for x_i in x_plot])
plt.plot(x_plot, np.square(x_plot), "k-")  # Ground truth
plt.plot(x_plot, y_plot, "k--")            # Model predicted
plt.legend(("Ground truth", "MLP predicted"), loc='upper center')
plt.title(rf"MLP tries to predict $x^2$ with {num_trainable_variables} trainable variables")
plt.show()

#%% Plot loss
start = 100
end = len(loss_list)
plt.plot(range(start, end), loss_list[start:end])
plt.ylabel("Loss (MSE)")
plt.xlabel("Training examples")
plt.show()

# %% Test
x, y = get_data(*range_tr)
y_pred = model(x)
print("x={:.2f}, y={:.3f}, y_pred={:.3f}, diff={:.3f}".format(*(i.numpy().squeeze() for i in (x, y, y_pred, y-y_pred))))


import numpy as np

from mvdr.linalg_utils import svd_wrapper

def get_principal_angles(A, B, n_components=None, is_ortho=True, deg=False):
    """
    Returns the vector of principal angles between two subspaces.

    Parameters
    ----------
    A: array-like, (n, p)
        Orthonormal basis for the first subspace.

    B: array-like, (n, q)
        Orthonormal basis for the second subspace.

    n_components: None, int
        Only compute the first few principal angles.

    is_ortho: bool
        Are A, B orthonormal? If they are not, we will orthonormalized them with a QR decomposition.

    deg: bool
        Return the vector of principal angles in degrees. If False, will use radians.

    Output
    ------
    theta: array-like, (n_components, )
        The principal angles in radians.
    """
    A = np.array(A)
    B = np.array(B)

    A_is_vec = False
    B_is_vec = False
    if A.ndim == 1:
        A = A.reshape(-1, 1)
        A_is_vec = True

    if B.ndim == 1:
        B = B.reshape(-1, 1)
        B_is_vec = True

    if not is_ortho:
        if A_is_vec:
            A /= np.linalg.norm(A)
        else:
            A = np.linalg.qr(A)[0]

        if A_is_vec:
            A /= np.linalg.norm(A)
        else:
            B = np.linalg.qr(B)[0]

    if A_is_vec and B_is_vec:
        cos_theta = A.T @ B
        cos_theta = cos_theta.reshape(-1)
    else:
        _, cos_theta, __ = svd_wrapper(A.T @ B, rank=n_components)

    cos_theta = np.clip(cos_theta, a_min=-1, a_max=1)
    theta = np.arccos(cos_theta)

    if deg:
        theta = np.rad2deg(theta)

    return theta

def subspace_dist(A, B, method='proj_f', is_ortho=True):
    """
    Computes  distance between two subspaces.

    See section 4.3 of (Edelman et al, 1998) for definitions
    http://www.cnbc.cmu.edu/cns/papers/edelman98.pdf

    Parameters
    ----------
    A, B: array-like, (n, p)
        Bases of the two subspace

    method: str
        Must be one of ['arc', 'fubini',
                        'proj_2', 'proj_f',
                        'chordal_2', 'chordal_f']

    is_ortho: bool
        Are A, B orthonormal? If they are not, we will orthonormalized them with a QR decomposition.
    """
    # TODO: check for edge cases e.g. n = 1 etc

    method = method.lower()
    if method not in ['arc_length', 'fubini',
                      'proj_2', 'proj_f',
                      'chordal_2', 'chordal_f']:
        raise ValueError(" method = {} is not currently an option".
                         format(method))

    A = np.array(A)
    B = np.array(B)

    if A.ndim == 1:
        A = A.reshape(-1, 1)

    if B.ndim == 1:
        B = B.reshape(-1, 1)

    if not is_ortho:
        A = np.linalg.qr(A)[0]
        B = np.linalg.qr(B)[0]

    if method in ['arc_length', 'fubini', 'chordal_2', 'chordal_f']:
        theta = get_principal_angles(A, B)

    if method == 'arc_length':
        return np.linalg.norm(theta)

    elif method == 'fubini':
        cos_theta = np.cos(theta)
        return np.arccos(np.product(cos_theta))

    elif method == 'chordal_2':
        return max(2 * np.sin(.5 * theta))

    elif method == 'chordal_f':
        return np.linalg.norm(2 * np.sin(.5 * theta))

    if method == 'proj_f':
        diff = A @ A.T - B @ B.T
        return np.sqrt(.5) * np.linalg.norm(diff, ord='fro')

    elif method == 'proj_2':
        diff = A @ A.T - B @ B.T
        return np.linalg.norm(diff, ord=2)