# # Finding eigenvalues of matrices with neural networks.
# Script for finding the eigenvectors corresponding to the largest eigenvalue of a matrix with a neural network.

import tensorflow.compat.v1 as tf
tf.disable_v2_behavior()
tf.reset_default_graph()
# tf.set_random_seed(343)

# import tensorflow as tf
import numpy as np
from matplotlib import cm
from matplotlib import pyplot as plt
from mpl_toolkits.mplot3d import axes3d
#from lib import compute_dx_dt

matrix_size = 6

A = np.random.random_sample(size=(matrix_size,matrix_size))
A = (A.T + A)/2.0
start_matrix = A

eigen_vals, eigen_vecs =  np.linalg.eig(A)

A = tf.convert_to_tensor(A)
print("A = ", A)

x_0 = tf.convert_to_tensor(np.random.random_sample(size = (1,matrix_size)))
print("x0 = ", x_0)

## The construction phase

num_iter = 10000
num_hidden_neurons = [50]
num_hidden_layers = np.size(num_hidden_neurons)

with tf.variable_scope('dnn'):

    previous_layer = x_0

    for l in range(num_hidden_layers):
        current_layer = tf.layers.dense(previous_layer, num_hidden_neurons[l],activation=tf.nn.sigmoid)
        previous_layer = current_layer

    dnn_output = tf.layers.dense(previous_layer, matrix_size)

with tf.name_scope('loss'):
    print("dnn_output = ", dnn_output)

    x_trial = tf.transpose(dnn_output)
    print("x_trial = ", x_trial)

    temp1 = (tf.tensordot(tf.transpose(x_trial), x_trial, axes=1)*A)
    temp2 = (1- tf.tensordot(tf.transpose(x_trial), tf.tensordot(A, x_trial, axes=1), axes=1))*np.eye(matrix_size)
    func = tf.tensordot((temp1-temp2), x_trial, axes=1)

    print(temp1)
    print(temp2)
    print(func)

    func = tf.transpose(func)
    x_trial = tf.transpose(x_trial)

    loss = tf.losses.mean_squared_error(func, x_trial)

learning_rate = 0.001

with tf.name_scope('train'):
    optimizer = tf.train.GradientDescentOptimizer(learning_rate)
    traning_op = optimizer.minimize(loss)

init = tf.global_variables_initializer()

g_dnn = None

losses = []

with tf.Session() as sess:
    init.run()
    for i in range(num_iter):
        sess.run(traning_op)

        if i % 100 == 0:
            l = loss.eval()
            print("Step:", i, "/",num_iter, "loss: ", l)
            losses.append(l)

    x_dnn = x_trial.eval()
x_dnn = x_dnn.T

# ## Plotting loss over time

plt.plot(losses[:5])
plt.xlabel("Iteration")
plt.ylabel("Loss")

print("Eigenvector NN = \n", (x_dnn/(x_dnn**2).sum()**0.5), "\n")

eigen_val_nn = x_dnn.T @ (start_matrix @ x_dnn) / (x_dnn.T @ x_dnn)

print("Eigenvalue NN = \n", eigen_val_nn, "\n \n")
print("Eigenvector analytic = \n", eigen_vecs)
print("\n")
print("Eigenvalues analytic = \n",eigen_vals)


import numpy as np
import pandas as pd
from numpy import sign

def bisect_(f, a, b, i_max, tol):
    """
    :param f: function
    :param a: beginning of the interval
    :param b: end of the interval
    :param i_max: maximal number of iterations
    :param tol: tolerance
    :return: (approximation of the root, function value of root approximation, DataFrame with data from each iteration)
    """

    if i_max < 1:
        raise ValueError("i_max must be greater than 0")

    # starting values at endpoints of interval
    ya = f(a)
    yb = f(b)

    # check if function has proper parameters
    if ya * yb > 0:
        raise ValueError("Function doesn't have different signs at beginning and end of interval")

    df = pd.DataFrame(columns=["Step", "a", "b", "c", "yc", "err"])

    for i in range(i_max):
        c = (a + b) / 2
        yc = f(c)
        err = (b - a) / 2

        df = df.append(pd.Series(data={"Step": i + 1, "a": a, "b": b, "c": c, "yc": yc, "err": err}),
                       ignore_index=True)

        # check for convergence
        if abs(yc) < tol:
            print(f"Bisect method convergent after {i+1} steps")
            break

        if sign(ya) == sign(yc):
            a = c
            ya = yc
        else:
            b = c
            yb = yc

    else:
        raise StopIteration("Function couldn't find approximation of root with given tolerance")

    df = df.astype({"Step": "int64"})
    df = df.set_index("Step")

    return c, yc, df

def rfalsi(f, a, b, i_max, tol):
    """
    :param f: function
    :param a: beginning of the interval
    :param b: end of the interval
    :param i_max: maximal number of iterations
    :param tol: tolerance
    :return: (approximation of the root, function value of root approximation, DataFrame with data from each iteration)
    """

    if i_max < 1:
        raise ValueError("i_max must be greater than 0")

    # starting values at endpoints of interval
    ya = f(a)
    yb = f(b)

    # check if function has proper parameters
    if ya * yb > 0:
        raise ValueError("Function doesn't have different signs at beginning and end of interval")

    df = pd.DataFrame(columns=["Step", "a", "b", "c", "yc"])

    for i in range(i_max):
        c = b - ((b - a) / (yb - ya)) * yb
        yc = f(c)

        df = df.append(pd.Series(data={"Step": i + 1, "a": a, "b": b, "c": c, "yc": yc}),
                       ignore_index=True)

        # Sprawdzenie zbieżności
        if abs(yc) < tol:
            print(f"Regula falsi method convergent after {i+1} steps")
            break

        if sign(ya) == sign(yc):
            a = c
            ya = yc
        else:
            b = c
            yb = yc

    else:
        raise StopIteration("Function couldn't find approximation of root with given tolerance")

    df = df.astype({"Step": "int64"})
    df = df.set_index("Step")

    return c, yc, df

def secant(f, xn0, xn1, i_max, tol):
    """
    :param f: function
    :param xn0: first approximation
    :param xn1: second approximation
    :param i_max: maximal number of iterations
    :param tol: tolerance
    :return: (approximation of the root, function value of root approximation, list of results in each step)
    """

    if i_max < 1:
        raise ValueError("i_max must be greater than 0")

    # y values of first two approximations
    yn0 = f(xn0)
    yn1 = f(xn1)

    df = pd.DataFrame(columns=["Step", "x_n", "x_n+1", "x_n+2", "f(x_n+2)"])

    for i in range(i_max):
        xn2 = xn1 - ((xn1 - xn0) / (yn1 - yn0)) * yn1

        yn2 = f(xn2)

        df = df.append(pd.Series(data={"Step": i + 1, "x_n": xn0, "x_n+1": xn1, "x_n+2": xn2, "f(x_n+2)": yn2}),
                                 ignore_index=True)

        # check for convergence
        if abs(yn2) < tol:
            print(f"Secant method convergent after {i+1} steps")
            break

        xn0, yn0 = xn1, yn1
        xn1, yn1 = xn2, yn2

    else:
        raise StopIteration("Function couldn't find approximation of root with given tolerance")

    df = df.astype({"Step": "int64"})
    df = df.set_index("Step")

    return xn2, yn2, df

def brent(f, a, b, tol):
    """
    :param f: function
    :param a: beginning of the interval
    :param b: end of the interval
    :param tol: tolerance
    :return: (approximation of the root, function value of root approximation, number of iterations)
    """
    eps = np.finfo(float).eps

    fa = f(a)
    fb = f(b)

    c, fc = a, fa
    d = e = b - a

    i = 0
    while(True):
        i += 1
        if (abs(fc) < abs(fb)):
            # swap
            b, c = c, b
            fb, fc = fc, fb

        tol = 2 * abs(b) * eps + tol
        m = 0.5 * (c - b)

        if abs(m) > tol and fb != 0:
            if abs(e) < tol or abs(fa) <= abs(fb):
                d = e = m
            else:
                s = fb / fa
                if (a == c):
                    p = 2 * m * s
                    q = 1 - s
                else:
                    q = fa / fc
                    r = fb / fc
                    p = s * (2 * m * q * (q - r) - (b - a) * (r - 1))
                    q = (q - 1) * (r - 1) * (s - 1)
                if (p > 0):
                    q = -q
                else:
                    p = -p

                s = e
                e = d

                if (2 * p < 3 * m * q - abs(tol * q) and p < abs(0.5 * s * q)):
                    d = p / q
                else:
                    d = e = m

            a = b
            fa = fb
            if (abs(d) > tol):
                dtol = d
            elif (m > 0):
                dtol = tol
            else:
                dtol = -tol

            b = b + dtol

            fb = f(b)
            print(f"a={a}, b={b}, c={c}")
            if (np.sign(fb) == np.sign(fc)):
                c, fc = a, fa
                d = e = b - a
        else:
            break
    return b, f(b), i

def newton(f, dfun, xn0, i_max, tol):
    """
    :param f: function
    :param df: derivative of a function f
    :param xn0: first approximation
    :param i_max: maximal number of iterations
    :param tol: tolerance
    :return: (approximation of the root, function value of root approximation, number of iterations)
    """

    if i_max < 1:
        raise ValueError("i_max must be greater than 0")

    df = pd.DataFrame(columns=["Step", "x_n", "x_n+1", "f(x_n+1)"])

    for i in range(i_max):
        xn1 = xn0 - f(xn0) / dfun(xn0)
        yn1 = f(xn1)

        df = df.append(pd.Series(data={"Step": i + 1, "x_n": xn0, "x_n+1": xn1, "f(x_n+1)": yn1}),
                                 ignore_index=True)

        if abs(yn1) < tol:
            print(f"Newton's method convergent after {i+1} steps")
            break

        xn0 = xn1
    else:
        raise StopIteration("Function couldn't find approximation of root with given tolerance")

    df = df.astype({"Step": "int64"})
    df = df.set_index("Step")

    return xn1, yn1, df

def newton_nles(fun: np.array, jacobian: np.array, X,  i_max, tol):
    """
    :param fun: system of X equations
    :param jacobian: Jacobian
    :param X: table of X first approximations
    :param i_max: maximal number of iterations
    :param tol: tolerance
    :return: (approximations of the roots, function values of roots aproximatons, list of results in each step)
    """

    if i_max < 1:
        raise ValueError("i_max must be greater than 0")

    df = pd.DataFrame(columns=["Step", "X_n", "dX_n", "f(X_n)"])

    for i in range(i_max):
        J = jacobian(*X)
        Y = fun(*X)
        dX = np.linalg.solve(J, Y)
        X -= dX

        df = df.append(pd.Series(data={"Step": i + 1, "X_n": X, "dX_n": dX, "f(X_n)": Y}),
                                 ignore_index=True)

        if np.linalg.norm(dX) < tol:
            print(f"Newton's method convergent after {i+1} steps")
            break

    else:
        raise StopIteration("Function couldn't find approximation of solution with given tolerance")

    df = df.astype({"Step": "int64"})
    df = df.set_index("Step")

    return X, df

def banach_nles(g: np.array, Xn0,  i_max, tol):
    """
    :param g: mapping function
    :param Xn0: table of first approximations
    :param i_max: maximal number of iterations
    :param tol: tolerance
    :return: (approximations of the roots, list of results in each step)
    """

    if i_max < 1:
        raise ValueError("i_max must be greater than 0")

    df = pd.DataFrame(columns=["Step", "X_n"])

    for i in range(i_max):
        Xn1 = g(*Xn0)

        row = pd.Series(data={"Step": i + 1, "X_n": Xn1})
        df = df.append(row, ignore_index=True)

        if np.linalg.norm(Xn1-Xn0) / np.linalg.norm(Xn1) < tol:
            print(f"Banach's method convergent after {i+1} steps")
            break

        Xn0 = Xn1

    else:
        raise StopIteration("Function couldn't find approximation of solution with given tolerance")

    df = df.astype({"Step": "uint32"})
    df = df.set_index("Step")

    return Xn1, df


# -*- coding: utf-8 -*-
"""
SYS-611 Queuing Markov Model Example

@author: Paul T. Grogan, pgrogan@stevens.edu
"""

# import the python3 behavior for importing, division, and printing in python2
from __future__ import absolute_import, division, print_function

# import the numpy package and refer to it is `np`
import numpy as np
# import the matplotlib pyplot package and refer to it is `plt`
import matplotlib.pyplot as plt

_lambda = 1/1.5 # arrival rate, 1.5 minutes per customer or 2/3 customer per minute
_mu = 1/0.75 # service rate, 0.75 minutes per customer or 4/3 customer per minute

# define process generator for inter-arrival duration
def gen_t_arrival():
    r = np.random.rand()
    return -np.log(1-r)/_lambda
    # note: could also use
    # return np.random.exponential(scale=1/_lambda)

# define process generator for service duration
def gen_t_service():
    r = np.random.rand()
    return -np.log(1-r)/_mu
    # note: could also use
    # return np.random.exponential(scale=1/_mu)

# define the number of events
NUM_EVENTS = 1000
# create lists to store variables of interest
t = np.zeros(NUM_EVENTS+1) # time
q = np.zeros(NUM_EVENTS+1) # number of customers in system
t_arrival = np.zeros(NUM_EVENTS) # sampled inter-arrival durations
t_service = np.zeros(NUM_EVENTS) # sampled service durations
delta_t = np.zeros(NUM_EVENTS) # duration of each event

# initialize time and state variables
t[0] = 0
q[0] = 0

for i in range(NUM_EVENTS):
    # generate samples for inter-arrival and service durations
    t_arrival[i] = gen_t_arrival()
    t_service[i] = gen_t_service()
    # process state transitions / updates for q and t
    if q[i] == 0 or t_arrival[i] < t_service[i]:
        # if no customers in queue or arrival happens before service
        # event is an arrival
        delta_t[i] = t_arrival[i]
        q[i+1] = q[i] + 1
    else:
        # otherwise event is a service
        delta_t[i] = t_service[i]
        q[i+1] = q[i] - 1
    t[i+1] = t[i] + delta_t[i]

print('{:>10s}{:>10s}{:>10s}{:>10s}{:>10s}{:>10s}{:>10s}'.format(
        'i', 't(i)', 'q(i)', 't_arrival', 't_service', 'delta_t', 'q(i+1)'))
for i in range(NUM_EVENTS):
    print('{:10.0f}{:10.2f}{:10.0f}{:10.2f}{:10.2f}{:10.2f}{:10.0f}'.format(
            i, t[i], q[i], t_arrival[i], t_service[i], delta_t[i], q[i+1]))

plt.figure()
plt.xlabel('Time, $t$')
plt.ylabel('Customers in System, $q$')
plt.step(t,q,'-r',where='post')


import numpy as np
import copy

def activation(values, func='', derivative=False):
    """Primary function to handle neuron activations.

    Parameters
    ----------
    values : :obj:`float`, :obj:`int`, :obj:`ndarray`
        Inputted value.
    func : str
        Keyword conveying type of activation function to use.
    derivative : bool
        Whether or not to use derivative form of activation function.

    Returns
    -------
    float
        If `val` is `float` or `int`.
    :obj:`ndarray:
        If `val` is :obj:`ndarray`.

    Raises
    ------
    KeyError
        If `func` input is unsupported.

    """
    val = copy.deepcopy(values)
    val[val < 1e-8] = 0
    if func == 'sigmoid':
        if derivative:
            return (np.exp(-val)) / ((1 + np.exp(-val)) ** 2)
        else:
            return 1 / (1 + np.exp(-val))

    elif func == 'relu':
        if derivative:
            val[val <= 0] = 0
            val[val > 0] = 1
        else:
            val[val < 0] = 0
        return val

    elif func == 'softmax':
        # d is meant for numerical stability, keeps vals in val close
        # to zero.
        d = -np.max(val)
        exp_sum = np.sum(np.exp(val + d), axis=0)
        softmax = np.exp(val + d) / exp_sum
        if derivative:
            jacobian = softmax @ -softmax.reshape(1, -1)
            diag = softmax - softmax**2
            np.fill_diagonal(jacobian, diag)
            return jacobian
        else:
            return softmax

    else:
        raise KeyError(f"Unrecognized activation function: {func}")

def loss(prediction, truth, loss_type=''):

    true = np.argmax(truth)

    if loss_type == 'mean-squared':
        return prediction - truth

    elif loss_type == 'cross-entropy':
        return -truth / prediction


from sympy import *
import sys

print '1D analysis:'
x, t, A, B, w, b, c, k, q = symbols('x t A B w b c k q')
u = (A*cos(w*t) + B*sin(w*t))*exp(-c*t)*cos(k*x)

# Constrain B from the initial condition u_t=0
u_t = diff(u, t)
u_t = simplify(u_t.subs(t, 0))
print 'u_t(x,0)=0:', u_t
# B = A*c/w
u = u.subs(B, A*c/w)
print 'u:', u

R = diff(u, t, t) + b*diff(u, t) - q*diff(u, x, x)
R = simplify(R)
terms = collect(R, [cos(w*t), sin(w*t)])
print 'factored terms:', terms
print 'latex terms:', latex(terms, mode='plain')
#  A*((-c**2*w + k**2*q*w - w**3)*cos(t*w) + (-b*c**2 - b*w**2 + c**3 + c*k**2*q + c*w**2)*sin(t*w))*exp(-c*t)*cos(k*x)/w
cos_eq = -c**2*w + k**2*q*w - w**3
sin_eq = -b*c**2 - b*w**2 + c**3 + c*k**2*q + c*w**2
print 'cos_eq has solution', solve(cos_eq, w)
# w = sqrt(k**2*q - c**2), assume c < k*sqrt(q)
sin_eq = sin_eq.subs(w, sqrt(k**2*q - c**2))
sin_eq = simplify(sin_eq)
print 'sin_eq:', sin_eq
print 'sin_eq has solution', solve(sin_eq, c)
# c = b/2

print '2D analysis:'
y, kx, ky = symbols('y kx ky')
u = (A*cos(w*t) + B*sin(w*t))*exp(-c*t)*cos(kx*x)*cos(ky*y)

# Constrain B from the initial condition u_t=0
u_t = diff(u, t)
u_t = simplify(u_t.subs(t, 0))
print 'u_t(x,0)=0:', u_t
# B = A*c/w
u = u.subs(B, A*c/w)
print 'u:', u

R = diff(u, t, t) + b*diff(u, t) - q*diff(u, x, x) - q*diff(u, y, y)
R = simplify(R)
terms = collect(R, [cos(w*t), sin(w*t)])
print 'factored terms 2D:', terms
print 'latex terms 2D:', latex(terms, mode='plain')
#  A*((-c**2*w + kx**2*q*w + ky**2*q*w - w**3)*cos(t*w) + (-b*c**2 - b*w**2 + c**3 + c*kx**2*q + c*ky**2*q + c*w**2)*sin(t*w))*exp(-c*t)*cos(kx*x)*cos(ky*y)/w
cos_eq = -c**2*w + kx**2*q*w + ky**2*q*w - w**3
sin_eq = -b*c**2 - b*w**2 + c**3 + c*kx**2*q + c*ky**2*q + c*w**2
print 'cos_eq has solution', solve(cos_eq, w)
# w = sqrt(k**2*q - c**2), assume c < k*sqrt(q)
sin_eq = sin_eq.subs(w, sqrt(kx**2*q + ky**2*q - c**2))
sin_eq = simplify(sin_eq)
print 'sin_eq:', sin_eq
print 'sin_eq, b and c terms:', collect(sin_eq, [b, c])
print 'sin_eq has solution', solve(sin_eq, c)


"""
Trajectory Planning Module

Point to Point Cubic
Point to Point Quintic
Point to Point Linear with Blend
"""
import numpy as np

def cubic_trajectory_planning(q0, qf, qd0, qdf, m = 100):
    """
    Point to Point Cubic Trajectory Planning

    ...

    Parameters
    ---
    q0  : Initial Position (Dof x 1)
    qf  : Final Position (Dof x 1)
    qd0 : Initial Velocity (Dof x 1)
    qf0 : Final Velocity (Dof x 1)
    m(Optional) : Discrete Time Steps

    Returns
    ---
    q, qd, qdd : Position, Velocity and Acceleration (Dof x m)
    """
    n = q0.shape[0]

    # Polynomial Parameters
    a0 = np.copy(q0)
    a1 = np.copy(qd0)
    a2 = 3 * (qf - q0) - 2 * qd0 - qdf
    a3 = -2 * (qf - q0) + qd0 + qdf

    timesteps = np.linspace(0, 1, num = m)

    q = np.zeros((n, m))
    qd = np.zeros((n, m))
    qdd = np.zeros((n, m))

    for i in range(len(timesteps)):
        t = timesteps[i]
        t_2 = t * t
        t_3 = t * t * t

        q[:, i] = (a0) + (a1 * t) + (a2 * t_2) + (a3 * t_3)
        qd[:, i] = (a1) + (2 * a2 * t) + (3 * a3 * t_2)
        qdd[:, i] = (2 * a2) + (6 * a3 * t)

    return q, qd, qdd

def quintic_trajectory_planning(q0, qf, qd0, qdf, qdd0, qddf, m = 100):
    """
    Point to Point Quintic Trajectory Planning

    ...

    Parameters
    ---
    q0  : Initial Position (Dof x 1)
    qf  : Final Position (Dof x 1)
    qd0 : Initial Velocity (Dof x 1)
    qdf : Final Velocity (Dof x 1)
    qdd0 : Initial Acceleration (Dof x 1)
    qddf : Final Acceleration (Dof x 1)
    m(Optional) : Discrete Time Steps

    Returns
    ---
    q, qd, qdd : Position, Velocity and Acceleration (Dof x m)
    """
    n = q0.shape[0]

    # Polynomial Parameters
    a0 = np.copy(q0)
    a1 = np.copy(qd0)
    a2 = np.copy(qdd0) / 2
    a3 = (20 * qf - 20 * q0 - 8 * qdf - 12 * qd0 - 3 * qdd0 + qddf) / 2
    a4 = (30 * q0 - 30 * qf + 14 * qdf + 16 * qd0 + 3 * qdd0 - 2 * qddf) / 2
    a5 = (12 * qf - 12 * q0 - 6 * qdf - 6 * qd0 - qdd0 + qddf) / 2

    timesteps = np.linspace(0, 1, num = m)

    q = np.zeros((n, m))
    qd = np.zeros((n, m))
    qdd = np.zeros((n, m))

    for i in range(len(timesteps)):
        t = timesteps[i]
        t_2 = t * t
        t_3 = t_2 * t
        t_4 = t_3 * t
        t_5 = t_4 * t

        q[:, i] = (a0) + (a1 * t) + (a2 * t_2) + (a3 * t_3) + (a4 * t_4) + (a5 * t_5)
        qd[:, i] = (a1) + (2 * a2 * t) + (3 * a3 * t_2) + (4 * a4 * t_3) + (5 * a5 * t_4)
        qdd[:, i] = (2 * a2) + (6 * a3 * t) + (12 * a4 * t_2) + (20 * a5 * t_3)

    return q, qd, qdd


##%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
##                          Process Monitoring of Etch data
## %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

#%% import required packages
import numpy as np
import matplotlib.pyplot as plt
from sklearn.mixture import GaussianMixture

#%% fetch data
import scipy.io

matlab_data = scipy.io.loadmat('MACHINE_Data.mat', struct_as_record = False)
Etch_data = matlab_data['LAMDATA']
calibration_dataAll = Etch_data[0,0].calibration # calibration_dataAll[i,0] corresponds to a 2D data from ith batch where columns correspond to different variables

variable_names = Etch_data[0,0].variables

##%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
##                          perform Multiway PCA
## %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

#%% generate unfolded data matrix
n_vars = variable_names.size - 2 # first 2 columns are not process variables
n_samples = 85 # following the work of He et al.

unfolded_dataMatrix = np.empty((1,n_vars*n_samples))
for expt in range(calibration_dataAll.size):
    calibration_expt = calibration_dataAll[expt,0][5:90,2:] # removing first 5 measurements as done in He et al.

    if calibration_expt.shape[0] < 85:
        continue

    unfolded_row = np.ravel(calibration_expt, order='F')[np.newaxis,:]
    unfolded_dataMatrix = np.vstack((unfolded_dataMatrix, unfolded_row))

unfolded_dataMatrix = unfolded_dataMatrix[1:,:]

#%% scale data
from sklearn.preprocessing import StandardScaler

scaler = StandardScaler()
data_train_normal = scaler.fit_transform(unfolded_dataMatrix)

#%% fit PCA model to calibration data
from sklearn.decomposition import PCA

pca = PCA(n_components = 3) # following the work of He et al.
score_train = pca.fit_transform(data_train_normal)

#%% visualize in 2D
plt.figure()
plt.scatter(score_train[:,0],score_train[:,1])
plt.xlabel('PC1 scores')
plt.ylabel('PC2 scores')

##%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
##                          GMM on PCA scores
## %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
#%% finding # of components via BIC
BICs = []
lowestBIC = np.inf
for n_cluster in range(1, 10):
    gmm = GaussianMixture(n_components = n_cluster, random_state = 100)
    gmm.fit(score_train)
    BIC = gmm.bic(score_train)
    BICs.append(BIC)

    if BIC < lowestBIC:
        optimal_n_cluster = n_cluster
        lowestBIC = BIC

plt.figure()
plt.plot(range(1,10), BICs, marker='o')
plt.xlabel('Number of components')
plt.ylabel('BIC')
plt.show()

#%% fit GMM model to metal-etch data
gmm = GaussianMixture(n_components = optimal_n_cluster, random_state = 100)
cluster_label = gmm.fit_predict(score_train)

plt.figure()
plt.scatter(score_train[:, 0], score_train[:, 1], c = cluster_label, s=20, cmap='viridis')
plt.xlabel('PC1 scores')
plt.ylabel('PC2 scores')

cluster_centers = gmm.means_
cluster_plot_labels = ['Cluster ' + str(i+1) for i in range(optimal_n_cluster)]
for i in range(optimal_n_cluster):
    plt.scatter(cluster_centers[i, 0], cluster_centers[i, 1], c='red', s=20, marker = '*', alpha=0.5)
    plt.annotate(cluster_plot_labels[i], (cluster_centers[i,0], cluster_centers[i,1]))

##%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
##                          Fault detection metric for training data
## %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

#%% global mahalonobis distance metric
Dglobal_train = np.zeros((score_train.shape[0],))

for i in range(score_train.shape[0]):
    x = score_train[i,:,np.newaxis]
    probs = gmm.predict_proba(x.T)

    for component in range(3):
        Dlocal =  np.dot(np.dot((x-gmm.means_[component,:,np.newaxis]).T,np.linalg.inv(gmm.covariances_[component,:])),(x-gmm.means_[component,:,np.newaxis]))
        Dglobal_train[i] =  Dglobal_train[i] + probs[0,component]*Dlocal

#%% Dglobal control limit
N = score_train.shape[0]
r = 3

alpha = 0.05 # 95% control limit
Dglobal_CL = r*(N**2-1)*scipy.stats.f.ppf(1-alpha,r,N-r)/(N*(N-r))

#%% Dglobal plot with CL
plt.figure()
plt.plot(Dglobal_train)
plt.plot([1,len(Dglobal_train)],[Dglobal_CL, Dglobal_CL], color='red')
plt.xlabel('Sample #')
plt.ylabel('D_global for training data')
plt.show()

##%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
##                          test data
## %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

#%% fetch test data and unfold
test_dataAll = Etch_data[0,0].test

unfolded_TestdataMatrix = np.empty((1,n_vars*n_samples))
for expt in range(test_dataAll.size):
    test_expt = test_dataAll[expt,0][5:90,2:]

    if test_expt.shape[0] < 85:
        continue

    unfolded_row = np.ravel(test_expt, order='F')[np.newaxis,:]
    unfolded_TestdataMatrix = np.vstack((unfolded_TestdataMatrix, unfolded_row))

unfolded_TestdataMatrix = unfolded_TestdataMatrix[1:,:]

#%% PCA on fault data
data_test_normal = scaler.transform(unfolded_TestdataMatrix)
score_test = pca.transform(data_test_normal)

#%% visualize in 2D (both test and calibration data)
plt.figure()
plt.scatter(score_train[:,0],score_train[:,1], c='blue', alpha=0.1)
plt.scatter(score_test[:,0],score_test[:,1], c='red', marker = '*')
plt.xlabel('PC1 scores')
plt.ylabel('PC2 scores')

#%% compute Dglobal_test
Dglobal_test = np.zeros((score_test.shape[0],))

for i in range(score_test.shape[0]):
    x = score_test[i,:,np.newaxis]
    probs = gmm.predict_proba(x.T)

    for component in range(3):
        Dlocal =  np.dot(np.dot((x-gmm.means_[component,:,np.newaxis]).T,np.linalg.inv(gmm.covariances_[component,:])),(x-gmm.means_[component,:,np.newaxis]))
        Dglobal_test[i] =  Dglobal_test[i] + probs[0,component]*Dlocal

#%% Dglobal plot with CL
plt.figure()
plt.plot(Dglobal_test, marker = '*')
plt.plot([1,len(Dglobal_test)],[Dglobal_CL,Dglobal_CL], color='red')
plt.xlabel('Sample #')
plt.ylabel('D_global for test data')
plt.show()

print('Number of faults identified: ', np.sum(Dglobal_test > Dglobal_CL), ' out of ', len(Dglobal_test))


import numpy as np

def normal_eqn(X, Y):
    a = X.T.dot(X)
    theta = np.linalg.pinv(a).dot(X.T).dot(Y)
    return theta


##!/usr/bin/env python3
##Реализация процедур, составляющих протокол подписи Эль-Гамаля с сокращённой длинной параметров.
import Crypto.Util.number as num
from Crypto import Random
from Crypto.Util import number
from sympy.ntheory import factorint
import random
import sympy
import math
import hashlib
import sys
from hashlib import sha256

def MillerRabin(n):
        if n!=int(n):
            return False
        n=int(n)
        #Miller-Rabin test for prime
        if n==0 or n==1 or n==4 or n==6 or n==8 or n==9:
            return False
        if n==2 or n==3 or n==5 or n==7:
            return True
        s = 0
        d = n-1
        while d%2==0:
            d>>=1
            s+=1
        assert(2**s * d == n-1)

        def trial_composite(a):
            if pow(a, d, n) == 1:
                return False
            for i in range(s):
                if pow(a, 2**i * d, n) == n-1:
                    return False
            return True
        for i in range(8):#number of trials
            a = random.randrange(2, n)
            if trial_composite(a):
                return False
        return True
def mod(a,p):                 #приведение по модулю
    mod = (a % p + p) % p
    return mod

def fast_pow(a, w, n):
    s = 1
    v = w
    c = a
    while not v == 0:
        if mod(v,2) == 1:
            s = mod(s*c,n)
            v = (v-1)//2
            c = mod(c**2,n)
        else:
            v = v//2
            c = mod(c*c,n)
    return s
def getPrimeFactors(n):
    prime_factor=[]
    while n%2==0:
        prime_factor.append(2)
        n=n/2
    for i in range(3, int(math.sqrt(n))+1,2):
        while n%i==0:
            prime_factor.append(int(i))
            n=n/i
    if n>2:
        prime_factor.append(int(n))
    return prime_factor

def main_menu():
    while True:
        print("\nLab 2:ELGAMAL DIGITAL SIGNATURE (SIMPLE IMPLEMENTATION)")
        print("1 - Keys generation")
        print("2 - Signing")
        print("3 - Verifying signature")
        print("4 - Test case invalid signature")
        s = int(input('> Enter your choice: '))
        if (s==1):
            print("\nKEYS GENERATION:")
            l_p = int(input("Enter length bits of prime p -> "))
            l_q=int(l_p/12.8)
            q=random.randint((2**l_q)/2,(2**l_q)-1)
            while not MillerRabin(q):
                q=random.randint((2**l_q)/2,(2**l_q)-1)
            i=1
            p=4
            dif =int(l_p-l_q)
            while not MillerRabin(p): #p=2*r*q+1
                p=q*2
                r=random.randint((2**dif)//2,(2**dif)-1) #r- нечетное число
                p=int(p*r)+1
                i+=1
            g=1 #g- число относящееся к q как к показателю
            while g==1:
                aa=random.randint(2,p-1)
                qq=(p-1)//q
                g=fast_pow(aa,qq,p)

            #while 1:
            #    #q= genprimeBits(l)
            #    p=2*q+1 #выбор показатель q, q|p-1
            #    if num.isPrime(p):
            #        break
            #r=getPrimeFactors(q)
            #print(r)
            #while 1:
            #    g=num.getRandomRange(3,p)
            #    safe=1
            #    if pow(g,2,p)==1:
            #        safe==0
            #    if safe and pow(g,q,p)==1:
            #        safe=0
            #    if safe and divmod(p-1,g)[1]==0:
            #        safe=0
            #        # g^(-1) must not divide p-1 because of Khadir's attack
            #    ginv=num.inverse(g,p)
            #    if safe and divmod(p-1,ginv)[1]==0:
            #        safe=0
            #    if safe:
            #        break
            #while(num.GCD(g,p-1)!=1):
            #    g=num.getRandomRange(3,p)

            print("Prime number p ->", p,"\nBinary p ->","(",bin(p),")")
            print("\nGenerator g ->",g)

            x=num.getRandomRange(1,p-1)
            while (num.GCD(x,p-1)!=1):
                x=num.getRandomRange(1,p-1)
            print("\nPrivate key x ->",x)

            y=pow(g,x,p)
            print("\nCalculating y=g^x mod p -> y=",y)
            print("\nCombo public key (p,g,y)->",p,g,y)

        elif (s==2):
            print("\nSIGNING:")
            input_message = input("Message: ")
            inputbytes = str.encode(input_message)
            while 1:
                k=num.getRandomRange(1,p-2)#k-random number in range(1,p-2), HOD(k,p-1)=1
                if (num.GCD(k,p-1)==1):
                    break
            #length of hash=256 bits
            h = hashlib.sha256(input_message.encode('utf-8')).hexdigest()
            mes = int(h,16)
            print("\nHashed message  -> (",mes.bit_length(),"bits)",mes) #(length bits M must < length bits p)
            R=pow(g,k,p)
            print("\nParameter signature R=g^k mod p (",R.bit_length(),"bits) -> ",R)
            t=num.inverse(k,q)
            S=t*(mes-R*x)%(q)
            print("\nParameter signature S=(M-Rx)k^(-1) mod p (",S.bit_length(),"bits)-> ",S)
            print("\nSinging signature (R,S).... -> (",R, S,")")
            print("\nMessage is signed! [M,R,S] -> [",input_message,"(",R,S,")]")
            print("\nSecret parameter k ->",k)

        elif (s==3):
            print("\nVERIFYING A SIGNATURE:")
            if (R>p) or (S>(p-1)) or (num.GCD(g,(p-1))!=1) :
                print ("\nWrong Signature! Invalid parameters R or S")
            D1=pow(g,mes,p) #D1=g^m mod p
            D2=(pow(y,R,p)*pow(R,S,p))%p #D2=y^R*R^S(mod p)
            print ("\nD1=g^m mod p->",D1)
            print("\nD2=y^R*R^Smod p->",D2)
            if (D1==D2):
                print("\nCorrect Signature!")
            else:
                print("\nWrong signature! Invalid parameters R or S")

        elif (s==4):
            print("\n4.TEST CASE INVALID SIGNATURE")
            k1=num.getRandomRange(1,p-1) #k random but is not checked
            print("\nNew k' ->",k1)
            R1=pow(g,k1,p) #R changed because of k changed
            print("\nParameter signature (changed) R'=g^k' mod p (",R1.bit_length(),"bits) -> ",R1)
            #t1=num.inverse(k1,q)
            #S1=t1*(mes-R1*x)%(q)
            #print("\nParameter signature S (changed) S'=(M-R'x)k'^(-1) mod p (",S1.bit_length(),"bits)-> ",S1)
            print("\nSinging signature (R',S).... -> (",R1, S,")")
            print("\nMessage is signed! [M,R',S] -> [",input_message,"(",R1,S,")]")
            D1_d=pow(g,mes,p) #D1=g^m mod p
            D2_d=(pow(y,R1,p)*pow(R1,S,p))%p #D2=y^R*R^S(mod p)
            print ("\nD1'=g^m mod p->",D1_d)
            print("\nD2'=y^R'*R'^S'mod p->",D2_d)
            if (D1_d==D2_d):
                print("\nCorrect Signature!")
            else:
                print("\nWrong signature! Invalid parameters R or S")

main_menu()


import numpy as np
import scipy as sp
import scipy.linalg as linalg
from numpy.linalg import inv

def my_LDA(X, Y):
    """
    Train a LDA classifier from the training set
    X: training data
    Y: class labels of training data

    """

    classLabels = np.unique(Y) # different class labels on the dataset
    classNum = len(classLabels) # number of classes
    datanum, dim = X.shape # dimensions of the dataset
    totalMean = np.mean(X,0) # total mean of the data

    # ====================== YOUR CODE HERE ======================
    # Instructions: Implement the LDA technique, following the
    # steps given in the pseudocode on the assignment.
    # The function should return the projection matrix W,
    # the centroid vector for each class projected to the new
    # space defined by W and the projected data X_lda.

    Mj = np.zeros((classNum, dim))
    Nj = np.zeros((classNum))

    Sw = np.zeros((dim, dim))
    for j_index in range(0,classNum):
        j = classLabels[j_index]
        index_class = np.where(Y == j)[0]
        Nj[j_index] = index_class.shape[0]
        Mj[j_index,:] = np.mean(X[index_class, :],0)
        for i in index_class:
            Sw += np.outer((X[i,:] - Mj[j_index,:]),(X[i,:] - Mj[j_index,:]))

    Sb = np.zeros((dim,dim))
    for j in range(0,classNum):
        Sb += Nj[j] * np.outer((Mj[j,:] - totalMean),(Mj[j,:] - totalMean))

    St = np.zeros((dim,dim))
    for i in range(0, datanum):
        St += np.outer((X[i,:] - totalMean),(X[i,:] - totalMean))

    e_values, U = np.linalg.eig(inv(Sw).dot(Sb))
    index_sort = np.argsort(e_values)[::-1]
    e_values = e_values[index_sort]
    U = U[:, index_sort]

    W = U[:, :classNum-1]
    X_lda = X.dot(W)
    projected_centroid = Mj.dot(W)

    W = np.real(W)
    X_lda = np.real(X_lda)
    projected_centroid = np.real(projected_centroid)

    # =============================================================

    return W, projected_centroid, X_lda


# Q2
# Student number:
import numpy as np
import statsmodels.api as sm
from scipy import stats
import pandas as pd

def ridge(y,X,a):
    """
    function which receives two arrays and a regularization value
    as inputs and implements ridge regression.

    :param y: a response vector with shape (n,1)
    :param X: an array of predictors with shape (n,p)
    :param a: a single number or a list of numbers which represent the regularization strength

    :return: returns a number or a list of numbers if a list has been provided for a
    which are the beta coefficient vectors.
    """
    n = X.shape[0]
    y.shape = n,1 # define shape of y
    x_ones = np.hstack((X, np.ones((X.shape[0], 1), dtype = X.dtype))) # append collumn with intercepts
    transp = x_ones.T # matrix transpose
    try:
        # ridge regression implementation if the a argument is a list
        inverse = (np.linalg.inv(transp @ x_ones + [np.identity(x_ones.shape[1]) * scalar for scalar in a])) # inverse matrix
        B_list = inverse @ transp @ y
        return B_list
    except:
        # implementation of ridge regression if the argument a is a single number
        inverse1 = (np.linalg.inv(transp @ x_ones + np.identity(x_ones.shape[1]) * a )) # inverse matrix
        B = inverse1 @ transp @ y
    return B

def remove_variables(y, X,  threshold , variable_names):
    """
    function which receives a response and outcome array, a  threshold  and a list of variable names
    ,performs regression tests and eliminates a variable with p value higher than the p argument
    after each test untill no variable p value is higher than the threshold p.

    :param y:  1-D array
    :param X: an array of predictors with shape (n,p)
    :param p: a threshold p value
    :param variable_names: a list of column names or None

    :return: returns a the matrix of columns after the non-signifficant ones
    have been eliminated and their names as a tuple.
    """
    # convert input numpy arrays to pandas dataframe for manipulation
    df=pd.DataFrame(data=X[0:,0:],
        index=[i for i in range(X.shape[0])],
        columns=['x' + str(i+1) if variable_names == None else variable_names[i] for i in range(X.shape[1])])
    df2=pd.DataFrame(data=y[0:,0:])
    # add of intercept
    df["intercept"] = 1
    # list with column names for later iteration over those columns
    cols = df.columns.tolist()
    cols = cols[-1:] + cols[:-1]
    df = df[cols]
    # iteration over the pandas dataframes  to implement the linear regression
    for i in range(df.shape[1]):
        model1 = sm.OLS(df2, df).fit()

    # determine the highest p value excluding the one of intercept
        maxPval = max(model1.pvalues[1:], default=0)
        cols = df.columns.tolist()

        # delete the column-predictor with the highest p value and
        # repeating this process until no further predictor's p value is higher than the threshold p.
        if maxPval >= threshold:
            for j in cols:
                if j != 'intercept':
                    if (model1.pvalues[j] == maxPval):
                        #if j != 'intercept':
                        del df[j]
        else:
            break # end of loop

    if len(cols[1:]) == 0:
        return (None, None)
    else:
        # turn pandas dataframe into numpy array and drop the intercept column
        new_x = df.to_numpy()
        new_X = new_x[:,1:] #exclude intercept
        new_variable_names = cols[1:]
        return (new_X, new_variable_names)

import numpy,from scipy,import statsmodels


import numpy as np

#############################################################################################################
# Kalman filters and smoothers
#############################################################################################################

# Kalman Filter - Time invariant model without inputs
# See Shumway & Stoffer (2017), ch. 6.
# https://rdrr.io/cran/astsa/src/R/Kfilter0.R
def myKalmanFilter0(y, # data np.array
                    cQ, cR,
                    mu0 = np.asarray([30,20]), #first estimate
                    Sigma0 = np.array([[1,0],[0,1]]),
                    A = np.asarray([[1,0],[0,1]]), #Observation matrix. We want every state from our state vector.
                    Phi = np.array([[1,1],[0,1]]), #Transition matrix. Displacement is updated with prev disp + curr vel while velocity is updated with prev vel (assuming no acc.)
                    fullOut = 0, # provide denerated quantities as well as predictions/filtering [0 - False; 1 - True]
                   ):
    N, M = y.shape
    Q = cQ @ (cQ.T) # cholesky decomp: cQ=chol(Q), cR=chol(R)
    R = cR @ (cR.T)
    # Initialize
    xp=[] # xp=x_t^{t-1} (prediction)
    Pp=[] # Pp=P_t^{t-1} (prediction covariance)
    xf=[] # xf=x_t^t (filtering)
    Pf=[] # Pf=x_t^t (filtering covariance)
    innov=[] # innovations
    sig=[] # innov var-cov matrix
    negLogLike=0
    for t in range(N):
        if t==0:
            xp.append( Phi @ mu0 ) # Eq. (6.18)
            Pp.append( Phi @ Sigma0 @ (Phi.T) + Q ) # Eq. (6.19)
        else:
            xp.append( Phi @ xf[t-1] ) # Eq. (6.18)
            Pp.append( Phi @ Pf[t-1] @ (Phi.T) + Q) # Eq. (6.19)

        sigtemp = A @ Pp[t] @ (A.T) + R # Eq. (6.24)
        sig.append( (sigtemp.T+sigtemp)/2 ) # innov var - make sure it's symmetric
        siginv = np.linalg.inv( sig[t] )
        K = Pp[t] @ (A.T) @ siginv # Eq. (6.22)
        innov.append( y[t,] - A @ xp[t] ) # Eq. (6.23)
        xf.append( xp[t] + K @ innov[t] ) # Eq. (6.20)
        Pf.append( Pp[t] - K @ A @ Pp[t] )
        negLogLike += np.log(np.linalg.det(sig[t])) + (innov[t].T) @ siginv @ innov[t] # Eq. (6.60)
    negLogLike *= 0.5

    if (fullOut == 1):
        return(xp, Pp, xf, Pf, negLogLike, K)
    else:
        return(xp, Pp, xf, Pf, negLogLike)

# Kalman Filter and Smoother - Time invariant model without inputs
# See Shumway & Stoffer (2017), ch. 6.
# https://rdrr.io/cran/astsa/src/R/Ksmooth0.R
def myKalmanSmoother0(y, # data np.array
                      cQ, cR,
                      mu0 = np.asarray([30,20]), #first estimate
                      Sigma0 = np.array([[1,0],[0,1]]),
                      A = np.asarray([[1,0],[0,1]]), #Observation matrix. We want every state from our state vector.
                      Phi = np.array([[1,1],[0,1]]), #Transition matrix. Displacement is updated with prev disp + curr vel while velocity is updated with prev vel (assuming no acc.)
                      fullOut = 0, # provide denerated quantities as well as predictions/filtering/smoothing
                     ):
    N, M = y.shape
    # Kalman filering
    if (fullOut == 1):
        xp, Pp, xf, Pf, negLogLike, K = myKalmanFilter0(y, cQ, cR, mu0, Sigma0, A, Phi, fullOut) # Call to kalman filter function
    else:
        xp, Pp, xf, Pf, negLogLike = myKalmanFilter0(y, cQ, cR, mu0, Sigma0, A, Phi) # Call to kalman filter function
    # Kalman smoothing
    xs = [None]*N # xs=x_t^n
    Ps = [None]*N # Ps=P_t^n
    J = [None]*N # J=J_t
    for t in range(N-1,0,-1):
        if t==N-1: # last time step, which is the first iteration
            xs[t] = xf[t]
            Ps[t] = Pf[t]
        # Kalman smooting equations
        J[t-1] = Pf[t-1] @ (Phi.T) @ np.linalg.inv(Pp[t]) # Eq. (6.47)
        xs[t-1] = xf[t-1] + J[t-1] @ (xs[t] - xp[t]) # Eq. (6.45)
        Ps[t-1] = Pf[t-1] + J[t-1] @ (Ps[t] - Pp[t]) @ (J[t-1].T) # Eq. (6.46)

    if (fullOut == 1):
        # First time step
        J0 = Sigma0 @ (Phi.T) @ np.linalg.inv(Pp[0])
        x0n = mu0 + J0 @ (xs[0] - xp[0])
        P0n = Sigma0 + J0 @ (Ps[0] - Pp[0]) @ (J0.T)
        # Output
        return(xp, Pp, xf, Pf, xs, Ps, negLogLike, K, J, J0, x0n, P0n)
    else:
        # Output
        return(xp, Pp, xf, Pf, xs, Ps, negLogLike)

#############################################################################################################
# Forward filtering, backward sampling
#############################################################################################################

# Forward filtering, backward samping of state space (displacement, velocity)
# See Shumway & Stoffer (2017), ch. 6. (sec. 6.12; forward filtering backward sampling)
# See also Petris et al (2009), ch. 4. (sec. 4.4; FFBS)
# https://github.com/cran/dlm/blob/master/R/DLM.R
def myFFBS(y, # data np.array
           cQ, cR,
           mu0 = np.asarray([30,20]), #first estimate
           Sigma0 = np.array([[1,0],[0,1]]),
           A = np.asarray([[1,0],[0,1]]), #Observation matrix. We want every state from our state vector.
           Phi = np.array([[1,1],[0,1]]), #Transition matrix. Displacement is updated with prev disp + curr vel while velocity is updated with prev vel (assuming no acc.)
          ):
    N, M = y.shape
    D = mu0.shape[0]
    # Kalman filering
    xp, Pp, xf, Pf, _ = myKalmanFilter0(y, cQ, cR, mu0, Sigma0, A, Phi) # Call to kalman filter function
    # Backward sampling
    xb = [None]*N # xb=x_t^n
    jitter = np.eye(D) * 1e-8
    for t in range(N-1,0,-1):
        if t==N-1: # last time step, which is the first iteration
            xb[t] = xf[t] + np.linalg.cholesky(Pf[t] + jitter) @ np.random.normal(size=(D,1))
        # Backward filtering equations (note that t-1 corr. to t in Eq. 6.218)
        Jt = Pf[t-1] @ (Phi.T) @ np.linalg.inv(Pp[t]) # Eq. (6.47)
        mt = xf[t-1] + Jt @ (xb[t] - xp[t]) # Eq. (6.218 (and 6.45))
        Vt = Pf[t-1] - Jt @ Pp[t] @ (Jt.T) # Eq. (6.218 (and 6.46))
        xb[t-1] = mt + np.linalg.cholesky(Vt + jitter) @ np.random.normal(size=(D,1))

    return(xb)

##############################################################################
# Gibbs sampler for one-parameter base Q-kernel
##############################################################################
def myGibbsQbase(xb, mu0, Phi, Qbase, a0=1, b0=1):
    N, M, _ = xb.shape
    xb_pred = np.concatenate([mu0.T, xb[:-1,:,0]]) # matrix of predictors
    invQbase = np.linalg.inv(Qbase)
    # Sum of squared errors
    errors = (xb[:,:,0] - xb_pred @ Phi.T)
    SStot = np.sum(np.diag((errors[1:,:] @ invQbase.T) @ errors[1:,:].T)) # skip the first prediction (numerical more stable), as m0 is not related to x_1
    # Draw parameter
    para_samp = np.random.gamma( shape=(a0 + (N*M)/2), scale=(1/(b0 + 0.5*SStot)) )

    return(para_samp)

def myGibbsQbase_slow(xb, mu0, Phi, Qbase, a0=1, b0=1):
    N, M, _ = xb.shape
    xb_pred = np.concatenate([mu0.T, xb[:-1,:,0]]) # matrix of predictors
    invQbase = np.linalg.inv(Qbase)
    # Sum of squared errors
    SStot = 0
    for t in range(1,N): # skip the first prediction (numerical more stable), as m0 is not related to x_1
        error_t = xb[t,:,0].T - Phi @ xb_pred[t,:].T
        SStot += (error_t.T) @ invQbase @ error_t
    # Draw parameter
    para_samp = np.random.gamma( shape=(a0 + (N*M)/2), scale=(1/(b0 + 0.5*SStot)) )

    return(para_samp)

# Posterior sampling of states (FFBS) and parameters (Gibbs)
def myPosSamplerQbase(y, # data np.array
                      cQ, cR,
                      mu0 = np.asarray([30,20]), #first estimate
                      Sigma0 = np.array([[1,0],[0,1]]),
                      A = np.asarray([[1,0],[0,1]]), #Observation matrix. We want every state from our state vector.
                      Phi = np.array([[1,1],[0,1]]), #Transition matrix. Displacement is updated with prev disp + curr vel while velocity is updated with prev vel (assuming no acc.)
                      Qbase = np.diag([0., 0., 0.1]),
                      a0=1, b0=1, # hyperprior parameters
                      Nsamp = 1000,
                     ):

    xb_list = []
    para_list = []
    Qbase += np.eye(Qbase.shape[0])*1e-8 # adding a little jitter
    for i in range(Nsamp):
        if (i>0):
            Q = Qbase * (1/para)
            cQ = np.sqrt(Q)
        # Forward filtering, backward sampling (FFBS)
        xb = myFFBS(y=y, cQ=cQ, cR=cR, mu0=mu0, Sigma0=Sigma0, A=A, Phi=Phi)
        xb = np.array(xb)
        # Gibbs sampling of parameters
        para = myGibbsQbase(xb=xb, mu0=mu0, Phi=Phi, Qbase=Qbase); # print(para)
        # para = myGibbsQbase_slow(xb=xb, mu0=mu0, Phi=Phi, Qbase=Qbase); # print(para)

        # Book keeping
        xb_list.append(xb)
        para_list.append(para)

    return(xb_list, para_list)

####################################################################################
# Gibbs sampler for d-inverse gamma (DIG) model
####################################################################################
# See also Petris et al (2009), ch. 4. (sec. 4.4; FFBS)
# https://github.com/cran/dlm/blob/master/R/DLM.R
def myGibbsDIG(xb, mu0, Phi, av=np.array([1,1,1]), bv=np.array([1,1,1])):
    N = xb.shape[0]
    xb_pred = np.concatenate([mu0.T, xb[:-1,:,0]]) # matrix of predictors
    errors = xb[:,:,0] - xb_pred @ Phi.T
    SSs = np.sum( errors**2, axis=0 ) # sum of squared errors
    # note that np.random.gamma uses 'scale' and Petris et al (2009) defines 'rate' (inverse scale)
    para_samp = np.random.gamma( shape=(av + N/2), scale=1/(bv + 0.5*SSs) )

    return(para_samp)

def myGibbsDIG_slow(xb, mu0, Phi, av=np.array([1,1,1]), bv=np.array([1,1,1])):
    N, M, _ = xb.shape
    xb_pred = np.concatenate([mu0.T, xb[:-1,:,0]]) # matrix of predictors
    SSs=np.zeros(M)
    for t in range(1,N): # skip the first prediction (numerical more stable), as m0 is not related to x_1
        error_t = xb[t,:,0].T - Phi @ xb_pred[t,:].T # errors
        SSs += error_t**2 # sum of squared errors
    # note that np.random.gamma uses 'scale' and Petris et al (2009) defines 'rate' (inverse scale)
    para_samp = np.random.gamma( shape=(av + N/2), scale=1/(bv + 0.5*SSs) )

    return(para_samp)

# Posterior sampling of states (FFBS) and parameters (Gibbs)
def myPosSamplerDIG(y, # data np.array
                    cQ, cR,
                    mu0 = np.asarray([30,20]), #first estimate
                    Sigma0 = np.array([[1,0],[0,1]]),
                    A = np.asarray([[1,0],[0,1]]), #Observation matrix. We want every state from our state vector.
                    Phi = np.array([[1,1],[0,1]]), #Transition matrix. Displacement is updated with prev disp + curr vel while velocity is updated with prev vel (assuming no acc.)
                    av=np.array([1]*3), bv=np.array([1]*3), # hyperprior parameters
                    Nsamp = 1000,
                   ):

    xb_list = []
    para_list = []
    for i in range(Nsamp):
        if (i>0):
            Q = np.diag(1/para) + np.eye(cQ.shape[0])*1e-8 # adding a little jitter
            cQ = np.sqrt(Q)
        # Forward filtering, backward sampling (FFBS)
        xb = myFFBS(y=y, cQ=cQ, cR=cR, mu0=mu0, Sigma0=Sigma0, A=A, Phi=Phi)
        xb = np.array(xb)
        # Gibbs sampling of parameters
        para = myGibbsDIG(xb=xb, mu0=mu0, Phi=Phi); # print(para)
        # para = myGibbsDIG_slow(xb=xb, mu0=mu0, Phi=Phi); # print(para)

        # Book keeping
        xb_list.append(xb)
        para_list.append(para)

    return(xb_list, para_list)


from __future__ import print_function  # For printing no newline
import sympy
import re

def fixPowers(s):
    return re.sub(r'q(\d+)\*\*(\d+)', r'pow( q\1, \2 )', s)

meqn = 5

# Ratio of specific heats
gamma = sympy.symbols("gamma")

# Conserved variables (mass, momentum and energy)
q1 = sympy.symbols("q1")
q2 = sympy.symbols("q2")
q3 = sympy.symbols("q3")
q4 = sympy.symbols("q4")
q5 = sympy.symbols("q5")

# Flux variables
#f1 = sympy.symbols("f1")
#f2 = sympy.symbols("f2")
#f3 = sympy.symbols("f3")
#f4 = sympy.symbols("f4")
#f5 = sympy.symbols("f5")

# Primitive variables:
u1 = sympy.symbols("u1")
u2 = sympy.symbols("u2")
u3 = sympy.symbols("u3")
en = sympy.symbols("en")  # energy
pr = sympy.symbols("pr")  # pressure

# Primitive variables ( velocity, pressure and energy )
u1 = q2/q1
u2 = q3/q1
u3 = q4/q1
pr = (gamma-1)*(en-q1/2*(u1**2+u2**2+u3**2))
en = q5

# Flux values
f1 = q1*u1
f2 = q1*u1**2 + pr
f3 = q1*u1*u2
f4 = q1*u1*u3
f5 = u1*(en+pr)

# Vector of conserved variables, and components of the flux function.
Q = [q1, q2, q3, q4, q5]
F = [f1, f2, f3, f4, f5]

# A = sympy.Matrix( meqn, meqn )
print("Computing the Jacobian of the flux function, f'(q)")
for j in range(meqn):
  for k in range(meqn):
      print( ('Dflux.set(i,%d,%d, ' % (j+1,k+1) ), end="" )
      tmp = fixPowers( str( sympy.simplify( sympy.expand( sympy.diff( F[j], Q[k]))).evalf() ) )
      print( tmp, end=");\n")
print(' ')

print("Computing the Hessian of the flux function: f''(q)")
for m1 in range(meqn):
  print(' ')
  for m2 in range(meqn):
    for m3 in range(meqn):
      print( ('D2flux.set(i,%d,%d,%d, ' % (m1+1,m2+1,m3+1) ), end="" )
      tmp = fixPowers( str( sympy.expand( sympy.diff( F[m1], Q[m2], Q[m3])).evalf() ) )
      print( tmp, end=");\n")
print(' ')


# -*- coding: utf-8 -*-
"""
@author: john
"""

from pygaussmarkov.ParametricSolver import ParametricSolver
import numpy as np
import matplotlib.pyplot as plt
import pygaussmarkov.GaussMarkovUtilities as GMUtils

class NthOrderFit(ParametricSolver):
    """ Nth Order Fitter """

    def __init__(self,order, xData, yData, uncertainties, deltaChiSqToStop = 0.01,dampingFactor = 1,useDampedGaussNeutonLineSearch = False, recordHistory = False):
        """
        order = the order of the fit, order 2 is for a*x**2 +b*x +c
        xData = np array with the x data
        yData = np array with the y data
        uncertainties = np array of the measurement uncertainties. If length=1 then the uncertainty will be assumed to be constant
        initialValues = the starting point for the fit
        deltaChiSqToStop = if the change in chiSq is smaller than this the fitter will stop. Default = 0.01
        dampingFactor = constant to chagne convergence speed. Default 1
        useDampedGaussNeutonLineSearch  weather to use the damped gauss neuton method. default = False
        """
        self.order = order
        super().__init__(deltaChiSqToStop = deltaChiSqToStop,dampingFactor = dampingFactor,useDampedGaussNeutonLineSearch = useDampedGaussNeutonLineSearch,recordHistory=recordHistory)
        self.xData = xData
        self.yData = yData
        self.uncert = uncertainties

    def getListOfParameterNames(self):
        return ['P'+str(i) for i in range(self.order+1)]

    def getAandAT(self,variables):
        m = np.matrix(np.zeros([len(self.xData),self.getNumberOfFreeParameters()]))
        skipped = 0
        parList = self.getListOfParameterNames()
        for i,x in enumerate(self.xData):
            for j in range(self.order+1):
                if not self.fixed[parList[j]]:
                    m[i,j-skipped] = x**(self.order - j)
                else:
                    skipped+=1
        mt = m.transpose()
        return np.matrix(m), np.matrix(mt)

    def getFx(self,variables):
        out = np.array(self.xData)
        for i,x in enumerate(self.xData):
            out[i]=0
            for j in range(self.order+1):
                out[i]=out[i]+variables[j]*x**(self.order - j)
        return out

    def getP(self):
        return GMUtils.generatePMatrix(self.uncert,len(self.xData))

    def getMeasurements(self):
        return self.yData

def run_example():
    xData = np.array(range(300))
    mTrue = 2.0
    cTrue = 0.0
    nTrue = 4.0
    noiseWidth = 10
    noise = np.random.normal(0, noiseWidth, len(xData))
    yData = nTrue*xData**2 + mTrue * xData + cTrue
    yData = yData+noise
    uncert = np.array([noiseWidth])

    fitter = NthOrderFit(2,xData,yData,uncert,useDampedGaussNeutonLineSearch=True)

    # run the fit and display the results
    out, err = fitter.solve(np.array([1.0,1.0,1.0]))
    print('Number of iterations = ',fitter.noInterations)
    print('Solution = ',out)
    print('Uncertainties = ',err)
    print('Final ChiSq = ', fitter.finalChi_)

    # plot data
    plt.plot(xData,yData,'*r')
    plt.plot(xData,fitter.getFx(out),'-k')

if __name__ == '__main__':
    run_example()


"""Figure of the Runge function in 3D.

x is evaluated on [-1, 1]
y is the result of applying the Runge function on each element in x

"""
import matplotlib.pyplot as plt
import numpy as np
from numpy.polynomial import Polynomial as Poly

from temfpy.interpolation import runge

def get_interpolator_runge_baseline(func, degree):
    xnodes = np.linspace(-1, 1, degree)
    poly = Poly.fit(xnodes, func(xnodes), degree)
    return poly

xvalues = np.linspace(-1, 1, 10000)

interpolant1 = get_interpolator_runge_baseline(runge, 5)
yfit1 = interpolant1(xvalues)

interpolant2 = get_interpolator_runge_baseline(runge, 10)
yfit2 = interpolant2(xvalues)

fig, ax = plt.subplots()
ax.plot(xvalues, runge(xvalues), label="Runge Function")
ax.plot(xvalues, yfit1, label="Approx. Degree 5")
ax.plot(xvalues, yfit2, label="Approx. Degree 10")
ax.set_xlabel("$x$")
ax.set_ylabel("$f(x)$")
ax.set_title("Runge Function and Polynomial Approximations over Uniform Grid")
ax.legend()

fig.savefig("fig-runge")


import numpy as np
import matplotlib.pyplot as plt
import scipy.optimize as optimize

def sigmoid(z):
    a = 1 + np.exp(-z)
    return 1 / a

def cost_function(theta, X, y):
    m = len(X)
    X = np.concatenate((np.ones((m, 1)), X), axis=1)
    # Calculate Cost J
    h_theta = sigmoid(np.dot(X, theta[np.newaxis].T))

    a = - np.dot(y, np.log(h_theta))
    b = - np.dot((1 - y), np.log(1 - h_theta))
    c = a + b
    J_val = c / m

    return J_val[0]

def gradient(theta, X, y):
    m = len(X)
    X = np.concatenate((np.ones((m, 1)), X), axis=1)
    h_theta = sigmoid(np.dot(X, theta[np.newaxis].T))

    # Calculate gradient descent
    a = h_theta - y[np.newaxis].T
    gradience = np.dot(X.T, a) / m

    return gradience.T[0]

def main():
    data = np.loadtxt('./ex2/ex2/ex2data1.txt', delimiter=',')
    X = data[:, :-1]
    y = data[:, -1]
    theta = np.zeros((3, ))

    # Plot data
    plot_1 = plt.scatter(X[y == 1][:, 0], X[y == 1][:, 1], marker='*')
    plot_2 = plt.scatter(X[y == 0][:, 0], X[y == 0][:, 1], marker='x')
    plt.xlabel('Exam 1 score')
    plt.ylabel('Exam 2 score')
    plt.legend([plot_1, plot_2], ['Admitted', 'Not admitted'], loc='upper right')

    res = optimize.minimize(cost_function, theta, (X, y), 'BFGS', gradient)
    theta = res.x

    x_1 = np.linspace(20, 100, 1000)
    x_2 = - (theta[0] + theta[1] * x_1) / theta[2]
    plt.plot(x_1, x_2, c='red')
    plt.show()

if __name__ == '__main__':
    main()


# -*- coding: utf-8 -*-
"""ME18B181_ME18B182_Q2.ipynb

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/14_vBgEa4frIFzK7P2LMuV6su7-M1exEe
"""

# Commented out IPython magic to ensure Python compatibility.
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
# %matplotlib inline

ages = [15, 15, 15, 18, 28, 29, 37, 37, 44, 50, 50, 60, 61, 64, 65, 65, 72, 75, 75, 82, 85, 91, 91, 97, 98, 125, 142, 142, 147, 147, 150, 159, 165, 183, 192, 195, 218, 218, 219, 224, 225, 227, 232, 232, 237, 246, 258, 276, 285, 300, 301, 305, 312, 317, 338, 347, 354, 357, 375, 394, 513, 535, 554, 591, 648, 660, 705, 723, 756, 768, 860]
weights = [21.66, 22.75, 22.3, 31.25, 44.79, 40.55, 50.25, 46.88, 52.03, 63.47, 61.13, 81, 73.09, 79.09, 79.51, 65.31, 71.9, 86.1, 94.6, 92.5, 105, 101.7, 102.9, 110, 104.3, 134.9, 130.68, 140.58, 155.3, 152.2, 144.5, 142.15, 139.81, 153.22, 145.72, 161.1, 174.18, 173.03, 173.54, 178.86, 177.68, 173.73, 159.98, 161.29, 187.07, 176.13, 183.4, 186.26, 189.66, 186.09, 186.7, 186.8, 195.1, 216.41, 203.23, 188.38, 189.7, 195.31, 202.63, 224.82, 203.3, 209.7, 233.9, 234.7, 244.3, 231, 242.4, 230.77, 242.57, 232.12, 246.7]

data = pd.DataFrame.from_dict({"ages": ages, "weights": weights})
data

# hyperparameters
intermediate_size = 65
epochs = 450
batch_size = 4
lr = 3e-3

import tensorflow as tf

inputs = tf.constant(data["ages"], dtype=tf.float32)[:, None]
targets = tf.constant(data["weights"], dtype=tf.float32)

model = tf.keras.models.Sequential([
        tf.keras.layers.Dense(intermediate_size, input_dim=1, activation="tanh"),
        tf.keras.layers.Dense(1, activation='linear'),
])

optimizer = tf.keras.optimizers.Adam(learning_rate=lr)
model.compile(loss='mean_squared_error', optimizer=optimizer)
history = model.fit(inputs, targets, epochs=epochs, batch_size=batch_size)

training_loss = history.history["loss"]
plt.plot(np.arange(len(training_loss)), training_loss)
plt.ylabel("Training loss")
plt.xlabel("Number of epochs")
plt.show()

predictions = model.predict(inputs)
plt.scatter(data["ages"], data["weights"], label="Original targets")
plt.plot(data["ages"], predictions, color="r", label="Trained ANN")

y = lambda x: 233.846*(1 - np.exp(-0.006042*x))
plt.plot(data["ages"], y(data["ages"]), color="g", label="y = 233.846 ∗ (1 − exp(−0.006042 x))")

plt.xlabel("Ages")
plt.ylabel("Weights")
plt.legend()
plt.show()


# Get a feeling of the eigenvalue calculation algorithm
# det(A-l*I) = 0
# for different lambda
# Code plots the linear transformation of [A - lambda * I] of the unit circle
# and the unit girds for different lambda

import numpy as np
import matplotlib.pyplot as plt
from matplotlib.animation import FuncAnimation
from pylab import *

################################################################################
# Read me:
# If you want to play for fun, alter the "A", "min_lambda", and "min_lambda"
# the animation will display the linear transformation of [A - lambda * I]
# Code only works for 2x2 matrix
A = np.array([[2, 1.5], [0.4, 3]]) # The matrix to be analyzed
min_lambda = 0
max_lambda = 5
################################################################################

x_max = 8 # where does the grid go on +/-x directions
y_max = 5 # where does the grid go on +/-y directions
gx = x_max * 2 - 1 # number of vertical grid lines
gy = y_max * 2 - 1 # number of horizontal grid lines

# locate the points on the outside box, used for grid plot
l = np.zeros((2, gy))
B_l = l.copy()
r = l.copy()
B_r = r.copy()
t = np.zeros((2, gx))
B_t = t.copy()
b = t.copy()
B_b = b.copy()

l[1, :] = np.linspace(-y_max+1, y_max-1, gy)
l[0, :] = l[1, :] * 0 - x_max
r[1, :] = np.linspace(-y_max+1, y_max-1, gy)
r[0, :] = r[1, :] * 0 + x_max
t[0, :] = np.linspace(-x_max+1, x_max-1, gx)
t[1, :] = t[0, :] * 0 + y_max
b[0, :] = np.linspace(-x_max+1, x_max-1, gx)
b[1, :] = t[0, :] * 0 - y_max

B = A.copy() # placeholder for the matrix: A - Lambda * I
th = np.arange(0.0, 2.1*np.pi, 0.1) # angle of the unit circle
c = np.array([np.cos(th), np.sin(th)]) # circle vector to be transformed
B_c = c.copy() # placeholder for vector: (A - Lambda * I) * unit circle

e, v = np.linalg.eig(A) # calculate eigven values (e) and eigen vectors (v)

fig, ax = plt.subplots()
fig.set_figheight(6)
fig.set_figwidth(9)
plt.plot(c[0, :], c[1, :], 'r-') # plot original unit circle
thismanager = get_current_fig_manager()
thismanager.window.wm_geometry("+500+0") # move figure window position
xdata, ydata = [], []
ln, = plt.plot([], [], color='xkcd:sky blue', linestyle='-', linewidth=0.5)
g, = [plt.plot([], [], color='xkcd:sky blue', linestyle='-', linewidth=1)]
for i in range(0, gx+gy):
    if i < gx - 1:
        if i < (gx - 1) / 2 - 1:
            lw_temp = 0.5 # vertical grid width whoes original x < 0
        else:
            lw_temp = 2.5 # vertical grid width whoes original x >= 0
        g_temp, = plt.plot([], [], color='xkcd:sky blue',
                       linewidth=lw_temp, linestyle='-')
    else:
        if i - gx < ((gy +-1) / 2 - 1):
            lw_temp = 0.5 # horizontal grid width whoes original y < 0
        else:
            lw_temp = 2.5 # horizontal grid width whoes original y >= 0
        g_temp, = plt.plot([], [], color=(0.2, 0.8, 0.2),
                       linewidth=lw_temp, linestyle='-') # plot placeholder
    g.append(g_temp) # append list

# graph properties
ax.axis('equal') # make the plot aspect ratio = 1
ax.set_facecolor('k') # background color = black
plt.xlabel('Linear Transform of $A - \lambda  I$')
plt.title('Grids: unit grid after transformation of $A - \lambda  I$\n' +\
          'Red circle: unit circle $u$ before transformation\n' +\
          'Blue ellipse: unit circle after transformation: $(A - \lambda  I)u$')
tbox_1_t = '' # lambda value update text placeholder
tbox_1 = plt.text(7, -5, tbox_1_t,
                  {'color': 'xkcd:sky blue', 'fontsize': 18},
         va="top", ha="right")
tbox_1.set_bbox(dict(facecolor='grey', alpha=0.8, edgecolor='grey'))
tbox_2_t = 'eigenvalues of A: ' +\
    np.array2string(e, precision=2, separator=' ', suppress_small=True)
tbox_2 = plt.text(11.5, 7, tbox_2_t,
                  {'color': 'xkcd:sky blue', 'fontsize': 16},
         va="top", ha="right")
tbox_2.set_bbox(dict(facecolor='grey', alpha=0.8, edgecolor='grey'))
tbox_2_t = 'eigenvectors of A: \n' +\
    np.array2string(v, precision=2, separator=' ', suppress_small=True)
tbox_3 = plt.text(11.5, 5.5, tbox_2_t,
                  {'color': 'xkcd:sky blue', 'fontsize': 16},
         va="top", ha="right")
tbox_3.set_bbox(dict(facecolor='grey', alpha=0.8, edgecolor='grey'))
tbox_4_t = 'A = ' +\
    np.array2string(A, precision=2, separator=' ', suppress_small=True)
tbox_4 = plt.text(-1, -5, tbox_4_t,
                  {'color': 'xkcd:sky blue', 'fontsize': 18},
         va="top", ha="right")
tbox_4.set_bbox(dict(facecolor='grey', alpha=0.8, edgecolor='grey'))

# to create the animation
def init(): # initialize the animation
    plot_max = 12
    ax.set_xlim(-plot_max, plot_max)
    ax.set_ylim(-plot_max, plot_max)
    return ln,

def update(frame): # animation update function
    B = A - frame * np.identity(2) # calculate A - Lambda * I
    for i in range(len(th)):
        B_c[:, i] = np.dot(B, c[:, i]) # linear transform of unit circle
    for i in range(0, gy):
        B_l[:, i] = np.dot(B, l[:, i]) # linear transform of left box line
    for i in range(0, gy):
        B_r[:, i] = np.dot(B, r[:, i]) # linear transform of right box line
    for i in range(0, gx):
        B_t[:, i] = np.dot(B, t[:, i]) # linear transform of top box line
    for i in range(0, gx):
        B_b[:, i] = np.dot(B, b[:, i]) # linear transform of bottom box line

    xdata.append(B_c[0, :])
    ydata.append(B_c[1, :])
    ln.set_data(xdata, ydata)
    yield ln

    for i in range(0, gx):
        g[i].set_data([B_t[0, i], B_b[0, i]],
                      [B_t[1, i], B_b[1, i]])
        yield g[i] # return g[i]
    for i in range(gx, gx+gy):
        g[i].set_data([B_l[0, i-gx], B_r[0, i-gx]],
                      [B_l[1, i-gx], B_r[1, i-gx]])
        yield g[i] # return g[i]

    tbox_1.set_text('$\lambda$ =%.1f' % frame) # update lambda on animation
    yield tbox_1

    return ln,

ani = FuncAnimation(fig, update, frames=np.linspace(min_lambda,max_lambda,500),
                   init_func=init, blit=True, repeat=True, interval=10)
plt.show()
ani.save('animation_1_1_1_1.gif', writer='imagemagick', fps=30)


import indexedexp as ixp
import sympy as sp

'''
Conversion FROM ADM variables gamma_{ij}, beta^i, and alpha TO 4-metric g_{mu nu} and/or inverse 4-metric g^{mu nu}
'''
def ADM_to_four_metric(gammaDD,betaU,alpha, returng4DD=True, returng4UU=False):
    # The ADM formulation decomposes Einstein's 4D equations into 3+1 dimensional form, with
    #     the 3 spatial dimensions separated from the 1 temporal dimension. DIM here refers to
    #     the spatial dimension.
    DIM = 3

    # Eq 4.47 in [Gourgoulhon](https://arxiv.org/pdf/gr-qc/0703035.pdf):

    # g_{tt} = -\alpha^2 + \beta^k \beta_k
    # g_{ti} = \beta_i
    # g_{ij} = \gamma_{ij}

    # Eq. 2.121 in B&S
    betaD = ixp.zerorank1()
    for i in range(DIM):
        for j in range(DIM):
            betaD[i] += gammaDD[i][j] * betaU[j]

    # Now compute the beta contraction.
    beta2 = sp.sympify(0)
    for i in range(DIM):
        beta2 += betaU[i] * betaD[i]

    g4DD = ixp.zerorank2(DIM=4)
    g4DD[0][0] = -alpha ** 2 + beta2
    for i in range(DIM):
        g4DD[i + 1][0] = g4DD[0][i + 1] = betaD[i]
        for j in range(DIM):
            g4DD[i + 1][j + 1] = gammaDD[i][j]

    if returng4DD == True and returng4UU == False:
        return g4DD

    gammaUU, gammaDET = ixp.symm_matrix_inverter3x3(gammaDD)
    # Eq. 4.49 in [Gourgoulhon](https://arxiv.org/pdf/gr-qc/0703035.pdf):

    # g^{tt} = -1 / alpha^2
    # g^{ti} = beta^i / alpha^2
    # g^{ij} = gamma^{ij} - beta^i beta^j / alpha^2

    g4UU = ixp.zerorank2(DIM=4)
    g4UU[0][0] = - 1 / alpha ** 2
    for i in range(DIM):
        g4UU[i + 1][0] = g4UU[0][i + 1] = betaU[i] / alpha ** 2
        for j in range(DIM):
            g4UU[i + 1][j + 1] = gammaUU[i][j] - betaU[i]*betaU[j] / alpha ** 2

    if returng4DD == True and returng4UU == True:
        return g4DD,g4UU
    if returng4DD == False and returng4UU == True:
        return g4UU
    print("Error: ADM_to_four_metric() called without requesting anything being returned!")
    exit(1)

'''
Conversion FROM 4-metric g_{mu nu} TO ADM variables gamma_{ij}, beta^i, and alpha
'''
def four_metric_to_ADM(g4DD):
    # The ADM formulation decomposes Einstein's 4D equations into 3+1 dimensional form, with
    #     the 3 spatial dimensions separated from the 1 temporal dimension. DIM here refers to
    #     the spatial dimension.
    DIM = 3

    # Eq 4.47 in [Gourgoulhon](https://arxiv.org/pdf/gr-qc/0703035.pdf):
    # g_{ij} = \gamma_{ij}

    gammaDD = ixp.zerorank2()
    for i in range(DIM):
        for j in range(DIM):
            gammaDD[i][j] = g4DD[i + 1][j + 1]

    # Eq 4.47 in [Gourgoulhon](https://arxiv.org/pdf/gr-qc/0703035.pdf):
    # g_{ti} = \beta_i
    betaD = ixp.zerorank1()
    for i in range(DIM):
        betaD[i] = g4DD[i+1][0]

    #Eq. 2.121 in B&S:
    # beta^i = gamma^{ij} beta_j
    betaU = ixp.zerorank1()
    gammaUU, gammaDET = ixp.symm_matrix_inverter3x3(gammaDD)
    for i in range(DIM):
        for j in range(DIM):
            betaU[i] += gammaUU[i][j]*betaD[j]

    # Eq 4.47 in [Gourgoulhon](https://arxiv.org/pdf/gr-qc/0703035.pdf):
    # g_{tt} = -\alpha^2 + \beta^k \beta_k
    # -> alpha = sqrt(beta^2 - g_tt)
    # Now compute the beta contraction.
    beta2 = sp.sympify(0)
    for i in range(DIM):
        beta2 += betaU[i] * betaD[i]

    alpha = sp.sqrt(beta2 - g4DD[0][0])

    return gammaDD,betaU,alpha


import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import matplotlib.patches as mpatches
from mpl_toolkits.mplot3d import Axes3D
from sklearn.linear_model import LinearRegression

np.random.seed(5)
trainDataRange = np.array([-3,6])

#True model: y=x^3-5x^2+3+9
a = 1
b = -5
c = 3
d = 9

#Generate random training data within the trainDataRange
#Parameter for showing plots and number of samples
def generateTrainData(numSamples=20,showPlot=True):
    #Generate X data
    trainX = np.linspace(trainDataRange[0],trainDataRange[1],numSamples)
    trainX = np.sort(trainX)
    trueY = transform_x(trainX)
    #Add noise with variance 10
    noisyY = trueY+np.random.randn(numSamples)*20
    #Plot the data
    if showPlot:
        plt.plot(trainX,noisyY,'bo')
        plt.plot(trainDataRange, transform_x(trainDataRange), 'r-', lw=2)
        plt.ylabel("Noisy Y")
        plt.xlabel("Input X")
        plt.title("Clean X and Noisy Y from Linear Relationship")
        plt.ylim(ymin=min(noisyY-20),ymax=max(noisyY)*2)
        plt.show()
        plt.clf()
    return trainX,noisyY

def transform_x(inputX):
    return a*np.power(inputX,3)+b*np.power(inputX,2)+c*np.power(inputX,1)+d

#Validate the data on a larger range
#Default is -2 to 9
#Training region is denoted by dotted lines
def validate(model,trainX,noisyY,numSamples=500,dataRange=[-4,7],showPlot=True,coeffs={},fig_size=(7,5)):
    dataRange=np.arange(dataRange[0],dataRange[1],0.1)
    transformed_dataRange = transform_x(dataRange)
    #Generate x values from the data range
    inputX = np.random.random_sample((numSamples,))*(dataRange[-1]-dataRange[0])+dataRange[0]
    inputX = np.sort(inputX)
    trueY = transform_x(dataRange)
    predY=predict(model,dataRange)
    #Plot graphs
    if showPlot:
        #A bit of math to determine where to draw the dotted lines
        plt.figure(figsize = fig_size)
        #plt.axes().clear()
        coordX1 = [trainDataRange[0]]*2
        coordX2 = [trainDataRange[1]]*2
        minY = min(min(transformed_dataRange),min(predY))
        maxY = max(max(transformed_dataRange),max(predY))
        line1, = plt.plot(coordX1, [minY,maxY], 'k-', lw=1,linestyle="--", label="Training Data Range")
        plt.plot(coordX2, [minY,maxY], 'k-', lw=1,linestyle="--")
        #TrainX
        train, = plt.plot(trainX,noisyY,'ko', lw=2, label="Training Data")
        #Prediction
        pred, = plt.plot(dataRange,predY,'b-', lw=2, label="Predicted Model")
        #True Data
        trueData, = plt.plot(dataRange, transform_x(dataRange),'r-', lw=1.5, label="True Data Distribution",alpha=0.7)

        plt.ylabel("Y")
        plt.xlabel("Input X")
        plt.ylim(ymin=min(noisyY-20),ymax=max(noisyY)*2)
        title = "Degree " + str (len(model)-1)+ " Model"
        for i,key in enumerate(coeffs.keys()):
            title += " "+str(key)+"="+str(coeffs[key])
            if i<len(coeffs.keys())-1:
                title +=","
        plt.title(title)
        # Create a legend for the first line.
        legend = plt.legend(handles=[line1,train,pred,trueData])
        plt.show()
    return error(trueY,predY)

#Train the data
def model(trainX,trainY,degree=1):
    #Creates the vandermonde matrix https://en.wikipedia.org/wiki/Vandermonde_matrix
    powers=np.vander(trainX,degree+1)
    A=powers
    #Solves the normal equation
    model = np.linalg.solve(A.T@A,A.T@trainY)
    return model

#Predicts given x values based on a model
def predict(model,x):
    degree=len(model)-1
    powers=np.vander(x,degree+1)
    return powers@model

#Determines the error between true Y values and predicted
def error(trueY,predY):
    return np.linalg.norm((trueY-predY))/len(trueY)

#Generates graphs of different degree models
#Plots training error and test error
def overfittingDemo(degrees = [0,1,2,3,4,5,7,10,13]):
    trainX,trainY = generateTrainData(showPlot=False)
    trainError = []
    testError = []
    #Iterate over all model orders
    for deg in degrees:
        currModel = model(trainX,trainY,degree=deg)
        predTrainY = predict(currModel,trainX)
        currTrainErr = error(trainY,predTrainY)
        currTestErr = validate(currModel,trainX,trainY,showPlot=True)
        trainError.append(currTrainErr)
        testError.append(currTestErr)

    #Plot the errors
#     plt.figure(figsize=(10,4))
#     plt.subplot(1,2,1)
#     plt.plot(degrees,trainError)
#     plt.ylabel("Error")
#     plt.xlabel("Degree of Model")
#     plt.title("Training Error")

#     plt.subplot(1,2,2)
#     plt.plot(degrees,np.log(testError))
#     plt.title("Test Error")
#     plt.xlabel("Degree of Model")
#     plt.ylabel("Log Error")
#     plt.show()
#     Uncomment if you are curious about the actual error values
#     print("Training Errors:",trainError)
#     print("Test Errors:",testError)

def get_features(data, col_list, y_name):
    """
    Function to return a numpy matrix of pandas dataframe features, given k column names and a single y column
    Outputs X, a n X k dimensional numpy matrix, and Y, an n X 1 dimensional numpy matrix.
    This is not a smart function - although it does drop rows with NA values. It might break.

    data(DataFrame): e.g. mpg, mpg_train
    col_list(list): list of columns to extract data from
    y_name(string): name of the column you to treat as the y column

    Ideally returns one np.array of shape (len(data), len(col_list)), and one of shape (len(data), len(col_list))
    """

    # keep track of numpy values
    feature_matrix = data[col_list + [y_name]].dropna().values
    return feature_matrix[:, :-1], feature_matrix[:, -1]

def plot_multiple_linear_regression(data, x_1_name, x_2_name, y_name):
    """
    This function makes a 3D plot for multliple linear regression with two explanatory variables.
    Thanks to: https://stackoverflow.com/a/26434204

    data(DataFrame): e.g. mpg, mpg_train
    x_1_name(string): the name of the column representing the first explanatory variable
    x_2_name(string): the name of the column representing the second explanatory variable
    y_name(string): the name of the column representing the dependent/response variable

    returns None but outputs an interactive 3D plot
    """
    model = LinearRegression()
    X, Y = get_features(data, [x_1_name, x_2_name], y_name)
    fit = model.fit(X, Y)

    fig = plt.figure(figsize=(10,10))
    ax = fig.add_subplot(111, projection='3d')

    x_surf = np.arange(0, max(data[x_1_name]), max(data[x_1_name])/10)                # generate a mesh
    y_surf = np.arange(0, max(data[x_2_name]), max(data[x_2_name])/10)
    x_surf, y_surf = np.meshgrid(x_surf, y_surf)

    exog = pd.core.frame.DataFrame({x_1_name: x_surf.ravel(), x_2_name: y_surf.ravel()})
    out = fit.predict(exog)
    print(exog.shape)
    ax.plot_surface(x_surf, y_surf,
                    out.reshape(x_surf.shape),
                    rstride=1,
                    cstride=1,
                    color='None',
                    alpha = 0.4)

    ax.scatter(data[x_1_name], data[x_2_name], data[y_name],
               marker='o',
               alpha=1)

    ax.set_xlabel(x_1_name)
    ax.set_ylabel(x_2_name)
    ax.set_zlabel(y_name)
    plt.title("Linear Model with 2 Explanatory Variables: " + x_1_name + " and " + x_2_name + " vs. " + y_name, y=1.08, fontsize=16)

    plt.show();

def plot_simple_residuals(data, x_name, y_name, linear_model):
    """
    This function plots a residual plot based off of a simple linear model
    on top of the scatterplot of the data it was fit to.

    data(DataFrame): e.g. mpg_train
    x_name(string): the name of the column representing the predictor variable
    y_name(string): the name of the column representing the dependent/response variable
    linear_model

    returns None but outputs residual plot resulting from linear model overlaid on scatterplot
    """
    X = data[[x_name]].values
    Y = data[[y_name]].values
    residuals = Y - linear_model.predict(X)

#     axes = plt.gca()
    plt.scatter(X, residuals)
    plt.axhline(y=0, color='r', linestyle='-')
    plt.title("Residual Plot: " + x_name + " vs. " + y_name)
    plt.xlabel(x_name)
    plt.ylabel("Residuals")
    plt.show()

# This function will just plot a scatter plot with the linear model overlaid.
def overlay_simple_linear_model(data, x_name, y_name, linear_model):
    """
    This function plots a simple linear model on top of the scatterplot of the data it was fit to.

    data(DataFrame): e.g. mpg_train
    x_name(string): the name of the column representing the predictor variable
    y_name(string): the name of the column representing the dependent/response variable
    linear_model

    returns None but outputs linear model overlaid on scatterplot
    """

    x = np.arange(max(data[x_name])).reshape(-1, 1) # a 2D array of integers between 0 and the maximum value of the x_name column
    y = linear_model.predict(x) # replace ___ with correct function

    data.plot.scatter(x_name, y_name) # scatter plot of x_name vs. y_name

    plt.plot(x, y, color='red')
    plt.title("Linear Model vs. Data: " + x_name + " vs. " + y_name)
    plt.show()


import numpy as np

"""
A simple propagation of uncertainty module.
# http://en.wikipedia.org/wiki/Propagation_of_uncertainty

For more sophisticated handling of uncertainties use the `uncertainties`
package.
"""

def EPadd(a, da, b, db, covar=0):
    """
    C = A + B
    """
    return (a + b), np.sqrt(da ** 2 + db ** 2 + 2 * covar)

def EPsub(a, da, b, db, covar=0):
    """
    C = A - B
    """
    return a - b, np.sqrt(da ** 2 + db ** 2 - 2 * covar)

def EPmul(a, da, b, db, covar=0):
    """
    C = A * B
    """
    return a * b, np.sqrt((b * da) ** 2 + (a * db) ** 2 + 2 * a * b * covar)

def EPmulk(a, da, k):
    """
    C = A * k
    """
    return a * k, np.absolute(da * k)

def EPdiv(a, da, b, db, covar=0):
    """
    C = A / B
    """
    return (
        a / b,
        np.sqrt(
            ((da / b) ** 2 + ((a ** 2) * (db ** 2) / (b ** 4)))
            - (2 * covar * a / (b ** 3))
        ),
    )

def EPpow(a, da, b, db, covar=0):
    """
    C = A ** B
    """
    c = a ** b
    dc = np.absolute(c) * np.sqrt(
        ((da * b / a) ** 2)
        + (np.log(a) * db) ** 2
        + 2 * b * np.log(a) * covar / a
    )
    return c, dc

def EPpowk(a, da, k, n=1):
    """
    C = pow(k, A * n)
    """
    return (
        np.power(k, a * n),
        np.absolute(np.power(k, a * n) * n * da * np.log(k)),
    )

def EPlog(a, da, k=1, n=1):
    """
    C = n * log(k * A )
    """
    return n * np.log(k * a), np.absolute(n * da / a)

def EPlog10(a, da, k=1, n=1):
    """
    C = n * log10(k * A)
    """
    return n * np.log10(k * a), np.absolute(n * da / (a * np.log(10)))

def EPexp(a, da, k=1, n=1):
    """
    C = n * exp(k * A)
    """
    return n * np.exp(k * a), np.absolute(k * da * n * np.exp(k * a))

def EPsin(a, da):
    """
    C = sin (A)
    """
    return np.sin(a), np.absolute(np.cos(a) * da)

def EPcos(a, da):
    """
    C = cos (A)
    """
    return np.cos(a), np.absolute(-np.sin(a) * da)

def EPtan(a, da):
    """
    C = tan (A)
    """
    return np.tan(a), np.absolute(da / (np.cos(a) ** 2))


# Logistic Regression

import numpy as np
np.set_printoptions(suppress=True)
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn import datasets

In this section we will construct binary and multiclass logistic regression models. We will try our binary model on the {doc}`breast cancer </content/appendix/data>` dataset and the multiclass model on the {doc}`wine </content/appendix/data>` dataset.

## Binary Logistic Regression

# import data
cancer = datasets.load_breast_cancer()
X = cancer['data']
y = cancer['target']

Let's first define some helper functions: the logistic function and a standardization function, equivalent to `scikit-learn`'s `StandardScaler`

def logistic(z):
    return (1 + np.exp(-z))**(-1)

def standard_scaler(X):
    mean = X.mean(0)
    sd = X.std(0)
    return (X - mean)/sd

The binary logistic regression class is defined below. First, it (optionally) standardizes and adds an intercept term. Then it estimates $\boldsymbol{\beta}$ with gradient descent, using the gradient of the negative log-likelihood derived in the {doc}`concept section </content/c3/s1/logistic_regression>`,

$$
\frac{\partial \mathcal{L}(\boldsymbol{\beta})}{\partial \boldsymbol{\beta}} = \frac{\partial - \log L (\boldsymbol{\beta})}{\partial \boldsymbol{\beta}} = -\mathbf{X}^\top (\mathbf{y} - \mathbf{p}).
$$

class BinaryLogisticRegression:

    def fit(self, X, y, n_iter, lr, standardize = True, has_intercept = False):

        ### Record Info ###
        if standardize:
            X = standard_scaler(X)
        if not has_intercept:
            ones = np.ones(X.shape[0]).reshape(-1, 1)
            X = np.concatenate((ones, X), axis = 1)
        self.X = X
        self.N, self.D = X.shape
        self.y = y
        self.n_iter = n_iter
        self.lr = lr

        ### Calculate Beta ###
        beta = np.random.randn(self.D)
        for i in range(n_iter):
            p = logistic(np.dot(self.X, beta)) # vector of probabilities
            gradient = -np.dot(self.X.T, (self.y-p)) # gradient
            beta -= self.lr*gradient

        ### Return Values ###
        self.beta = beta
        self.p = logistic(np.dot(self.X, self.beta))
        self.yhat = self.p.round()

The following instantiates and fits our logistic regression model, then assesses the in-sample accuracy. Note here that we predict observations to be from class 1 if we estimate $P(Y_n = 1)$ to be above 0.5, though this is not required.

binary_model = BinaryLogisticRegression()
binary_model.fit(X, y, n_iter = 10**4, lr = 0.0001)
print('In-sample accuracy: '  + str(np.mean(binary_model.yhat == binary_model.y)))

Finally, the graph below shows a distribution of the *estimated* $P(Y_n = 1)$ based on each observation's *true* class. This demonstrates that our model is quite confident of its predictions.

fig, ax = plt.subplots()
sns.distplot(binary_model.p[binary_model.yhat == 0], kde = False, bins = 8, label = 'Class 0', color = 'cornflowerblue')
sns.distplot(binary_model.p[binary_model.yhat == 1], kde = False, bins = 8, label = 'Class 1', color = 'darkblue')
ax.legend(loc = 9, bbox_to_anchor = (0,0,1.59,.9))
ax.set_xlabel(r'Estimated $P(Y_n = 1)$', size = 14)
ax.set_title(r'Estimated $P(Y_n = 1)$ by True Class', size = 16)
sns.despine()

## Multiclass Logistic Regression

# import data
wine = datasets.load_wine()
X = wine['data']
y = wine['target']

Before fitting our multiclass logistic regression model, let's again define some helper functions. The first (which we don't actually use) shows a simple implementation of the softmax function. The second applies the softmax function to each row of a matrix. An example of this is shown for the matrix

$$
\mathbf{Z} = \begin{bmatrix} 1 & 1 \\ 0 & 1 \end{bmatrix}.
$$

The third function returns the $I$ matrix discussed in the concept section, whose $(n, k)^\text{th}$ element is a 1 if the $n^\text{th}$ observation belongs to the $k^\text{th}$ class and a 0 otherwise. An example is shown for

$$
\mathbf{y} = \begin{bmatrix} 0 & 0 & 1 & 1 & 2 \end{bmatrix}^\top.
$$

def softmax(z):
    return np.exp(z)/(np.exp(z).sum())

def softmax_byrow(Z):
    return (np.exp(Z)/(np.exp(Z).sum(1)[:,None]))

def make_I_matrix(y):
    I = np.zeros(shape = (len(y), len(np.unique(y))), dtype = int)
    for j, target in enumerate(np.unique(y)):
        I[:,j] = (y == target)
    return I

Z_test = np.array([[1, 1],
              [0,1]])
print('Softmax for Z:\n', softmax_byrow(Z_test).round(2))

y_test = np.array([0,0,1,1,2])
print('I matrix of [0,0,1,1,2]:\n', make_I_matrix(y_test), end = '\n\n')

The multiclass logistic regression model is constructed below. After standardizing and adding an intercept, we estimate $\hat{\mathbf{B}}$ through gradient descent. Again, we use the gradient discussed in the {doc}`concept section </content/c3/s1/logistic_regression>`,

$$
\frac{\partial \mathcal{L}(\mathbf{B})}{\partial \mathbf{B}} = \frac{\partial -\log L (\mathbf{B})}{\partial \mathbf{B}} = \mathbf{X}^\top (\mathbf{I} - \mathbf{P}).
$$

class MulticlassLogisticRegression:

    def fit(self, X, y, n_iter, lr, standardize = True, has_intercept = False):

        ### Record Info ###
        if standardize:
            X = standard_scaler(X)
        if not has_intercept:
            ones = np.ones(X.shape[0]).reshape(-1, 1)
            X = np.concatenate((ones, X), axis = 1)
        self.X = X
        self.N, self.D = X.shape
        self.y = y
        self.K = len(np.unique(y))
        self.n_iter = n_iter
        self.lr = lr

        ### Fit B ###
        B = np.random.randn(self.D*self.K).reshape((self.D, self.K))
        self.I = make_I_matrix(self.y)
        for i in range(n_iter):
            Z = np.dot(self.X, B)
            P = softmax_byrow(Z)
            gradient = np.dot(self.X.T, self.I - P)
            B += lr*gradient

        ### Return Values ###
        self.B = B
        self.Z = np.dot(self.X, B)
        self.P = softmax_byrow(self.Z)
        self.yhat = self.P.argmax(1)

The multiclass model is instantiated and fit below. The `yhat` value returns the class with the greatest estimated probability. We are again able to classify all observations correctly.

multiclass_model = MulticlassLogisticRegression()
multiclass_model.fit(X, y, 10**4, 0.0001)
print('In-sample accuracy: '  + str(np.mean(multiclass_model.yhat == y)))

The plots show the distribution of our estimates of the probability that each observation belongs to the class it actually belongs to. E.g. for observations of class 1, we plot $P(y_n = 1)$. The fact that most counts are close to 1 shows that again our model is confident in its predictions.

fig, ax = plt.subplots(1, 3, figsize = (17, 5))
for i, y in enumerate(np.unique(y)):
    sns.distplot(multiclass_model.P[multiclass_model.y == y, i],
                 hist_kws=dict(edgecolor="darkblue"),
                 color = 'cornflowerblue',
                 bins = 15,
                 kde = False,
                 ax = ax[i]);
    ax[i].set_xlabel(xlabel = fr'$P(y = {y})$', size = 14)
    ax[i].set_title('Histogram for Observations in Class '+ str(y), size = 16)
sns.despine()


# Question 6 Lab 03
# AB Satyaprakash (180123062)

# imports ----------------------------------------------------------------------
from math import factorial
from fractions import Fraction
import numpy as np
import sympy as sp
# ------------------------------------------------------------------------------
# functions --------------------------------------------------------------------

def forwardDiff(fArray):
    sz = len(fArray)
    fdArray = [fArray]
    for i in range(1, sz):
        temp = []
        for j in range(sz-i):
            temp.append(fdArray[i-1][j+1]-fdArray[i-1][j])
        fdArray.append(temp)
    return fdArray

def newtonFDPoly(fArray, xArray):
    x0, x1 = xArray[0], xArray[1]
    h = x1-x0
    u = np.array([1/h, -x0/h])
    fdArray = forwardDiff(fArray)
    sz = len(fArray)
    px = np.array([0])

    for i in range(sz):
        term = np.array([1])
        for j in range(i):
            term = np.polymul(term, np.polyadd(u, np.array([-j])))
            term = term/(j+1)
        term = term*fdArray[i][0]
        px = np.polyadd(px, term)
    return px

# ------------------------------------------------------------------------------
# The following data are given for a polynomial P(x) of unknown degree.
# P(0) = 4, P(1) = 9, P(2) = 15, P(3) = 18
# Determine the coefficient of x^3 in P(x), if all fourth-order forward differences are 1.

print('As we can simply observe, if all 4th order forward differences are 1, this means')
print('All 5th order forward differences will be 0, in other words degree of polynmial is 4')
X = [0, 1, 2, 3]
P = [4, 9, 15, 18]

px = newtonFDPoly(P, X)
# Since we also have informatio about the 4th order FD, we can add 1 more term to P(x)
term = np.polymul([1, 0], np.polymul([1, -1], np.polymul([1, -2], [1, -3])))
term = (term*1)/factorial(4)
px = np.polyadd(px, term)
print('The polynmial is thus given as \n{}'.format(np.poly1d(px)))
print('Clearly, the coefficient of x^3 is {} or {}'.format(px[1], '-11/12'))

# Question 6 ends --------------------------------------------------------------

Semester 6/MA 322 (Scientific Computing Theory)/Labs/Lab 3/Code/q6.py


import numpy as np

def create_dataset(func, x_range, n_samples, mu, sigma, equispaced=True):
    """Creates ``n_samples`` samples using function ``func``
    and adding noise from Gaussian with mean ``mu`` and
    variance ``sigma``.
    """
    if equispaced:
        xs = np.linspace(x_range[0], x_range[1], n_samples)
    else:
        xs = np.random.uniform(x_range[0], x_range[1], size=n_samples)
    ts = func(xs) + np.random.normal(mu, sigma, n_samples)
    return xs, ts

def augment_dataset(xs, deg):
    """Augments monodimentional list of samples.
    """
    assert(xs.ndim == 1)
    xs = np.expand_dims(xs, 1)
    xs_augmented = np.ones(xs.shape)
    for i in range(1, deg+1):
        xs_augmented = np.hstack((xs_augmented, xs**i))
    return xs_augmented

def polynomial(x, coeffs):
    """ Returns a polynomial for ``x`` values for the ``coeffs`` provided.

    The coefficients must be in ascending order (``x**0`` to ``x**deg``).
    """
    y = 0
    for i in range(len(coeffs)):
        y += coeffs[i]*x**i
    return y


from app.utils.methods import BaseMethod
from sympy import Function, Symbol, diff
from sympy.parsing.sympy_parser import parse_expr

class Newton(BaseMethod):

    def __init__(self, function, x0, tolerance, iterations):
        self.function = function
        self.x0 = x0
        self.tol = tolerance
        self.iterations = int(iterations)
        self.array = []

    def run(self):
        f = parse_expr(self.function)
        x = Symbol("x")
        df = diff(f, x)
        fx = f.subs(x, self.x0)
        dfx = f.subs(x, self.x0)
        error = self.tol + 1
        self.array.append(["0", float(self.x0), float(fx), float(dfx), "0"])
        for i in range(1, self.iterations):
            if fx == 0 or error <= self.tol or dfx == 0:
                break

            x1 = self.x0 - fx / dfx
            fx = f.subs(x, x1)
            dfx = df.subs(x, x1)
            error = abs(x1 - self.x0)
            rel_error = error
            self.x0 = x1

            self.array.append([str(i), float(self.x0), float(fx), float(dfx), float(rel_error)])

        if fx == 0:
            return {
                "method_status": "success",
                "result": f"{float(self.x0)} is a root",
                "iterations": self.array
            }
        elif error < self.tol:
            return {
                "method_status": "success",
                "result": f"{float(self.x0)} approaches a root of the function with a tolerance of {self.tol}",
                "iterations": self.array
            }
        elif dfx == 0:
            return {
                "method_status": "success",
                "result": f"{float(self.x0)} is a possible multiple root",
                "iterations": self.array
            }
        else:
            return {
                "method_status": "failed",
                "result": "Max interactions exceeded",
                "iterations": self.array
            }


import numpy as np
import json

class SlamMap:
    def __init__(self, markers = np.zeros((2,0)), covariance = np.zeros((0,0)), taglist = []):
        self.markers = markers
        self.covariance = covariance
        self.taglist = taglist
        self.check_valid()

    def check_valid(self):
        assert (markers.shape[0] == 2), "Markers must be 2xn."
        n = markers.shape[1]
        assert (n == len(taglist)), "No. of markers and tags do not match."
        assert (2*n == self.covariance.shape[0]), "Covariance matrix does not match size of markers."
        assert (self.covariance.shape[0] == self.covariance.shape[1]), "Covariance matrix is not square."
        cov_sym_score = np.linalg.norm(self.covariance - self.covariance.T)
        assert(cov_sym_score < 0.01), "Covariance matrix is not symmetric."

    def save(self, fname="slam_map.txt"):
        map_attributes = {"taglist":self.taglist,
                          "markers":self.markers.tolist(),
                          "covariance":self.covariance.tolist()}
        with open(fname,'w') as map_file:
            json.dump(map_attributes, map_file, indent=2)

    def load(self, fname="slam_map.txt"):
        with open(fname,'r') as map_file:
            map_attributes = json.load(map_file)
        self.taglist = map_attributes["taglist"]
        self.markers = np.array(map_attributes["markers"])
        self.covariance = np.array(map_attributes["covariance"])
        self.check_valid()

    def compute_tf(self, other):
        markers1 = self.markers.copy()
        markers2 = other.markers.copy()

        # re-order the other markers to match self
        idxlist = [other.taglist.index(tag) for tag in self.taglist]
        markers2 = markers2[:, idxlist]

        armse, R, t = self.compute_armse(markers1, markers2)

        return armse, R, t

    @staticmethod
    def compute_armse(markers1, markers2):
        # An implementation of Umeyama's method
        n = markers1.shape[1]
        mu1 = 1.0/n * np.sum(markers1, axis=1, keepdims=True)
        mu2 = 1.0/n * np.sum(markers2, axis=1, keepdims=True)
        Sigma = 1.0/n * np.dot((markers2 - mu2), (markers1 - mu1).transpose())

        S = np.eye(2)

        U, D, VH = np.linalg.svd(Sigma)

        R = np.dot(U, np.dot(S, VH))
        t = mu2 - np.dot(R, mu1)

        errors = markers2 - np.dot(R,markers1) - t

        armse = np.sqrt( 1.0/n * np.sum(errors**2) )
        return armse, R, t


import numpy as np
import bluefog.torch as bf
import torch
from bluefog.common import topology_util
import networkx as nx
np.set_printoptions(precision=3, suppress=True, linewidth=200)

bf.init()
network_size = bf.size()

G = topology_util.ExponentialTwoGraph(network_size)
W = np.zeros((network_size, network_size))

for rank in range(network_size):
    self_weight, neighbor_weights = topology_util.GetRecvWeights(G, rank)
    W[rank,rank] = self_weight
    for r, v in neighbor_weights.items():
        W[rank, r] = v

if bf.rank() == 0:
    print(W)
    print()
    print('The sum of each col is:', np.sum(W,axis=0))
    print('The sum of each row is:', np.sum(W,axis=1))
    if np.sum(W, axis=1).all() and np.sum(W, axis=0).all():
        print('W is doubly stochastic.')

    w, _ = np.linalg.eig(W)
    w = np.abs(w)
    w_sorted = np.sort(w)
    print('The second largest eigenvalue is:', w_sorted[-2])


"""
Q3: Fourier filtering and smoothing
Plot the actual data, then calculate Fourier coefficients
Then plot the transformed data
"""

import numpy as np
import matplotlib.pyplot as plt

# import the Dow data
dow = np.loadtxt("dow.txt", float)

# plot it all onto a graph
dow_l = np.arange(len(dow))

plt.figure(1)
plt.title('Closing values of DOW index per day')
plt.xlabel('Number of days since start of data set')
plt.ylabel('Closing value')
plt.xlim(0,(len(dow)))
plt.plot(dow_l, dow, 'g', label='Unprocessed data')
# we won't display this plot yet, since we want to add to it later

# B
# calculate Fourier coefficients
dowft = np.fft.rfft(dow)
#print dowft #and, oh! look. they're complex.

# C
# what point do we cut off for 10%? LET'S FIND OUT
dowft_l = len(dowft)
dowft_10pc_l = (((dowft_l)//10)) # no need to +1 because we're operating on zero

dowft_f10 = np.copy(dowft)
dowft_f10[(dowft_10pc_l):(dowft_l)] = 0.0 # set all values past the 1/10th point to zero

# D
# inverse transform
inv_dowft_f10 = np.fft.irfft(dowft_f10)
# plot
plt.plot(dow_l,inv_dowft_f10, 'r', label='Inverse transform, 10% of coefficients') # plot individual ones too for clarity
plt.legend(loc=0)
plt.show()

# E
# what point do we cut off for 2%? LET'S FIND OUT
dowft_2pc_l = (((dowft_l)//50))

dowft_f2 = np.copy(dowft)
dowft_f2[(dowft_2pc_l):(dowft_l)] = 0.0 # set all values past the 1/10th point to zero

# inverse transform again
inv_dowft_f2 = np.fft.irfft(dowft_f2)

# plot
plt.figure(2)
plt.plot(dow_l,dow,'g', label='Unprocessed data')
plt.plot(dow_l,inv_dowft_f2, 'b', label='Inverse transform, 2% of coefficients')
plt.title('Closing values of DOW index per day')
plt.xlabel('Number of days since start of data set')
plt.ylabel('Closing value')
plt.xlim(0,(len(dow)))
plt.legend(loc=0)
plt.show()

#plot the whole lot too
plt.figure(3)
plt.plot(dow_l,dow,'g', label='Unprocessed data')
plt.plot(dow_l,inv_dowft_f10, 'r', label='Inverse transform, 10% of coefficients') # plot individual ones too for clarity
plt.plot(dow_l,inv_dowft_f2, 'b', label='Inverse transform, 2% of coefficients')
plt.title('Closing values of DOW index per day')
plt.xlabel('Number of days since start of data set')
plt.ylabel('Closing value')
plt.legend(loc=0)
plt.xlim(0,(len(dow)))
plt.show()