import numpy as np
import pandas as pd

def dft(time_series):  # don't call this
    '''DFT   Discrete Fourier transform
    DFS(time_series) computes the Discrete Fourier transform of an input
    time-series -- time_series.
    Uses numpy.fft method

    :param time_series:  must be a vector with a length N that is >1 and odd.
    :returns alpha0: mean of the time-series
    :returns table: a pandas dataframe containging alpha, beta, power
        table.alpha  = coefficients of cosine terms for k=1:Nyquist
        table.beta   = coefficients of sine terms for k=1:Nyquist
        table.power  = normalised power-spectrum of the time-series
    '''
    N = len(time_series)
    if N % 2 == 0:
        # print('even')
        time_series = time_series[0:-1]
        N -= 1
    elif N <= 1:
        raise ValueError("ValueError: length must be greater than 1")
    # print(time_series)
    # Ensure that time_series is a column vector
    if len(np.array(time_series).shape) != 1:
        raise ValueError("ValueError: needs to be 1D")

    G = np.fft.fft(time_series)

    return G

def dfs(time_series):  # yes
    '''DFS   Discrete Fourier series
    DFS(time_series) computes the Discrete Fourier series of an input
    time-series -- time_series.

    :param time_series:  must be a vector with a length N that is >1 and odd.
    :returns alpha0:mean of the time-series
    :returns table: a pandas dataframe containging alpha, beta, power
        table.alpha  = coefficients of cosine terms for k=1:(N-1)/2
        table.beta   = coefficients of the sine terms for k=1:(N-1)/2
        table.power  = normalised power-spectrum of the time-series
    '''

    # Check length of time-series.  Must be odd and longer than 1

    N = len(time_series)
    # print(N)
    if N % 2 == 0:
        # print('even')
        time_series = time_series[0:-1]
        N -= 1
    elif N <= 1:
        raise ValueError("ValueError: length must be greater than 1")

    # Ensure that time_series is a column vector
    if len(np.array(time_series).shape) != 1:
        raise ValueError("ValueError: needs to be 1D")

    # Create index of time-series entries
    j = np.arange(0, N)

    # Calculate the coefficients at each frequenctime_series
    G = np.zeros((int((N-1)/2), 2))
    # 2 = sin, cos
    # N = number of timesteps (5)
    # k_max = int(N-1)/2 = maximum harmonic number
    alpha0 = np.mean(time_series)
    for k in range(1, 1+int((N-1)/2)):
        C = np.cos(2*np.pi*j*k/N)
        S = np.sin(2*np.pi*j*k/N)
        Z = np.vstack((C, S))
        # print time_series
        G[k-1, :] = (2./N) * np.matmul(Z, time_series - alpha0)
    # Assemble structure containing results
    alpha = G[:, 0]
    beta = G[:, 1]
    power = 0.5 * (alpha.T**2 + beta.T**2)/np.var(time_series, ddof=1)
    d = {'alpha': alpha, 'beta': beta, 'power': power}
    table = pd.DataFrame(d)
    return alpha0, table

def fourier_to_freq_spectrum(time_series, test_time):  # yes
    '''fourier_to_freq_spectrum
    :param time_series: time-series data.
        must be a vector with a length N that is >1 and odd.
    :param test_time: times corresponding to time-series data
    :returns alpha0: mean of the time-series
    '''
    sampling_rate = test_time[1]-test_time[0]
    fourier_transform = dft(time_series)
    abs_fourier_transform = np.abs(fourier_transform)
    power_spectrum = np.square(abs_fourier_transform)
    frequency = np.linspace(0, sampling_rate/2, len(power_spectrum))
    # pd dataframe columns freq, power
    d = {'frequency': frequency, 'power': power_spectrum}
    df = pd.DataFrame(d)
    return df

def fourier_to_coefficients(time_series):  # yes
    '''
    :param time_series:  must be a vector with a length N that is >1 and odd.
    :returns alpha0: mean of the time-series
    :returns table: a pandas dataframe containging alpha, beta, power
        table.alpha  = coefficients of cosine terms for k=1:Nyquist
        table.beta   = coefficients of sine terms for k=1:Nyquist
        table.power  = normalised power-spectrum of the time-series
    '''
    N = len(time_series)
    if N % 2 == 0:
        # print('even')
        time_series = time_series[0:-1]
        N -= 1
    elif N <= 1:
        raise ValueError("ValueError: length must be greater than 1")
    G = dft(time_series)

    # freq = np.fft.fftfreq(len(y), t[1] - t[0])
    N = len(time_series)
    Nu = int(np.ceil((N+1)/2))
    G = G[0:Nu]

    alpha0 = np.real(G[0])/N
    alpha = 2*np.real(G[1:])/N
    beta = -2*np.imag(G[1:])/N
    power = 0.5*(alpha**2 + beta**2)/np.var(time_series, ddof=1)

    d = {'alpha': alpha, 'beta': beta, 'power': power}
    table = pd.DataFrame(d)
    return alpha0, table

def fourier_approx(alpha0, table, data, k=[]):  # yes, total approx
    ''' fourier_approx
    calculates approximated data values

    :param alpha0: mean of time series
    :param alpha: coefficients of cosine terms for k=1:(N-1)/2
    :param beta: coefficients of the sine terms for k=1:(N-1)/2
    :param data: 1 time-series of data
    :param k: harmonic numbers associated with alpha and beta
    :return approximation: approximated time series
    '''
    alpha = table.alpha
    beta = table.beta
    N = len(data)
    if len(k) == 0:
        k = np.arange(1, len(alpha)+1)
    approximation = np.zeros(N)
    for j in range(N):
        approximation[j] = alpha0 + np.sum(
            alpha*np.cos(2.*np.pi*k/N * j)
            + beta*np.sin(2.*np.pi*k/N * j))
    # print(approximation)
    # print(approximation.size)
    return approximation

def calc_residuals(alpha0, table, data, data_times, components=0):
    '''calc_residuals
    plots the top contributing components to the approximation
    :param alpha0: mean of time series
    :param table: a pandas dataframe containging alpha, beta, power
    :param data: time-series data
    :param data_times: times corresponding to the time-series data
    :param components: number of components to include in approximation.
        If not entered then uses optimise_residuals to find best value to use.
    '''
    # time series
    # print(table)
    # print(type(table))
    if components == 0:
        components = optimise_residuals(alpha0, table, data)
    # print("number of components being used: " + str(components))
    top_indices = np.flip(np.argsort(np.array(table.power))[-components:])
    top_alpha = [table.alpha[i] for i in top_indices]
    top_beta = [table.beta[i] for i in top_indices]
    top_power = [table.power[i] for i in top_indices]
    top_table = pd.DataFrame(
        {'alpha': top_alpha, 'beta': top_beta, 'power': top_power}
        )
    approximation = fourier_approx(
        alpha0, top_table, data, k=top_indices+1
        )
    # difference
    residual = data-approximation
    residual_df = pd.DataFrame(
        {'time': data_times, 'raw_data': data, 'approx': approximation, 'residuals': residual}
        )
    # time, data, residual column
    # plt.plot(residual)
    # plt.plot(data)
    # plt.show()
    # plot residual against components
    N = len(data)
    top_components_for_approx_dict = {'times': data_times}
    freq = {}
    amplitude = {}
    power = {}
    for i in range(0, components):
        y = (
            alpha0
            + top_alpha[i]*np.cos(2.*np.pi*top_indices[i]/N * np.arange(0, N))
            + top_beta[i]*np.sin(2.*np.pi*top_indices[i]/N * np.arange(0, N))
        )
        # print('y', y)
        freq['Comp. '+str(i+1)] = round(2.*np.pi*top_indices[i]/N, 2)
        amplitude['Comp. '+str(i+1)] = round(max(y)-np.mean(y), 2)
        power['Comp. '+str(i+1)] = round(top_power[i], 2)
        top_components_for_approx_dict['comp.'+str(i+1)] = y
        # plt.plot(y)
    # plt.show()
    top_components_for_approx = pd.DataFrame(
        top_components_for_approx_dict
        )  # one column for time, then a column for each component
    summary_table = pd.DataFrame(
        {'frequency': freq, 'amplitude': amplitude, 'power': power}
        )
    return top_components_for_approx, summary_table, residual_df
    # THINGS YOU COULD RETURN
    # top_components_for_approx -
    #   pd.df with one column for time, then a column for each key component
    # summary_table -
    #   pd.df indexed by component, columns for frequency, amplitude, power
    # residual_df -
    #   pd.df with columns for time, data, residual

def optimise_residuals(alpha0, table, data):  # no
    '''optimise_residuals
    find the minimum number of components to use
        to suitably approximate the data
    :param alpha0: mean of time series
    :param table: a pandas dataframe containging alpha, beta, power
    :param data: time-series data
    :returns best_index: number of components to include in approximation.
    '''
    # print('optimising residuals')
    mean_residual = []
    x = np.arange(1, len(table.power), 1)
    for components in x:
        top_indices = np.argsort(np.array(table.power))[-components:]
        top_alpha = [table.alpha[i] for i in top_indices]
        top_beta = [table.beta[i] for i in top_indices]
        top_power = [table.power[i] for i in top_indices]
        top_table = pd.DataFrame(
            {'alpha': top_alpha, 'beta': top_beta, 'power': top_power}
            )
        approximation = fourier_approx(
            alpha0, top_table, data, top_indices+1
            )
        residual = data - approximation
        mean_residual.append(np.mean(abs(residual)))
    diff = np.gradient(np.gradient(mean_residual))
    sorted_indices = np.argsort(diff)
    if abs(max(mean_residual, key=abs)) > 1:
        best_index = len(sorted_indices)-1
    else:
        for i in sorted_indices:
            if abs(mean_residual[i]) < 1:
                best_index = i
                break
    return best_index

# example usage without Bokeh
# dataframe = pd.read_csv('bokeh_app/data/test_timeseries_noisy.csv')

# selected_column = 'y'

# time = dataframe['time'].values.tolist()
# data = dataframe[selected_column].values.tolist()

# This is where you pass it the appropriate data

# given name of header of pd

# alpha0, table = dfs(data)
# frequency, power_spectrum = fourier_to_freq_spectrum(data, time)
# print(len(frequency), len(power_spectrum))

# calc_residuals(alpha0, table, data, time, components = 0)

# plt.plot(data)
# plt.plot(fourier_approx(alpha0, table, data))
# plt.show()

# print(fourier_to_coefficients([1,2,3,4,5,6]))


from math import exp
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from sklearn.datasets import load_iris
from sklearn.model_selection import train_test_split
class LR():
    def __init__(self,max_iterator=200,learning_rate = 0.01):
        self.max_iterator = max_iterator
        self.learning_rate = learning_rate
    def sigmoid(self,x):
        return 1/(1+exp(-x))
    def data_add_one_dim(self,data_feature): #回归系数比特征数多一个，即常数项 故原来的特征向量统统在前面添加一个1
        #data_mat = []
        pad = np.ones((len(data_feature),1),dtype=np.float32) #直接用numpy的水平拼接
        data_mat = np.hstack((pad,data_feature))
        #for d in range(len(data_feature)):
         #   data_mat.append([1.0,*data_feature[d]])
        return data_mat
    def fit(self,x,y):
        data_feature = self.data_add_one_dim(x)
        data_label = y
        self.weights = np.zeros((len(data_feature[0]),1),dtype=np.float32)
        for ite in range(self.max_iterator):
            for i in range(len(data_feature)):
                result = self.sigmoid(np.dot(data_feature[i],self.weights))
                error = data_label[i] - result
                self.weights += self.learning_rate * error * np.transpose([data_feature[i]])
        print('LogisticRegression Model(learning_rate={},max_iter={})'.format(self.learning_rate, self.max_iterator))
    def score(self, X_test, y_test):
        right = 0
        X_test = self.data_add_one_dim(X_test)
        for x, y in zip(X_test, y_test):
            result = np.dot(x, self.weights)
            if (result > 0 and y == 1) or (result < 0 and y == 0):
                right += 1
        return right / len(X_test)


from scipy import stats
import matplotlib.pyplot as plt

'''Linear regression uses the relationship between
the data-points to draw a straight line through all them.
'''

# x is car age, y is speed
x = [5, 7, 8, 7, 2, 17, 2, 9, 4, 11, 12, 9, 6]
y = [99, 86, 87, 88, 111, 86, 103, 87, 94, 78, 77, 85, 86]

slope, intercept, r, p, std_err = stats.linregress(x, y)

def myfunc(x):
    return slope * x + intercept

mymodel = list(map(myfunc, x))

# draw simple x, y graph using above arrays
plt.scatter(x, y)
plt.xlabel("car age")
plt.ylabel("speed")
plt.plot(x, mymodel)
plt.show()

# R-squared
'''
The r-squared value ranges from 0 to 1, where 0 means no relationship,
and 1 means 100% related.
'''
print(r)

'''
The result -076 shows that there is a relationship, not perfect,
but it indicates that we could use linear regression in future predictions.
If the result is like 0.01, it indicates a very bad relationship,
and tells us that this data set is not suitable for linear regression.
'''

# predicting future values
speed = myfunc(10)

print(speed)


import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
from linear_classification.basic_logistic_unit import sigmoid

sns.set()

def cross_entropy(y, y_hat):
    E = 0
    for i in range(len(y)):
        if y[i] == 1:
            E -= np.log(y_hat[i])
        else:
            E -= np.log(1-y_hat[i])
    return E

if __name__ == '__main__':
    number_of_samples = 100
    dimensions = 2
    X = np.random.randn(number_of_samples, dimensions)

    # Center the first 50 points at (-2,-2)
    X[:50,:] = X[:50,:] - 2*np.ones((50,dimensions))

    # Center the second 50 points at (2,2)
    X[50:,:] = X[50:,:] + 2*np.ones((50,dimensions))

    y = np.array([0]*50 + [1]*50)

    X = np.concatenate((np.ones((number_of_samples, 1)), X), axis=1 )

    W = np.random.randn(dimensions+1)
    z = X.dot(W)

    y_hat = sigmoid(z)

    print("Cross Entropy:", cross_entropy(y, y_hat))
    w_closed_form = np.array([0, 4, 4])
    y_hat_closed_form = sigmoid(X.dot(w_closed_form))
    print("Cross Entropy (closed form):", cross_entropy(y, y_hat_closed_form))

    plt.figure()
    plt.scatter(X[:,1], X[:,2], c=y, s=100, alpha=0.5)
    x_axis = np.linspace(-6, 6, 100)
    y_axis = -x_axis
    plt.plot(x_axis, y_axis, '--r')
    plt.show()


# -*- coding: utf-8 -*-
"""
@author: Diego Pérez Reyes
"""

import numpy as np

"""
Convertidor coordenadas cartesianas a polares
"""
def convert_car(x,y):
    r = np.sqrt(x ** 2 + y ** 2)

    if x != 0:
        theta = np.arctan(y/x)
    else:
        theta = np.pi / 2
        if y < 0:
            theta *= 3

    return round(r,2), round(theta,2)

#Ejemplo
#print(convert_car(0, 1))

"""
Convertidor coordenadas polares a cartesianas
"""
def convert_pol(r,theta):
    x = r * np.cos(theta)
    y = r * np.sin(theta)

    return round(x,2), round(y,2)

#Ejemplo
#print(convert_pol(2, (11 * np.pi) / 6))


import numpy as np
import matplotlib.pyplot as plt

# Write with LaTeX
from matplotlib import rc
rc('text', usetex=True)
rc('font', family='serif')

# State the Problem

''' We will solve the problem of throwing a point-like sphere with initial position:

(x_o, y_o) = (0 [m], 0 [m]) ,

initial velocity:

v_o = 20 [m/s]

and throwing angle:

phi = pi/6 [rad]. '''

''' So the problem is seperated to the following:

1. dv_x/dt = 0 = func_v_x(t, x, y, v_x, v_y),
2. dv_y/dt = -g = func_v_y(t, x, y, v_x, v_y),
3. dx/dt = v_x = func_x(t, x, y, v_x, v_y),
4. dy/dt = v_y = func_y(t, x, y, v_x, v_y). '''

def func_x(t, x, y, v_x, v_y):
    return v_x

def func_v_x(t, x, y, v_x, v_y):
    return 0

def func_y(t, x, y, v_x, v_y):
    return v_y

def func_v_y(t, x, y, v_x, v_y):
    g = 10 # Gravitational Intensity Near Earth
    return -g

def shot_Euler(τ):

    # τ : Time Step
    t = np.zeros(200)

    y, v_y = np.zeros(200), np.zeros(200)
    x, v_x = np.zeros(200), np.zeros(200)

    # Errors
    e_x, e_y = np.zeros(200), np.zeros(200)

    # Initial Conditions
    t[0] = 0
    y[0], v_y[0] = 0, 10
    x[0], v_x[0] = 0, 10 * np.sqrt(3)

    i = 0 # Counter
    while y[i-1] >= 0:
        # x - Direction
        x[i+1] = x[i] + τ*func_x(t, x, y, v_x, v_y)[i]
        v_x[i+1] = v_x[i] + τ*func_v_x(t[i], x[i], y[i], v_x[i], v_y[i])
        # y - Direction
        y[i+1] = y[i] + τ*func_y(t, x, y, v_x, v_y)[i]
        v_y[i+1] = v_y[i] + τ*func_v_y(t[i], x[i], y[i], v_x[i], v_y[i])
        # Time
        t[i+1] = t[i] + τ

        # Errors
        e_x[i] = 10*np.sqrt(3)*t[i] - x[i]
        e_y[i] = -5*t[i]**2 + 10*t[i] - y[i]
        # Counter
        i += 1

    # We Kepp Values != 0 for x and y respectively
    y, x = y[:i], x[:i]
    t = t[:i]
    e_x, e_y = e_x[:i], e_y[:i]

    return x, y, t, e_x, e_y

if __name__ == '__main__':

    Exy, (Ex, Ey) = plt.subplots(1, 2)

    Exy.suptitle(r'Results from Euler Method')

    Ex.plot(shot_Euler(0.05)[2], shot_Euler(0.05)[0], color='black', linewidth=1)
    # How Graph is Shown (axis titles)
    Ex.set_ylabel(r'$x(t)$ $[m]$')
    Ex.set_xlabel(r'$t$ $[s]$')
    # Show the major grid lines
    Ex.grid(b=True, which='major', color='#666666', linestyle='--')
    # Show the minor grid lines
    Ex.minorticks_on()
    Ex.grid(b=True, which='minor', color='#999999', linestyle='--', alpha=0.1)

    Ey.plot(shot_Euler(0.05)[2], shot_Euler(0.05)[1], color='red', linewidth=1)
    # How Graph is Shown (axis titles)
    Ey.set_ylabel(r'$y(t)$ $[m]$')
    Ey.set_xlabel(r'$t$ $[s]$')
    # Show the major grid lines
    Ey.grid(b=True, which='major', color='#666666', linestyle='--')
    # Show the minor grid lines
    Ey.minorticks_on()
    Ey.grid(b=True, which='minor', color='#999999', linestyle='--', alpha=0.1)

    # Fix Quality
    Exy.tight_layout()
    #Save the Graph
    Exy.savefig('Euler-Method-Results.pdf')

    Eexyt, (Eext, Eeyt) = plt.subplots(1, 2)

    Eexyt.suptitle('Erros from Euler Method with Respect to Time')

    Eext.plot(shot_Euler(0.05)[2], shot_Euler(0.05)[3], color='b', linewidth=1)
    # How Graph is Shown (axis titles)
    Eext.set_ylabel(r'$e_x(t)$ $[m]$')
    Eext.set_xlabel(r'$t$ $[s]$')
    # Show the major grid lines
    Eext.grid(b=True, which='major', color='#666666', linestyle='--')
    # Show the minor grid lines
    Eext.minorticks_on()
    Eext.grid(b=True, which='minor', color='#999999', linestyle='--', alpha=0.1)

    Eeyt.plot(shot_Euler(0.05)[2], shot_Euler(0.05)[4], color='m', linewidth=1)
    # How Graph is Shown (axis titles)
    Eeyt.set_ylabel(r'$e_y(t)$ $[m]$')
    Eeyt.set_xlabel(r'$t$ $[s]$')
    # Show the major grid lines
    Eeyt.grid(b=True, which='major', color='#666666', linestyle='--')
    # Show the minor grid lines
    Eeyt.minorticks_on()
    Eeyt.grid(b=True, which='minor', color='#999999', linestyle='--', alpha=0.1)

    # Fix Quality
    Eexyt.tight_layout()
    #Save the Graph
    Eexyt.savefig('Erros-with-Respect-to-Time.pdf')

    # Sfalmata Synarthsei toy τ (Bhma)
    Eulere_x, Eulere_y = np.zeros(200), np.zeros(200)
    bhma = np.linspace(0.02, 1, 100)

    i = 0  # Metrhths
    for τ in bhma:
        Eulere_x[i] = shot_Euler(τ)[3][-1]
        Eulere_y[i] = shot_Euler(τ)[4][-1]
        i += 1

    Eulere_x = Eulere_x[:i]
    Eulere_y = Eulere_y[:i]

    Eexyτ, (Eexτ, Eeyτ) = plt.subplots(1, 2)

    Eexyτ.suptitle('Erros from Euler Method with Respect to Time Step')

    Eexτ.plot(bhma, Eulere_x, color='g', linewidth=1)
    # How Graph is Shown (axis titles)
    Eexτ.set_ylabel(r'$e_x(\tau)$ $[m]$')
    Eexτ.set_xlabel(r'$\tau$ $[s]$')
    # Show the major grid lines
    Eexτ.grid(b=True, which='major', color='#666666', linestyle='--')
    # Show the minor grid lines
    Eexτ.minorticks_on()
    Eexτ.grid(b=True, which='minor', color='#999999', linestyle='--', alpha=0.1)

    Eeyτ.plot(bhma, Eulere_y, color='c', linewidth=1)
    # How Graph is Shown (axis titles)
    Eeyτ.set_ylabel(r'$e_y(\tau)$ $[m]$')
    Eeyτ.set_xlabel(r'$\tau$ $[s]$')
    # Show the major grid lines
    Eeyτ.grid(b=True, which='major', color='#666666', linestyle='--')
    # Show the minor grid lines
    Eeyτ.minorticks_on()
    Eeyτ.grid(b=True, which='minor', color='#999999', linestyle='--', alpha=0.1)

    # Fix Quality
    Eexyτ.tight_layout()
    #Save the Graph
    Eexyτ.savefig('Erros-with–Respect-to-Time-Step.pdf')


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Wed Sep 16 21:29:16 2020

@author: zhwa2432
"""

import numpy as np
import scipy.fftpack
import matplotlib.pyplot as plt
from scipy.signal import savgol_filter

N = 100
x = np.linspace(0,2*np.pi,N)
y = np.sin(x) + np.random.random(N) * 0.2

w = scipy.fftpack.rfft(y)
f = scipy.fftpack.rfftfreq(N, x[1]-x[0])
spectrum = w**2

cutoff_idx = spectrum < (spectrum.max()/5)
w2 = w.copy()
w2[cutoff_idx] = 0

y2 = scipy.fftpack.irfft(w2)+0.06

plt.plot(x,y, label='Orignial')  # Plot some data on the (implicit) axes.
plt.plot(x,y2, label='Smoothed')

#####
x = np.linspace(0,2*np.pi,100)
y = np.sin(x) + np.random.random(100) * 0.8

def smooth(y, box_pts):
    box = np.ones(box_pts)/box_pts
    y_smooth = np.convolve(y, box, mode='same')
    return y_smooth

plt.figure(0)  # start a new window
plt.plot(x, y,'o')
plt.plot(x, smooth(y,3), 'r-', lw=2)
plt.plot(x, smooth(y,19), 'g-', lw=2)

###
yhat = savgol_filter(y, 51, 3) # window size 51, polynomial order 3

plt.plot(x,y)
plt.plot(x,yhat, color='red',lw=3)
plt.show()

import pylab as plt
import statsmodels.api as sm

x = np.linspace(0,2*np.pi,100)
y = np.sin(x) + np.random.random(100) * 0.2
lowess = sm.nonparametric.lowess(y, x, frac=0.1)

plt.plot(x, y, '+')
plt.plot(lowess[:, 0], lowess[:, 1])
plt.show()

import numpy,import scipy,from scipy,import statsmodels


'''
Loss functions and their gradients
crysx_nn
Manas Sharma
feedback@bragitoff.com
https://www.bragitoff.com
'''
from numba import vectorize,jit,njit,prange,set_num_threads,get_num_threads
import numpy as np
try:
    import cupy as cp
except ImportError:
    print('Warning: CuPy could not be imported! You can only use CPU for computations.')

@njit(cache=True,fastmath=False, parallel=True)
def MSE_loss(outi, out0):
    """
    Computes Mean Squared error/loss between targets
    and predictions.
    Input: predictions (N, k) ndarray (N: no. of samples, k: no. of output nodes)
          targets (N, k) ndarray        (N: no. of samples, k: no. of output nodes)
    Returns: scalar
    Note: The averaging is only done over the output nodes and not over the samples in a batch.
    Therefore, to get an answer similar to PyTorch, one must divide the result by the batch size.
    """
    #TODO : There is big problem that I noticed today. For large arrays, the NUmba compiled version of this
    # doesn't give correct results.
    # but then the pure numpy version is not as fast as Numba.
    # Alternative 1
    # return np.sum((outi-out0)**2)/outi.shape[1]
    # Alternative 2
    # return np.mean((outi - out0)**2)*outi.shape[0]
    # Alternative 3 (Extremely fast if there are a lot of cores available)
    loss = 0.0
    for i in prange(outi.shape[0]):
        for j in prange(outi.shape[1]):
            loss = loss + (outi[i,j] - out0[i,j])**2 # should have given a race condition but somehow numba is able to avoid it
    # Average over number of output nodes
    loss = loss / outi.shape[1]
    return loss

@njit(cache=True,fastmath=True)
def MSE_loss_grad(outi, out0):
    """
    Computes mean squared error gradient between targets
    and predictions.
    Input: predictions (N, k) ndarray (N: no. of samples, k: no. of output nodes)
          targets (N, k) ndarray        (N: no. of samples, k: no. of output nodes)
    Returns: (N,k) ndarray
    Note: The averaging is only done over the output nodes and not over the samples in a batch.
    Therefore, to get an answer similar to PyTorch, one must divide the result by the batch size.
    """

    return 2*(outi-out0)/outi.shape[1]

@njit(cache=True,fastmath=False, parallel=True)
def MAE_loss(predictions, targets):
    """
    Computes Mean Absolute error/loss between targets
    and predictions.
    Input: predictions (N, k) ndarray (N: no. of samples, k: no. of output nodes)
          targets (N, k) ndarray        (N: no. of samples, k: no. of output nodes)
    Returns: scalar
    Note: The averaging is only done over the output nodes and not over the samples in a batch.
    Therefore, to get an answer similar to PyTorch, one must divide the result by the batch size.
    """
    # Alternative 1
    # return np.sum(np.abs(predictions-targets))/predictions.shape[1]
    # Alternative 2
    # return np.mean(np.abs(predictions - targets))*predictions.shape[0]
    # Alternative 3 (Extremely fast if there are a lot of cores available)
    loss = 0.0
    for i in prange(predictions.shape[0]):
        for j in prange(predictions.shape[1]):
            loss = loss + np.abs(predictions[i,j] - targets[i,j]) # should have given a race condition but somehow numba is able to avoid it
    # Average over number of output nodes
    loss = loss / predictions.shape[1]
    return loss

@njit(cache=True,fastmath=True,parallel=False)
def MAE_loss_grad(predictions, targets):
    """
    Computes mean absolute error gradient between targets
    and predictions.
    Input: predictions (N, k) ndarray (N: no. of samples, k: no. of output nodes)
          targets (N, k) ndarray        (N: no. of samples, k: no. of output nodes)
    Returns: (N,k) ndarray
    Note: The averaging is only done over the output nodes and not over the samples in a batch.
    Therefore, to get an answer similar to PyTorch, one must divide the result by the batch size.
    """
    # References: https://stats.stackexchange.com/questions/313235/can-i-use-gradient-descent-for-least-absolute-deviation-regression
    # https://stats.stackexchange.com/questions/312737/mean-absolute-error-mae-derivative
    # Alternative 1
    loss_grad = np.where(predictions >= targets, 1.0, -1.0)
    return loss_grad/predictions.shape[1]
    # Alternative 2 (The following is wrong (only returns a scalar). Needs to be corrected (return an (N,k) array). )
    loss_grad = 0.0
    for i in prange(predictions.shape[0]):
        for j in prange(predictions.shape[1]):
            # if np.abs(predictions[i,j] - targets[i,j])<0.0000001:
            #     loss_grad = loss_grad + 0.0 # should have given a race condition but somehow numba is able to avoid it
            if predictions[i,j] < targets[i,j]:
                loss_grad = loss_grad + (-1.0) # should have given a race condition but somehow numba is able to avoid it
            elif predictions[i,j] > targets[i,j]:
                loss_grad = loss_grad + (1.0) # should have given a race condition but somehow numba is able to avoid it
    # Average over number of output nodes
    loss_grad = loss_grad / predictions.shape[1]
    return loss_grad

@njit(cache=True,fastmath=True, parallel=True)
def BCE_loss(predictions, targets, epsilon=1e-7):
    """
    Computes binary cross entropy between targets (encoded as one-hot vectors)
    and predictions.
    Input: predictions (N, k) ndarray (N: no. of samples, k: no. of output nodes)
          targets (N, k) ndarray        (N: no. of samples, k: no. of output nodes)
    Returns: scalar
    Note: The averaging is only done over the output nodes and not over the samples in a batch.
    Therefore, to get an answer similar to PyTorch, one must divide the result by the batch size.
    """
    # Reference Formula: https://peltarion.com/knowledge-center/documentation/modeling-view/build-an-ai-model/loss-functions/binary-crossentropy
    # This should be averaged over outputs/output nodes/ classes
    predictions = np.clip(predictions, epsilon, 1. - epsilon)
#    N = predictions.shape[0]
    # Alternative 1 (This is probably as fast as the other one)
    # ce = -np.sum(targets*np.log(predictions+epsilon)+(1.-targets)*np.log(1.-predictions+epsilon))/predictions.shape[1]#/N
    # return ce
    # Alternative 2 (Parallelized)
    loss = 0.0
    for i in prange(predictions.shape[0]):
        for j in prange(predictions.shape[1]):
            temp = -(targets[i,j]*np.log(predictions[i,j]+epsilon) + (1.-targets[i,j])*np.log(1.-predictions[i,j]+epsilon))
            # temp = -(targets[i,j]*np.log(predictions[i,j]) + (1.-targets[i,j])*np.log(1.-predictions[i,j]))
            loss = loss + temp # should have given a race condition but somehow numba is able to avoid it
    # Average over number of output nodes
    loss = loss / predictions.shape[1]
    return loss

# @njit(cache=True,fastmath=True, parallel=True)
def BCE_loss_grad(predictions, targets):
    """
    Computes binary cross entropy gradient between targets (encoded as one-hot vectors)
    and predictions.
    Input: predictions (N, k) ndarray (N: no. of samples, k: no. of output nodes)
          targets (N, k) ndarray        (N: no. of samples, k: no. of output nodes)
    Returns: (N,k) ndarray
    Note: The averaging is only done over the output nodes and not over the samples in a batch.
    Therefore, to get an answer similar to PyTorch, one must divide the result by the batch size.
    """
    # https://math.stackexchange.com/questions/2503428/derivative-of-binary-cross-entropy-why-are-my-signs-not-right
    # Alternative 1 (Converts NAN resulting from division by 0 to 0 using Numpy, is incompatible with NUMBA)
    return -(np.nan_to_num(np.divide(targets,predictions,dtype=targets.dtype))-np.nan_to_num(np.divide(1.-targets,1.-predictions,dtype=targets.dtype)))/predictions.shape[1]
    # Alternative 2 (Converts NAN resulting from division by 0 to 0 using Numpy, is compatible with NUMBA) !!! DOESNT WORK
    # grad = np.zeros(predictions.shape)
    # # grad = predictions
    # for i in prange(predictions.shape[0]):
    #     for j in prange(predictions.shape[1]):
    #         temp = targets[i,j]/predictions[i,j]  -  (1.-targets)/(1.-predictions[i,j])
    #         if not np.isnan(temp):
    #             grad[i,j] = temp
    # # Average over number of output nodes
    # grad = grad / predictions.shape[1]
    # return grad
    # Alternative 3 !! DOESNT WORK
    # grad = -(np.divide(targets,predictions)-np.divide(1.-targets,1.-predictions))/predictions.shape[1]
    # for i in prange(predictions.shape[0]):
    #     for j in prange(predictions.shape[1]):
    #         if np.isnan(grad[i,j]):
    #             grad[i,j]=0.
    # return grad

@njit(cache=True,fastmath=True)
def CCE_loss(predictions, targets, epsilon=1e-9):
    """
    Computes categorical cross entropy between targets (encoded as one-hot vectors)
    and predictions.
    Input: predictions (N, k) ndarray
          targets (N, k) ndarray
    Returns: scalar
    """
    # Reference: https://stackoverflow.com/questions/47377222/what-is-the-problem-with-my-implementation-of-the-cross-entropy-function
    # Formula: https://peltarion.com/knowledge-center/documentation/modeling-view/build-an-ai-model/loss-functions/categorical-crossentropy
    # No averaging over the no. of outputs/output nodes/classes is needed
    predictions = np.clip(predictions, epsilon, 1. - epsilon)
    cce = -np.sum(targets*np.log(predictions+epsilon))
    return cce

# @njit(cache=True,fastmath=True)
def CCE_loss_grad(predictions, targets, epsilon=1e-9):
    """
    Computes categorical cross entropy gradient between targets (encoded as one-hot vectors)
    and predictions.
    Input: predictions (N, k) ndarray
          targets (N, k) ndarray
    Returns: matrix
    """
    predictions = np.clip(predictions, epsilon, 1. - epsilon)
    # https://math.stackexchange.com/questions/2503428/derivative-of-binary-cross-entropy-why-are-my-signs-not-right
    return -np.nan_to_num(np.divide(targets,predictions,dtype=targets.dtype))
    # return -np.nan_to_num(np.divide(targets,predictions))
    # return -np.divide(targets,predictions)

##-----------CUPY---------------
def MSE_loss_cupy(outi, out0):
    """
    Computes Mean Squared error/loss between targets
    and predictions.
    Input: predictions (N, k) ndarray (N: no. of samples, k: no. of output nodes)
          targets (N, k) ndarray        (N: no. of samples, k: no. of output nodes)
    Returns: scalar
    Note: The averaging is only done over the output nodes and not over the samples in a batch.
    Therefore, to get an answer similar to PyTorch, one must divide the result by the batch size.
    """

    return cp.sum((outi-out0)**2)/outi.shape[1]

def MSE_loss_grad_cupy(outi, out0):
    """
    Computes mean squared error gradient between targets
    and predictions.
    Input: predictions (N, k) ndarray (N: no. of samples, k: no. of output nodes)
          targets (N, k) ndarray        (N: no. of samples, k: no. of output nodes)
    Returns: (N,k) ndarray
    Note: The averaging is only done over the output nodes and not over the samples in a batch.
    Therefore, to get an answer similar to PyTorch, one must divide the result by the batch size.
    """

    return 2*(outi-out0)/outi.shape[1]

def MAE_loss_cupy(predictions, targets):
    """
    Computes Mean Absolute error/loss between targets
    and predictions.
    Input: predictions (N, k) ndarray (N: no. of samples, k: no. of output nodes)
          targets (N, k) ndarray        (N: no. of samples, k: no. of output nodes)
    Returns: scalar
    Note: The averaging is only done over the output nodes and not over the samples in a batch.
    Therefore, to get an answer similar to PyTorch, one must divide the result by the batch size.
    """
    # Alternative 1
    return cp.sum(cp.abs(predictions-targets))/predictions.shape[1]
    # Alternative 2
    # return cp.mean(cp.abs(predictions - targets))*predictions.shape[0]

def MAE_loss_grad_cupy(predictions, targets):
    """
    Computes mean absolute error gradient between targets
    and predictions.
    Input: predictions (N, k) ndarray (N: no. of samples, k: no. of output nodes)
          targets (N, k) ndarray        (N: no. of samples, k: no. of output nodes)
    Returns: (N,k) ndarray
    Note: The averaging is only done over the output nodes and not over the samples in a batch.
    Therefore, to get an answer similar to PyTorch, one must divide the result by the batch size.
    """
    # References: https://stats.stackexchange.com/questions/313235/can-i-use-gradient-descent-for-least-absolute-deviation-regression
    # https://stats.stackexchange.com/questions/312737/mean-absolute-error-mae-derivative
    # Alternative 1
    loss_grad = cp.where(predictions >= targets, 1.0, -1.0)
    return loss_grad/predictions.shape[1]

def BCE_loss_cupy(predictions, targets, epsilon=1e-7):
    """
    Computes binary cross entropy between targets (encoded as one-hot vectors)
    and predictions.
    Input: predictions (N, k) ndarray (N: no. of samples, k: no. of output nodes)
          targets (N, k) ndarray        (N: no. of samples, k: no. of output nodes)
    Returns: scalar
    Note: The averaging is only done over the output nodes and not over the samples in a batch.
    Therefore, to get an answer similar to PyTorch, one must divide the result by the batch size.
    """
    predictions = cp.clip(predictions, epsilon, 1. - epsilon)
#    N = predictions.shape[0]
    ce = -cp.sum(targets*cp.log(predictions+epsilon)+(1-targets)*cp.log(1-predictions+epsilon))/predictions.shape[1]#/N
    return ce

def BCE_loss_grad_cupy(predictions, targets, epsilon=1e-7):
    """
    Computes binary cross entropy gradient between targets (encoded as one-hot vectors)
    and predictions.
    Input: predictions (N, k) ndarray (N: no. of samples, k: no. of output nodes)
          targets (N, k) ndarray        (N: no. of samples, k: no. of output nodes)
    Returns: (N,k) ndarray
    Note: The averaging is only done over the output nodes and not over the samples in a batch.
    Therefore, to get an answer similar to PyTorch, one must divide the result by the batch size.
    """
    # https://math.stackexchange.com/questions/2503428/derivative-of-binary-cross-entropy-why-are-my-signs-not-right
    predictions = cp.clip(predictions, epsilon, 1. - epsilon)
    # return -(cp.divide(targets,predictions)-cp.divide(1-targets,1-predictions))/predictions.shape[1]
    return -(cp.nan_to_num(cp.divide(targets,predictions,dtype=targets.dtype))-cp.nan_to_num(cp.divide(1.-targets,1.-predictions,dtype=targets.dtype)))/predictions.shape[1]

def CCE_loss_cupy(predictions, targets, epsilon=1e-9):
    """
    Computes categorical cross entropy between targets (encoded as one-hot vectors)
    and predictions.
    Input: predictions (N, k) ndarray
          targets (N, k) ndarray
    Returns: scalar
    """
    # Reference: https://stackoverflow.com/questions/47377222/what-is-the-problem-with-my-implementation-of-the-cross-entropy-function
    # Formula: https://peltarion.com/knowledge-center/documentation/modeling-view/build-an-ai-model/loss-functions/categorical-crossentropy
    # No averaging over the no. of outputs/output nodes/classes is needed
    predictions = cp.clip(predictions, epsilon, 1. - epsilon)
    cce = -cp.sum(targets*cp.log(predictions))
    return cce

def CCE_loss_grad_cupy(predictions, targets, epsilon=1e-8):
    """
    Computes cross entropy gradient between targets (encoded as one-hot vectors)
    and predictions.
    Input: predictions (N, k) ndarray
          targets (N, k) ndarray
    Returns: matrix
    """
    predictions = cp.clip(predictions, epsilon, 1. - epsilon)
    # https://math.stackexchange.com/questions/2503428/derivative-of-binary-cross-entropy-why-are-my-signs-not-right
    return -cp.divide(targets,predictions)


import numpy as np

def degrees_to_radians(degrees):
    """
    Converts degrees to radians
    @param: degrees
    """
    radians = (degrees/360.0)*2*np.pi
    return radians

def V2T( V ):
    """
    V2T converts 1x6 vector into 4x4 transformation matrix
    @param: V - 1x6 vector of form [x,y,z,rx,ry,rz] where x,y,z is the translation
    and rx,ry,rz is an angle-axis representation of the angle where the
    unit vector representing the axis has been multipled by the angle of
    rotation about it.
    @returns: T - a standard 4x4 transformation matrix
    """
    assert(V.shape == (1,6))
    T = np.eye(4)
    T[0:3, 3] = V[0, 0:3]
    T[0:3, 0:3] = V2R(V[0,3:6])
    return T

def V2R( V ):
    """
    V2R converts a 1x3 angle-axis vector into a 3x3 rotation matrix
    @param: V - 1x3 vector of form [rx,ry,rz] where rx,ry,rz is an angle-axis
    representation of the angle where the unit vector representing the axis
    has been multipled by the angle of rotation about it.
    @returns: R - a standard 3x3 transformation matrix
    """
    V = V.transpose()
    s = np.linalg.norm(V)
    if s == 0:
        R = eye(3)
    else:
        V = V[:] / s
        V = V.reshape(3,1)
        V = np.insert(V, 3, s)
        R = vrrotvec2mat(V)

    return R

def T2V( T ):
    """
    T2V converts 4x4 transformation matrix into a 1x6 vector
    @param: T - a standard 4x4 transformation matrix
    @returns:V - 1x6 vector of form [x,y,z,rx,ry,rz] where x,y,z is the translation
    and rx,ry,rz is an angle-axis representation of the angle where the
    unit vector representing the axis has been multipled by the angle of
    rotation about it
    """
    assert(T.shape == (4,4))
    V = np.zeros((1,6))
    V[0, 0:3] = T[0:3, 3]
    V[0, 3:6] = R2V(T[0:3, 0:3])

    return V

def R2V( R ):
    """
    R2V converts 3x3 rotation matrix into a 1x3 angle-axis vector
    @param: R - a standard 3x3 transformation matrix
    @returns: V - 1x3 vector of form [rx,ry,rz] where rx,ry,rz is an angle-axis
    representation of the angle where the unit vector representing the axis
    has been multipled by the angle of rotation about it
    """
    assert(R.shape == (3,3))
    R = vrrotmat2vec(R)
    V = R[0, 0:3] * R[0, 2]

    return V

def vrrotvec2mat(ax_ang):
    """
    Create a rotation matrix corresponding to the rotation around a general
    axis by a specified angle.
    """
    if ax_ang.ndim == 1:
        if np.size(ax_ang) == 5:
            ax_ang = np.reshape(ax_ang, (5, 1))
            msz = 1
        elif np.size(ax_ang) == 4:
            ax_ang = np.reshape(np.hstack((ax_ang, np.array([1]))), (5, 1))
            msz = 1
        else:
            raise Exception('Wrong Input Type')
    elif ax_ang.ndim == 2:
        if np.shape(ax_ang)[0] == 5:
            msz = np.shape(ax_ang)[1]
        elif np.shape(ax_ang)[1] == 5:
            ax_ang = ax_ang.transpose()
            msz = np.shape(ax_ang)[1]
        else:
            raise Exception('Wrong Inpuqt Type')
    else:
        raise Exception('Wrong Input Type')

    direction = ax_ang[0:3, :]
    angle = ax_ang[3, :]

    d = np.array(direction, dtype=np.float64)
    d /= np.linalg.norm(d, axis=0)
    x = d[0, :]
    y = d[1, :]
    z = d[2, :]
    c = np.cos(angle)
    s = np.sin(angle)
    tc = 1 - c

    mt11 = tc*x*x + c
    mt12 = tc*x*y - s*z
    mt13 = tc*x*z + s*y

    mt21 = tc*x*y + s*z
    mt22 = tc*y*y + c
    mt23 = tc*y*z - s*x

    mt31 = tc*x*z - s*y
    mt32 = tc*y*z + s*x
    mt33 = tc*z*z + c

    mtx = np.column_stack((mt11, mt12, mt13, mt21, mt22, mt23, mt31, mt32, mt33))

    inds1 = np.where(ax_ang[4, :] == -1)
    mtx[inds1, :] = -mtx[inds1, :]

    if msz == 1:
        mtx = mtx.reshape(3, 3)
    else:
        mtx = mtx.reshape(msz, 3, 3)

    return mtx

def vrrotmat2vec(mat1, rot_type='proper'):
    """
    Create an axis-angle np.array from Rotation Matrix:
    ====================

    @param mat:  The nx3x3 rotation matrices to convert
    @type mat:   nx3x3 numpy array

    @param rot_type: 'improper' if there is a possibility of
                      having improper matrices in the input,
                      'proper' otherwise. 'proper' by default
    @type  rot_type: string ('proper' or 'improper')

    @return:    The 3D rotation axis and angle (ax_ang)
                5 entries:
                   First 3: axis
                   4: angle
                   5: 1 for proper and -1 for improper
    @rtype:     numpy 5xn array

    """
    mat = np.copy(mat1)
    if mat.ndim == 2:
        if np.shape(mat) == (3, 3):
            mat = np.copy(np.reshape(mat, (1, 3, 3)))
        else:
            raise Exception('Wrong Input Typef')
    elif mat.ndim == 3:
        if np.shape(mat)[1:] != (3, 3):
            raise Exception('Wrong Input Typez')
    else:
        raise Exception('Wrong Input Type')

    msz = np.shape(mat)[0]
    ax_ang = np.zeros((5, msz))

    epsilon = 1e-12
    if rot_type == 'proper':
        ax_ang[4, :] = np.ones(np.shape(ax_ang[4, :]))
    elif rot_type == 'improper':
        for i in range(msz):
            det1 = np.linalg.det(mat[i, :, :])
            if abs(det1 - 1) < epsilon:
                ax_ang[4, i] = 1
            elif abs(det1 + 1) < epsilon:
                ax_ang[4, i] = -1
                mat[i, :, :] = -mat[i, :, :]
            else:
                raise Exception('Matrix is not a rotation: |det| != 1')
    else:
        raise Exception('Wrong Input parameter for rot_type')

    mtrc = mat[:, 0, 0] + mat[:, 1, 1] + mat[:, 2, 2]

    ind1 = np.where(abs(mtrc - 3) <= epsilon)[0]
    ind1_sz = np.size(ind1)
    if np.size(ind1) > 0:
        ax_ang[:4, ind1] = np.tile(np.array([0, 1, 0, 0]), (ind1_sz, 1)).transpose()

    ind2 = np.where(abs(mtrc + 1) <= epsilon)[0]
    ind2_sz = np.size(ind2)
    if ind2_sz > 0:
        # phi = pi
        # This singularity requires elaborate sign ambiguity resolution

        # Compute axis of rotation, make sure all elements >= 0
        # real signs are obtained by flipping algorithm below
        diag_elems = np.concatenate((mat[ind2, 0, 0].reshape(ind2_sz, 1),
                                     mat[ind2, 1, 1].reshape(ind2_sz, 1),
                                     mat[ind2, 2, 2].reshape(ind2_sz, 1)), axis=1)
        axis = np.sqrt(np.maximum((diag_elems + 1)/2, np.zeros((ind2_sz, 3))))
        # axis elements that are <= epsilon are set to zero
        axis = axis*((axis > epsilon).astype(int))

        # Flipping
        #
        # The algorithm uses the elements above diagonal to determine the signs
        # of rotation axis coordinate in the singular case Phi = pi.
        # All valid combinations of 0, positive and negative values lead to
        # 3 different cases:
        # If (Sum(signs)) >= 0 ... leave all coordinates positive
        # If (Sum(signs)) == -1 and all values are non-zero
        #   ... flip the coordinate that is missing in the term that has + sign,
        #       e.g. if 2AyAz is positive, flip x
        # If (Sum(signs)) == -1 and 2 values are zero
        #   ... flip the coord next to the one with non-zero value
        #   ... ambiguous, we have chosen shift right

        # construct vector [M23 M13 M12] ~ [2AyAz 2AxAz 2AxAy]
        # (in the order to facilitate flipping):    ^
        #                                  [no_x  no_y  no_z ]

        m_upper = np.concatenate((mat[ind2, 1, 2].reshape(ind2_sz, 1),
                                  mat[ind2, 0, 2].reshape(ind2_sz, 1),
                                  mat[ind2, 0, 1].reshape(ind2_sz, 1)), axis=1)

        # elements with || smaller than epsilon are considered to be zero
        signs = np.sign(m_upper)*((abs(m_upper) > epsilon).astype(int))

        sum_signs = np.sum(signs, axis=1)
        t1 = np.zeros(ind2_sz,)
        tind1 = np.where(sum_signs >= 0)[0]
        t1[tind1] = np.ones(np.shape(tind1))

        tind2 = np.where(np.all(np.vstack(((np.any(signs == 0, axis=1) == False), t1 == 0)), axis=0))[0]
        t1[tind2] = 2*np.ones(np.shape(tind2))

        tind3 = np.where(t1 == 0)[0]
        flip = np.zeros((ind2_sz, 3))
        flip[tind1, :] = np.ones((np.shape(tind1)[0], 3))
        flip[tind2, :] = np.copy(-signs[tind2, :])

        t2 = np.copy(signs[tind3, :])

        shifted = np.column_stack((t2[:, 2], t2[:, 0], t2[:, 1]))
        flip[tind3, :] = np.copy(shifted + (shifted == 0).astype(int))

        axis = axis*flip
        ax_ang[:4, ind2] = np.vstack((axis.transpose(), np.pi*(np.ones((1, ind2_sz)))))

    ind3 = np.where(np.all(np.vstack((abs(mtrc + 1) > epsilon, abs(mtrc - 3) > epsilon)), axis=0))[0]
    ind3_sz = np.size(ind3)
    if ind3_sz > 0:
        phi = np.arccos((mtrc[ind3]-1)/2)
        den = 2*np.sin(phi)
        a1 = (mat[ind3, 2, 1]-mat[ind3, 1, 2])/den
        a2 = (mat[ind3, 0, 2]-mat[ind3, 2, 0])/den
        a3 = (mat[ind3, 1, 0]-mat[ind3, 0, 1])/den
        axis = np.column_stack((a1, a2, a3))
        ax_ang[:4, ind3] = np.vstack((axis.transpose(), phi.transpose()))

    return ax_ang


import matplotlib.pyplot as plt
import numpy as np
from numpy.linalg import cholesky
from scipy.spatial.distance import cdist

def gaussSample(mu, sigma, n):
    A = cholesky(sigma)
    Z = np.random.normal(loc=0, scale=1, size=(len(mu), n))
    return np.dot(A, Z).T + mu

def mu_fun(x):
    return np.zeros_like(np.ravel(x))

def kfn(x, z):
    L = 1
    return 1*np.exp(-(cdist(x/L, z/L)**2)/2)

np.random.seed(10)
L = 1
xs = np.arange(-5, 5.2, 0.2)[..., np.newaxis]
ns = len(xs)
keps = 1e-8
color_1 = '#984ea3'
color_2 = '#e41a1c'
fig, ax1 = plt.subplots(1, 1, figsize=(9, 6))

# plot sampled functions from the prior
for i in range(3):
    mu = mu_fun(xs)
    sigma = kfn(xs, xs) + 1e-15 * np.eye(len(xs))
    fs = gaussSample(mu, sigma, 1)
    ax1.plot(np.ravel(xs), np.ravel(fs), color=color_1, linestyle='-', linewidth=2)

fig.savefig('../figures/gprDemoNoiseFreePrior.png')

# generate noise-less training data
Xtrain = np.array([[-4], [-3], [-2], [-1], [1]])
ftrain = np.sin(Xtrain)

K = kfn(Xtrain, Xtrain)
Ks = kfn(Xtrain, xs)
Kss = kfn(xs, xs) + keps * np.eye(len(xs))
Ki = np.linalg.inv(K)
postMu = mu_fun(xs)[..., np.newaxis] + np.dot(Ks.T, np.dot(Ki, (ftrain - mu_fun(Xtrain)[..., np.newaxis])))
postCov = Kss - np.dot(Ks.T, np.dot(Ki, Ks))

fig, ax2 = plt.subplots(1, 1, figsize=(9, 6))

mu = np.ravel(postMu)
S2 = np.diag(postCov)
f = np.concatenate([mu + 2 * np.sqrt(S2), np.flip(mu - 2 * np.sqrt(S2), axis=0)])
ax2.fill(np.concatenate([xs, np.flip(xs, axis=0)]), f, color=color_2, alpha=0.2)

for i in range(3):
    mu = np.ravel(postMu)
    sigma = postCov
    fs = gaussSample(mu, sigma, 1)
    ax2.plot(np.ravel(xs), np.ravel(fs), color=color_1, linestyle='-', linewidth=2)
    ax2.scatter(Xtrain, ftrain, color=color_1, marker='x', linewidth=12)
fig.savefig('../figures/gprDemoNoiseFreePost.png')

# Generate sequence of plots using subsets of data
XtrainAll = np.array([[-4], [-3], [-2], [-1], [4]])

fig, ax = plt.subplots(1, 5, figsize=(20, 4))

for n in range(len(XtrainAll)):
    Xtrain = XtrainAll[:n + 1]
    ftrain = np.sin(Xtrain)

    K = kfn(Xtrain, Xtrain)
    Ks = kfn(Xtrain, xs)
    Kss = kfn(xs, xs) + keps * np.eye(len(xs))
    Ki = np.linalg.inv(K)
    postMu = mu_fun(xs)[..., np.newaxis] + np.dot(Ks.T, np.dot(Ki, (ftrain - mu_fun(Xtrain)[..., np.newaxis])))
    postCov = Kss - np.dot(Ks.T, np.dot(Ki, Ks))

    mu = np.ravel(postMu)
    S2 = np.diag(postCov)
    f = np.concatenate([mu + 2 * np.sqrt(S2), np.flip(mu - 2 * np.sqrt(S2), axis=0)])
    ax[n].fill(np.concatenate([xs, np.flip(xs, axis=0)]), f, color=color_2, alpha=0.2)

    for i in range(3):
        mu = np.ravel(postMu)
        sigma = postCov
        fs = gaussSample(mu, sigma, 1)
        ax[n].plot(np.ravel(xs), np.ravel(fs), color=color_1, linestyle='-', linewidth=2)
        ax[n].scatter(Xtrain, ftrain, color=color_1, marker='x', linewidth=12)

    ax[n].set_title('N={}'.format(n + 1))
fig.savefig('../figures/gprDemoNoiseFreePost_subplots.png')


import numpy as np
import matplotlib.pyplot as plt
from matplotlib import rc
import pandas as pd
from scipy import integrate
from scipy.optimize import curve_fit

# Write with LaTeX
rc('text', usetex=True)
rc('font', family='serif')

#Read .csv file (the .csv files are produced with the Logger Pro Software in Lab)
file1 = pd.read_csv('Otto-1.csv', header=None)
file2 = pd.read_csv('Otto-2.csv', header=None)

v1 = file1[11][1:]
v1 = np.array([float(i)*0.1 for i in v1])
v2 = file2[11][1:]
v2 = np.array([float(i)*0.1 for i in v2])

p1 = file1[8][1:]
p1 = np.array([float(i)*10**3 for i in p1])
p2 = file2[8][1:]
p2 = np.array([float(i)*10**3 for i in p2])

# Integrals
W43_1 = integrate.simps(p1[180:186], v1[180:186], even="avg")
W21_1 = integrate.simps(p1[567:573], v1[567:573], even="avg")
W_1 = integrate.trapz(p1, v1)

W43_2 = integrate.simps(p2[145:153], v2[145:153], even='avg')
W21_2 = integrate.simps(p2[495:502], v2[495:502], even='avg')
W_2 = integrate.trapz(p2, v2)

# log Axis
logv43_1 = np.log(v1[180:186])
logp43_1 = np.log(p1[180:186])
logv21_1 = np.log(v1[567:573])
logp21_1 = np.log(p1[567:573])

logv43_2 = np.log(v2[145:153])
logp43_2 = np.log(p2[145:153])
logv21_2 = np.log(v2[495:502])
logp21_2 = np.log(p2[495:502])

# Fitting
def func(x, a, b):
    return (a * x) + b

popt43_1, pcov43_1 = curve_fit(func, logv43_1, logp43_1)
err43_1 = np.sqrt(np.diag(pcov43_1))
popt21_1, pcov21_1 = curve_fit(func, logv21_1, logp21_1)
err21_1 = np.sqrt(np.diag(pcov21_1))

popt43_2, pcov43_2 = curve_fit(func, logv43_2, logp43_2)
err43_2 = np.sqrt(np.diag(pcov43_2))
popt21_2, pcov21_2 = curve_fit(func, logv21_2, logp21_2)
err21_2 = np.sqrt(np.diag(pcov21_2))

# Fill Between
def f(x, a, b):
    return (a / (b+x))

popt1, pcov1 = curve_fit(f, v1[567:573], p1[567:573])
popt11, pcov11 = curve_fit(f, v1[180:186], p1[180:186])

popt2, pcov2 = curve_fit(f, v2[495:502], p2[495:502])
popt22, pcov22 = curve_fit(f, v2[145:153], p2[145:153])

x = np.linspace(0.00097, 0.00036, 100)

# P - V
fig1, ax1 = plt.subplots(2,1)

ax1[0].fill_between(x, f(x, *popt1), f(x, *popt11), color='royalblue', label="$W_1$", alpha=0.7)
ax1[0].scatter(v1, p1, color='black', marker=".")
ax1[0].plot(v1, p1, color = 'royalblue', lw=2)
ax1[0].set_ylabel("$P_1$ $(\\mathrm{kPa})$")
ax1[0].set_xlabel("$V_1$ $(10^{-4}\,\\mathrm{m^3})$")
ax1[0].set_title("First Cycle")
ax1[0].legend()
ax1[0].grid(b=True, which='major', color='#666666', linestyle='--')
ax1[0].minorticks_on()
ax1[0].grid(b=True, which='minor', color='#999999', linestyle='--', alpha=0.2)

ax1[1].fill_between(x, f(x, *popt1), f(x, *popt11), color='darkorange', label="$W_2$", alpha=0.7)
ax1[1].scatter(v2, p2, color='black', marker=".", lw=2)
ax1[1].plot(v2, p2, color = 'darkorange')
ax1[1].set_ylabel("$P_2$ $(\\mathrm{kPa})$")
ax1[1].set_title("Second Cycle")
ax1[1].set_xlabel("$V_2$ $(10^{-4}\,\\mathrm{m^3})$")
ax1[1].legend()
ax1[1].grid(b=True, which='major', color='#666666', linestyle='--')
ax1[1].minorticks_on()
ax1[1].grid(b=True, which='minor', color='#999999', linestyle='--', alpha=0.2)

fig1.tight_layout()

# logP - logV
fig2, ax2 = plt.subplots(2,2)

ax2[0][0].scatter(logv43_1, logp43_1, color='black', marker=".")
ax2[0][0].plot(logv43_1, func(logv43_1, *popt43_1), color = 'dodgerblue', label=f'$y = m_1x + c_1$')
ax2[0][0].set_ylabel("$\\log P_1$")
ax2[0][0].set_xlabel("$\\log V_1$")
ax2[0][0].legend()
ax2[0][0].set_title("First Adiabatic Compression")
ax2[0][0].grid(b=True, which='major', color='#666666', linestyle='--')
ax2[0][0].minorticks_on()
ax2[0][0].grid(b=True, which='minor', color='#999999', linestyle='--', alpha=0.2)

ax2[0][1].scatter(logv21_1, logp21_1, color='black', marker=".")
ax2[0][1].plot(logv21_1, func(logv21_1, *popt21_1), color = 'gold', label=f'$y = M_1x + C_1$')
ax2[0][1].set_ylabel("$\\log P_1$")
ax2[0][1].set_xlabel("$\\log V_1$")
ax2[0][1].legend()
ax2[0][1].set_title("First Adiabatic Relaxation")
ax2[0][1].grid(b=True, which='major', color='#666666', linestyle='--')
ax2[0][1].minorticks_on()
ax2[0][1].grid(b=True, which='minor', color='#999999', linestyle='--', alpha=0.2)

ax2[1][0].scatter(logv43_2, logp43_2, color='black', marker=".")
ax2[1][0].plot(logv43_2, func(logv43_2, *popt43_2), color = 'red', label=f'$y = m_2x + c_2$')
ax2[1][0].set_ylabel("$\\log P_2$")
ax2[1][0].set_xlabel("$\\log V_2$")
ax2[1][0].legend()
ax2[1][0].set_title("Second Adiabatic Compression")
ax2[1][0].grid(b=True, which='major', color='#666666', linestyle='--')
ax2[1][0].minorticks_on()
ax2[1][0].grid(b=True, which='minor', color='#999999', linestyle='--', alpha=0.2)

ax2[1][1].scatter(logv21_2, logp21_2, color='black', marker=".")
ax2[1][1].plot(logv21_2, func(logv21_2, *popt21_2), color = 'blueviolet', label=f'$y = M_2x + C_2$')
ax2[1][1].set_ylabel("$\\log P_2$")
ax2[1][1].set_xlabel("$\\log V_2$")
ax2[1][1].legend()
ax2[1][1].set_title("Second Adiabatic Relaxation")
ax2[1][1].grid(b=True, which='major', color='#666666', linestyle='--')
ax2[1][1].minorticks_on()
ax2[1][1].grid(b=True, which='minor', color='#999999', linestyle='--', alpha=0.2)

fig2.tight_layout()

plt.show()

# Print the Work Done
print(f"W43_1 = {W43_1} J, W21_1={W21_1} J, W_1 = {W43_1+W21_1} J =? {W_1} J \n")
print(f"W43_2 = {W43_2} J, W21_2={W21_2} J, W_2 = {W43_2+W21_2} J =? {W_2} J \n\n")

# Print the Lines' Slope
print(f"a43_1 ± δa43_1 = {popt43_1[0]} ± {err43_1[0]}, a21_1 ± δa21_1 = {popt21_1[0]} ± {err21_1[0]}\n")
print(f"a43_2 ± δa43_2 = {popt43_2[0]} ± {err43_2[0]}, a21_1 ± δa21_1 = {popt21_2[0]} ± {err21_2[0]}\n")


import numpy as np
import matplotlib.pyplot as plt
import random

class bandit():

	def __init__(self, m, initial_val):

		self.true_mean = m
		self.sample_mean = initial_val
		self.N = 0
		self.temp_val = 0

	def pull(self):

		mn = self.true_mean
		val = np.random.normal(mn, 1, None)
		self.update(val)

	def update(self, x):
		self.N += 1

		new_sample_mean = (1 - (1/self.N))*(self.sample_mean) + (1/self.N)*x
		self.sample_mean = new_sample_mean
"""
class bandit_time():

	def __init__(self, m, initial_val):

		self.true_mean = m
"""
# uses epsilon greedy algorithm to learn
def three_armed_bandit(mn1, mn2, mn3, eps, n):

	N = n
	epsilon = eps
	bandits = [bandit(mn1,0), bandit(mn2,0), bandit(mn3,0)]
	tot = 0
	data = np.ones([N,1])

	for i in range(N):
		p = np.random.rand()

		if p < epsilon:
			# explore bandits
			use = np.random.randint(3)
		else:
			# exploit one with max sample mean
			bandits_xbar = [bandits[0].sample_mean, bandits[1].sample_mean, bandits[2].sample_mean]
			use = np.argmax(bandits_xbar)

		bandits[use].pull()
		data[i][0] = bandits[use].sample_mean

	cumulative_avg = np.cumsum(data)/(np.arange(N) + 1)
	x_vals = np.arange(0,N,1)

	return cumulative_avg
#	print("bandit1 = ",bandits[0].sample_mean,"bandit2 = ",bandits[1].sample_mean,"bandit3 = ",bandits[2].sample_mean)

# simply greedy algorithm, but initial value is very large, so it encourage exploration.
def optimistic_initial_val(mn1,mn2,mn3,n):
	N = n
	bandits = [bandit(mn1,10),bandit(mn2,10),bandit(mn3,10)]
	tot = 0
	data = np.ones([N,1])

	for i in range(N):
		bandits_xbar = [bandits[0].sample_mean, bandits[1].sample_mean, bandits[2].sample_mean]
		use = np.argmax(bandits_xbar)
		bandits[use].pull()
		data[i][0] = bandits[use].sample_mean

	cumulative_avg = np.cumsum(data)/(np.arange(N) + 1)
	x_vals = np.arange(0,N,1)

	return cumulative_avg

# Using Chernoff-Hoefding bounds to learn. Need to do more reading on this.
def UCH_bound(mn1,mn2,mn3,n):
	N = n
	bandits = [bandit(mn1,10),bandit(mn2,10),bandit(mn3,10)]
	tot = 0
	data = np.ones([N,1])

	for i in range(1,N+1):
		bandits_CH_bound = [np.sqrt( (2*np.log(i)/(bandits[0].N + 0.0001)) ),
							np.sqrt( (2*np.log(i)/(bandits[1].N + 0.0001)) ),
							np.sqrt( (2*np.log(i)/(bandits[2].N + 0.0001)) )]
		bandits_xbar = [bandits[0].sample_mean + bandits_CH_bound[0],
						bandits[1].sample_mean + bandits_CH_bound[1],
						bandits[2].sample_mean + bandits_CH_bound[2]]
		use = np.argmax(bandits_xbar)
		bandits[use].pull()
		data[i-1][0] = bandits[use].sample_mean

	cumulative_avg = np.cumsum(data)/(np.arange(N) + 1)
	x_vals = np.arange(0,N,1)

	return cumulative_avg

# compare different methods
def compare_eps():

	mn1 = 1
	mn2 = 2
	mn3 = 3
	n = 4000

	c_01 = three_armed_bandit(mn1,mn2,mn3,0.1,n)
	c_005 = three_armed_bandit(mn1,mn2,mn3,0.05,n)
	c_001 = three_armed_bandit(mn1,mn2,mn3,0,n)

	# optimistic initial value (works on static problems)
	c_OIV = optimistic_initial_val(mn1,mn2,mn3,n)

	# Chernoff-Hoefding bound as epsilon
	c_UCHB = UCH_bound(mn1,mn2,mn3,n)

	x_vals = np.arange(0,c_01.shape[0],1)

	plt.plot(x_vals,c_001,label="eps = 0.01",color="black")
	plt.plot(x_vals,c_005,label = "eps = 0.05",color="orange")
	plt.plot(x_vals,c_01,label = "eps = 0.1",color="red")
#	plt.plot(x_vals,c_OIV,label = "OIV = 10",color="blue")
	plt.plot(x_vals,c_UCHB,label = "UCHB",color="blue")
	plt.plot(x_vals, np.ones([n,1])*mn1, linestyle="--")
	plt.plot(x_vals, np.ones([n,1])*mn2, linestyle="--")
	plt.plot(x_vals, np.ones([n,1])*mn3, linestyle="--")
	plt.xscale("log")
	plt.legend()
	plt.show()

	plt.plot(x_vals,c_001,label="eps = 0.01",color="black")
	plt.plot(x_vals,c_005,label = "eps = 0.05",color="orange")
	plt.plot(x_vals,c_01,label = "eps = 0.1",color="red")
#	plt.plot(x_vals,c_OIV,label = "OIV = 10",color="blue")
	plt.plot(x_vals,c_UCHB,label = "UCHB",color="blue")
	plt.plot(x_vals, np.ones([n,1])*mn1, linestyle="--")
	plt.plot(x_vals, np.ones([n,1])*mn2, linestyle="--")
	plt.plot(x_vals, np.ones([n,1])*mn3, linestyle="--")
	plt.legend()
	plt.show()


import scipy as sp
import numpy as np
import scipy.sparse as sparse
import scipy.sparse.linalg as sla
import matplotlib.pyplot as plt

def hnorm(r):
    """define ||r||_h = h ||r||_2"""
    n = len(r)
    h = 1.0 / (n+1)
    hrnorm = h * np.linalg.norm(r)
    return hrnorm

def poissonop(n):
    """
    Poisson operator h^{-2} * [-1 2 1]
    """
    A = (n+1)**2 * sparse.diags([-1, 2, -1], [-1, 0, 1], shape=(n,n), format='csr')
    return A

def residual(u, f):
    """
    f - A u
    """
    n = len(f)
    r = np.zeros(len(u))
    r[1:-1] = f[1:-1] - ((n+1)**2) * (2 * u[1:-1] - u[2:] - u[:-2])
    r[0]    = f[0]    - ((n+1)**2) * (2 * u[0]    - u[1])
    r[-1]   = f[-1]   - ((n+1)**2) * (2 * u[-1]   - u[-2])
    return r

def relax(u, f, nu):
    """
    Weighted Jacobi
    """
    n = len(u)
    Dinv = 1.0 / (2.0 * ((n+1)**2))
    omega = 2.0 / 3.0
    unew = u.copy()

    for steps in range(nu):
        unew = unew + omega * Dinv * residual(unew, f)

    return unew

def interpolate(uc):
    """interpolate u of size 2**(k-1)-1 to 2**(k)-1"""
    uf = np.zeros((2*len(uc) + 1,))
    uf[:-1:2] = 0.5 * uc
    uf[1::2] = uc
    uf[2::2] += 0.5 * uc
    return uf

def restrict(uf):
    """restrict u of size 2**(k)-1 to 2**(k-1)-1"""
    uc = 0.25 * uf[:-1:2] + 0.5 * uf[1::2] + 0.25 * uf[2::2]
    return uc

def vcycle(kmax, kmin, u, f, nu):
    ulist = [None for k in range(kmax+1)]
    flist = [None for k in range(kmax+1)]

    print('grid: ', end=' ')
    # down cycle
    for k in range(kmax, kmin, -1):
        print(k, end=' ')
        u = relax(u, f, nu)
        ulist[k] = u
        flist[k] = f

        f = restrict(residual(u, f))
        u = np.zeros(f.shape)
        ulist[k-1] = u
        flist[k-1] = f

    # coarsest grid
    print(kmin, end=' ')
    Ac = poissonop(2**kmin - 1)
    flist[kmin] = f
    ulist[kmin] = sla.spsolve(Ac, f)

    # up cycle
    for k in range(kmin+1, kmax+1, 1):
        print(k, end=' ')
        u = ulist[k]
        f = flist[k]
        uc = ulist[k-1]
        u += interpolate(uc)
        u = relax(u, f, nu)
    print('.')
    return u

kmax = 7
kmin = 2

# set up fine problem
n = 2**kmax - 1

xx = np.linspace(0, 1, n+2)[1:-1]

f = 2 - 12 * xx**2

ustar = xx**4 - xx**2
A = poissonop(len(f))
udstar = sla.spsolve(A, f)
print(hnorm(ustar - udstar))
u = np.random.rand(len(f))

# set up smoothing sweeps
nu = 2

res = []
err = []

res.append(hnorm(residual(u, f)))
for i in range(12):
    u = vcycle(kmax, kmin, u, f, nu)
    #u = vcycle3(u, f, nu)
    res.append(hnorm(residual(u, f)))
    err.append(hnorm(u - ustar))
print(res)


#This file provides a function getMatrices which returns the dot products of logsignature
#basis elements as a matrix for each level.
#This allows you to find the L2 dot product of log signatures in tensor space without
#ever expressing them in tensor space.
#This might be useful if you have to calculate such a dot product very many times.
#The demo() function provides an example.

#Don't use for dimension bigger than 9, because it uses the basis() function

import numpy as np, numpy, sys, os, scipy.linalg
from six import print_

#add the parent directory, so we find our iisignature build if it was built --inplace
sys.path.append(os.path.dirname(os.path.abspath(os.path.dirname(__file__))))
import iisignature

#the following 3 functions for interpreting basis strings
#are copied from the tests

#text, index -> (either number or [res, res]), newIndex
def parseBracketedExpression(text,index):
    if(text[index]=='['):
        left, m = parseBracketedExpression(text,index+1)
        right, n = parseBracketedExpression(text,m+1)
        return [left,right],n+1
    else:
        n = 0
        while(n<len(text) and text[index+n] in ['1','2','3','4','5','6','7','8','9']): #this should always just happen once if input is a bracketed expression of letters
            n = n + 1
        return int(text[index:index+n]), index+n

#print (parseBracketedExpression("[23,[2,[[22,1],2]]]",0))

#bracketed expression, dim -> numpy array of its value, depth
def multiplyOut(expn, dim):
    if isinstance(expn,list):
        left, leftDepth  = multiplyOut(expn[0],dim)
        right,rightDepth = multiplyOut(expn[1],dim)
        a = numpy.outer(left,right).flatten()
        b = numpy.outer(right,left).flatten()
        return a-b, leftDepth+rightDepth
    else:
        a = numpy.zeros(dim)
        a[expn-1]=1
        return a,1

#string of bracketed expression, dim -> numpy array of its value, depth
#for example:
#valueOfBracket("[1,2]",2) is ([0,1,-1,0],2)
def valueOfBracket(text,dim):
    return multiplyOut(parseBracketedExpression(text,0)[0],dim)

def getMatrices(s):
    info=iisignature.info(s)
    m=info["level"]
    d=info["dimension"]
    values=[[] for i in range(m)]
    for exp in iisignature.basis(s):
        vec,level = valueOfBracket(exp,d)
        values[level-1].append(vec)
    out=[]
    for v in values:
        def f(x,y): return numpy.inner(v[x],v[y])
        out.append(numpy.fromfunction(
            numpy.vectorize(f),(len(v),len(v)),dtype=int))
#        store=[]
#        for x1 in v:
#            store.append([])
#            for x2 in v:
#                store[-1].append(numpy.inner(x1,x2))
#        out.append(numpy.array(store))
    return out

def demo():
    #1. generate some dummy data
    d=3
    m=6
    path1=np.random.uniform(size=(20,d))
    path2=np.random.uniform(size=(20,d))
    paths = (path1,path2)
    s=iisignature.prepare(d,m)

    #2. print the dot product of the log signatures in tensor space
    #(The "x" means that the log signature is returned in tensor space.)
    expandedLogSig1,expandedLogSig2=(iisignature.logsig(i,s,"x") for i in paths)
    target = np.inner(expandedLogSig1,expandedLogSig2)
    print_ ("Target:", float(target))

    #3. use getMatrices to act on the log signatures expressed in a basis
    #    and get the same answer.
    matrices=getMatrices(s)
    logsig1,logsig2=(iisignature.logsig(i,s) for i in paths)
    adjustment=scipy.linalg.block_diag(*matrices)
    #print(np.dot(adjustment,logsig2).shape)
    print_ ("We get:", float(np.inner(logsig1,np.dot(adjustment,logsig2))))

if __name__=="__main__":
    demo()


import numpy

def get_lead_ind(row):
    for entry in range(len(row)):
        if row[entry] != 0:
            return entry
    return len(row)

def get_row_to_swap(M, start_i):
    leftmost = start_i
    howleft = get_lead_ind(M[start_i])
    for row in range(start_i+1, len(M)):
        if(get_lead_ind(M[row])<howleft):
            howleft = get_lead_ind(M[row])
            leftmost = row
    return leftmost

def add_rows_coefs(r1, c1, r2, c2):
    result = [0]*len(r1)
    #r1*c1 + r2*c2
    for i in range(len(result)):
        # if(i < len(r1)):
            result[i] += c1*r1[i]
        # if(i < len(r2)):
            result[i] += c2*r2[i]
    return result

def eliminate(M, row_to_sub, best_lead_ind):
    for row in range(row_to_sub+1, len(M)):
        if(M[row][best_lead_ind] != 0):
            coeff = M[row][get_lead_ind(M[row])] / M[row_to_sub][get_lead_ind(M[row_to_sub])]
            for item in range(best_lead_ind, len(M[row])):
                M[row][item] -= coeff*M[row_to_sub][item]

def forward_step(M):
    for row in range(len(M)):
        swap = get_row_to_swap(M, row)
        r = M[row]
        M[row] = M[swap]
        M[swap] = r
        eliminate(M, row, get_lead_ind(M[row]))

M = [[5, 6, 7, 8],
[0,0, 1, 1],
[0, 0, 5, 2],
[0, 0, 7, 0]]

# M = [[5, 6, 7, 8],
# [0, 0, 0, 1],
# [0, 0, 5, 2],
# [0, 1, 0, 0]]

forward_step(M)

print(numpy.array(M))


import read_data as RD
import numpy as np
import matplotlib.pyplot as plt
from PIL import Image

X = RD.read_data()
print('X = ',X.shape)
X_mean = np.reshape(np.sum(X,1)/X.shape[1],[ X.shape[0],1])
X = X-X_mean
print('X_centerred = ',X.shape)
[U,S,V] = np.linalg.svd(X, full_matrices=False)
print('U = ',U.shape)
print('S = ',S.shape)
print('V = ',V.shape)

N = 10#N=number of eigenvectors to consider
A = U[:,0:N]
Y = np.matmul(np.transpose(A),X)
print('Y = ',Y.shape)

class_mean = list()
class_cov = list()
for i in range(0,26):
	class_images = []
	count = 0
	for j in range(i,Y.shape[1],26):
		class_images.append(Y[:,j])
		count += 1

	class_images = np.array(class_images)
	class_images = class_images.T
	cm = np.reshape(np.sum(class_images,1)/count,[N,1])
	class_mean.append(cm)
	class_cov.append(np.matmul((class_images-cm),np.transpose(class_images-cm))/(count-1))

class_mean = (np.array(class_mean)).squeeze()
class_cov = (np.array(class_cov)).squeeze()
print('Class Mean = ',class_mean.shape)
print('Class Covariance = ',class_cov.shape)

############ Test data Read  ########################
datadir='../test_data'    # directory where the data files reside
dataset=['veranda']
datachar='abcdefghijklmnopqrstuvwxyz'
Rows=64    # all images are 64x64
Cols=64
n=len(dataset)*len(datachar)  # total number of images
p=Rows*Cols   # number of pixels

X_test=np.zeros((p,n))  # images arranged in columns of X
k=0
for dset in dataset:
	for ch in datachar:
		fname='/'.join([datadir,dset,ch])+'.tif'
		im=Image.open(fname)
		img = np.array(im)
		X_test[:,k]=np.reshape(img,(1,p))
		k+=1
print('X_test = ',X_test.shape)
X_test -= X_mean
Y_test = np.matmul(np.transpose(A),X_test)
print('Y_test = Y',Y_test.shape)

def calculate_distance(Y_test,class_mean,class_cov):
	correct_table = list()
	incorrect_table = list()
	for i in range(0,Y_test.shape[1]):
		distance = list()
		for c in range(class_mean.shape[0]):
			T = np.reshape(Y_test[:,i] - np.transpose(class_mean[c,:]), [N,1])
			distance.append(np.matmul(np.transpose(T),np.matmul(np.linalg.inv(class_cov[c,:,:]),T)) + np.log(np.linalg.norm(class_cov[c,:,:])))
		distance = list((np.array(distance)).squeeze())
		predicted_class = datachar[distance.index(np.min(distance))]
		if datachar[i] == predicted_class:
			correct_table.append([datachar[i],predicted_class])
		else:
			incorrect_table.append([datachar[i],predicted_class])
	return correct_table,incorrect_table

[correct_table,incorrect_table] = calculate_distance(Y_test,class_mean,class_cov)
print('\n\nOriginal Covariance - Incorrect Classes')
for i in range(0,len(incorrect_table)):
	print(incorrect_table[i][0],' ------------------------- ',incorrect_table[i][1])
print('\n')
########### Modification 1 ##############
modified_cov = list()
for i in range(0,class_cov.shape[0]):
	modified_cov.append(np.diag(np.diag(class_cov[i,:,:])))
modified_cov = np.array(modified_cov)
[correct_table,incorrect_table] = calculate_distance(Y_test,class_mean,modified_cov)
print('Modification 1 - Incorrect Classes')
for i in range(0,len(incorrect_table)):
	print(incorrect_table[i][0],' ------------------------- ',incorrect_table[i][1])
print('\n')

########### Modification 2 ##############
modified_cov = list()
avg_cov = np.sum(class_cov,0)/class_cov.shape[0]
for i in range(0,class_cov.shape[0]):
	modified_cov.append(avg_cov)
modified_cov = np.array(modified_cov)
print('Modification 2 - Incorrect Classes')
[correct_table,incorrect_table] = calculate_distance(Y_test,class_mean,modified_cov)
for i in range(0,len(incorrect_table)):
	print(incorrect_table[i][0],' ------------------------- ',incorrect_table[i][1])
print('\n')
########### Modification 3 ##############
modified_cov = list()
avg_cov = np.sum(class_cov,0)/class_cov.shape[0]
for i in range(0,class_cov.shape[0]):
	modified_cov.append(np.diag(np.diag(avg_cov)))
modified_cov = np.array(modified_cov)
print('Modification 3 - Incorrect Classes')
[correct_table,incorrect_table] = calculate_distance(Y_test,class_mean,modified_cov)
for i in range(0,len(incorrect_table)):
	print(incorrect_table[i][0],' ------------------------- ',incorrect_table[i][1])
print('\n')
########### Modification 4 ##############
modified_cov = list()
for i in range(0,class_cov.shape[0]):
	modified_cov.append(np.identity(class_cov.shape[1]))
modified_cov = np.array(modified_cov)
print('Modification 4 - Incorrect Classes')
[correct_table,incorrect_table] = calculate_distance(Y_test,class_mean,modified_cov)
for i in range(0,len(incorrect_table)):
	print(incorrect_table[i][0],' ------------------------- ',incorrect_table[i][1])
print('\n')


'''
Adapted from VisPy example volume rendering here: https://github.com/vispy/vispy/blob/master/examples/basics/scene/volume.py

NOTE: Normalization approach credited to Etienne Cmb on Stack Overflow: https://stackoverflow.com/questions/51306488/transparency-with-voxels-in-vispy/51309283#51309283
'''

from numba import jit
import numpy as np

import imageio
from vispy import app, scene
from vispy.color import Colormap

from timeit import default_timer as timer

start = timer()

'''
Computing Fractal
'''

# PARAMETERS TO CHANGE THE FRACTAL GENERATED
anim = True     # change whether to produce a .gif animation of fractal rotating
seq = "ABC"     # sequence to alternate r values
a_lb = 2        # a lower bound
a_ub = 4        # a upper bound
b_lb = 2        # b lower bound
b_ub = 4        # b upper bound
c_lb = 2        # c lower bound
c_ub = 4        # c upper bound

# PARAMETERS REFINING ACCURACY OF FRACTAL PICTURE GENERATED
num_warmups = 100             # number of "warmups" or throwaway iterations before computing lyapunov exponent
num_lyap_iterations = 100      # number of iterations used to compute the lyapunov exp
steps = 100                   # steps between b1 and b2 values on axes -- higher it is, the better the picture

# LOGISTIC MAP THAT GIVES US THE NEXT X
@jit
def F(x, curr_r):
    return (curr_r * x) * (1 - x)

# DERIVATIVE OF F -- USED TO COMPUTE THE LYAPUNOV EXPONENT
@jit
def Fprime(x, curr_r):
    ans = curr_r * (1 - (2 * x))
    ans[ans == 0] = 0.00001
    ans[ans == -np.inf] = -1000
    ans[ans == np.inf] = 1000
    return ans

# RETURNS THE CORRECT B-VALUE BASED ON THE CURRENT ITERATION
@jit
def getseqval(curr_iteration, a, b, c):
    index = np.mod(curr_iteration, len(seq))
    if (seq[index] == 'A'):
        return a
    elif (seq[index] == 'B'):
        return b
    else:
        return c

# RETURNS THE LYAPUNOV EXPONENT BASED ON THE SPECIFIED B1 AND B2 VALUES
@jit
def getlyapexponent(a, b, c):
    x = .5          # initial value of x
    lyap_sum = 0     # initializing lyapunov sum for use later

    # do warmups, to discard the early values of the iteration to allow the orbit to settle down
    for i in range(num_warmups):
        x = F(x, getseqval(i, a, b, c))

    for i in range(num_warmups, num_lyap_iterations + num_warmups):
        lyap_sum += np.log( np.abs(Fprime(x, getseqval(i, a, b, c) ) ) )

        # get next x
        x = F(x, getseqval(i, a, b, c))

    return (lyap_sum / num_lyap_iterations)

# RETURNS DATA NORMALIZED TO VALUES BETWEEN 0 AND 1, AS WELL AS THE NORMALIZED VALUE OF BOUNDARY_OLD
@jit
def normalize(data, boundary_old):
    orig_max = data.max()
    orig_min = data.min()

    # normalized boundary
    boundary_norm = boundary_old - orig_min
    boundary_norm = boundary_norm / (orig_max - orig_min)

    data = np.subtract(data, orig_min)
    data = np.divide(data, orig_max - orig_min)

    return data, boundary_norm

'''
Creating and Preparing 3D Fractal Data
'''

# CREATING FRACTAL IMAGE
a = np.linspace(a_lb, a_ub, steps)   #range of b1 values
b = np.linspace(b_lb, b_ub, steps)   #range of b2 values
c = np.linspace(c_lb, c_ub, steps)

aa, bb, cc = np.meshgrid(a, b, c, indexing='ij')

fractal_3D = getlyapexponent(aa, bb, cc)

# normalize data between 0 and 1 to be displayed and return chaotic boundary
fractal_3D, chaotic_boundary = normalize(fractal_3D, 0.0)
print("chaotic boundary:", chaotic_boundary)

'''
Creating 3D projection of data
'''

# Prepare canvas
canvas = scene.SceneCanvas(keys='interactive', size=(800, 600), show=True)
canvas.measure_fps()

# Set up a viewbox to display the image with interactive pan/zoom
view = canvas.central_widget.add_view()
camera = scene.cameras.ArcballCamera(parent=view.scene, fov=60, scale_factor=steps*3, center = (0, 0, 0))
view.camera = camera

# Create the volume
volume = scene.visuals.Volume(fractal_3D, clim=(0, 1), method='translucent', parent=view.scene, threshold=0.225,emulate_texture=False)
volume.transform = scene.STTransform(translate=(-steps//2, -steps//2, -steps//2))

# Creating color map to display fractal
fractal_colors = [(1, 0, 1, .5), (0, 0, 1, .5), (.1, .8, .8, .3), (.1, 1, .1, .3), (1, 1, 0, .2), (1, 0, 0, .1), (0, 1, 1, (1 - chaotic_boundary) / 7), (0, 1, .8, (1 - chaotic_boundary) / 8), (0, 0, 0, 0), (0, 0, 0, 0)]
color_control_pts = [0, (0.6 * chaotic_boundary), (0.7 * chaotic_boundary), (0.8 * chaotic_boundary), (0.9 * chaotic_boundary), (0.95 * chaotic_boundary), (0.97 * chaotic_boundary), (0.99 * chaotic_boundary), chaotic_boundary, chaotic_boundary, 1.0]

fractal_map = Colormap(fractal_colors, controls=color_control_pts, interpolation='zero')

# Assigning newly made color map to volume data
volume.cmap = fractal_map

''' Creating animation of rotating fractal '''
if anim:
    file_name = "Anim_3D_Fractal_" + seq + ".gif"
    writer = imageio.get_writer(file_name)

    # Parameters to change animation
    angle_delta = 10.0  # amount to rotate fractal by each frame
    axes = [[1, 1, 0], [1, .5, .5], [1, 0, 1], [.5, 0, 1], [1, .5, .5]]  # axes to rotate fractal on, in succession

    for axis in axes:
        for rotate in range(int(360/angle_delta)):
            im = canvas.render()
            writer.append_data(im)
            view.camera.transform.rotate(angle_delta, axis)
    writer.close()

''' Run program '''

if __name__ == '__main__':
    print(__doc__)
    app.run()

end = timer()
print("elapsed time: " + str(end - start))


import numpy as np

def quantile(values, quantiles):
    """Compute MATLAB-alike quantiles

    Arguments:
        values - (np.ndarray) Input array or object that can be converted to an
        array.
        quantiles - (float, np.ndarray) float in range of [0,1] (or sequence
        of floats). Quantile to compute, which must be between 0 and 1
        inclusive.

    location of first element as a quantile -> 0.5 / n
    location of last element as a quantile -> (n - 0.5) / n

         ^
         |     (n-0.5) / n    1.0
     1.0 ----------------|---|
         |              /
         |             /
         |            /
         |           /                  <- this is how quantiles look in MATLAB
         |          /
         |         /
         |        /
         |       /
     0.0 ----|---|------------->
         | 0.0   0.5 / n

    y = ax + b

    0 = a * 0.5 / n + b         <- 0th element is treated as 0.5 / n quantile
    1 = a * (n - 0.5) / n + b   <- last element is treated as (n - 0.5) / n quantile

    a = n / (n - 1)
    b = - 0.5 / (n - 1)

    """
    values = np.array(values)
    n = float(values.size)
    a = n / (n - 1)
    b = -0.5 / (n - 1)
    quantiles = np.array(quantiles)
    matlab_alike_quantiles = np.clip(a * quantiles + b, a_min=0.0, a_max=1.0)
    return np.percentile(values, q=100.0 * matlab_alike_quantiles)

def n_quantiles(values, N, unbiased=True, backend=quantile):
    return backend(values, np.arange(1, N + 1, dtype=float) / (N + int(unbiased)))

def iqr(values, rng=(25, 75)):
    q1, q3 = quantile(values, 0.01 * np.array(rng))
    return q3 - q1


#!/usr/bin/python
'''
This function generates the data for a 1D motion considering a
Piecewise constant, white noise acceleration model, equation (14)

var_a is the variance of the process noise
R is the variance of measurement noise
z is the measured data
x are the true values of the system states

example of use
python gen_data14.py 1e-3 1
'''

import sys
import numpy as np
import matplotlib.pyplot as plt

def get_generated_data_eq_14(Q, R):
    var_a = float(Q)
    R = float(R)

    N = 100  # data size
    T = 1.0  # [s] Sampling time interval

    x = np.zeros((2, N))  # states are [position; speed; acceleration]
    x[:, 0] = [0, 10]  # state initialization, change to give your own initial values

    A = np.array([[1, T],
                  [0, 1]], dtype=float)  # Transition matrix

    G = np.array([[T ** 2 / 2],
                  [T]], dtype=float)  # Vector gain for the process noise

    w = np.random.normal(0.0, np.sqrt(var_a), N)  # process noise
    for ii in range(1, N):  # simulate system dynamics
        x[:, ii] = A.dot(x[:, ii - 1]) + G.dot(w[ii]).T

    v = np.random.normal(0.0, np.sqrt(R), N)  # measurement noise
    z = x[0, :] + v  # position measurements assuming C = [1 0 0]
    return z, x


import math
import numpy as np
import itertools
from functools import partial
import matplotlib.pyplot as plt

def parzen_window(query, sample, window_size):
    """
    Parameters
    ----------
    query: float
    sample: float
    window_size: float,
        size of the parzen window

    Returns
    ----------
    indicator: int
        1 if the sample falls within a unit hypercube around the query
        0 otherwise
    """

    return 1 if abs((query - sample) / window_size) < .5 else 0

def gaussian_kernel(query, sample, window_size):
    """
    Parameters
    ----------
    query: float
    sample: float
    window_size: float,
        size of the parzen window

    Returns
    ----------
    prob: float
    """

    unit_distance = (query - sample) / window_size

    return math.pow(2*math.pi, -0.5) * \
        np.exp(-0.5*np.dot(unit_distance, unit_distance))

def kernel_function_wrapper(window_size, kernel):
    """Wrapper for kernel function

    Parameters
    ----------
    window_size: float
        size of the kernel band width
    kernel: string
        kernel to use for estimation

    Returns
    ---------
    handle: partial method
    """

    if kernel == "parzen":
        return partial(parzen_window, window_size=window_size)
    elif kernel == "gaussian":
        return partial(gaussian_kernel, window_size=window_size)

def kernel_density_estimation(query, samples, window_size, kernel="gaussian"):
    """Evaluate value of the probability density function at position query

    Parameters
    ----------
    query: float
    samples: (m, )
        numpy array of float, m is the number of samples
    window_size: float
        size of the kernel band width
    kernel: string
        kernel to use for estimation

    Returns
    ---------
    prob: float
    """

    n_samples = samples.shape[0]

    kernel_function_handler = kernel_function_wrapper(window_size, kernel)

    return sum([
        kernel_function_handler(query, samples[idx])
        for idx in range(n_samples)
    ]) / (n_samples * window_size)

def kernel_density_estimation_wrapper(samples, window_size, kernel="gaussian"):
    """Wrapper for kernel density estimation function

    Parameters
    ----------
    samples: (m, )
        numpy array of float, m is the number of samples
    window_size: float
        size of the kernel band width
    kernel: string
        kernel to use for estimation

    Returns
    ---------
    handle: partial method
    """

    return partial(
        kernel_density_estimation,
        samples=samples,
        window_size=window_size,
        kernel=kernel)

#########################################
# Standard normal distribution
#########################################

sample_size = 100
samples = np.random.normal(loc=0, scale=1, size=sample_size)

grid_1d = np.linspace(-10, 10, num=500, endpoint=True)

from scipy.stats import norm

window_size = [1, 2, 4, 8]

fig, axarr = plt.subplots(2, 2, sharex=True, sharey=True)

vectorized_norm_pdf = np.vectorize(norm.pdf)

# plot effect of window size
for i, j in itertools.product(range(2), range(2)):

    window_index = i + j

    parzen_window_estimation = np.vectorize(
        kernel_density_estimation_wrapper(
            samples=samples,
            window_size=window_size[window_index],
            kernel="parzen"),
        otypes=[np.float])
    gaussian_kernel_estimation = np.vectorize(
        kernel_density_estimation_wrapper(
            samples=samples,
            window_size=window_size[window_index],
            kernel="gaussian"),
        otypes=[np.float])

    pdf = vectorized_norm_pdf(grid_1d)
    pdf_parzen = parzen_window_estimation(grid_1d)
    pdf_gaussian = gaussian_kernel_estimation(grid_1d)

    line1, = axarr[i, j].plot(grid_1d, pdf, 'r', label="pdf")
    line2, = axarr[i, j].plot(grid_1d, pdf_parzen, 'g', label="parzen")
    line3, = axarr[i, j].plot(grid_1d, pdf_gaussian, 'b', label="gaussian")
    axarr[i, j].set_title("window size: {}".format(window_size[window_index]))

fig.legend(handles=[line1, line2, line3], loc="lower right")

plt.show()

#########################################
# Mixture of gaussian distributions
#########################################

means = [0, 3, 5]
stds = [1, 4, 2]

sample_size = 1000

n_distributions = 3
distribution_choice = np.random.choice(
    np.arange(3), p=[0.3, 0.4, 0.3], size=sample_size)

# Generate samples from a mixture of gaussians
def generate_samples(means, stds, distribution_choice):

    ret = []

    bin_counts = np.bincount(distribution_choice)

    for i in range(len(means)):
        ret += np.random.normal(
            loc=means[i], scale=stds[i], size=bin_counts[i]).tolist()

    return np.array(ret)

samples = generate_samples(means, stds, distribution_choice)

grid_1d = np.linspace(-10, 10, num=500, endpoint=True)

window_size = [1, 2, 4, 8]

fig, axarr = plt.subplots(2, 2, sharex=True, sharey=True)

# plot effect of window size
for i, j in itertools.product(range(2), range(2)):

    window_index = i + j

    parzen_window_estimation = np.vectorize(
        kernel_density_estimation_wrapper(
            samples=samples,
            window_size=window_size[window_index],
            kernel="parzen"),
        otypes=[np.float])
    gaussian_kernel_estimation = np.vectorize(
        kernel_density_estimation_wrapper(
            samples=samples,
            window_size=window_size[window_index],
            kernel="gaussian"),
        otypes=[np.float])

    pdf_parzen = parzen_window_estimation(grid_1d)
    pdf_gaussian = gaussian_kernel_estimation(grid_1d)

    line1 = axarr[i, j].hist(
        samples, bins=100, label="empirical", normed=True, color='r')
    line2, = axarr[i, j].plot(grid_1d, pdf_parzen, 'g', label="parzen")
    line3, = axarr[i, j].plot(grid_1d, pdf_gaussian, 'b', label="gaussian")
    axarr[i, j].set_title("window size: {}".format(window_size[window_index]))

fig.legend(handles=[line2, line3], loc="lower right")

plt.show()


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Tue Mar  5 12:13:53 2019
Statistic lib contains functions to undertake statistic analysis
@author: chunan
"""
import numpy
import pylab

# define useful constants
pi = numpy.pi

# calculation Standard Deviation (SD) ------------------------------------------
def standard_deviation(sig=0, i_test=0):
  if i_test == 1:
    sig = test_data(dim=1, size=50)

  SD = 0
  s_mean = numpy.mean(sig)
  N = len(sig)
  for i in range(0, N):
    SD = SD + (sig[i] - s_mean)**2
  SD = numpy.sqrt(SD/N)

  if i_test == 1:
    x = numpy.arange(0, N)
    pylab.figure()
    pylab.plot(x, sig,'o', color='blue', markersize=5)
    pylab.errorbar(0, s_mean, yerr=SD, fmt='s', color='green', markersize=5, capsize=3)
    pylab.legend(['sig', 'SD'], loc='upper right')
    pylab.xlabel('number')
    pylab.ylabel('value')
    pylab.title('standard deviation')
    print('mean', 'SD')
    print(s_mean, SD)
  return SD

# calculation Standard Error (SE) ----------------------------------------------
def standard_error(sig=0, i_test=0):
  '''
  standard error calculate just how certain the mean value we got shall be trusted!
  Reference:
  [1] http://berkeleysciencereview.com/errorbars-anyway/
  '''
  if i_test == 1:
    sig = test_data(dim=1, size=80)

  SD = standard_deviation(sig=sig)
  SE= SD/numpy.sqrt(len(sig))
  if i_test == 1:
    s_mean = numpy.mean(sig)
    x = numpy.arange(0, len(sig))
    pylab.figure()
    pylab.plot(x, sig,'o', color='blue', markersize=5)
    pylab.errorbar(0, s_mean, yerr=SD, fmt='s', color='green', markersize=5, capsize=3)
    pylab.errorbar(numpy.int(len(sig)/2), s_mean, yerr=SE, fmt='s', color='red', markersize=5, capsize=3)
    pylab.legend(['sig', 'SD', 'SE'], loc='upper right')
    pylab.xlabel('number')
    pylab.ylabel('value')
    pylab.title('standard error')
    print('mean', 'SD', 'SE')
    print(s_mean, SD, SE)

  return SE

def test_data(dim=1, size=30):
  if dim == 1:
    s = 5 + 0.1*numpy.random.randn(size)

  return s

i_test = 1
if i_test == 1:
  pylab.close('all')
  #standard_deviation(i_test=1)
  standard_error(i_test=1)


import numpy

def cofi_cost_func(params, Y, R, num_users, num_movies, num_features, reg_lambda=0.0):
    """
    Collaborative filtering cost function.

    Parameters
    ----------
    params : array_like
        The parameters which will be optimized. This is a one
        dimensional vector of shape (num_movies x num_users, 1). It is the
        concatenation of the feature vectors X and parameters Theta.

    Y : array_like
        A matrix of shape (num_movies x num_users) of user ratings of movies.

    R : array_like
        A (num_movies x num_users) matrix, where R[i, j] = 1 if the
        i-th movie was rated by the j-th user.

    num_users : int
        Total number of users.

    num_movies : int
        Total number of movies.

    num_features : int
        Number of features to learn.

    reg_lambda : float, optional
        The regularization coefficient.

    Returns
    -------
    cost_J : float
        The value of the cost function at the given params.

    grad : array_like
        The gradient vector of the cost function at the given params.
        grad has a shape (num_movies x num_users, 1)

    Instructions
    ------------
    Compute the cost function and gradient for collaborative filtering.
    Concretely, you should first implement the cost function (without
    regularization) and make sure it is matches our costs. After that,
    you should implement thegradient and use the checkCostFunction routine
    to check that the gradient is correct. Finally, you should implement
    regularization.

    Notes
    -----
    - The input params will be unraveled into the two matrices:
        x_array : (num_movies  x num_features) matrix of movie features
        Theta : (num_users  x num_features) matrix of user features

    - You should set the following variables correctly:

        x_grad : (num_movies x num_features) matrix, containing the
                 partial derivatives w.r.t. to each element of x_array
        Theta_grad : (num_users x num_features) matrix, containing the
                     partial derivatives w.r.t. to each element of Theta

    - The returned gradient will be the concatenation of the raveled
      gradients x_grad and Theta_grad.
    """
    # Unfold the U and W matrices from params
    cost_J = 0
    prod_mov_feat = num_movies*num_features
    x_array = params[:prod_mov_feat].reshape(num_movies, num_features)
    x_grad = numpy.zeros(x_array.shape)
    Theta = params[prod_mov_feat:].reshape(num_users, num_features)
    Theta_grad = numpy.zeros(Theta.shape)

    hypothesis = numpy.dot(x_array, numpy.transpose(Theta))
    error = (1/2)*numpy.sum(numpy.square((hypothesis - Y) * R))
    reg_term = (reg_lambda/2)*(numpy.sum(numpy.square(x_array)) + numpy.sum(numpy.square(Theta)))
    cost_J = error + reg_term

    for i in range(R.shape[0]):
        idx = numpy.where(R[i, :] == 1)[0]
        Theta_temp = Theta[idx, :]
        Y_temp = Y[i, idx]
        term1 = numpy.dot(numpy.dot(x_array[i, :], Theta_temp.T) - Y_temp, Theta_temp)
        term2 = reg_lambda*x_array[i, :]
        x_grad[i, :] = term1 + term2

    for j in range(R.shape[1]):
        idx = numpy.where(R[:, j] == 1)[0]
        X_temp = x_array[idx, :]
        Y_temp = Y[idx, j]
        term1 = numpy.dot(numpy.dot(X_temp, Theta[j, :]) - Y_temp, X_temp)
        term2 = reg_lambda * Theta[j, :]
        Theta_grad[j, :] = term1 + term2

    grad = numpy.concatenate([x_grad.ravel(), Theta_grad.ravel()])

    return cost_J, grad


#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Time     : 2018/10/09 04:05
# @Author   : Iydon
# @File     : course3.3.py

import numpy as np
from Poly import *

def divided_difference(xs:list, fxs:list):
    if len(xs)==1:
        return fxs[0]
    elif len(xs)>1:
        f = divided_difference
        return (f(xs[1:],fxs[1:])-f(xs[:-1],fxs[:-1]))/(xs[-1]-xs[0])

def newton_divided_difference(xs:list, fxs:list, forward:bool=True):
    """
    Newton's Divided-Difference Formula.
    """
    length = len(xs)
    result = [i for i in fxs]
    sign   = -1 if forward else 1
    for i in range(length-1):
        for j in range(length-i-1, 0, -1):
            idx = -length+i+j if forward else length-i-j-1
            result[idx]  = result[idx+sign]-result[idx]
            result[idx] /= xs[idx+sign*(i+1)]-xs[idx]
        # print(result)
    return result[::-sign]

def newton_forward(xs:list, fxs:list, degree:int, x0:float):
    def prod(lst, result=1):
        return prod(lst[:-1], result*lst[-1]) if lst else result
    def s_k(s, k):
        return prod([s-i for i in range(k)])
    # body
    h = xs[1] - xs[0]
    s = (x0-xs[0]) / h
    result = fxs[0]
    for k in range(1, degree+1):
        # print(s_k(s,k), divided_difference(xs[:k+1],fxs[:k+1]), h**k)
        result += s_k(s,k) * divided_difference(xs[:k+1],fxs[:k+1]) * h**k
    return result

def newton_backward(xs:list, fxs:list, degree:int, x0:float):
    def prod(lst, result=1):
        return prod(lst[:-1], result*lst[-1]) if lst else result
    def s_k(s, k):
        return prod([s-i for i in range(k)])
    # body
    h = xs[1] - xs[0]
    s = (x0-xs[-1]) / h
    result = fxs[-1]
    print(result)
    for k in range(1, degree+1):
        print((-1)**k, s_k(-s,k), divided_difference(xs[-k-1:],fxs[-k-1:]), h**k)
        result += (-1)**k * s_k(-s,k) * divided_difference(xs[-k-1:],fxs[-k-1:]) * h**k
    return result

"""
xs  = [1.0, 1.3, 1.6, 1.9, 2.2]
fxs = [0.7651977, 0.6200860, 0.4554022, 0.2818186, 0.1103623]
n   = newton_divided_difference(xs, fxs, forward=True)
print(n)
"""

xs  = [-0.75, -0.5, -0.25, 0]
fxs = [-0.07181250, -0.02475000, 0.33493750, 1.10100000]
n_f = newton_forward(xs, fxs, 3, -1/3)
print(n_f)

print("*"*10)

xs  = [1.0, 1.3, 1.6, 1.9, 2.2]
fxs = [0.7651977, 0.6200860, 0.4554022, 0.2818186, 0.1103623]
n_b = newton_backward(xs, fxs, 4, 2)
print(n_b)

print("*"*10)

# Newton
def newton_divided_difference(xs:list, fxs:list, degree:int=0):
    """
    Newton's Divided-Difference Formula.
    """
    def prod(lst, result=1):
        return prod(lst[:-1], result*lst[-1]) if lst else result
    result = fxs[0]
    for i in range(1, degree+1):
        print(divided_difference(xs[:i+1],fxs[:i+1]), [Poly([1,-x]) for x in xs[:i]])
        result += divided_difference(xs[:i+1],fxs[:i+1]) * prod([Poly([1,-x]) for x in xs[:i]])
    return result

xs  = [-.1, .0, .2, .3, 0.35]
fxs = [5.30, 2.00, 3.19, 1.00, 0.9726]
n = newton_divided_difference(xs, fxs, 4)
print(n)

print("*"*10)

from sympy import *

x = symbols("x")
a,b,c,d,e = symbols("a,b,c,d,e")
a,b,c,d = 1,-5,8,0
f = a*x**4 + b*x**3 + c*x**2 + d*x + e
d = lambda g: g.subs(x, x+1) - g
dd = lambda g: d(d(g))
ddd = lambda g: d(dd(g))
dddd = lambda g: dd(dd(g))
print(expand(dd(f)).subs(x, 0))


# 10.007 1D Project PCR Fluorescence Model
# Created by James Raphael Tiovalen (2020)

# Import libraries
import numpy as np
from scipy.optimize import curve_fit
from sklearn import metrics

# Define exponential function
def exponential(C, alpha, beta):
    return alpha * np.exp(beta * C)

# Define logistic function
def logistic(C, k, d):
    return 450 / (1 + np.exp(- k * (C - d)))

# Define cube root function
def cube_root(C, alpha, beta):
    return (((alpha * C) - beta) ** (1 / 3)) + 219.1416667

if __name__ == '__main__':
    cycles = np.array([8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, 32, 34, 36]).astype(np.float64)
    fluorescences = np.array([8.2, 19.1, 37.6, 70.4, 145.9, 219.1416667, 296.7, 367.8, 378.2, 403.3, 424.5, 436.6, 445.1, 446.8, 448.0]).astype(np.float64)

    # Question A2
    old_popt, old_pcov = curve_fit(exponential, cycles[0:5], fluorescences[0:5], method="dogbox", bounds=(0, np.inf))
    old_r_squared = round(metrics.r2_score(exponential(cycles[0:5], *old_popt), fluorescences[0:5]), 3)
    print(f"Equation: F = {old_popt[0]} * e^({old_popt[1]} * C)")
    print(f"R^2 = {old_r_squared}")

    # Question A6
    popt, pcov = curve_fit(logistic, cycles, fluorescences, method="trf", bounds=(0, np.inf))
    r_squared = round(metrics.r2_score(logistic(cycles, *popt), fluorescences), 3)
    print(f"Equation: F = 450 / (1 + e^(- {popt[0]} * (C - {popt[1]})))")
    print(f"R^2 = {r_squared}")

    # Question A8
    new_popt, new_pcov = curve_fit(cube_root, cycles, fluorescences, method="lm")
    new_r_squared = round(metrics.r2_score(cube_root(cycles, *new_popt), fluorescences), 3)
    print(f"Equation: F = ((({new_popt[0]} * C) - {new_popt[1]}) ** (1 / 3)) + 219.1416667")
    print(f"R^2 = {new_r_squared}")


'''
    Problem 87 - Prime power triples
'''
import numpy as np
from numba import jit

@jit
def getPrimes(n):
    arr = np.ones(n + 1, dtype=bool)
    arr[0], arr[1] = 0, 0
    for i in range(2, n + 1):
        if arr[i]:
            arr[i * i:n + 1:i] = False
    return arr.nonzero()[0]

if __name__ == "__main__":
    # print(getPrimes(29))
    N = 50 * (10**6)
    m = int(N**0.5)
    print("m : ", m)
    sol = set()
    primes = getPrimes(m)
    print("Number of primes = ", len(primes))

    sqrs = primes**2
    tripples = primes**3
    tripples = tripples[tripples < N]
    fourth = primes**4
    fourth = fourth[fourth < N]
    for s in sqrs:
        for t in tripples:
            for f in fourth:
                x = s + t + f
                if x < N:
                    sol.add(x)

    print(len(sol))


#cross-entropy.py
# answer - https://classroom.udacity.com/nanodegrees/nd101/parts/94643112-2cab-46f8-a5be-1b6e4fa7a211/modules/4d61ad35-d227-460e-8be6-16407f273e7d/lessons/00912248-2445-4713-ad9c-76b8536e1959/concepts/760235e0-a3ec-4e56-8cdb-56d762886690#
import numpy as np
#%%
# Write a function that takes as input two lists Y, P,
# and returns the float corresponding to their cross-entropy.
def cross_entropy(Y,P):
    Y = np.float_(Y)
    P = np.float_(P)
    return -np.sum(Y * np.log(P) + (1 - Y) * np.log(1 - P))

cross_entropy([],[])

# %%
# natural logorithms

np.log(.999)