import numpy as np
import math

from .utils import *

 # A lot of comments and documentation is directly copied from Raphael Vallat (https://github.com/raphaelvallat/entropy)

# Permutation Entropy
"""Permutation Entropy.

  Parameters
  ----------
  x : np.array
      One-dimensional time series of shape (n_times)
  order : int
      Order of permutation entropy. Default is 3.
  delay : int
      Time delay (lag). Default is 1.
  normalize : bool
      If True, divide by log2(order!) to normalize the entropy between 0
      and 1. Otherwise, return the permutation entropy in bit. Default is true.
  Returns
  -------
  pe : float
      Permutation Entropy.
  Notes
  -----
  The permutation entropy is a complexity measure for time-series first
  introduced by Bandt and Pompe in 2002.
  The permutation entropy of a signal :math:`x` is defined as:
  .. math:: H = -\\sum p(\\pi)\\log_2(\\pi)
  where the sum runs over all :math:`n!` permutations :math:`\\pi` of order
  :math:`n`. This is the information contained in comparing :math:`n`
  consecutive values of the time series. It is clear that
  :math:`0 ≤ H (n) ≤ \\log_2(n!)` where the lower bound is attained for an
  increasing or decreasing sequence of values, and the upper bound for a
  completely random system where all :math:`n!` possible permutations appear
  with the same probability.
  The embedded matrix :math:`Y` is created by:
  .. math::
      y(i)=[x_i,x_{i+\\text{delay}}, ...,x_{i+(\\text{order}-1) *
      \\text{delay}}]
  .. math:: Y=[y(1),y(2),...,y(N-(\\text{order}-1))*\\text{delay})]^T
  References
  ----------
  Bandt, Christoph, and Bernd Pompe. "Permutation entropy: a
  natural complexity measure for time series." Physical review letters
  88.17 (2002): 174102.
  Examples
  --------
  Permutation entropy with order 2
  >>> from OrdinalEntroPy import *
  >>> import numpy as np
  >>> x = [4, 7, 9, 10, 6, 11, 3]
  >>> # Return a value in bit between 0 and log2(factorial(order))
  >>> print(PE(x, order=2, normalize=False))
  0.9182958340544896
  Normalized permutation entropy with order 3
  >>> from OrdinalEntroPy import *
  >>> import numpy as np
  >>> x = [4, 7, 9, 10, 6, 11, 3]
  >>> # Return a value comprised between 0 and 1.
  >>> print(PE(x, order=3, normalize=True))
  0.5887621559162939
"""
def PE(values,order=3,delay=1,normalize=True):

  # get all the permuations
  str_permutations = get_str_permutation_ordinal(values,order,delay)

  # get set of indices for each unique permutation
  permutation_indexes = get_permutation_index(str_permutations)

  # get frequency of each permutation pattern
  permutation_frequency = get_permutation_frequency(permutation_indexes,len(values),order)

  # get shannon entropy of frequencies
  entropy = get_shanon_entropy(permutation_frequency)

  #Normalize
  if normalize:
    entropy = entropy/math.log2(math.factorial(order))

  return entropy

"""Dispersion Entropy.

  Parameters
  ----------
  x : np.array
      One-dimensional time series of shape (n_times)
  order : int
      Order of permutation entropy. Default is 3.
  classes : int
      Number of classes. Default is 3.
  delay : int
      Time delay (lag). Default is 1.
  normalize : bool
      If True, divide by log2(classes**order) to normalize the entropy between 0
      and 1. Otherwise, return the permutation entropy in bit. Default is true.
  Returns
  -------
  pe : float
      Permutation Entropy.
  Notes
  -----
  Dispersion Entropy (DE) was introduced in the year 2016 by Azami and Rostaghi
  to quantify the complexity of time series.
  The Dispersion entropy of a signal :math:`x` is defined as:
  .. math:: H = -\\sum p(\\pi)\\log_2(\\pi)
  where the sum runs over all :math:`classes**order` permutations :math:`\\pi` of order
  :math:`n` and consisting of classes :math:`c`. This is the information contained in comparing :math:`n`
  consecutive values of the time series. It is clear that
  :math:`0 ≤ H (n) ≤ \\log_2(classes^order)` where the lower bound is attained for an
  increasing or decreasing sequence of values, and the upper bound for a
  completely random system where all :math:`classes**order` possible dispersion patterns appear
  with the same probability.
  The embedded matrix :math:`Y` is created by:
  .. math::
      y(i)=[x_i,x_{i+\\text{delay}}, ...,x_{i+(\\text{order}-1) *
      \\text{delay}}]
  .. math:: Y=[y(1),y(2),...,y(N-(\\text{order}-1))*\\text{delay})]^T
  References
  ----------
  M. Rostaghi and H. Azami, "Dispersion Entropy: A Measure for Time-Series Analysis,"
  in IEEE Signal Processing Letters, vol. 23, no. 5, pp. 610-614, May 2016, doi: 10.1109/LSP.2016.2542881.
  Examples
  --------
  Dispersion entropy with order=3 and classes=3
  >>> from OrdinalEntroPy import *
  >>> import numpy as np
  >>> np.random.seed(1234567)
  >>> x = np.random.rand(3000)
  >>> # Return a value in bit between 0 and log2(factorial(order))
  >>> print(DE(x, order=3,classes=3,normalize=True))
  0.9830685145488814
"""

# Dispersion Entropy
def DE(values,order=3,classes=3,delay=1,normalize=True):
  # map TS to classes using cummulative distributive function
  mapped_values = get_ncdf_values(values,classes)
  # get all the permuations
  str_permutations = get_str_permutation(mapped_values,order,delay)

  # get set of indices for each unique permutation
  permutation_indexes = get_permutation_index(str_permutations)

  # get frequency of each permutation pattern
  permutation_frequency = get_permutation_frequency(permutation_indexes,len(values),order)

  # get shannon entropy of frequencies
  entropy = get_shanon_entropy(permutation_frequency)
  if normalize:
    entropy = entropy/math.log2(classes**order)

  return entropy

# Reverse Dispersion Entropy
def RDE(values,order=3,classes=3,delay=1,normalize=True):
  mapped_values = get_ncdf_values(values,classes)
  # get all the permuations
  str_permutations = get_str_permutation(mapped_values,order,delay)

  permutation_indexes = get_permutation_index(str_permutations)

  permutation_frequency = get_permutation_frequency(permutation_indexes,len(values),order)

  entropy = np.square(permutation_frequency).sum() - (1/(classes**order))
  if normalize:
    entropy = entropy/(1 - (1/(classes**order)))

  return entropy

# Reverse Permutation Entropy
def RPE(values,order=3,delay=1,normalize=True):
  str_permutations = get_str_permutation_ordinal(values,order,delay)

  permutation_indexes = get_permutation_index(str_permutations)
  #print(set(str_permutations))

  permutation_frequency = get_permutation_frequency(permutation_indexes,len(values),order)
  #print(permutation_frequency)
  entropy = np.square(permutation_frequency).sum() - (1/math.factorial(order))
  if normalize:
    entropy = entropy/(1 - (1/math.factorial(order)))

  return entropy

# Weighted Permutation Entropy
def WPE(values,order=3,delay=1,normalize=True):
  weights = get_weights(values,order,delay)

  str_permutations = get_str_permutation_ordinal(values,order,delay)

  permutation_indexes = get_permutation_index(str_permutations)

  weighted_permutations = cal_weighted_permutations(np.array(weights),permutation_indexes)

  entropy = get_shanon_entropy(weighted_permutations)

  if normalize:
    entropy = entropy/math.log2(math.factorial(order))

  return entropy

  # Reverse Weighted Permutation Entropy
def RWPE(values,order,delay=1,normalize=True):
  weights = get_weights(values,order,delay)

  str_permutations = get_str_permutation_ordinal(values,order,delay)

  permutation_indexes = get_permutation_index(str_permutations)

  weighted_permutations = cal_weighted_permutations(np.array(weights),permutation_indexes)

  entropy = np.square(weighted_permutations).sum() - (1/math.factorial(order))
  if normalize:
    entropy = entropy/(1 - (1/math.factorial(order)))

  return entropy

# Reverse weighted Dispersion Entropy
def RWDE(values,order,classes,delay=1,normalize=True):
  # find variance of each permutation
  weights = get_weights(values,order,delay)
  # convert values with NCDF and assign them class
  mapped_values = get_ncdf_values(values,classes)
  # get all the permuations
  str_permutations = get_str_permutation(mapped_values,order,delay)

  #find the indices of a permutation
  permutation_indexes = get_permutation_index(str_permutations)

  # get the weight for each permutation
  weighted_permutations = cal_weighted_permutations(np.array(weights),permutation_indexes)

  entropy = np.square(weighted_permutations).sum() - (1/(classes**order))
  # calculate final RWDE entropy
  if normalize:
    entropy = entropy/(1 - (1/(classes**order)))

  return entropy


import numpy as np
from scipy import linalg
from numpy import dot

def nmf(X, latent_features, max_iter=100, error_limit=1e-6, fit_error_limit=1e-6):
	"""
	Decompose X to A*Y
	"""
	eps = 1e-5
	print 'Starting NMF decomposition with {} latent features and {} iterations.'.format(latent_features, max_iter)
	#X = X.toarray()  # I am passing in a scipy sparse matrix

	# mask
	mask = np.sign(X)
	rows, columns = X.shape
	A = np.random.rand(rows, latent_features)
	A = np.maximum(A, eps)

	Y = linalg.lstsq(A, X)[0]
	Y = np.maximum(Y, eps)

	masked_X = mask * X
	X_est_prev = dot(A, Y)
	for i in range(1, max_iter + 1):
		top = dot(masked_X, Y.T)
		bottom = (dot((mask * dot(A, Y)), Y.T)) + eps
		A *= top / bottom

		A = np.maximum(A, eps)
		top = dot(A.T, masked_X)
		bottom = dot(A.T, mask * dot(A, Y)) + eps
		Y *= top / bottom
		Y = np.maximum(Y, eps)
		if i % 5 == 0 or i == 1 or i == max_iter:
			print 'Iteration {}:'.format(i),
			X_est = dot(A, Y)
			err = mask * (X_est_prev - X_est)
			fit_residual = np.sqrt(np.sum(err ** 2))
			X_est_prev = X_est
			curRes = linalg.norm(mask * (X - X_est), ord='fro')
			print 'fit residual', np.round(fit_residual, 4),
			print 'total residual', np.round(curRes, 4)
			if curRes < error_limit or fit_residual < fit_error_limit:
				break
	return A, Y

if __name__ == "__main__":
    R = [
         [5,3,0,1],
         [4,0,0,1],
         [1,1,0,5],
         [1,0,0,4],
         [0,1,5,4],
        ]

    R = np.array(R)
    print R

    nP, nQ = nmf(R, 2)
    print np.dot(nP, nQ)
	# To restore R, execute numpy.dot(np, nQ.T)


#coding:utf8
import numpy as np

def sigmoid(x):
    """
    Compute the sigmoid of x

    Arguments:
    x -- A scalar or numpy array of any size

    Return:
    s -- sigmoid(x)
    """
    s = 1 / (1 + np.exp(-x))
    return s

def sigmoid_derivative(x):
    """
    Compute the gradient (also called the slope or derivative) of the sigmoid function with respect to its input x.
    You can store the output of the sigmoid function into variables and then use it to calculate the gradient.

    Arguments:
    x -- A scalar or numpy array

    Return:
    ds -- Your computed gradient.
    """
    s = sigmoid(x)
    ds = s * (1 - s)
    return ds

# GRADED FUNCTION: image2vector
def image2vector(image):
    """
    Argument:
    image -- a numpy array of shape (length, height, depth)

    Returns:
    v -- a vector of shape (length*height*depth, 1)
    """
    v = image.reshape((image.shape[0] * image.shape[1] * image.shape[2]), 1)

    return v

# GRADED FUNCTION: normalizeRows

def normalizeRows(x):
    """
    Implement a function that normalizes each row of the matrix x (to have unit length).

    Argument:
    x -- A numpy matrix of shape (n, m)

    Returns:
    x -- The normalized (by row) numpy matrix. You are allowed to modify x.
    """
    # Compute x_norm as the norm 2 of x. Use np.linalg.norm(..., ord = 2, axis = ..., keepdims = True)
    # keepdims is keep the np.dims, in this case, [2, 3] will become [2, 1], but not [2, ]
    x_norm = np.linalg.norm(x, ord=2, axis=1, keepdims=True)
    # x_norm = np.linalg.norm(x, ord = 2, axis = 1)
    # print(x_norm.shape)
    # Divide x by its norm.
    x = x / x_norm
    return x

# GRADED FUNCTION: softmax

def softmax(x):
    """Calculates the softmax for each row of the input x.

    Your code should work for a row vector and also for matrices of shape (n, m).

    Argument:
    x -- A numpy matrix of shape (n,m)

    Returns:
    s -- A numpy matrix equal to the softmax of x, of shape (n,m)
    """
    # Apply exp() element-wise to x. Use np.exp(...).
    x_exp = np.exp(x)
    # Create a vector x_sum that sums each row of x_exp. Use np.sum(..., axis = 1, keepdims = True).
    x_sum = np.sum(x_exp, axis=1, keepdims=True)
    # Compute softmax(x) by dividing x_exp by x_sum. It should automatically use numpy broadcasting.
    s = x_exp / x_sum
    # print("x_exp: {}, x_sum: {}".format(x_exp.shape, x_sum.shape))

    return s

# The loss is used to evaluate the performance of your model. The bigger your loss is,
# the more different your predictions ($ \hat{y} $) are from the true values ($y$). In deep learning,
# you use optimization algorithms like Gradient Descent to train your model and to minimize the cost.
# GRADED FUNCTION: L1
# L1正则
def L1(yhat, y):
    """
    Arguments:
    yhat -- vector of size m (predicted labels)
    y -- vector of size m (true labels)

    Returns:
    loss -- the value of the L1 loss function defined above
    """
    ### START CODE HERE ### (≈ 1 line of code)
    loss = np.sum(np.abs(yhat - y), axis=0)
    ### END CODE HERE ###

    return loss

# GRADED FUNCTION: L2

def L2(yhat, y):
    """
    Arguments:
    yhat -- vector of size m (predicted labels)
    y -- vector of size m (true labels)

    Returns:
    loss -- the value of the L2 loss function defined above
    """
    loss = np.dot(abs(y - yhat), abs(y - yhat).T)
    # loss = np.square(np.linalg.norm(abs(y - yhat), ord = 2, axis = 0))
    return loss


import math
from scipy.special import erf
from numpy import poly1d
from numpy import pi, sin, linspace
from numpy import exp, cos

# scipy erfc does not support complex numbers but erf does
def erfc(z):
        if z == complex(0,0):
          return 1.0
        else:
          return 1.0-erf(z)

def analytic_solution_simple(t, x, a_0, L, g, eta): # time, x_value, initial maximum perturbation (at the very left), wavelength, gravity, viscosity
        k= 2*pi/L # wave number 2pi/wavelength
        gamma=2*eta*k**2
        a=a_0*cos(pi*x)
        return a*exp(-gamma*t)

# Formulas were compared with the same formula hacked into matlab
# based on
# Motion of two superposed viscous fluids
# A Prosperetti - Physics of Fluids, 1981
# doi:10.1063/1.863522
# assumes that fluid in the tank has same viscosity as fluid above
def analytic_solution(t, x, a_0, L, g, eta): # time, x_value, initial maximum perturbation (at the very left), wavelength, gravity, viscosity
  debug=False
  eta=eta/2.0
  k= 2.0*pi/L # wave number 2pi/wavelength
  omega_0sq = g*k  # inviscid natural frequency
  a=a_0*cos(pi*x)

  p1 = poly1d([1.0,0.0,2*k**2*eta,4*k**3*eta**(1.5),eta**2*k**4+omega_0sq],r=0)

  p1roots = p1.r
  z1=p1roots[0]
  z2=p1roots[1]
  z3=p1roots[2]
  z4=p1roots[3]
  Z1=(z2-z1)*(z3-z1)*(z4-z1)
  Z2=(z1-z2)*(z3-z2)*(z4-z2)
  Z3=(z1-z3)*(z2-z3)*(z4-z3)
  Z4=(z1-z4)*(z2-z4)*(z3-z4)

  if debug:
                print 'Calculate analytic solution:'
                print 't=', t
                print 'a=', a
                print 'k=', k
                print 'g=', g
                print 'eta=', eta
                print 'omega_0sq=', omega_0sq

  t0=4*eta**2*k**4/(8*eta**2*k**4+omega_0sq)*a*erfc((eta*k**2*t)**0.5)
  t1=z1/Z1*omega_0sq*a/(z1**2-eta*k**2)*exp((z1**2-eta*k**2)*t)*erfc(z1*t**0.5)
  t2=z2/Z2*omega_0sq*a/(z2**2-eta*k**2)*exp((z2**2-eta*k**2)*t)*erfc(z2*t**0.5)
  t3=z3/Z3*omega_0sq*a/(z3**2-eta*k**2)*exp((z3**2-eta*k**2)*t)*erfc(z3*t**0.5)
  t4=z4/Z4*omega_0sq*a/(z4**2-eta*k**2)*exp((z4**2-eta*k**2)*t)*erfc(z4*t**0.5)

  a=t0+t1+t2+t3+t4

  if debug:
                print 'a(t)=', a.real
  if (a.imag>0.000001):
                print 'Warning: Imaginary part of a(t) is not zero!'
  return a.real


import numpy as np

def sigmoid(z):
    return 1 / (1 + np.exp(-z))

def predict(X, w):
    linear_combination = np.matmul(X, w)
    return sigmoid(linear_combination)

def classify(X, w):
    return np.round(predict(X, w))

def loss (X, Y, w):
    predictions = predict(X, w)
    first_term = Y * np.log(predictions)
    second_term = (1 - Y) * np.log(1 - predictions)
    return (-1) * np.average(first_term + second_term)

def gradient(X, Y, w):
    predictions = predict(X, w)
    return np.matmul(X.T, (predictions - Y)) / (2 * X.shape[0])

def train(X, Y, iterations, learning_rate):
    w = np.zeros((X.shape[1], 1))
    for i in range(iterations):
        print("Iterations: %4d => Loss: %.20f" % (i, loss(X, Y, w)))
        w -= gradient(X, Y, w) * learning_rate
    return w

def test(X, Y, w):
    total_examples = X.shape[0]
    correct_results = np.sum(classify(X, w) == Y)
    success_percent = correct_results * 100 / total_examples

    print("\nSuccess: %d/%d (%.2f%%)" % (correct_results, total_examples, success_percent))

if __name__ == "__main__":
    X1, X2, X3, Y = np.loadtxt("examples.txt", skiprows=1, unpack=True)
    X = np.column_stack((np.ones(X1.size), X1, X2, X3))
    Y = Y.reshape(-1, 1)
    iterations = 10000
    learning_rate = 0.001
    w = train(X, Y, iterations, learning_rate)

    test(X, Y, w)

    input("Press <Enter>")


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Sat May  2 16:13:25 2020

@author: tyrion
"""

import numpy as np

def vectors_to_polar(vectors, deg = False):
    '''
    Transform a set of vectors to polar coordinates

    Arguments
    ----------
    vectors : ndarray (n, 3)
        Vector data
    deg : bool, optional, default False
        If True, returns output polar angles in degrees. If False, returns angles in radian.
    Returns
    ----------
    theta : ndarray (size, )
        Polar angle
    phi : ndarray (size, )
        Azimuthal angle
    '''

    phi = np.arctan2(vectors[:, 1],vectors[:, 0])
    theta = np.arccos(vectors[:, 2])

    if deg == True:

        phi = np.rad2deg(phi)
        theta = np.rad2deg(theta)

    return theta, phi

def polar_to_vectors(theta, phi, deg = False):
    '''
    Transform a set of polar coordinates to vectors

    Arguments
    ----------
    theta : ndarray (n, )
        Polar angle
    phi : ndarray (n, )
        Azimuthal angle
    deg : bool, optional, default False
        If True, assumes that input is in degrees. If False, assumes that input is in radian.
    Returns
    ----------
    vectors : ndarray (n, 3)
        Vector data
    '''

    if deg == True:

        phi = np.deg2rad(phi)
        theta = np.deg2rad(theta)

    sintheta = np.sin(theta)

    vecs = np.empty((theta.shape[0],3))
    vecs[:, 0] = sintheta*np.cos(phi)
    vecs[:, 1] = sintheta*np.sin(phi)
    vecs[:, 2] = np.cos(theta)

    return vecs

def vectors_to_geographical(vectors, deg = False):
    '''
    Transform a set of vectors to geographical coordinates

    Arguments
    ----------
    vectors : ndarray (n, 3)
        Vector data
    deg : bool, optional, default False
        If True, returns output geographical angles in degrees. If False, returns angles in radian.
    Returns
    ----------
    latitude : ndarray (n, )
        Geographical latitude
    longitude : ndarray (n, )
        Geographical longitude
    '''

    longitude = np.arctan2(vectors[:, 1],vectors[:, 0])
    latitude = np.arcsin(vectors[:, 2])

    if deg == True:

        latitude = np.rad2deg(latitude)
        longitude = np.rad2deg(longitude)

    return latitude, longitude

def geographical_to_vectors(latitude, longitude, deg = False):
    '''
    Transform a set of geographical coordinates to vectors

    Arguments
    ----------
    latitude : ndarray (n, )
        Polar angle
    longitude : ndarray (n, )
        Azimuthal angle
    deg : bool, optional, default False
        If True, assumes that input is in degrees. If False, assumes that input is in radian.
    Returns
    ----------
    vectors : ndarray (n, 3)
        Vector data
    '''

    if deg == True:

        latitude = np.deg2rad(latitude)
        longitude = np.deg2rad(longitude)

    vectors = np.empty((latitude.shape[0],3))

    coslat = np.cos(latitude)

    vectors[:, 0] = coslat * np.cos(longitude)
    vectors[:, 1] = coslat * np.sin(longitude)
    vectors[:, 2] = np.sin(latitude)

    return vectors


from typing import Callable
import numpy as np

def simpsonsRule(f: Callable, N: float, start: float=-1e6,
                 stop: float=1e6) -> float:
    """Function to approximate the numeric integral of a function, f, using
    Simpson's rule.

    Arguments:
        f {Callable} -- Function for which the integral is to be estimated.
        N {float} -- Number of nodes to consider.

    Keyword Arguments:
        start {float} -- Starting point (default: {-1e6}).
        stop {float} -- Stopping point (default: {1e6}).

    Returns:
        float -- Approximation of the area under the function.
    """

    # Building values for approximation, and getting step size
    x, h = np.linspace(start=start, stop=stop, num=N, retstep=True)

    # Computing midpoints
    x_mid = np.array([(x[i - 1] + x[i]) / 2 for i in range(1, N)])

    # Estimating using Simpson's rule
    area = np.sum(2 * f(x)) - (f(start) + f(stop)) + (4 * np.sum(f(x_mid)))

    # Scaling area
    area *= (h / 6)

    return area


import numpy as np

'''

Cross-Entropy is an error function that we want to minimize.
To calculate Cross-Entropy(CE):

CE = negative summation of the logarithms of the probabilities whenever there is or there's none
simply
 = - summ [(y_i)ln(p_i) + (1-y_i)ln(1-p_i)]
where:
y is either 1 if there is or 0 if there's none
p is the probability that there is

'''

def cross_entropy(Y, P):
    result = []
    for i in range(0,len(Y)):
        result.append(-1*(Y[i]*np.log(P[i]) + (1-Y[i])*np.log(1-P[i])))
    CE = np.sum(result)
    return CE

'''
Udacity's solution which is cleaner

import numpy as np

def cross_entropy(Y, P):
    Y = np.float_(Y)
    P = np.float_(P)
    return -np.sum(Y * np.log(P) + (1 - Y) * np.log(1 - P))
'''

Lesson 3: Introduction to Neural Networks/3 Cross Entropy.py


from math import copysign, isnan
import numpy as np

test = False

with open(f"../input/{'test_' if test else ''}day7.txt") as f:
    # :pinchers:
    crabs = np.array([int(x) for x in f.read().strip().split(",")])

def sim_p1(h):
    # d/dh | x - h | = sgn(h - x)
    return np.sum(np.abs(crabs - h)), np.sum(np.sign(h - crabs))

def sim_p2(h):
    # let k = |x_i - h|
    # cost = k * (k + 1) / 2
    # so d/dh = sgn(h - x) / 2 + (h - x)

    d   = np.abs(crabs - h)
    err = np.sum((d * (d + 1)) / 2)
    dh  = np.sum(.5 * np.sign(h - crabs) + (h - crabs))

    return err, dh

def solve(sim):
    # very random
    sol = 5
    alpha = .00005

    for i in range(100000):
        err, dh = sim(sol)

        if abs(dh) < 1e-4:
            break

        sol -= dh * alpha

    err, _ = sim(round(sol))
    print(f"sol: {sol} -> {round(sol)}, err: {err}")

if __name__ == "__main__":
    solve(sim_p1)
    solve(sim_p2)


# Q.2 How do the values of options at time t = 0 compare for various values of M? Compute and plot graphs (of the
# initial option prices) varying M in steps of 1 and in steps of 5. What do you observe about the convergence of
# option prices?

# Pandas : pip install pandas
# Matplotlib: pip install matplotlib
# Numpy: pip install numpy
# Ipython: pip install ipython

import math
import numpy as np
import matplotlib.pyplot as plt

# Given data
S0=100
K=105
T=5
r=0.05
sig=0.3

# Function to get Option Price for a given M
def getOptionPrice(M):
    dt = T/M
    u = math.exp(sig*math.sqrt(dt)+(r-sig*sig/2)*dt)
    d = math.exp(-sig*math.sqrt(dt)+(r-sig*sig/2)*dt)
    p = (math.exp(r*dt)-d)/(u-d)

    # Check if No Arbitrage Principle has got violated
    if p < 0 or p > 1:
        print("No Arbitrage Principle has been Violated")
        return '-','-'

    callList = [0]*(M+1)
    putList = [0]*(M+1)

    for i in range(M+1):
        callList[i] = max(S0*(u**i)*(d**(M-i)) - K, 0)
        putList[i] = max(0, K - S0*(u**i)*(d**(M-i)))

    for i in range(M):
        for j in range(M-i):
            callList[j] = ((1-p)*callList[j] + p*callList[j+1])*math.exp(-r*T/M)
            putList[j] = ((1-p)*putList[j] + p*putList[j+1])*math.exp(-r*T/M)
    return callList[0], putList[0]

# Lists to store the option prices
callPrices = []
putPrices = []
M=0
# Compute initial option prices in steps of 1
while M < 400:
    M += 1
    call, put = getOptionPrice(M)
    callPrices.append(call)
    putPrices.append(put)
MList = np.linspace(1, 400, 400)

plt.plot(MList, callPrices)
plt.xlabel('Value of M')
plt.ylabel('Call Option Price')
plt.title('Varying Price of Call Option with Value of M (Step Size 1)')
plt.show()

plt.plot(MList, putPrices)
plt.xlabel('Value of M')
plt.ylabel('Price of Put Option')
plt.title('Varying Price of Put Option with Value of M (Step Size 1)')
plt.show()

# Lists to store the option prices
callPrices = []
putPrices = []

# Compute initial option prices in steps of 5
M=0
while M < 400:
    M += 5
    call, put = getOptionPrice(M)
    callPrices.append(call)
    putPrices.append(put)
MList = np.linspace(1, 400, 80)

plt.plot(MList, callPrices)
plt.xlabel('Value of M')
plt.ylabel('Call Option Price')
plt.title('Varying Call Option Price with Value of M (Step Size 5)')
plt.show()

plt.plot(MList, putPrices)
plt.xlabel('Value of M')
plt.ylabel('Price of Put Option')
plt.title('Varying Put Option Price with Value of M (Step Size 5)')
plt.show()

Semester 6/MA 374 (Financial Engg. Lab)/Lab 1/180123062_ABSatyaprakash_q2 1.py


import numpy as np
import math
from scipy import stats

x = np.array([14.4,7.2,27.5,53.8,38.0,15.9,4.9])
y = np.array([54,64,44,32,37,68,62])

n = x.size

rank_x = np.array([5,6,3,2,1,4,7])
rank_y = np.array([4,2,5,7,6,1,3])

d = rank_x - rank_y
d_square = np.square(d)
sum_d_square = np.sum(d_square)

print("d : ",d)
print("d_square : ",d_square)

# if there are no duplicates
coeff_numerator = 6*sum_d_square

# if there are duplicates
coeff_numerator_dup = 6*(sum_d_square+((6+24)/12))

coeff_denominator = n*(math.pow(n,2)-1)
coeff = 1-(coeff_numerator/coeff_denominator)

print("rank co-relation : ",coeff)


# Databricks notebook source
# MAGIC %md
# MAGIC ScaDaMaLe Course [site](https://lamastex.github.io/scalable-data-science/sds/3/x/) and [book](https://lamastex.github.io/ScaDaMaLe/index.html)
# MAGIC
# MAGIC This is a 2019-2021 augmentation and update of [Adam Breindel](https://www.linkedin.com/in/adbreind)'s initial notebooks.
# MAGIC
# MAGIC _Thanks to [Christian von Koch](https://www.linkedin.com/in/christianvonkoch/) and [William Anzén](https://www.linkedin.com/in/william-anz%C3%A9n-b52003199/) for their contributions towards making these materials Spark 3.0.1 and Python 3+ compliant._

# COMMAND ----------

# MAGIC %md
# MAGIC #### We can also implement the model with mini-batches -- this will let us see matrix ops in action:
# MAGIC
# MAGIC (N.b., feed_dict is intended for small data / experimentation. For more info on ingesting data at scale, see https://www.tensorflow.org/api_guides/python/reading_data)

# COMMAND ----------

# we know these params, but we're making TF learn them

REAL_SLOPE_X1 = 2 # slope along axis 1 (x-axis)
REAL_SLOPE_X2 = 3 # slope along axis 2 (y-axis)
REAL_INTERCEPT = 5 # intercept along axis 3 (z-axis), think of (x,y,z) axes in the usual way

# COMMAND ----------

import numpy as np
# GENERATE a batch of true data, with a little Gaussian noise added

def make_mini_batch(size=10):
  X = np.random.rand(size, 2) #
  Y = np.matmul(X, [REAL_SLOPE_X1, REAL_SLOPE_X2]) + REAL_INTERCEPT + 0.2 * np.random.randn(size)
  return X.reshape(size,2), Y.reshape(size,1)

# COMMAND ----------

# MAGIC %md
# MAGIC To digest what's going on inside the function above, let's take it step by step.

# COMMAND ----------

 Xex = np.random.rand(10, 2) # Xex is simulating PRNGs from independent Uniform [0,1] RVs
 Xex # visualize these as 10 orddered pairs of points in the x-y plane that makes up our x-axis and y-axis (or x1 and x2 axes)

# COMMAND ----------

Yex = np.matmul(Xex, [REAL_SLOPE_X1, REAL_SLOPE_X2]) #+ REAL_INTERCEPT #+ 0.2 * np.random.randn(10)
Yex

# COMMAND ----------

# MAGIC %md
# MAGIC The first entry in Yex is obtained as follows (change the numbers in the produc below if you reevaluated the cells above) and geometrically it is the location in z-axis of the plane with slopes given by REAL_SLOPE_X1 in the x-axis and REAL_SLOPE_X2 in the y-aixs with intercept 0 at the point in the x-y or x1-x2 plane given by (0.68729439,  0.58462379).

# COMMAND ----------

0.21757443*REAL_SLOPE_X1 +  0.01815727*REAL_SLOPE_X2

# COMMAND ----------

# MAGIC %md
# MAGIC The next steps are adding an intercept term to translate the plane in the z-axis and then a scaled (the multiplication by 0.2 here) gaussian noise from independetly drawn pseudo-random samples from the standard normal or Normal(0,1) random variable via `np.random.randn(size)`.

# COMMAND ----------

Yex = np.matmul(Xex, [REAL_SLOPE_X1, REAL_SLOPE_X2]) + REAL_INTERCEPT # + 0.2 * np.random.randn(10)
Yex

# COMMAND ----------

Yex = np.matmul(Xex, [REAL_SLOPE_X1, REAL_SLOPE_X2])  + REAL_INTERCEPT + 0.2 * np.random.randn(10)
Yex # note how each entry in Yex is jiggled independently a bit by 0.2 * np.random.randn()

# COMMAND ----------

# MAGIC %md
# MAGIC Thus we can now fully appreciate what is going on in `make_mini_batch`. This is meant to substitute for pulling random sub-samples of batches of the real data during stochastic gradient descent.

# COMMAND ----------

make_mini_batch() # our mini-batch of Xx and Ys

# COMMAND ----------

import tensorflow as tf

batch = 10 # size of batch

tf.reset_default_graph() # this is important to do before you do something new in TF

# we will work with single floating point precision and this is specified in the tf.float32 type argument to each tf object/method
x = tf.placeholder(tf.float32, shape=(batch, 2)) # placeholder node for the pairs of x variables (predictors) in batches of size batch
x_aug = tf.concat( (x, tf.ones((batch, 1))), 1 ) # x_aug is a concatenation of a vector of 1`s along the first dimension

y = tf.placeholder(tf.float32, shape=(batch, 1)) # placeholder node for the univariate response y with batch many rows and 1 column
model_params = tf.get_variable("model_params", [3,1]) # these are the x1 slope, x2 slope and the intercept (3 rows and 1 column)
y_model = tf.matmul(x_aug, model_params) # our two-factor regression model is defined by this matrix multiplication
# note that the noise is formally part of the model and what we are actually modeling is the mean response...

error = tf.reduce_sum(tf.square(y - y_model))/batch # this is mean square error where the sum is computed by a reduce call on addition

train_op = tf.train.GradientDescentOptimizer(0.02).minimize(error) # learning rate is set to 0.02

init = tf.global_variables_initializer() # our way into running the TF session

errors = [] # list to track errors over iterations

with tf.Session() as session:
    session.run(init)
    for i in range(1000):
      x_data, y_data = make_mini_batch(batch) # simulate the mini-batch of data x1,x2 and response y with noise
      _, error_val = session.run([train_op, error], feed_dict={x: x_data, y: y_data})
      errors.append(error_val)

    out = session.run(model_params)
    print(out)

# COMMAND ----------

REAL_SLOPE_X1, REAL_SLOPE_X2, REAL_INTERCEPT # compare with rue parameter values - it's not too far from the estimates

# COMMAND ----------

import matplotlib.pyplot as plt

fig, ax = plt.subplots()
fig.set_size_inches((4,3))
plt.plot(errors)
display(fig)

# COMMAND ----------


# -*- coding: utf-8 -*-
"""
Computes the Legendre-Gauss-Lobatto nodes, weights and the LGL Vandermonde
matrix. The LGL nodes are the zeros of (1-x**2)*P'_N(x). Useful for numerical
integration and spectral methods.

Reference on LGL nodes and weights:
  C. Canuto, M. Y. Hussaini, A. Quarteroni, T. A. Tang, "Spectral Methods
  in Fluid Dynamics," Section 2.3. Springer-Verlag 1987

Originally implemented in MATLAB by Greg von Winckel
http://www.mathworks.com/matlabcentral/fileexchange/4775-legende-gauss-lobatto-nodes-and-weights

@author: Nicolas Guarin-Zapata
"""
from __future__ import division, print_function
from numpy import pi, sin, zeros, amax, abs, square, array, linspace
from numpy.linalg import norm
import numpy as np

def gauss_lobatto(N, tol=1e-15):
    """
    Use Chebyshev nodes as first guess.

    Compute P_(N) using the recursion relation
    Compute its first and second derivatives and
    update x using the Newton-Raphson method.

    Chebyshev nodes are computed using the symmetric
    formula that involves sine as presented in:

      Nick Trefethen. Aproximation Theory and
      Aproximation Practice, Chapter 2, exercise 2.

    """
    x = sin(linspace(-pi/2, pi/2, N))
    P = zeros((N, N))  # Vandermonde Matrix
    x_old = 2
    while amax(abs(x - x_old)) > tol:
        x_old = x
        P[:, 0] = 1
        P[:, 1] = x
        for k in range(2, N):
            P[:, k] = ((2 * k - 1) * x * P[:, k - 1] -
                       (k - 1) * P[:, k - 2]) / k
        x = x_old - (x * P[:, N - 1] - P[:, N - 2]) / (N * P[:, N - 1])

    w = 2.0 / ((N - 1) * N * square(P[:, N - 1]))

    return array(x), array(w)

if __name__ == "__main__":
    x, _ = gauss_lobatto(5)
    x_exact = array([-1, -np.sqrt(3/7), 0, np.sqrt(3/7), 1])
    print("Relative error: {:g}".format(norm(x - x_exact)/norm(x_exact)))


# TODO: Create a synthetic dataset - how would the data need to look like if we were able to fit the sine wave on them?

# You want to reverse engineer data which would fit onto your 'ideal' curve:
# Eq: y = amp * np.sin (2* np.pi / 24 * x + shift_h) + shift_v
# Without shift_h:  2.63 * np.sin (2 * np.pi / 24 * x) + 17.18

import numpy as np
import matplotlib.pyplot as plt

def sine_function(x, amp, per, shift_h, shift_v):
    return amp * np.sin(2*np.pi/per * x + shift_h) + shift_v

amp, per, shift_h, shift_v = 2.63, 24.0, 0, 17.18

repeats = 4
x_sine = np.linspace(0, repeats * 24 + 1, 100)
y_sine = sine_function(x_sine, amp, per, shift_h, shift_v)

# Now do the reverse engineering for 5-generational families with phases ranging between 0-24:
def SynthesizeData(phase):
    """ Phase ranges from 0 to 24 with increments of 0.2. """

    x_list = [phase]
    y_list = []

    while len(x_list) < 5 or len(y_list) < 5:
        x = x_list[-1]
        y = sine_function(x=x, amp=amp, per=per, shift_h=shift_h, shift_v=shift_v)
        x_list.append(y+x)
        y_list.append(y)
    x_list = x_list[:-1]

    return x_list, y_list

phase_range = np.linspace(0, 24, 120)
print (phase_range)

x_data_0 = []
y_data_0 = []
for phase in list(phase_range.tolist()):
    x_list, y_list = SynthesizeData(phase=phase)
    plt.scatter(x=x_list, y=y_list, color="purple", alpha=0.2, zorder=2) # label="Fake Data Point",
    y_data_0.append(y_list)
    x_data_0.append([value - x_list[0] for value in x_list])

# Now plot the thing:
plt.plot(x_sine, y_sine, color="turquoise", label="Ideal Sine Wave", zorder=1)
plt.axhline(y=shift_v, color="grey", linestyle="dashed", zorder=0)
plt.axvline(x=24.0, color="gold", linestyle="dashed", zorder=0)
plt.axvline(x=48.0, color="gold", linestyle="dashed", zorder=0)
plt.axvline(x=72.0, color="gold", linestyle="dashed", zorder=0)
plt.xticks(np.arange(0, repeats * 24 + 1, 4))
plt.xlabel("Oscillation Period [hours]")
plt.ylabel("Cell Cycle Duration [hours]")
plt.title("Ideal Sine Wave")
plt.legend(loc='upper center', bbox_to_anchor=(0.5, -0.15), ncol=1)
plt.savefig("/Users/kristinaulicna/Documents/Rotation_2/Sine_Wave/Summary_All_Cells/Ideal_Wave.png", bbox_inches="tight")
plt.show()
plt.close()

# Now plot the non-phased data:
for x, y in zip(x_data_0, y_data_0):
    plt.scatter(x=x, y=y)

plt.xlabel("Oscillation Period [hours]")
plt.ylabel("Cell Cycle Duration [hours]")
plt.title("Relationships between generations in an ideal family\n"
          "Equation: {} * np.sin (2 * np.pi / {} * x + {}) + {}"
          .format(amp, per, shift_h, shift_v))
plt.savefig("/Users/kristinaulicna/Documents/Rotation_2/Sine_Wave/Summary_All_Cells/Ideal_Wave_Relationships.png",
            bbox_inches="tight")
plt.show()
plt.close()


#Determining a linear trend line for Bitcoin
import numpy as np
from scipy.linalg import lstsq

year = np.array([2010.91666666666, 2011.41666666666, 2011.91666666666,
             2012.41666666666, 2012.91666666666, 2013.41666666666,
             2013.91666666666, 2014.41666666666, 2014.91666666666,
             2015.41666666666, 2015.91666666666, 2016.41666666666,
             2016.91666666666, 2017.41666666666])
btc_price = np.array([0.23, 9.57, 3.06, 5.27, 12.56, 129.3, 946.92, 629.02,
	              378.64, 223.31, 362.73, 536.42, 753.25, 2452.18])

M = year[:, np.newaxis]**[0, 1]
model, _, _, _ = lstsq(M,btc_price)
print "Intercept =", model[0]
print "year coefficient =", model[1]


"""

Common Algorithms

@author: Naimish Agarwal
"""

import math
import fundamental_data_structures as fds
from numba import jit

@jit
def gcd(p, q):

    """
    Find Greatest Common Factor of integers p, q

    """
    if q == 0:
        return p
    r = p % q
    return gcd(q, r)

@jit
def is_prime(n):
    """

    Returns whether n is a prime number or not

    """

    if n < 2:
        return False

    for i in xrange(2, int(math.sqrt(n))):
        if n % i == 0:
            return False

    return True

def evaluate_arithmetic_expression(expression):
    """

    Evaluate an arithmetic expression using Dijkstra's two stack algorithm.
    The function assumes that each subexpression is enclosed in parenthesis.

    e.g. ( 1 + ( 2 * ( 3 / 7 ) ) )

    It is also assumed that at least one operator and two operands are present.

    e.g. 1 + 2 (valid)

    e.g. 2 (invalid)

    Parameters:
    -----------

    expression: arithmetic expression which can comprise the following:
        (, ), +, -, *, /, numbers

    all separated by whitespace.

    Returns:
    --------

    Value of expression after evaluation

    """

    expression = "( " + expression + " )"
    tokens = expression.split()
    operator_stack = fds.LinkedListStack()
    operand_stack = fds.LinkedListStack()

    for token in tokens:
        if token == "(":
            pass
        elif token in ["+", "-", "*", "/"]:
            operator = token
            operator_stack.push(operator)
        elif token == ")":
            operator = operator_stack.pop()
            num2 = operand_stack.pop()
            num1 = operand_stack.pop()

            print num1, operator, num2
            if operator == "+":
                operand = num1 + num2
            elif operator == "-":
                operand = num1 - num2
            elif operator == "*":
                operand = num1 * num2
            elif operator == "/":
                operand = num1 / num2
            operand_stack.push(operand)
        else:
            operand = float(token)
            operand_stack.push(operand)

    return operand_stack.pop()


# This program was originally authored by Delaney Granizo and Andrei Kirilenko
as a part of the Master of Finance curriculum at MIT Sloan.

"""
In this notebook, we mainly explores the statistical method of computing maximum likelyhood function
for common distributions. A subsequent financial application will be examined through fitting the normal
distributions to asset returns using MLE.
"""

# Basic Imports
import math
import matplotlib.pyplot as plt
import numpy as np
import scipy
import scipy.stats

"""
== Normal Distribution ==

We first explores some basic sampling from a normal distribution. Given the data available, a
function can be defined so that it will compute the MLE for the \mu and \sigma parameters of the
normal distribution.

"""

True_Mean = 80
True_Std = 10
X = np.random.normal(True_Mean, True_Std, 1000)

def normal_mu_MLE(X):
  """Return the MLE given mu in a normal distribution.
  """
  n_obs = len(X)
  sum_obs = sum(x)
  return 1.0/n_obs*sum_obs

def normal_sigma_MLE(X):
  """Return
  """
  n_obs = len(X)
  mu = normal_mu_MLE(X)
  sum_sqd = sum(np.power((X - mu), 2)) # sum up the squared differences.
  sigma_sq = 1.0/ n_obs * sum_sqd
  return math.sqrt(sigma_sq)

pdf = scipy.stats.norm.pdf
x = np.linespace(0, 80, 80)
plt.hist(x, bins=x, normed='true')
plt.plot(pdf(x, loc=mu, scale=std))
plt.xlabel('Value')
plt.ylabel('Observed Frequency')
plt.legend(['Fitted Distribution PDF', 'Observed Data', ])

# Exponential Distribution
TRUE_LAMDA = 5
X = np.random.exponetial(TRUE_LAMDA, 1000)

def exp_lamda_MLE(X):
  T = len(X)
  s = sum(X)
  return s/T

pdf = scipy.stats.exon.pdf
x = range(0, 80)
plt.hist(X, bins=x, normed='true')
plt.plot(pdf(x, scale=1))
plt.xlabel('Value')
plt.ylabel('Observed Frequency')
plt.legend(['Fitted Distribution PDF', 'Observed Data', ])

prices = get_pricing('AAPL', fields='price', start_date='2018-01-01', end_date='2019-12-31')
absolute_returns = np.diff(prices)
returns = absolute_returns / prices[:-1]

# Using the scipy's fit function to get the mu and sigma
mu, std = scipy.stats.norm.fit(returns)
pdf = scipy.stats.norm.pdf
x = np.linspace(-1, 1, num= 100)
h = plt.hist(returns, bins=x, normed='true')
l = plt.plot(x, pdf(X, loc=mu, scale=std))

# Note that fitting this sample to normal would not work if they do not follow a normal in the
# first place. Such, Jarque-Bera normality test can be carried out. The hypothesis of normal
# distribution will be rejected if the p-value is under a threshold c.


import numpy as np
import sys

sys.path.append("..")
from fd_partial_derivative import fd_partial_derivative

# Choose grid size
gs = np.array([19,15])
h = 1.0/(gs-1)

# Build a grid
x, y = np.meshgrid(np.linspace(0,1,gs[0]),np.linspace(0,1,gs[1]))
V = np.concatenate((np.reshape(x,(-1, 1)),np.reshape(y,(-1, 1))),axis=1)

# Build staggered grid in x direction
x, y = np.meshgrid(np.linspace(0,1,gs[0]-1),np.linspace(0,1,gs[1]))
Vx = np.concatenate((np.reshape(x,(-1, 1)),np.reshape(y,(-1, 1))),axis=1)
# Build staggered grid in y direction
x, y = np.meshgrid(np.linspace(0,1,gs[0]),np.linspace(0,1,gs[1]-1))
Vy = np.concatenate((np.reshape(x,(-1, 1)),np.reshape(y,(-1, 1))),axis=1)

# Build partial derivative matrices
Dx = fd_partial_derivative(gs=gs,h=h,direction=0)
Dy = fd_partial_derivative(gs=gs,h=h,direction=1)

# all rows must sum up to zero (i.e. a constant function has zero derivative)
assert((np.isclose(Dx.sum(axis=1),np.zeros((Dx.shape[0],1)))).all())
assert((np.isclose(Dy.sum(axis=1),np.zeros((Dy.shape[0],1)))).all())

# Build linear function
f = 2*V[:,0] + 5*V[:,1]
computed_derivative_x = Dx*f
computed_derivative_y = Dy*f
# Derivatives must be 2.0 and 5.0, respectively
assert((np.isclose(computed_derivative_x,2.0*np.ones((computed_derivative_x.shape[0])))).all())
assert((np.isclose(computed_derivative_y,5.0*np.ones((computed_derivative_y.shape[0])))).all())

# Convergence test
linf_norm_x = 100.0
linf_norm_y = 100.0
print("This experiment should print a set of decreasing values, converging")
print("towards zero and decreasing roughly by half in each iteration")
for power in range(3,13,1):
    gs = np.array([2**power,2**power - 2])
    h = 1.0/(gs-1)
    # Build a grid
    x, y = np.meshgrid(np.linspace(0,1,gs[0]),np.linspace(0,1,gs[1]))
    V = np.concatenate((np.reshape(x,(-1, 1)),np.reshape(y,(-1, 1))),axis=1)

    # Build staggered grid in x direction
    x, y = np.meshgrid(np.linspace(0,1,gs[0]-1),np.linspace(0,1,gs[1]))
    Vx = np.concatenate((np.reshape(x,(-1, 1)),np.reshape(y,(-1, 1))),axis=1)
    # Build staggered grid in y direction
    x, y = np.meshgrid(np.linspace(0,1,gs[0]),np.linspace(0,1,gs[1]-1))
    Vy = np.concatenate((np.reshape(x,(-1, 1)),np.reshape(y,(-1, 1))),axis=1)

    # Build partial derivative matrices
    Dx = fd_partial_derivative(gs=gs,h=h,direction=0)
    Dy = fd_partial_derivative(gs=gs,h=h,direction=1)
    # Build non-linear function
    f = np.cos(V[:,0]) + np.sin(V[:,1])
    # Derivatives on staggered grids
    fx = -np.sin(Vx[:,0])
    fy =  np.cos(Vy[:,1])
    # Computed derivatives using our matrices
    computed_derivative_x = Dx*f
    computed_derivative_y = Dy*f
    # Print L infinity norm of difference
    # Make sure norm is decreasing
    assert(linf_norm_x>np.max(np.abs(computed_derivative_x - fx)))
    assert(linf_norm_y>np.max(np.abs(computed_derivative_y - fy)))
    linf_norm_x = np.max(np.abs(computed_derivative_x - fx))
    linf_norm_y = np.max(np.abs(computed_derivative_y - fy))
    # Print L infinity norm of difference
    print(np.array([np.max(np.abs(computed_derivative_x - fx)),np.max(np.abs(computed_derivative_y - fy))]))

print("Unit test passed, all asserts passed")


import numpy as np
import matplotlib.pyplot as plt

#coiefficient calculation
def regress(x, x_s, t_s, M, N,lamda = 0):
    order_list =  np.arange((M + 1))
    order_list =  order_list[ :, np.newaxis]
    exponent = np.tile(order_list,[1,N])
    h = np.power(x_s,exponent)
    a =  np.matmul(h, np.transpose(h)) + lamda*np.eye(M+1)
    b =  np.matmul(h, t_s)
    w = np.linalg.solve(a, b) #calculate the coefficent

    exponent2 = np.tile(order_list,[1,200])
    h2 = np.power(x,exponent2)
    p = np.matmul(w, h2)
    return p

##task 1
x =  np.linspace(0, 1, 200)
t =  np.sin(np.pi*x*2)

N = 10
sigma = 0.2;
x_10 = np.linspace(0, 1, N)
t_10 = np.sin(np.pi*x_10*2) + np.random.normal(0,sigma,N)

plt.figure(1)
plt.plot(x, t, 'g',  x_10, t_10, 'bo',linewidth = 2)
plt.ylabel('t',rotation='horizontal')
plt.xlabel('x')
plt.savefig('1.png', dpi=300)

##task 2
M = 3
p3_10 =  regress(x, x_10, t_10, M, N)

M = 9
p9_10 =  regress(x, x_10, t_10, M, N)

plt.figure(2)
plt.plot(x, t, 'g',  x_10, t_10, 'bo', x, p3_10, 'r', linewidth = 2)
plt.ylabel('t',rotation='horizontal')
plt.xlabel('x')
plt.text(0.7, 0.7,'M=3', fontsize=16)
plt.savefig('2.png', dpi=300)

plt.figure(3)
plt.plot(x, t, 'g',  x_10, t_10, 'bo', x, p9_10, 'r', linewidth = 2)
plt.ylabel('t',rotation='horizontal')
plt.xlabel('x')
plt.text(0.7, 0.7,'M=9', fontsize=16)
plt.savefig('3.png', dpi=300)

##task3
N = 15
x_15 =  np.linspace(0, 1,  N)
t_15 = np.sin(np.pi*x_15*2) + np.random.normal(0,sigma, N)

M = 9
p9_15 =  regress(x, x_15, t_15, M, N)

N = 100
x_100 =  np.linspace(0, 1,  N)
t_100 = np.sin(np.pi*x_100*2) + np.random.normal(0,sigma, N)

M = 9
p9_100 =  regress(x, x_100, t_100, M, N)

plt.figure(4)
plt.plot(x, t, 'g',  x_15, t_15, 'bo', x, p9_15, 'r', linewidth = 2)
plt.ylabel('t',rotation='horizontal')
plt.xlabel('x')
plt.text(0.7, 0.7,'N=15', fontsize=16)
plt.savefig('4.png', dpi=300)

plt.figure(5)
plt.plot(x, t, 'g',  x_100, t_100, 'bo', x, p9_100, 'r', linewidth = 2)
plt.ylabel('t',rotation='horizontal')
plt.xlabel('x')
plt.text(0.7, 0.7,'N=100', fontsize=16)
plt.savefig('5.png', dpi=300)

##task4
N = 10
M = 9
p9_10 =  regress(x, x_10, t_10, M, N, np.exp(-18))

plt.figure(6)
plt.plot(x, t, 'g',  x_10, t_10, 'bo', x, p9_10, 'r', linewidth = 2)
plt.ylabel('t',rotation='horizontal')
plt.xlabel('x')
plt.text(0.7, 0.7,'ln$\lambda$ = -18', fontsize=16)
plt.savefig('6.png', dpi=300)
plt.show()


import math
import numpy as np

### Note ###

# z is weighted input

### Functions ###

def linear(z,m):
	return m*z

def elu(z,alpha):
	return z if z >= 0 else alpha*(e^z -1)

def leakyrelu(z, alpha):
	return max(alpha * z, z)

def relu(z):
  return max(0, z)

def sigmoid(z):
  return 1.0 / (1 + np.exp(-z))

def tanh(z):
	return (np.exp(z) - np.exp(-z)) / (np.exp(z) + np.exp(-z))

### Derivatives ###

def linear_prime(z,m):
	return m

def elu_prime(z,alpha):
	return 1 if z > 0 else alpha*np.exp(z)

def leakyrelu_prime(z, alpha):
	return 1 if z > 0 else alpha

def sigmoid_prime(z):
  return sigmoid(z) * (1-sigmoid(z))

def relu_prime(z):
  return 1 if z > 0 else 0

def tanh_prime(z):
	return 1 - np.power(tanh(z), 2)


import numpy as np
from matplotlib import pyplot as plt
import time

def bezier_curve_3(xpoints, ypoints, nTimes=1000):
    ret_x = []
    ret_y = []
    for i in np.arange(0, nTimes + 1):
        t = i / nTimes
        x = (1 - t) ** 3 * xpoints[0] + 3.0 * t * (1 - t) ** 2 * xpoints[1] + 3.0 * (t) ** 2 * (1 - t) * xpoints[
            2] + t ** 3 * xpoints[3]
        ret_x.append(x)
        y = (1 - t) ** 3 * ypoints[0] + 3.0 * t * (1 - t) ** 2 * ypoints[1] + 3.0 * (t) ** 2 * (1 - t) * ypoints[
            2] + t ** 3 * ypoints[3]
        ret_y.append(y)
    return ret_x, ret_y

def temp_curve_3(x, y, nTimes=1000):
    dx_0 = (x[1] - x[0]) * 2 / nTimes
    dy_0 = (y[1] - y[0]) * 2 / nTimes

    tdx = (x[2] - 2 * x[1] + x[0]) * 2 / nTimes ** 2
    tdy = (y[2] - 2 * y[1] + y[0]) * 2 / nTimes ** 2

    tddx = (x[3] - 3 * (x[2] - x[1]) - x[0]) * 2 / nTimes ** 3
    tddy = (y[3] - 3 * (y[2] - y[1]) - y[0]) * 2 / nTimes ** 3

    M_0 = x[0]
    N_0 = y[0]

    TM = (x[1] - x[0]) / nTimes  # TMS
    TN = (y[1] - y[0]) / nTimes  # TNS

    ret_x = []
    ret_y = []
    ret_x.append(M_0)
    ret_y.append(N_0)
    C = nTimes

    atdx = tdx / 2
    atdy = tdy / 2

    atddx = tddx / 2
    atddy = tddy / 2
    for i in np.arange(1.0, C+1):
        M_0 += dx_0 + atdx
        N_0 += dy_0 + atdy

        atdx += tdx
        atdy += tdy

        TM += tdx + atddx
        TN += tdy + atddy

        atddx += tddx
        atddy += tddy

        M = M_0 + (i * (TM))
        N = N_0 + (i * (TN))

        ret_x.append(M)
        ret_y.append(N)
    return ret_x, ret_y

xpoints = [-6.0000000001, -4.00000000011, -0.000000000111, -0.0000000001111]
ypoints = [-5.0000000001, 3.00000000011, 0.000000000111, -4.0000000001111]

plt.rcParams["figure.figsize"] = (10, 10)
plt.axis([-8, 2, 2, -8])

plt.plot(xpoints, ypoints, "b")

loop_times = 1000
proc_times = 1000

start = time.time()
for i in np.arange(loop_times):
    X, Y = bezier_curve_3(xpoints, ypoints, nTimes=proc_times)

end = time.time()
plt.plot(X, Y, "k")
print("Bezier_3 time : ", end - start, "(start : ", start, " end : ", end)

start = time.time()
for i in np.arange(loop_times):
    X, Y = temp_curve_3(xpoints, ypoints, nTimes=proc_times)

end = time.time()
plt.plot(X, Y, "m")
print("temp_3 time : ", end - start, "(start : ", start, " end : ", end)

plt.show()


import numpy as np
from sklearn import preprocessing

data = np.array([[3, -1.5,  2, -5.4], [0,  4,  -0.3, 2.1], [1,  3.3, -1.9, -4.3]])

print(data)

#Mean removal
print("Mean: ",data.mean(axis=0))
print("Standard Deviation: ",data.std(axis=0))

data_standardized = preprocessing.scale(data)

print("Mean standardized data: ",data_standardized.mean(axis=0))
print("Standard Deviation standardized data: ",data_standardized.std(axis=0))

#Scaling
data_scaler = preprocessing.MinMaxScaler(feature_range=(0, 1))
data_scaled = data_scaler.fit_transform(data)

print("Min: ",data.min(axis=0))
print("Max: ",data.max(axis=0))

print("Min: ",data_scaled.min(axis=0))
print("Max: ",data_scaled.max(axis=0))

print(data_scaled)

#Normalization
data_normalized = preprocessing.normalize(data, norm='l1',axis=0)

print(data_normalized)

data_norm_abs = np.abs(data_normalized)

print(data_norm_abs.sum(axis=0))

#Binarization
data_binarized = preprocessing.Binarizer(threshold=1.4).transform(data)

print(data_binarized)

#One Hot Encoding
data = np.array([[1, 1, 2], [0, 2, 3], [1, 0, 1], [0, 1, 0]])
print(data)

encoder = preprocessing.OneHotEncoder()
encoder.fit(data)
encoded_vector = encoder.transform([[1, 2, 3]]).toarray()

print(encoded_vector)


"""
A perfect number is a number for which the sum of its proper divisors is exactly equal to the number.
For example, the sum of the proper divisors of 28 would be 1 + 2 + 4 + 7 + 14 = 28,
which means that 28 is a perfect number.

A number n is called deficient if the sum of its proper divisors is less than n and it is called abundant
if this sum exceeds n.

As 12 is the smallest abundant number, 1 + 2 + 3 + 4 + 6 = 16, the smallest number that can be written as the sum
of two abundant numbers is 24.
By mathematical analysis, it can be shown that all integers greater than 28123 can be written as the sum of two
abundant numbers. However, this upper limit cannot be reduced any further by analysis even though it is known that
the greatest number that cannot be expressed as the sum of two abundant numbers is less than this limit.

Find the sum of all the positive integers which cannot be written as the sum of two abundant numbers.
"""

import numpy as np

# all numbers over this can be written as sums of abundant numbers
limit = 28123

def d(n):
    divisors = set()
    divisors.add(1)

    less_half = np.math.floor(n / 2)

    for i in range(2, less_half + 1):

        if n % i == 0:
            divisors.add(i)

    return sum(divisors)

def all_abundant():
    abundant = list()

    for i in range(12, limit + 1):
        if d(i) > i:
            abundant.append(i)

    return abundant

abundant_numbers = all_abundant()

# Sieve like algorithm to zero out all numbers that can be written as the sum of abundant numbers
# then take the sum of the remaining
rang = [x for x in range(0, limit)]

for i in abundant_numbers:
    for j in abundant_numbers:
        if (i + j) < limit:
            # print(i+j)
            rang[i + j] = 0
        else:
            break

print(f'The sum of positive integers that cannot be written as the sum of two abundant numbers is: {sum(rang)}')

Solutions/Problem 23 - Non-abundant sums.py


import numpy
import scipy.io
'''
This code was originally in Matlab
function generatePoissonTrains
%% author: Paula Kuokkanen
%% 5. Mai 2011

traslated to Python: Nikolay Chenkov
June 2013

homogeneous Poisson process

A homogeneous Poisson process generates events (spikes) with a constant
probability or rate r(t) = r. Spikes are independent. Thus, at each
time point t, the probability P of observing a spike within
a sufficiently small temporal window from t to t + delta_t is
P= r*delta_t.

There are two simple ways of implementing a homogeneous Poisson process:
1.    Progress through time in small steps of size delta_t,
      draw a random number x (from a uniform distribution
      between 0 and 1, Matlab function 'rand'
      at each time step. If x < P, generate a spike.

2.    Generate interspike intervals from an exponential probability
      density. When x is uniformly distributed between 0 and 1,
      its negative logarithm is exponentially distributed. Thus, we
      can generate spike times t(i) iteratively from the
      formula t(i+1) = t(i) - log(x)/r.

Using implementation #2, generate 1000 spikes by a homogeneous
Poisson process with a rate of r=100 (Hz)
'''

n = 1000   # desired number of spikes
rate = 100  # rate in spikes per second
randNumbers = numpy.random.rand(n)  # n random numbers, uniform distribution
ISIs = -numpy.log(randNumbers)/rate   # ISIs in seconds
ISIs = ISIs*1000   # ISIs in milliseconds

# compute spike times from ISIs
SpikeTimes = numpy.cumsum(ISIs)

'''
# inhomogeneous Poisson process

 An inhomogeneous Poisson process is characterized by a time-dependent
 rate r(t). There are several ways of generating spikes.
 Two basic ways are:
 1.    Progress through time in small intervals of width delta_t.
       For sufficiently small delta_t, the probability P(i) of
       observing a spike in interval i from t(i) = i*delta_t to
       t_(i) + delta_t is then given by
           P(i) = integral_t(i)^[t_(i)+delta_t)] r(tau) d tau .
       Then, draw a random number x(i) from a uniform distribution
       for each time interval. If x(i) < P(i)$, generate a spike.

 2.    `Thinning' the spike train of a homogeneous Poisson process:
       First, a spike sequence is generated with a homogeneous
       Poisson process at rate r_max = max[r(t)]. The spike
       sequence is then thinned by generating a random number
       x(i) for each spike and removing the spike at time
       t(i) from the train if r(t(i))/r_max < x(i).

 We use now method 2, thinning. Our probability distribution r is
     r(t) = A* sin(2*pi*f*t) + r_0
 with A = 50 Hz, r_0 = 100 Hz and f = 10 Hz. We generate 1000 spikes.

 First, start with the homogeneous, but with more spikes so that you can
 thin out later on
'''

n = 10000   # desired number of spikes
maxRate = 150  # rate in spikes per second
randNumbers2 = numpy.random.rand(n)  # n random numbers, uniform distribution
ISIs2 = -numpy.log(randNumbers2)/maxRate   # ISIs in seconds
ISIs2 = ISIs2*1000   # ISIs in milliseconds

# compute spike times from ISIs
SpikeTimes2 = numpy.cumsum(ISIs2)

# thin the spike train

randNumbers = numpy.random.rand(n)
rates = 50.*numpy.sin(SpikeTimes2*2*numpy.pi/100) + 100
Ps = rates/maxRate
SpikeTimes_inh = SpikeTimes2[randNumbers>Ps]
n = 1000
SpikeTimes_inh = SpikeTimes_inh[0:n]     # take just 1000 spikes

'''
Poisson process with absolute refractory period

Real neurons are in a refractory state immediately after a spike.
Thus, during a certain 'refractory period' after a spike, it is
impossible or much harder to evoke the next spike.
When the effect of refractoriness is taken into account, spikes
can no longer considered to be independent because the probability
of observing the next spike then also depends on the time that has
passed since the last spike.

In point process models, an approximation of refractoriness is to
set the probability of generating a spike to 0 for a certain absolute
refractory period tr after a spike, and then to r afterwards.

We simulate a homogeneous Poisson process and add an absolute refractory
period of t_r = 5 ms. We use different 'driving' rates
r = 10, 50, 100, 200, 500, and 1000Hz.
We simulate 1000 spikes for each rate r.

Adding an absolute refractory period corresponds to simply adding a
fixed value to each interspike interval of a 'normal'
homogeneous Poisson process.
'''

rates_ref = [10, 50, 100, 200, 500, 1000]  # "driving" rates in spikes per second
n = 1000   # desired number of spikes
ISIs_ref = numpy.zeros((n,len(rates_ref)))
randNumbers3 = numpy.random.rand(n)  #n random numbers, uniform distribution
t_r = 5    # absolute refractory period in ms

for i,r in enumerate(rates_ref):
    ISIs_ref[:,i] = -numpy.log(randNumbers3)/r*1000 + t_r  #ISIs in milliseconds

# compute spike times from ISIs
SpikeTimes_ref = numpy.zeros((n,len(rates_ref)))
for i,r in enumerate(rates_ref):
    SpikeTimes_ref[:,i] = numpy.cumsum(ISIs_ref[:,i])

scipy.io.savemat('PoissonSpikeTrains.mat',
                    {'SpikeTimes_hom': SpikeTimes,
                    'SpikeTimes_inh': SpikeTimes_inh,
                    'SpikeTimes_ref': SpikeTimes_ref,
                    'rates_ref': rates_ref})


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
from math import sqrt, fabs

import dcstats.statistics_EJ as s
from dcstats.basic_stats import mean, sd, sdm, ttestPDF

def isclose(a, b, rel_tol=1e-09, abs_tol=0.0):
    return abs(a-b) <= max(rel_tol * max(abs(a), abs(b)), abs_tol)

def test_beta_incomplete():

    bi = s.incompleteBeta(0.5, 1, 2)
    assert isclose(0.75, bi, rel_tol=0.0001)

    try:
        from scipy.special import betainc
        spbi = betainc(1, 2, 0.5)
        assert isclose(spbi, bi, rel_tol=0.0001)
    except:
        print('scipy not available')

def test_mean():
    X = [1, 2, 3]
    assert mean(X) == 2

def test_sd():
    X = [1, 2, 3]
    assert isclose(sd(X), 1.0, rel_tol=0.0001)

def test_sdm():
    X = [1, 2, 3]
    print(sdm(X))
    assert isclose(sdm(X), 0.57735026918963, rel_tol=0.000001)

def test_ttest_P_paired():
    D = [1, 2, 3]
    df = len(D) - 1
    dbar, dsdm = mean(D), sdm(D)
    print(dbar, dsdm)
    tval = dbar / dsdm
    print(tval)
    assert isclose(tval, 3.464101615137754, rel_tol=0.0000001)

    P = ttestPDF(tval, df)
    print(P)
    assert isclose(P, 0.07417990022744862, rel_tol=0.0000001)

def test_ttest_P_unpaired():
    X = [1, 2, 3]
    Y = [1, 4, 6]
    df = len(X) + len(Y) - 2

    xbar, sdx = mean(X), sd(X)
    ybar, sdy = mean(Y), sd(Y)
    s = (sdx * sdx * (len(X)-1) + sdy * sdy * (len(Y)-1)) / df
    sdiff = sqrt(s * (1.0 / len(X) + 1.0 / len(Y)))
    adiff = fabs(xbar - ybar)

    print(sdiff, adiff)
    tval = sdiff / adiff
    print(tval)
    assert isclose(tval, 0.9380831519646861, rel_tol=0.0000001)

    P = ttestPDF(tval, df)
    print(P)
    assert isclose(P, 0.40131270580971734, rel_tol=0.0000001)


import numpy as np
import linalg_toolbox as linalg_tools

def get_projection_matrix(data, classes, ndim, normalize=False):
    """
    Performs a LDA over the data and return the projection matrix.
    data: the data to be projected. Datapoints are given by rows while features
          are given by columns.
    ndim: the dimension of the lower dimentional subspace the data should be
          projected on.
    normalize: set to True if the data should be centered and whitened before
               computing the projection matrix
    """
    n_datapoints, n_features = data.shape
    data = data.T # to retrieve the usual notation: datapoints are arranged by columns.
    if normalize:
        data = linalg_tools.normalize_data(data) # center and whiten the data

    assert ndim <= n_features, 'We can\'t project onto a higher dimentional space.'
    assert len(classes) == n_datapoints, 'Some points have no associated class, or ' \
                                         'too many classes.'

    # Computing the mean of all classes. That is to say, the mean image of
    # every class.
    mu_c = compute_mu_classes(data, classes)

    # Computing the covariance matrix 'between classes': how much are centers
    # of different classes apart from each other
    Sb = compute_Sb(data, classes, mu_c)

    # Computing the covariance matrix 'within classes': how much points within
    # clusters are separated from each other
    Sw = compute_Sw(data, classes, mu_c)

    # This is the matrix we need to compute the eigenvectors from
    fisher_matrix = np.linalg.inv(Sw) * Sb

    # Getting the eigen vectors (they correspond to the projection matrix we are
    # looking for).
    projection_matrix = linalg_tools.get_largest_eigen_vectors(fisher_matrix, ndim)
    return projection_matrix.T

def project_using_projection_matrix(data_to_transform, projection_matrix):
    """
    Projects given data into lower dimentional subspace using the provided
    projection_matrix.
    WARNING: the data is supposed to be in standard data science notation:
    rows: datapoints/individuals
    columns: variables/features
    """
    projected_data = projection_matrix * data_to_transform.T;
    return projected_data.T # the .T is used to get back to standard data-science
                            # layout: datapoints are arranged by rows

def project(data, classes, ndim, normalize):
    """
    Projects an array of data onto a lower dimentional subspace.
    The procedure here is the 'naive' one (as seen in the lectures).
    data: the data to be projected. Datapoints are given by rows while features
          are given by columns.
    ndim: the dimension of the lower dimentional subspace the data should be
          projected on.
    normalize: set to True if the data should be centered and whitened before
               computing the projection matrix
    """
    projection_matrix = get_projection_matrix(data, classes, ndim, normalize)
    return project_using_projection_matrix(data, projection_matrix)

def get_subdata(data, classes, class_to_select):
    """
    Given a dataset and the associate class for each point, returns
    the points whose class is exactly class_to_select).
    """
    subdata = data[:,classes == class_to_select]
    return subdata

def compute_mu_classes(data, classes):
    """
    Given a dataset and the associate class for each point, returns
    the mean of each of the classes as matrix mu_c where:
    each column is associated to a class
    each row is associated to a coordinate
    """
    n_features,_ = data.shape
    unique_classes = np.unique(classes)
    mu_c = np.matrix(np.zeros([n_features,len(unique_classes)]))
    for class_num,class_name in enumerate(unique_classes):
        subdata = get_subdata(data, classes, class_name)
        mu_subdata = subdata.mean(axis=1)
        mu_c[:,class_num] = mu_subdata

    return mu_c

def compute_Sb(data, classes, mu_c):
    """
    Computes the variance 'between' classes. That is to say, how well are
    each class separated from each other.
    data: the dataset
    mu_c: the mean value of each cluster. One can use compute_mu_classes to get it.
    """
    mu = data.mean(axis=1) # mean of the whole dataset

    n_features,_ = data.shape
    unique_classes = np.unique(classes)
    Sb = np.matrix(np.zeros([n_features, n_features]))
    for class_num,class_name in enumerate(unique_classes):
        subdata = get_subdata(data, classes, class_name)
        _,size_subdata = subdata.shape
        mu_subdata = mu_c[:,class_num]
        diff_mu = mu_subdata - mu
        Sb += size_subdata * (diff_mu * diff_mu.T)

    return Sb

def compute_Sw(data, classes, mu_c):
    """
    Computes the variance 'within' classes. That is to say, how spread are each
    class by themselves (how points are separated from each other within their own class).
    data: the dataset
    mu_c: the mean value of each cluster. One can use compute_mu_classes to get it.
    """
    unique_classes = np.unique(classes)
    n_features,_ = data.shape
    Sw = np.matrix(np.zeros([n_features, n_features]))
    for class_num,class_name in enumerate(unique_classes):
        subdata = get_subdata(data, classes, class_name)
        mu_subdata = mu_c[:,class_num]
        _,size_subdata = subdata.shape
        Sw += np.cov(subdata)

    return Sw


#!/usr/bin/env python3

import tensorflow as tf
import numpy as np
import matplotlib.pyplot as plt
import sys
from mpl_toolkits.mplot3d import Axes3D

# Define hyperparameters
TRAINING_EPOCH = 400
LEARNING_RATE = 1
REAL_PARAMS = [1.2, 2.5]
INIT_PARAMS = [[5, 4], [5, 1], [2, 4.5]][2]

''' Select different target function '''
def targetFunction(option, x):
	if option == '1':
		# Use a simple linear function with two parameters
		return lambda w, b: w * x + b
	elif option == '2':
		# Use Tensorflow as a calibrating tool for empirical formula like following
		return lambda w, b: w * x**3 + b * x**2
	elif option == '3':
		# Use the most simplest two parameters and two layers Neural Net, and their local and global minimum
		return lambda w, b: np.sin(b * np.cos(w * x))
	return None

''' Select different training function '''
def trainFunction(option, x):
	if option == '1':
		# Use a simple linear function with two parameters
		return lambda w, b: w * x + b
	elif option == '2':
		# Use Tensorflow as a calibrating tool for empirical formula like following
		return lambda w, b: w * x**3 + b * x**2
	elif option == '3':
		# Use the most simplest two parameters and two layers Neural Net, and their local and global minimum
		return lambda w, b: tf.sin(b * tf.cos(w * x))
	return None

def main(args1, args2):
	''' Create data '''
	# Training data
	x = np.linspace(-1, 1, 200, dtype=np.float32)
	noise = np.random.randn(200) / 10
	y_fun = targetFunction(args1, x)
	y = y_fun(*REAL_PARAMS) + noise

	''' Create TensorFlow model '''
	# Set the learning rate
	global LEARNING_RATE
	LEARNING_RATE = float(args2)

	# Define Weights and biases
	Weights, biases = [tf.Variable(initial_value=val,dtype=tf.float32) for val in INIT_PARAMS]

	# Prediction
	y_pred = trainFunction(args1, x)
	pred = y_pred(Weights, biases)

	# Define the loss function and the optimizer
	loss = tf.reduce_mean(tf.square(y - pred))
	train = tf.train.GradientDescentOptimizer(LEARNING_RATE).minimize(loss)

	''' Start training '''
	Weights_list, biases_list, loss_list = [], [], []
	with tf.Session() as sess:
		# Initialize all variables in TensorFlow
		sess.run(tf.global_variables_initializer())

		# Train 400 times
		for epoch in range(TRAINING_EPOCH):
			w, b, l = sess.run([Weights, biases, loss])

			# Record the changes of parameters
			Weights_list.append(w)
			biases_list.append(b)
			loss_list.append(l)

			# Training
			result, _ = sess.run([pred, train])

	''' Visualization '''
	print('Weight = ', w, 'bias = ', b)

	# Plot the input data and the training result
	plt.figure(1)
	plt.scatter(x, y, c='#74BCFF', s=50, alpha=0.5, label='Train')
	plt.legend(loc='upper left')
	plt.plot(x, result, 'r-', lw=2)
	plt.savefig('input_%s_%s.png' % (args1, args2))

	# Plot loss rate in 3D figure
	fig = plt.figure(2)
	ax = Axes3D(fig)
	weight3D, bias3D = np.meshgrid(np.linspace(-2, 7, 30), np.linspace(-2, 7, 30))      # Parameter space
	loss3D = np.array([np.mean(np.square(y_fun(w, b) - y)) for w, b in zip(weight3D.flatten(), bias3D.flatten())]).reshape(weight3D.shape)
	ax.plot_surface(weight3D, bias3D, loss3D, rstride=1, cstride=1, cmap=plt.get_cmap('rainbow'), alpha=0.5)
	ax.scatter(Weights_list[0], biases_list[0], zs=loss_list[0], s=300, c='r')          # Initial parameter
	ax.set_xlabel('Weight')
	ax.set_ylabel('bias')
	ax.plot(Weights_list, biases_list, zs=loss_list, zdir='z', c='r', lw=3)             # Plot 3D gradient descent
	plt.savefig('%s_%s.png' % (args1, args2))
	plt.show()

''' ENTRY POINT '''
if __name__ == "__main__":
	if len(sys.argv) != 3:
		print('[ERROR] No argument')
		print('[INFO] FORMAT: "python3 main.py 1 [FLOAT]" or "python3 main.py 2 [FLOAT]" or "python3 main.py 3 [FLOAT]"')
		sys.exit()
	else:
		# For option
		if sys.argv[1] == '1':
			print('[INFO] Using linear function as target and training function')
		elif sys.argv[1] == '2':
			print('[INFO] Using non-linear function as target and training function')
		elif sys.argv[1] == '3':
			print('[INFO] Using sin/cos function as target and training function')
		else:
			print('[ERROR] Invalid argument')
			print('[INFO] FORMAT: "python3 main.py 1" or "python3 main.py 2" or "python3 main.py 3"')
			sys.exit()

		# For learning rate
		print('[INFO] Using %s learning rate' % sys.argv[2])

		main(sys.argv[1], sys.argv[2])


import sympy

x = sympy.Symbol('x')
y = sympy.Symbol('y')

print(type(x))
# <class 'sympy.core.symbol.Symbol'>

expr = x**2 + y + 1

print(expr)
# x**2 + y + 1

z = sympy.Symbol('ZZZZ')

expr_z = z**2 + 3 * z

print(expr_z)
# ZZZZ**2 + 3*ZZZZ

print(expr)
# x**2 + y + 1

print(expr.subs(x, 1))
# y + 2

print(expr.subs(x, y))
# y**2 + y + 1

print(expr.subs([(x, 1), (y, 2)]))
# 4

expr = (x + 1)**2

print(expr)
# (x + 1)**2

expr_ex = sympy.expand(expr)

print(expr_ex)
# x**2 + 2*x + 1

expr_factor = sympy.factor(expr_ex)

print(expr_factor)
# (x + 1)**2

print(sympy.factor(x**3 - x**2 - 3 * x + 3))
# (x - 1)*(x**2 - 3)

print(sympy.factor(x * y + x + y + 1))
# (x + 1)*(y + 1)

print(sympy.solve(x**2 - 3 * x + 2))
# [1, 2]

print(sympy.solve(x**2 + x + 1))
# [-1/2 - sqrt(3)*I/2, -1/2 + sqrt(3)*I/2]

expr = x + y**2 - 4

print(sympy.solve(expr, x))
# [-y**2 + 4]

print(sympy.solve(expr, y))
# [-sqrt(-x + 4), sqrt(-x + 4)]

expr1 = 3 * x + 5 * y - 29
expr2 = x + y - 7

print(sympy.solve([expr1, expr2]))
# {x: 3, y: 4}

print(sympy.diff(x**3 + 2 * x**2 + x))
# 3*x**2 + 4*x + 1

expr = x**3 + y**2 - y

print(sympy.diff(expr, x))
# 3*x**2

print(sympy.diff(expr, y))
# 2*y - 1

print(sympy.integrate(3 * x**2 + 4 * x + 1))
# x**3 + 2*x**2 + x

print(sympy.diff(sympy.cos(x)))
# -sin(x)

print(sympy.diff(sympy.exp(x)))
# exp(x)

print(sympy.diff(sympy.log(x)))
# 1/x

print(sympy.integrate(sympy.cos(x)))
# sin(x)

print(sympy.integrate(sympy.exp(x)))
# exp(x)

print(sympy.integrate(sympy.log(x)))
# x*log(x) - x


from sympy import ImmutableMatrix as Matrix
from sympy.vector import CoordSys3D, matrix_to_vector

import numpy as np
import sympy as sy

m3_1_x = Matrix([-5, 5, -2])
m3_1_y = Matrix([ 2, 2, -5])

C = CoordSys3D('C')
x_3_1 = matrix_to_vector(m3_1_x, C)
y_3_1 = matrix_to_vector(m3_1_y, C)

print("x+y = ", x_3_1 + y_3_1)
print("2x = ", 2*x_3_1)
print("x-y = ", x_3_1 - y_3_1)
print("2x+10y = ", 2*x_3_1 + 10*y_3_1)

x_3_2 = np.array([[-2, 2, 1],
                 [-1, 0, 1],
                 [1, -1, 0]])
y_3_2 = ([18, 9, -7])
print("coefficients of linear combo are: ", np.linalg.solve(x_3_2, y_3_2) )

m_3_2 = sy.Matrix([[-2, 2, 1,18],
                  [-1, 0, 1, 9],
                  [1, -1, 0, -7]])

print(m_3_2.rref(pivots=True))

m_3_3 = sy.Matrix([[-1,  1, -1,  13],
                   [ 1,  0,  1,  -8],
                   [ 1, -1,  2, -16]])

print(m_3_3.rref(pivots=True))

m_3_4 = sy.Matrix([[ 5,  5,  0,  -2],
                   [ 1, -1, -2,   3],
                   [-1, -1,  3,  -1]])

print(m_3_4.rref(pivots=True))

m_3_5 = sy.Matrix([[ 3,  1, -5,   26],
                   [ 2,  4,  1,    9]])

print(m_3_5.rref(pivots=True))

m_3_6 = sy.Matrix([[ 0,   0,  1,   -5], #note each column is a vector
                   [ -1,  2, -3,   12],
                   [ -1,  1, -2,   11]])

print(m_3_6.rref(pivots=True))

m_3_7 = sy.Matrix([[ -5,  5, -10,  -30],
                   [ -4,  2,  -6,  -16],
                   [  1,  0,   2,    3]])

print(m_3_7.rref(pivots=True))

m3_8_u = Matrix([ 6, -2,  9]) # this is one vector as a 1 x 3 matrix
m3_8_v = Matrix([ 6,  3,  1])
m3_8_w = Matrix([ 1, -3,  1])

C = CoordSys3D('C')
x_8_u = matrix_to_vector(m3_8_u, C)
x_8_v = matrix_to_vector(m3_8_v, C)
x_8_w = matrix_to_vector(m3_8_w, C)

print("2u+3v-9w = ", 2*x_8_u + 3*x_8_v - 9*x_8_w)

m_3_9 = sy.Matrix([[  -2,   2, -6,  0],
                    [ -1,  -5,  9,  0]])

print(m_3_9.rref(pivots=True))

m_3_10 = sy.Matrix([[  9,  -6,  0,  0],
                    [  9, -10,  2,  0],
                    [ -6,  -8,  6,  0]])

print(m_3_10.rref(pivots=True))

m_3_13_1 = sy.Matrix([[  4,  -2,  8, -10, -5, 0],
                      [ -9,   5, -9,  -4,  5, 0],
                      [ -5,  -1,  7,  -4,  9, 0],
                      [ 10,   9, -8,   1, -7, 0]])

print(m_3_13_1.rref(pivots=True))

m_3_13_2 = sy.Matrix([[  4,   8, 0],
                      [  4,   8, 0],
                      [  2,   4, 0],
                      [  5,   10, 0]])

print(m_3_13_2.rref(pivots=True))

k = sy.symbols('k')
m_3_14 = sy.Matrix([ [ -1,    1,      3, 0],
                     [ -2,   -2,     -5, 0],
                     [ -5,  -15 + k, -7, 0]])

print(m_3_14.rref(pivots=True))

m_4_2 = sy.Matrix([[-2,  2,  3, -9],
                   [-1,  0,  0, -1],
                   [ 1, -1, -1,  5]])

print(m_4_2.rref(pivots=True)) #coefficients of linear combo are in last column

m_4_3 = sy.Matrix([[-1,  5,  1, -5],
                   [-4, -1, -2, -5],
                   [ 1, -4,  2,  2]])

print(m_4_3.rref(pivots=True)) #coefficients of linear combo are in last column

m_4_4 = sy.Matrix([[ 2,  2,  8],
                   [-1, -2, -9],
                   [ 1,  1,  4]])

print(m_4_4.rref(pivots=True)) #coefficients of linear combo are in last column

m_4_5 = sy.Matrix([[-2, -2,  3, -6],
                   [ 1,  0,  0, -5],
                   [ 1,  1, -1,  2]])

print(m_4_5.rref(pivots=True)) #coefficients of linear combo are in last column

m_4_6 = sy.Matrix([[ 1,  1, -3, 0],
                   [ 2, -1, -3, 3],
                   [-4, -5, 13, 18]])

print(m_4_6.rref(pivots=True)) #coefficients of linear combo are in last column

m4_7_x = Matrix([4, -2, 0])
m4_7_y = Matrix([-5, 1, 5])

C = CoordSys3D('C')
x4_7 = matrix_to_vector(m4_7_x, C)
y4_7 = matrix_to_vector(m4_7_y, C)

print("3x = ", 3*x4_7 )
print("x+y = ", x4_7+y4_7)
print("3x+y = ", 3*x4_7 + y4_7)

m_4_8 = sy.Matrix([[ -1, -1, -2],
                   [ -2,  1, -1],
                   [  3,  5,  8]])

print(m_4_8.rref(pivots=True)) #coefficients of linear combo are in last column

m_4_10 = sy.Matrix([[ -16,  16, -5,  -2],
                    [ -9,  -19, -10, -18],
                    [  4,   12, -16,  12],
                    [-11,   -7, -15,  -4]])

print(m_4_10.rref(pivots=True)) #check pivots

m_4_10_w = sy.Matrix([[ -16,  16, -5, -11],
                      [ -9,  -19, -10, -27],
                      [  4,   12, -16,  36],
                      [-11,   -7, -15, -14]])

print(m_4_10_w.rref(pivots=True)) #check pivots

m_4_11 = sy.Matrix([[ -15,  0, 15, 0], #added 0s to check independence
                    [ -10, -8,  2, 0],
                    [  14, 31, 17, 0]])

print(m_4_11.rref(pivots=True)) #coefficients of linear combo are in last column

m_4_12 = sy.Matrix([[   4, -4,  8, 0], #added 0s to check independence
                    [  12,  3, 15, 0],
                    [   9,  6,  9, 0]])

print(m_4_12.rref(pivots=True)) #coefficients of linear combo are in last column

m_4_13 = sy.Matrix([[  1,    k,  0], #added 0s to check independence
                    [  k, 6*k+7, 0]])

print(m_4_13.rref(pivots=True)) #coefficients of linear combo are in last column

m_4_14 = sy.Matrix([  [  4,  -3,  1, -5, 0],
                      [  5,  -1, -3,  2, 0],
                      [ -3,  -4,  5, -3, 0],
                      [ -5,  -2,  2,  1, 0]])

print(m_4_14.rref(pivots=True)) #check pivots

m_4_15 = sy.Matrix([[  -1,  2,    -1, 0], #added 0s to check independence
                    [  -5, -12+k, -3, 0],
                    [  -3, -2,    -1, 0]])

print(m_4_15.rref(pivots=True)) #coefficients of linear combo are in last column