"""
  Question 2: Rossler attractor
"""
# Library import
import numpy as np # Data structures, vector, matrices, etc
import matplotlib.pyplot as plt # Plots
from mpl_toolkits.mplot3d import Axes3D
import pathlib
#%%
# RHS definition
def F(t, u, **kwargs):
  a, b, c = kwargs['a'], kwargs['b'], kwargs['c']
  return np.array([-u[1] - u[2], u[0] + a * u[1], b + u[2] * (u[0] - c)])

# Fourth-order Runge Kutta method
def RK4(t, u0, **kwargs):
  L = len(t)
  U = np.zeros((L, len(u0)))
  U[0] = u0
  dt = t[1] - t[0]
  for k in range(L-1):
    k1 = F(t[k], U[k], **kwargs)
    k2 = F(t[k] + 0.5 * dt, U[k] + 0.5 * dt * k1, **kwargs)
    k3 = F(t[k] + 0.5 * dt, U[k] + 0.5 * dt * k2, **kwargs)
    k4 = F(t[k] + dt, U[k] + dt * k3, **kwargs)
    U[k + 1] = U[k] + (1/6) * dt * (k1 + 2 * k2 + 2 * k3 + k4)

  return U

# Euler method for testing...
def Euler(t, u0, **kwargs):
  L = len(t)
  U = np.zeros((L, len(u0)))
  U[0] = u0
  dt = t[1] - t[0]
  for k in range(L - 1):
    U[k + 1] = U[k] + dt * F(t[k], U[k], **kwargs)

  return U

# Plot x(t)
def plot(t, x):
  plt.plot(t, x)
  plt.grid(True)
  plt.show()

# Plot solution
def plot3D(x, y, z):
  fig = plt.figure(figsize=(12, 12))
  ax = fig.add_subplot(111, projection='3d')
  ax.plot(x, y, z)
  ax.set_xlabel('x')
  ax.set_ylabel('y')
  ax.set_zlabel('z')
  plt.show()
#%% Reproducing Rossler Paper Experiment
L_r = 5000 # 340000
T_max_r = 339.249 #813.312#339.249
t_r = np.linspace(0, T_max_r, L_r + 1)
a_r, b_r, c_r = 0.2, 0.2, 5.7
u0_r = np.array([0, -6.78, 0.02])
Ur = RK4(t_r, u0_r, a=a_r, b=b_r, c=c_r)

#%% Plot Rossler experiment
plot3D(Ur[:,0], Ur[:,1], Ur[:,2])
print("Rossler experiment - (x, y, z) at t_end: ", Ur[-1])

#%% Question 4
L_1 = 5000
T_max_1 = 300
t_1 = np.linspace(0, T_max_1, L_1 + 1)
u0_1 = np.array([4, 0, 0])

#%% Experiment 1
a_, b_, c_1 = 0.1, 0.1, 4
U1 = RK4(t_1, u0_1, a=a_, b=b_, c=c_1)

#%% Experiment 2
c_2 = 6
U2 = RK4(t_1, u0_1, a=a_, b=b_, c=c_2)

#%% Experiment 3
c_3 = 8.5
U3 = RK4(t_1, u0_1, a=a_, b=b_, c=c_3)

#%% Experiment 4
c_4 = 11
U4 = RK4(t_1, u0_1, a=a_, b=b_, c=c_4)

#%% Experiment 5
c_5 = 12.5
U5 = RK4(t_1, u0_1, a=a_, b=b_, c=c_5)

#%% Experiment 6
c_6 = 14
U6 = RK4(t_1, u0_1, a=a_, b=b_, c=c_6)

#%% Plots question 4
plot3D(U1[:,0], U1[:,1], U1[:,2])
plot3D(U2[:,0], U2[:,1], U2[:,2])
plot3D(U3[:,0], U3[:,1], U3[:,2])
plot3D(U4[:,0], U4[:,1], U4[:,2])
plot3D(U5[:,0], U5[:,1], U5[:,2])
plot3D(U6[:,0], U6[:,1], U6[:,2])

#%% Question 5
L_2 = 5000
T_max_2 = 300
t_2 = np.linspace(0, T_max_1, L_1 + 1)
a_, b_, c_ = 0.2, 0.2, 5.7
u0_2a = np.array([1, 0, 0])
u0_2b = np.array([1.01, 0, 0])

#%% Solve
U7a = RK4(t_2, u0_2a, a=a_, b=b_, c=c_)
U7b = RK4(t_2, u0_2b, a=a_, b=b_, c=c_)

#%% Plots
plot3D(U7a[:,0], U7a[:,1], U7a[:,2])
plot3D(U7b[:,0], U7b[:,1], U7b[:,2])

#%% Plot only x_1(t), x_2(t) and difference
plot(t_2, U7a[:,0])
plot(t_2, U7b[:,0])
plot(t_2, U7a[:,0] - U7b[:,0])

#%% Data structure to save
X1 = np.zeros((len(U7a), 2)); X1[:,0] = t_2; X1[:,1] = U7a[:,0]
X2 = np.zeros((len(U7b), 2)); X2[:,0] = t_2; X2[:,1] = U7b[:,0]
X1X2 = np.zeros((len(U7b), 2)); X1X2[:,0] = t_2; X1X2[:,1] = U7a[:,0] - U7b[:,0]

#%% Save data
DIR = 'data/2/' # Directory name
pathlib.Path(DIR).mkdir(parents=True, exist_ok=True) # Create Folder
# Write files
np.savetxt(DIR + 'Ur.csv', Ur, fmt='%.8f', delimiter=',', header='x,y,z', comments="")
np.savetxt(DIR + 'U1.csv', U1, fmt='%.8f', delimiter=',', header='x,y,z', comments="")
np.savetxt(DIR + 'U2.csv', U2, fmt='%.8f', delimiter=',', header='x,y,z', comments="")
np.savetxt(DIR + 'U3.csv', U3, fmt='%.8f', delimiter=',', header='x,y,z', comments="")
np.savetxt(DIR + 'U4.csv', U4, fmt='%.8f', delimiter=',', header='x,y,z', comments="")
np.savetxt(DIR + 'U5.csv', U5, fmt='%.8f', delimiter=',', header='x,y,z', comments="")
np.savetxt(DIR + 'U6.csv', U6, fmt='%.8f', delimiter=',', header='x,y,z', comments="")
np.savetxt(DIR + 'U7a.csv', U7a, fmt='%.8f', delimiter=',', header='x,y,z', comments="")
np.savetxt(DIR + 'U7b.csv', U7b, fmt='%.8f', delimiter=',', header='x,y,z', comments="")
np.savetxt(DIR + 'X1.csv', X1, fmt='%.8f', delimiter=',', header='t,x', comments="")
np.savetxt(DIR + 'X2.csv', X2, fmt='%.8f', delimiter=',', header='t,x', comments="")
np.savetxt(DIR + 'X1X2.csv', X1X2, fmt='%.8f', delimiter=',', header='t,x', comments="")


import unittest

import numpy as np
import scipy.stats as stats

def my_entropy(px):
    return -np.sum(px * np.log2(px), axis=0)

def my_cross_entropy(p_true, p_predict):
    return -np.sum(p_true * np.log2(p_predict), axis=0)

# Also known as residual entropy.
def my_kullback_leibler(px, py):
    return np.sum(px * np.log2(px / py), axis=0)

def my_kullback_leibler2(px, py):
    return my_cross_entropy(px, py) - my_entropy(px)

def my_encoding_entropy(px, eb):
    return np.sum(px * eb)

#
# Unit Tests.
#

pk0 = np.asarray([.5, .5])
pk0_1 = np.asarray([.75, .25])
pk1 = np.asarray([.3, .4, .1, .2])
pk2 = np.asarray([.3, .4, .1, .2])
pk3 = np.asarray([.31, .39, .09, .21])

# Weather : Sun, Overcast, Sun+Cloud, Sun+HeavyCloud, LightRain, Rain, HeavyRain, Storm
w1 = np.asarray([1 / 8, 1 / 8, 1 / 8, 1 / 8, 1 / 8, 1 / 8, 1 / 8, 1 / 8])
w2 = np.asarray([.35, .35, .1, .1, 0.04, 0.04, 0.01, 0.01])
w3 = np.asarray([.01, .01, .04, .04, 0.1, 0.1, 0.35, 0.35])

# but used to encode each weather state
eb1 = np.asarray([3, 3, 3, 3, 3, 3, 3, 3])
eb2 = np.asarray([2, 2, 3, 3, 4, 4, 5, 5])

class TestEntropy(unittest.TestCase):

    def test_0(self):
        # two equally probable events = 1 bit of information
        self.assertAlmostEqual(my_entropy(pk0), 1)

        # Information is reduced if probabilities not equal, as in this case information is reduced
        # when we learn it is sunny as it sunny 75% of the time.
        self.assertAlmostEqual(my_entropy(pk0_1), 0.81, places=2)

        return

    def test_1(self):
        self.assertAlmostEqual(my_entropy(pk1), stats.entropy(pk=pk1, base=2))
        return

    # Divergence from self = 0
    def test_2(self):
        self.assertAlmostEqual(my_kullback_leibler(pk1, pk1), 0)
        return

    def test_3(self):
        self.assertAlmostEqual(my_kullback_leibler(pk1, pk2), stats.entropy(pk=pk1, qk=pk2, base=2))
        return

    def test_4(self):
        self.assertAlmostEqual(my_kullback_leibler(pk1, pk3), stats.entropy(pk=pk1, qk=pk3, base=2))
        return

    # Eight equally probable weather events = 3 bits
    def test_w0(self):
        self.assertAlmostEqual(my_entropy(w1), 3)
        return

    # Cross entropy is a function of bits used on average to encode a state. If less bits can be used
    # for most common states then cross entropy is closer to the entropy of the probability distribution
    # our ideal state is for cross entropy to be = entropy (lower limit)
    def test_w1(self):
        self.assertAlmostEqual(my_encoding_entropy(w2, eb2), 2.42, places=2)
        self.assertGreater(my_encoding_entropy(w2, eb2), my_entropy(w2))
        self.assertAlmostEqual(my_encoding_entropy(w3, eb2), 4.58, places=2)
        self.assertGreater(my_encoding_entropy(w3, eb2), my_entropy(w2))
        self.assertGreater(my_encoding_entropy(w3, eb2), my_encoding_entropy(w2, eb2))
        return

    # KL Divergence = Cross Entropy - Entropy.
    # So we test the difference between our expected distribution of weather events based on the
    # number of bits we chose to encode with vs. the actual distribution
    #
    def test_w2(self):
        p_true = w3
        p_predict = 1 / np.power(2, eb2)
        self.assertAlmostEqual(my_kullback_leibler2(p_true, p_predict), 2.35, places=2)
        return

    # Cross entropy loss function for NN classification.
    def test_cross_entropy_loss(self):
        p_one_hot = np.asarray([0.0, 0.0, 1.0, 0.0, 0.0])
        p_nn_pred = np.asarray([0.1, 0.2, 0.4, 0.1, 0.2])
        self.assertAlmostEqual(my_cross_entropy(p_one_hot, p_nn_pred), -np.log2(.4))
        return

# Execute the UnitTests.
#

if __name__ == "__main__":
    tests = TestEntropy()
    suite = unittest.TestLoader().loadTestsFromModule(tests)
    unittest.TextTestRunner().run(suite)


import numpy as np
from matplotlib import pyplot as plt
from optimizer import simplex_problem, verbose_print

import seaborn as sns

sns.set_palette('Set1')

paths = []

def feasible_region(A, b):
    """
    calculate the feasible region of LP prblem
    :param A: coef of constrains
    :param b: upper bound of constrains
    :return: 2D coordinate of point in feasible region
    """
    b = np.expand_dims(b, axis=1)
    x0 = np.linspace(0, 100, 500)
    x = np.array(np.meshgrid(x0, x0))
    # Construct grid
    x = x.reshape(2, -1)
    checked_cons = np.matmul(A, x) <= np.repeat(b, x.shape[1], axis=1)
    # and logic product in column
    mask = checked_cons.all(axis=0).reshape((x0.shape[0], x0.shape[0]))
    x = x.reshape((2, x0.shape[0], x0.shape[0]))
    x = (x.T[mask.T]).T
    return x

def visual_problem(c, A, b, optimal=None):
    """
    visualize for 2 variable constrains and feasible region for LP problem

    Given A.T * x <= b,that mean
    A[0]*x0 + A[1]*x1 <= b[0]
    Minimize
    c.T * x
    :param optimal: optimal solution
    :param A: constrain coefs
    :param b: constrain value
    :return: plt figure
    """

    if c.shape[0] != 2:
        raise Exception('function must be 2 dimension')

    epsilon = 0.000000001

    if A.shape[0] != b.shape[0]:
        raise Exception('A and b must be the same shape')

    fig, ax = plt.subplots(figsize=(8, 8))
    x1 = np.linspace(0, 100)

    for i in range(A.shape[0]):
        plt.plot(x1, (b[i] - A[i][0] * x1) / (A[i][1] + epsilon), lw=3,
                 label=r'{} : ${}x_1 + {}x_2 \leq{}$'.format(i, A[i][0], A[i][1], b[i]))
        plt.fill_between(x1, 0, (b[i] - A[i][0] * x1) / (A[i][1] + epsilon), alpha=0.1)

    plt.plot(np.zeros_like(x1), x1, lw=3, label=r'$x_1 : non-negative$')
    plt.plot(x1, np.zeros_like(x1), lw=3, label=r'$x_2 : non-negative$')
    # calculate the feasible
    fx, fy = feasible_region(A, b)
    z = np.dot(c.T, [fx, fy])
    plt.scatter(fx, fy, c=z, cmap='jet', zorder=-1)
    if optimal is not None:
        plt.plot(optimal[0], optimal[1], 'ro', label='optimal point')
    # labels and stuff
    cb = plt.colorbar()
    cb.set_label('objective value', fontsize=14)
    plt.xlabel('x1', fontsize=16)
    plt.ylabel('x2', fontsize=16)
    plt.xlim(-0.5, 100)
    plt.ylim(-0.5, 100)
    plt.legend(fontsize=14)
    plt.show()

if __name__ == '__main__':
    c = np.array([-8, -2])
    A = np.array([[1, 1],
                  [2, 1],
                  [1, 0]])
    b = np.array([80, 100, 40])
    res = simplex_problem(c, A, b, print_func=verbose_print)
    visual_problem(c, A, b, res['x'])


from numpy import *

# matrix function is deprecated check this link -> https://stackoverflow.com/questions/53254738/deprecation-status-of-the-numpy-matrix-class
#m1 = matrix(arr1) # using 2D array
#m2 = matrix('1 2 ; 3 4;5 6;') (2 x 3) using string as arg
#print("diagonal matrix : ",diagonal(m))
'''m1= matrix('1 2 ; 3 4;5 6;')
m2 =matrix('1 2 ; 3 4;5 6;')
m3 = m1 + m2
m4 = m1 * m2'''

'''# matrix multiplication
arr1 = array([[1,2,3,4],[5,6,7,8]])
arr2 = array([[1,2],[3,4],[5,6],[7,8]])

row1 , col1 = arr1.shape
row2 , col2 = arr2.shape
if col1 == row2:
    print("Matrix Multiplication is possible because (%d x %d[row1,col1]) X (%d x %d[row2,col2])"%(row1 , col1,row2,col2),"where %d == %d(col1==row2)"%(col1,row2),"So that it is not possible (%d x %d[row1,col1]) X (%d x %d[row2,col2]) -> (%d x %d[row1,col2])"%(row1 , col1,row2,col2,row1,col2))
    print("Multiplication is : ",arr1.dot(arr2))
else:
    print("Matrix Multiplication is not possible because  (%d x %d[row1,col1]) X (%d x %d[row2,col2])"%(row1 , col1,row2,col2),"where %d != %d(col1!=row2)"%(col1,row2),"So that it is not possible %d x %d X %d x %d -> %d x %d"%(row1 , col1,row2,col2,col1,row2))

# addition
arr3 = array([[1,2],[3,4]])
arr4 = array([[1,2],[3,4]])
row1 , col1 = arr3.shape
row2 , col2 = arr4.shape
if arr3.shape == arr4.shape:
    print("Matrix addition is possible because","shape is (%d,%d) == (%d,%d)"%(row1 , col1,row2,col2))
    print("addition is : ",arr3+arr4) # also use this add(arr3,arr4)
else:
    print("Matrix addition is not possible because","shape is (%d,%d) != (%d,%d)"%(row1 , col1,row2,col2))

# subtraction
arr5 = array([[1,2],[3,4]])
arr6 = array([[2,3],[1,2]])
row1 , col1 = arr5.shape
row2 , col2 = arr6.shape
if arr5.shape == arr6.shape:
    print("Matrix subtraction is possible because","shape is (%d,%d) == (%d,%d)"%(row1 , col1,row2,col2))
    print("Subtraction is : ",arr5-arr6) # subtract(arr3,arr4)
else:
    print("Matrix subtraction is not possible because","shape is (%d,%d) != (%d,%d)"%(row1 , col1,row2,col2))

# element vise multiplication
arr7 = array([[1,2],[3,4]])
arr8 = array([[1,2],[3,4]])
row1 , col1 = arr7.shape
row2 , col2 = arr8.shape
if arr7.shape == arr8.shape:
    print("Matrix element wise multiplication is possible because","shape is (%d,%d) == (%d,%d)"%(row1 , col1,row2,col2))
    print("Multiplication is : ",multiply(arr7,arr8)) # also use arr7*arr8 same for divide
else:
    print("Matrix element wise multiplication is not possible because","shape is (%d,%d) != (%d,%d)"%(row1 , col1,row2,col2))

# Transpose of matrix
arr9 = array([[1, 1], [2, 1], [3, -3]])
print("Transpose of matrix : ",arr9.transpose())'''

# Slicing of Matrix

# 1D array slicing
'''arr10 = array([1, 3, 5, 7, 9, 11, 13])
print("full : ",arr10[:])
print("first 4 elements : ",arr10[:4])
# slicing include upper limit and lower exclude and range exclude upper limit
print("4th elements : ",arr10[3:4])
# when we use - indexing upper limit is excluded
print("last 2 elements not consider : ",arr10[:-2])
print("from last 2nd plus 3 elements : ",arr10[-5:-2])
print("last 3rd element : ",arr10[-3:-2])
# reverse
print("reveres array : ",arr10[::-1])'''

# 2D array slicing
'''arr11 = array([[1, 4, 5, 12, 14],
    [-5, 8, 9, 10, 17],
    [-6, 7, 11, 19, 21]])

print("Full : ",arr11[:,:]) # arr11[row slice,col slice]
# lower limit is excluded
print("arr11[2,4] : ",arr11[1:2,3:4])
print("first 2 rows and 3 columns",arr11[:2,:3])
print("third row with all columns: ",arr11[2:3,])
# in - indexing upper limit is excluded
print("arr11[1,2] : ",arr11[-3:-2,-4:-3])'''

arr12 = array([[1, 4, 5, 12,21,22],
    [-5, 8, 9, 10,23,24],
    [-6, 7, 11, 19,25,26]])

arr13 = arr12.reshape(2,3,3)
print("full : ",arr13)
''' output of above
[[[ 1  4  5]
  [12 21 22]
  [-5  8  9]]

 [[10 23 24]
  [-6  7 11]
  [19 25 26]]]

from above first 3*3 matrix is
  [ 1  4  5]
  [12 21 22]
  [-5  8  9]
  second 3*3 matrix is
  [10 23 24]
  [-6  7 11]
  [19 25 26] '''
# check below description for understand all parameters
print("all matrices only, first column : ",arr13[:,:,:1])
'''B[:,:,1] we are passing : , : and 1. First : indicates that we are selecting both the 3X3 matrices.
The second : indicates that we are selecting all the rows from both the 3X3 matrices. The third parameter
1 indicates that we are selecting only the second column values of all the rows from both the 3X3 matrices'''

print("all matrices ,second row , second and third column : ",arr13[:,1:2,1:3])
print("2 matrix ,second and third row , second and third column : ",arr13[1:2,1:3,1:3])


 #!/usr/bin/env python

import numpy as np
import random

from q1_softmax import softmax
from q2_sigmoid import sigmoid, sigmoid_grad
from q2_gradcheck import gradcheck_naive

def forward_backward_prop(data, labels, params, dimensions):
    """
    Forward and backward propagation for a two-layer sigmoidal network

    Compute the forward propagation and for the cross entropy cost,
    and backward propagation for the gradients for all parameters.

    Arguments:
    data -- M x Dx matrix, where each row is a training example.
    labels -- M x Dy matrix, where each row is a one-hot vector.
    params -- Model parameters, these are unpacked for you.
    dimensions -- A tuple of input dimension, number of hidden units
                  and output dimension
    """

    ### Unpack network parameters (do not modify)
    ofs = 0
    Dx, H, Dy = (dimensions[0], dimensions[1], dimensions[2])

    W1 = np.reshape(params[ofs:ofs+ Dx * H], (Dx, H))
    ofs += Dx * H
    b1 = np.reshape(params[ofs:ofs + H], (1, H))
    ofs += H
    W2 = np.reshape(params[ofs:ofs + H * Dy], (H, Dy))
    ofs += H * Dy
    b2 = np.reshape(params[ofs:ofs + Dy], (1, Dy))

    ### YOUR CODE HERE: forward propagation

    #This is for the Input of the hidden layer H
    # h = sigmoid(np.dot(data,W1)+b1)
    z1 = np.dot(data,W1) + b1
    a1 = sigmoid(z1)

    #This is the output Layer
    z2 = np.dot(a1,W2) +b2
    y_pred = softmax(z2)

    cost = - np.sum(labels * np.log(y_pred))

    ### END YOUR CODE

    ### YOUR CODE HERE: backward propagation

    d3 = y_pred - labels
    gradW2 = np.dot(a1.T,d3)
    gradb2 = np.sum(d3,axis=0)

    da1 = np.dot(d3,W2.T)
    dz1 = sigmoid_grad(a1)*da1

    gradW1 = np.dot(data.T,dz1)
    gradb1 = np.sum(dz1,axis=0)
    ### END YOUR CODE

    ### Stack gradients (do not modify)
    grad = np.concatenate((gradW1.flatten(), gradb1.flatten(),
        gradW2.flatten(), gradb2.flatten()))

    return cost, grad

def sanity_check():
    """
    Set up fake data and parameters for the neural network, and test using
    gradcheck.
    """
    print("Running sanity check...")

    N = 20 #Number of Training examples
    dimensions = [10, 5, 10]
    data = np.random.randn(N, dimensions[0])   # each row will be a datum
    #print("Data :: ",data.shape)
    labels = np.zeros((N, dimensions[2]))
    print("Labels Shape:: ",labels.shape)
    for i in list(range(N)):
        #Making it a one-hot vector from All Zeros
        labels[i, random.randint(0,dimensions[2]-1)] = 1

    #print("Labels Value :: ",labels)

    ## Unfolding All the Parameters
    params = np.random.randn((dimensions[0] + 1) * dimensions[1] + (
        dimensions[1] + 1) * dimensions[2], )

    print("Param Shape:: " , params.shape)
    gradcheck_naive(lambda params:
        forward_backward_prop(data, labels, params, dimensions), params)

def your_sanity_checks():
    """
    Use this space add any additional sanity checks by running:
        python q2_neural.py
    This function will not be called by the autograder, nor will
    your additional tests be graded.
    """
    print("Running your sanity checks...")
    ### YOUR CODE HERE
    #raise NotImplementedError
    ### END YOUR CODE

if __name__ == "__main__":
    sanity_check()
    your_sanity_checks()


# coding: utf-8
import numpy as np
from cv2 import cv2
import matplotlib.pyplot as plt

# Consider an image whose pixel values are confined to some specific range of values only. For example, brighter image will have all pixels confined to high values. But a good image will have pixels from all regions of the image. So you need to stretch this histogram to either ends and that is what Histogram Equalization does (in simple words). This normally improves the contrast of the image.

img = cv2.imread(r'pictures\equalization_opencv.jpg', 0)

hist, bins = np.histogram(img.flatten(), 256, [0, 256])

cdf = hist.cumsum()
cdf_normalized = cdf * hist.max() / cdf.max()

# You can see histogram lies in brighter region. We need the full spectrum. For that, we need a transformation function which maps the input pixels in brighter region to output pixels in full region. That is what histogram equalization does.
# Now we find the minimum histogram value (excluding 0) and apply the histogram equalization equation as given in wiki page. But I have used here, the masked array concept array from Numpy. For masked array, all operations are performed on non-masked elements. You can read more about it from Numpy docs on masked arrays.
cdf_m = np.ma.masked_equal(cdf, 0)
cdf_m = (cdf_m - cdf_m.min())*255/(cdf_m.max()-cdf_m.min())
cdf = np.ma.filled(cdf_m, 0).astype('uint8')

img2 = cdf[img]
# Now calculate its histogram and cdf as before
hist2, bins = np.histogram(img2.flatten(), 256, [0, 256])

cdf2 = hist2.cumsum()
cdf_normalized2 = cdf2 * hist2.max() / cdf2.max()

plt.subplot(221), plt.imshow(img, 'gray')
plt.subplot(222), plt.plot(cdf_normalized, color='b'), plt.hist(img.flatten(), 256, [
    0, 256], color='r'), plt.xlim([0, 256]), plt.legend(('cdf', 'histogram'), loc='upper left')
plt.subplot(223), plt.imshow(img2, 'gray')
plt.subplot(224), plt.plot(cdf_normalized2, color='b'), plt.hist(img2.flatten(), 256, [
    0, 256], color='r'), plt.xlim([0, 256]), plt.legend(('cdf', 'histogram'), loc='upper left')

plt.show()

# Another important feature is that, even if the image was a darker image (instead of a brighter one we used), after equalization we will get almost the same image as we got. As a result, this is used as a “reference tool” to make all images with same lighting conditions. This is useful in many cases. For example, in face recognition, before training the face data, the images of faces are histogram equalized to make them all with same lighting conditions.

# About numpy.ma : https://blog.csdn.net/wzy628810/article/details/103833856
# Numpy Doc : https://numpy.org/doc/stable/reference/maskedarray.html
# OR : https://docs.scipy.org/doc/numpy-1.13.0/reference/maskedarray.generic.html#maskedarray-generic-constructing


import sympy

A = sympy.Matrix([[1, 2], [2, -1]])
B = sympy.Matrix([[1, 0, 3], [2, 1, 1]])
C = sympy.Matrix([[1, 2], [3, 1], [2, -1]])
D = sympy.Matrix([[1, 0, 3], [1, 1, -2], [2, -1, 1]])
E = sympy.Matrix([[1], [2], [-1]])
F = sympy.Matrix([[1, 2, 3]])

print(A*B)
print(D**2)
print(D*C)
print(C*B)
print(B*C)
print(F*E)
print(E*F)
print()

C = sympy.Matrix([[2, 3], [1, 2]])
D = sympy.Matrix([[1, 1, 0], [0, 1, 1], [1, 0, 1]])
E = sympy.Matrix([[3, 0, 2], [2, -1, -2], [-1, 2, 6]])

print(A**-1)
print(C**-1)
print(D**-1)
print()
print((A**-1)*sympy.Matrix([[1], [-2]]))
print((D**-1)*E)


import numpy as np

# stretched exponential decay function
def stretched_exp(x, x0, a, beta):
    return x0*np.exp(-a*(x**beta))

# 'exp exp' decay function
def expexp(x, a0, a1, lam, beta):
    return a0*np.exp(-a1*(1-np.exp(-lam*x**beta)))


# Based on http://d2l.ai/chapter_computer-vision/transposed-conv.html

import superimport

import jax
import jax.numpy as jnp

def trans_conv(X, K):
    h, w = K.shape
    Y = jnp.zeros((X.shape[0] + h - 1, X.shape[1] + w - 1))
    for i in range(X.shape[0]):
        for j in range(X.shape[1]):
            Y = Y.at[i : i + h, j : j + w].add(X[i, j] * K)
    return Y

# Example from D2L fig 13.10.1
X = jnp.array([[0.0, 1], [2, 3]])
K = jnp.array([[0.0, 1], [2, 3]])
Y = trans_conv(X, K)
print(Y)

X_ = X.reshape(1, 1, 2, 2)
K_ = jnp.rot90(K, 2).reshape(1, 1, 2, 2)
Y2 = jax.lax.conv_general_dilated(X_, K_, window_strides=(1, 1), padding=((1, 1), (1, 1)), lhs_dilation=(1, 1))
# print(Y2)
assert jnp.allclose(Y, Y2)

"""
X_ = X.reshape(1, 1, 2, 2)
K_ = jnp.rot90(K, 2).reshape(1, 1, 2, 2)
Y2 = jax.lax.conv_general_dilated(X_, K_, window_strides=(1, 1), padding=((0, 0), (0, 0)), lhs_dilation=(1, 1))
print('Y2', Y2)
"""

# Transposed Matrix multiplication

K = jnp.array([[1, 2], [3, 4]])

def kernel2matrix(K):
    k, W = jnp.zeros(5), jnp.zeros((4, 9))
    k = k.at[:2].set(K[0, :])
    k = k.at[3:5].set(K[1, :])
    W = W.at[0, :5].set(k)
    W = W.at[1, 1:6].set(k)
    W = W.at[2, 3:8].set(k)
    W = W.at[3, 4:].set(k)
    return W

W = kernel2matrix(K)

X = jnp.array([[0.0, 1], [2, 3]])
Y = trans_conv(X, K)
Y2 = jnp.dot(W.T, X.reshape(-1)).reshape(3, 3)
assert jnp.allclose(Y, Y2)

# Example from Geron fig 14.27

X = jnp.ones((2, 3))
K = jnp.ones((3, 3))
X_, K_ = X.reshape(1, 1, 2, 3), jnp.rot90(K, 2).reshape(1, 1, 3, 3)
Y2 = jax.lax.conv_general_dilated(X_, K_, window_strides=(1, 1), padding=((2, 2), (2, 2)), lhs_dilation=(2, 2))
print(Y2)
print(Y2.shape)


import numpy as np
import pandas as pd

def meanX(dataX):
    return np.mean(dataX, axis=0)

def variance(X):
    m, n = np.shape(X)
    mu = meanX(X)
    muAll = np.tile(mu, (m, 1))
    X1 = X - muAll
    variance = 1. / m * np.diag(X1.T * X1)
    return variance

def normalize(X):
    m, n = np.shape(X)
    mu = meanX(X)
    muAll = np.tile(mu, (m, 1))
    X1 = X - muAll
    X2 = np.tile(np.diag(X.T * X), (m, 1))
    XNorm = X1 / X2
    return XNorm

def pca(XMat, k):
    average = meanX(XMat)
    m, n = np.shape(XMat)
    avgs = np.tile(average, (m, 1))
    data_adjust = XMat - avgs
    covX = np.cov(data_adjust.T)
    featValue, featVec = np.linalg.eig(covX)
    index = np.argsort(-featValue)
    if k > n:
        print("k must lower than feature number")
        return
    else:
        selectVec = featVec.T[index[:k]]
        finalData = np.matmul(data_adjust, selectVec.T)
        reconData = np.matmul(finalData, selectVec) + average
        real_finalData = finalData.real
        real_reconData = reconData.real
    return real_finalData, real_reconData

def loaddata(datafile):
    return np.array(pd.read_csv(datafile, sep="\t", header=None)).astype(np.float)

def reduction(CS, DS, FLAGS):
    k_CS = int(CS.shape[1] * 0.8)
    k_DS = int(DS.shape[1] * 0.8)
    CS, reconMat = pca(CS, k_CS)
    DS, reconMat = pca(DS, k_DS)
    return CS, DS


# coding: utf-8
import numpy as np
import matplotlib.pylab as plt

def softmax(a):
    exp_a = np.exp(a)
    sum_exp_a = np.sum(exp_a)
    return exp_a / sum_exp_a

def soft_max_improved(a):
    c = np.max(a)
    exp_a = np.exp(a - c)
    return exp_a / np.sum(exp_a)

# X = np.arange(-5.0, 5.0, 0.1)
# Y = softmax(X)
# plt.plot(X, Y)
# # plt.ylim(-0.1, 1.1)
# plt.show()

# Test:
a = np.array([0.3, 2.9, 4.0])
# a1 = np.array([1010, 1000, 990]) # overflow

exp_a = np.exp(a)
print(exp_a)

sum_exp_a = np.sum(exp_a)
print(sum_exp_a)

print(exp_a / sum_exp_a)


import numpy as np

def unit_step(v):
	if v >= 0:
		return 1
	else:
		return 0

def perceptron(x, w, b):
    v = np.dot(w, x)+b
    y =unit_step(v)
    return y

def NOR_percep(x):
    w = np.array([-1, -1])
    b = 0.5
    result = perceptron(x, w, b)
    return result

example1 = np.array([1, 1])
example2 = np.array([1, 0])
example3 = np.array([0, 1])
example4 = np.array([0, 0])

print("1 NOR 1: ",NOR_percep(example1))
print("1 NOR 0: ",NOR_percep(example2))
print("0 NOR 1: ",NOR_percep(example3))
print("0 NOR 0: ",NOR_percep(example4 ))


import numpy as np

def calc_v(mu, r, a):
    ''' Calculate velocity at a point in an orbit given
    the gravitational parameter of the primary body, radius
    at the point of interest, and the semi-major axis of
    the orbit. '''
    return np.sqrt(2*(mu/r - mu/2/a))  # (km/s)

def calc_a(h, mu, e):
    ''' Calculate semi-major axis given specific angular momentum,
    gravitational parameter of primary body, and eccentricity. '''
    return (h**2/mu)*(1/(1 - e**2))

def calc_T(mu, a):
    ''' Calculate orbital period given gravitational parameter
    and semi-major axis. '''
    return 2*np.pi*a**1.5/np.sqrt(mu)

def calc_RA_dot(mu, J2, R, e, a, i):
    ''' Calculate time rate of change of Right Ascension (RA)
    of an orbit given gravitational parameter, J2, equatorial
    radius of primary body, eccentricity, semi-major axis,
    and inclination. '''
    return -1.5*(np.sqrt(mu)*J2*R**2)/((1 - e**2)**2*a**3.5)*np.cos(i)

def calc_AP_dot_1(mu, J2, R, e, a, i):
    ''' Calculate time rate of change of Argument of Periapsis (AP)
    given gravitational parameter, J2, equatorial radius of primary
    body, eccentricity, semi-major axis, and inclination. '''
    return -1.5*(np.sqrt(mu)*J2*R**2)/((1 - e**2)**2*a**3.5)*(2.5*(np.sin(i))**2 - 2)

def calc_AP_dot_2(RAdot, i):
    ''' Calculate time rate of change of Argument of Periapsis (AP)
    given time rate of change of Right Ascension (RAdot) and
    inclination. '''
    return RAdot*(2.5*(np.sin(i))**2 - 2)/(np.cos(i))

def find_orbital_elements(r_, v_, mu):
    ''' Calculate all six orbital elements given a position and
    velocity vector at some point in the orbit, and the gravitational
    parameter of the primary body. A variable with a trailing
    underscore denotes a vector: x_ is a vector, x is the magnitude
    of x_. Applies Algorithm 4.2 from Orbital Mechanics for Engineering
    Students, 4 ed, Curtis. Returns angles in radians. '''
    r = np.linalg.norm(r_)
    v = np.linalg.norm(v_)
    vr = np.dot(r, v)/r

    # Specific Angular Momentum
    h_ = np.cross(r_, v_)
    h = np.linalg.norm(h_)

    # Inclination
    i = np.arccos(h_[2]/h)

    # Right Ascension
    Khat_ = np.array([0.0, 0.0, 1.0])
    N_ = np.cross(Khat_, h_)
    N = np.linalg.norm(N_)

    if N != 0:
        RA = np.arccos(N_[0]/N)
        RA = RA if N_[1] >= 0 else 2*np.pi - RA
    else:
        RA = 0.0

    # Eccentricity
    e_ = (1/mu)*(np.cross(v_, h_) - mu*r_/r)
    e = np.linalg.norm(e_)

    # Argument of Periapsis
    if N != 0:
        if e > 1e-7:
            AP = np.arccos(np.dot(N_/N, e_/e))
            AP = AP if e_[2] >= 0 else 2*np.pi - AP
        else:
            AP = 0.0
    else:
        AP = 0.0

    # True Anomaly
    if e > 1e-7:
        TA = np.arccos(np.dot(e_/e, r_/r))
        TA = TA if vr >= 0 else 2*np.pi - TA
    else:
        cp = np.cross(N_, r_)
        TA = np.arccos(np.dot(N_, r_)/N/r)
        TA = TA if cp[2] >= 0 else 2*np.pi - TA

    return h, i, e, RA, AP, TA

def calc_v_infinity(mu, R1, R2):
    ''' Calculate the hyperbolic excess speed for an escape trajectory
    from planet 1 with circular orbit radius R1 about the primary with
    gravitational parameter mu headed toward planet 2 with radius R2
    about the primary.
    :param R1: `float` (km) circular orbital radius, planet 1
    :param R2: `float` (km) circular orbital radius, planet 2
    :return: `float` (km/s) hyperbolic excess speed of escape trajectory
    '''
    return np.sqrt((mu/R1)*(np.sqrt((2*R2)/(R1 + R2)) - 1))

def calc_dv_hohmann_coplanar_common_apseline(mu, ra1, rp1, ra2, rp2):
    ''' Calculate total delta-V (m/s) required for a Hohmann transfer between
    two coplanar orbits with a common apse line.
    :param mu: `float` (km**3/s**2) gravitational parameter of primary body
    :param ra1: `float` (km) apoapsis radius, initial orbit
    :param rp1: `float` (km) periapsis radius, initial orbit
    :param ra2: `float` (km) apoapsis radius, final orbit
    :param rp2: `float` (km) periapsis radius, final orbit
    :return: `float` (m/s) total delta-V required for maneuver
    '''
    a1 = 0.5*(ra1 + rp1)
    ai = 0.5*(ra2 + rp1)
    a2 = 0.5*(ra2 + rp2)

    vp1 = calc_v(mu, rp1, a1)
    vpi = calc_v(mu, rp1, ai)
    vai = calc_v(mu, ra2, ai)
    va2 = calc_v(mu, ra2, a2)

    dv1 = vpi - vp1  # (km/s) delta-V of first burn
    dv2 = va2 - vai  # (km/s) delta-V of second burn

    return (np.abs(dv1) + np.abs(dv2))  # (km/s) total maneuver delta-V


# =========================================================
# @purpose: principal components analysis
# @date：   2019/12
# @version: v1.0
# @author： Xu Huasheng
# @github： https://github.com/xuhuasheng/pca
# =========================================================

from sklearn.externals import joblib
import numpy as np
import glob
import os
import time

def pca(dataMat, r, autoset_r=False, autoset_rate=0.9):
    """
    purpose: principal components analysis
    """
    print("Start to do PCA...")
    t1 = time.time()
    meanVal = np.mean(dataMat, axis=0) # 竖着求平均值，数据格式是m×n
    meanRemoved = dataMat - meanVal
    # normData = meanRemoved / np.std(dataMat) #标准差归一化
    covMat = np.cov(meanRemoved, rowvar=0)   # 求协方差矩阵 n×n维
    eigVals, eigVects = np.linalg.eig(np.mat(covMat)) # 求特征值和特征向量  eigVals: 1xn维, eigVects: n×n维
    eigValIndex = np.argsort(-eigVals) # 特征值由大到小排序，eigValIndex是特征值索引的重排列 1×n维

    # 自动设置r的取值
    if autoset_r:
        r = autoset_eigNum(eigVals, autoset_rate)
        print("autoset: take top {} of {} features".format(r, meanRemoved.shape[1]))

    r_eigValIndex = eigValIndex[:r] # 选取前r个特征值的索引  1×r维
    r_eigVect = eigVects[:, r_eigValIndex] # 把前r个特征值对应的特征向量组成投影变换矩阵P  n×r维
    lowDDataMat = meanRemoved * r_eigVect # 矩阵点乘投影变换矩阵, 得降维后的矩阵: m×r维 公式Y=X*P
    reconMat = (lowDDataMat * r_eigVect.T) + meanVal   # 转换新空间的数据  m×n维
    t2 = time.time()
    print("PCA takes %f seconds" %(t2-t1))
    joblib.dump(r_eigVect, './pca_args_save/r_eigVect.eig')
    joblib.dump(meanVal, './pca_args_save/meanVal.mean') # save mean value
    return lowDDataMat, reconMat

def autoset_eigNum(eigValues, rate=0.99):
    """
    purpose: Automatically set the optimal number of eigen vectors
    """
    eigValues_sorted = sorted(eigValues, reverse=True)
    eigVals_total = eigValues.sum()
    for i in range(1, len(eigValues_sorted)+1):
        eigVals_sum = sum(eigValues_sorted[:i])    # 前i个特征向量求和
        if eigVals_sum / eigVals_total >= rate:
            break
    return i


import numpy as np
import pandas as pd
import matplotlib.pyplot as plt

df = pd.read_csv("china_gdp.csv", sep=",")
print(df.head(8))
print(df.columns)
print(df.dtypes)

#we can see it is similar to Logistic Function.
plt.figure(figsize=(8,5))
x_data, y_data = (df["Year"].values, df["Value"].values) #your X and Y
plt.plot(x_data, y_data, 'ro')
plt.ylabel('GDP')
plt.xlabel('Year')
plt.show()

"""Sigmoid Function"""
def sigmoid(x, Beta_1, Beta_2):
     y = 1 / (1 + np.exp(-Beta_1*(x-Beta_2)))
     return y

"""Model"""
beta_1 = 0.10
beta_2 = 1990.0

#logistic function
Y_pred = sigmoid(x_data, beta_1 , beta_2) #call function

#plot initial prediction against datapoints
plt.plot(x_data, Y_pred*15000000000000., 'b')
plt.plot(x_data, y_data, 'ro')
plt.show()

# Lets normalize our data
xdata =x_data/max(x_data)
ydata =y_data/max(y_data)
#print(len(xdata),max(x_data))

from scipy.optimize import curve_fit
popt, pcov = curve_fit(sigmoid, xdata, ydata)
#print the final parameters
print(" beta_1 = %f, beta_2 = %f" % (popt[0], popt[1]))

x = np.linspace(1960, 2015, 55)
x = x/max(x)
plt.figure(figsize=(8,5))
y = sigmoid(x, *popt)
plt.plot(xdata, ydata, 'ro', label='data')
plt.plot(x,y, linewidth=3.0, label='fit')
plt.legend(loc='best')
plt.ylabel('GDP')
plt.xlabel('Year')
plt.show()

###################################Model data using sklearn #################################
# split data into train/test
msk = np.random.rand(len(df)) < 0.8
train_x = xdata[msk]
test_x = xdata[~msk]
train_y = ydata[msk]
test_y = ydata[~msk]

# build the model using train set
popt, pcov = curve_fit(sigmoid, train_x, train_y)

# predict using test set
y_hat = sigmoid(test_x, *popt)

# evaluation
print("Mean absolute error: %.2f" % np.mean(np.absolute(y_hat - test_y)))
print("Residual sum of squares (MSE): %.2f" % np.mean((y_hat - test_y) ** 2))
from sklearn.metrics import r2_score
print("R2-score: %.2f" % r2_score(y_hat , test_y) )


import numpy as np
from scipy import integrate

f1 = lambda x, y: 1 / (np.sqrt(x+y) * (1+x+y))
f2 = lambda x, y: x**2 + y ** 2

def trapezoidal_rule_2D(f, xmin, xmax, xnum, ymin, ymax, ynum):
    integral = 0
    hx = (xmax - xmin) / xnum

    for i in range(0, xnum + 1):
        x = xmin + i * hx
        hy = (ymax(x) - ymin(x)) / ynum

        part_int = 0
        for j in range(0, ynum + 1):
            y = ymax(x) + j * hy

            if j in {0, ynum}:
                part_int += 0.5 * f(x, y)
            else:
                part_int += f(x, y)

        part_int *= hy
        if i in {0, xnum}:
            integral += 0.5 * part_int
        else:
            integral += part_int

    integral *= hx
    return integral

f = f1
xmin = 0
xmax = 1
ymin = lambda x: 0
ymax = lambda x: 1 - x
xnum = 1000
ynum = 1000
custom_int = trapezoidal_rule_2D(f, xmin, xmax, xnum, ymin, ymax, ynum) * 10
lib_int = integrate.dblquad(f, xmin, xmax, ymin, ynum)[0]

print("Custom: " + str(custom_int))
print("Library: " + str(lib_int))

f = f2
xmin = -3
xmax = 3
ymin = lambda x: -5
ymax = lambda x: 5
xnum = 1000
ynum = 1000
custom_int = trapezoidal_rule_2D(f, xmin, xmax, xnum, ymin, ymax, ynum) / 10
lib_int = integrate.dblquad(f, xmin, xmax, ymin, ynum)[0]

print("Custom: " + str(custom_int))
print("Library: " + str(lib_int))


import matplotlib.pyplot as graph
import numpy as np
import pandas as pd
import scipy.optimize as opt

# --------------------------------------------------------------------
# Logistic growth model to predict COVID-19 prevalence and
# incidence rate using existing data from John Hopkins University and
# New York Times Github (https://github.com/nytimes/covid-19-data)
#
# Written Arnel Aguinaldo, PhD (Point Loma Nazarene University)
# Applicable rights for free reproduction of this program are defined in
# MIT License
# --------------------------------------------------------------------

# growth rates (vary by region)
b0 = 0.20   #full exposure
b1 = 0.12   #60% exposure due to mitigation measures (i.e., social distancing)
b2 = 0.08   #40% exposure due to mitigation measures (i.e., social distancing)

# population max
Nmin=1
Nmax=10000
a=(Nmax-Nmin)/Nmin
dt=1

# time interval
t=np.arange(1,120,2)

# infection prevalence (total infected per day)
def n0(t):
    return Nmax/(1+a*np.exp(-b0*t))
def n1(t):
    return Nmax/(1+a*np.exp(-b1*t))
def n2(t):
    return Nmax/(1+a*np.exp(-b2*t))

# infection incidence rates (number of new cases per day)
def r0(t):
    return (n0(t)-n0(t-1))/dt
def r1(t):
    return (n1(t)-n1(t-1))/dt
def r2(t):
    return (n2(t)-n2(t-1))/dt

# graph style
graph.style.use('seaborn-talk')

# Prevalence plot
graph.figure(1)
graph.plot(t, n0(t),'b', t, n1(t), 'r', t, n2(t), 'g')
graph.xlabel('Days')
graph.ylabel('Prevalence (Total Cases)')
graph.legend([ 'Full Exposure','60% Exposure', '40% Exposure'])
graph.title('COVID19')
graph.grid('true')

# Incidence rate plot
graph.figure(2)
graph.plot(t, r0(t),'b', t, r1(t), 'r', t, r2(t), 'g')
graph.xlabel('Days')
graph.ylabel('Incidence Rate')
graph.legend([ 'Full Exposure','60% Exposure', '40% Exposure'])
graph.title('COVID19')
graph.grid('true')

graph.show()

#Day 5 and 30 prevalence by exposure
#Full exposure
print(n0(5), n0(30))
#60% exposure
print(n1(5), n1(30))
#40% exposure
print(n2(5), n2(30))

# ----------------------------------------------------------
# Fit logistic model to real data (USA or state)
# ----------------------------------------------------------

#import case data (update with the local path of the csv file)
sPath='total-cases-covid=19-04052020.csv'
alldata=pd.read_csv(
        sPath,
        sep=',',
        usecols=['Days', 'California']
        )
alldata.head()
alldata=alldata.dropna()

#logistic function to be fitted
def r(t,a,b,c):
    return c/(1+a*np.exp(-b0*t))

# initialize coefficients using known or random values
#init_guess = [1, 1.5, Nmax]
init_guess = np.random.exponential(size=3)

# set array bounds (adjust according to region or target population)
bounds = (0, [1000., 0.5, 100000.])

t = np.array(alldata['Days']) + 1
y = np.array(alldata['California'])

# real infection rate
rate = (y[1:]-y[:-1])        #daily incidence (growth)
t_max = t.size

# ----------------------------------------------------------
# Logistic model to predict prevalence (total cases)
# ----------------------------------------------------------

#non-linear optimization to estimate best fit model on case data
#uncomment next line to find best fit with new regional data
#(a,b,c),covar = opt.curve_fit(r, t, y, bounds=bounds, p0=init_guess)

#BEST FIT FOR CALIFORNIA PREVALENCE (r^2=0.992):
(a,b,c)=(234.7722, 0.2414, 19362.9526)

# time intervals
tn=np.arange(1,40,2)
tr=np.arange(1,t_max)

# fitted infection prevalence (total infected per day)
def n_lg(tn):
    return c/(1+a*np.exp(-b*tn))

# prevalence plot
graph.figure(1)
graph.scatter(t,y)
graph.plot(tn, n_lg(tn), 'r')
graph.xlabel('Days')
graph.ylabel('Prevalence (Total Cases)')
graph.legend([ 'Model','California'])
graph.title('COVID19')
graph.grid('true')

# ----------------------------------------------------------
# Logistic model to predict incidence (daily growth) rate
# ----------------------------------------------------------

y=rate

#non-linear optimization to estimate best fit model on daily rate
#uncomment next line to find best fit with new regional data
#(a,b,c),covar = opt.curve_fit(r, tr, y, bounds=bounds, p0=init_guess)

#BEST FIT FOR CALIFORNIA INCIDENCE (r^2=0.812):
(a,b,c)=(204.7359, 0.2551, 3021.3831)

def rate_lg(tr):
    return c/(1+a*np.exp(-b*tr))

# Incidence rate (%) plot
graph.figure(2)
graph.scatter(tr,rate)
graph.plot(tr, rate_lg(tr), 'r')
graph.xlabel('Days')
graph.ylabel('Incidence Rate')
graph.legend([ 'Model','California'])
graph.title('COVID19')
graph.grid('true')


## module gaussSeidel
''' x,numIter,omega = gaussSeidel(iterEqs,x,tol = 1.0e-9)
    Gauss-Seidel method for solving [A]{x} = {b}.
    The matrix [A] should be sparse. User must supply the
    function iterEqs(x,omega) that returns the improved {x},
    given the current {x} ('omega' is the relaxation factor).
'''
import numpy as np
import math

def gaussSeidel(iterEqs,x,tol = 1.0e-9):
    omega = 1.0
    k = 10
    p = 1
    for i in range(1,501):
        xOld = x.copy()
        x = iterEqs(x,omega)
        dx = math.sqrt(np.dot(x-xOld,x-xOld))
        if dx < tol: return x,i,omega
      # Compute relaxation factor after k+p iterations
        if i == k: dx1 = dx
        if i == k + p:
            dx2 = dx
            omega = 2.0/(1.0 + math.sqrt(1.0 - (dx2/dx1)**(1.0/p)))
    print('Gauss-Seidel failed to converge')


# What happens to quantization error when you fourier transform?
# before figuring out analytically we experiment computationally to find out what to expect.
import numpy as np
from numpy.fft import fft,rfft,irfft,ifft

def quantize_uniform_noise(signal,delta):
    # input signal simulated getting 'quantized' by adding noise
    noise = np.random.uniform(-delta/2,delta/2,len(signal))
    return signal + noise

def mse(signal,noisy_signal):
    return np.mean(abs(signal - noisy_signal)**2)

if __name__=="__main__":
    mse_signals = []
    mse_rft_signals = []
    sqrt = np.sqrt
    delta = sqrt(12) # 1.0
    n = 100000
    print("Simulating the effects of quantization on signals and their fourier transforms...\n\n")
    for i in range(1000):
        signal = np.random.normal(0,1,n)
        noisy_signal = quantize_uniform_noise(signal,delta)
        mse_signal = mse(signal,noisy_signal)
        mse_signals.append(mse_signal)

        rft_s = rfft(signal)/sqrt(n)
        rft_ns = rfft(noisy_signal)/sqrt(n)
        mse_rft_signal = mse(rft_s,rft_ns)
        mse_rft_signals.append(mse_rft_signal)
    print("mean squared error for the singal is {}\nmean squared error for the rftsig is {}".format(np.mean(mse_signals),np.mean(mse_rft_signals)))
    print("")
    print("the respective stds are {} and {}".format(np.std(mse_signals),np.std(mse_rft_signals)))


import numpy as np
import matplotlib.pyplot as plt

def f(x):
    return 2*x**2

# np.arange(start, stop, step) to give us smoother line
x = np.arange(0, 5, 1e-3)
y = f(x)

plt.figure()
plt.plot(x, y)

# the point and the 'close enough' point
p2_delta = 1e-4
x1 = 2
x2 = x1 + p2_delta
y1 = f(x1)
y2 = f(x2)

print((x1,y1), (x2,y2))

# derivative approximation and y-intercept for the tangent line
approximate_derivative = (y2-y1)/(x2-x1)
b = y2 -  approximate_derivative*x2

# we put the tangent line calculation into a function so we can call
# it multiple times for different values of x
# approximate_derivative and b are constant for given function
# thus calculated once above this function
def tangent_line(x):
    return approximate_derivative*x + b

# plotting the tangent line
# +- 1 to draw the tangent line on our graph
# then we calculate the y for given x using the tangent line function
# Matplotlib will draw a line for us through these points
to_plot = [x1-1, x1, x1+1]
plt.plot(to_plot, [tangent_line(i) for i in to_plot])
plt.savefig('graph1.png')

print('approximate derivative for f(x)',
    f'where x = {x1} is {approximate_derivative}')


# Use Runge-Kutta 45 to numerically solve the differential equation of the voltage discharge from a simple RC circuit
# by gunnar pope
# 01/24/19
# Usage: $ python RK4.py

# references: https://docs.scipy.org/doc/scipy/reference/generated/scipy.integrate.solve_ivp.html

from scipy.integrate import solve_ivp

R = 1e6 #Ohms
C = 1e-6 #Farads

# dV/dt = -1/(R*C)*Vc
def f(t,y):
    dydt = -1.0/(R*C)*y
    return dydt

# solve_ivp(fun, t_span, y0, method='RK45', t_eval=None, dense_output=False, events=None, vectorized=False, **options)
t_start = 0
t_end   = 10

# initial conditions (this could be a vector too [a, b, c, etc]
ics = [2] #volts

# apply RK45 to numerically solve the differential equation, given the initial conditions and time span.
sol = solve_ivp(f, [t_start,t_end],ics, method='RK45')
print(sol.t)
print()
print(sol.y)

import matplotlib.pyplot as plt
import numpy as np

# compare the numerical solution to the solution in the time-domain.
t = np.linspace(t_start,t_end,100)
vout = ics*np.exp(-1/(R*C)*t)

plt.plot(t,vout, label='Exact Solution')
plt.plot(sol.t, sol.y[0], label='RK4 Solution')
plt.title("A Numerical Methods Example Using RK4 In Python")
plt.xlabel("Time (s)")
plt.ylabel("Voltage (v)")
plt.grid(True)
plt.legend()
plt.show()


import numpy as np
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
from matplotlib import cm
from mpl_toolkits.mplot3d import Axes3D
import math
import sys
import os

data_dir = sys.argv[1]
out_dir = sys.argv[2]
out = os.path.join(sys.argv[2], 'Q2b.txt')
outfile = open(out, "w")

# 2. Sampling and Stochastic Gradient
print("\n################ 2. Sampling and Stochastic Gradient ################", file=outfile)

# (a) Sampling

w = np.array([[3],[1],[2]])
X1 = np.random.normal(3, 2, (int)(1e6))
X2 = np.random.normal(-1, 2, (int)(1e6))
noise = np.random.normal(0, np.sqrt(2), int(1e6)).reshape(-1,1)

X = np.column_stack((X1, X2))
X = np.column_stack((np.ones(X.shape[0]), X))
Y = np.dot(X, w) + noise
m = len(Y)

# (b) Stochastic Gradient

def hw(theta, i, r):
	x = X[i*r:(i+1)*r,]
	return np.dot(x, theta)

def Jw(h, i, r):
	y = Y[i*r:(i+1)*r,]
	return (0.5/r)*np.sum((y - h)**2)

def dJw(h, i, r):
	x = X[i*r:(i+1)*r,]
	y = Y[i*r:(i+1)*r,]
	return (1.0/r)*np.dot(x.T, (y - h))

def stochasticGradientDescent(x, y, r):
	print('\nBatch Size (r) = {}'.format(r), file=outfile)
	b = (int)(m/r)
	eta = 0.001
	theta = np.zeros((x.shape[1], 1))
	thetaL = [theta]
	prevCost = 0
	converged = False
	itr = 0
	while not converged:
		for i in range(b):
			h = hw(theta, i, r)
			cost = Jw(h, i, r)
			theta = theta + eta*dJw(h, i, r)
			thetaL.append(theta)
		itr += 1
		error = abs(cost - prevCost)
		prevCost = cost
		if error < 1e-7 or itr > 25000:
			converged = True

		if r == m and itr % 1000 == 0:
			print('iteration {}: error = {} '.format(itr, error), end = '', file=outfile)
			print('w = {0},{1},{2}'.format(theta[0], theta[1], theta[2]), file=outfile)
		elif r != m:
			print('iteration {}: error = {} '.format(itr, error), end = '', file=outfile)
			print('w = {0},{1},{2}'.format(theta[0], theta[1], theta[2]), file=outfile)
	print("Stopping Criteria: Epoch > 7 and Error < 1e-7", file=outfile)
	print("Max Iterations = ", itr, file=outfile)
	print("Error =", error, file=outfile)
	print("Final Parameters = {0},{1},{2}\n".format(theta[0], theta[1], theta[2]), file=outfile)
	return thetaL, itr

thetaL1, maxit1 = stochasticGradientDescent(X, Y, 1)
thetaL2, maxit2 = stochasticGradientDescent(X, Y, 100)
thetaL3, maxit3 = stochasticGradientDescent(X, Y, 10000)
thetaL4, maxit4 = stochasticGradientDescent(X, Y, 1000000)


'''
  File: lorenz.py

  Author: Thomas Kost

  Date: 08 August 2021

  @breif function for lorenz chaotic system, and plotting script
 '''
import numpy as np
import matplotlib.pyplot as plt
def lorenz(x :np.array, beta: np.array)->np.array:
    '''
    x: position state vector of lorenz system

    beta: vector of lorenz parameters

    return: derivative of state vector

    '''

    dx = np.array([ beta[0]*(x[1]-x[0]),
                    x[0]*(beta[1]-x[2])-x[1],
                    x[0]*x[1]-beta[2]*x[2]])
    return dx

def step_lorenz(x:np.array, beta:np.array, dt:float)->np.array:
    next_x = x + lorenz(x,beta)*dt
    return next_x

if __name__ == "__main__":
    dt = 0.001
    t = np.arange(0,50,dt)
    x = np.zeros((3,len(t)))
    x_0 = np.array((0,1,20))
    b = np.array((10,28,8/3.0))

    for i in range(len(t)):
      if(i == 0):
        x[:,i] = x_0
      else:
        x[:,i] = step_lorenz(x[:,i-1],b,dt)

    ax = plt.figure().add_subplot(projection='3d')
    ax.plot(x[0,:], x[1,:], x[2,:], lw=0.5)
    ax.set_xlabel("X Axis")
    ax.set_ylabel("Y Axis")
    ax.set_zlabel("Z Axis")
    ax.set_title("Lorenz Attractor")
    plt.show()


import numpy as np

def getSSM(X):
    """
    Compute a Euclidean self-similarity image between a set of points
    :param X: An Nxd matrix holding the d coordinates of N points
    :return: An NxN self-similarity matrix
    """
    D = np.sum(X**2, 1)[:, None]
    D = D + D.T - 2*X.dot(X.T)
    D[D < 0] = 0
    D = 0.5*(D + D.T)
    D = np.sqrt(D)
    return D

def getW(D, K, Mu = 0.5):
    """
    Return affinity matrix
    [1] Wang, Bo, et al. "Similarity network fusion for aggregating data types on a genomic scale."
        Nature methods 11.3 (2014): 333-337.
    :param D: Self-similarity matrix
    :param K: Number of nearest neighbors
    """
    #W(i, j) = exp(-Dij^2/(mu*epsij))
    DSym = 0.5*(D + D.T)
    np.fill_diagonal(DSym, 0)

    Neighbs = np.partition(DSym, K+1, 1)[:, 0:K+1]
    MeanDist = np.mean(Neighbs, 1)*float(K+1)/float(K) #Need this scaling
    #to exclude diagonal element in mean
    #Equation 1 in SNF paper [1] for estimating local neighborhood radii
    #by looking at k nearest neighbors, not including point itself
    Eps = MeanDist[:, None] + MeanDist[None, :] + DSym
    Eps = Eps/3
    W = np.exp(-DSym**2/(2*(Mu*Eps)**2))
    return W

def getCSM(X, Y):
    """
    Return the Euclidean cross-similarity matrix between the M points
    in the Mxd matrix X and the N points in the Nxd matrix Y.
    :param X: An Mxd matrix holding the coordinates of M points
    :param Y: An Nxd matrix holding the coordinates of N points
    :return D: An MxN Euclidean cross-similarity matrix
    """
    C = np.sum(X**2, 1)[:, None] + np.sum(Y**2, 1)[None, :] - 2*X.dot(Y.T)
    C[C < 0] = 0
    return np.sqrt(C)

def getGreedyPerm(X, M, Verbose = False):
    """
    Purpose: Naive O(NM) algorithm to do the greedy permutation
    :param X: Nxd array of Euclidean points
    :param M: Number of points in returned permutation
    :returns: (permutation (N-length array of indices), \
            lambdas (N-length array of insertion radii))
    """
    #By default, takes the first point in the list to be the
    #first point in the permutation, but could be random
    perm = np.zeros(M, dtype=np.int64)
    lambdas = np.zeros(M)
    ds = getCSM(X[0, :][None, :], X).flatten()
    for i in range(1, M):
        idx = np.argmax(ds)
        perm[i] = idx
        lambdas[i] = ds[idx]
        ds = np.minimum(ds, getCSM(X[idx, :][None, :], X).flatten())
        if Verbose:
            interval = int(0.05*M)
            if i%interval == 0:
                print("Greedy perm %i%s done..."%(int(100.0*i/float(M)), "%"))
    Y = X[perm, :]
    return {'Y':Y, 'perm':perm, 'lambdas':lambdas}


import numpy as np
from scipy.stats import boxcox,boxcox_normmax

def BoxCox(x,lamda,shift=None):
	if shift is None:
		xs,shift = ShiftX(x)
	else:
		xs = x + shift
	if lamda == 0:
		xt = np.log(xs)
	else:
		xt = (xs**lamda - 1)/lamda
	return xt

def BoxCoxLogLikelihood(x,lamda):
	n = np.size(x)
	xl = BoxCox(x,lamda)
	xlbar = np.mean(xl)

	fxl = -(n/2)*np.log(np.sum((xl - xlbar)**2)/n) + (lamda-1)*np.sum(np.log(x))

	return fxl

def MaximizeBoxCoxLikelihood(x,LamdaRange=[-4.0,4.0],Tol=0.001):
	#import matplotlib.pyplot as plt

	LR = np.array(LamdaRange)
	dL = 1.0
	Repeat = True
	out = 1.0
	while Repeat:
		nL = np.int32((LR[1]-LR[0])/dL)
		L = np.arange(nL)*dL + LR[0]
		ll = np.zeros(nL,dtype='float64')
		for j in range(0,nL):
			ll[j] = BoxCoxLogLikelihood(x,L[j])
		#print(ll)
		bad = np.where(np.isfinite(ll) == False)[0]
		ll[bad] = np.nan
		good = np.where(np.isfinite(ll))[0]
		if good.size == 0:
			return np.nan
		best = np.where(ll == np.nanmax(ll))[0]
		#print(best,ll[best[0]])
		if best[0] == 0:
			best[0] = 1
		if best[0] == nL-1:
			best[0] -= 1
		if dL < Tol:
			out = L[best[0]]
			Repeat = False
		#plt.plot(L,ll)
		LR = np.array([L[best[0]-1],L[best[0]+1]])
		dL = (LR[1]-LR[0])/nL
	return out

def ShiftX(xin):
	x = xin[np.isfinite(xin)]
	shift = 0.0
	if (x < 0).any():
		newx = x - np.min(x)
		shift += -np.min(x)
	else:
		newx = x
	if (newx == 0).any():
		gt0 = np.where(newx > 0)[0]
		if gt0.size == 0:
			return x,np.nan
		minx = np.min(newx[newx > 0])/2.0
		newx += minx
		shift += minx
	return newx,shift

def FitBoxCox(x):

	#first of all - shift the data so that there are no negatives!
	xs,shift = ShiftX(x)
	if np.isnan(shift):
		return x,1.0,1.0
	#now use scipy.stats.boxcox to fit function
	#lamda = boxcox_normmax(xs,brack=(-2.0,1.9),method='all').min()
	lamda = MaximizeBoxCoxLikelihood(xs)
	xt = boxcox(xs,lamda)

	return xt,lamda,shift

def ReverseBoxCox(xt,lamda,shift):
	if lamda == 0:
		xs = np.exp(xt)
	else:
		xs = (xt*lamda + 1)**(1/lamda)

	x = xs-shift
	return x

def BoxCoxNormalize(x,nSigma=1.0,NotFin='ignore'):
	#copy input array
	X = np.copy(x.astype('float64'))

	#deal with bad data - NotFin to value to replace infs and NaNs with
	if NotFin is 'ignore':
		good = np.where(np.isfinite(X))[0]
		Xg = X[good]
	else:
		bad = np.where(np.isfinite(X) == False)[0]
		X[bad] = NotFin
		Xg = X

	if Xg.size == 0:
		return X,0.0,1.0,1.0,1.0

	#perform BC transform
	Xbc,lamda,shift = FitBoxCox(Xg)

	Xbc[np.isfinite(Xbc) == False] = np.nan

	#calculate mean and std
	mu = np.nanmean(Xbc)
	nsig = np.nanstd(Xbc)*nSigma

	#rescale data
	Xs = (Xbc - mu)/(nsig)

	#Add bad data back in if necessary
	if NotFin is 'ignore':
		Xout = np.zeros(X.size,dtype='float64')+np.nan
		Xout[good] = Xs
	else:
		Xout = Xs
	return Xout,mu,nsig,lamda,shift

def BoxCoxUnNormalize(x,mu,nsig,lamda,shift):
	Xs = np.copy(x)
	Xbc = Xs*nsig + mu
	Xout = ReverseBoxCox(Xbc,lamda,shift)

	return Xout


import numpy as np

X = np.array([1, 2, 3, 4, 5, 6, 7])
Y = np.array([2, 3, 4, 5, 6, 7, 8])
w_gradient = 0
b_gradient = 0
w, b = 0.5, 0.5

learning_rate = .01
loss = 0
EPOCHS = 2000
N = len(Y)

for i in range(EPOCHS):

    # Predict
    Y_pred = (w * X) + b

    # Loss
    loss = np.square(Y_pred - Y).sum() / (2.0 * N)
    if i % 100 == 0:
        print(loss)

    # Backprop
    grad_y_pred = (2 / N) * (Y_pred - Y)
    w_gradient = (grad_y_pred * X).sum()
    b_gradient = (grad_y_pred).sum()

    # Optimize
    w -= (w_gradient * learning_rate)
    b -= (b_gradient * learning_rate)

print("\n\n")
print("LEARNED:")
print(w, b)
print("\n")
print("TEST:")
print(np.round(b + w * (-2)))
print(np.round(b + w * 0))
print(np.round(b + w * 1))
print(np.round(b + w * 6))
print(np.round(b + w * 3000))

# Expected: 30001, but gives 30002.
# Training for 3000 epochs will give expected result.
# For simple demo with less training data and small input range 2000 in enough
print(np.round(b + w * 30000))


import pandas as pd
import numpy as np
import math
import pdb

def average(series):
    """
    implements the average of a pandas series from scratch
    suggested functions:
    len(list)
    sum(list)
    you should get the same result as calling .mean() on your series
    https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.Series.mean.html
    See numpy documenation for implementation details:
    https://docs.scipy.org/doc/numpy/reference/generated/numpy.mean.html
    """
    pass
    avgnum = sum(series) / len(series)
    return avgnum

def standard_deviation(series):
    """
    implements the sample standard deviation of a series from scratch
    you may need a for loop and your average function
    also the function math.sqrt
    you should get the same result as calling .std() on your data
    https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.Series.std.html
    See numpy documenation for implementation details:
    https://docs.scipy.org/doc/numpy/reference/generated/numpy.std.html
    """
    pass
    avgnum = sum(series) / len(series)
    sumsq = 0
    for each in series:
        t = each - avgnum
        sumsq = sumsq + pow(t,2)
    stdnum = math.sqrt(sumsq/(len(series)-1))
    return stdnum

def median(series):
    """
    finds the median of the series from scratch
    you may need to sort your values and use
    modular division
    this number should be the same as calling .median() on your data
    See numpy documenation for implementation details:
    https://docs.scipy.org/doc/numpy/reference/generated/numpy.median.html
    https://pandas.pydata.org/pandas-docs/version/0.23.0/generated/pandas.Series.median.html
    """
    pass
    n = len(series)
    for j in range(n - 1):
        count = 0
        for i in range(0, n - 1 - j):
            if series[i] > series[i + 1]:
                series[i], series[i + 1] = series[i + 1], series[i]
                count = count + 1
        if 0 == count:
            break

    if int(n/2) == n/2:
        mednum = (series[n//2-1]+series[n//2])/2
    else:
        mednum = series[n//2]

    return mednum


import numpy as np
import util

def main(train_path, valid_path, save_path, plot_path):
    """Problem: Logistic regression with Newton's Method.

    Args:
        train_path: Path to CSV file containing dataset for training.
        valid_path: Path to CSV file containing dataset for validation.
        save_path: Path to save predicted probabilities using np.savetxt().
        plot_path: Path to save plots.
    """
    x_train, y_train = util.load_dataset(train_path, add_intercept=True)
    x_valid, y_valid = util.load_dataset(valid_path, add_intercept=True)
    clf = LogisticRegression()
    clf.fit(x_train, y_train)
    util.plot(x_valid, y_valid, clf.theta, plot_path)
    np.savetxt(save_path, clf.predict(x_valid))

class LogisticRegression:
    """Logistic regression with Newton's Method as the solver.

    Example usage:
        > clf = LogisticRegression()
        > clf.fit(x_train, y_train)
        > clf.predict(x_eval)
    """
    def __init__(self, step_size=0.01, max_iter=1000000, eps=1e-5,
                 theta_0=None, verbose=True):
        """
        Args:
            step_size: Step size for iterative solvers only.
            max_iter: Maximum number of iterations for the solver.
            eps: Threshold for determining convergence.
            theta_0: Initial guess for theta. If None, use the zero vector.
            verbose: Print loss values during training.
        """
        self.theta = theta_0
        self.step_size = step_size
        self.max_iter = max_iter
        self.eps = eps
        self.verbose = verbose

    def fit(self, x, y):
        """Run Newton's Method to minimize J(theta) for logistic regression.

        Args:
            x: Training example inputs. Shape (n_examples, dim).
            y: Training example labels. Shape (n_examples,).
        """
        self.theta = np.zeros(x.shape[1])
        while True:
            y_pred = self.predict(x)
            grad = ((y_pred - y) * x.T).mean(axis=1)
            hess = ((y_pred * (1 - y_pred)) * x.T) @ x / x.shape[1]
            diff = grad @ np.linalg.inv(hess.T)
            self.theta = self.theta - diff
            if np.abs(diff).sum() < self.eps:
                return self

    def predict(self, x):
        """Return predicted probabilities given new inputs x.

        Args:
            x: Inputs of shape (n_examples, dim).

        Returns:
            Outputs of shape (n_examples,).
        """
        z = self.theta @ x.T
        return 1 / (1 + np.exp(-z))

if __name__ == '__main__':
    main(train_path='ds1_train.csv',
         valid_path='ds1_valid.csv',
         save_path='logreg_pred_1.txt',
         plot_path='logreg_1.jpg')

    main(train_path='ds2_train.csv',
         valid_path='ds2_valid.csv',
         save_path='logreg_pred_2.txt',
         plot_path='logreg_2.jpg')


""" Perform forward stepwise feature selection to narrow down predictors in linear regression
 Use k-fold cross-validation method and Root Mean Squared Error (RMSE) for evaluating models.
"""
import copy
import logging
from matplotlib import pyplot as plt
import numpy as np
import pandas as pd
from sklearn import datasets
from statsmodels.formula.api import ols

def forward_select(df: pd.DataFrame, response: str, num_folds=5, test_data_frac=0.25) -> list:
    """ Forward selection for linear regression
    :param df: pandas dataframe with all candidate predictors and the response variable
    :param response: response/independent variable
    :param num_folds: number of folds for dividing data to cross validation and train sets
    :param test_data_frac: fraction of input dataframe kept for final testing
    :return: an statsmoodel regression result with optimal predictors.
    Predictors are selected by forward stepwise selection method and evaluated by Root Mean Squared Error
    """
    remaining_predictors = set(df.columns)
    remaining_predictors.remove(response)
    selected_predictors = []
    test_data = df.sample(frac=test_data_frac)
    train_cv = df.drop(test_data.index)
    folds = kfold(train_cv, num_folds)
    best_predictors = []
    while remaining_predictors:
        rmses_candidates = []
        for candidate in remaining_predictors:
            formula = "{} ~ {} + 1".format(response, ' + '.join(selected_predictors + [candidate]))
            rmse = calculate_average_error(folds, formula, response)
            rmses_candidates.append((rmse, candidate))
        rmses_candidates.sort(key=lambda pair: -pair[0])
        cv_rmse, best_candidate = rmses_candidates.pop()
        selected_predictors = selected_predictors + [best_candidate]
        remaining_predictors.remove(best_candidate)
        formula = "{} ~ {} + 1".format(response, ' + '.join(selected_predictors))
        regression = ols(formula, train_cv).fit()
        test_rmse = calculate_rmse(regression, test_data, response)
        best_predictors.append((selected_predictors, cv_rmse, test_rmse))
    return best_predictors

def kfold(df: pd.DataFrame, k: int) -> dict:
    """ Form a dictionary of k folds. Each dictionary item is a dataframe
    :param df: input dataframe
    :param k: number of folds
    :return: dictionary of folds.
    """
    folds = {}
    fold_size = int(len(df)/k)
    df_copy = copy.deepcopy(df)
    for i in range(k - 1):
        folds[i] = df_copy.sample(n=fold_size)
        df_copy.drop(folds[i].index, inplace=True)
    folds[k - 1] = df_copy
    assert len(pd.concat([fold for fold in folds.values()], axis=0)) == len(df)
    return folds

def calculate_average_error(
        folds: dict,
        formula: str,
        response: str,
) -> np.ndarray:
    """ Calculate average RMSE of cross validation folds
    :param folds: dictionary of folds
    :param formula: regression formula
    :param response: response variable
    :return: average RMSE of folds
    """
    num_folds = len(folds)
    rmse_list = []
    for k in range(num_folds):
        regression = ols(formula, folds[k]).fit()
        cv_data = pd.concat([fold_df for (fold, fold_df) in folds.items() if fold != k], axis=0)
        rmse_list.append(calculate_rmse(regression, cv_data, response))

    avg_rmse = np.mean(rmse_list)
    return avg_rmse

def calculate_rmse(regression, data: pd.DataFrame, response: str) -> float:
    """ Calculate RMSE of the given regression on input data
    :param regression: regression results from running statsmodel ols
    :param data: dataframe
    :param response: response variable
    :return:
    """
    predicted = regression.predict(data)
    actual = data[f"{response}"]
    error = actual - predicted
    rss = np.square(error.values)
    rmse = np.sqrt(np.sum(rss)/len(error))
    return rmse

if __name__ == '__main__':
    data = datasets.load_diabetes()
    df = pd.DataFrame(data.data, columns=data.feature_names)
    response = "y_var"
    df[response]=data.target
    logging.debug(f"The dataframe: \n {df.head()}")

    # Run forward stepwise selection
    best_predictors = forward_select(df, response, num_folds=5, test_data_frac=0.2)
    best_predictors.sort(key=lambda formula_errors: formula_errors[1])
    best_formula, cv_error, test_errors = best_predictors[0]
    logging.debug(f"Best model (in terms of test RMSE): {best_formula} \n"
                 f" with test RMSE of {test_errors}"
                 )
    logging.debug(f"All models and their cv errors: \n {best_predictors}")

    # Plot the results
    best_predictors.sort(key=lambda formula_error: len(formula_error[0]))
    plt.figure()
    plt.xlabel("Number of predictors")
    plt.ylabel("RMSE")
    x = list(range(1, len(best_predictors) + 1))
    cv_errors = [formula_error[1] for formula_error in best_predictors]
    plt.xticks(ticks=x)
    plt.plot(x, cv_errors, "-rx", label="CV data")

    test_errors = [formula_error[2] for formula_error in best_predictors]
    plt.gcf()
    plt.plot(x, test_errors, "-bo", label="test data")
    plt.legend()
    plt.show()


import cv2
import numpy as np

'''
Numpy has an FFT package to do this.
np.fft.fft2() provides us the frequency transform which will be a complex array.
Its first argument is the input image, which is grayscale.
Second argument is optional which decides the size of output array.
If it is greater than size of input image, input image is padded with zeros before calculation of FFT.
If it is less than input image, input image will be cropped.
If no arguments passed, Output array size will be same as input.

Now once you got the result, zero frequency component (DC component) will be at top left corner.
If you want to bring it to center, you need to shift the result by \frac{N}{2} in both the directions.
This is simply done by the function, np.fft.fftshift().
(It is more easier to analyze).
Once you found the frequency transform, you can find the magnitude spectrum.
'''

def fourier_numpy(image):
    f = np.fft.fft2(image)
    fshift = np.fft.fftshift(f)
    return 20 * np.log(np.abs(fshift))

'''
OpenCV provides the functions cv2.dft() and cv2.idft() for this.
It returns the same result as previous, but with two channels.
First channel will have the real part of the result and second channel will have the imaginary part of the result.
The input image should be converted to np.float32 first.
'''

def fourier_open_cv(image):
    dft = cv2.dft(np.float32(image), flags=cv2.DFT_COMPLEX_OUTPUT)
    dft_shift = np.fft.fftshift(dft)
    return 20 * np.log(cv2.magnitude(dft_shift[:, :, 0], dft_shift[:, :, 1]))


# reference to Tensorflow's implement
import numpy as np

def sigmoid_cross_entropy_loss(labels, logits):
    r"""used when objects is not mutually exclusive. For instance,
    a picture can have a dog and a cat at the same time

    for brevity, let `x = logits`, `z = labels`. The logistic loss is

        z * -log(sigmoid(x)) + (1 - z) * -log(1 - sigmoid(x))

    equivalent formulation:

        max(x, 0)  -  x * z + log(1 + exp(-abs(x)))
    """
    zeros = np.zeros_like(logits, dtype=logits.dtype)
    cond = (logits >= zeros)
    relu_logits = np.where(cond, logits, zeros)             # max(x, 0)
    neg_abs_logits = np.where(cond, -logits, logits)        # -abs(x)
    return relu_logits - logits * labels + np.log1p(np.exp(neg_abs_logits))

# real tensorflow implement
input = np.array([[.1, .2, .3, .4],
                  [.3, .5, .7, .8],
                  [.9, .3, .8, .1]])
target = np.array([[.1, .0, .2, .1],
                   [.3, .4, .6, .7],
                   [.0, .1, .9, .2]])
import tensorflow as tf
with tf.Session() as sess:
    loss = tf.nn.sigmoid_cross_entropy_with_logits(labels=target, logits=input)
    loss1 = sess.run(loss)

loss2 = sigmoid_cross_entropy_loss(logits=input, labels=target)

print("loss compute by Tensorflow: \n", loss1)
print("loss compute by Mory: \n", loss2)

# test in pytorch
import torch
input = torch.from_numpy(input)
target = torch.from_numpy(target)
loss_fn = torch.nn.BCEWithLogitsLoss(reduce=False)
loss3 = loss_fn(input, target)
print(loss3)


from sympy import *
import math

def single_point_truncation(f, x_0, x_1, eps=0, eta=0, max_steps=20, verbose=False):
    """单点弦截法

    Args:
        f: function, 迭代函数
        x_0, x_1: float, 初值
        eps: float, 根的容许误差, default 0.
        eta: float, abs(f(x)) 的容许误差, default 0.

        max_steps: int, 最大迭代次数, default 20.
        verbose: bool, 打印出每一步的值, default False.

    Returns:
        x_final: float, 最终的近似根 x
    """

    f_x0 = f(x_0)

    x = x_1
    for i in range(max_steps):
        if verbose:
            print(i, x)

        x_next = x - f(x) / (f(x) - f_x0) * (x - x_0)

        if abs(x_next - x) <= eps or abs(f(x)) <= eta:
            break

        x = x_next

    return x


# Computational Linear Algebra Ep#2 ex1
# By: Nick Space Cowboy

import numpy as np
import time

if __name__ == "__main__":
	print("A:")
	# Generates a random 10x10  upper triangular matrix with values between zero to 100
	A = np.array(np.random.randint (0,100,(10,10)))
	A1 = np.triu(A)
	print(A)
	print("A1:")
	print(A1)
	print("")
	b = np.array(np.random.randint(0, 100, (10,1)))
	print("b: ")
	print(b)
	print("")
	s1 = time.time()
	# Solve the normal A matrix system
	x = np.linalg.solve(A,b)
	s1e = time.time()
	print("x:")
	print(x)
	print("Full A matrix time: " + str(s1e - s1))
	s2 = time.time()
	# Solve the Triangular matrix system
	x1 = np.linalg.solve(A1,b)
	s2e = time.time()
	print("x1:")
	print(x1)
	print("Triangle matrix time: " + str(s2e - s2))


"""
  Name     : c12_14_stock_price_movement.py
  Book     : Python for Finance (2nd ed.)
  Publisher: Packt Publishing Ltd.
  Author   : Yuxing Yan
  Date     : 6/6/2017
  email    : yany@canisius.edu
             paulyxy@hotmail.com
"""
import scipy as sp
import matplotlib.pyplot as plt
# input area
stock_price_today = 9.15 # stock price at time zero
T =1.                    # maturity date (in years)
n_steps=100.             # number of steps
mu =0.15                 # expected annual return
sigma = 0.2              # annualized volatility
sp.random.seed(12345)    # fixed our seed
n_simulation = 5         # number of simulations
dt =T/n_steps
#
S = sp.zeros([n_steps], dtype=float)
x = range(0, int(n_steps), 1)
for j in range(0, n_simulation):
    S[0]= stock_price_today
    for i in x[:-1]:
        e=sp.random.normal()
        S[i+1]=S[i]+S[i]*(mu-0.5*pow(sigma,2))*dt+sigma*S[i]*sp.sqrt(dt)*e;
    plt.plot(x, S)
#
plt.figtext(0.2,0.8,'S0='+str(S[0])+',mu='+str(mu)+',sigma='+str(sigma))
plt.figtext(0.2,0.76,'T='+str(T)+', steps='+str(int(n_steps)))
plt.title('Stock price (number of simulations = %d ' % n_simulation +')')
plt.xlabel('Total number of steps ='+str(int(n_steps)))
plt.ylabel('stock price')
plt.show()


"""
Funzioni per il calcolo degli zeri di funzioni non lineari
"""
import numpy as np
import math

'''
Il core Python non possiede la funzione sign.
La funzione copysign(a,b)  del modulo math restituisce un valore numerico che ha il valore assoluto di
a e segno di b.
Per avere il segno di un valore numerico b si può usare math.copysign(1,b)
che resistuisce 1 se b>0, -1 se b<0, 0 se b è zero
'''

def sign(x): return math.copysign(1, x)

#Bisezione
def bisez(fname,a,b,tol):
    eps=np.spacing(1)      # np.spacing(x) Restituisce la distanza tra x e il numero adiacente più vicino.
                           # np.spacing(1)  restituisce quindi l' eps di macchina.
    fa=fname(a)
    fb=fname(b)
    if sign(fa)==sign(fb):
       print('intervallo non corretto --Metodo non applicabile')
       return [],0,[]
    else:
        maxit=int(math.ceil(math.log((b-a)/tol)/math.log(2)))
        print('n. di passi necessari=',maxit,'\n');
        xk=[]
        it=0
        while it<maxit and  abs(b-a)>=tol+eps*max(abs(a),abs(b)):
            c=a+(b-a)*0.5   #formula stabile per il calcolo del punto medio dell'intervallo
            xk.append(c)
            fxk=fname(c)
            if fxk==0:
                break
            elif sign(fxk)==sign(fa):
                a=c
                fa=fxk
            elif sign(fxk)==sign(fb):
                b=c
                fb=fxk

            it+=1

        x=c

    return x,it,xk

def regula_falsi(fname,a,b,tol,nmax):
#Regula Falsi
        eps=np.spacing(1)
        xk=[]
        fa=fname(a)
        fb=fname(b)
        if sign(fa)==sign(fb):
          print('intervallo non corretto --Metodo non applicabile')
          return [],0,[]
        else:
            it=0
            fxk=fname(a)
            while it<nmax and  abs(b-a)>=tol+eps*max(abs(a),abs(b)) and abs(fxk)>=tol :
                x1=a-fa*(b-a)/(fb-fa);
                xk.append(x1)
                fxk=fname(x1);
                if fxk==0:
                    break
                elif sign(fxk)==sign(fa):
                    a=x1;
                    fa=fxk;
                elif sign(fxk)==sign(fb):
                    b=x1;
                    fb=fxk;
                it+=1

            if it==nmax :
                print('Regula Falsi: Raggiunto numero max di iterazioni')

        return x1,it,xk

def corde(fname,fpname,x0,tolx,tolf,nmax):
 #Corde
        xk=[]
        m=fpname(x0)               #m= Coefficiente angolare della tangente in x0
        fx0=fname(x0)
        d=fx0/m
        x1=x0-d
        fx1=fname(x1)
        xk.append(x1)
        it=1
        while it<nmax and  abs(fx1)>=tolf and abs(d)>=tolx*abs(x1) :
           x0=x1
           fx0=fname(x0)
           d=fx0/m
           '''
           #x1= ascissa del punto di intersezione tra  la retta che passa per il punto
           (xi,f(xi)) e ha pendenza uguale a m  e l'asse x
           '''
           x1=x0-d
           fx1=fname(x1)
           it=it+1
           xk.append(x1)

        if it==nmax:
            print('raggiunto massimo numero di iterazioni \n')

        return x1,it,xk

#Secanti
def secanti(fname,xm1,x0,tolx,tolf,nmax):
        xk=[]
        fxm1=fname(xm1);
        fx0=fname(x0);
        d=fx0*(x0-xm1)/(fx0-fxm1)
        x1=x0-d;
        xk.append(x1)
        fx1=fname(x1);
        it=1

        while it<nmax and abs(fx1)>=tolf and abs(d)>=tolx*abs(x1):
            xm1=x0
            x0=x1
            fxm1=fname(xm1)
            fx0=fname(x0)
            d=fx0*(x0-xm1)/(fx0-fxm1)
            x1=x0-d
            fx1=fname(x1)
            xk.append(x1);
            it=it+1;

        if it==nmax:
           print('Secanti: raggiunto massimo numero di iterazioni \n')

        return x1,it,xk

def newton(fname,fpname,x0,tolx,tolf,nmax):
#Newton
        xk=[]
        fx0=fname(x0)
        dfx0=fpname(x0)
        if abs(dfx0)>np.spacing(1):
            d=fx0/dfx0
            x1=x0-d
            fx1=fname(x1)
            xk.append(x1)
            it=0

        else:
            print('Newton:  Derivata nulla in x0 - EXIT \n')
            return [],0,[]

        it=1
        while it<nmax and abs(fx1)>=tolf and  abs(d)>=tolx*abs(x1):
            x0=x1
            fx0=fname(x0)
            dfx0=fpname(x0)
            if abs(dfx0)>np.spacing(1):
                d=fx0/dfx0
                x1=x0-d
                fx1=fname(x1)
                xk.append(x1)
                it=it+1
            else:
                 print('Newton: Derivata nulla in x0 - EXIT \n')
                 return x1,it,xk

        if it==nmax:
            print('Newton: raggiunto massimo numero di iterazioni \n');

        return x1,it,xk

def stima_ordine(xk,iterazioni):
      p=[]

      for k in range(iterazioni-3):
         p.append(np.log(abs(xk[k+2]-xk[k+3])/abs(xk[k+1]-xk[k+2]))/np.log(abs(xk[k+1]-xk[k+2])/abs(xk[k]-xk[k+1])));

      ordine=p[-1]
      return ordine

#Newton Modificato
def newton_m(fname,fpname,x0,m,tolx,tolf,nmax):
        eps=np.spacing(1)
        xk=[]
        #xk.append(x0)
        fx0=fname(x0)
        dfx0=fpname(x0)
        if abs(dfx0)>eps:
            d=fx0/dfx0
            x1=x0-m*d
            fx1=fname(x1)
            xk.append(x1)
            it=0

        else:
            print('Newton:  Derivata nulla in x0  \n')
            return [],0,[]

        it=1
        while it<nmax and abs(fx1)>=tolf and  abs(d)>=tolx*abs(x1):
            x0=x1
            fx0=fname(x0)
            dfx0=fpname(x0)
            if abs(dfx0)>eps:
                d=fx0/dfx0
                x1=x0-m*d
                fx1=fname(x1)
                xk.append(x1)
                it=it+1
            else:
                 print('Newton Mod: Derivata nulla   \n')
                 return x1,it,xk

        if it==nmax:
            print('Newton Mod: raggiunto massimo numero di iterazioni \n');

        return x1,it,xk

def iterazione(gname,x0,tolx,nmax):

        xk=[]
        xk.append(x0)
        x1=gname(x0)
        d=x1-x0
        xk.append(x1)
        it=1
        while it<nmax and  abs(d)>=tolx*abs(x1) :
            x0=x1
            x1=gname(x0)
            d=x1-x0
            it=it+1
            xk.append(x1)

        if it==nmax:
            print('Raggiunto numero max di iterazioni \n')

        return x1, it,xk


from cvxpy import *
import numpy as np
from posdef import *

mu = np.matrix([[-0.01132331],[-0.05061149],[-0.00729714],[-0.06206554],[-0.01079347],[0.0025]])
Sigma = np.array(
	[[2.52879395e-03, 1.85425637e-03, 9.20436767e-04, 1.95914715e-03, 4.28396944e-03, 0.00000000e+00],
	 [1.85425637e-03, 2.15246031e-03, 9.42088127e-04, 1.58969365e-03, 3.30104182e-03, 0.00000000e+00],
	 [  9.20436767e-04, 9.42088127e-04, 2.17054891e-03, 7.91879289e-04, 1.39817285e-03, 0.00000000e+00],
	 [  1.95914715e-03, 1.58969365e-03, 7.91879289e-04, 1.67831424e-03, 3.77524505e-03, 0.00000000e+00],
	 [  4.28396944e-03, 3.30104182e-03, 1.39817285e-03, 3.77524505e-03, 8.72500053e-03, 0.00000000e+00],
	 [  0.00000000e+00, 0.00000000e+00, 0.00000000e+00, 0.00000000e+00, 0.00000000e+00, 1.73472348e-18]])

print(isPD(Sigma))

min_return = 0.0
n = 6

w = Variable(n)
ret = mu.T*	w
risk = quad_form(w, Sigma)
min_ret = Parameter(sign='positive')
min_ret.value = min_return

prob = Problem(Minimize(risk),
               [ret >= min_ret,
               sum_entries(w) == 1,
               w >= 0])
prob.solve()
print(w.value)


""""
README
======
This file contains Python codes.
======
"""

""" An incremental search algorithm """
import numpy as np

def incremental_search(f, a, b, dx):
    """
    :param f: The function to solve
    :param a: The left boundary x-axis value
    :param b: The right boundary x-axis value
    :param dx: The incremental value in searching
    :return: The x-axis value of the root,
                number of iterations used
    """
    fa = f(a)
    c = a + dx
    fc = f(c)
    n = 1

    while np.sign(fa) == np.sign(fc):
        if a >= b:
            return a - dx, n

        a = c
        fa = fc
        c = a + dx
        fc = f(c)
        n += 1

    if fa == 0:
        return a, n
    elif fc == 0:
        return c, n
    else:
        return (a + c)/2., n

if __name__ == "__main__":
    """
    The keyword 'lambda' creates an anonymous function
    with input argument x
    """
    y = lambda x: x**3 + 2.0*x**2 - 5.
    root, iterations = incremental_search(y, -5., 5., 0.001)
    print "Root is:", root
    print "Iterations:", iterations


from scipy.io.wavfile import read
from scipy.signal import convolve
import numpy as np
import matplotlib.pyplot as plt
import time

# Answer to the question len(x) = 200, len(h) = 100. What is len(y)?
# Length of y would be: len(x) + len(h) - 1 = 299.
def myTimeConv(x,h):

    y = np.zeros(len(x) + len(h) - 1)

    padded_x = np.hstack((np.zeros(len(h)-1), x, np.zeros(len(h)-1)))
    reverse_h = np.flip(h)

    for i in range(len(y)):
        y[i] = np.dot(padded_x[i:i+len(h)], reverse_h)

    return y

def CompareConv(x, h):

    timer = np.zeros(2)

    current_time = time.time()
    my_conv = myTimeConv(x, h)
    timer[0] = time.time() - current_time

    current_time = time.time()
    scipy_conv = convolve(x, h)
    timer[1] = time.time() - current_time

    m = np.mean(my_conv - scipy_conv)
    mabs = np.mean(np.abs(my_conv - scipy_conv))
    stdev = np.std(my_conv - scipy_conv)

    return m, mabs, stdev, timer

def readwav(filename):
    fs, x = read(filename)
    if x.dtype == 'float32':
        audio = x
    else:
        # change range to [-1,1)
        if x.dtype == 'uint8':
            nbits = 8
        elif x.dtype == 'int16':
            nbits = 16
        elif x.dtype == 'int32':
            nbits = 32
        else:
            nbits = 1  # No conversion

        audio = x / float(2 ** (nbits - 1))

    # special case of unsigned format
    if x.dtype == 'uint8':
        audio = audio - 1.

    if audio.ndim > 1:
        audio = audio[:, 0]

    return audio

if __name__ == "__main__":

    x = np.ones(200)

    rampup = np.linspace(0,1, 26)
    rampdown = np.linspace(1,0,26)
    h = np.hstack((rampup[:-1],rampdown))

    y_time = myTimeConv(x, h)

    piano = readwav("piano.wav")
    impulse_response = readwav("impulse-response.wav")

    compConv = CompareConv(piano, impulse_response)

    fo = open("./results/02-convolutionStats.txt", "w+")

    fo.writelines("Mean: " + str(compConv[0]) + '\n')
    fo.writelines("Mean Absolute: " + str(compConv[1]) + '\n')
    fo.writelines("Standard Deviation: " + str(compConv[2]) + '\n')
    fo.writelines("Time of my convolution: " + str(compConv[3][0]) + '\n')
    fo.writelines("Time of Scipy convolution: " + str(compConv[3][1]) + '\n')

    time = np.linspace(0, len(y_time)-1, len(y_time))

    plt.plot(time, y_time)
    plt.xlabel("Time (In Samples)")
    plt.ylabel("Signal Amplitude")
    plt.title("Time Domain Convolution of two signals")
    plt.show()


from typing import Callable, List, Optional
from lab1.src.onedim.one_dim_search import dichotomy_method

import numpy as np

DEFAULT_EPS = 1e-6
DEFAULT_MAX_ITERS = 100

def conjugate_direction_method(q: np.ndarray,
                               b: np.ndarray,
                               start: np.ndarray,
                               max_iters=DEFAULT_MAX_ITERS,
                               eps: float = DEFAULT_EPS,
                               trajectory: Optional[List] = None):
    if np.all(b == 0):
        return b, 0

    if trajectory is not None:
        trajectory.append(start)
    w_prev = - q @ start - b
    u_prev = w_prev
    if np.linalg.norm(u_prev) == 0:
        return start, 0
    h_prev = np.dot(u_prev, u_prev) / np.dot(q @ u_prev, u_prev)
    x_prev = start + h_prev * u_prev
    if trajectory is not None:
        trajectory.append(x_prev)

    k = 1
    while k < max_iters:
        w_k = -q @ x_prev - b
        u_k = w_k - (np.dot(q @ u_prev, w_k) / np.dot(q @ u_prev, u_prev)) * u_prev
        if np.linalg.norm(u_k) < eps:
            return x_prev, k
        h_k = np.dot(w_k, u_k) / np.dot(q @ u_k, u_k)
        x_k = x_prev + h_k * u_k

        u_prev = u_k
        x_prev = x_k
        k += 1
        if trajectory is not None:
            trajectory.append(x_prev)

    return x_prev, k

def conjugate_gradient_method(f: Callable[[np.ndarray], float],
                              f_grad: Callable[[np.ndarray], np.ndarray],
                              start: np.ndarray,
                              eps=DEFAULT_EPS,
                              max_iters=DEFAULT_MAX_ITERS,
                              trajectory: Optional[List] = None):
    if trajectory is not None:
        trajectory.append(start)

    w_prev = (-1) * f_grad(start)
    u_prev = w_prev
    if np.linalg.norm(w_prev) < eps:
        return start, 0
    x_prev = start
    k = 1

    while k < max_iters:
        w_k = (-1) * f_grad(x_prev)
        y_k = max(0, np.dot(w_k - w_prev, w_k) / np.dot(w_prev, w_prev))
        u_k = w_k + y_k * u_prev
        h_k, _, _ = dichotomy_method(lambda h: f(x_prev + h * u_k), 0, 50, 1e-8)
        x_k = x_prev + h_k * u_k

        if np.linalg.norm(w_k) < eps or np.linalg.norm(x_k - x_prev) < eps:
            return x_prev, k

        x_prev = x_k
        u_prev = u_k
        w_prev = w_k
        k += 1

        if trajectory is not None:
            trajectory.append(x_prev)

    return x_prev, k


"""
Demonstration module for quadratic interpolation.

Sample solutions for Homework 2 problems #2 through #7.
"""

import numpy as np
import matplotlib.pyplot as plt
from numpy.linalg import solve

def quad_interp(xi,yi):
    """
    Quadratic interpolation.  Compute the coefficients of the polynomial
    interpolating the points (xi[i],yi[i]) for i = 0,1,2.
    Returns c, an array containing the coefficients of
      p(x) = c[0] + c[1]*x + c[2]*x**2.

    """

    # check inputs and print error message if not valid:

    error_message = "xi and yi should have type numpy.ndarray"
    assert (type(xi) is np.ndarray) and (type(yi) is np.ndarray), error_message

    error_message = "xi and yi should have length 3"
    assert len(xi)==3 and len(yi)==3, error_message

    # Set up linear system to interpolate through data points:

    A = np.vstack([np.ones(3), xi, xi**2]).T
    c = solve(A,yi)

    return c

def plot_quad(xi, yi):
    """
    Perform quadratic interpolation and plot the resulting function along
    with the data points.
    """

    # Compute the coefficients:
    c = quad_interp(xi,yi)

    # Plot the resulting polynomial:
    x = np.linspace(xi.min() - 1,  xi.max() + 1, 1000)
    y = c[0] + c[1]*x + c[2]*x**2

    plt.figure(1)       # open plot figure window
    plt.clf()           # clear figure
    plt.plot(x,y,'b-')  # connect points with a blue line

    # Add data points  (polynomial should go through these points!)
    plt.plot(xi,yi,'ro')   # plot as red circles
    plt.ylim(-2,8)         # set limits in y for plot

    plt.title("Data points and interpolating polynomial")

    plt.savefig('quadratic.png')   # save figure as .png file

def cubic_interp(xi,yi):
    """
    Cubic interpolation.  Compute the coefficients of the polynomial
    interpolating the points (xi[i],yi[i]) for i = 0,1,2,3
    Returns c, an array containing the coefficients of
      p(x) = c[0] + c[1]*x + c[2]*x**2 + c[3]*x**3.

    """

    # check inputs and print error message if not valid:

    error_message = "xi and yi should have type numpy.ndarray"
    assert (type(xi) is np.ndarray) and (type(yi) is np.ndarray), error_message

    error_message = "xi and yi should have length 4"
    assert len(xi)==4 and len(yi)==4, error_message

    # Set up linear system to interpolate through data points:

    A = np.vstack([np.ones(4), xi, xi**2, xi**3]).T
    c = solve(A,yi)

    return c

def plot_cubic(xi, yi):
    """
    Perform cubic interpolation and plot the resulting function along
    with the data points.
    """

    # Compute the coefficients:
    c = cubic_interp(xi,yi)

    # Plot the resulting polynomial:
    x = np.linspace(xi.min() - 1,  xi.max() + 1, 1000)
    y = c[0] + c[1]*x + c[2]*x**2 + c[3]*x**3

    plt.figure(1)       # open plot figure window
    plt.clf()           # clear figure
    plt.plot(x,y,'b-')  # connect points with a blue line

    # Add data points  (polynomial should go through these points!)
    plt.plot(xi,yi,'ro')   # plot as red circles
    plt.ylim(-2,8)         # set limits in y for plot

    plt.title("Data points and interpolating polynomial")

    plt.savefig('cubic.png')   # save figure as .png file

def poly_interp(xi,yi):
    """
    General polynomial interpolation.

    Compute the coefficients of the polynomial
    interpolating the points (xi[i],yi[i]) for i = 0,1,2,...,n-1
    where n = len(xi) = len(yi).

    Returns c, an array containing the coefficients of
      p(x) = c[0] + c[1]*x + c[2]*x**2 + ... + c[N-1]*x**(N-1).

    """

    # check inputs and print error message if not valid:

    error_message = "xi and yi should have type numpy.ndarray"
    assert (type(xi) is np.ndarray) and (type(yi) is np.ndarray), error_message

    error_message = "xi and yi should have the same length "
    assert len(xi)==len(yi), error_message

    # Set up linear system to interpolate through data points:
    # Uses a list comprehension, see
    # http://docs.python.org/2/tutorial/datastructures.html#list-comprehensions

    n = len(xi)
    A = np.vstack([xi**j for j in range(n)]).T
    c = solve(A,yi)

    return c

def plot_poly(xi, yi):
    """
    Perform polynomial interpolation and plot the resulting function along
    with the data points.
    """

    # Compute the coefficients:
    c = poly_interp(xi,yi)

    # Plot the resulting polynomial:
    x = np.linspace(xi.min() - 1,  xi.max() + 1, 1000)

    # Use Horner's rule:
    n = len(xi)
    y = c[n-1]
    for j in range(n-1, 0, -1):
        y = y*x + c[j-1]

    plt.figure(1)       # open plot figure window
    plt.clf()           # clear figure
    plt.plot(x,y,'b-')  # connect points with a blue line

    # Add data points  (polynomial should go through these points!)
    plt.plot(xi,yi,'ro')   # plot as red circles
    plt.ylim(yi.min()-1, yi.max()+1)       # set limits in y for plot

    plt.title("Data points and interpolating polynomial")

    plt.savefig('poly.png')   # save figure as .png file

def test_quad1():
    """
    Test code, no return value or exception if test runs properly.
    """
    xi = np.array([-1.,  0.,  2.])
    yi = np.array([ 1., -1.,  7.])
    c = quad_interp(xi,yi)
    c_true = np.array([-1.,  0.,  2.])
    print "c =      ", c
    print "c_true = ", c_true
    # test that all elements have small error:
    assert np.allclose(c, c_true), \
        "Incorrect result, c = %s, Expected: c = %s" % (c,c_true)

    # Also produce plot:
    plot_quad(xi,yi)

def test_quad2():
    """
    Test code, no return value or exception if test runs properly.
    """

    # Generate a test by specifying c_true first:
    c_true = np.array([7., 2., -3.])
    # Points to interpolate:
    xi = np.array([-1.,  0.,  2.])
    # Function values to interpolate:
    yi = c_true[0] + c_true[1]*xi + c_true[2]*xi**2

    # Now interpolate and check we get c_true back again.
    c = quad_interp(xi,yi)
    print "c =      ", c
    print "c_true = ", c_true
    # test that all elements have small error:
    assert np.allclose(c, c_true), \
        "Incorrect result, c = %s, Expected: c = %s" % (c,c_true)

    # Also produce plot:
    plot_quad(xi,yi)

def test_cubic1():
    """
    Test code, no return value or exception if test runs properly.
    """

    # Generate a test by specifying c_true first:
    c_true = np.array([7., -2., -3., 1.])
    # Points to interpolate:
    xi = np.array([-1.,  0.,  1., 2.])
    # Function values to interpolate:
    yi = c_true[0] + c_true[1]*xi + c_true[2]*xi**2 + c_true[3]*xi**3

    # Now interpolate and check we get c_true back again.
    c = cubic_interp(xi,yi)
    print "c =      ", c
    print "c_true = ", c_true
    # test that all elements have small error:
    assert np.allclose(c, c_true), \
        "Incorrect result, c = %s, Expected: c = %s" % (c,c_true)

    # Also produce plot:
    plot_cubic(xi,yi)

def test_poly1():
    """
    Test code, no return value or exception if test runs properly.
    Same points as test_cubic1.
    """
    # Generate a test by specifying c_true first:
    c_true = np.array([7., -2., -3., 1.])
    # Points to interpolate:
    xi = np.array([-1.,  0.,  1., 2.])
    # Function values to interpolate:
    # Use Horner's rule:
    n = len(xi)
    yi = c_true[n-1]
    for j in range(n-1, 0, -1):
        yi = yi*xi + c_true[j-1]

    # Now interpolate and check we get c_true back again.
    c = poly_interp(xi,yi)

    print "c =      ", c
    print "c_true = ", c_true
    # test that all elements have small error:
    assert np.allclose(c, c_true), \
        "Incorrect result, c = %s, Expected: c = %s" % (c,c_true)

    # Also produce plot:
    plot_poly(xi,yi)

def test_poly2():
    """
    Test code, no return value or exception if test runs properly.
    Test with 5 points (quartic interpolating function).
    """
    # Generate a test by specifying c_true first:
    c_true = np.array([0., -6., 11., -6., 1.])
    # Points to interpolate:
    xi = np.array([-1.,  0.,  1., 2., 4.])
    # Function values to interpolate:
    # Use Horner's rule:
    n = len(xi)
    yi = c_true[n-1]
    for j in range(n-1, 0, -1):
        yi = yi*xi + c_true[j-1]

    # Now interpolate and check we get c_true back again.
    c = poly_interp(xi,yi)

    print "c =      ", c
    print "c_true = ", c_true
    # test that all elements have small error:
    assert np.allclose(c, c_true), \
        "Incorrect result, c = %s, Expected: c = %s" % (c,c_true)

    # Also produce plot:
    plot_poly(xi,yi)

if __name__=="__main__":
    print "Running test..."
    test_quad1()
    test_quad2()
    test_cubic1()
    test_poly1()
    test_poly2()


import numpy as np
import torch
import torch.nn

def sum_of_squares_error(y, t):
    return 0.5 * torch.sum((y-t)**2)

t = torch.tensor([0, 0, 1, 0, 0, 0, 0, 0, 0, 0])
y = torch.tensor([0.1, 0.05, 0.6, 0.0, 0.05, 0.1, 0.0, 0.1, 0.0, 0.0])
print(sum_of_squares_error(y, t), "'2'일 확률이 가장 높은 경우")

y = torch.tensor([0.1, 0.05, 0.1, 0.0, 0.05, 0.1, 0.0, 0.6, 0.0, 0.0])
print(sum_of_squares_error(y, t), "'7'일 확률이 가장 높은 경우")
print("---------------------------------------------------")

def default_cross_entropy_error(y, t):
    delta = 1e-7
    return -torch.sum(t * torch.log(y + delta))

t = torch.tensor([0, 0, 0, 0, 0, 0, 0, 1, 0, 0])
y = torch.tensor([0.1, 0.05, 0.6, 0.0, 0.05, 0.1, 0.0, 0.1, 0.0, 0.0])
print(default_cross_entropy_error(y, t))

y = torch.tensor([0.1, 0.05, 0.1, 0.0, 0.05, 0.1, 0.0, 0.6, 0.0, 0.0])
print(default_cross_entropy_error(y, t))
print("---------------------------------------------------")

def numpy_cross_entropy_error(y, t):
    if y.ndim == 1:
        y = y.reshape(1, y.size)
        t = t.reshape(1, t.size)

    batch_size = y.shape[0]
    return -np.sum(t * np.log(y + 1e-7)) / batch_size

t = np.array([0, 0, 0, 0, 0, 0, 0, 1, 0, 0])
y = np.array([0.1, 0.05, 0.6, 0.0, 0.05, 0.1, 0.0, 0.1, 0.0, 0.0])
print(numpy_cross_entropy_error(y, t))

def torch_cross_entropy_error(y, t):
    if y.dim() == 1:
        y = y.view(1, y.size()[0])
        t = t.view(1, t.size()[0])

    batch_size = y.shape[0]
    return -torch.sum(t * torch.log(y + 1e-7)) / batch_size

t = torch.tensor([0, 0, 0, 0, 0, 0, 0, 1, 0, 0])
y = torch.tensor([0.1, 0.05, 0.6, 0.0, 0.05, 0.1, 0.0, 0.1, 0.0, 0.0])
print(torch_cross_entropy_error(y, t))

def numpy_cross_entropy_error_1(y, t):
    if y.ndim == 1:
        y = y.reshape(1, y.size)
        t = t.reshape(1, t.size)

    if t.size == y.size:
        t = t.argmax(axis=1)

    batch_size = y.shape[0]
    return -np.sum(np.log(y[np.arange(batch_size), t] + 1e-7)) / batch_size

t = np.array([0, 0, 0, 0, 0, 0, 0, 1, 0, 0])
y = np.array([0.1, 0.05, 0.6, 0.0, 0.05, 0.1, 0.0, 0.1, 0.0, 0.0])
print(numpy_cross_entropy_error_1(y, t))

def cross_entropy_error_1(y, t):
    if y.dim() == 1:
        y = y.view(1, y.size()[0])
        t = t.view(1, t.size()[0])

    if t.size() == y.size():
        t = t.argmax(axis=1)

    batch_size = y.shape[0]
    return -torch.sum(torch.log(y[torch.arange(batch_size), t] + 1e-7)) / batch_size

t = torch.tensor([0, 0, 0, 0, 0, 0, 0, 1, 0, 0])
y = torch.tensor([0.1, 0.05, 0.6, 0.0, 0.05, 0.1, 0.0, 0.1, 0.0, 0.0])
print(cross_entropy_error_1(y, t))
print("---------------------------------------------------")


#!/usr/bin/env python
# coding: utf-8

# In[1]:

import math
import scipy.stats as st

def ExpectedLoss(PD, EA, LR):
    ## PD is the probability of default or Expected Default frequency (EDF) ##
    ## EA is the Exposure Amount (Total loan amount) ##
    ## LR is the loss rate (severity of default) ##

    EL = EA*PD*LR

    return EL

def RecoveryRate(LR):
    RR = 1 - LR

    return RR

def UnexpectedLoss(EA, PD, LR, stdLR):
    ## Assuming 2 state model ##

    VarPD = PD*(1 - PD)

    UL = EA*math.sqrt(PD*(stdLR**2) + (LR**2)*VarPD)

    return UL

def PortfolioExpectedLoss(PD1, EA1, LR1, PD2, EA2, LR2):
    EL1 = ExpectedLoss(PD1, EA1, LR1)
    EL2 = ExpectedLoss(PD2, EA2, LR2)

    ELp = EL1 + EL2

    return ELp

def PortfolioUnexpectedLoss(EA1, PD1, LR1, stdLR1, EA2, PD2, LR2, stdLR2, rho):
    ## rho is the correlation between the assets ##

    UL1 = UnexpectedLoss(EA1, PD1, LR1, stdLR1)
    UL2 = UnexpectedLoss(EA2, PD2, LR2, stdLR2)

    ULp = math.sqrt((UL1**2) + (UL2**2) + 2*rho*UL1*UL2)

    return ULp

def RiskContribution(EA1, PD1, LR1, stdLR1, EA2, PD2, LR2, stdLR2, rho):

    ## Enter x as 1 or 2 for risk contribution of those assets ##
    x = int(input('Enter a number: '))
    UL1 = UnexpectedLoss(EA1, PD1, LR1, stdLR1)
    UL2 = UnexpectedLoss(EA2, PD2, LR2, stdLR2)
    ULp = PortfolioUnexpectedLoss(EA1, PD1, LR1, stdLR1, EA2, PD2, LR2, stdLR2, rho)

    if x == 1:

        RC1 = UL1*(UL1 + (rho*UL2))/ULp

        return RC1

    elif x == 2:

        RC2 = UL2*(UL2 + (rho*UL1))/ULp

        return RC2

def FirmSizeScaleAdjustement(Xr, Yl, Yr):

    ## Operational Risk ##
    ## This is a formula when you want to see your scale of severity by viewing data from others banks ##
    ## Xr is the Bank X revenue ##
    ## Yl is the Bank Y loss from that event ##
    ## Yr is the Bank Y revenue ##
    ## We want to calculate the Bank X loss from that event by readjusting the scale ##

    Xl = Yl*((Xr/Yr)**0.23)

    return Xl

def ValueAtRisk(V, alpha, mu, sigma):
    ## Assuming VAR as Normal Distribution ##
    ## V is portfolio value ##
    ## mu is the Expected Portfolio return ##
    ## alpha is the significance level ##

    z = st.norm.ppf(alpha)

    var = (mu - z*sigma)*V

    return var

def ExpectedShortFall(V, alpha, mu, sigma):
    ES = (-mu + sigma * stats.norm.pdf(stats.norm.ppf(alpha))/(1-(alpha)))*V

    return ES

def LognormalValueAtRisk(V, alpha, mu, sigma):
    Lvar = V*(1 - math.exp(mu - (st.norm.ppf(alpha))*sigma ))

    return Lvar