# ============================== Answer ==============================

import numpy as np
from typing import List
class MyMLP():
    def __init__(
        self,
        input_size: int,
        hidden_size: int,
        num_classes: int,
        lr: float,
        batch_size: int = 1,
    ):
        self.input_size = input_size
        self.num_classes = num_classes
        self.batch_size = batch_size
        self.lr = lr

        self.w1 = np.random.randn(hidden_size, input_size)
        self.w2 = np.random.randn(num_classes, hidden_size)

    def generate_data(self):
        # Genereate Pseudo data
        # Randn for x
        # Randint one-hot for y
        x = np.random.randn(self.batch_size, self.input_size)
        y = np.random.randint(0, self.num_classes, (self.batch_size, ))
        y = np.eye(self.num_classes)[y]
        return x, y

    def sigmoid(self, x: np.ndarray) -> np.ndarray:
        # Sigmoid Activation
        return 1 / (1 + np.exp(-x))

    def partial_sigmoid(self, x: np.ndarray) -> np.ndarray:
        # Derivatives of sigmoid activation
        return self.sigmoid(x) * (1 - self.sigmoid(x))

    def softmax(self, x: np.ndarray) -> np.ndarray:
        # Softmax Layer
        return np.exp(x) / np.sum(np.exp(x), axis=1, keepdims=True)

    def forward(self, x: np.ndarray) -> np.ndarray:
        # Feed Forward for mlp
        self.a1 = x @ self.w1.T
        self.h1 = self.sigmoid(self.a1)
        self.a2 = self.h1 @ self.w2.T
        self.out = self.softmax(self.a2)
        return self.out

    def loss(self, y_hat: np.ndarray, y: np.ndarray) -> np.ndarray:
        # cross entropy loss
        return np.sum(np.log(y_hat) * y)

    def bp(self, x: np.ndarray, y:np.ndarray) -> np.ndarray:
        # See the process in mlp_bp.md

        self.dl_da2 = (-y + self.softmax(self.a2)) # dl/da2
        self.dl_dw2 = self.dl_da2.T @ self.h1 # dl/dw2
        self.dl_dh1 = self.dl_da2 @ self.w2 # dl/dh1
        self.dl_da1 = (self.dl_da2 @ self.w2) * self.partial_sigmoid(self.a1) # dl/da1
        self.dl_dw1 = self.dl_da1.T @ x # dl/dw1

        self.w1 = self.w1 - self.lr * self.dl_dw1
        self.w2 = self.w2 - self.lr * self.dl_dw2

        return self.w1, self.w2

    def matrix_differential_propogation(self, x: np.ndarray, y: np.ndarray) -> np.ndarray:
        # Matrix-Matrix function derivatives
        # Double check for gradient
        dl_dw2 = -np.kron(self.h1, y.T) + np.kron(self.h1, np.exp(self.a2).T) / np.sum(np.exp(self.a2))
        return dl_dw2

if __name__ == '__main__':
    np.random.seed(2022)
    mymlp = MyMLP(input_size=4, hidden_size=10, num_classes=3, lr=0.01, batch_size=1)
    x, y = mymlp.generate_data()

    # Step 1
    # check your forward result:
    out = mymlp.forward(x)
    assert np.linalg.norm(out - np.array([[0.36235674, 0.40728367, 0.23035959]])) < 1e-7

    # Step 2
    # check your loss
    loss = mymlp.loss(out, y)
    assert np.linalg.norm(loss - -1.015126071106058) < 1e-7

    w1, w2 = mymlp.bp(x, y)
    # Step 2
    # check your gradients:
    assert np.linalg.norm(mymlp.dl_da2 - np.array([[-0.63764326,  0.40728367,  0.23035959]])) < 1e-7
    assert np.linalg.norm(mymlp.dl_dw2 - np.array([[-0.40732181, -0.47931036, -0.15217421, -0.03860381, -0.09738246, -0.36582423,
        -0.57306758, -0.36619898, -0.53017721, -0.32491802],
        [ 0.2601698,   0.30615125,  0.09719866,  0.02465752,  0.06220137,  0.23366394,
        0.36603706,  0.2339033,   0.33864158,  0.2075358 ],
        [ 0.14715201,  0.17315911,  0.05497555,  0.01394629,  0.03518109,  0.13216029,
        0.20703051,  0.13229568,  0.19153563,  0.11738222]])) < 1e-7
    assert np.linalg.norm(mymlp.dl_dh1 - np.array([[-0.22072836,  0.54220196,  0.2838984,  -0.24103344, -1.74668689, -0.65096623,
        -0.33832413, -0.18948642,  0.76756801,  0.07541716]]))
    assert np.linalg.norm(mymlp.dl_da1 - np.array([[-0.05093011,  0.10120303,  0.05158341, -0.01370905, -0.22601836, -0.15920446,
        -0.03079302, -0.04632553,  0.1075607,   0.0188474 ]]))
    assert np.linalg.norm(mymlp.dl_dw1 - np.array([[ 0.03771251, -0.04810758, -0.05592065, -0.02520829],
        [-0.07493837,  0.09559438,  0.11111971,  0.05009129],
        [-0.03819625,  0.04872466,  0.05663796,  0.02553164],
        [ 0.01015122, -0.0129493,  -0.01505237, -0.00678541],
        [ 0.16736108, -0.21349247, -0.24816545, -0.11186968],
        [ 0.11788702, -0.15038138, -0.17480459, -0.07879958],
        [ 0.02280148, -0.02908648, -0.03381036, -0.01524126],
        [ 0.03430292, -0.04375818, -0.05086487, -0.02292921],
        [-0.07964608,  0.10159971,  0.11810036,  0.05323807],
        [-0.01395604,  0.01780288,  0.02069421,  0.00932868]])) < 1e-7

    # Step 3
    # check your updated weight result
    assert np.linalg.norm(w1 - np.array([[-0.00090502, -0.27442035, -0.13872636,  1.98493824],
            [ 0.28285871,  0.75985271,  0.29987041,  0.53979636],
            [ 0.37387925,  0.37732615, -0.09077957, -2.30619859],
            [ 1.14265851, -1.53552479, -0.86360149,  1.01661279],
            [ 1.03229027, -0.8223573,   0.02138651, -0.38222486],
            [-0.30536435,  0.99879532, -0.12552579, -1.47509791],
            [-1.94113434,  0.83393979, -0.56687978,  1.17463937],
            [ 0.3187258,   0.19130801,  0.36977883, -0.10091857],
            [-0.94101303, -1.40515771,  2.07946601, -0.12084862],
            [ 0.75993144,  1.82725411, -0.66093403, -0.80789955]])) < 1e-7
    assert np.linalg.norm(w2 - np.array([[ 0.89187333, -0.21265435, -0.93800277,  0.59992435,  2.22408652,  1.0036637,
        1.15540522, -0.1519143,  -1.64527408, -1.45186499],
        [ 0.31815989,  0.80828812, -0.24177658,  0.16487451, -0.03412259,  0.08552794,
        1.03048616, -1.06316559, -1.01697087, -0.42230754],
        [ 0.93068173,  0.31559234, -0.943021,    0.32112784, -1.36990554, -0.21448964,
        -0.11681657,  0.62104636,  0.55332894, -2.95860606]])) < 1e-7

    # Bonus Point
    dl_dw2 = mymlp.matrix_differential_propogation(x, y)
    assert np.linalg.norm(dl_dw2 - np.array([[-0.40732181, -0.47931036, -0.15217421, -0.03860381, -0.09738246, -0.36582423,
        -0.57306758, -0.36619898, -0.53017721, -0.32491802],
        [ 0.2601698,   0.30615125,  0.09719866,  0.02465752,  0.06220137,  0.23366394,
        0.36603706,  0.2339033,   0.33864158,  0.2075358 ],
        [ 0.14715201,  0.17315911,  0.05497555,  0.01394629,  0.03518109,  0.13216029,
        0.20703051,  0.13229568,  0.19153563,  0.11738222]])) < 1e-7


from typing import Tuple
import matplotlib.pyplot as plt
import numpy as np

import pandas as pd
import math

import json

import os

from mpl_toolkits import mplot3d

import argparse

def MaxNorm(X):
    #! Expects X to be a np.array
    # dim = X.shape[1]
    # m = X.shape[0]

    max = -1

    for xi in X:
        norm = 0
        for v in xi:
            norm = norm + v*v
        norm = math.sqrt(norm)
        if norm > max:
            max = norm

    return max

def AproximatePerceptron(X: np.ndarray, Y: np.ndarray, maxIterations: int) -> Tuple[np.ndarray, float, int]:
    weights = np.zeros(X.shape[1])      #| Parameters
    bias = 0                            #| Bias, separated for implementation simplicity

    totalErrors = 0                     #| Total errors in the entire process, less or equal than R^2/delta^2
    currentErrors = -1                  #| Errors in the current iterations

    it = 0

    while currentErrors != 0 and it <= maxIterations:    #| Stops updating the weights when there are no more errors or until a maxIteration in case something goes wrong
        currentErrors = 0

        for xi, yi in zip(X, Y):
            if yi*(np.dot(weights, xi) + bias) <= 0:    #| Checks for an error
                totalErrors += 1
                currentErrors += 1

                weights += yi*xi                        #| Updating weights and bias
                bias += yi

        it += 1

    return (weights, bias, totalErrors)

def RenderPointsAndSolution2D(X: np.ndarray, Y: np.ndarray, weights: np.ndarray, bias: float):
    X_1, X_2 = zip(*X)

    df = pd.DataFrame(dict(X1=X_1, X2=X_2, Y=Y))
    colors = {-1:'lightcoral', 1:'steelblue'}

    fig, ax = plt.subplots()
    ax.scatter(df['X1'], df['X2'], c=df['Y'].map(colors))

    solX = np.linspace(min(X_1), max(X_1), 10)
    solY = (-bias - solX*weights[0])/weights[1]

    plt.plot(solX, solY, '#9467bd')
    plt.savefig('result2D.png')
    plt.show()

def RenderPointsAndSolution3D(X: np.ndarray, Y: np.ndarray, weights: np.ndarray, bias: float):
    X_1, X_2, X_3 = zip(*X)

    fig = plt.figure()
    ax = plt.axes(projection='3d')

    df = pd.DataFrame(dict(X1=X_1, X2=X_2, X3=X_3, Y=Y))
    colors = {-1:'lightcoral', 1:'steelblue'}

    ax.scatter(df['X1'], df['X2'], df['X3'], c=df['Y'].map(colors))
    ax.view_init(elev=10., azim=-110)

    solX = np.linspace(min(X_1), max(X_1), 10)
    solY = np.linspace(min(X_2), max(X_2), 10)

    pX, pY = np.meshgrid(solX, solY)
    pZ = (-bias - pY*weights[1]- pX*weights[0])/weights[2]

    ax.contour3D(pX, pY, pZ, 50, cmap='binary')

    plt.savefig('result3D.png')
    plt.show()

def CheckDataIntegrity(points: list, labels: list) -> bool:
    if len(points) <= 0 or len(points) != len(labels):
        return False

    #| Check first element
    x1 = points[0]

    if type(x1) != list:
        return False

    isPointsValid = all(
        isinstance(x, list) and
        len(x)== len(x1) and
        all(isinstance(xi, (int, float)) for xi in x)
        for x in points)    #| Probably not eficient but check if it's composed of list of the same size with numeric values

    isLabelsValid = all(
        isinstance(x, (int, float)) for x in labels     #| Allows floats for experimenting
    )

    return isPointsValid and isLabelsValid

def LoadData(fileName:str) -> Tuple[np.ndarray, np.ndarray, int]:
    dir_path = os.path.dirname(os.path.realpath(__file__))
    fileDir = os.path.join(dir_path, 'Data', fileName)

    if not os.path.exists(fileDir):
        raise Exception("File doesn't exists!")

    f = open(fileDir)

    data = json.load(f)

    points = data['points']
    labels = data['labels']

    if not CheckDataIntegrity(points, labels):
        raise Exception("Data is not well organized")

    return (np.array(points), np.array(labels), len(points[0]))

if __name__ == "__main__":
    descText = 'This is a basic implementation of the Perceptron algorithm, it will load the labeld data from a .json and will calculate the hyperplane dividing the data.\nIt only generates a plot for 2D and 3D data.'

    parser = argparse.ArgumentParser(description=descText)

    parser.add_argument('-v', '--version', help='shows program version', action='store_true')
    parser.add_argument('-d', '--data', help='Specify the name of your .json file with your data.\nThe file must be inside the "Data" folder and should follow the same sintaxys as the examples given.')
    parser.add_argument('-it', '--iterations', help='Maximum number of iterations the program can do.')

    args = parser.parse_args()

    if args.version:
        print("Perceptron example version 0.1 by Daniel Eliasib")

    dataFile = ""
    maxIter = 1500

    if args.data:
        dataFile = str(args.data)
    else:
        dataFile = "example2D-1.json"

    if args.iterations:
        val = int(args.iterations)
        maxIter = 1 if val <= 1 else val

    print("Loading data example %s" % dataFile)
    print("Aproximating with maximum iterations %d" % maxIter)

    X, Y, dim = LoadData(dataFile)

    W, b, k = AproximatePerceptron(X, Y, maxIter)

    if dim == 2:
        RenderPointsAndSolution2D(X, Y, W, b)
    elif dim == 3:
        RenderPointsAndSolution3D(X, Y, W, b)
    else:
        print("The points have dimention %d, this programm will only render 2D and 3D data." % dim)

    print("Results: ")
    print("\tWeights: " + str(W))
    print("\tBias: " + str(b))
    print("\tTotal errors: " + str(k))
# RenderPointsAndSolution2D(X, Y, W, b)


import numpy as np

def TP0(dv, u):
    '''Demo problem 0 for horsetail matching, takes two input vectors of any size
    and returns a single output'''
    return np.linalg.norm(np.array(dv)) + np.linalg.norm(np.array(u))

def TP1(x, u, jac=False):
    '''Demo problem 1 for horsetail matching, takes two input vectors of size 2
    and returns just the qoi if jac is False or the qoi and its gradient if jac
    is True'''
    factor = 0.1*(u[0]**2 + 2*u[0]*u[1] + u[1]**2)
    q = 0 + factor*(x[0]**2 + 2*x[1]*x[0] + x[1]**2)
    if not jac:
        return q
    else:
        grad = [factor*(2*x[0] + 2*x[1]), factor*(2*x[0] + 2*x[1])]
        return q, grad

def TP2(dv, u, jac=False):
    '''Demo problem 2 for horsetail matching, takes two input vectors of size 2
    and returns just the qoi if jac is False or the qoi and its gradient if jac
    is True'''
    y = dv[0]/2.
    z = dv[1]/2. + 12

    q = 0.25*((y**2 + z**2)/10 + 5*u[0]*u[1] - z*u[1]**2) + 0.2*z*u[1]**3 + 7

    if not jac:
        return q
    else:
        dqdx1 = (1./8.)*( 2*y/10. )
        dqdx2 = (1./8.)*( 2*z/10. - u[1]**2) + 0.1*u[1]**3
        return q, [dqdx1, dqdx2]

def TP2b(dv, u, jac=False):
    '''Demo problem 2 for horsetail matching, takes two input vectors of size 2
    and returns just the qoi if jac is False or the qoi and its gradient if jac
    is True'''
    y = dv[0]/2.
    z = dv[1]/2. + 12

    q = 0.25*((y**2 + z**2)/10 + 5*u[0]*u[1] - z*u[1]**2) +\
            0.2*z*u[1]**3 + 7 + u[0]*(y + z)*0.02

    if not jac:
        return q
    else:
        dqdx1 = (1./8.)*( 2*y/10.) + 0.01*u[0]
        dqdx2 = (1./8.)*( 2*z/10. - u[1]**2) + 0.1*u[1]**3 + 0.01*u[0]
        return q, [dqdx1, dqdx2]

def TP3(x, u, jac=False):
    '''Demo problem 1 for horsetail matching, takes two input values of
    size 1'''

    q = 2 + 0.5*x + 1.5*(1-x)*u
    if not jac:
        return q
    else:
        grad = 0.5 -1.5*u
        return q, grad


import numpy as np
import matplotlib.pyplot as plt

class Newtons_Divided_Differences:
    def __init__(self, differences, data_x):
        self.differences = differences
        self.data_x = data_x
        self.multiplierstring = ""
        # self.generalequation = "y = f(x) = " # TBI
        print(f"x = {data_x}\n")

    def __call__(self, x):
        """
        this function is for calculating y from given x using all the difference coefficients
        x can be a single value or a numpy
        the formula being used:
        f(x) = f [x0] + (x-x0) f[x0,x1] + (x-x0) (x-x1) f[x0,x1,x2] + . . . + (x-x0) (x-x1) . . . (x-xk-1) f[x0, x1, . . ., xk]

        work on this after implementing 'calc_div_diff'. Then you should have
        f[x0], f[x0,x1]. . . . . ., f[x0, x1, . . ., xk] stored in self.differences

        'res' variable must return all the results (corresponding y for x)
        """

        y = np.zeros(len(x)) #Initialization to avoid runtime error. You can change this line if you wish

        # place your code here!!!!!!!!!!!!!!!!!!!!!!!
        for i in range(len(y)):
            y[i] = self.newtonDiv(x[i])
        return y

    def newtonDiv(self, xi):
        res = 0
        addstr = f"y = f({xi}) = "
        for i in range(len(self.differences)):
            res += self.differences[i][0] * self.multiplier(xi, i)
            if i is not 0:
                addstr += " + "
            if self.differences[i][0] < 0:
                addstr += f'({self.differences[i][0]})'
            else:
                addstr += f'{self.differences[i][0]}'
            if self.multiplierstring is not "":
                addstr += f' * {self.multiplierstring}' #+ f' = {self.multiplier(xi, i)}'
        print(addstr + "\n")
        return res

    def multiplier(self, xi, i):
        res = 1
        mulstr = ""
        for j in range(i):
            res *= xi - self.data_x[j]
            if j is not 0:
                mulstr += " * "
            if(self.data_x[j] < 0):
                mulstr += f"({xi} - ({self.data_x[j]}))"
            else:
                mulstr += f"({xi} - {self.data_x[j]})"
        self.multiplierstring = mulstr
        return res

#basic rule for calculating the difference, implanted in the lambda function. You may use it if you wish
difference = lambda y2, y1, x2, x1: (y2-y1)/(x2-x1)

def calc_div_diff(x,y):
    assert(len(x)==len(y))
    #write this function to calculate all the divided differences in the list 'b'
    b = [[] for i in range(len(x))]

    #place your code here!!!!!!!!!!!!!!!!!!!!!!!!!
    for i in range(len(x)):
        for j in range(len(x)-i):
            if(i==0):
                b[i].append(y[j])
            else:
                b[i].append(difference(b[i-1][j+1],b[i-1][j],x[j+i],x[j]))
    return b

def print_diff(diff):
    for i in range(len(diff)):
        if i is 0:
            print('y = ', end = '')
        print(diff[i])
    print()

data_x = np.array([-3.,-2.,-1.,0.,1.,3.,4.])
data_y = np.array([-60.,-80.,6.,1.,45.,30.,16.])
# data_x = np.array([10.,15.,20.,22.5])
# data_y = np.array([227.4,362.78,517.35,602.97])
differences = calc_div_diff(list(data_x), list(data_y))
obj = Newtons_Divided_Differences(list(differences), list(data_x))
print_diff(differences)

# #generating 50 points from -3 to 4 in order to create a smooth line
X = np.linspace(-3, 4, 50, endpoint=True)
# X = np.array([-2.999999, -2.01231, -0.93213, 0.6757757, 1.97578, 2.9466, 3.998654])
# X = np.array([10.,15.,20.,22.5])
# X = np.linspace(10.1, 22.49999, 4, endpoint=True)
F = obj(X)
plt.plot(X,F)
plt.plot(data_x, data_y, 'ro')
plt.show()


'''A Collection of trigonometric orthogonal transforms.'''

import numpy as np

# DCT-II of 1D signal (type 2) [equivalent to scipy.fftpack.dct]
def dctII(din):
  # dct size
  N = len(din)
  # create output array
  dout = np.zeros(din.shape)
  # rearrange input in output array
  for i in range(0,N/2):
    dout[i] = din[2*i]
    dout[-i-1] = din[2*i+1]
  # twiddle
  W = np.exp(-1j*2*np.pi*np.arange(0,N)/(4*N))
  # return DCT
  return 2*np.real(W * np.fft.fft(dout))

# DCT-III of 1D signal (type 2) [equivalent to scipy.fftpack.idct divided by 2N]
def dctIII(din):
  # dct-III size
  N = len(din)
  # create output array
  temp = np.zeros(din.shape, dtype=complex)
  dout = np.zeros(din.shape)
  # twiddle
  W = np.exp(1j*2*np.pi*np.arange(0,N)/(4*N))
  # create output array
  # first point
  temp[0]  = 0.5 * W[0] * din[0]
  temp[1:] = 0.5 * W[1:] * (din[1:] - 1j*din[:0:-1])
  # compute iFFT
  temp = np.real(np.fft.ifft(temp))
  # output mapping
  for i in range(0,N/2):
    dout[2*i] = temp[i]
    dout[2*i+1] = temp[-i-1]
  # return DCT-III
  return dout

# DST-III of 1D signal (type 3) [see Shao and Johnson, Type-IV DCT, DST and MDCT algorithms with reduced numbers of arithmetic operations, 2009]
def dstIII(x):
  # size and output array
  N = len(x)
  X = np.zeros(N)
  # reverse array
  X[0] = x[0]
  X[1:] = x[-1:0:-1]
  # perform DCT-III
  X = dctIII(X)
  # invert sign of odd indices
  X[1::2] *= -1
  # return DST-III
  return X

# DCT-IV of 1D signal (type 4) [see Z. Wang, On Computing the Discrete Fourier and Cosine Transforms, 1985]
def dctIV(x):
  # size and output array
  N = len(x)
  X = np.zeros(N)
  # Multiplication by matrix Bn from paper, combined with reordering from Pn, permutation matrix
  X[0] = x[0] # factor sqrt(2) cancelled by later DCT-III
  for n in range(1,N-1,2):
    X[N-1-(n-1)/2] = x[n] - x[n+1]    # odd indices
    X[(n+1)/2] = x[n] + x[n+1]        # even indices
  X[N/2] = x[N-1] # factor sqrt(2) cancelled by later DST-III
  # Half-length DCT-III
  X[0] *= 2     # to conform to our definition of DCT-III
  X[0:N/2] = dctIII(X[0:N/2])
  # Half-length DST-III of reversed data, in reversed order
  X[N/2] *= 2   # to conform to our definition of DST-III
  X[N/2+1:] = X[-1:N/2:-1]
  X[N-1:N/2-1:-1] = dstIII(X[N/2:])
  # Multiplication by Tn from paper (Rotation matrix)
  for n in range(0,N/2):
    c = np.cos((2*n+1)*np.pi/4./N)
    s = np.sin((2*n+1)*np.pi/4./N)
    X[[n, N-1-n]]  = np.dot(np.array([[c,s],[s,-c]]), X[[n,N-1-n]])
  # Scaling was adjusted compared to paper so that dctIV is orthonormal (its own inverse)
  X *= np.sqrt(N/2.)
  # return DCT-IV
  return X

# DCT-II of 1D signal (type 2) [equivalent to scipy.fftpack.dct]
def dct(din):
  return dctII(din)

# iDCT-II (<=> DCT-III) of 1D signal (type 2) [equivalent to scipy.fftpack.idct divided by 2N]
def idct(din):
  return dctIII(din)

# Modified Discrete Cosine Transform
# see ref: Shao and Johnson, Type-IV DCT, DST, and MDCT algo..., 2009
# uses DCT-IV to compute MDCT
# takes as input 2N array, outputs N array
def mdct(x):
  N = len(x)/2 # output is half length

  # create array
  X = np.zeros(N)

  # fill the array according to Shao and Johnson
  for n in range(N/2):
    X[n] = -x[3*N/2-1-n] - x[3*N/2+n]
  for n in range(N/2,N):
    X[n] = x[n-N/2] - x[3*N/2-1-n]

  # take the DCT-IV of this array
  X = dctIV(X)

  # return MDCT
  return X

# inverse Modified Discrete Cosine Transform
# see ref: Shao and Johnson, Type-IV DCT, DST, and MDCT algo..., 2009
# uses DCT-IV to compute iMDCT
# takes N array as input, outputs 2N array
def imdct(X):
  N = len(X)

  # create array
  x = np.zeros(2*N)

  # take DCT-IV
  x[:N] = dctIV(X)

  # we know DCT-IV output is mirror antisymmetric, the iMDCT is shifted by N/2
  x[N:3*N/2] = -x[N/2-1::-1]
  x[3*N/2:] = -x[:N/2]
  x[:N/2] = x[N/2:N]
  x[N/2:N] = -x[N/2-1::-1]

  # return iMDCT
  return x

# Discrete Hartley transform
def dht(din):
  # dht size
  N = len(din)
  # create output array
  H = np.zeros(N)
  # Fourier transform
  F = np.fft.fft(din)
  # mapping to DHT
  for k in range(N):
    H[k] = np.real(F[k]) - np.imag(F[k])
  # return DHT
  return H

# inverse Discrete Hartley transform
def idht(din):
  # dht size
  N = len(din)
  # create output array
  F = np.zeros(N,dtype=complex)
  # mapping to DFT
  F[0] = din[0]
  for k in range(1,N):
    F[k] = 0.5*(din[k] + din[N-k]) - 1j*(din[k] - din[N-k])
  # inverse Fourier transform
  F = np.fft.ifft(F)
  # return iDFT
  return np.real(F)

## Return the DFT to DCT transform matrix
def spectral_dct_mtx(N):

  D2C = np.zeros((N,N), dtype=complex)

  # DC component
  D2C[0,0] = 2*N

  # l even, right part
  k = np.arange(N/2+1,N)
  l = 2*(N-k)
  D2C[l,k] = N*np.exp(1j*np.pi*l/2./N)

  # l even, left part
  k = np.arange(1, N/2)
  l = 2*k
  D2C[l,k] = N*np.exp(-1j*np.pi*l/2./N)

  # l odd
  K, L = np.meshgrid(np.arange(N), np.arange(1,N,2))
  D2C[1::2,:] = 1j*np.exp(-1j*np.pi*K/N)*(1/np.sin(np.pi*(K+L/2.)/N) + 1/np.sin(np.pi*(K-L/2.)/N))

  return D2C


# notes for this course can be found at:
# https://deeplearningcourses.com/c/data-science-linear-regression-in-python
# https://www.udemy.com/data-science-linear-regression-in-python

import numpy as np
import matplotlib.pyplot as plt

def make_poly(X, deg):
    n = len(X)
    data = [np.ones(n)]
    for d in xrange(deg):
        data.append(X**(d+1))
    return np.vstack(data).T

def fit(X, Y):
    return np.linalg.solve(X.T.dot(X), X.T.dot(Y))

def fit_and_display(X, Y, sample, deg):
    N = len(X)
    train_idx = np.random.choice(N, sample)
    Xtrain = X[train_idx]
    Ytrain = Y[train_idx]

    plt.scatter(Xtrain, Ytrain)
    plt.show()

    # fit polynomial
    Xtrain_poly = make_poly(Xtrain, deg)
    w = fit(Xtrain_poly, Ytrain)

    # display the polynomial
    X_poly = make_poly(X, deg)
    Y_hat = X_poly.dot(w)
    plt.plot(X, Y)
    plt.plot(X, Y_hat)
    plt.scatter(Xtrain, Ytrain)
    plt.title("deg = %d" % deg)
    plt.show()

def get_mse(Y, Yhat):
    d = Y - Yhat
    return d.dot(d) / len(d)

def plot_train_vs_test_curves(X, Y, sample=20, max_deg=20):
    N = len(X)
    train_idx = np.random.choice(N, sample)
    Xtrain = X[train_idx]
    Ytrain = Y[train_idx]

    test_idx = [idx for idx in xrange(N) if idx not in train_idx]
    # test_idx = np.random.choice(N, sample)
    Xtest = X[test_idx]
    Ytest = Y[test_idx]

    mse_trains = []
    mse_tests = []
    for deg in xrange(max_deg+1):
        Xtrain_poly = make_poly(Xtrain, deg)
        w = fit(Xtrain_poly, Ytrain)
        Yhat_train = Xtrain_poly.dot(w)
        mse_train = get_mse(Ytrain, Yhat_train)

        Xtest_poly = make_poly(Xtest, deg)
        Yhat_test = Xtest_poly.dot(w)
        mse_test = get_mse(Ytest, Yhat_test)

        mse_trains.append(mse_train)
        mse_tests.append(mse_test)

    plt.plot(mse_trains, label="train mse")
    plt.plot(mse_tests, label="test mse")
    plt.legend()
    plt.show()

    plt.plot(mse_trains, label="train mse")
    plt.legend()
    plt.show()

if __name__ == "__main__":
    # make up some data and plot it
    N = 100
    X = np.linspace(0, 6*np.pi, N)
    Y = np.sin(X)

    plt.plot(X, Y)
    plt.show()

    for deg in (5, 6, 7, 8, 9):
        fit_and_display(X, Y, 10, deg)
    plot_train_vs_test_curves(X, Y)


"""
Please see
https://computationalmindset.com/en/neural-networks/ordinary-differential-equation-solvers.html#ode1
for details
"""

import numpy as np
import matplotlib.pyplot as plt

from scipy.integrate import solve_ivp

ode_fn = lambda t, x: np.sin(t) + 3. * np.cos(2. * t) - x

an_sol = lambda t : (1./2.) * np.sin(t) - (1./2.) * np.cos(t) + \
                    (3./5.) * np.cos(2.*t) + (6./5.) * np.sin(2.*t) - \
                    (1./10.) * np.exp(-t)

t_begin=0.
t_end=10.
t_nsamples=100
t_space = np.linspace(t_begin, t_end, t_nsamples)
x_init = 0.

x_an_sol = an_sol(t_space)

method = 'RK45' #available methods: 'RK45', 'RK23', 'DOP853', 'Radau', 'BDF', 'LSODA'
num_sol = solve_ivp(ode_fn, [t_begin, t_end], [x_init], method=method, dense_output=True)
x_num_sol = num_sol.sol(t_space).T

plt.figure()
plt.plot(t_space, x_an_sol, '--', linewidth=2, label='analytical')
plt.plot(t_space, x_num_sol, linewidth=1, label='numerical')
plt.title('ODE 1st order IVP solved by SciPy with method=' + method)
plt.xlabel('t')
plt.ylabel('x')
plt.legend()
plt.show()


# -*- coding: utf-8 -*-
"""
Created on Wed Jun 16 11:34:21 2021

@author: Luigi
"""

import matplotlib.pyplot as plt
import numpy as np
import math
import scipy as sci
from scipy.optimize import fsolve
import sympy as sym

def puntofisso(g, x0, tol, nMax):
    xks = []
    xks.append(x0)
    x1 = g(x0)
    xks.append(x1)
    it = 1
    while it < nMax and abs(x1 - x0) >= tol * abs(x1):
        x0 = x1
        x1 = g(x0)
        xks.append(x1)
        it += 1
    return x1, it, xks

def ordine(xks, iteraz):
    num = math.log(abs(xks[iteraz - 1] - xks[iteraz - 2]) / abs(xks[iteraz - 3] - xks[iteraz - 2]))
    den = math.log(abs(xks[iteraz - 3] - xks[iteraz - 2]) / abs(xks[iteraz - 3] - xks[iteraz - 4]))
    return num / den

x = sym.Symbol("x")
gx = sym.tan((3/2) * x) - 2 * sym.cos(x) - x * (6 - x)
g = sym.utilities.lambdify(x, gx, np)
dgx = sym.diff(gx, x, 1)
dg = sym.utilities.lambdify(x, dgx, np)

# Plotto la funzione per cercare zeri
xx = np.linspace(-1, 1, 100)
plt.plot(xx, xx, xx, g(xx))
plt.show()

radice = fsolve(g, -1)

print(radice)
"""
La |derivata| è < 1 in un intorno quindi rispetta il th di convergenza locale
"""
xrad = np.linspace(radice - 0.1, radice + 0.1, 100)

plt.plot(xrad, abs(dg(xrad)), xrad, np.zeros(100,))
plt.show()

geqx = (sym.tan((3/2) * x) - 2 * sym.cos(x)) / (6 - x)
geq = sym.utilities.lambdify(x, geqx, np)
dgeqx = sym.diff(geqx, x, 1)
dgeq = sym.utilities.lambdify(x, dgeqx, np)
plt.plot(xx, xx, xx, geq(xx))
plt.show()
radice = fsolve(geq, -0.20)

print(abs(dgeq(radice)))
plt.plot(xrad, abs(dgeq(xrad)), xrad, np.zeros(100,))
plt.show()

# Testo puntofisso

x0 = 0
tol = 1.e-7
nmax = 500

x1, iteraz, xks = puntofisso(g, x0, tol, nmax)
plt.plot(np.arange(iteraz + 1), xks)
print(ordine(xks, iteraz + 1))

import numpy,import scipy,from scipy,import sympy


import numpy as np
import matplotlib.pyplot as plt

# Plot 2D points
def displaypoints2d(points):
  plt.figure(0)
  plt.plot(points[0,:],points[1,:], '.b')
  plt.xlabel('Screen X')
  plt.ylabel('Screen Y')
  plt.show()

# Plot 3D points
def displaypoints3d(points):
  fig = plt.figure(1)
  ax = fig.add_subplot(111, projection='3d')
  ax.scatter(points[0,:], points[1,:], points[2,:], 'b')
  ax.set_xlabel("World X")
  ax.set_ylabel("World Y")
  ax.set_zlabel("World Z")
  plt.show()

def cart2hom(points):
  """ Transforms from cartesian to homogeneous coordinates.

  Args:
    points: a np array of points in cartesian coordinates

  Returns:
    points_hom: a np array of points in homogeneous coordinates
  """

  shape = points.shape
  points_hom = np.concatenate((points, np.ones((1, shape[1]))), axis=0)
  # points_hom = np.ones((shape[0]+1, shape[1]))
  # points_hom[:-1, :] = points

  return points_hom

def hom2cart(points):
  """ Transforms from homogeneous to cartesian coordinates.

  Args:
    points: a np array of points in homogenous coordinates

  Returns:
    points_hom: a np array of points in cartesian coordinates
  """

  # shape = points.shape
  # points_car = np.zeros((shape[0], shape[1]-1))
  # for i in range(shape[0]):
  #       points_car[i, :] = points[i, :-1] / points[i, -1]
  # devide the last row 4*n
  return np.delete(points/points[-1], -1, axis=0)

def gettranslation(v):
  """ Returns translation matrix T in homogeneous coordinates for translation by v.

  Args:
    v: 3d translation vector

  Returns:
    T: translation matrix in homogeneous coordinates
  """
  n = len(v)
  T = np.identity(n+1)
  for i in range(n):
      T[i, -1] = v[i]

  return T

def getxrotation(d):
  """ Returns rotation matrix Rx in homogeneous coordinates for a rotation of d degrees around the x axis.

  Args:
    d: degrees of the rotation

  Returns:
    Rx: rotation matrix
  """

  d_radian = d * (np.pi/180)
  Rx = np.identity(4)
  Rx[1, 1] = np.cos(d_radian)
  Rx[1, 2] = - np.sin(d_radian)
  Rx[2, 1] = - Rx[1, 2]
  Rx[2, 2] = Rx[1, 1]

  return Rx

def getyrotation(d):
  """ Returns rotation matrix Ry in homogeneous coordinates for a rotation of d degrees around the y axis.

  Args:
    d: degrees of the rotation

  Returns:
    Ry: rotation matrix
  """
  d_radian = d * (np.pi/180)
  Ry = np.identity(4)
  Ry[0, 0] = np.cos(d_radian)
  Ry[0, 2] = np.sin(d_radian)
  Ry[2, 0] = - Ry[0, 2]
  Ry[2, 2] = Ry[0, 0]

  return Ry

def getzrotation(d):
  """ Returns rotation matrix Rz in homogeneous coordinates for a rotation of d degrees around the z axis.

  Args:
    d: degrees of the rotation

  Returns:
    Rz: rotation matrix
  """
  d_radian = d * (np.pi/180)
  Rz = np.identity(4)
  Rz[0, 0] = np.cos(d_radian)
  Rz[0, 1] = -np.sin(d_radian)
  Rz[1, 0] = - Rz[0, 1]
  Rz[1, 1] = Rz[0, 0]

  return Rz

def getcentralprojection(principal, focal):
  """ Returns the (3 x 4) matrix L that projects homogeneous camera coordinates on homogeneous
  image coordinates depending on the principal point and focal length.

  Args:
    principal: the principal point, 2d vector
    focal: focal length

  Returns:
    L: central projection matrix
  """

  L = np.zeros((3, 4))
  L[0, 0] = focal
  L[1, 1] = focal
  L[0, 2] = principal[0]
  L[1, 2] = principal[1]
  L[2, 2] = 1

  return L

def getfullprojection(T, Rx, Ry, Rz, L):
  """ Returns full projection matrix P and full extrinsic transformation matrix M.

  Args:
    T: translation matrix
    Rx: rotation matrix for rotation around the x-axis
    Ry: rotation matrix for rotation around the y-axis
    Rz: rotation matrix for rotation around the z-axis
    L: central projection matrix

  Returns:
    P: projection matrix
    M: matrix that summarizes extrinsic transformations
  """

  M = Rz.dot(Rx).dot(Ry).dot(T)
  P = L.dot(M)

  return P, M

def projectpoints(P, X):
  """ Apply full projection matrix P to 3D points X in cartesian coordinates.

  Args:
    P: projection matrix
    X: 3d points in cartesian coordinates

  Returns:
    x: 2d points in cartesian coordinates
  """

  X_hom = cart2hom(X)
  X_pro = P.dot(X_hom)    # 像素坐标系 齐次三维坐标
  x = hom2cart(X_pro)

  return x

def loadpoints():
  """ Load 2D points from obj2d.npy.

  Returns:
    x: np array of points loaded from obj2d.npy
  """

  return np.load(r'data\obj2d.npy')

def loadz():
  """ Load z-coordinates from zs.npy.

  Returns:
    z: np array containing the z-coordinates
  """

  return np.load(r'data\zs.npy')

def invertprojection(L, P2d, z):
  """
  Invert just the projection L of cartesian image coordinates P2d with z-coordinates z.

  Args:
    L: central projection matrix
    P2d: 2d image coordinates of the projected points
    z: z-components of the homogeneous image coordinates

  Returns:
    P3d: 3d cartesian camera coordinates of the points
  """

  P2d_hom = P2d * z
  L = np.delete(L, -1, axis=1)
  P3d = np.linalg.inv(L).dot(np.concatenate((P2d_hom, z), axis=0))

  return P3d

def inverttransformation(M, P3d):
  """ Invert just the model transformation in homogeneous coordinates
  for the 3D points P3d in cartesian coordinates.

  Args:
    M: matrix summarizing the extrinsic transformations
    P3d: 3d points in cartesian coordinates

  Returns:
    X: 3d points after the extrinsic transformations have been reverted
  """

  P3d_hom = cart2hom(P3d)
  X = np.linalg.inv(M).dot(P3d_hom)

  return X

def p3multiplecoice():
  '''
  Change the order of the transformations (translation and rotation).
  Check if they are commutative. Make a comment in your code.
  Return 0, 1 or 2:
  0: The transformations do not commute.
  1: Only rotations commute with each other.
  2: All transformations commute.
  '''

  return 0

if __name__ == "__main__":
    t = np.array([-27.1, -2.9, -3.2])
    principal_point = np.array([8, -10])
    focal_length = 8

    # model transformations
    T = gettranslation(t)
    Ry = getyrotation(135)
    Rx = getxrotation(-30)
    Rz = getzrotation(90)
    print(T)
    print(Ry)
    print(Rx)
    print(Rz)

    K = getcentralprojection(principal_point, focal_length)

    P,M = getfullprojection(T, Rx, Ry, Rz, K)
    # print(P)
    # print(M)
    # print('test')

    points = loadpoints()
    displaypoints2d(points)

    z = loadz()

    Xt = invertprojection(K, points, z)

    Xh = inverttransformation(M, Xt)

    worldpoints = hom2cart(Xh)
    displaypoints3d(worldpoints)

    points2 = projectpoints(P, worldpoints)
    displaypoints2d(points2)

    plt.show()


r"""
Solve Poisson equation in 2D with periodic bcs in one direction
and homogeneous Neumann in the other

    \nabla^2 u = f,

Use Fourier basis for the periodic direction and Shen's Neumann basis for the
non-periodic direction.

The equation to solve is

     (\nabla^2 u, v) = (f, v)

"""
import sys
import os
from sympy import symbols, cos, sin, pi
import numpy as np
from shenfun import inner, div, grad, TestFunction, TrialFunction, \
    TensorProductSpace, FunctionSpace, Array, Function, comm, la, dx, \
    chebyshev

# Collect basis and solver from either Chebyshev or Legendre submodules
assert len(sys.argv) == 3, "Call with two command-line arguments"
assert sys.argv[-1].lower() in ('legendre', 'chebyshev')
assert isinstance(int(sys.argv[-2]), int)

family = sys.argv[-1].lower()
Solver = chebyshev.la.Helmholtz if family == 'chebyshev' else la.SolverGeneric1ND

# Use sympy to compute a rhs, given an analytical solution
x, y = symbols("x,y", real=True)
#ue = (1-x**3)*cos(2*y)
ue = cos(2*pi*x)
fe = -ue.diff(x, 2)-ue.diff(y, 2)

# Size of discretization
N = int(sys.argv[-2])
N = (N, N)
bc = {'left': ('N', ue.diff(x, 1).subs(x, -1)), 'right': ('N', ue.diff(x, 1).subs(x, 1))}
SN = FunctionSpace(N[0], family=family, bc=bc)
K1 = FunctionSpace(N[1], family='F', dtype='d')
T = TensorProductSpace(comm, (SN, K1), axes=(0, 1))
u = TrialFunction(T)
v = TestFunction(T)

# Get f on quad points
fj = Array(T, buffer=fe)

# Compute right hand side of Poisson equation
f_hat = inner(v, fj)

# Get left hand side of Poisson equation
matrices = inner(v, -div(grad(u)))

# Create Helmholtz linear algebra solver
sol = Solver(matrices)

constraint = ((0, dx(Array(T, buffer=ue), weighted=True)/dx(Array(T, val=1), weighted=True)),)

# Solve and transform to real space
u_hat = Function(T).set_boundary_dofs()           # Solution spectral space
u_hat = sol(f_hat, u_hat, constraints=constraint)

uq = T.backward(u_hat).copy()

# Compare with analytical solution
uj = Array(T, buffer=ue)
print(abs(uj-uq).max())
assert np.allclose(uj, uq)

if 'pytest' not in os.environ:
    import matplotlib.pyplot as plt
    plt.figure()
    X = T.local_mesh(True) # With broadcasting=True the shape of X is local_shape, even though the number of datapoints are still the same as in 1D
    plt.contourf(X[0], X[1], uq)
    plt.colorbar()

    plt.figure()
    plt.contourf(X[0], X[1], uj)
    plt.colorbar()

    plt.figure()
    plt.contourf(X[0], X[1], uq-uj)
    plt.colorbar()
    plt.title('Error')

    plt.show()


import numpy as np
import sys

# lambda_input = int(sys.argv[1])
# sigma2_input = float(sys.argv[2])
# X_train = np.genfromtxt(sys.argv[3], delimiter=",")
# y_train = np.genfromtxt(sys.argv[4])
# X_test = np.genfromtxt(sys.argv[5], delimiter=",")

lambda_input = 2
sigma2_input = 3
X_train = np.loadtxt('X_train.csv', delimiter=",")
Y_train = np.loadtxt('y_train.csv', delimiter=",")
X_test = np.loadtxt('X_test.csv', delimiter=",")

def part1(x, y, lbd):
    """
    Solution part 1: Compute the ridge regression weights

    np.linalg.inv:

    Calcula o inverso de uma matriz.
    O inverso de uma matriz eh tal que, se for multiplicado pela matriz original,
    resulta em matriz de identidade.

    np.dot:
    Calcula o produto de duas matrizes

    np.eye:
    Calcula a matriz identidade

    :param lbd: lambda input
    :param x: train set
    :param y: labels train set
    :return: wRR, list with ridge regression wights
    """
    _, d = x.shape

    # inputs x identity matrix from x_train[1] + scalar product form x train matrix
    l_identity = lbd * np.eye(d)
    wrr = np.linalg.inv(l_identity + x.T.dot(x)).dot(x.T).dot(y)
    return wrr

def part2(x_train, x_test, lmbd, sigma):
    """
    Solution for Part 2, implemented  Active Learning
    :return: active,  points chosen using active learning
    """
    _, x = x_train.shape

    x_test_index = list(range(x_test.shape[0]))

    indexes = []

    # Calculate covariance
    covariance = np.linalg.inv(lmbd * np.eye(x) + 1 / sigma * (x_train.T.dot(x_train)))
    print("Cov matrix", covariance.shape)

    for _ in range(10):
        temp_cov = [sigma + x_test[i].dot(covariance).dot(x_test[i].T) for i in x_test_index]
        print("preds", temp_cov[:5])
        print("argmax", np.argmax(temp_cov[:5]))

        pos_max_variance = np.argmax(temp_cov)
        el = x_test_index.pop(pos_max_variance)
        indexes.append(el)

        # update covariance with test values
        covariance = np.linalg.inv(covariance +
                                   sigma * x_test[pos_max_variance].T.dot(x_test[pos_max_variance]))

        last_index = indexes[len(indexes) - 1]

        print("x_test shape:", x_test[last_index].shape)
        print("x_train shape:", x_train.shape)
        x_train = np.concatenate((x_train, x_test[last_index].reshape(1, x)), axis=0)

    return (np.array(indexes) + 1).reshape(1, len(indexes))

if __name__ == '__main__':
    wRR = part1(X_train, Y_train, lambda_input)  # Assuming wRR is returned from the function
    np.savetxt("wRR_" + str(lambda_input) + ".csv", wRR, delimiter="\n")  # write output to file

    active = part2(X_train, X_test, lambda_input,
                   sigma2_input)  # Assuming active is returned from the function
    print("Active", active)
    np.savetxt("active_" + str(lambda_input) + "_" + str(int(sigma2_input)) + ".csv", active,
               delimiter=",")  # write output to file


import numpy as np
A_var = np.array([
    [5, 2, 1],
    [6, 8, 0],
    [2, 5, 8]
    ])
B_var = np.array([
    [5, 3, 2],
    [7, 9, 1],
    [3, 6, 9]
    ])
C_var = np.array([
    [6, 3, 2],
    [8, 0, 2],
    [4, 7, 0]
    ])

x = np.array([5, 2, 1, 6, 8])

x_1 = np.array([[5, 2, 1],
                [6, 8, 8]])

x_2 = np.array([[5, 2, 1],
                [6, 8, 8]])

x_3 = np.array([5, 2, 1])

y = np.array([5, 3, 2, 7, 9])

y_1 = np.array([[5, 3, 2],
                [7, 9, 4]])

y_2 = np.array([[5, 3, 2],
                [7, 9, 4]])

matrix = np.array([
    [1, 2, 0],
    [2, 2, 0],
    [3, 2, 0],
    [4, 2, 0]
    ])

matrix_two = np.array([
    [4, 3, 2, 1],
    [2, 2, 2, 2],
    [0, 0, 0, 0]
    ])

def problem_1a (A, B):
    return A + B

def problem_1b (A, B, C):
    return (np.dot(A, B) - C)

def problem_1c (A, B, C):
    return ((A * B) + C.T)

def problem_1d (x, y):
    return np.inner(x, y)

def problem_1e (A):
    return np.zeros(np.shape(A_var))

def problem_1f (A, x):
    return np.linalg.solve(A, x)

def problem_1g (A, x):
    return (np.linalg.solve(A.T, x.T)).T

def problem_1h (A, alpha):
    num_rows, num_cols = A.shape
    print("This is A \n", A)
    print("\nThis is I \n", np.eye(num_rows))
    return A + (np.eye(num_rows) * alpha)

def problem_1i (A, i):
    return np.sum(A[i, ::2])

def problem_1j (A, c, d):
    b = np.where(np.logical_and(c <= A, A <= d))
    return np.mean(A[b])

def problem_1k (A, k):
    eigen_value, eigen_vector = np.linalg.eig(A) #get eigenvalues and vectors
    descen_arr_indices = eigen_value.argsort() #sort indices in descending order
    matching_indices = descen_arr_indices[len(descen_arr_indices) - k:]
    return eigen_vector[:,matching_indices] #use indices to get respective eigen vectors

def problem_1l (x, k, m, s):
    return np.random.multivariatenormal(x+m, np.eye(x.shape[0]) * s, k).T

def problem_1m (A):
    return np.random.permutation(A)

def problem_1n (x):
    x_bar = np.mean(x)
    sigma = np.std(x)
    y_vec = []
    for i in range(len(x)):
        x_i = x[i]
        y_i =  (x_i - x_bar) / sigma
        y_vec.append(y_i)

    return y_vec

def problem_1o (x, k):
    return np.repeat(np.atleast_2d(x),k)

def problem_1p (X):
    m, n = np.shape(X)
    three_dim_mat = np.repeat(np.atleast_3d(X), n, axis=2)
    swap_mat = np.swapaxes(three_dim_mat, 1, 2)
    D = np.sqrt(np.sum(np.square(three_dim_mat-swap_mat), axis=0))
    return D

def linear_regression (X_tr, y_tr):
    X_t = X_tr.T
    X_y = np.matmul(X_tr.T, y_tr)
    X_XT = np.matmul(X_t, X_tr)
    return np.linalg.solve(X_XT, X_y)

def mean_square_error(y_hat, y):
    dist_diff = np.square(y_hat-y)
    err = np.mean(dist_diff)
    return err

def train_age_regressor():
    # Load data
    X_tr = np.reshape(np.load("age_regression_Xtr.npy"), (-1, 48*48))
    ytr = np.load("age_regression_ytr.npy")
    X_te = np.reshape(np.load("age_regression_Xte.npy"), (-1, 48*48))
    yte = np.load("age_regression_yte.npy")

    w = linear_regression(X_tr, ytr)
    y_hat_tr = np.matmul(X_tr, w)
    y_hat_te = np.matmul(X_te, w)

    train = mean_square_error(y_hat_tr, ytr)
    test = mean_square_error(y_hat_te, yte)

    return train, test


import numpy as np

# Transform Cartesian to polar coordinates
def cart2pol(x, y):
    theta = np.arctan2(y, x)
    rho = np.hypot(x, y)
    return rho, theta

# Converts rad to theta in range 0 - 360
def positive_deg_theta(theta):
    theta = np.rad2deg(theta)
    return np.mod(theta + 360, 360)

# Transform polar to Cartesian coordinates
def pol2cart(rho, theta):
    x = rho * np.cos(theta)
    y = rho * np.sin(theta)
    return x, y


import numpy
import random
import math

def getDirection(sdev, p_cardinal):
	#sdev = 15
	#p_cardinal = .4

	cardinal = numpy.random.binomial(1, p_cardinal) # cardinal axis: up/down vs left/right
	pole = numpy.random.binomial(1, .5) # pol of cardinal axis up vs down, left vs right

	if cardinal==1: #up/down
		if pole==1: #up
			deg_mean = 0
		else: #down
			deg_mean =	180

	else: #left/righrt
		if pole==1: #left
			deg_mean = 270
		else: #right
			deg_mean = 90

	gaze_angle= random.gauss(deg_mean, sdev)
	return gaze_angle

'''
plotting the distribution
pipe output to data.txt
in R

data <- read.table("C:/Users/Ulysses/Desktop/PT/data.txt", quote="\"")
hist(data$V1, breaks=50, main="sdev = 35, p_cardinal=.5")
text(0, y=0, "up")
text(90, y=0, "right")
text(180, y=0, "down")
text(270, y=0, "left")

OR as polar histogram:

library(circular)
data <- read.table("C:/Users/Ulysses/Desktop/PT/data.txt", quote="\"")
rose.diag(circular(data$V1), bins=36, prop=3, zero=pi/2, main="sdev = 15, p_cardinal=.4")

'''


from itertools import combinations

import numpy as np

transition_matrix = np.array([[0.2,0.5,0.2,0.1],
                                [0.3,0.4,0.2,0.1],
                                [0.1,0.3,0.4,0.2],
                                [0.1,0.1,0.3,0.5]])

initial_state = np.array([[0.5, 0.3, 0.2, 0.0]])

def prob_dist_after_n_steps(t_mat, init_mat, num_steps):
    prob_dist = init_mat
    for i in range(num_steps-1):
        prob_dist = np.matmul(prob_dist, t_mat)

    return prob_dist / prob_dist.sum()

def find_count_prob(prob_dist, count):
    if count > len(prob_dist[0]) or count < 0:
        print("Invalid Input")
        raise NotImplementedError("Invalid input.")
    combs = combinations(range(len(prob_dist[0])), count)

    total_prob = 0
    for comb in combs:
        prob = 1
        for i in range(len(prob_dist[0])):
            if i in comb:
                prob *= prob_dist[0,i]
            else:
                prob *= (1 - prob_dist[0,i])
        total_prob += prob
    return total_prob

def main():
    print("Probability of exactly 3 lines being busy after 4 steps:")
    prob_dist = prob_dist_after_n_steps(transition_matrix, initial_state, 4)
    print(prob_dist[0,3])

    print("\n\nProbability distribution after 4 steps:" + str(prob_dist[0]))
    print("Most probable #lines being busy after time step 4: " + str(np.argmax(prob_dist[0])))

if __name__ == "__main__":
    main()


import numpy as np
from matplotlib import pyplot as plt

# Score
# 0 : Fail, 1 : Pass
data = np.array([
    [45, 0],
    [50, 0],
    [55, 0],
    [60, 1],
    [65, 1],
    [70, 1],
])

x = data[:, 0]
y = data[:, 1]

import math

# array (numpy)
def sigmoid(x):  # 시그모이드 함수 정의
    return 1/(1+np.exp(-x))

# x is scalar 1 EA
# def sigmoid(x):  # 시그모이드 함수 정의
#     return 1/(1+math.exp(-x))
x = np.arange(-10.0, 10.0, 0.1)
print(x.shape, x.ndim, x.size)
print(type(x))
# print(x)
y = sigmoid(x)
# plt.scatter(data[:, 0], data[:, 1])
plt.plot(x, y, 'g')
plt.plot([0, 0], [1.0, 0.0], ':')  # 가운데 점선 추가
plt.title('Sigmoid Function')
plt.show()

# W 값에 따른 경사도 변화
x = np.arange(-5.0, 5.0, 0.1)
y1 = sigmoid(0.5*x)
y2 = sigmoid(x)
y3 = sigmoid(2*x)

plt.plot(x, y1, 'r', linestyle='--')  # W의 값이 0.5일때
plt.plot(x, y2, 'g')  # W의 값이 1일때
plt.plot(x, y3, 'b', linestyle='--')  # W의 값이 2일때
plt.plot([0, 0], [1.0, 0.0], ':')  # 가운데 점선 추가
plt.title('Sigmoid Function')
plt.show()

# b 값에 따른 좌, 우 이동
x = np.arange(-5.0, 5.0, 0.1)
y1 = sigmoid(x+0.5)
y2 = sigmoid(x+1)
y3 = sigmoid(x+1.5)

plt.plot(x, y1, 'r', linestyle='--')  # x + 0.5
plt.plot(x, y2, 'g')  # x + 1
plt.plot(x, y3, 'b', linestyle='--')  # x + 1.5
plt.plot([0, 0], [1.0, 0.0], ':')  # 가운데 점선 추가
plt.title('Sigmoid Function')
plt.show()


# Logistic Regression

## Regression and Classification

regression answers how much? or how many? questions, for example:

1. predict the number of dollors at which a house will be sold.
2. predict the revenue of a restaurant.

in practice, we are more often interested in classification: asking not "how much", but "which one":

1. does this email belong in the spam foler or the inbox?
2. does this image depict a donkey, a dog, a cat, or a rooster?
3. which movie is Jean most likely to watch next?

linear regression not fit to solve classification problem for two reasons:

1. linear regression range in $\mathbb{R}$, while classification label is discrete.
2. linear regression uses euclid distance, result in d(class 3, class 1) > d(class 2, class 1), this is often not true in classification.

## Logistic Regression Model

we first foucus on binary classification

suppose dataset $D=\left \{ (x^{(1)},y^{(1)}),...,(x^{(n)},y^{(n)}) \right \} $, where $x^{(i)} \in \mathbb{R}^{d},\ y^{(i)} \in \left \{ 0, 1\right \}$.

to tackle this problem, after deriving $\theta^{T}x$

we uses a function that transform value in $\mathbb{R}$ into value in $[0, 1]$ and view this as the probability of being positive

we choose:

$$\sigma(x) = \frac{1}{1 + e^{-x}}$$

as that function, it is the so called sigmoid function, we choose sigmoid function because:

1. it is a monotony increase, differentiable, symmetric function that maps $\mathbb{R}$ into $[0,1]$.
2. simple form
3. derivative can calculate easily: ${\sigma}'(x) = \sigma(x)(1 - \sigma(x))$

now the model is:

$$h_{\theta}(x) = \frac{1}{1 + exp(-\theta^{T}x)}$$

this is the logistic regression model.

additionally, in our view, we have:

$$h_{\theta}(x) = p(y=1|x)$$

## Entropy

self information $I(x)$ indicates the amount of information an event $x$ to happen.

we want $I(x)$ to satisfy:

1. $I(x) \ge 0$
2. $I(x) = 1 \text{ if }p(x)=1$
3. $\text{if }p(x_{1}) > p(x_{2}) \text{, then } I(x_{1}) < I(x_{2})$
4. $I(x_{1}, x_{2}) = I(x_{1}) + I(x_{2}) \text{ for independent }x_{1},x_{2}$

this leads to $I(x) = -log\ p(x)$

while self-information measures the information of a single discrete event, entropy measures the information of a random variable:

$$
\begin{equation}
\begin{split}
H(X)=&E(I(x))\\
=&E(-log\ p(x))\\
=&-\sum_{x \in \mathcal{X}}log\ p(x)
\end{split}
\end{equation}
$$

it is exactly the optimal encoding length of $X$.

cross entropy $H(p, q)$ is the encoding length of $p$ by optimal encoding of $q$:

$$H(p,q)=E_{p}\left[-log\ q(x)\right] = -\sum_{x}p(x)log\ q(x)$$

fix $p$, the closer $q$ is to $p$, the less is $H(p,q)$.

we can use $H(p,q)$ to define the distance of $q$ to $p$.

turn to our binary classification problem, for $y^{(i)} \in \left\{0,1\right\}, \hat{y}^{(i)} \in (0, 1)$, we have:

$$
H(y^{(i)}, \hat{y}^{(i)}) =
\begin{cases}
-log(\hat{y}^{(i)}),\text{ if } y^{(i)} = 1\\
-log(1 - \hat{y}^{(i)}),\text{ if } y^{(i)} = 0
\end{cases}
$$

combine the two cases:

$$H(y^{(i)}, \hat{y}^{(i)}) = -y^{(i)}log(\hat{y}^{(i)}) - (1-y^{(i)})log(1 - \hat{y}^{(i)})$$

we use cross entropy to define the loss of logistic regression model:

$$
\begin{equation}
\begin{split}
J(\theta) =& \sum_{i=1}^{n}H(y^{(i)}, H(\hat{y}^{(i)})) \\
=& \sum_{i=1}^{n}-y^{(i)}log(\hat{y}^{(i)}) - (1-y^{(i)})log(1 - \hat{y}^{(i)}) \\
=& \sum_{i=1}^{n}-y^{(i)}log(h_{\theta}(x^{(i)})) - (1 - y^{(i)})log(1 - h_{\theta}(x^{(i)}))
\end{split}
\end{equation}
$$

this is the so called logistic regression.

in addition, we can write cross entropy loss in matrix form:

$$J(\theta) = -y^{T}log(\sigma(\theta^{T}X)) - (1 - y)^{T}log(1 - \sigma(\theta^{T}X))$$

import numpy as np
from sklearn.datasets import load_iris
from sklearn.model_selection import train_test_split

iris = load_iris()
X = iris["data"][:, 2:]
y = (iris["target"] == 2).astype(np.int)

X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=1)

from sklearn.linear_model import LogisticRegression

log_reg = LogisticRegression(solver="lbfgs", random_state=42)
log_reg.fit(X_train, y_train)

from sklearn.metrics import accuracy_score

y_pred = log_reg.predict(X_test)
accuracy_score(y_test, y_pred)

## Probability Interpretation of Cross Entropy Loss

as we suppose

$$p(y=1|x) = h_{\theta}(x)$$

and

$$p(y=0|x) = 1 - h_{\theta}(x)$$

combine these two, in our prediction:

$$p(y^{(i)}|x^{(i)}) = \left [ h_{\theta}(x^{(i)}) \right ]^{y^{(i)}}\left [ 1 - h_{\theta}(x^{(i)}) \right ]^{1 - y^{(i)}} $$

log likelikhood function:

$$
\begin{equation}
\begin{split}
L(\theta) &= log\prod_{i=1}^{n} \left [ h_{\theta}(x^{(i)}) \right ]^{y^{(i)}}\left [ 1 - h_{\theta}(x^{(i)}) \right ]^{1 - y^{(i)}} \\
&= \sum_{i=1}^{n}\left [y^{(i)}log\ h_{\theta}(x^{(i)}) + (1 - y^{(i)})log(1 - h_{\theta}(x^{(i)})) \right ]
\end{split}
\end{equation}
$$

maximum the above likelihood is equal to minimize the cross entropy loss:

$$J(\theta) = \sum_{i=1}^{n}-y^{(i)}log(h_{\theta}(x^{(i)})) - (1 - y^{(i)})log(1 - h_{\theta}(x^{(i)}))$$

## Update-rule

we have:

$$
\begin{equation}
\begin{split}
\frac{\partial }{\partial \theta_{j}}J(\theta ) &= \frac{\partial }{\partial \theta_{j}}\sum_{i=1}^{n}-log(\sigma(\theta^{T}x^{(i)}))y^{(i)} - log(1 - \sigma(\theta^{T}x^{(i)}))(1 - y^{(i)}) \\
&= \sum_{i=1}^{n} \left (-y^{(i)}\frac{1}{\sigma(\theta^{T}x^{(i)})} + (1 - y^{(i)})\frac{1}{1 - \sigma(\theta^{T}x^{(i)})} \right )\frac{\partial }{\partial \theta_{j}}\sigma(\theta^{T}x^{(i)})\\
&=\sum_{i=1}^{n} \left (-y^{(i)}\frac{1}{\sigma(\theta^{T}x^{(i)})} + (1 - y^{(i)})\frac{1}{1 - \sigma(\theta^{T}x^{(i)})} \right )\sigma(\theta^{T}x^{(i)})(1-\sigma(\theta^{T}x^{(i)}))\frac{\partial }{\partial \theta_{j}}\theta^{T}x^{(i)} \\
&=\sum_{i=1}^{n}(\sigma(\theta^{T}x^{(i)}) - y^{(i)})x_{j}^{(i)} \\
&=\sum_{i=1}^{n}(h_{\theta}(x^{(i)}) - y^{(i)})x_{j}^{(i)}
\end{split}
\end{equation}
$$

the same form with linear regression!

as linear regression, we have the update rule for logistic regression:

$$\theta_{j}: =\theta_{j} - \alpha\sum_{i=1}^{n} (h_{\theta }(x^{(i)}) - y^{(i)})x_{j}^{(i)} $$

combine all dimensions, we have:

$$\theta: =\theta - \alpha\sum_{i=1}^{n} (h_{\theta }(x^{(i)}) - y^{(i)})\cdot x^{(i)} $$

write in matrix form：
$$
\frac{\partial }{\partial \theta}J(\theta ) = X^{T}(\sigma(X\theta) -y)
$$
matrix form of update formula：
$$
\theta: =\theta - \alpha X^{T}(\sigma(X\theta)-\mathbf{y} )
$$

## Regularization

like linear regression, we add penalty term on $J(\theta)$ for regularization

$l2$ penalty：
$$J(\theta) := J(\theta) + \lambda \left \| \theta \right \|_{2}^{2} $$

$l1$ penalty：
$$J(\theta) := J(\theta) + \lambda \left \| \theta \right \|_{1} $$

## Softmax Regression

now we turn to multi-class classification.

we start off with a simple image classification problem, each input consists of a $2\times{2}$ grayscale image, represent each pixel with a scalar, giving us features $\left\{x_{1},x_{2},x_{3}, x_{4}\right\}$. assume each image belong to one among the categories "cat", "chiken" and "dog".

we have a nice way to represent categorical data: the one-hot encoding, i.e a vector with as many components as we have categories, the component corresponding to particular instance's category is 1 and all others are 0.

for our problem, "cat" represents by $(1,0,0)$, "chicken" by $(0, 1, 0)$, "dog" by $(0, 0, 1)$.

to estimate the conditional probabilities of all classes, we need a model with multiple outputs, one per class.

address classification with linear models, we will need as many affine functions as we have outputs:

$$o_{1} = x_{1}w_{11} + x_{2}w_{12} + x_{3}w_{13} + x_{4}w_{14}$$
$$o_{2} = x_{1}w_{21} + x_{2}w_{22} + x_{3}w_{23} + x_{4}w_{24}$$
$$o_{3} = x_{1}w_{31} + x_{2}w_{32} + x_{3}w_{33} + x_{4}w_{34}$$

depict as:

![jupyter](./images/softmaxreg.svg)

we would like output $\hat{y_{j}}$ to be interpreted as probability that a given item belong to class $j$.

to transform our current outputs $\left\{o_{1},o_{2},o_{3},o_{4}\right\}$ to probability distribution $\left\{\hat{y}_{1},\hat{y}_{2},\hat{y}_{3},\hat{y}_{4}\right\}$, we use the softmax operation:

$$\hat{y}_{j} = \frac{exp(o_{j})}{\sum_{k}exp(o_{k})}$$

when predicting:

$$\text{predict class} = \underset{j}{argmax}\ \hat{y}_{j} = \underset{j}\ o_{j}$$

$\hat{y}_{j}$ is necessary when compute loss.

as logistic regression, we use the cross entropy loss:

$$H(y,\hat{y}) = -\sum_{k}y_{j}log\ \hat{y}_{j} = -log\ \hat{y}_{\text{category of y}}$$

now complete the construction of softmax regression.

y = iris["target"]
X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=1)

softmax_reg = LogisticRegression(multi_class="multinomial", solver="lbfgs", C=10)
softmax_reg.fit(X_train, y_train)

y_pred = softmax_reg.predict(X_test)
accuracy_score(y_test, y_pred)


"""
Created on Fri Feb 29 14:33:17 2015
@author: open risk
Purpose: Concentration metrics library
Version: 0.1
"""
import numpy as np

# calculate total exposure
def exposure(portfolio):
    return np.sum(portfolio)

# calculate portfolio exposure weights
def weights(portfolio):
    return np.true_divide(portfolio,exposure(portfolio))

# calculate the concentration ratio
def cr(weights, n):
    return weights[:n].sum()

# calculate the normalized hhi concentration index
def hhi(weights):
    n = weights.size
    h = np.square(weights).sum()
    return (h - 1.0 / n) / (1.0 - 1.0 / n)

# calculate the inverted Hannah Kay index
def hk(weights, a):
    n = weights.size
    h1 = np.power(weights,a).sum()
    h2 = np.power(h1, 1.0/(a-1.0))
    return (h2 - 1.0 / n) / (1.0 - 1.0 / n)

# calculate the Gini index
def gini(weights):
    n = weights.size
    i = np.arange(1,n+1)
    return (1.0 - 2.0 * np.multiply(i,weights).sum())/n + 1.0

# calculate the Shannon entropy index
def shannon(weights):
    weights_nz = weights[weights!=0]
    n = weights_nz.size
    logweights = np.log(weights_nz)
    h = - np.multiply(weights_nz,logweights).sum()
    return 1.0 - h / np.log(n)


import scipy.integrate as spi
import numpy as np

print('Single integral computed by SciPy cumulative_trapezoid')
print('Example 1-01 cumulative_trapezoid')
print('Integral of 2xe^-x from x=1 to x=5')

integrand = lambda x : 2 * x * np.exp(-x)
a = 1.
b = 5.
step = 1e-4

xs = np.arange(a, b, step)
ys = integrand(xs)

result = spi.cumulative_trapezoid(ys, xs)
result = result[-1]
print('Result is ', result)


import numpy as np

def gen_fft_twiddles_lut(Nfft, Inverse=False, rad4=False):
    Phi = (np.pi * 2 / Nfft) * np.arange(0, Nfft)
    if Inverse:
        Twiddles_real = np.cos(Phi)
        Twiddles_imag = np.sin(Phi)
    else:
        Twiddles_real = np.cos(-Phi)
        Twiddles_imag = np.sin(-Phi)
    if rad4:
        twid = np.empty((2 * int(3*Nfft/4), ), dtype=Twiddles_real.dtype)
        twid[0::2] = Twiddles_real[:int(3*Nfft/4)]
        twid[1::2] = Twiddles_imag[:int(3*Nfft/4)]
        return twid
    twid = np.empty((2 * int(Nfft//2), ), dtype=Twiddles_real.dtype)
    twid[0::2] = Twiddles_real[:int(Nfft//2)]
    twid[1::2] = Twiddles_imag[:int(Nfft//2)]
    return twid

def gen_rfft_twiddles_lut(Nfft):
    Phi = (np.pi * 2 / Nfft) * np.arange(0, Nfft//2)
    Twiddles_real = np.sin(Phi)
    Twiddles_imag = np.cos(Phi)
    twid = np.empty((Twiddles_real.size + Twiddles_imag.size, ), dtype=Twiddles_real.dtype)
    twid[0::2] = Twiddles_real
    twid[1::2] = Twiddles_imag
    return twid

def gen_fft_swaptable_lut(Ni, rad4=False):
    if rad4:
        log4 = int(np.log2(Ni) / 2)
        iL = Ni / 4
        iM = 1
        SwapTable = np.zeros(Ni)
        for i in range(log4):
            for j in range(iM):
                SwapTable[j +   iM] = SwapTable[j] +   iL
                SwapTable[j + 2*iM] = SwapTable[j] + 2*iL
                SwapTable[j + 3*iM] = SwapTable[j] + 3*iL
            iL /= 4
            iM *= 4
    else:
        log2 = int(np.log2(Ni))
        iL = Ni / 2
        iM = 1
        SwapTable = np.zeros(Ni)
        for i in range(log2):
            for j in range(iM):
                SwapTable[j + iM] = SwapTable[j] + iL
            iL /= 2
            iM *= 2
    return SwapTable

def gen_dct_twiddles_lut(Ndct, dct_type=2, norm=None):
    norm_factor = np.ones((Ndct, Ndct))
    if norm == "ortho" and dct_type == 2:
        norm_factor   *= np.sqrt(1/(2*Ndct))
        norm_factor[0] = np.sqrt(1/(4*Ndct))
    if norm == "ortho" and dct_type == 3:
        norm_factor   *= np.sqrt(1/(2*Ndct))
        norm_factor[0] = np.sqrt(1/(Ndct))
    DCT_Coeff = np.zeros((Ndct, Ndct))
    for k in range(Ndct):
        for i in range(Ndct):
            if dct_type == 2:
                coeff = 2*np.cos(np.pi / (2*Ndct) * k * (2*i + 1))
            elif dct_type == 3:
                coeff = 1 if i == 0 else 2*np.cos(np.pi / (2*Ndct) * i * (2*k + 1))
            else:
                raise NotImplementedError("DCT type 2 and 3 only supported")
            DCT_Coeff[k, i] = coeff

    return (DCT_Coeff * norm_factor)


# -*- coding: utf-8 -*-
"""
Implementing Linear Regression from scratch
(okay, not exactly scatch, using Numpy)

Date: 2 June 2020 12:52 PM
Author: Gurpinder Singh
"""
import numpy as np
import matplotlib.pyplot as plt

#Read the data
x = []
y = []
with open("ex1data1.txt") as data:
    lines = data.readlines()
for line in lines:
    line = line.replace("\n", "")
    x.append([float(line.split(",")[0])])
    y.append([float(line.split(",")[1])])

#Make X vector, Y vector out of them
ones = [[1] for i in range(len(x))]
X = np.array(x)
Y = np.array(y)
X = np.append(ones, X, axis=1)
theta = np.array([[0], [0]])

#implement cost function
def cost_function(X, Y, theta):
    """
    Cost function for particular theta values for X and Y
    """
    h_theta = X.dot(theta)
    SSE = np.square(h_theta-Y)
    MSE = sum(SSE)/(2*len(Y))
    return MSE

#implement gradient descent
def gradient_descent(theta, X, Y, alpha):
    """
    Implementing one step of gradient descent algorithm.
    """
    Err = X.dot(theta) - y
    inside_term = Err.transpose().dot(X)
    theta = theta - ((alpha/len(Y))*inside_term.transpose())
    return theta

alpha = 0.01
J = []
Lines = []
for j in [10, 100, 500, 1000, 1500, 3000]:
    for i in range(j):
        J.append(cost_function(X, Y, theta))
        theta = gradient_descent(theta, X, Y, alpha)

    Lines.append([theta[0] + theta[1]*j for j in np.linspace(1, 30, num= 50)])

    #data visualization
plt.figure()
plt.plot(X[:,1].transpose(), Y[:, 0].transpose(), "rx", label = "Data points")
for Line, j in zip(Lines, [10, 100, 500, 1000, 1500, 3000]):
    plt.plot(Line, "-", label = "{} interations".format(j))

plt.title("Visualizing Gradient Descent Fit over multiple iterations")
plt.xlabel("Population Size (in 10,000)")
plt.ylabel("Profits (in $10,000)")
plt.legend(loc = "best")
plt.savefig("linear_regression.jpg")


# Coding the Softmax function

## used to compute the probability of each class
## when the classification problem involves more than 2 classes
## when 2 classes: if discrete => step function / if continuous => sigmoid function

import numpy as np

# Write a function that takes as input a list of numbers, and returns
# the list of values given by the softmax function.
def softmax(L):

    results_list = []

    denominator_sum = 0
    for i in range(len(L)):
        denominator_sum += np.exp(L[i])

    for score in L:
        result_score = np.exp(score) / denominator_sum

        results_list.append(result_score)

    return results_list

Neural Networks/Lesson 1 - Intro to neural networks/Coding the Softmax function/softmax.py


#------------------------------------------------------------------------------------------------------------#
#Chapter 2 - Visualization with hierarchical clustering and t-SNE

#------------------------------------------------------------------------------------------------------------#
#Hierarchical clustering of the grain data

# Perform the necessary imports
from scipy.cluster.hierarchy import linkage, dendrogram
import matplotlib.pyplot as plt

# Calculate the linkage: mergings
mergings = linkage(samples, method='complete')

# Plot the dendrogram, using varieties as labels
dendrogram(mergings,
           labels=varieties,
           leaf_rotation=90,
           leaf_font_size=6,
)
plt.show()

#------------------------------------------------------------------------------------------------------------#

#Hierarchies of stocks
# Import normalize
from sklearn.preprocessing import normalize

# Normalize the movements: normalized_movements
normalized_movements = normalize(movements)

# Calculate the linkage: mergings
mergings = linkage(normalized_movements, method='complete')

# Plot the dendrogram
dendrogram(mergings,
           labels=companies,
           leaf_rotation=90,
           leaf_font_size=6,
)

plt.show()

#------------------------------------------------------------------------------------------------------------#
#Different linkage, different hierarchical clustering!
# Perform the necessary imports
import matplotlib.pyplot as plt
from scipy.cluster.hierarchy import dendrogram, linkage

# Calculate the linkage: mergings
mergings = linkage(samples,method='single')

# Plot the dendrogram
dendrogram(mergings,
           labels=country_names,
           leaf_rotation=90,
           leaf_font_size=6,
)
plt.xlabel('European Nations')
plt.ylabel('Number of votes')
plt.title('EuroVision 2017 Denogram ')
plt.legend(loc='upper right')
plt.show()

#------------------------------------------------------------------------------------------------------------#

#Extracting the cluster labels
# Perform the necessary imports
import pandas as pd
from scipy.cluster.hierarchy import fcluster

# Use fcluster to extract labels: labels
labels = fcluster(mergings,6,criterion='distance')

# Create a DataFrame with labels and varieties as columns: df
df = pd.DataFrame({'labels': labels, 'varieties': varieties})

# Create crosstab: ct
ct = pd.crosstab(df['labels'],df['varieties'])

# Display ct
print(ct)

#------------------------------------------------------------------------------------------------------------#
#t-SNE visualization of grain dataset
# Import TSNE
from sklearn.manifold import TSNE

# Create a TSNE instance: model
model = TSNE(learning_rate=200)

# Apply fit_transform to samples: tsne_features
tsne_features = model.fit_transform(samples)

# Select the 0th feature: xs
xs = tsne_features[:,0]

# Select the 1st feature: ys
ys = tsne_features[:,1]

# Scatter plot, coloring by variety_numbers
plt.scatter(xs,ys,c=variety_numbers)
plt.show()

#------------------------------------------------------------------------------------------------------------#
#A t-SNE map of the stock market

# Import TSNE
from sklearn.manifold import TSNE

# Create a TSNE instance: model
model = TSNE(learning_rate=50)

# Apply fit_transform to normalized_movements: tsne_features
tsne_features = model.fit_transform(normalized_movements)

# Select the 0th feature: xs
xs = tsne_features[:,0]

# Select the 1th feature: ys
ys = tsne_features[:,1]

# Scatter plot
plt.scatter(xs,ys,alpha=0.5)

# Annotate the points
for x, y, company in zip(xs, ys, companies):
    plt.annotate(company, (x, y), fontsize=5, alpha=0.75)
plt.show()

#------------------------------------------------------------------------------------------------------------#

#------------------------------------------------------------------------------------------------------------#

#------------------------------------------------------------------------------------------------------------#

#------------------------------------------------------------------------------------------------------------#

#------------------------------------------------------------------------------------------------------------#

#------------------------------------------------------------------------------------------------------------#

#------------------------------------------------------------------------------------------------------------#

#------------------------------------------------------------------------------------------------------------#

#------------------------------------------------------------------------------------------------------------#

#------------------------------------------------------------------------------------------------------------#

18.Unsupervised Learning in Python/Chapter 2 - Visualization with hierarchical clustering and t-SNE.py


# Code by Desh Iyer

# ----------------------------------------------------------------

# PROBLEM STATEMENT

# Create a dataset of a single variable comprising of only numerical values. Create a Normal distribution for it. Convert it to Z distribution and plot both of the distribution.

# ----------------------------------------------------------------

# TODO

# [X] - Create a dataset of a single variable comprising of only numerical values.
# [X] - Create a Normal distribution for it.
# [X] - Convert it to Z distribution
# [X] - Plot both of the distribution.

# ----------------------------------------------------------------

# SOLUTION

# Import libs
import matplotlib.pyplot as plt
import numpy as np

# Define a function to calculate the respective Z-Score for every data point
def calcZScore(x, mean, std):
    return ((x - mean) / std)

# Take user input for number of samples
numberSamples = int(input("Enter number of samples: "))

# Generate a normal distribution with arbitrary mean 5 and std 2
normallyDistributedData = np.random.normal(5, 2, numberSamples)

# Calculate summary stats
normalMean = np.mean(normallyDistributedData)
normalStd = np.std(normallyDistributedData)

# Convert the above generated normally distributed data to Z-distributed data using list comprehension
zDistributedData = [calcZScore(i, normalMean, normalStd) for i in normallyDistributedData]

# Plot two graphs

plt.title("Normal distribution w/ Mean: 5 & Std. Deviation: 2")
plt.xlabel('x')
plt.ylabel('Normal Distribution')
plt.grid()
plt.hist(normallyDistributedData)
plt.show()

plt.title("Z - Distribution for above graph")
plt.xlabel('x')
plt.ylabel('Standard Normal / Z - Distribution')
plt.grid()
plt.hist(zDistributedData)
plt.show()


import numpy as np

class Bohachevsky:

    def __init__(self, bounds=None):

        if bounds == None:
            self.bounds = [[-1, 1]] * 2
        self.min = [0, -0.24]
        self.fmin = -0.24

    def f(self, x):
        '''
        :param x: 一维数组
        :return:  函数值
        '''

        x1 = x[0]
        x2 = x[1]

        f_value = x1 ** 2 + x2 ** 2 - 0.3 * np.cos(3 * np.pi * x1) + 0.3 * np. cos(4 * np.pi * x2) + 0.3
        return f_value

class branin:
    '''
    Branin function

    :param bounds: the box constraints to define the domain in which the function is optimized.
    :param sd: standard deviation, to generate noisy evaluations of the function.
    '''

    def __init__(self, bounds=None, a=None, b=None, c=None, d=None, e=None, r=None):
        self.input_dim = 2
        if bounds == None:
            self.bounds = [[-5, 10], [0, 15]]

        if a == None: self.a = 1
        else: self.a = a
        if b == None: self.b = 5.1/(4*np.pi**2)
        else: self.b = b
        if c == None: self.c = 5/np.pi
        else: self.c = c
        if d == None: self.d = 6
        else: self.d = d
        if e == None: self.e = 10
        else: self.e = e
        if r == None: self.r = 1 / (8 * np.pi)
        else: self.r = r

        self.min = [(-np.pi, 12.275), (np.pi, 2.275), (9.42478, 2.475)]
        self.fmin = 0.397887
        self.name = 'Branin'

    def f(self, Y):

        X = np.array(Y).reshape(1, self.input_dim)

        x1 = X[:, 0]
        x2 = X[:, 1]

        fval = self.a * (x2 - self.b * x1 ** 2 + self.c * x1 - self.d) ** 2 + self.e * (1 - self.r) * np.cos(
            x1) + self.e
        return fval[0]


import os
import math
import numpy as np

def part_one_helper(angle, change):
    change = math.radians(change)
    angle = angle + change

    if angle > 2 * math.pi:
        angle = angle - (2 * math.pi)
    elif angle < 0:
        angle = angle + (2 * math.pi)

    return angle

def part_one(entries):
    # Ship properties
    x, y = (0, 0)
    a = math.pi / 2

    # For each instruction
    for instr, value in entries:
        # Switch on instruction
        if instr == 'N':
            y = y + value
        elif instr == 'S':
            y = y - value
        elif instr == 'E':
            x = x + value
        elif instr == 'W':
            x = x - value
        elif instr == 'L':
            a = part_one_helper(a, -1 * value)
        elif instr == 'R':
            a = part_one_helper(a, value)
        elif instr == 'F':
            x = x + round((math.sin(a) * value), 3)
            y = y + round((math.cos(a) * value), 3)

    return int(abs(x) + abs(y))

def part_two_helper(x, y, change):
    return part_one_helper(math.atan2(x, y), change)

def part_two(entries):
    # Ship properties
    x, y = (0, 0)
    wx, wy = (10, 1)

    # For each instruction
    for instr, value in entries:
        # Switch on instruction
        if instr == 'N':
            wy = wy + value
        elif instr == 'S':
            wy = wy - value
        elif instr == 'E':
            wx = wx + value
        elif instr == 'W':
            wx = wx - value
        elif instr == 'L':
            wa = part_two_helper(wx, wy, -1 * value)
            radius = math.sqrt(wx ** 2 + wy ** 2)
            wx = round(math.sin(wa) * radius, 3)
            wy = round(math.cos(wa) * radius, 3)
        elif instr == 'R':
            wa = part_two_helper(wx, wy, value)
            radius = math.sqrt(wx ** 2 + wy ** 2)
            wx = round(math.sin(wa) * radius, 3)
            wy = round(math.cos(wa) * radius, 3)
        elif instr == 'F':
            for _ in range(value):
                x = x + wx
                y = y + wy

    return int(abs(x) + abs(y))

if __name__ == "__main__":
    # Get input from txt file
    with open(os.getcwd() + '\\2020\\Day 12\\input.txt', 'r') as file_obj:
        file_input = file_obj.readlines()

    # Clean input
    entries = []
    for entry in file_input:
        entries.append((
            entry[0],
            int(entry[1:len(entry)].rstrip())
        ))

    # Part one
    print(part_one(entries))

    # Part two
    print(part_two(entries))


import math
import numpy as np
def interest1(b, p, n):
    """
    INTEREST1(b, p, n) computes the new balance after n years for an initial
    balance b and an annual interest rate p in per cent
    """
    return b*(1 + p/100)**n

def gauss3(x, mu, sigma):
    """
    GAUSS3 Evaluates Gaussian normal distribution with mean mu and standard
    deviation sigma at x
    """
    return math.exp(-1/2*((x - mu)/sigma)**2)/(sigma*math.sqrt(2*math.pi))

def gauss4(x_arr, mu, sigma):
    """
    GAUSS4 Evaluates Gaussian normal distribution with mean mu and standard
    deviation sigma at x, where x may be an array
    """
    fd = []
    for x in x_arr:
            a=1/2*((x - mu)/sigma)**2
            e=math.exp(-a)
            fd.append((1/(sigma*math.sqrt(2*math.pi))) * e)
    return np.array(fd)

def gauss5(x, mu, sigma):
    """
    GAUSS5 Evaluates Gaussian normal distribution with mean mu and standard
    deviation sigma at x, where x may be an array
    """
    return math.exp(-1/2*(np.linalg.matrix_power((x - mu)/sigma),2))/(sigma*math.sqrt(2*math.pi))

def angle(a, b):
    """
    ANGLE computes the angle between two vectors a and b in R^2
    """
    adotb = np.sum(a*b)            # or np.dot(a,b)
    norma = np.sqrt(np.sum(a*a));  # or np.linalg.norm(a)
    normb = np.sqrt(np.sum(b*b));  # or np.linalg.norm(b)
    return np.arccos(adotb/(norma*normb));


# Copyright 2017 Battelle Energy Alliance, LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# from https://en.wikipedia.org/wiki/Test_functions_for_optimization
#
# takes input parameters x,y
# returns value in "ans"
# optimal minimum at f(512,404.2319) = -959.6407
# parameter range is -512 <= x,y <= 512
import math
import numpy as np
def evaluate(x,y):
  """
    evaluates the Eggholder function at each given x,y.
    @ In, x, float, first input dimention.
    @ In, y, float, second input dimention.
    @ Out, evaluate, float, f(x,y)
  """
  return (-(y + 47) * np.sin(np.sqrt(abs(x/2 + (y  + 47))))
          -x * np.sin(np.sqrt(abs(x - (y  + 47)))))

def run(self,Inputs):
  """
    RAVEN API
    @ In, self, object, RAVEN container
    @ In, Inputs, dict, additional inputs
    @ Out, None
  """
  self.ans = evaluate(self.x,self.y)

if __name__ == '__main__':
  import sys
  x = float(sys.argv[1])
  y = float(sys.argv[2])
  print(evaluate(x, y))


"""
Example: Cafe Java Demand Distribution and Process Generator.

This example develops a discrete process generator for coffee demand data
observed in Cafe Java.

@author: Paul T. Grogan <pgrogan@stevens.edu>
"""

# import the python3 behavior for importing, division, and printing in python2
from __future__ import absolute_import, division, print_function

# import the numpy package and refer to it as `np`
# see http://docs.scipy.org/doc/numpy/reference/ for documentation
import numpy as np

# import the matplotlib pyplot package and refer to it as `plt`
# see http://matplotlib.org/api/pyplot_api.html for documentation
import matplotlib.pyplot as plt

# create a numpy array with the demand sizes
demands = np.array([0, 1, 2, 3])
labels = ["None", "Small", "Medium", "Large"]
print("demands = {}".format(demands))

# create a numpy array with the observed frequencies
frequency = np.array([8, 10, 22, 10])

# the probability mass function is the number of demands (frequency) divided
# by the total number of demands (sub of all frequency)
pmf = frequency/np.sum(frequency)
print("pmf = {}".format(pmf))

# create a new figure for a bar plot using the demand indices, pmf values,
# desired bar width, and color
plt.figure()
bar_width = 0.5
plt.bar(demands, pmf, bar_width, align='center', color='b')
plt.ylabel('p(x)')
plt.ylim([0,1])
plt.xlabel('Coffee Demand (x)')
plt.title('PMF for Cafe Java Demand')
plt.xticks(demands, labels)

# the cumulative distribution function is the cumulative sum of the PDF
cdf = np.cumsum(pmf)
print("cdf = {}".format(cdf))

# create a new figure for a step plot using the demand indices, cdf values,
# desired format (solid red line, -r), and option for post-steps
plt.figure()
plt.step(demands, cdf, '-r', where='post')
plt.ylabel('F(x)')
plt.ylim([0,1])
plt.xlabel('Coffee Demand (x)')
plt.title('CDF for Cafe Java Demand')
plt.xticks(demands, labels)

# define a function to generate demands following the accept-reject method
def generate_demand_arm():
    """Generates a demand following the accept-reject method.

    Returns:
        demand (int): the size of coffee demanded
    """
    # create a CDF for alternatives
    alternatives = [0.25, 0.50, 0.75, 1.00]
    # store the accepted demand (-1 until accepted)
    accepted = -1
    # perform accept-reject while no demand is yet accepted
    while accepted < 0:
        # generate random U(0,1) samples
        r_1 = np.random.rand()
        r_2 = np.random.rand()
        # perform check for alternative 0
        if r_1 <= alternatives[0]:
            # check if second sample is less than PMF value
            if r_2 <= pmf[0]:
                accepted = 0
        # perform check for alternative 1
        elif r_1 <= alternatives[1]:
            # check if second sample is less than PMF value
            if r_2 <= pmf[1]:
                accepted = 1
        # perform check for alternative 2
        elif r_1 <= alternatives[2]:
            # check if second sample is less than PMF value
            if r_2 <= pmf[2]:
                accepted = 2
        # otherwise no need to check, it's the final CDF entry (alternative 3)
        else:
            # check if second sample is less than PMF value
            if r_2 <= pmf[3]:
                accepted = 3
        """
        note: the code above could be replaced with the following for
        loop which iterates over each array index i in demands

        r_1 = np.random.rand()
        r_2 = np.random.rand()
        for i in range(len(demands)):
            if r_1 <= alternatives[i]:
                if r_2 <= pmf[i]:
                    accepted = i
        """
    return accepted

# define number of samples
num_samples = 1000

# fill the samples arrays with samples from the generators
samples_arm = [generate_demand_arm() for i in range(num_samples)]

# count the number of each demand: use a generator expression to count the
# number of samples which match each demand level i
counts_arm = np.array([sum(samples_arm==i) for i in demands])
frequency_arm = counts_arm/np.sum(counts_arm)

# create a new figure recreating the PMF bar plot
plt.figure()
bar_width = 0.3
plt.bar(demands, pmf, bar_width, align='center', color='k', label='Observed')
# also display the generated frequency (offset bars by bar_width)
plt.bar(demands+bar_width, frequency_arm,
        bar_width, color='g', label='Generated (ARM)')
plt.ylabel('p(x)')
plt.ylim([0,1])
plt.xlabel('Coffee Demand (x)')
plt.title('Cafe Java Demand Process Generator Results (n={})'.format(num_samples))
plt.xticks(demands + bar_width, labels)
plt.legend()


%matplotlib inline
import numpy as np
import matplotlib.pyplot as plt

def function_for_roots(x):
    a = 1.01
    b = -3.04
    c = 2.07
    return a*x**2 + b*x + c

def check_initial_values(f, x_min, x_max, tol):
    y_min = f(x_min)
    y_max = f(x_max)

    if ((y_min*y_max)>=0):
        print('no zero crossing in this range', x_min, x_max)
        s = 'f(f%) = %f, f(f%) = %f' % (x_min, y_min, x_min, x_max)
        print(s)
        return 0

    if(np.fabs(y_min)< tol):
        return 1
    if (np.fabs(y_max)< tol):
        return 2
    return 3

def bisection_root_finding(f, x_min_start, x_max_start, tol):
    x_min = x_min_start
    x_max = x_max_start
    x_mid = 0.0

    y_min = f(x_min)
    y_max = f(x_max)
    y_mid = 0.0

    i_max = 10000
    i = 0

    flag = check_initial_values(f, x_min, x_max, tol)
    if (flag == 0):
        print('Error in bisection_root_finding().')
        raise ValueError('Initial values invalid', x_min, x_max)
    elif(flag == 1):
        return(x_min)
    elif(flag == 2):
        return(x_max)

    flag = 1

    while(flag):
        x_mid = .5*(x_min+x_max)
        y_mid = f(x_mid)

        if (np.fabs(y_mid) < tol):
            flag = 0
        else:
            if (f(x_mid)*(f(x_min))>0):
                x_min = x_mid
            else:
                x_max = x_mid

        print(x_min, f(x_min), x_max, f(x_max))

        i += 1

        if(i>=i_max):
            print("exceeded max number of iterations = ", i)
            s = "Min bracket f(%f) = %f" % (x_min, f(x_min))
            print(s)
            s = "Max bracket f(%f) = %f" % (x_max, f(x_max))
            print(s)
            s = "Mid bracket f(%f) = %f" % (x_mid, f(x_mid))
            print(s)
            raise StopIteration("Stopping iterations after ", i)
        print('iteration number = ' + str(i))
    return x_mid

#Root 1
x_min = 0.0
x_max = 1.5
tolerance = 1.0e-6

print(x_min, function_for_roots(x_min))
print(x_max, function_for_roots(x_max))

x_root = bisection_root_finding(function_for_roots, x_min, x_max, tolerance)
y_root = function_for_roots(x_root)

s = 'Root found with y(%f) = %f' % (x_root, y_root)
print(s)

#Root 2
x_min = 1.5
x_max = 3.0
tolerance = 1.0e-6

print(x_min, function_for_roots(x_min))
print(x_max, function_for_roots(x_max))

x_root = bisection_root_finding(function_for_roots, x_min, x_max, tolerance)
y_root = function_for_roots(x_root)

s = 'Root found with y(%f) = %f' % (x_root, y_root)
print(s)

x = np.linspace(0, 3, 1000)

points = [0.0, 1.5, 3.0]
roots = [1.040869, 1.969032]

plt.ylim(-0.5, 2.1)
plt.xlim(0, 3)

y = 1.01 * x ** 2 - 3.04 * x + 2.07
plt.plot(x, y)
plt.plot(x, 0 * x)

plt.scatter(points, [function_for_roots(x) for x in points], color='red')
plt.scatter(roots, [function_for_roots(x) for x in roots], color='green')

plt.ylabel('f(x)')
plt.xlabel('x')

plt.show()


import numpy as np
import matplotlib.pyplot as plt

dataset = np.load('housingprices.npy')
plt.plot(dataset[:,0],dataset[:,1],'o')
plt.show()

#simple linear
X=np.ones((42,2))
X[:,1]=dataset[:,1]
Y=dataset[:,0]

betahat = np.linalg.inv(X.T.dot(X)).dot(X.T.dot(Y))
plt.plot(X[:,1],Y,'o')
xseq=np.arange(0,12,.1)
plt.plot(xseq,betahat[0]+betahat[1]*xseq)
plt.show()

#cubic
X=np.ones((42,4))
X[:,1]=dataset[:,1]
X[:,2]=X[:,1]**2
X[:,3]=X[:,1]**3
Y=dataset[:,0]

betahat = np.linalg.inv(X.T.dot(X)).dot(X.T.dot(Y))
plt.plot(X[:,1],Y,'o')
xseq=np.arange(0,12,.1)
plt.plot(xseq,betahat[0]+betahat[1]*xseq+betahat[2]*xseq**2+betahat[3]*xseq**3)
plt.show()

#cubic
X=np.ones((42,5))
X[:,1]=dataset[:,1]
X[:,2]=X[:,1]**2
X[:,3]=X[:,1]**3
X[:,4]=X[:,1]**4
Y=dataset[:,0]

betahat = np.linalg.inv(X.T.dot(X)).dot(X.T.dot(Y))
plt.plot(X[:,1],Y,'o')
xseq=np.arange(0,12,.1)
plt.plot(xseq,betahat[0]+betahat[1]*xseq
         +betahat[2]*xseq**2+betahat[3]*xseq**3+betahat[4]*xseq**4)
plt.ylim([0,600000])
plt.show()


import numpy as np

def sigmoid(arr):
    """
    Passes each element of the numpy array arr through the sigmiod function elementwise.

    :param arr: numpy array
    :return: arr passed through sigmoid function elementwise
    """

    return 1/(1 + np.exp(-1 * arr))

def sigmoid_gradient(arr):
    """
    Compute the gradient of the sigmoid function for each element in arr,

    :param arr: numpy array
    :return: arr passed through the gradient of the sigmoid function elementwise
    """

    return sigmoid(arr) * (1-sigmoid(arr))


import random
import math
import numpy

def chisquarevariate(k):
	"""Generates a variate from chi square distrubuton with paramater k"""

	s = 0
	for i in range(k):
		s += math.pow(random.normalvariate(0, 1), 2)

	return s

def estimate(k, n):
	"""Estimate expected value and variance over n runs"""
	variates = []
	for i in range(n):
		variates.append(chisquarevariate(k))

	return (numpy.mean(variates), numpy.var(variates))

if __name__ == "__main__":
	k = 9
	n = 10000

	random.seed(123)

	for i in range(1, k):
		res = estimate(i, n)
		print "K = ", i, ": ", res


import os
import math
import numpy as np
from scipy import linalg
from mimoclasses import Channel

# ---------- CHANNEL ------------
def awgnChannel(x,N0):
    """
    Generates the AWGN channel
    Input parameters
    - Signal x (should be avg unit power)
    - Noise variance N0
    Other parameters
    - Thermal noise = -174dBm/Hz
    - Variance N0/2 per real symbol
    """
    N0_r = np.random.normal(0, N0/2, x.shape)
    N0_i = np.random.normal(0, N0/2, x.shape)
    return (x+N0_r+1j*N0_i)

def generateChMatrix(Nr,Nt,chtype):
    # Current threshold for good/bad condition number
    cond_num_thr = 5

    # Condition number based channel will not make sense for Nr=Nt=1
    # since the condition number is always 1
    if Nr==1 and Nt==1:
        if chtype > Channel.RAND_UNIT:
            chtype = Channel.RAND_UNIT

    # TODO: support different channel models in future.
    if chtype == Channel.NONE:
        if Nr==Nt:
            H_r = np.identity(Nr)
            H_i = np.zeros((Nr,Nt))
        else:
            raise ValueError('For channel type-'+str(chtype)+', Nr=Nt is needed.')
            return -1
    elif chtype == Channel.RAND_UNIT:
        # Using complex gaussian random variable
        # Real and Img part: mean = 0, variance = 1
        H_r = np.random.normal(0, 1, size=(Nr,Nt))
        H_i = np.random.normal(0, 1, size=(Nr,Nt))
    elif chtype == Channel.RAND_UNIT_GOOD:
        cond_num = cond_num_thr + 1
        while cond_num > cond_num_thr:
            # Using complex gaussian random variable
            # Real and Img part: mean = 0, variance = 1
            H_r = np.random.normal(0, 1, size=(Nr,Nt))
            H_i = np.random.normal(0, 1, size=(Nr,Nt))
            cond_num = np.linalg.cond(np.asmatrix((H_r + 1j*H_i)))
    elif chtype == Channel.RAND_UNIT_BAD:
        cond_num = 0
        while cond_num < cond_num_thr:
            # Using complex gaussian random variable
            # Real and Img part: mean = 0, variance = 1
            H_r = np.random.normal(0, 1, size=(Nr,Nt))
            H_i = np.random.normal(0, 1, size=(Nr,Nt))
            cond_num = np.linalg.cond(np.asmatrix((H_r + 1j*H_i)))
    else:
        raise ValueError('Channel type-'+str(chtype)+' is not supported.')
        return -1

    cond_num = np.linalg.cond(np.asmatrix((H_r + 1j*H_i)))
    print('Condition number of the generated channel: '+str(cond_num))
    return np.asmatrix((H_r + 1j*H_i))


import numpy as np
import taylor as ta

def quad_vec(x,A,b):
    # computes the n dimensional quadratic of the form:
    #    (1/2) x^T A x - b^T x
    return 0.5*(x.T@A@x) - b.T@x

if (__name__ == "__main__"):
    print()
    print(f'--------------------- QUADRATIC TEST ---------------------')
    print(f'    f(A,b,x) = 0.5 x^T A x - x^T b\n')

    # set matrix and vector
    A = np.array([[1.0,2.0,3.0],
                  [2.0,4.0,5.0],
                  [3.0,5.0,6.0]])
    x = np.array([1.0,2.0,3.0])
    b = np.array([4.0,5.0,6.0])

    # print relevant quantities
    print(f'A :\n{A}\n')
    print(f'x : {x}\n')
    print(f'b : {b}\n')
    print(f'A x - b : {A@x-b}\n')

    # compute first derivative of quadratic with respect to x
    deriv_quad_vec_1 = ta.diff(quad_vec,x,1,args=(A,b))
    print(f'df / dx : {deriv_quad_vec_1}\n')

    # compute second derivative of quadratic with respect to x
    deriv_quad_vec_2 = ta.diff(quad_vec,x,2,args=(A,b))
    print(f'd^2 f / dx^2 :\n{deriv_quad_vec_2}\n')


import numpy as np
import matplotlib.pyplot as plt

N = 10000

def martingale_call(S0, S_q, S_n, K, T, r):
    """Martingale call option pricing"""
    q = np.sum([S_n > K]) / len(S_n)
    qs = np.sum([S_q > K]) / len(S_q)
    return S0 * qs - K * np.exp(-r * T) * q

def mc_call_rn(S_n, K, T, r):
    """Monte-Carlo call option pricing with risk-neutral probabilities"""
    return np.exp(-r * T) * np.mean(call_payoff(S_n, K))

def mc_call_martingale(S0, S_q, K):
    return S0 * np.mean(np.maximum(1 - K * S_q ** (-1), 0))

def call_payoff(S_n, K):
    return np.maximum(S_n - K, 0)

def problem1():
    S0 = 10
    K = 9.3
    r = 0.025
    sg = 0.3
    T = 1
    eps = np.random.normal(size=(N))

    S_n = np.zeros(N)  # risk neutral
    S_q = np.zeros(N)  # prob measure s
    for i in range(N):
        # S_t[i] = S0 * np.exp((r - 0.5 * s ** 2) * T  + s * epsilon[i] * np.sqrt(T))
        S_n[i] = S0 * np.exp((r - 0.5 * sg ** 2) * T + sg * eps[i] * np.sqrt(T))
        S_q[i] = S0 * np.exp((r + sg ** 2 - 0.5 * sg ** 2) * T + sg * eps[i] * np.sqrt(T))

    # plt.hist(S_n)
    # plt.hist(S_q)
    # plt.show()

    print('Martingale price:', martingale_call(S0, S_q, S_n, K, T, r))
    print('MC price:', mc_call_rn(S_n, K, T, r))

def problem2():
    S0 = 10
    K = 9.3
    r = 0.025
    sg = 0.3
    T = 1
    eps = np.random.normal(size=(N))

    S_n = np.zeros(N)  # risk neutral
    S_q = np.zeros(N)  # prob measure s
    for i in range(N):
        # S_t[i] = S0 * np.exp((r - 0.5 * s ** 2) * T  + s * epsilon[i] * np.sqrt(T))
        S_n[i] = S0 * np.exp((r - 0.5 * sg ** 2) * T + sg * eps[i] * np.sqrt(T))
        S_q[i] = S0 * np.exp((r + sg ** 2 - 0.5 * sg ** 2) * T + sg * eps[i] * np.sqrt(T))

    # plt.hist(S_n)
    # plt.hist(S_q)
    # plt.show()

    print('MC price risk-neutral:', mc_call_rn(S_n, K, T, r))
    print('MC price stock:', mc_call_martingale(S0, S_q, K))

if __name__ == '__main__':
    problem2()


import math
from scipy.stats import norm,t

def ztest(X:float, mu:float, sd:float,n:float,alpha=0.05,two_tail=False):

  """
  ztest(): Calculates Z test for single sample, two tailed as well as single.

  :param X: Sample mean
  :type X: float
  :param mu: Population mean/ Data mean
  :type mu: float
  :param sd: Standard Deviation
  :type sd: float
  :param n: Sample size
  :type n: float
  :param alpha: P value Limit/Significance value (default:0.05)
  :type alpha: float
  :param two_tail: Enter true for conducting two-tailed test. (default: False)
  :type two_tail: Boolean
  :return: [Z value, Pvalue, True/False (Test was conclusive or not)]
  :rtype: list
  """
  tail=1

  if(two_tail):
    tail=2
  if(sd==0 or n==0):
    return None
  z=(X-mu)/(sd/(math.sqrt(n)))
  pval=tail*(norm.sf(abs(z)))
  conclusion = (pval<=alpha)

  return [round(z, 3), round(pval, 4), conclusion]

def ztest2(X1:float, X2:float, mudiff:float, sd1:float, sd2:float, n1:float, n2:float,alpha=0.05,two_tail=False):
  """
  ztest2(): Calculates Z test for two sampled data, two tailed as well as single.

  :param X1: Sample mean 1
  :type X: float
  :param X2: Sample mean 2
  :type X: float
  :param mudiff: mean difference
  :type mu: float
  :param sd1: Standard Deviation 1
  :type sd1: float
  :param sd2: Standard Deviation 2
  :type sd2: float
  :param n1: Sample size 1
  :type n1: float
  :param n2: Sample size 2
  :type n2: float
  :param alpha: P value Limit/Significance value (default:0.05)
  :type alpha: float
  :param two_tail: Enter true for conducting two-tailed test. (default: False)
  :type two_tail: Boolean
  :return: [Z score, P value, True/False (Test was conclusive or not)]
  :rtype: list
  """

  tail=1
  if(two_tail):
    tail=2

  if(n1+n2<=0):
    return None

  pooledSE = math.sqrt(sd1**2/n1 + sd2**2/n2)
  if (pooledSE==0):
    return None
  z = ((X1 - X2) - mudiff)/pooledSE
  pval = tail*norm.sf(abs(z))
  conclusion = (pval<=alpha)
  return [round(z, 3), round(pval, 4), conclusion]

def ztestp(P1:float, P2:float,P:float, mudiff:float, n1:float, n2:float, alpha=0.05,two_tail=False):
  """
  ztestp(): Calculates Z test for two population proportions, two tailed as well as single.

  :param P1: Proportion 1
  :type P1: float
  :param P2: Proportion 2
  :type P2: float
  :param P: Overall Proportion
  :type P: float
  :param mudiff: mean difference
  :type mudiff: float
  :param n1: Sample size 2
  :type n1: float
  :param n2: Sample size 2
  :type n2: float
  :param alpha: P value Limit/Significance value (default:0.05)
  :type alpha: float
  :param two_tail: Enter true for conducting two-tailed test. (default: False)
  :type two_tail: Boolean
  :return: [Z value, Pvalue, True/False (Test was conclusive or not)]
  :rtype: list
  """
  tail=1
  if(two_tail):
    tail=2
  if(n1+n2<=0):
    return None
  z = ((P1-P2)-mudiff)/math.sqrt(P*(1-P)*((1/n1)+(1/n2)))
  pval = tail*norm.sf(abs(z))
  conclusion = (pval<=alpha)

  return [round(z, 3), round(pval, 4), conclusion]

def ttest(X:float, mu:float, sd:float, n:float, alpha=0.5,two_tail=False):
  """ttest(): Calculates T test for single sample, two tailed as well as single. This function works for Paired tests as well as the formula is same.

  :param X: Sample mean
  :type X: float
  :param mu: Population mean/ Data mean
  :type mu: float
  :param sd: Standard Deviation
  :type sd: float
  :param n: Sample size
  :type n: float
  :param alpha: P value Limit/Significance value (default:0.05)
  :type alpha: float
  :param two_tail: Enter true for conducting two-tailed test. (default: False)
  :type two_tail: Boolean
  :return: [T score, P value, True/False (Test was conclusive or not)]
  :rtype: list
  """
  tail=1
  if(two_tail):
    tail=2
  if(sd==0 or n==0):
    return None

  tt=(X-mu)/(sd/(math.sqrt(n)))
  pval=tail*(t.sf(abs(tt),n-1))
  conclusion = (pval<=alpha)
  return [round(tt, 3), round(pval, 4), conclusion]

def ttest2(X1:float, X2:float, mudiff:float, sd1:float, sd2:float, n1:float, n2:float,alpha=0.05,V=False,two_tail=False):
  """ttest2(): Calculates T test for two sampled data (independent), two tailed as well as single.

  :param X1: Sample mean 1
  :type X: float
  :param X2: Sample mean 2
  :type X: float
  :param mudiff: mean difference
  :type mu: float
  :param sd1: Standard Deviation 1
  :type sd1: float
  :param sd2: Standard Deviation 2
  :type sd2: float
  :param n1: Sample size 1
  :type n1: float
  :param n2: Sample size 2
  :type n2: float
  :param alpha: P value Limit/Significance value (default:0.05)
  :type alpha: float
  :param V: Enter True assuming equal variances. (default: False)
  :type V: Boolean
  :param two_tail: Enter true for conducting two-tailed test. (default: False)
  :type two_tail: Boolean
  :return: [T score, P value, True/False (Test was conclusive or not)]
  :rtype: list
  """
  tail=1
  if(two_tail):
    tail=2
  if(n1+n2<=0):
    return None
  if(V):
    pooledSE = math.sqrt((((sd1**2)*(n1-1) +(sd2**2)*(n2-1))/(n1+n2-2)))*math.sqrt(1/n1 + 1/n2)
  else:
    pooledSE = math.sqrt(sd1**2/n1 + sd2**2/n2)
  print(pooledSE)

  if (pooledSE==0):
    return None
  tt = ((X1 - X2) - mudiff)/pooledSE
  pval = tail*(t.sf(abs(tt),n1+n2-2))
  conclusion = (pval<=alpha)

  return [round(tt, 3), round(pval, 4), conclusion]


import numpy as np
from scipy import linalg as la

def ref(A):
    for j in xrange(A.shape[0]-1):
        for i in xrange(j+1, A.shape[0]):
            A[i,j:] -= A[i,j] * A[j,j:] / A[j,j]

def ref2(A):
    # This is an alternate version
    # It is faster but less intuitive.
    for i in xrange(A.shape[0]):
        A[i+1:,i:] -= np.outer(A[i+1:,i]/A[i,i], A[i,i:])

def LU(A):
    U = A.copy()
    L = np.eye(A.shape[0])
    for j in xrange(A.shape[0]-1):
        for i in xrange(j+1, A.shape[0]):
            # operation corresponding to left mult by
            # the elementary matrix desired
            L[i,j] = U[i,j] / U[j,j]
            # now we apply the change to U
            U[i,j:] -= L[i,j] * U[j,j:]
    return L, U

def LU2(A):
    # This is an alternate similar to ref2.
    U = A.copy()
    L = np.eye(A.shape[0])
    for i in xrange(A.shape[0]-1):
        L[i+1:,i] = U[i+1:,i] / U[i,i]
        U[i+1:,i:] -= np.outer(L[i+1:,i], U[i,i:])
    return L, U

def LU_inplace(A):
    '''
    this same as the LU function above, just replace the
    L's and U's by A. Also, we need to make sure to not
    set the value of entries below the main diagonal twice.
    '''
    for j in xrange(A.shape[0]-1):
        for i in xrange(j+1, A.shape[0]):
			# change to L
            A[i,j] /= A[j,j]
                # we just set entries of jth column below the main diagonal
            # change to U
            A[i,j+1:] -= A[i,j] * A[j,j+1:]
                # start from column j+1 to avoid setting entires in jth column again

def Solve():
    # the students should have code to generate the random matrices, inverse, LU, and solve
    A = np.random.rand(1000,1000)
    B = np.random.rand(1000,500)
    Ai = la.inv(A)
    (lu,piv) = la.lu_factor(A)

    # the students should report the time for the following operations
    np.dot(Ai,B)
    la.lu_solve((lu,piv),B)

def LU_solve(A,B):
    for j in xrange(A.shape[0]-1):
        for i in xrange(j+1, A.shape[0]):
            B[i] -= A[i,j] * B[j]
    for j in xrange(A.shape[0]-1, -1, -1):
        B[j] /= A[j,j]
        for i in xrange(j):
            B[i] -= A[i,j] * B[j]

def LU_det(A):
    (lu,piv) = la.lu_factor(A)

    # determine whether an even or odd number of row swaps
    s = (piv != np.arange(A.shape[0])).sum() % 2

    return ((-1)**s) * lu.diagonal().prod()

def cholesky(A):
    L = np.zeros_like(A)
    for i in xrange(A.shape[0]):
        for j in xrange(i):
            L[i,j]=(A[i,j] - np.inner(L[i,:j], L[j,:j])) / L[j,j]
        sl = L[i,:i]
        L[i,i] = sqrt(A[i,i] - np.inner(sl, sl))
    return L

def cholesky_inplace(A):
    for i in xrange(A.shape[0]):
        A[i,i+1:] = 0.
        for j in range(i):
            A[i,j] = (A[i,j] - np.inner(A[i,:j],A[j,:j])) / A[j,j]
        sl = A[i,:i]
        A[i,i] = sqrt(A[i,i] - np.inner(sl, sl))

def cholesky_solve(A, B):
    for j in xrange(A.shape[0]):
        B[j] /= A[j,j]
        for i in xrange(j+1, A.shape[0]):
            B[i] -= A[i,j] * B[j]
    for j in xrange(A.shape[0]-1, -1, -1):
        B[j] /= A[j,j]
        for i in xrange(j):
            B[i] -= A[j,i] * B[j]