import numpy as np
import numpy.linalg as linear_alg

def ml_weights(input_matrix, targets):
    """
    This method returns the weights that give the best linear fit between
    the processed inputs and the targets.
    """

    # phi = np.matrix(input_matrix)
    phi = np.array(input_matrix)
    # targets = np.matrix(targets).reshape((len(targets), 1))
    targets = np.array(targets).reshape((len(targets), 1))
    # weights = linear_alg.inv(phi.transpose()*phi)*phi.transpose()*targets
    # a = phi.transpose()* phi
    a = np.dot(phi.T, phi)
    inv = linear_alg.inv(a)
    dot = np.dot(inv, phi.T)
    weights = np.dot(dot, targets)
    # weights = np.poly1d(np.polyfit(input_matrix, targets, 1))

    return np.array(weights).flatten()

def regularised_ml_weights(
        input_matrix, targets, reg_param):
    """
    This method returns the weights that give the best linear fit between
    the processed inputs and the targets, penalised by some regularisation term
    (reg_param).
    """

    phi = np.matrix(input_matrix)
    targets = np.matrix(targets).reshape((len(targets), 1))
    i = np.identity(phi.shape[1])
    weights = linear_alg.inv(reg_param*i + phi.transpose()*phi)*phi.transpose()*targets

    return np.array(weights).flatten()

def linear_model_predict(design_matrix, weights):
    ys = np.matrix(design_matrix)*np.matrix(weights).reshape((len(weights), 1))

    return np.array(ys).flatten()

# this is the feature mapping for a polynomial of given degree in 1d
def expand_to_monomials(inputs, degree):
    """
    Create a design matrix from a 1d array of input values, where columns
    of the output are powers of the inputs from 0 to degree (inclusive).

    So if input is: inputs=np.array([x1, x2, x3])  and degree = 4 then
    output will be design matrix:
        np.array( [[  1.    x1**1   x1**2   x1**3   x1**4   ]
                   [  1.    x2**1   x2**2   x2**3   x2**4   ]
                   [  1.    x3**1   x3**2   x3**3   x3**4   ]]).
    """

    expanded_inputs = []
    for i in range(degree+1):  # because inclusive
        expanded_inputs.append(inputs**i)

    return np.array(expanded_inputs).transpose()

def construct_polynomial_approx(degree, weights):
    """
    This function creates and returns a prediction function based on a
    feature mapping and some weights.

    The returned prediction function takes a set of input values and returns
    the predicted output for each.
    """

    # here is a function that is created on the fly from the input feature
    # mapping and weights
    def prediction_function(xs):
        expanded_xs = np.matrix(expand_to_monomials(xs, degree))
        ys = expanded_xs*np.matrix(weights).reshape((len(weights), 1))

        return np.array(ys).flatten()

    # We return the function reference (handle) itself. This can be used like
    # any other function.
    return prediction_function

def construct_feature_mapping_approx(feature_mapping, weights):
    """
    This function creates and returns a prediction function based on a
    feature mapping and some weights.

    The returned prediction function takes a set of input values and returns
    the predicted output for each.
    """
    # here is a function that is created on the fly from the input feature
    # mapping and weights
    def prediction_function(xs):
        design_matrix = np.matrix(feature_mapping(xs))
        return linear_model_predict(design_matrix, weights)

    # returning the function reference (handle) itself,
    # which can be used like any other function
    return prediction_function

def construct_rbf_feature_mapping(centres, scale):
    """
    parameters
    ----------
    centres - a DxM matrix (numpy array) where D is the dimension of the space
        and each row is the central position of an rbf basis function.
        For D=1 can pass an M-vector (numpy array).
    scale - a float determining the width of the distribution. Equivalent role
        to the standard deviation in the Gaussian distribution.

    returns
    -------
    feature_mapping - a function which takes an NxD data matrix and returns
        the design matrix (NxM matrix of features)
    """

    #  to enable python's broadcasting capability we need the centres
    # array as a 1xDxM array
    if len(centres.shape) == 1:
        centres = centres.reshape((1, 1, centres.size))
    else:
        centres = centres.reshape((1, centres.shape[1], centres.shape[0]))
    # the denominator
    denom = 2*scale**2

    # now create a function based on these basis functions
    def feature_mapping(datamtx):
        #  to enable python's broadcasting capability we need the datamtx array
        # as a NxDx1 array
        if len(datamtx.shape) == 1:
            # if the datamtx is just an array of scalars, turn this into
            # a Nx1x1 array
            datamtx = datamtx.reshape((datamtx.size, 1, 1))
        else:
            # if datamtx is NxD array, then we reshape matrix as a
            # NxDx1 array
            datamtx = datamtx.reshape((datamtx.shape[0], datamtx.shape[1], 1))
        return np.exp(-np.sum((datamtx - centres)**2,1)/denom)
    # return the created function
    return feature_mapping

def construct_knn_approx(train_inputs, train_targets, k):
    """
    For 1 dimensional training data, it produces a function:reals-> reals
    that outputs the mean training value in the k-Neighbourhood of any input.
    """

    # creating Euclidean distance
    distance = lambda x, y: (x-y)**2
    train_inputs = np.resize(train_inputs, (1, train_inputs.size))

    def prediction_function(inputs):
        inputs = inputs.reshape((inputs.size, 1))
        distances = distance(train_inputs, inputs)
        predicts = np.empty(inputs.size)
        for i, neighbourhood in enumerate(np.argpartition(distances, k)[:, :k]):
            # the neighbourhood is the indices of the closest inputs to xs[i]
            # the prediction is the mean of the targets for this neighbourhood
            predicts[i] = np.mean(train_targets[neighbourhood])
        return predicts

    # returning a handle to the locally defined function
    return prediction_function

def calculate_weights_posterior(designmtx, targets, beta, m0, S0):
    """
    Calculates the posterior distribution (multivariate gaussian) for weights
    in a linear model.

    parameters
    ----------
    designmtx - 2d (N x M) array of inputs (data-matrix or design-matrix) where
        N is the number of data-points and each row is that point's representation
    targets - 1d (N)-array of target values
    beta - the known noise precision
    m0 - prior mean (vector) 1d-array (or array-like) of length M
    S0 - the prior covariance matrix 2d-array

    returns
    -------
    mN - the posterior mean (vector)
    SN - the posterior covariance matrix
    """

    N, M = designmtx.shape
    Phi = np.matrix(designmtx)
    t = np.matrix(targets).reshape((N, 1))
    m0 = np.matrix(m0).reshape((M, 1))
    S0_inv = np.matrix(np.linalg.inv(S0))
    SN = np.linalg.inv(S0_inv + beta * Phi.transpose() * Phi)
    mN = SN * (S0_inv * m0 + beta * Phi.transpose() * t)

    return np.array(mN).flatten(), np.array(SN)

def predictive_distribution(designmtx, beta, mN, SN):
    """
    Calculates the predictive distribution a linear model. This amounts to a
    mean and variance for each input point.

    parameters
    ----------
    designmtx - 2d (N x K) array of inputs (data-matrix or design-matrix) where
        N is the number of data-points and each row is that point's
        representation
    beta - the known noise precision
    mN - posterior mean of the weights (vector) 1d-array (or array-like)
        of length K
    SN - the posterior covariance matrix for the weights 2d (K x K)-array

    returns
    -------
    ys - a vector of mean predictions, one for each input datapoint
    sigma2Ns - a vector of variances, one for each input data-point
    """

    N, K = designmtx.shape
    Phi = np.matrix(designmtx)
    mN = np.matrix(mN).reshape((K, 1))
    SN = np.matrix(SN)
    ys = Phi * mN
    # create an array of the right size with the uniform term
    sigma2Ns = np.ones(N) / beta
    for n in range(N):
        # now calculate and add in the data dependent term
        # NOTE: I couldn't work out a neat way of doing this without a for-loop
        # NOTE: but if anyone can please share the answer.
        phi_n = Phi[n, :].transpose()
        sigma2Ns[n] += phi_n.transpose() * SN * phi_n
    return np.array(ys).flatten(), np.array(sigma2Ns)


import numpy as np

def euclidean_dist(a, b):
    dist = 0
    for i in range(len(a)):
        dist += np.power(a[i] - b[i], 2)
    return np.sqrt(dist)

def manhattan_dist(a, b):
    dist = 0
    for i in range(len(a)):
        dist += abs(a[i] - b[i])
    return dist


"""
This module performs a comparision between my algorithm and Sklearn's
implementation. In order to facilitate this comparison, I use the spam
data set from The Elements of Statistical Learning. I compare the objective
value and the misclassification errors for the training and the testing data.
"""
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import sklearn.preprocessing
import linearsvm as svm
from sklearn.svm import LinearSVC

# Downloading the data from the Stanford website
print("Downloading the data\n")

spam = pd.read_table('https://web.stanford.edu/~hastie/ElemStatLearn/'
                     'datasets/spam.data', sep=' ', header=None)
traintestind = pd.read_table('https://web.stanford.edu/~hastie/'
                             'ElemStatLearn/datasets/spam.traintest', sep=' ',
                               header=None)
X = np.asarray(spam)[:, 0:-1]

y = spam[57]
y = y.map({0: -1, 1: 1})
y = np.asarray(y)

# Splitting the data according to the split given on the website
indvar = np.array(traintestind).T[0]
X_train = X[indvar == 0, :]
X_test = X[indvar == 1, :]
y_train = y[indvar == 0]
y_test = y[indvar == 1]

print("Standardizing the data\n")

# Using Standard Scaler to scale the data
scaler = sklearn.preprocessing.StandardScaler()
scaler.fit(X_train)
X_train = scaler.transform(X_train)
X_test = scaler.transform(X_test)

# First train using sklearn
print("Training Sklearn model\n")
sklearn_model = LinearSVC(penalty='l2', loss='squared_hinge', C=0.1)
sklearn_model.fit(X_train, y_train)
predictions_sklearn_test = sklearn_model.predict(X_test)
predictions_sklearn_train = sklearn_model.predict(X_train)
sklearn_coefficients = sklearn_model.coef_
misclass_train_sklearn = 1 - np.mean(predictions_sklearn_train == y_train)
misclass_test_sklearn = 1 - np.mean(predictions_sklearn_test == y_test)

# Perform training on my own model
print("Training my own model\n")
betas, objs = svm.mylinearsvm(0.1, 0.1, 100, X_train, y_train)
misclass_train_own = svm.calc_misclass(betas[-1], X_train, y_train)
misclass_test_own = svm.calc_misclass(betas[-1], X_test, y_test)

objval_sk = svm.objective(sklearn_coefficients.T, 0.1, X_train, y_train)

# Output results to stdout
print("Outputting data\n")
print("The value of the objective for my implementation is:", objs[-1])
print("The value of the objective for sklearn implementation is:", objval_sk)
print("The misclassification error for training set using sklearn is:", misclass_train_sklearn)
print("The misclassification error for test set using sklearn is:", misclass_test_sklearn)
print("The misclassification error for the training set using my own ", misclass_train_own)
print("The misclassification error for the test set using my own", misclass_test_own)


import numpy as np
# Note: please don't import any new package. You should solve this problem using only the package(s) above.
#-------------------------------------------------------------------------

'''
    Problem 2: Softmax Regression (30 points)
    In this problem, you will implement the softmax regression for multi-class classification problems.
The main goal of this problem is to extend the logistic regression method to solving multi-class classification problems.
We will get familiar with computing gradients of vectors/matrices.
We will use multi-class cross entropy as the loss function and stochastic gradient descent to train the model parameters.

    A list of all variables being used in this problem is provided at the end of this file.
'''

#----------------------------------------------------
'''
    (Value Forward Function 1) Given a softmax regression model with parameters W and b, please compute the linear logit z(x) on a data sample x.
    ---- Inputs: --------
        * x: the feature vector of a data instance, a float numpy vector of length p.
        * W: the weight matrix of softmax regression, a float numpy matrix of shape (c by p).
        * b: the bias values of softmax regression, a float numpy vector of length c.
    ---- Outputs: --------
        * z: the linear logits, a float numpy vector of length c..
    ---- Hints: --------
        * This problem can be solved using 1 line(s) of code.
'''
#---------------------
def compute_z(x, W, b):
    #########################################
    ## INSERT YOUR CODE HERE (1 points)
    z = W.dot(x) + b
    #########################################
    return z
    #-----------------
    '''
        TEST: Now you can test the correctness of your code above by typing the following in the terminal:
        ---------------------------------------------------
        nosetests -v test2.py:test_compute_z
        --- OR ----
        python3 -m nose -v test2.py:test_compute_z
        --- OR ----
        python -m nose -v test2.py:test_compute_z
        ---------------------------------------------------
    '''

#----------------------------------------------------
'''
    (Local Gradient 1.1) Suppose we are given a softmax regression model with parameters W and b. Suppose we have already computed the linear logits z(x) on a training sample x.  Please compute partial gradients of the linear logits z(x) w.r.t. the biases b.
    ---- Inputs: --------
        * c: the number of classes in the classification task, an integer scalar.
    ---- Outputs: --------
        * dz_db: the partial gradient of the logits z w.r.t. the biases b, a float matrix of shape c by c.  Each element dz_db[i,j] represents the partial gradient of the i-th logit z[i] w.r.t. the j-th bias b[j],   d_z[i] / d_b[j].
    ---- Hints: --------
        * This problem can be solved using 1 line(s) of code.
'''
#---------------------
def compute_dz_db(c):
    #########################################
    ## INSERT YOUR CODE HERE (1 points)
    dz_db = np.diag(np.full(c, 1))
    #########################################
    return dz_db
    #-----------------
    '''
        TEST: Now you can test the correctness of your code above by typing the following in the terminal:
        ---------------------------------------------------
        nosetests -v test2.py:test_compute_dz_db
        --- OR ----
        python3 -m nose -v test2.py:test_compute_dz_db
        --- OR ----
        python -m nose -v test2.py:test_compute_dz_db
        ---------------------------------------------------
    '''

#----------------------------------------------------
'''
    (Global Gradients 1.1) Suppose we are given a softmax regression model with parameters (W and b) and we have a training data sample (x,y).  Suppose we have already computed the local gradient 1.1 (dz_db) and the global gradients of the loss L w.r.t. the linear logits z(x) (dL_dz). Please compute the partial gradients of the loss L w.r.t. biases b using chain rule.
    ---- Inputs: --------
        * dL_dz: the partial gradients of the loss function L w.r.t. the linear logits z, a float numpy vector of length c.  The i-th element dL_dz[i] represents the partial gradient of the loss function L w.r.t. the i-th linear logit z[i],  d_L / d_z[i]..
        * dz_db: the partial gradient of the logits z w.r.t. the biases b, a float matrix of shape c by c.  Each element dz_db[i,j] represents the partial gradient of the i-th logit z[i] w.r.t. the j-th bias b[j],   d_z[i] / d_b[j].
    ---- Outputs: --------
        * dL_db: the partial gradient of the loss function L w.r.t. the biases b, a float numpy vector of length c.  The i-th element dL_db[i] represents the partial gradient of the loss function w.r.t. the i-th bias,  d_L / d_b[i].
    ---- Hints: --------
        * This problem can be solved using 1 line(s) of code.
'''
#---------------------
def compute_dL_db(dL_dz, dz_db):
    #########################################
    ## INSERT YOUR CODE HERE (1 points)
    dL_db = dz_db.dot(dL_dz)
    #########################################
    return dL_db
    #-----------------
    '''
        TEST: Now you can test the correctness of your code above by typing the following in the terminal:
        ---------------------------------------------------
        nosetests -v test2.py:test_compute_dL_db
        --- OR ----
        python3 -m nose -v test2.py:test_compute_dL_db
        --- OR ----
        python -m nose -v test2.py:test_compute_dL_db
        ---------------------------------------------------
    '''

#----------------------------------------------------
'''
    (Local Gradient 1.2) Suppose we are given a softmax regression model with parameters W and b. Suppose we have already computed the linear logits z(x) on a training sample x.  Please compute partial gradients of the linear logits z(x) w.r.t. the weights W.
    ---- Inputs: --------
        * x: the feature vector of a data instance, a float numpy vector of length p.
        * c: the number of classes in the classification task, an integer scalar.
    ---- Outputs: --------
        * dz_dW: the partial gradient of logits z w.r.t. the weight matrix W, a numpy float tensor of shape (c by c by p).  The (i,j,k)-th element of dz_dW represents the partial gradient of the i-th logit (z[i]) w.r.t. the weight W[j,k],   d_z[i] / d_W[j,k].
    ---- Hints: --------
        * This problem can be solved using 3 line(s) of code.
'''
#---------------------
def compute_dz_dW(x, c):
    #########################################
    ## INSERT YOUR CODE HERE (1 points)
    dz_db = np.diag(np.full(c, 1))
    dz_dW = np.array([[x if j == 1 else np.zeros((len(x))) for j in i] for i in dz_db])
    #########################################
    return dz_dW
    #-----------------
    '''
        TEST: Now you can test the correctness of your code above by typing the following in the terminal:
        ---------------------------------------------------
        nosetests -v test2.py:test_compute_dz_dW
        --- OR ----
        python3 -m nose -v test2.py:test_compute_dz_dW
        --- OR ----
        python -m nose -v test2.py:test_compute_dz_dW
        ---------------------------------------------------
    '''

#----------------------------------------------------
'''
    (Global Gradient 1.2) Suppose we are given a softmax regression model with parameters (W and b) and we have a training data sample (x,y).  Suppose we have already computed the local gradients local gradient 1.2 (dz_dW) and the global gradients of the loss L w.r.t. the linear logits z(x) (dL_dz). Please compute the partial gradient of the loss L w.r.t. the weights W using chain rule.
    ---- Inputs: --------
        * dL_dz: the partial gradients of the loss function L w.r.t. the linear logits z, a float numpy vector of length c.  The i-th element dL_dz[i] represents the partial gradient of the loss function L w.r.t. the i-th linear logit z[i],  d_L / d_z[i]..
        * dz_dW: the partial gradient of logits z w.r.t. the weight matrix W, a numpy float tensor of shape (c by c by p).  The (i,j,k)-th element of dz_dW represents the partial gradient of the i-th logit (z[i]) w.r.t. the weight W[j,k],   d_z[i] / d_W[j,k].
    ---- Outputs: --------
        * dL_dW: the partial gradients of the loss function L w.r.t. the weight matrix W, a numpy float matrix of shape (c by p).  The i,j-th element dL_dW[i,j] represents the partial gradient of the loss function L w.r.t. the i,j-th weight W[i,j],  d_L / d_W[i,j].
    ---- Hints: --------
        * You could use np.tensordot(A,B) to compute the dot product of two tensors A and B.
        * This problem can be solved using 1 line(s) of code.
'''
#---------------------
def compute_dL_dW(dL_dz, dz_dW):
    #########################################
    ## INSERT YOUR CODE HERE (2 points)
    dL_dW = np.tensordot(dL_dz, dz_dW, 1)
    #########################################
    return dL_dW
    #-----------------
    '''
        TEST: Now you can test the correctness of your code above by typing the following in the terminal:
        ---------------------------------------------------
        nosetests -v test2.py:test_compute_dL_dW
        --- OR ----
        python3 -m nose -v test2.py:test_compute_dL_dW
        --- OR ----
        python -m nose -v test2.py:test_compute_dL_dW
        ---------------------------------------------------
    '''

#----------------------------------------------------
'''
    (Value Forward Function 2) Suppose we are given a softmax regression model and we have already computed the linear logits z(x) on a data sample x. Please compute the softmax activation on the data sample, i.e., a(x).
    ---- Inputs: --------
        * z: the linear logits, a float numpy vector of length c..
    ---- Outputs: --------
        * a: the softmax activations, a float numpy vector of length c..
    ---- Hints: --------
        * You could use np.exp(x) to compute the element-wise exponentials of vector x.
        * When computing exp(z), you need to be careful about overflowing cases. When an element of z (say z[i]) is a large number (say 1000),  the computer can no longer store the result of exp(z[i]) in a floating-point number. In this case, we may want to avoid computing exp(z) directly. Instead, you could find the largest value in z (say max_z) and subtract every element with max_z and then you could compute exp() on the vector (z-max_z) directly. The result will be correct, but will no longer suffer from overflow problems.
        * This problem can be solved using 2 line(s) of code.
'''
#---------------------
def compute_a(z):
    #########################################
    ## INSERT YOUR CODE HERE (2 points)
    zprime = (z - max(z)) if abs(max(z)) >= 1000 else z
    sigma = sum([np.exp(zi) for zi in zprime])
    a = np.exp(zprime) / sigma
    #########################################
    return a
    #-----------------
    '''
        TEST: Now you can test the correctness of your code above by typing the following in the terminal:
        ---------------------------------------------------
        nosetests -v test2.py:test_compute_a
        --- OR ----
        python3 -m nose -v test2.py:test_compute_a
        --- OR ----
        python -m nose -v test2.py:test_compute_a
        ---------------------------------------------------
    '''

#----------------------------------------------------
'''
    (Local Gradients 2) Suppose we are given a softmax regression model and we have already computed the linear logits z(x) and activations a(x) on a training sample (x). Please compute the partial gradients of the softmax activations a(x) w.r.t. the linear logits z(x).
    ---- Inputs: --------
        * a: the softmax activations, a float numpy vector of length c..
    ---- Outputs: --------
        * da_dz: the partial gradient of the activations a w.r.t. the logits z, a float numpy matrix of shape (c by c).  The (i,j)-th element of da_dz represents the partial gradient ( d_a[i]  / d_z[j] ).
    ---- Hints: --------
        * This problem can be solved using 3 line(s) of code.
'''
#---------------------
def compute_da_dz(a):
    #########################################
    ## INSERT YOUR CODE HERE (2 points)
    da_dz = np.array([[a[i] * (1 - a[i]) if i == j else -a[i] * a[j] for j in range(len(a))] for i in range(len(a))])
    #########################################
    return da_dz
    #-----------------
    '''
        TEST: Now you can test the correctness of your code above by typing the following in the terminal:
        ---------------------------------------------------
        nosetests -v test2.py:test_compute_da_dz
        --- OR ----
        python3 -m nose -v test2.py:test_compute_da_dz
        --- OR ----
        python -m nose -v test2.py:test_compute_da_dz
        ---------------------------------------------------
    '''

#----------------------------------------------------
'''
    (Global Gradients 2) Suppose we are given a softmax regression model with parameters (W and b) and we have a training data sample (x,y).  Suppose we have already computed the gradient of the loss L w.r.t. the activations a(x) and the partial gradients of activations a(x) w.r.t. the linear logits z(x). Please compute the partial gradients of the loss L w.r.t. the linear logits z(x) using chain rule.
    ---- Inputs: --------
        * dL_da: the partial gradients of the loss function L w.r.t. the activations a, a float numpy vector of length c.  The i-th element dL_da[i] represents the partial gradient of the loss function L w.r.t. the i-th activation a[i],  d_L / d_a[i]..
        * da_dz: the partial gradient of the activations a w.r.t. the logits z, a float numpy matrix of shape (c by c).  The (i,j)-th element of da_dz represents the partial gradient ( d_a[i]  / d_z[j] ).
    ---- Outputs: --------
        * dL_dz: the partial gradients of the loss function L w.r.t. the linear logits z, a float numpy vector of length c.  The i-th element dL_dz[i] represents the partial gradient of the loss function L w.r.t. the i-th linear logit z[i],  d_L / d_z[i]..
    ---- Hints: --------
        * This problem can be solved using 1 line(s) of code.
'''
#---------------------
def compute_dL_dz(dL_da, da_dz):
    #########################################
    ## INSERT YOUR CODE HERE (2 points)
    dL_dz = dL_da.dot(da_dz)
    #########################################
    return dL_dz
    #-----------------
    '''
        TEST: Now you can test the correctness of your code above by typing the following in the terminal:
        ---------------------------------------------------
        nosetests -v test2.py:test_compute_dL_dz
        --- OR ----
        python3 -m nose -v test2.py:test_compute_dL_dz
        --- OR ----
        python -m nose -v test2.py:test_compute_dL_dz
        ---------------------------------------------------
    '''

#----------------------------------------------------
'''
    (Value Forward Function 3) Suppose we are given a softmax regression model and we have already computed the activations a(x) on a training sample x. Suppose the label of the training sample is y. Please compute the loss function of the softmax regression model on the training sample.
    ---- Inputs: --------
        * a: the softmax activations, a float numpy vector of length c..
        * y: the label of one data sample, an integer scalar value. The values can be 0,1,2, ..., or (c-1).
    ---- Outputs: --------
        * L: the multi-class cross entropy loss, a float scalar..
    ---- Hints: --------
        * You could use np.log(x) to compute the natural log of x.
        * When computing log(a[i]), you need to be careful about a corner case where log(0) is not defined in math. Now the question is how can any activation (a) become 0? It is mathematically impossible, because the output of the softmax function (activation) should be 0<a[i]<1. However, in the above function (compute_a), when some linear logit z[i] is a number much larger than all the other elements in z (say z[j]), the activation of all the other elements (a[j]) will be very very small. Then the computer can no longer store these small numbers accurately in floating-point numbers. Instead, computer will store 0 as the activation a[i]. Then we have a problem in this function. We need to handle the specially case when a[j] = 0. To solve this problem, we need to avoid computing log(0) by assigning the final result of L directly. In this case, the log(a[j]) should be a very large negative number (say -10000...000 ), though it should not be negative infinity. So the loss L = -log(a[j]) should be a very large positive number.
        * This problem can be solved using 1 line(s) of code.
'''
#---------------------
def compute_L(a, y):
    #########################################
    ## INSERT YOUR CODE HERE (2 points)
    L = 10000000000000000000000 if a[y] == 0 else -np.log(a[y])
    #########################################
    return L
    #-----------------
    '''
        TEST: Now you can test the correctness of your code above by typing the following in the terminal:
        ---------------------------------------------------
        nosetests -v test2.py:test_compute_L
        --- OR ----
        python3 -m nose -v test2.py:test_compute_L
        --- OR ----
        python -m nose -v test2.py:test_compute_L
        ---------------------------------------------------
    '''

#----------------------------------------------------
'''
    (Local/Global Gradient 3) Suppose we are given a softmax regression model and we have already computed the activations a(x) on a training sample x. Suppose the label of the training sample is y. Please compute the partial gradients of the loss function (L) w.r.t. the activations (a).
    ---- Inputs: --------
        * a: the softmax activations, a float numpy vector of length c..
        * y: the label of one data sample, an integer scalar value. The values can be 0,1,2, ..., or (c-1).
    ---- Outputs: --------
        * dL_da: the partial gradients of the loss function L w.r.t. the activations a, a float numpy vector of length c.  The i-th element dL_da[i] represents the partial gradient of the loss function L w.r.t. the i-th activation a[i],  d_L / d_a[i]..
    ---- Hints: --------
        * If you want to create an all-zero array of the same shape as x, you could use np.zeros_like(x) to create the all-zero matrix.
        * When computing 1/a[i], you need to be careful about a corner case where 1/0 is not defined in math. Now the question is how can any of the activations (a) become 0? It is mathematically impossible, because the output of the softmax function (activations) should be 0<a[i]<1. However, in the compute_a() function, when some element of linear logits z (say z[i]) is much larger than all the other linear logits (say z[j]), the activations of all the other elements (a[j]) will be very very small. Then the computer can no longer store these small numbers accurately in floating-point numbers. Instead, computer will store 0 as the activation. Then we have a problem in this function. We need to handle the specially case when a[j] = 0.  To solve this problem, we need to avoid computing 1/a[j] by assigning the final result of 1/a[j] directly. In this case, when a[j] is a very small positive number (say exp(-900)), then 1/a[j] should be a very large positive number, though it should not be positive infinity. In this case, (-1/a[j]) will be a very large negative number (say -100000...000), though it should not be negative infinity.
        * This problem can be solved using 2 line(s) of code.
'''
#---------------------
def compute_dL_da(a, y):
    #########################################
    ## INSERT YOUR CODE HERE (2 points)
    dL_da = np.zeros_like(a)
    dL_da[y] = -100000000000000000 if abs(a[y]) == 0 else -1/a[y]
    #########################################
    return dL_da
    #-----------------
    '''
        TEST: Now you can test the correctness of your code above by typing the following in the terminal:
        ---------------------------------------------------
        nosetests -v test2.py:test_compute_dL_da
        --- OR ----
        python3 -m nose -v test2.py:test_compute_dL_da
        --- OR ----
        python -m nose -v test2.py:test_compute_dL_da
        ---------------------------------------------------
    '''

#----------------------------------------------------
'''
    (Forward Pass) Suppose we are given a softmax regression model with parameter W and b. Given a data sample (x), please compute the activations a(x) on the sample.
    ---- Inputs: --------
        * x: the feature vector of a data instance, a float numpy vector of length p.
        * W: the weight matrix of softmax regression, a float numpy matrix of shape (c by p).
        * b: the bias values of softmax regression, a float numpy vector of length c.
    ---- Outputs: --------
        * a: the softmax activations, a float numpy vector of length c..
    ---- Hints: --------
        * This problem can be solved using 2 line(s) of code.
'''
#---------------------
def forward(x, W, b):
    #########################################
    ## INSERT YOUR CODE HERE (1 points)
    a = compute_a(compute_z(x, W, b))
    #########################################
    return a
    #-----------------
    '''
        TEST: Now you can test the correctness of your code above by typing the following in the terminal:
        ---------------------------------------------------
        nosetests -v test2.py:test_forward
        --- OR ----
        python3 -m nose -v test2.py:test_forward
        --- OR ----
        python -m nose -v test2.py:test_forward
        ---------------------------------------------------
    '''

#----------------------------------------------------
'''
    (Back Propagation) Suppose we are given a softmax regression model with parameters (W and b) and we have a training data sample (x) with label (y).  Suppose we have already computed the activations a(x) on the data sample in the forward-pass. Please compute the global gradients of the loss w.r.t. the parameters W and b on the data sample using back propagation.
    ---- Inputs: --------
        * x: the feature vector of a data instance, a float numpy vector of length p.
        * y: the label of one data sample, an integer scalar value. The values can be 0,1,2, ..., or (c-1).
        * a: the softmax activations, a float numpy vector of length c..
    ---- Outputs: --------
        * dL_dW: the partial gradients of the loss function L w.r.t. the weight matrix W, a numpy float matrix of shape (c by p).  The i,j-th element dL_dW[i,j] represents the partial gradient of the loss function L w.r.t. the i,j-th weight W[i,j],  d_L / d_W[i,j].
        * dL_db: the partial gradient of the loss function L w.r.t. the biases b, a float numpy vector of length c.  The i-th element dL_db[i] represents the partial gradient of the loss function w.r.t. the i-th bias,  d_L / d_b[i].
    ---- Hints: --------
        * Step 1: compute all the local gradients by re-using the above functions.
        * Step 2: use the local gradients to build global gradients for the parameters W and b.
        * This problem can be solved using 7 line(s) of code.
'''
#---------------------
def backward(x, y, a):
    #########################################
    ## INSERT YOUR CODE HERE (4 points)
    dL_da = compute_dL_da(a, y)
    da_dz = compute_da_dz(a)
    dL_dz = compute_dL_dz(dL_da, da_dz)
    dz_dW = compute_dz_dW(x, len(a))
    dL_dW = compute_dL_dW(dL_dz, dz_dW)
    dz_db = compute_dz_db(len(a))
    dL_db = compute_dL_db(dL_dz, dz_db)
    #########################################
    return dL_dW, dL_db
    #-----------------
    '''
        TEST: Now you can test the correctness of your code above by typing the following in the terminal:
        ---------------------------------------------------
        nosetests -v test2.py:test_backward
        --- OR ----
        python3 -m nose -v test2.py:test_backward
        --- OR ----
        python -m nose -v test2.py:test_backward
        ---------------------------------------------------
    '''

#----------------------------------------------------
'''
    (Gradient Descent 1) Suppose we are given a softmax regression model with parameters (W and b) and we have a training data sample (x,y).  Suppose we have already computed the partial gradients of the loss w.r.t. the biases b on the data sample. Please update the biases b using gradient descent.
    ---- Inputs: --------
        * b: the bias values of softmax regression, a float numpy vector of length c.
        * dL_db: the partial gradient of the loss function L w.r.t. the biases b, a float numpy vector of length c.  The i-th element dL_db[i] represents the partial gradient of the loss function w.r.t. the i-th bias,  d_L / d_b[i].
        * alpha: the step-size parameter of gradient descent, a float scalar.
    ---- Outputs: --------
        * b: the bias values of softmax regression, a float numpy vector of length c.
    ---- Hints: --------
        * This problem can be solved using 1 line(s) of code.
'''
#---------------------
def update_b(b, dL_db, alpha=0.001):
    #########################################
    ## INSERT YOUR CODE HERE (1 points)
    b = b - alpha * dL_db
    #########################################
    return b
    #-----------------
    '''
        TEST: Now you can test the correctness of your code above by typing the following in the terminal:
        ---------------------------------------------------
        nosetests -v test2.py:test_update_b
        --- OR ----
        python3 -m nose -v test2.py:test_update_b
        --- OR ----
        python -m nose -v test2.py:test_update_b
        ---------------------------------------------------
    '''

#----------------------------------------------------
'''
    (Gradient Descent 2) Suppose we are given a softmax regression model with parameters (W and b) and we have a training data sample (x,y).  Suppose we have already computed the partial gradients of the loss w.r.t. the weights W on the data sample. Please update the weights W using gradient descent.
    ---- Inputs: --------
        * W: the weight matrix of softmax regression, a float numpy matrix of shape (c by p).
        * dL_dW: the partial gradients of the loss function L w.r.t. the weight matrix W, a numpy float matrix of shape (c by p).  The i,j-th element dL_dW[i,j] represents the partial gradient of the loss function L w.r.t. the i,j-th weight W[i,j],  d_L / d_W[i,j].
        * alpha: the step-size parameter of gradient descent, a float scalar.
    ---- Outputs: --------
        * W: the weight matrix of softmax regression, a float numpy matrix of shape (c by p).
    ---- Hints: --------
        * This problem can be solved using 1 line(s) of code.
'''
#---------------------
def update_W(W, dL_dW, alpha=0.001):
    #########################################
    ## INSERT YOUR CODE HERE (1 points)
    W = W - alpha * dL_dW
    #########################################
    return W
    #-----------------
    '''
        TEST: Now you can test the correctness of your code above by typing the following in the terminal:
        ---------------------------------------------------
        nosetests -v test2.py:test_update_W
        --- OR ----
        python3 -m nose -v test2.py:test_update_W
        --- OR ----
        python -m nose -v test2.py:test_update_W
        ---------------------------------------------------
    '''

#----------------------------------------------------
'''
    (Training Softmax Regression) Given a training dataset X (features), Y (labels), train the softmax regression model using stochastic gradient descent: iteratively update the weights W and biases b using the gradients on each random data sample.  We repeat n_epoch passes over all the training instances.
    ---- Inputs: --------
        * X: the feature matrix of training instances, a float numpy matrix of shape (n by p).
        * Y: the labels of training instance, a numpy integer numpy array of length n. The values can be 0, 1, ..., or c-1.
        * alpha: the step-size parameter of gradient descent, a float scalar.
        * n_epoch: the number of passes to go through the training dataset in the training process, an integer scalar.
    ---- Outputs: --------
        * W: the weight matrix of softmax regression, a float numpy matrix of shape (c by p).
        * b: the bias values of softmax regression, a float numpy vector of length c.
    ---- Hints: --------
        * Step 1 Forward pass: compute the linear logits, activations and loss.
        * Step 2 Back propagation: compute the gradients of W and b.
        * Step 3 Gradient descent: update the parameters W and b using gradient descent.
        * This problem can be solved using 4 line(s) of code.
'''
#---------------------
def train(X, Y, alpha=0.001, n_epoch=100):
    n,p = X.shape # n: the number of training samples, p: the number of input features
    c = max(Y) + 1 # the number of classes
    W = np.random.randn(c,p) # initialize W randomly using standard normal distribution
    b= np.zeros(c) # initialize b as all zeros
    for _ in range(n_epoch): # iterate through the dataset n_epoch times
        indices = np.random.permutation(n) # shuffle the indices of all samples
        for i in indices: # iterate through each random training sample (x,y)
            x=X[i] # the feature vector of the i-th random sample
            y=Y[i] # the label of the i-th random sample
            #########################################
            ## INSERT YOUR CODE HERE (1 points)
            a = forward(x, W, b)
            dL_dW, dL_db = backward(x, y, a)
            W = update_W(W, dL_dW, alpha)
            b = update_b(b, dL_db, alpha)
            #########################################
    return W, b
    #-----------------
    '''
        TEST: Now you can test the correctness of your code above by typing the following in the terminal:
        ---------------------------------------------------
        nosetests -v test2.py:test_train
        --- OR ----
        python3 -m nose -v test2.py:test_train
        --- OR ----
        python -m nose -v test2.py:test_train
        ---------------------------------------------------
    '''

#----------------------------------------------------
'''
    (Using Softmax Regression 1) Given a trained softmax regression model with parameters W and b. Suppose we have a test sample x. Please use the softmax regression model to predict the label of x and the probabilities of the label being in each of the classes, i.e. the activation a(x). .
    ---- Inputs: --------
        * x: the feature vector of a data instance, a float numpy vector of length p.
        * W: the weight matrix of softmax regression, a float numpy matrix of shape (c by p).
        * b: the bias values of softmax regression, a float numpy vector of length c.
    ---- Outputs: --------
        * y: the label of one data sample, an integer scalar value. The values can be 0,1,2, ..., or (c-1).
        * a: the softmax activations, a float numpy vector of length c..
    ---- Hints: --------
        * If we have multiple elements in the activations being the largest at the same time (for example, [0.5, 0.5,0] have two largest values), we can break the tie by choosing the element with the smallest index.
        * This problem can be solved using 2 line(s) of code.
'''
#---------------------
def inference(x, W, b):
    #########################################
    ## INSERT YOUR CODE HERE (1 points)
    a = compute_a(compute_z(x, W, b))
    y = np.argmax(a)
    #########################################
    return y, a
    #-----------------
    '''
        TEST: Now you can test the correctness of your code above by typing the following in the terminal:
        ---------------------------------------------------
        nosetests -v test2.py:test_inference
        --- OR ----
        python3 -m nose -v test2.py:test_inference
        --- OR ----
        python -m nose -v test2.py:test_inference
        ---------------------------------------------------
    '''

#----------------------------------------------------
'''
    (Using Softmax Regression 2) Given a trained softmax regression model with parameters W and b. Suppose we have a test dataset Xtest (features). For each data sample x in Xtest, use the softmax regression model to predict the label of x and the probabilities of the label being in each of the classes, i.e. the activation a(x).
    ---- Inputs: --------
        * Xtest: the feature matrix of test instances, a float numpy matrix of shape (n_test by p).
        * W: the weight matrix of softmax regression, a float numpy matrix of shape (c by p).
        * b: the bias values of softmax regression, a float numpy vector of length c.
    ---- Outputs: --------
        * Ytest: the predicted labels of test data samples, an integer numpy array of length ntest. Ytest[i] represents the predicted label on the i-th test sample.
        * P: the predicted probabilities of test data samples to be in different classes, a float numpy matrix of shape (n_test,c).  P[i,j] is the probability of the i-th data sample to have the j-th class label.
    ---- Hints: --------
        * This problem can be solved using 1 line(s) of code.
'''
#---------------------
def predict(Xtest, W, b):
    n_test = Xtest.shape[0] # the number of test samples
    c = W.shape[0] # the number of classes
    Ytest = np.zeros(n_test) # initialize label vector as all zeros
    P = np.zeros((n_test,c)) # initialize label probability matrix as all zeros
    for i in range(n_test): # iterate through each test sample
        x=Xtest[i] # the feature vector of the i-th data sample
        #########################################
        ## INSERT YOUR CODE HERE (5 points)
        Ytest[i], P[i] = inference(x, W, b)
        #########################################
    return Ytest, P
    #-----------------
    '''
        TEST: Now you can test the correctness of your code above by typing the following in the terminal:
        ---------------------------------------------------
        nosetests -v test2.py:test_predict
        --- OR ----
        python3 -m nose -v test2.py:test_predict
        --- OR ----
        python -m nose -v test2.py:test_predict
        ---------------------------------------------------
    '''

#--------------------------------------------

'''
    TEST problem 2:
        Now you can test the correctness of all the above functions by typing the following in the terminal:
        ---------------------------------------------------
        nosetests -v test2.py
        --- OR ----
        python3 -m nose -v test2.py
        --- OR ----
        python -m nose -v test2.py
        ---------------------------------------------------

        If your code passed all the tests, you will see the following message in the terminal:
        ----------- Problem 2 (30 points in total)--------------------- ... ok
        * (1 points) compute_z ... ok
        * (1 points) compute_dz_db ... ok
        * (1 points) compute_dL_db ... ok
        * (1 points) compute_dz_dW ... ok
        * (2 points) compute_dL_dW ... ok
        * (2 points) compute_a ... ok
        * (2 points) compute_da_dz ... ok
        * (2 points) compute_dL_dz ... ok
        * (2 points) compute_L ... ok
        * (2 points) compute_dL_da ... ok
        * (1 points) forward ... ok
        * (4 points) backward ... ok
        * (1 points) update_b ... ok
        * (1 points) update_W ... ok
        * (1 points) train ... ok
        * (1 points) inference ... ok
        * (5 points) predict ... ok
        ----------------------------------------------------------------------
        Ran 17 tests in 1.189s

        OK
'''

#--------------------------------------------

#--------------------------------------------
'''
    List of All Variables

* p:  the number of input features.
* c:  the number of classes in the classification task, an integer scalar.
* x:  the feature vector of a data instance, a float numpy vector of length p.
* y:  the label of one data sample, an integer scalar value. The values can be 0,1,2, ..., or (c-1).
* W:  the weight matrix of softmax regression, a float numpy matrix of shape (c by p).
* b:  the bias values of softmax regression, a float numpy vector of length c.
* z:  the linear logits, a float numpy vector of length c..
* a:  the softmax activations, a float numpy vector of length c..
* L:  the multi-class cross entropy loss, a float scalar..
* dL_da:  the partial gradients of the loss function L w.r.t. the activations a, a float numpy vector of length c.  The i-th element dL_da[i] represents the partial gradient of the loss function L w.r.t. the i-th activation a[i],  d_L / d_a[i]..
* dL_dz:  the partial gradients of the loss function L w.r.t. the linear logits z, a float numpy vector of length c.  The i-th element dL_dz[i] represents the partial gradient of the loss function L w.r.t. the i-th linear logit z[i],  d_L / d_z[i]..
* da_dz:  the partial gradient of the activations a w.r.t. the logits z, a float numpy matrix of shape (c by c).  The (i,j)-th element of da_dz represents the partial gradient ( d_a[i]  / d_z[j] ).
* dz_dW:  the partial gradient of logits z w.r.t. the weight matrix W, a numpy float tensor of shape (c by c by p).  The (i,j,k)-th element of dz_dW represents the partial gradient of the i-th logit (z[i]) w.r.t. the weight W[j,k],   d_z[i] / d_W[j,k].
* dz_db:  the partial gradient of the logits z w.r.t. the biases b, a float matrix of shape c by c.  Each element dz_db[i,j] represents the partial gradient of the i-th logit z[i] w.r.t. the j-th bias b[j],   d_z[i] / d_b[j].
* dL_dW:  the partial gradients of the loss function L w.r.t. the weight matrix W, a numpy float matrix of shape (c by p).  The i,j-th element dL_dW[i,j] represents the partial gradient of the loss function L w.r.t. the i,j-th weight W[i,j],  d_L / d_W[i,j].
* dL_db:  the partial gradient of the loss function L w.r.t. the biases b, a float numpy vector of length c.  The i-th element dL_db[i] represents the partial gradient of the loss function w.r.t. the i-th bias,  d_L / d_b[i].
* alpha:  the step-size parameter of gradient descent, a float scalar.
* n_epoch:  the number of passes to go through the training dataset in the training process, an integer scalar.
* n:  the number of data instance in the training set.
* n_test:  the number of data instance in the test set.
* X:  the feature matrix of training instances, a float numpy matrix of shape (n by p).
* Y:  the labels of training instance, a numpy integer numpy array of length n. The values can be 0, 1, ..., or c-1.
* Xtest:  the feature matrix of test instances, a float numpy matrix of shape (n_test by p).
* Ytest:  the predicted labels of test data samples, an integer numpy array of length ntest. Ytest[i] represents the predicted label on the i-th test sample.
* P:  the predicted probabilities of test data samples to be in different classes, a float numpy matrix of shape (n_test,c).  P[i,j] is the probability of the i-th data sample to have the j-th class label.

'''
#--------------------------------------------


#! /usr/bin/env python

"""
File: yx_ODE
Copyright (c) 2016 Chinmai Raman
License: MIT
Course: PHYS227
Assignment: C.3
Date: April 4th, 2016
Email: raman105@mail.chapman.edu
Name: Chinmai Raman
Description: Uses a forward Euler method to solve an ODE problem for a varying step size in x
"""

from __future__ import division
import matplotlib.pyplot as plt
import numpy as np
import sympy as sp

def euler(step, y0):
    """
    Implements Euler's method for the differential equation dy/dx = 1/(2(y-1)) on the interval [0,4]
    """
    x = [0]
    index_x = 0
    while x[index_x] < 4:
        x.append(x[index_x] + step)
        index_x += 1
    index_y = 0
    y = [y0]
    def yprime(y):
        yprime = 1 / (2 * (y - 1))
        return yprime
    while index_y < index_x:
        y.append(y[index_y] + step * yprime(y[index_y]))
        index_y += 1
    return x, y

def graph_euler():
    """
    Graphs the differential equation dy/dx = 1/(2(y-1)) given y(0) = 1 + sqrt(epsilon) for vary
    """
    fig = plt.figure(1)
    x1 = euler(1, 1 + np.sqrt(1e-3))[0]
    y1 = euler(1, 1 + np.sqrt(1e-3))[1]
    x2 = euler(0.25, 1 + np.sqrt(1e-3))[0]
    y2 = euler(0.25, 1 + np.sqrt(1e-3))[1]
    x3 = euler(0.01, 1 + np.sqrt(1e-3))[0]
    y3 = euler(0.01, 1 + np.sqrt(1e-3))[1]
    x_actual = np.linspace(0, 6, 1001)
    y_actual = 1 + np.sqrt(x_actual + 1e-3)
    plt.plot(x1, y1, 'r-', label = "line 1")
    plt.plot(x2, y2, 'g-')
    plt.plot(x3, y3, 'b-')
    plt.plot(x_actual, y_actual, 'y-')
    plt.xlabel('x')
    plt.ylabel('y')
    plt.title('f(x)')
    plt.show()

def sym_solve():
    x = sp.symbols('x')
    f = sp.Function('f')
    return sp.dsolve(sp.Eq(sp.Derivative(f(x), x), 1 / (2 * (f(x) - 1))))[1]

def sym_euler(step, y0):
    x = np.linspace(0, 4, 4 / step)

def test_euler():
    assert euler(0.0001, 1 + np.sqrt(1e-3))[1][-1] - 3.0003019764418086 < 1e-10


#!/usr/bin/python
"""
author  : Bhekimpilo Ndhlela
author  : 18998712
module  : Applied Mathematics(Numerical Methods) TW324
task    : computer assignment 04 question 5(a)
since   : Friday-23-03-2018
"""
def question_a(n=8.0, debug=True):
    h = (1.0 / (n - 1.0))
    f = array([exp(x) for x in linspace(0.0, 1.0, 8.0)])
    A = matrix([[1,-2, 1, 0, 0, 0, 0, 0],\
                [0, 1,-2, 1, 0, 0, 0, 0],\
                [0, 0, 1,-2, 1, 0, 0, 0],\
                [0, 0, 0, 1,-2, 1, 0, 0],\
                [0, 0, 0, 0, 1,-2, 1, 0],\
                [0, 0, 0, 0, 0, 1,-2, 1]])
    f_ = (1. / h**2.) * matmul(A, f).T

    """
    plot the approximation of the curve unnecessary for question
    5. I am doing it as a means of viewing the solution of the
    approximation.
    """
    plt.plot(linspace(0.0, 1.0, 6.0), f_, 'ro')
    plt.plot(linspace(0.0, 1.0, 6.0), f_, 'k--')
    plt.title("The Approximation of the Function exp(x) between [0;1]")
    plt.xlabel("x")
    plt.ylabel("f\"(x) = exp(x)")
    plt.show()

    if debug is True:
        print "DEBUG MODE: [ON] \t [QUESTION 5 A.)]"
        print "f\"(x) = ",
        for i in f_:
            print i,
        print "\nDim(f_) = ", shape(f_), "\t",\
              "Dim(A)  = ", shape(A),  "\t",\
              "Dim(f)  = ", shape(f)

if __name__ == "__main__":
    from numpy import (linspace, exp, array, shape, matrix, matmul)
    import matplotlib.pyplot as plt

    question_a()
else:
    from sys import exit
    exit("USAGE: python q5.py")


import numpy as np

def exp_softmax(x):
    """ Exponential softmax (numerically stable)
        Args:
            x (numpy.ndarray) : 1D vector.
        Returns:
            (numpy.ndarry) : softmax values of elements of x.
    """
    rebase_x = x - np.max(x)
    return np.exp(x) / np.sum(np.exp(x))

def log_softmax(q):
    """Logaritmic softmax (numerically stable)
        Args:
            q (numpy.ndarray) : 1D vector
        Returns:
            logarithmic softmax
    """
    max_q = max(0.0, np.max(q))
    rebased_q = q - max_q
    return rebased_q - np.logaddexp(-max_q, np.logaddexp.reduce(rebased_q))

class SoftMax:

    def __init__(self,Q,temp=1000):
        """ Softmax action-selection according to a Boltzmann/Gibbs distribution.
            Args:
                Q (numpy.ndarray) : Q-value table (states,actions)
                temp (int) : temperature
        """
        if len(Q.shape) == 2:
            self._num_states  = Q.shape[0]
            self._num_actions = Q.shape[1]
        else:
            raise ValueError('Q is not a 2-dimensional numpy.ndarray')

        self._Q = Q
        self.temp = temp
        self._P = np.ones(self._num_actions) / self._num_actions
        self._action_idx = range(0,self._num_actions)

    def action(self,state):
        """ Returns an action sampled from a Gibbs probability distribution
            Args:
                state (int) : state index (row of Q).

            Return:
                    (int) : action index (column of Q).
        """
        return np.random.choice(self._action_idx,1,p=list(exp_softmax(self._Q[state,:]/self.temp)))[0]


import numpy as np
import matplotlib.pyplot as plt

def f1(x, y):
    """ Bottom boundary
    """
    return 0

def f2(x, y):
    """ Left boundary
    """
    return 0

def f3(x, y):
    """ Right boundary
    """
    return 0

def f4(x, y):
    """ Top boundary
    """
    return 10 * np.sin(np.pi * x)

def save_fig(x, y, U, file_name):
    """ Save the figure of the approximate solution.

    Args:
        x: ndarray, the split of x interval
        y: ndarray, the split of y interval
        U: ndarray, the value of the coordiantes
        file_name: string, the picture name to be saved
    """
    fig = plt.figure()
    ax = fig.add_subplot(111, projection='3d')
    X, Y = np.meshgrid(x, y)
    ax.plot_surface(X, Y, U)
    ax.set_xlabel('x')
    ax.set_ylabel('y')
    ax.set_zlabel('z')
    ax.set_title("Approximate solution of Laplace equation by five-points")
    plt.savefig(file_name)

def five_point_difference_scheme(x_inte, y_inte, M, N, func1, func2, func3, func4):
    """ Solve Laplace equation by five point difference scheme.

    Args:
        x_inte: list/ndarray, the boundary of x
        y_inte: list/ndarray, the boundary of y
        M: int, number of endpoints
        N: int, number of endpoints
        func1: function, bottom boundary
        func2: function object, left boundary
        func3: function object, right boundary
        func4: function object, top boundary
    """
    # cofficients matrix
    B = np.diag(4*np.ones(N-1)) + np.diag(-np.ones(N-2), 1) + np.diag(-np.ones(N-2), -1)
    I = - np.eye(N-1)
    A = np.kron(np.diag(np.ones(M-2), -1), I) + np.kron(np.diag(np.ones(M-2), 1), I) + np.kron(np.diag(np.ones(M-1)), B)

    x = np.linspace(x_inte[0], x_inte[1], M+1)
    y = np.linspace(y_inte[0], y_inte[1], N+1)
    # boundary condition
    b = np.zeros((M-1, N-1))
    for i in range(M-1):
        for j in range(N-1):
            if i == 0:
                if j == 0:
                    b[i, j] = func1(x[i], y[j]) + func2(x[i], y[j+1])
                elif j == N-2:
                    b[i, j] = func4(x[i+1], y[j+2])  + func2(x[i], y[j+1])
                else:
                    b[i, j] = func2(x[i], y[j+1])
            elif i == N-2:
                if j == 0:
                    b[i, j] = func1(x[i+1], y[j]) + func3(x[i+2], y[j+1])
                elif j == N-2:
                    b[i, j] = func4(x[i+1], y[j+2]) + func3(x[i+2], y[j+1])
                else:
                    b[i, j] = func3(x[i+2], y[j+1])
            else:
                if j == 0:
                    b[i, j] = func1(x[i+1], y[j])
                elif j == N-2:
                    b[i, j] = func4(x[i+1], y[j+2])
                else:
                    b[i, j] = 0

    U = (np.linalg.inv(A) @ b.T.reshape(-1, 1)).reshape(M-1, N-1)

    # show the approximate solution
    x1 = np.arange(x_inte[0]+(x_inte[1]-x_inte[0])/M, x_inte[1], (x_inte[1]-x_inte[0])/M)
    y1 = np.arange(y_inte[0]+(y_inte[1]-y_inte[0])/N, y_inte[1], (y_inte[1]-y_inte[0])/N)
    save_fig(x1, y1, U, 'figure_five_points.png')

if __name__ == '__main__':
    M = N = 100
    x_inte = y_inte = [0, 1]
    five_point_difference_scheme(x_inte, y_inte, M, N, f1, f2, f3, f4)


import numpy.linalg as lin
import math as m

def ccwRot(deg):
    rads = m.radians(deg)
    M = [[m.cos(rads), -m.sin(rads), 0],
     [m.sin(rads), m.cos(rads), 0],
     [0, 0, 1]]
    return M

def cwRot(deg):
    M = ccwRot(deg)
    M = lin.inv(M)
    return M

def solve():
    print("Problem 1, using a rotation of 45 degrees:")

    angle = 45
    for row in ccwRot(angle):
        print(row)
    print()
    for row in cwRot(angle):
        print(row)


# Author: Filipe Chagas Ferraz
# Description: Linear Regression program
# Programming Language: Python 2
# Email: filipe.ferraz0@gmail.com
# Github account: github.com/FilipeChagasDev
# License: MIT License

import numpy as np
import matplotlib.pyplot as plt
import math
import random as rand
import os.path
from os import path

# function that loads the text file and turns it into a list of tuples
def load_samples (path):
    samples_file = open(path, "r")
    samples_text = "[" + samples_file.read() + "]"
    return eval(samples_text)

# square error derivative in function of angular coefficient
def SE_angular_coeff_diff (angular_coeff,linear_coeff,sample_x,sample_y):
    # SE = ( a * x + b - y)^2
    # dSE/da = 2*x*(a*x + b - y)
    # where:
    #   a = angular_coeff
    #   b = linear_coeff
    #   x = sample_x
    #   y = sample_y

    return 2 * sample_x * ( angular_coeff * sample_x + linear_coeff - sample_y )

# square error derivative in function of linear coefficient
def SE_linear_coeff_diff (angular_coeff,linear_coeff,sample_x,sample_y):
    # SE = ( a * x + b - y)^2
    # dSE/db = 2*(a*x + b - y)
    # where:
    #   a = angular_coeff
    #   b = linear_coeff
    #   x = sample_x
    #   y = sample_y

    return 2 * ( angular_coeff * sample_x + linear_coeff - sample_y )

# mean square error derivative in function of angular coefficient
def MSE_angular_coef_diff (angular_coeff, linear_coeff, samples):
    # MSE = 1/N * SE_1 + SE_2 + ... + SE_N
    # dMSE/da = 1/N * dSE_1/da * dSE_2/da + ... + dSE_N/da
    # where:
    #   dSE/da = SE_angular_coeff_diff(angular_coeff, linear_coeff, x, y)
    summation = 0
    for sample in samples:
        summation += SE_angular_coeff_diff(angular_coeff, linear_coeff, sample[0], sample[1])

    return summation/samples.shape[0]

# mean square error derivative in function of linear coefficient
def MSE_linear_coef_diff (angular_coeff, linear_coeff, samples):
    # MSE = 1/N * SE_1 + SE_2 + ... + SE_N
    # dMSE/db = 1/N * dSE_1/db * dSE_2/db + ... + dSE_N/db
    # where:
    #   dSE/db = SE_angular_coeff_diff(angular_coeff, linear_coeff, x, y)
    summation = 0
    for sample in samples:
        summation += SE_linear_coeff_diff(angular_coeff, linear_coeff, sample[0], sample[1])

    return summation/samples.shape[0]

# function that returns true if v1 is close to v2 (within tolerance range)
def approximately (val1, val2, tolerance):
    return (val1 > val2 - tolerance ) and (val1 < val2 + tolerance)

# Gradient descent function. It returns the (angular_coeff, linear_coeff) tuple
def gradient_descent (samples, ratio, tolerance):
    current_angular_coeff = rand.random()*10
    current_linear_coeff = rand.random()*10

    while True:
        dMSE_over_da = MSE_angular_coef_diff(current_angular_coeff,current_linear_coeff,samples)
        dMSE_over_db = MSE_linear_coef_diff(current_angular_coeff,current_linear_coeff, samples)

        current_angular_coeff -= dMSE_over_da / ratio
        current_linear_coeff -=  dMSE_over_db / ratio

        if math.isnan(dMSE_over_da) or math.isnan(dMSE_over_db):
            raise Exception("Overflow problem in the gradient_descent iteration. Try a larger ratio.")

        if approximately(dMSE_over_da, 0, tolerance) and approximately(dMSE_over_db, 0, tolerance):
            return (current_angular_coeff, current_linear_coeff)

def main():
    samples_path = ""
    while True:
        samples_path = raw_input("enter the file path where the samples are (\"default\" or \"\" for testing): ")

        if samples_path in ["", "default"]:
            samples_path = "samples.txt"
            break
        elif path.exists(samples_path):
            break
        else:
            print("Invalid file path. Try again...")

    samples = np.array(load_samples(samples_path))

    ratio = float("nan")
    while True:
        ratio = float(raw_input("enter the gradient descent ratio (recommended 100): "))
        if ratio < 100:
            print("The ratio value cannot be less than 100. Try again...")
        else:
            break

    tolerance = float("nan")
    while True:
        tolerance = float(raw_input("enter the gradient descent tolerance (recommended 0.01): "))
        if tolerance >= 1:
            print("The tolerance most be less than 1. Try again...")
        else:
            break

    print("starting iteration...")

    (angular_coeff, linear_coeff) = gradient_descent(samples, ratio, tolerance)

    print("Regression successfully completed")
    print("f(x) = ax + b, where:")
    print("a = " + str(angular_coeff))
    print("b = " + str(linear_coeff))

    np_linear_func = np.vectorize(lambda x: angular_coeff * x + linear_coeff)

    plt.plot(samples[:,0],samples[:,1],"ro",
            samples[:,0],np_linear_func(samples[:,0]))

    plt.show()

main()


import numpy as np

def mishra_bird(x, *args):
    """Mishra's Bird constrained function with 2 parameters.

    To be used in the constrained optimization examples.

    When subject to:
        (x[0] + 5) ** 2 + (x[1] + 5) ** 2 < 25
    the global minimum is at f(-3.1302, -1.5821) = -106.7645

    Bounds: -10 <= x[0] <= 0
           -6.5 <= x[1] <= 0

    Reference:
    https://en.wikipedia.org/wiki/Test_functions_for_optimization
    """
    fx = np.sin(x[1]) * np.exp((1 - np.cos(x[0])) ** 2) + \
         np.cos(x[0]) * np.exp((1 - np.sin(x[1])) ** 2) + (x[0] - x[1]) ** 2

    return fx

def mishra_bird_constr(x, *args):
    """Constraint for the Mishra's Bird function."""
    fx = (x[0] + 5) ** 2 + (x[1] + 5) ** 2 - 25

    return fx * -1


#!/usr/bin/env python3
# -*- coding: utf-8 -*-

import numpy as np

"""
Ex)

c  h  r  i  s.
c  h  a  s  e.

Way #3:
Match c to c: cost 0
Match h to h: cost 0
Match r to a: cost 1
Delete i: cost 1
Match s to s: cost 0
Delete e: cost 1

Way #1:
Match c to c: cost 0
Match h to h: cost 0
Delete r: cost 1
Match i to a: cost 1
Match s to s: cost 0
Delete e: cost 1
Total cost: 3

Way #2:
match c to c
match h to h
match r to a (1)
match i to s (1)
match s to e (1)
Total cost: 3

dist("chris", "chris") = 0
dist("chris", "chase") > 0

1) Match characters.  If they're the same
   I don't pay a cost. If they're different
   pay a cost of 1
2) Delete a character at a cost of 1

Ex) school  fools

s  c  h  o  o  l.
f  o  o  l.  s

Delete s (cost 1)
Delete c (cost 1)
Match h to f (cost 1)
Match o to o
Match o to o
Match l to l
Delete s (cost 1)
Cost 4

"""

"""
How to solve edit distance recursively

fib(n) = fib(n-1) + fib(n-2)

edit("chris", "chase") = min(
    1 + edit("chri", "chase"), // Deleted s from "chris"
    1 + edit("chris", "chas"), // Deleted e from "chase"
    1 + edit("chri", "chas") // Match e to s
    )

edit("chri", "chase") = min(
                         1 + edit("chr", "chase")
                         1 + edit("chri", "chas")
                         1 + edit("chr", "chas")
                         )
Stopping condition:
edit("", "chas")
edit("", s) = # characters in s

"""

count = 0

def edit_naive(a, b):
    global count
    count += 1
    result = -1
    if len(a) == 0:
        result = len(b)
    elif len(b) == 0:
        result = len(a)
    else:
        # Delete the last character from a
        cost1 = 1 + edit_naive(a[0:-1], b)
        # Delete the last character from b
        cost2 = 1 + edit_naive(a, b[0:-1])
        # Match the last character from a to last in b
        cost3 = 0 + edit_naive(a[0:-1], b[0:-1])
        if a[-1] != b[-1]:
            cost3 += 1
        result = min(cost1, cost2, cost3)
    return result

def edit(a, b, table):
    global count
    count += 1
    i = len(a)
    j = len(b)
    result = -1
    if table[i, j] == -1:
        if len(a) == 0:
            result = len(b)
        elif len(b) == 0:
            result = len(a)
        else:
            # Delete the last character from a
            cost1 = 1 + edit(a[0:-1], b, table)
            # Delete the last character from b
            cost2 = 1 + edit(a, b[0:-1], table)
            # Match the last character from a to last in b
            cost3 = 0 + edit(a[0:-1], b[0:-1], table)
            if a[-1] != b[-1]:
                cost3 += 1
            result = min(cost1, cost2, cost3)
        table[i, j] = result
    else:
        result = table[i, j]
    return result

LEFT = 0
UP = 1
DIAG = 2

def edit_iter(s1, s2):
    global count
    table = -1*np.ones((len(s1)+1, len(s2)+1))
    moves = np.zeros_like(table)
    # Base cases / stopping conditions
    table[0, :] = np.arange(table.shape[1])
    moves[0, :] = LEFT
    table[:, 0] = np.arange(table.shape[0])
    moves[:, 0] = UP
    for i in range(1, len(s1)+1): # Row
        for j in range(1, len(s2)+1): # Col
            count += 1
            cost = 0
            if s1[i-1] != s2[j-1]:
                cost = 1
            costs = np.array([table[i, j-1] + 1, # left
                              table[i-1, j] + 1, # up
                              table[i-1, j-1] + cost]) # diag
            idx = np.argmin(costs)
            table[i, j] = costs[idx]
            moves[i, j] = idx

    # Backtrace
    # Start at the lower right, and follow the arrows
    # until we get to the upper left
    path = [[len(s1), len(s2)]]
    # while row of the last thing you did is not zero
    # OR the col of the last thing did is not zero
    while path[-1][0] != 0 or path[-1][1] != 0:
        i, j = path[-1]
        if moves[i, j] == LEFT:
            j = j - 1
        elif moves[i, j] == UP:
            i = i - 1
        elif moves[i, j] == DIAG:
            i = i - 1
            j = j - 1
        path.append([i, j])

    # Return element in lower right corner for
    # the cost of matching the entire strings
    return table[-1, -1], path

s1 = "school"
s2 = "fools"

count = 0
print(edit_naive(s1, s2))
print("count = ", count)

count = 0
table = -np.ones((len(s1)+1, len(s2)+1))
print(edit(s1, s2, table))
print("count = ", count)

count = 0
cost, path = edit_iter(s1, s2)
print(cost)
print(path)


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Wed Sep  4 09:57:04 2019

@author: amandaash
"""
import numpy as np
import matplotlib.pyplot as plt

def harmonic(N):
    """
    input:
        N - number of iterations in the summation.
    output:
        harmonic1 - upward harmonic series
        harmonic2 - downward harmonic series
    """
    harmonic1 = 0
    harmonic2 = 0
    for iteration in range(1,N+1,1):
        harmonic1 = harmonic1 + (1/iteration)
    for iteration in range(N,0,-1):
        harmonic2 = harmonic2 + 1/iteration
    return harmonic1, harmonic2

n_val = []
harm_err = []
for n in range(1000):
    n_val.append(n)
    harm1, harm2 = harmonic(n)
    #print(harmonic1, harmonic2)
    harmonic_error = (harm1-harm2)/(np.abs(harm1)+np.abs(harm2))
    harm_err.append(harmonic_error)

plt.plot(n_val, harm_err, '.', lw = 1)
plt.show()

"""
When N = 6, the analytical solution to the harmonic series is 2.45. The downward harmonic series is equal to
this value, making the downward solution more precise. The upward harmonic series has extra digits.

The reason this is the case is because when you start with a large number, in this case 1, if you are adding
a smaller number, the smaller number is converted to scientific notation with the same exponent as the larger number.
This can truncate numbers which are greater than 16 decimal points away. For example, small numbers which would
normally round to some number would be saved as 0. However, if you start with a small number, as in the downward
series, the exponents are smaller initially, so more f the decimal points are saved.

"""


"""
CSMM.101X - Columbia University - Artificial Intelligence - Week 7 Project 2 - Linear Regression
"""
# Import necessary libraries
import sys
from os.path import exists
from numpy import ones, zeros
import pandas as pd

class LinearRegression:
    __slots__ = ('weights', 'data', 'X', 'y', 'learning_rates', 'max_iterations')

    def __init__(self, csv_file=None, learning_rates=None, max_iterations=100):
        self.weights = None
        self.data = None
        self.X = None
        self.y = None
        self.learning_rates = [0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1, 5, 10]
        self.max_iterations = len(self.learning_rates) * [max_iterations]
        self.load_dataset(csv_file=csv_file)

    def load_dataset(self, csv_file=None, target_column=None, column_headers=None):

        if not isinstance(csv_file, str):
            raise TypeError(f"{csv_file} should be  {str}")

        if target_column is not None and isinstance(target_column, str):
            raise TypeError(f" {target_column} should be {str}")

        if not exists(csv_file):
            raise IOError(f"{csv_file} file path does not exist")

        self.data = pd.read_csv(csv_file, header=column_headers)

        # Setting the last column as target column
        if target_column is None:
            target_column = self.data.columns[len(self.data.columns) - 1]

        # Setting y value as target column
        self.y = self.data[target_column]

        # Dropping the target column from dataset
        self.data.drop(columns=[target_column], inplace=True)  # The data changed here.

        # Adding Bias Column to the Input Data
        column_index = len(self.data.columns)
        self.normalize_data()
        self.data.insert(column_index, column=column_index, value=ones(len(self.data)))
        self.X = self.data

    def normalize_data(self):

        # Mean of Each Column
        mean = self.data.mean(axis=0)
        standard_deviation = self.data.std(axis=0)
        self.data = self.data - mean
        self.data = self.data / standard_deviation

    def train_model(self, output_csv_file='linear_regression_output.csv'):
        d = len(self.data.columns)
        output_vector = []
        n = len(self.X)

        for alpha, iterations in zip(self.learning_rates, self.max_iterations):

            # Initialize the beta to 0
            self.weights = zeros(d)
            for i in range(iterations):
                fx = self.X.dot(self.weights)
                temp1 = fx - self.y
                temp2 = (self.X.T * temp1).sum(axis=1)
                self.weights = self.weights - ((alpha / n) * temp2)

            weights = self.weights.tolist()
            output_vector.append(",".join(map(str, [alpha, iterations, weights[-1], *weights[:-1]])) + "\n")

        if output_csv_file is not None:
            try:
                with open(output_csv_file, "w") as output_csv:
                    output_csv.writelines(output_vector)
            except Exception as e:
                print(e)

def main():
    if len(sys.argv) < 3:
        print("python3 problem2.py input2.csv output2.csv")
        sys.exit(1)

    input_csv = str(sys.argv[1])
    output_csv = str(sys.argv[2])

    linear_regression = LinearRegression(csv_file=input_csv)
    linear_regression.train_model(output_csv_file=output_csv)

if __name__ == "__main__":
    main()

Week 7 Projects/2. Linear Regression/problem2.py


import numpy as np
import math

def sum_squared_error( outputs, targets, derivative=False ):
    if derivative:
        return outputs - targets
    else:
        return 0.5 * np.mean(np.sum( np.power(outputs - targets,2), axis = 1 ))
#end cost function

def hellinger_distance( outputs, targets, derivative=False ):
    """
    The output signals should be in the range [0, 1]
    """
    root_difference = np.sqrt( outputs ) - np.sqrt( targets )

    if derivative:
        return root_difference/( np.sqrt(2) * np.sqrt( outputs ))
    else:
        return np.mean(np.sum( np.power(root_difference, 2 ), axis=1) / math.sqrt( 2 ))
#end cost function

def binary_cross_entropy_cost( outputs, targets, derivative=False, epsilon=1e-11 ):
    """
    The output signals should be in the range [0, 1]
    """
    # Prevent overflow
    outputs = np.clip(outputs, epsilon, 1 - epsilon)
    divisor = np.maximum(outputs * (1 - outputs), epsilon)

    if derivative:
        return (outputs - targets) / divisor
    else:
        return np.mean(-np.sum(targets * np.log( outputs ) + (1 - targets) * np.log(1 - outputs), axis=1))
#end cost function
cross_entropy_cost = binary_cross_entropy_cost

def softmax_categorical_cross_entropy_cost( outputs, targets, derivative=False, epsilon=1e-11 ):
    """
    The output signals should be in the range [0, 1]
    """
    outputs = np.clip(outputs, epsilon, 1 - epsilon)

    if derivative:
        return outputs - targets
    else:
        return np.mean(-np.sum(targets * np.log( outputs ), axis=1))
#end cost function
softmax_neg_loss = softmax_categorical_cross_entropy_cost


import math
import numpy

def gaussian(x, mu, sig):
    '''
    Gaussian function.
    '''
    return (1 / math.sqrt(2 * math.pi * sig ** 2)) * \
        numpy.exp(-(x - mu) ** 2 / (2 * sig ** 2))

def logistic(x, x0, L, M, k):
    '''
    Logistic function.
    '''
    return M + (L / (1 + numpy.exp(-k * (x - x0))))

def entropy (num1, num2):
    percnt1 = num1/(num1+num2)
    percnt2 = num2/(num1+num2)
    return (-(percnt1*math.log2(percnt1) + percnt2*math.log2(percnt2)))

if __name__ == "__main__":
    print(entropy(5,9))
    print(entropy(3,2))
    print(entropy(2,3))


## QR Algorithm
# Author: Kelly Rivera

# This code is an implementation of a PageRank Algorithm variant using QR Decomposition
# Implementing Householder and Gram-Schmidt Process

### Import Libraries
import numpy as npy
import scipy as sp
import scipy.linalg as spl  # SciPy Linear Algebra Library
import matplotlib.pyplot as plt

### Householder transformation method
def qr_householder(x):
    Q, R = npy.linalg.qr(x)
    return Q, R

def qr_Algorithm_HH(x,iterations):
    m, n = x.shape
    Q_last = npy.identity(n)
    for _ in range(iterations):
        Q, R = qr_householder(x)
        # This update matrix A, by dot product of reverse QR
        x = npy.dot(R, Q)

        eigenvectors = npy.dot(Q_last, Q)
        Q_last = eigenvectors

    eigenvalues = npy.diag(x)  #outputs only the values on the main diaginal
    return eigenvalues, eigenvectors

### Gram-Schmidt Process
def qr_GS(x):
    m, n = x.shape
    Q = npy.zeros((m, n)) # Q is all zero
    R = npy.zeros((n, n))

    for j in range(n):
        v = x[:, j] # x[:,0] returns the first* column* of x, x[:,1] reutrns the second column etc

        for i in range(j):
            q = Q[:, i]
            R[i, j] = q.dot(v)
            v = v - R[i, j] * q

        vNorm = npy.linalg.norm(v)
        R[j, j] = vNorm
        Q[:, j] = v / vNorm

    return Q, R

def qr_Algorithm(x,iterations):
    m, n = x.shape
    Q_last = npy.identity(n)
    for _ in range(iterations):
        Q, R = qr_GS(x)
        # This update matrix A, by dot product of reverse QR
        x = npy.dot(R, Q)

        eigenvectors = npy.dot(Q_last, Q)
        Q_last = eigenvectors
    eigenvalues = npy.diag(x)  #outputs only the values on the main diaginal
    return eigenvalues, eigenvectors

### QR Gram-Schmidt with Shift
def shiftedQR_Algorithm(x, iterations):
    m, n = x.shape
    I = npy.identity(n)
    Q_last = npy.identity(n)

    for _ in range(iterations):
        μ = x[[n-1],[n-1]]         # shift: μ = a_nn
        Q, R = qr_GS((x - (μ*I)))  # A − μI = QR: preforms the gram schmidt process with shift

        x = npy.dot(R,Q) + (μ*I)   # This update matrix A, by dot product of reverse QR and adding back the shift

        eigenvectors = npy.dot(Q_last, Q)
        Q_last = eigenvectors

    eigenvalues = npy.diag(x)  #outputs only the values on the main diaginal
    return eigenvalues, eigenvectors


# coding: utf-8

# In[11]:

from scipy.stats import beta
import matplotlib.pyplot as plt
import numpy as np
plt.rcParams['figure.figsize']=(7,5)

# In[3]:

a=np.array([0.1,1.0,2.0,8.0])
b=np.array([0.1,1.0,3.0,4.0])

# In[14]:

for (new_a,new_b) in zip(a,b):
    print(new_a,new_b)
    x = np.linspace(beta.ppf(0.0001,new_a,new_b),
                   beta.ppf(0.9999,new_a,new_b),100)
    plt.plot(x,beta.pdf(x,new_a,new_b),label='a={},b={}'.format(new_a,new_b))
ax = plt.gca()
ax.set_ylim((0,3))
ax.set_xlim((0,1))
ax.legend()


import numpy as np
import matplotlib.pyplot as plt

def lin_reg_coef(x_list, y_list):
    """
    function that returns Linear Regression coefficients

    Inputs:
    1. x_list : list of data on x-axis
    2. y_list : list of data on y-axis

    Output: Tuple of Linear Regression Coefficients
    """
    # converting the list into array
    x = np.array(x_list)
    y = np.array(y_list)

    # number of observations/points
    n = np.size(x)

    # mean of x and y vector
    m_x = np.mean(x)
    m_y = np.mean(y)

    # calculating cross-deviation and deviation about x
    SS_xy = np.sum(y*x) - n*m_y*m_x
    SS_xx = np.sum(x*x) - n*m_x*m_x

    # calculating regression coefficients
    b_1 = SS_xy / SS_xx
    b_0 = m_y - b_1*m_x
    #c = b_0, m = b_1 in y=mx+c
    return (b_0, b_1)


import numpy as np
import matplotlib.pyplot as plt
pixel=30
img = np.fromfunction(lambda u,v: np.sin(u)+np.cos(v) ,(pixel,pixel))

print(img)
fourier= np.fft.fft2(img)
fourier_s=np.fft.fftshift(fourier)
print(fourier)
plt.subplot(1,2,1)
plt.imshow(img)
plt.subplot(1,2,2)
plt.imshow(abs(fourier_s))
plt.show()
# magnitude_spectrum = 20*np.log(np.abs(F2))
# b.pltx(bw_img,magnitude_spectrum)


# loss function损失函数常用的mean squared error（均方误差） 0.5 * (for i in range(n): np.sum((yi - ti) ** 2)
import numpy as np
# y: softmax function output
# t：test data set label
def mean_squared_error(y , t):
    return 0.5 * np.sum((y - t) ** 2)

# 人为设定test label为2，即当前标签为digit 2
t = [0, 0, 1, 0, 0, 0, 0, 0, 0, 0]

# example 1
# y1中代表softmax predict 从0～9的概率
y1 = [0.1, 0.05, 0.6, 0.0, 0.05, 0.1, 0.0, 0.1, 0.0, 0.0]
print('y1: ', mean_squared_error(np.array(y1), np.array(t)))

# example 2
y2 = [0.1, 0.05, 0.1, 0.0, 0.05, 0.1, 0.0, 0.6, 0.0, 0.0]
print('y2: ', mean_squared_error(np.array(y2), np.array(t)))

'''
由mean_squared_error可以看出第一个神经网络输出的结果比第二个神经网络的误差较小，所以example1更加接近正解
'''


#!/usr/bin/env python
#
# Author: Ying Xiong.
# Created: Mar 19, 2014.

import numpy as np

def numerical_jacobian(fcn, x0, dx=1e-6, method=0, return_f0=False):
    """Compute the numerical Jacobian matrix of a given function.

    Parameters
    ----------
    fcn: function handle
        Takes an N-vector as input and return an M-vector.
    x0: ndarray
        An input N-vector.
    dx: scalar
        For small change in x0.
    method: int or string
        With following options:
          * {0, 'forward'}: compute the Jacobian as (f(x0+dx)-f(x0))/dx.
          * 1, 'central' : compute the Jacobian as (f(x0+dx)-f(x0-dx))/2/dx.
    return_f0: boolean
        If set to true, also return fcn(x0).

    Returns
    -------
    J : `ndarray`
        The `MxN` Jacobian matrix.
    f0 : `ndarray`
        The function value at x0.

    Examples
    --------
    >>> J = numerical_jacobian(fcn, x0, ...)
    >>> (J, f0) = numerical_jacobian(fcn, x0, ..., return_f0=True)
    """
    N = len(x0)
    if method==0 or method=="forward":
        # Forward difference.
        f0 = fcn(x0)
        M = len(f0)
        J = np.zeros((M,N))
        for j in xrange(N):
            xj = x0.copy()
            xj[j] += dx
            fj = fcn(xj)
            J[:,j] = (fj-f0) / dx
        if return_f0:
            return (J, f0)
        else:
            return J
    elif method==1 or method=="central":
        # Central difference.
        for j in xrange(N):
            xj1 = x0.copy()
            xj2 = x0.copy()
            xj1[j] -= dx
            xj2[j] += dx
            fj1 = fcn(xj1)
            fj2 = fcn(xj2)
            if j == 0:
                M = len(fj1)
                J = np.zeros((M,N))
            J[:,j] = (fj2-fj1) / 2 / dx
        if return_f0:
            return (J, fcn(x0))
        else:
            return J
    else:
        assert False, "Unknown method '" + method + "'."


class Solution:
    def climbStairs(self, n: int) -> int:
        # 我的思路是，不管前面怎么走，最后就剩1个台阶的时候，1种，2个台阶的时候2种；前面只是看得到多少条路径，分叉到最后一层，得到有多少种方法
        global comb_num
        comb_num = 0
        def count(s):
            global comb_num
            if s == 2:
                comb_num += 2
            elif s == 1:
                comb_num += 1
            else:
                steps = [1, 2]
                for step in steps:
                    count(s - step)
        count(n)
        return comb_num

    def climbStairs_Dynamic(self, n: int) -> int: # 动态规划
        import numpy as np
        steps = list(np.zeros(n + 1))
        for i, s in enumerate(steps):
            if i == 1:
                steps[i] = 1
            elif i == 2:
                steps[i] = 2
            else:
                steps[i] = steps[i - 1] + steps[i - 2]
        print(steps)
        return steps[n]

    def climbStairs_demo(self, n: int):
        if (n == 1):
            return 1
        if (n == 2):
            return 2
        output = self.climbStairs_demo(n - 1) + self.climbStairs_demo(n - 2)
        return output

nums = 38
solution = Solution()
output = solution.climbStairs(nums)
print(output)
output = solution.climbStairs_Dynamic(nums)
print(output)


"""
generate 5 different test signals of the form: a₁cos(ω₁t + φ₁)

John M. O' Toole, University College Cork
Started: 04-09-2019
last update: <2019-09-04 13:36:01 (otoolej)>
"""
import numpy as np
from matplotlib import pyplot as plt
import matplotlib

def gen_signals(sig_num=0, DBplot=True):
    """define some simple test signals of the form: a₁cos(ω₁t + φ₁)

    Parameters
    ----------
    sig_num: string, {'0', '1', '2', '3', '4'}
        which signal type
    DBplot: bool
        plot or not

    Returns
    -------
    x_dt : dict, keys: 'x' and 'fwe'
        returns test signal 'x' and the expected frequency-weighted energy 'fwe'
    """

    N = 256
    Nh = np.ceil(N / 2)
    n = np.arange(N)
    w1 = np.pi / (N / 32)
    w2 = np.pi / (N / 8)
    ph1 = -np.pi + 2 * np.pi * np.random.rand(1)
    ph2 = -np.pi + 2 * np.pi * np.random.rand(1)
    a1 = 1.3
    a2 = 3.1
    x1 = a1 * np.cos(w1 * n + ph1)
    x2 = a2 * np.cos(w2 * n + ph2)

    def sig0():
        # ---------------------------------------------------------------------
        # stationary signal: a₁cos(ω₁t + φ₁)
        # ---------------------------------------------------------------------
        x = x2  # / max(x2)
        extact_x = np.full(N, (a2 ** 2) * (w2 ** 2))
        return({'x': x, 'fwe': extact_x})

    def sig1():
        # ---------------------------------------------------------------------
        # concatentated (in time) signals: a₁cos(ω₁t + φ₁) PLUS a₂cos(ω₂t + φ₂)
        # ---------------------------------------------------------------------
        a1 = 1
        a2 = 1
        w1 = np.pi / (N / 32)
        w2 = np.pi / (N / 16)

        x = np.hstack((a1 * np.cos(w1 * np.arange(Nh + 1)),
                       a2 * np.cos(w2 * np.arange(Nh + 1, N))))
        extact_x = np.hstack((np.full((int(Nh)), a1 ** 2) * np.sin(w1) ** 2,
                              np.full((int(Nh)), a2 ** 2) * np.sin(w2) ** 2))
        return({'x': x, 'fwe': extact_x})

    def sig2():
        # ---------------------------------------------------------------------
        # signal with amplitude modulation: e^{rt} a₁cos(ω₁t + φ₁)
        # ---------------------------------------------------------------------
        r1 = 0.005
        x = np.exp(-r1 * n) * x1

        extact_x = (a1 ** 2) * np.exp(-2*r1*n) * (np.sin(w1) ** 2+r1 ** 2)
        return({'x': x, 'fwe': extact_x})

    def sig3():
        # ---------------------------------------------------------------------
        # signal with frequency modulation: a₁cos(φ(t))
        # ---------------------------------------------------------------------
        if_law = 0.1 + 0.3 * np.sin(n * np.pi / N)
        ph = np.cumsum(if_law)
        x = a1 * np.cos(ph + ph1)
        extact_x = (a1 ** 2) * (if_law) ** 2
        # extact_x = (a1 ** 2) * (np.sin(if_law)) ** 2
        return({'x': x, 'fwe': extact_x})

    def sig4():
        # ---------------------------------------------------------------------
        # sum of two signals: a₁cos(ω₁t + φ₁) + a₂cos(ω₂t + φ₂)
        # ---------------------------------------------------------------------
        x = x1 + x2
        edo_x1 = (a1 ** 2) * (w1 ** 2)
        edo_x2 = (a2 ** 2) * (w2 ** 2)
        a_12 = 2 * a1 * a2 * w1 * w2
        extact_x = edo_x1 + edo_x2 + a_12 * np.cos(n * (w1 - w2) + ph1 - ph2)
        return({'x': x, 'fwe': extact_x})

    def default_sig():
        # -------------------------------------------------------------------
        # default option
        # -------------------------------------------------------------------
        print('Invalid signal name; defaulting to sig0')
        return(sig0())

    # pick which function to execute
    which_sig = {'0': sig0, '1': sig1, '2': sig2, '3': sig3, '4': sig4}

    def get_sig(name):
        return which_sig.get(name, default_sig)()

    x_dt = get_sig(sig_num)

    if DBplot:
        plt.figure(10, clear=True)
        plt.plot(x_dt['x'], '-x')
        plt.plot(x_dt['fwe'], '-o')

    return(x_dt)

# testing :
# xd = gen_signals('0', True)


import numpy as np
from utils import to_ndarray
from utils.math_functions import euclidian_distance

class KMeans:

    k: int
    x: np.ndarray
    labels: np.ndarray
    gen_mode: str
    centroids: np.ndarray

    def __init__(self, k=3, gen_mode='rnd_element'):
        """
        Args:
            k (int, optional): number of centroids. Defaults to 3.
            gen_mode (str, optional): centroid generation mode. Defaults to 'rnd_element'.
                * 'rnd_element' will select k elements randomly.
                * 'rnd_values' generate random values between min and max of elements for each centroid
        """

        self.k = k
        self.gen_mode = gen_mode
        self.labels = np.array([])
        self._verify_params()

    def _verify_params(self):
        if self.gen_mode not in ['rnd_element', 'rnd_values']:
            raise Exception("incorrect centroid generation mode. available modes: 'rnd_element', and 'rnd_values'")

    def _init_centroids_random(self):
        min_values = self.x.min(axis=0)
        max_values = self.x.max(axis=0)

        centroid = []

        for i in range(self.k):
            centroid.append([])
            for j in range(self.x.shape[1]):
                centroid[i].append(np.random.uniform(low=min_values[j], high=max_values[j]))

        return np.array(centroid)

    def _init_centroids_by_select(self):
        attempts = 50
        centroids = None

        # Select k random points while not duplicates, or attempts == 0
        while attempts > 0:
            centroids_idx = np.random.choice(len(self.x), size=self.k, replace=False)
            centroids = self.x[centroids_idx]

            if len(np.unique(centroids, axis=0)) == self.k:
                return centroids

            attempts -= 1

        print('warn: two or more centroids are equal')

        return centroids

    def _init_centroids(self):
        if self.gen_mode == 'rnd_element':
            return self._init_centroids_by_select()
        elif self.gen_mode == 'rnd_values':
            return self._init_centroids_random()

        raise Exception("incorrect centroid generation mode. available modes: 'rnd_element', and 'rnd_values'")

    def _define_labels(self, elements):
        labels = []

        for element in elements:
            distances = euclidian_distance(element, self.centroids)
            labels.append(np.argmin(distances))

        return np.array(labels)

    def _calculate_centroids(self):
        centroids = []

        for index in range(self.k):
            mask = self.labels == index
            selected = self.x[mask]
            mean = np.mean(selected, axis=0)
            centroids.append(mean)

        return np.array(centroids)

    def fit(self, x, max_iter=1000):
        self._verify_params()

        self.x = to_ndarray(x)
        self.centroids = self._init_centroids()

        while max_iter > 0:
            max_iter -= 1

            labels = self._define_labels(self.x)
            if np.array_equal(labels, self.labels):
                break
            else:
                self.labels = labels
                self.centroids = self._calculate_centroids()

    def predict(self, x):
        values = to_ndarray(x)
        return self._define_labels(values)


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# Created by techno at 27/05/19

#Feature:  #Enter feature name here
# Enter feature description here

#Scenario:  # Enter scenario name here
# Enter steps here

import numpy as np

a = np.array([ True,True,False,False])
b = np.array([ True,False,True,False])

print ( np.logical_and(a,b))
print(np.logical_or(a,b))
print(np.logical_not(a,b))

"""
However, you can also compare entire arrays with each other! In this case,
 you use the np.array_equal() function. Just pass in the two arrays that
 you want to compare with each other, and you’re done.

Note that, besides comparing, you can also perform logical operations
on your arrays. You can start with np.logical_or(), np.logical_not()
and np.logical_and(). This basically works like your typical OR,
NOT and AND logical operations;

In the simplest example, you use OR to see whether your elements are the same
(for example, 1), or if one of the two array elements is 1. If both of them
are 0, you’ll return FALSE. You would use AND to see whether your second
element is also 1 and NOT to see if the second element differs from 1.
"""


import numpy as np
from datetime import datetime

a = np.array([[1, 2, 2],
	       [2, 7, 3],
	       [-1, 7, -3]],float)

b = np.array([3, 1, -14], float)

num_elements = len(b)

x = np.zeros(num_elements, float)

# Elimination

start_time = datetime.now()
# Go through the fixed rows
for i in range(num_elements-1):

# Take the rows under the fixed row, row by row inorder to apply the row operations
	for j in range(i + 1, num_elements):
		factor = a[i, i] / a[j, i]
		b[j] = b[i] - factor*b[j]
		# element subtraction
		for k in range(i, num_elements):
			a[j, k] = a[i, k] - factor*a[j, k]

# Back Substitution

# Start with last row
x[num_elements - 1] = b[num_elements - 1] / a[num_elements - 1, num_elements - 1]
for i in range(num_elements - 2, -1, -1):
	# Summation of terms
	terms = 0
	for j in range(i + 1, num_elements):
		terms += a[i, j] * x[j]
	x[i] = (b[i] - terms) / a[i, i]
total_time = datetime.now() - start_time
print(f'It took {total_time.microseconds} microseconds to solve the system.')
print('Solution saved to file')

# Save to file
np.savetxt('linear_sysFile.txt', x, fmt='%4.8f', delimiter=' ')


import numpy
import numpy as np

#Parametros de entrada
#A matriz nxn, #b matriz de valores independientes
#Salida
#x matriz  de las soluciones
def fact_lu(A, b):
    n = A.shape[0]#Obtiene el numero de filas de A
    m = A.shape[1]#Obtiene el numero de columnas de A
    t = b.shape[0]#Obtiene el numero de filas de b
    o = b.shape[1]#Obtiene el numero de columnas de B
    if n != m:#Comprueba que la matriz sea cuadrada
        raise ValueError("La matriz no es cuadrada")
    if t != n or o!=1:#Comprueba que las matriz b sea las dimensiones adecuadas
        raise ValueError("La matriz b no tiene las dimensiones correctas")
    if teorema_2_lu(A,n) == False:#Comprueba las submatrices sean invertibles
        raise ValueError("La matriz no cumple el teorema 2 de Fact LU")

    M = matriz_inf_lu(A, n)#Obtiene la matriz de inferior y superior en version LU

    y = hacia_Adelante(M[1], b, n)#Resuelve Ly=b
    x = hacia_Atras(M[0], y, n)#Resuleve Ux=y
    return x

#Parametros de entrada
#A matriz  cuadrada, #n largo de la matriz
#Salida
#Valor booleado que identifica si se cumple o no el teorema 2
#Evaluacion del Teorema de LU
def teorema_2_lu(A, n):#Comprueba que las submatrices tiene determinante diferente de 0
    Valor = True#Booleano que sirve para identificar si se cumplio o no el teorema
    for k in range(0, n):
        if np.linalg.det(A[0:k, 0:k]) == 0:
            Valor = False#Se detiene si una submatriz no cumple el teorema 2
            break
    return Valor#True se cumple, #False se incumple

#Parametros de entrada
#A matriz cuadrada, #n largo de la matriz
#Salida
#U matriz Triangular superior LU, #L Matriz Triangular inferior LU
#Matrices inferior y superior version Factorizacion de LU
def matriz_inf_lu(A,n):
    U=A
    L=np.zeros((n,n))

    for k in range(0,n-1):#Columnas
        for i in range(k+1,n):#Filas
            Mik=U[i,k]/U[k,k]
            L[i,k]=Mik#Agrega a la matriz los valores ocupuesto a los "Multiplicadores" de la matriz
            for j in range(k,n):#Ciclo para agregar los nuevos valores de la matriz U
                U[i,j]=U[i,j]-Mik*U[k,j]
                if i==j:
                    L[i,j]=1
    L[0,0]=1
    return U,L

#Parametros de entrada
#A matriz  triangular superior, #b nueva matriz de valores independientes, #n largo de la matriz
#Salida
#soluciones matriz de soluciones "x#"
#Sustitucion hacia atras
def hacia_Atras(A, b, n):
    soluciones = np.zeros((n, 1))
    i = n - 1;

    while (i >= 0):
        sumatoria = 0
        for j in range(i + 1, n):
            sumatoria = sumatoria + (A[i, j] * soluciones[j])

        x = (1 / A[i, i]) * (b[i] - sumatoria)
        soluciones[i] = x
        i = i - 1
    return soluciones

#Parametros de entrada
#A matriz  triangular inferior, #b nueva matriz de valores independientes, #n largo de la matriz
#Salida
#soluciones matriz de soluciones "x#"
#Sustitucion hacia adelante
def hacia_Adelante(A, b, n):
    soluciones = np.zeros((n, 1))#Matriz de soluciones
    soluciones[0] = b[0] / A[0, 0]#Primera solucion
    i = 1;

    while (i < n):#Mueve las filas
        sumatoria = 0
        for j in range(0, i):#Mueve columnas
            sumatoria = sumatoria + (A[i, j] * soluciones[j])#Sumatoria necesaria

        x = (b[i] - sumatoria) / A[i, i]#Calcular el valor de la solucion "x"
        soluciones[i] = x#Agrega la solucion a la matriz de soluciones
        i = i + 1
    return soluciones

if __name__ == '__main__':
    #Ejemplos de ingresos de datos
    A = np.matrix('4 -2 1; 20 -7 12;-8 13 17', float)
    A2 = np.matrix('25 15 -5 -10;15 10 1 -7;-5 1 21 4;-10 -7 4 18',float)

    b = numpy.matrix('11;70;17')
    b2 = np.matrix('-25;-19;-21;-5')
    print("Ejemplo 1 tiene como resultado:")
    print(fact_lu(A, b))
    print("Ejemplo 2 tiene como resultado:")
    print(fact_lu(A2, b2))

Catalogo/03. Sistemas de Ecuaciones/Python/fact_lu.py