###############################
######### Problem 2 ###########
###############################

""" Tools for calculating Gradient Descent for ||Ax-b||. """
import matplotlib.pyplot as plt
import numpy as np

def main():
    ################################################################################
    # Input Variables
    A = np.array([[15, 8], [6, 5]])  # do not change this until the last part
    b = np.array([4.5, 6])  # b in the equation ||Ax-b||
    initial_position = np.array([0, 0])  # position at iteration 0
    total_step_count = 200  # number of GD steps to take
    step_size = lambda i: 1/(1+i)  # step size at iteration i
    ################################################################################

    # computes desired number of steps of gradient descent
    positions = compute_updates(A, b, initial_position, total_step_count, step_size)

    # print out the values of the x_i
    print(positions)
    print(np.dot(np.linalg.inv(A), b))

    # plot the values of the x_i
    plt.scatter(positions[:, 0], positions[:, 1], c='blue')
    plt.scatter(np.dot(np.linalg.inv(A), b)[0],
                np.dot(np.linalg.inv(A), b)[1], c='red')
    plt.plot()
    plt.show()

def compute_gradient(A, b, x):
    """Computes the gradient of ||Ax-b|| with respect to x."""
    return np.dot(A.T, (np.dot(A, x) - b)) / np.linalg.norm(np.dot(A, x) - b)

def compute_update(A, b, x, step_count, step_size):
    """Computes the new point after the update at x."""
    return x - step_size(step_count) * compute_gradient(A, b, x)

def compute_updates(A, b, p, total_step_count, step_size):
    """Computes several updates towards the minimum of ||Ax-b|| from p.

    Params:
        b: in the equation ||Ax-b||
        p: initialization point
        total_step_count: number of iterations to calculate
        step_size: function for determining the step size at step i
    """
    positions = [np.array(p)]
    for k in range(total_step_count):
        positions.append(compute_update(A, b, positions[-1], k, step_size))
    return np.array(positions)

main()

###############################
######### Problem 4 ###########
###############################

from common import *

###############################
######### Part b ##############
###############################

########################################################################
#########  Gradient Computing and MLE ##################################
########################################################################
def compute_gradient_of_likelihood(single_obj_loc, sensor_loc, single_distance):
    """
    Compute the gradient of the loglikelihood function for part a.

    Input:
    single_obj_loc: 1 * d numpy array.
    Location of the single object.

    sensor_loc: k * d numpy array.
    Location of sensor.

    single_distance: k dimensional numpy array.
    Observed distance of the object.

    Output:
    grad: d-dimensional numpy array.

    """
    loc_diff = single_obj_loc - sensor_loc
    tmp = np.linalg.norm(loc_diff, axis=1)
    weight = (tmp - single_distance) / tmp
    grad = -np.sum(np.expand_dims(weight, axis=1) * loc_diff, axis=0)

    return grad

def find_mle_by_grad_descent_part_b(initial_obj_loc, sensor_loc, single_distance, lr=0.001, num_iters = 10000):
    """
    Compute the gradient of the loglikelihood function for part a.

    Input:
    initial_obj_loc: 1 * d numpy array.
    Initialized Location of the single object.

    sensor_loc: k * d numpy array. Location of sensor.

    single_distance: k dimensional numpy array.
    Observed distance of the object.

    Output:
    obj_loc: 1 * d numpy array. The mle for the location of the object.

    """
    obj_loc = initial_obj_loc
    for i in range(num_iters):
        obj_loc = obj_loc + lr * compute_gradient_of_likelihood(obj_loc, sensor_loc, single_distance)

    return obj_loc

if __name__ == "__main__":
    ########################################################################
    #########  MAIN ########################################################
    ########################################################################

    # Your code: set some appropriate learning rate here
    lr = 0.01

    np.random.seed(0)
    sensor_loc = generate_sensors()
    obj_loc, distance = generate_data(sensor_loc)
    single_distance = distance[0]
    print('The real object location is')
    print(obj_loc)
    # Initialized as [0,0]
    initial_obj_loc = np.array([[0.,0.]])
    estimated_obj_loc = find_mle_by_grad_descent_part_b(initial_obj_loc,
               sensor_loc, single_distance, lr=lr, num_iters = 10000)
    print('The estimated object location with zero initialization is')
    print(estimated_obj_loc)

    # Random initialization.
    initial_obj_loc = np.random.randn(1,2)
    estimated_obj_loc = find_mle_by_grad_descent_part_b(initial_obj_loc,
               sensor_loc, single_distance, lr=lr, num_iters = 10000)
    print('The estimated object location with random initialization is')
    print(estimated_obj_loc)

from common import *
#from part_b_starter import find_mle_by_grad_descent_part_b

#############################
######### Part c ############
#############################
def log_likelihood(obj_loc, sensor_loc, distance):
    """
    This function computes the log likelihood (as expressed in Part a).
    Input:
    obj_loc: shape [1,2]
    sensor_loc: shape [7,2]
    distance: shape [7]
    Output:
    The log likelihood function value.
    """
    diff = np.sqrt(np.sum((sensor_loc - obj_loc)**2, axis = 1)) - distance
    func_value = -sum((diff) ** 2) / 2
    return func_value

if __name__ == "__main__":
    ################################################################################
    ######### Compute the function value at local minimum for all experiments. #####
    ################################################################################
    num_sensors = 20

    np.random.seed(100)
    sensor_loc = generate_sensors(k=num_sensors)

    # num_data_replicates = 10
    num_gd_replicates = 100

    obj_locs = [[[i,i]] for i in np.arange(0,1000,100)]

    func_values = np.zeros((len(obj_locs),10, num_gd_replicates))
    # record sensor_loc, obj_loc, 100 found minimas
    minimas = np.zeros((len(obj_locs), 10, num_gd_replicates, 2))
    true_object_locs = np.zeros((len(obj_locs), 10, 2))

    for i, obj_loc in enumerate(obj_locs):
        for j in range(10):
            obj_loc, distance = generate_data_given_location(sensor_loc, obj_loc,
                                                           k = num_sensors, d = 2)
            true_object_locs[i, j, :] = np.array(obj_loc)

            for gd_replicate in range(num_gd_replicates):
                initial_obj_loc = np.random.randn(1,2) * (100 * i+1)
                obj_loc = find_mle_by_grad_descent_part_b(initial_obj_loc,
                         sensor_loc, distance[0], lr=0.1, num_iters = 1000)
                minimas[i, j, gd_replicate, :] = np.array(obj_loc)
                func_value = log_likelihood(obj_loc, sensor_loc, distance[0])
                func_values[i, j, gd_replicate] = func_value

    ########################################################################
    ######### Calculate the things to be plotted. ##########################
    ########################################################################
    local_mins = [[np.unique(func_values[i,j].round(decimals=2)) for j in range(10)] for i in range(10)]
    num_local_min = [[len(local_mins[i][j]) for j in range(10)] for i in range(10)]
    proportion_global = [[sum(func_values[i,j].round(decimals=2) == min(local_mins[i][j]))*1.0/100 \
                         for j in range(10)] for i in range(10)]

    num_local_min = np.array(num_local_min)
    num_local_min = np.mean(num_local_min, axis = 1)

    proportion_global = np.array(proportion_global)
    proportion_global = np.mean(proportion_global, axis = 1)

    ########################################################################
    ######### Plots. #######################################################
    ########################################################################
    fig, axes = plt.subplots(figsize=(8,6), nrows=2, ncols=1)
    fig.tight_layout()
    plt.subplot(211)

    plt.plot(np.arange(0,1000,100), num_local_min)
    plt.title('Number of local minimum found by 100 gradient descents.')
    plt.xlabel('Object Location')
    plt.ylabel('Number')
    #plt.show()
    #plt.savefig('num_obj.png')
    # Proportion of gradient descents that find the local minimum of minimum value.

    plt.subplot(212)
    plt.plot(np.arange(0,1000,100), proportion_global)
    plt.title('Proportion of GD that finds the global minimum among 100 gradient descents.')
    plt.xlabel('Object Location')
    plt.ylabel('Proportion')
    fig.tight_layout()
    #plt.show()
    #plt.savefig('prop_obj.png')

    ########################################################################
    ######### Plots of contours. ###########################################
    ########################################################################
    np.random.seed(0)
    # sensor_loc = np.random.randn(7,2) * 10
    x = np.arange(-10.0, 10.0, 0.1)
    y = np.arange(-10.0, 10.0, 0.1)
    X, Y = np.meshgrid(x, y)
    obj_loc = [[0,0]]
    obj_loc, distance = generate_data_given_location(sensor_loc,
                                                   obj_loc, k = num_sensors, d = 2)

    Z =  np.array([[log_likelihood((X[i,j],Y[i,j]),
                                 sensor_loc, distance[0]) for j in range(len(X))] \
                 for i in range(len(X))])

    plt.figure(figsize=(10,4))
    plt.subplot(121)
    CS = plt.contour(X, Y, Z)
    plt.clabel(CS, inline=1, fontsize=10)
    plt.title('With object at (0,0)')

    np.random.seed(0)
    # sensor_loc = np.random.randn(7,2) * 10
    x = np.arange(-400,400, 4)
    y = np.arange(-400,400, 4)
    X, Y = np.meshgrid(x, y)
    obj_loc = [[200,200]]
    obj_loc, distance = generate_data_given_location(sensor_loc,
                                                   obj_loc, k = num_sensors, d = 2)

    Z =  np.array([[log_likelihood((X[i,j],Y[i,j]),
                                 sensor_loc, distance[0]) for j in range(len(X))] \
                 for i in range(len(X))])

    # Create a simple contour plot with labels using default colors.  The
    # inline argument to clabel will control whether the labels are draw
    # over the line segments of the contour, removing the lines beneath
    # the label
    #plt.figure()
    plt.subplot(122)
    CS = plt.contour(X, Y, Z)
    plt.clabel(CS, inline=1, fontsize=10)
    plt.title('With object at (200,200)')
    #plt.savefig('likelihood_landscape.png')

    ########################################################################
    ######### Plots of Found local minimas. ###########################################
    ########################################################################
    #sensor_loc
    #minimas = np.zeros((len(obj_locs), 10, num_gd_replicates, 2))
    #true_object_locs = np.zeros((len(obj_locs), 10, 2))
    object_loc_i = 5
    trail = 0

    plt.figure()
    plt.plot(sensor_loc[:, 0], sensor_loc[:, 1], 'r+', label="sensors")
    plt.plot(minimas[object_loc_i, trail, :, 0], minimas[object_loc_i, trail, :, 1], 'g.', label="minimas")
    plt.plot(true_object_locs[object_loc_i, trail, 0], true_object_locs[object_loc_i, trail, 1], 'b*', label="object")
    plt.title('object at location (%d, %d), gradient descent recovered locations' % (object_loc_i*100, object_loc_i*100))
    plt.legend()
    plt.show()

###############################
######### Part f ##############
###############################

########################################################################
#########  Gradient Computing and MLE ##################################
########################################################################
def compute_grad_likelihood(sensor_loc, obj_loc, distance):
    """
    Compute the gradient of the loglikelihood function for part f.

    Input:
    sensor_loc: k * d numpy array.
    Location of sensors.

    obj_loc: n * d numpy array.
    Location of the objects.

    distance: n * k dimensional numpy array.
    Observed distance of the object.

    Output:
    grad: k * d numpy array.
    """
    grad = np.zeros(sensor_loc.shape)
    ### finish the grad loglike ###
    for i, j in enumerate(sensor_loc):
        d = distance[:, i]
        grad[i] = compute_gradient_of_likelihood(j, obj_loc, d)

    return grad

def find_mle_by_grad_descent(initial_sensor_loc,
           obj_loc, distance, lr=0.001, num_iters = 1000):
    """
    Compute the gradient of the loglikelihood function for part f.

    Input:
    initial_sensor_loc: k * d numpy array.
    Initialized Location of the sensors.

    obj_loc: n * d numpy array. Location of the n objects.

    distance: n * k dimensional numpy array.
    Observed distance of the n object.

    Output:
    sensor_loc: k * d numpy array. The mle for the location of the object.

    """
    sensor_loc = initial_sensor_loc
    ### finish the gradient descent ###
    for i in range(num_iters):
        sensor_loc = sensor_loc + lr * compute_grad_likelihood(sensor_loc, obj_loc, distance)

    return sensor_loc
########################################################################
#########  Gradient Computing and MLE ##################################
########################################################################

np.random.seed(0)
sensor_loc = generate_sensors()
obj_loc, distance = generate_data(sensor_loc, n = 100)
print('The real sensor locations are')
print(sensor_loc)
# Initialized as zeros.
initial_sensor_loc = np.zeros((7,2)) #np.random.randn(7,2)
estimated_sensor_loc = find_mle_by_grad_descent(initial_sensor_loc,
                                                obj_loc, distance, lr=0.001, num_iters = 1000)
print('The predicted sensor locations are')
print(estimated_sensor_loc)

########################################################################
#########  Estimate distance given estimated sensor locations. #########
########################################################################

def compute_distance_with_sensor_and_obj_loc(sensor_loc, obj_loc):
    """
    stimate distance given estimated sensor locations.

    Input:
    sensor_loc: k * d numpy array.
    Location of the sensors.

    obj_loc: n * d numpy array. Location of the n objects.

    Output:
    distance: n * k dimensional numpy array.
    """
    estimated_distance = scipy.spatial.distance.cdist(obj_loc, sensor_loc, metric='euclidean')
    return estimated_distance

########################################################################
#########  MAIN  #######################################################
########################################################################
np.random.seed(100)
########################################################################
#########  Case 1. #####################################################
########################################################################
mse =0

for i in range(100):
    obj_loc, distance = generate_data(sensor_loc, k = 7, d = 2, n = 1, original_dist = True)
    #obj_loc, distance = generate_data_given_location(estimated_sensor_loc, obj_loc, k = 7, d = 2)
    l = float('-inf')
    ### compute the mse for this case ###
    estimate = compute_distance_with_sensor_and_obj_loc(estimated_sensor_loc, obj_loc)
    mse = mse + np.mean(np.sum(estimate, axis = 1))

mse = mse / 100

print('The MSE for Case 1 is {}'.format(mse))

########################################################################
#########  Case 2. #####################################################
########################################################################
mse = 0

for i in range(100):
    obj_loc, distance = generate_data(sensor_loc, k = 7, d = 2, n = 1, original_dist = False)
    #obj_loc, distance = generate_data_given_location(estimated_sensor_loc, obj_loc, k = 7, d = 2)
    l = float('-inf')
    ### compute the mse for this case ###
    estimate = compute_distance_with_sensor_and_obj_loc(estimated_sensor_loc, obj_loc)
    mse = mse + np.mean(np.sum(estimate, axis = 1))

mse = mse / 100

print('The MSE for Case 2 is {}'.format(mse))

########################################################################
#########  Case 3. #####################################################
########################################################################
mse =0

for i in range(100):
    obj_loc, distance = generate_data(sensor_loc, k = 7, d = 2, n = 1, original_dist = False)
    obj_loc, distance = generate_data_given_location(estimated_sensor_loc, obj_loc, k = 7, d = 2)
    l = float('-inf')
    # Your code: compute the mse for this case
    estimate = compute_distance_with_sensor_and_obj_loc(estimated_sensor_loc, obj_loc)
    mse = mse + np.mean(np.sum(estimate, axis = 1))

mse = mse / 100

print('The MSE for Case 2 (if we knew mu is [300,300]) is {}'.format(mse))

###############################
######### Problem 5 ###########
###############################

import numpy as np
import matplotlib
import matplotlib.pyplot as plt

# Gradient descent optimization
# The learning rate is specified by eta
class GDOptimizer(object):
    def __init__(self, eta):
        self.eta = eta

    def initialize(self, layers):
        pass

    # This function performs one gradient descent step
    # layers is a list of dense layers in the network
    # g is a list of gradients going into each layer before the nonlinear activation
    # a is a list of of the activations of each node in the previous layer going
    #
    def update(self, layers, g, a):
        m = a[0].shape[1]
        for layer, curGrad, curA in zip(layers, g, a):

            ############################
            ########## PART F ##########
            ############################

            ########################################################################################
            # Compute the gradients for layer.W and layer.b using the gradient for the output of the
            # layer curA and the gradient of the output curGrad
            # Use the gradients to update the weight and the bias for the layer
            #
            # Normalize the learning rate by m (defined above), the number of training examples input
            # (in parallel) to the network.
            #
            # It may help to think about how you would calculate the update if we input just one
            # training example at a time; then compute a mean over these individual update values.
            # ######################################################################################

            WeightUpdate = curGrad @ curA.T
            BiasUpdate = np.sum(curGrad, axis=1).reshape(layer.b.shape)
            layer.updateWeights(-self.eta/m * WeightUpdate)
            layer.updateBias(-self.eta/m * BiasUpdate)

# Cost function used to compute prediction errors
class QuadraticCost(object):

    # Compute the squared error between the prediction yp and the observation y
    # This method should compute the cost per element such that the output is the
    # same shape as y and yp
    @staticmethod
    def fx(y,yp):

        ############################
        ########## PART B ##########
        ############################

        return 0.5 * (y - yp) ** 2

    # Derivative of the cost function with respect to yp
    @staticmethod
    def dx(y,yp):

        ############################
        ########## PART B ##########
        ############################

        return yp - y

# Sigmoid function fully implemented as an example
class SigmoidActivation(object):
    @staticmethod
    def fx(z):
        return 1 / (1 + np.exp(-z))

    @staticmethod
    def dx(z):
        return SigmoidActivation.fx(z) * (1 - SigmoidActivation.fx(z))

# Hyperbolic tangent function
class TanhActivation(object):

    # Compute tanh for each element in the input z
    @staticmethod
    def fx(z):

        ############################
        ########## PART C ##########
        ############################

        return np.tanh(z)

    # Compute the derivative of the tanh function with respect to z
    @staticmethod
    def dx(z):

        ############################
        ########## PART C ##########
        ############################

        return 1 - np.tanh(z) ** 2

# Rectified linear unit
class ReLUActivation(object):
    @staticmethod
    def fx(z):

        ############################
        ########## PART C ##########
        ############################

        return np.where(z < 0, 0, z)

    @staticmethod
    def dx(z):

        ############################
        ########## PART C ##########
        ############################

        return np.where(z <= 0, 0, 1)

# Linear activation
class LinearActivation(object):
    @staticmethod
    def fx(z):

        ############################
        ########## PART C ##########
        ############################

        return z

    @staticmethod
    def dx(z):

        ############################
        ########## PART C ##########
        ############################

        return np.ones(z.shape)

# This class represents a single hidden or output layer in the neural network
class DenseLayer(object):

    # numNodes: number of hidden units in the layer
    # activation: the activation function to use in this layer
    def __init__(self, numNodes, activation):
        self.numNodes = numNodes
        self.activation = activation

    def getNumNodes(self):
        return self.numNodes

    # Initialize the weight matrix of this layer based on the size of the matrix W
    def initialize(self, fanIn, scale=1.0):
        s = scale * np.sqrt(6.0 / (self.numNodes + fanIn))
        self.W = np.random.normal(0, s,
                                   (self.numNodes,fanIn))
        self.b = np.random.uniform(-1,1,(self.numNodes,1))

    # Apply the activation function of the layer on the input z
    def a(self, z):
        return self.activation.fx(z)

    # Compute the linear part of the layer
    # The input a is an n x k matrix where n is the number of samples
    # and k is the dimension of the previous layer (or the input to the network)
    def z(self, a):
        return self.W.dot(a) + self.b # Note, this is implemented where we assume a is k x n

    # Compute the derivative of the layer's activation function with respect to z
    # where z is the output of the above function.
    # This derivative does not contain the derivative of the matrix multiplication
    # in the layer.  That part is computed below in the model class.
    def dx(self, z):
        return self.activation.dx(z)

    # Update the weights of the layer by adding dW to the weights
    def updateWeights(self, dW):
        self.W = self.W + dW

    # Update the bias of the layer by adding db to the bias
    def updateBias(self, db):
        self.b = self.b + db

# This class handles stacking layers together to form the completed neural network
class Model(object):

    # inputSize: the dimension of the inputs that go into the network
    def __init__(self, inputSize):
        self.layers = []
        self.inputSize = inputSize

    # Add a layer to the end of the network
    def addLayer(self, layer):
        self.layers.append(layer)

    # Get the output size of the layer at the given index
    def getLayerSize(self, index):
        if index >= len(self.layers):
            return self.layers[-1].getNumNodes()
        elif index < 0:
            return self.inputSize
        else:
            return self.layers[index].getNumNodes()

    # Initialize the weights of all of the layers in the network and set the cost
    # function to use for optimization
    def initialize(self, cost, initializeLayers=True):
        self.cost = cost
        if initializeLayers:
            for i in range(0,len(self.layers)):
                if i == len(self.layers) - 1:
                    self.layers[i].initialize(self.getLayerSize(i-1))
                else:
                    self.layers[i].initialize(self.getLayerSize(i-1))

    # Compute the output of the network given some input a
    # The matrix a has shape n x k where n is the number of samples and
    # k is the dimension
    # This function returns
    # yp - the output of the network
    # a - a list of inputs for each layer of the newtork where
    #     a[i] is the input to layer i
    #     (note this does not include the network output!)
    # z - a list of values for each layer after evaluating layer.z(a) but
    #     before evaluating the nonlinear function for the layer
    def evaluate(self, x):
        curA = x.T
        a = [curA]
        z = []
        for layer in self.layers:
            z.append(layer.z(curA))
            curA = layer.a(z[-1])
            a.append(curA)
        yp = a.pop()
        return yp, a, z

    # Compute the output of the network given some input a
    # The matrix a has shape n x k where n is the number of samples and
    # k is the dimension
    def predict(self, a):
        a,_,_ = self.evaluate(a)
        return a.T

    # Computes the gradients at each layer. y is the true labels, yp is the
    # predicted labels, and z is a list of the intermediate values in each
    # layer. Returns the gradients and the forward pass outputs (per layer).
    #
    # In particular, we compute dMSE/dz_i. The reasoning behind this is that
    # in the update function for the optimizer, we do not give it the z values
    # we compute from evaluating the network.
    def compute_grad(self, x, y):
        # Feed forward, computing outputs of each layer and
        # intermediate outputs before the non-linearities
        yp, a, z = self.evaluate(x)

        # d represents (dMSE / da_i) that you derive in part (e);
        #   it is inialized here to be (dMSE / dyp)
        d = self.cost.dx(y.T, yp)
        grad = []

        # Backpropogate the error
        for layer, curZ in zip(reversed(self.layers),reversed(z)):

            ############################
            ########## PART D ##########
            ############################

            # Compute the gradient of the output of each layer with respect to the error
            # grad[i] should correspond with the gradient of the output of layer i
            #   before the activation is applied (dMSE / dz_i); be sure values are stored
            #   in the correct ordering!
            grad.insert(0, d * layer.dx(curZ))
            d = layer.W.T @ grad[0]

        return grad, a

    # Computes the gradients at each layer. y is the true labels, yp is the
    # predicted labels, and z is a list of the intermediate values in each
    # layer. Uses numerical derivatives to solve rather than symbolic derivatives.
    # Returns the gradients and the forward pass outputs (per layer).
    #
    # In particular, we compute dMSE/dz_i. The reasoning behind this is that
    # in the update function for the optimizer, we do not give it the z values
    # we compute from evaluating the network.
    def numerical_grad(self, x, y, delta=1e-4):

        # computes the loss function output when starting from the ith layer
        # and inputting z_i
        def compute_cost_from_layer(layer_i, z_i):
            cost = self.layers[layer_i].a(z_i)
            for layer in self.layers[layer_i+1:]:
                cost = layer.a(layer.z(cost))
            return self.cost.fx(y.T, cost)

        # numerically computes the gradient of the error with respect to z_i
        def compute_grad_from_layer(layer_i, inp):
            mask = np.zeros(self.layers[layer_i].b.shape)
            grad_z = []
            # iterate to compute gradient of each variable in z_i, one at a time
            for i in range(mask.shape[0]):
                mask[i] = 1
                delta_p_output = compute_cost_from_layer(layer_i, inp+mask*delta)
                delta_n_output = compute_cost_from_layer(layer_i, inp-mask*delta)
                grad_z.append((delta_p_output - delta_n_output) / (2 * delta))
                mask[i] = 0;

            return np.vstack(grad_z)

        _, a, _ = self.evaluate(x)

        grad = []
        i = 0
        curA = x.T
        for layer in self.layers:
            curA = layer.z(curA)
            grad.append(compute_grad_from_layer(i, curA))
            curA = layer.a(curA)
            i += 1

        return grad, a

    # Train the network given the inputs x and the corresponding observations y
    # The network should be trained for numEpochs iterations using the supplied
    # optimizer
    def train(self, x, y, numEpochs, optimizer):

        # Initialize some stuff
        n = x.shape[0]
        x = x.copy()
        y = y.copy()
        hist = []
        optimizer.initialize(self.layers)

        # Run for the specified number of epochs
        for epoch in range(0,numEpochs):

            # Compute the gradients
            grad, a = self.compute_grad(x, y)

            # Update the network weights
            optimizer.update(self.layers, grad, a)

            # Compute the error at the end of the epoch
            yh = self.predict(x)
            C = self.cost.fx(y, yh)
            C = np.mean(C)
            hist.append(C)
        return hist

    ############################
    ########## PART J ##########
    ############################

    def trainBatch(self, x, y, batchSize, numEpochs, optimizer):
        hist = []
        for epoch in np.arange(0,numEpochs):
            e = []
            for i in range(0, x.shape[0], batchSize):
                end = min(i+batchSize, x.shape[0])
                batchx = x[i:end, :]
                batchy = y[i:end, :]
                e = e + self.train(batchx, batchy, 1, optimizer)
            hist.append(np.mean(e))
        return hist

if __name__ == '__main__':
    # switch these statements to True to run the code for the corresponding parts
    # PART E
    DEBUG_MODEL = False
    # Part G
    BASE_MODEL = False
    # Part H
    DIFF_SIZES = False
    # Part I
    RIDGE = False
    # Part J
    SGD = False
    # Part K
    PARTk = False

    # Generate the training set
    np.random.seed(9001)
    x=np.random.uniform(-np.pi,np.pi,(1000,1))
    y=np.sin(x)
    xLin=np.linspace(-np.pi,np.pi,250).reshape((-1,1))
    yHats = {}

    activations = dict(ReLU=ReLUActivation,
                       tanh=TanhActivation,
                       linear=LinearActivation)
    lr = dict(ReLU=0.02,tanh=0.02,linear=0.005)
    names = ['ReLU','linear','tanh']

    #### PART F ####
    if DEBUG_MODEL:
        print('Debugging gradients..')
        # Build the model
        activation = activations["ReLU"]
        model = Model(x.shape[1])
        model.addLayer(DenseLayer(10,activation()))
        model.addLayer(DenseLayer(10,activation()))
        model.addLayer(DenseLayer(1,LinearActivation()))
        model.initialize(QuadraticCost())

        grad, _ = model.compute_grad(x, y)
        n_grad, _ = model.numerical_grad(x, y)
        for i in range(len(grad)):
            print('squared difference of layer %d:' % i, np.linalg.norm(grad[i] - n_grad[i]))

    #### PART G ####
    if BASE_MODEL:
        print('\n----------------------------------------\n')
        print('Standard fully connected network')
        for key in names:
            # Build the model
            activation = activations[key]
            model = Model(x.shape[1])
            model.addLayer(DenseLayer(100,activation()))
            model.addLayer(DenseLayer(100,activation()))
            model.addLayer(DenseLayer(1,LinearActivation()))
            model.initialize(QuadraticCost())

            # Train the model and display the results
            hist = model.train(x,y,500,GDOptimizer(eta=lr[key]))
            yHat = model.predict(x)
            yHats[key] = model.predict(xLin)
            error = np.mean(np.square(yHat - y))/2
            print(key+' MSE: '+str(error))
            plt.plot(hist)
            plt.title(key+' Learning curve')
            plt.show()

        # Plot the approximations
        font = {'family' : 'DejaVu Sans',
            'weight' : 'bold',
                'size'   : 12}
        matplotlib.rc('font', **font)
        y = np.sin(xLin)
        for key in activations:
            plt.plot(xLin,y)
            plt.plot(xLin,yHats[key])
            plt.title(key+' approximation')
            plt.savefig(key+'-approx.png')
            plt.show()

    # Train with different sized networks
    #### PART H ####
    if DIFF_SIZES:
        print('\n----------------------------------------\n')
        print('Training with various sized network')
        names = ['ReLU', 'tanh']
        sizes = [5,10,25,50]
        widths = [1,2,3]
        errors = {}
        y = np.sin(x)
        for key in names:
            error = []
            for width in widths:
                for size in sizes:
                    activation = activations[key]
                    model = Model(x.shape[1])
                    for _ in range(width):
                        model.addLayer(DenseLayer(size,activation()))
                    model.addLayer(DenseLayer(1,LinearActivation()))
                    model.initialize(QuadraticCost())
                    hist = model.train(x,y,500,GDOptimizer(eta=lr[key]))
                    yHat = model.predict(x)
                    yHats[key] = model.predict(xLin)
                    e = np.mean(np.square(yHat - y))/2
                    error.append(e)
            errors[key] = np.asarray(error).reshape((len(widths),len(sizes)))

        # Print the results
        for key in names:
            error = errors[key]
            print(key+' MSE Error')
            header = '{:^8}'
            for _ in range(len(sizes)):
                header += ' {:^8}'
            headerText = ['Layers'] + [str(s)+' nodes' for s in sizes]
            print(header.format(*headerText))
            for width,row in zip(widths,error):
                text = '{:>8}'
                for _ in range(len(row)):
                    text += ' {:<8}'
                rowText = [str(width)] + ['{0:.5f}'.format(r) for r in row]
                print(text.format(*rowText))

    # Perform ridge regression on the last layer of the network
    #### PART I ####
    if RIDGE:
        print('\n----------------------------------------\n')
        print('Running ridge regression on last layer')
        from sklearn.linear_model import Ridge
        errors = {}
        for key in names:
            error = []
            sizes = [5,10,25,50]
            widths = [1,2,3]
            for width in widths:
                for size in sizes:
                    activation = activations[key]
                    model = Model(x.shape[1])
                    for _ in range(width):
                        model.addLayer(DenseLayer(size,activation()))
                    model.initialize(QuadraticCost())
                    ridge = Ridge(alpha=0.1)
                    X = model.predict(x)
                    ridge.fit(X,y)
                    yHat = ridge.predict(X)
                    e = np.mean(np.square(yHat - y))/2
                    error.append(e)
            errors[key] = np.asarray(error).reshape((len(widths),len(sizes)))

        # Print the results
        for key in names:
            error = errors[key]
            print(key+' MSE Error')
            header = '{:^8}'
            for _ in range(len(sizes)):
                header += ' {:^8}'
            headerText = ['Layers'] + [str(s)+' nodes' for s in sizes]
            print(header.format(*headerText))
            for width,row in zip(widths,error):
                text = '{:>8}'
                for _ in range(len(row)):
                    text += ' {:<8}'
                rowText = [str(width)] + ['{0:.5f}'.format(r) for r in row]
                print(text.format(*rowText))

        # Plot the results
        for key in names:
            for width,row in zip(widths,errors[key]):
                layer = ' layers'
                if width == 1:
                    layer = ' layer'
                plt.semilogy(row,label=str(width)+layer)
            plt.title('MSE for ridge regression with '+key+' activation')
            plt.xticks(range(len(sizes)),sizes)
            plt.xlabel('Layer size')
            plt.ylabel('MSE')
            plt.legend()
            plt.savefig(key+'-ridge.png')
            plt.show()

    ############################
    ########## PART J ##########
    ############################

    if SGD:
        print('\n----------------------------------------\n')
        print('Running SGD')
        batchSizes = [1, 10, 20, 50, 100]
        for key in names:
            for batchSize in batchSizes:
                activation = activation[key]
                model = Model(x.shape[1])
                model.addLayer(DenseLayer(100,activation()))
                model.addLayer(DenseLayer(100,activation()))
                model.addLayer(DenseLayer(1,LinearActivation()))
                model.initialize(QuadraticCost())
                epoch = 25 * batchSize
                hist = model.trainBatch(x, y, batchSize, epoch, GDOptimizer(eta=lr[key]))
                yHat = model.predict(x)
                yHats[key] = model.predict(xLin)
                err = np.mean(np.square(yHat - y))/2
                print(key, 'batchsize:', batchSize, 'MSE:', err)

    ############################
    ########## PART K ##########
    ############################

    if PARTk:
        print('\n----------------------------------------\n')
        print('Running Bonus Question')
        # we only use ReLU in this question
        activation = activations['ReLU']
        model = Model(x.shape[1])
        model.addLayer(DenseLayer(100,activation()))
        model.addLayer(DenseLayer(100,activation()))
        model.addLayer(DenseLayer(1,LinearActivation()))
        model.initialize(QuadraticCost())

        # Train the model and display the results
        hist = model.train(x,y,100,GDOptimizer(eta=0.01))
        yHat = model.predict(x)
        error = np.mean(np.square(yHat - y))/2
        print('100 max epochs, 0.01 LR'+' MSE: '+str(error))

        hist = model.train(x,y,100,GDOptimizer(eta=0.05))
        yHat = model.predict(x)
        error = np.mean(np.square(yHat - y))/2
        print('100 max epochs, 0.05 LR'+' MSE: '+str(error))

        hist = model.train(x,y,100,GDOptimizer(eta=0.1))
        yHat = model.predict(x)
        error = np.mean(np.square(yHat - y))/2
        print('100 max epochs, 0.1 LR'+' MSE: '+str(error))

        hist = model.train(x,y,200,GDOptimizer(eta=0.01))
        yHat = model.predict(x)
        error = np.mean(np.square(yHat - y))/2
        print('200 max epochs, 0.01 LR'+' MSE: '+str(error))

        hist = model.train(x,y,200,GDOptimizer(eta=0.05))
        yHat = model.predict(x)
        error = np.mean(np.square(yHat - y))/2
        print('200 max epochs, 0.05 LR'+' MSE: '+str(error))

        hist = model.train(x,y,200,GDOptimizer(eta=0.1))
        yHat = model.predict(x)
        error = np.mean(np.square(yHat - y))/2
        print('200 max epochs, 0.1 LR'+' MSE: '+str(error))

        hist = model.train(x,y,500,GDOptimizer(eta=0.01))
        yHat = model.predict(x)
        error = np.mean(np.square(yHat - y))/2
        print('500 max epochs, 0.01 LR'+' MSE: '+str(error))

        hist = model.train(x,y,500,GDOptimizer(eta=0.05))
        yHat = model.predict(x)
        error = np.mean(np.square(yHat - y))/2
        print('500 max epochs, 0.05 LR'+' MSE: '+str(error))

        hist = model.train(x,y,100,GDOptimizer(eta=0.1))
        yHat = model.predict(x)
        error = np.mean(np.square(yHat - y))/2
        print('500 max epochs, 0.1 LR'+' MSE: '+str(error))


import numpy as np
import matplotlib.pyplot as plt

def load(filename):
    f = open(filename, 'rb')
    return np.loadtxt(f, delimiter=',')

data = load('ex1data2.txt')
X = data[:,0:2]
y = data[:,2:3]
m = len(y)

for i in range(10):
    print('x = [%.0f %.0f], y = %.0f \n' % (X[i,0], X[i,1], y[i, :]))

def featureNormalize(X):
    X_norm = X
    if (len(X) <= 1):
        return np.zeros(X.shape), np.zeros((1, X.shape[1])), np.zeros((1, X.shape[1]))

    mu = np.mean(X, axis=0, keepdims=True)
    # https://stackoverflow.com/questions/27600207/why-does-numpy-std-give-a-different-result-to-matlab-std
    sigma = np.std(X, axis=0, keepdims=True, ddof=1)
    X_norm = (X - mu) / sigma
    return X_norm, mu, sigma

# print('featureNormalize() test')
# def magic(n):
#   n = int(n)
#   if n < 3:
#     raise ValueError("Size must be at least 3")
#   if n % 2 == 1:
#     p = np.arange(1, n+1)
#     return n*np.mod(p[:, None] + p - (n+3)//2, n) + np.mod(p[:, None] + 2*p-2, n) + 1
#   elif n % 4 == 0:
#     J = np.mod(np.arange(1, n+1), 4) // 2
#     K = J[:, None] == J
#     M = np.arange(1, n*n+1, n)[:, None] + np.arange(n)
#     M[K] = n*n + 1 - M[K]
#   else:
#     p = n//2
#     M = magic(p)
#     M = np.block([[M, M+2*p*p], [M+3*p*p, M+p*p]])
#     i = np.arange(p)
#     k = (n-2)//4
#     j = np.concatenate((np.arange(k), np.arange(n-k+1, n)))
#     M[np.ix_(np.concatenate((i, i+p)), j)] = M[np.ix_(np.concatenate((i+p, i)), j)]
#     M[np.ix_([k, k+p], [0, k])] = M[np.ix_([k+p, k], [0, k])]
#   return M

# X_test, mu_test, sigma_test = featureNormalize(np.array([[1], [2], [3]]))
# print(X_test, mu_test, sigma_test)

# X_test, mu_test, sigma_test = featureNormalize(magic(3))
# print(X_test, mu_test, sigma_test)

# X_test, mu_test, sigma_test = featureNormalize(np.concatenate((-np.ones((1, 3)), magic(3))))
# print(X_test, mu_test, sigma_test)
# print('test over')

X, mu, sigma = featureNormalize(X)

print(X)
print(mu)
print(sigma)

X = np.concatenate((np.ones((m, 1)), X), 1)
alpha = 0.1
num_iters = 400

def computeCostMulti(X, y, theta):
    m = len(y)
    error = X @ theta - y;
    return error.T @ error / (2 * m)

# print('computeCostMulti() test')
# X_test = np.array([[2, 1, 3], [7, 1, 9], [1, 8, 1], [3, 7, 4]])
# y_test = np.array([[2], [5], [5], [6]])
# theta_test = np.array([[0.4], [0.6], [0.8]])
# print(computeCostMulti(X_test, y_test, theta_test))
# print('test over')

def gradientDescentMulti(X, y, theta, alpha, num_iters):
    m = len(y)
    J_history = np.zeros((num_iters, 1))
    for iters in range(num_iters):
        theta = theta - alpha * (X.T @ (X @ theta - y)) / m
        J_history[iters] = computeCostMulti(X, y, theta)
    return theta, J_history

# print('gradientDescentMulti() test')
# X_test = np.array([[2, 1, 3], [7, 1, 9], [1, 8, 1], [3, 7, 4]])
# y_test = np.array([[2], [5], [5], [6]])
# print(gradientDescentMulti(X_test, y_test, np.array([[0.1], [-0.2], [0.3]]), 0.01, 10))
# print('test over')

theta = np.zeros((3, 1))
theta, _ =  gradientDescentMulti(X, y, theta, alpha, num_iters)

print('Theta computed from gradient descent: [%f, %f %f]' % (theta[0], theta[1], theta[2]))

x = (np.array([1650, 3]) - mu) / sigma
price = np.concatenate(([[1]], x), 1) @ theta
print('Predicted price of a 1650 sq-ft, 3 br house (using gradient descent): $%f' % price)

alpha = [1, 0.3, 0.1, 0.03, 0.01]
num_iters = range(50, 201)
theta = np.zeros((3, 1))
_, J_history = gradientDescentMulti(X, y, theta, alpha[2], num_iters[0])
plt.plot(range(1, num_iters[0]+1), J_history, '-b', linewidth = 2)
plt.xlabel('Number of iterations')
plt.ylabel('Cost J')
plt.show()

# replace the value of alhpa below best alhpa value you found above
alpha = 0.1
num_iters = 50

theta = np.zeros((3, 1))
theta, _ = gradientDescentMulti(X, y, theta, alpha, num_iters)

print('Theta computed from gradent descent: [%f, %f, %f]' % (theta[0], theta[1], theta[2]))
x = (np.array([1650, 3]) - mu) / sigma
price = np.concatenate(([[1]], x), 1) @ theta
print('Predicted price of a 1650 sq-ft, 3 br house (using gradient descent): %f\n' % price)

data = load('ex1data2.txt')
X = data[:, 0:2]
y = data[:, 2:3]
m = len(y)

ax = plt.axes(projection = '3d')
x0 = X[:, 0:1]
x1 = X[:, 1:2]
x0 = np.linspace(np.min(x0), np.max(x0), 10)
x1 = np.linspace(np.min(x1), np.max(x1), 10)
x0, x1 = np.meshgrid(x0, x1)
X_norm = (np.array([x0,x1]).T - mu) / sigma
X_norm = np.concatenate((np.ones((X_norm.shape[0], X_norm.shape[1], 1)), X_norm), 2)
# z = x0 * theta[1] + x1 * theta[2] + theta[0]
z = (X_norm @ theta).T
ax.plot_surface(x0, x1, z.reshape(x0.shape), color='w', alpha=0.5)
ax.scatter3D(X[:,0:1], X[:,1:2], y, c='b', marker='^')
ax.scatter3D(1650, 3, price, c='r', marker='o')
ax.invert_xaxis()

plt.xlabel('sq-ft')
plt.ylabel('br')
plt.show()

data = load('ex1data2.txt')
X = data[:, 0:2]
y = data[:, 2:3]
m = len(y)

X = np.concatenate((np.ones((m, 1)), X), 1)

def normalEqn(X, y):
    return np.linalg.pinv((X.T @ X)) @ X.T @ y

# print('normalEqn() test')
# X_test = np.array([[2, 1, 3], [7, 1, 9], [1, 8, 1], [3, 7, 4]])
# y_test = np.array([[2], [5], [5], [6]])
# print(normalEqn(X_test, y_test))
# print('test over')

theta = normalEqn(X, y)
print('Theta computed from the normal equations: [%f, %f, %f]' % (theta[0], theta[1], theta[2]))

price = np.array([[1, 1650, 3]]) @ theta
print('Predicted price of a 1650 sq-ft, 3 br house (using norma equations): $%f' % price)

# plot normalEqn
ax = plt.axes(projection = '3d')
x0 = X[:, 1:2]
x1 = X[:, 2:3]
x0 = np.linspace(np.min(x0), np.max(x0), 10)
x1 = np.linspace(np.min(x1), np.max(x1), 10)
x0, x1 = np.meshgrid(x0, x1)
z = x0 * theta[1] + x1 * theta[2] + theta[0]
ax.plot_surface(x0, x1, z, color='w', alpha=0.5)
ax.scatter3D(X[:,1:2], X[:,2:3], y, c='b', marker='^')
ax.scatter3D(1650, 3, price, c='r', marker='o')
ax.invert_xaxis()

plt.xlabel('sq-ft')
plt.ylabel('br')
plt.show()


#!/usr/bin/env python

from scipy.optimize import linprog

"""
Solve a basic linear programming problem using SciPy
https://docs.scipy.org/doc/scipy/reference/generated/scipy.optimize.linprog.html
"""

def main():
    """
    Minimize:  c^T * x
    Subject to:
        A_ub * x <= b_ub
        A_eq * x == b_eq
    """

    print('\n')
    print('**************************************************')
    print('Example from scipy.optimize.linprog documentation:')
    print('**************************************************')
    print('Minimize: [-1; 4] * x')
    print('Subject to:  [[-3, 1]; [1, 2]] * x < [6; 4]')
    c = [-1, 4]
    A = [[-3, 1], [1, 2]]
    b = [6, 4]
    x0_bounds = (None, None)
    x1_bounds = (-3, None)
    print('Solving...')
    result = linprog(c, A_ub=A, b_ub=b, bounds=(x0_bounds, x1_bounds),
                  options={"disp": True})
    print('Result:')
    print(result)

    print('\n')
    print('*********************************')
    print('Infeasible Example:')
    print('  http://math.stackexchange.com/questions/2011787/linear-programming-excercise/')
    print('*********************************')
    print('Minimize: [1;2;3] * x')
    print('Subject to:')
    print(    'x1 + 3*x3 >= 3')
    print(    '-x1 + x2 + 3*x3 <= 4')
    print(    'x1 + 2*x2 - 3*x3 <= -6')
    print(    'x1 >= 0')
    print(    'x2 >= 0')
    print(    'x3 >= 0')
    c = [1, 2, 3]  # objective function
    # Inequality constraint matrix. Note that first inequality is negated.
    A = [[-1,0,-3], [-1, 1, 3], [1,2,-3]]
    b = [-3, 4, -6]
    x0_bnd = (0, None)
    x1_bnd = (0, None)
    x2_bnd = (0, None)
    print('Solving...')
    result = linprog(c, A_ub=A, b_ub=b, bounds=(x0_bnd, x1_bnd, x2_bnd),
                  options={"disp": True})
    print('Result:')
    print(result)

if __name__ == "__main__":
    main()


#The Hacker Within: Python Boot Camp 2010 - Session 07 - Using SciPy.
#Presented by Anthony Scopatz.
#
#SciPy Integration, run before you glide.

#Tools used to calculate numerical, definite integrals may be found in the 'integrate' module.
import scipy.integrate
#For kicks, let's also grab
import scipy.special
import numpy

#There are two basic ways you can integrate in SciPy:
#     1. Integrate a function, or
#     2. Integrate piecewise data.

#First Let's deal with integration of functions.
#Recall that in Python, functions are also objects.
#Therefore you can pass functions as arguments to other functions!
#Just make sure that the function that you want to integrate returns a float,
#or, at the very least, an object that has a __float__() method.

#The simplest way to compute a functions definite integral is via the quad(...) function.
def CrazyFunc(x):
	return (scipy.special.i1(x) - 1)**3

print("Try integrating CrazyFunc on the range [-5, 10]...")

val, err = scipy.integrate.quad(CrazyFunc, -5, 10)

print("A Crazy Function integrates to %.8E"%val)
print("And with insanely low error of %.8E"%err)
print("")

#You can also use scipy.integrate.Inf for infinity in the limits of integration
print("Now try integrating e^x on [-inf, 0]")
print("(val, err) = " + str( scipy.integrate.quad(scipy.exp, -scipy.integrate.Inf, 0.0) ))
print("")

#2D integrations follows similarly,
def dA_Sphere(phi, theta):
	return  scipy.sin(phi)

print("Integrate the surface area of the unit sphere...")
val, err = scipy.integrate.dblquad(dA_Sphere, 0.0, 2.0*scipy.pi, lambda theta: 0.0,  lambda theta: scipy.pi )
print("val = %.8F"%val)
print("err = %.8E"%err)
print("")

def dV_Sphere(phi, theta, r):
	return r * r * dA_Sphere(phi, theta)

print("Integrate the volume of a sphere with r=3.5...")
val, err = scipy.integrate.tplquad(dV_Sphere, 0.0, 3.5, lambda r: 0.0, lambda r: 2.0*scipy.pi, lambda x, y: 0.0, lambda x, y: scipy.pi)
print("val = %.8F"%val)
print("err = %.8E"%err)
print("")

#Now, only very rarely will scientists (and even more rarely engineers) will truely 'know'
#the function that they wish to integrate.  Much more often we'll have piecewise data
#that we wish numerically integrate (ie sum an array y(x), biased by array x).
#This can be done in SciPy through the trapz function.

y = range(0, 11)
print("Trapazoidally integrate y = x on [0,10]...")
val = scipy.integrate.trapz(y)
print("val = %F"%val)
print("")

#You can also define a domain to integrate over.
x = numpy.arange(0.0, 20.5, 0.5)
y = x * x
print("Trapazoidally integrate y = x^2 on [0,20] with half steps...")
val = scipy.integrate.trapz(y, x)
print("val = %F"%val)
print("")

print("Trapazoidally integrate y = x^2 with dx=0.5...")
val = scipy.integrate.trapz(y, dx=0.5)
print("val = %F"%val)
print("")

def dDecay(y, t, lam):
	return -lam*y

#Of course, sometimes we have simple ODEs that we want to integrate over time for...
#These are generally of the form:
#     dy / dt = f(y, t)
#For example take the decay equation...
#     f(y, t) = - lambda * y
#We can integrate this using SciPy's 'odeint'  This is of the form:
#     odeint( f, y0, [t0, t1, ...])
#Let's try it...
vals = scipy.integrate.odeint( lambda y, t: dDecay(y, t, 0.2), 1.0, [0.0, 10.0] )
print("If you start with a mass of y(0) = %F"%vals[0][0])
print("you'll only have y(t=10) = %F left."%vals[1][0])

#Check out http://docs.scipy.org/doc/scipy/reference/integrate.html for a complete listing.


"""
simplex_algorithm.py

Returns the minimizer of the function
"""

import numpy as np
np.set_printoptions(formatter={'float': lambda x: "{0:0.3f}".format(x)});

def pivoting(tabl, p, q):
    # pivoting about (p,q)-th element
    tabl_new = tabl - tabl[p] * (tabl[:, [q]] / tabl[p, [q]]);
    tabl_new[p] = tabl[p] / tabl[p, [q]];

    return tabl_new;

def simplex_algorithm(A, b, C, indx_cols, options):

    b_print = options['b_print'];
    b_blands_rule = options['b_blands_rule'];
    epsilon = -10e-6; # workaround for -0 issues in while loop

    if (b_print):
        # Forming initial tableau
        tabl_top = np.concatenate((A, b), axis = 1);
        tabl_bottom = np.concatenate((C.T, np.array([[0]])), axis = 1);
        tabl = np.concatenate((tabl_top, tabl_bottom), axis = 0);
        print(' ');
        print('Initial Tableau [A b; C.T 0]');
        print(tabl);

    m, n = A.shape;
    Binv = np.linalg.inv(A[:, indx_cols]); # inverse matrix of basic columns of A
    A = Binv @ A;
    b = Binv @ b;
    cB = C[indx_cols]; # vector of cost coefficients for basic cols of A
    r = C - A.T @ cB; # vector of relative cost coefficients
    cost = -cB.T @ b; # cost function

    # Forming canonical tableau
    tabl_top = np.concatenate((A, b), axis = 1);
    tabl_bottom = np.concatenate((r.T, cost), axis = 1);
    tabl = np.concatenate((tabl_top, tabl_bottom), axis = 0);

    if (b_print):
        print(' ');
        print('Canonical Tableau [A b; r.T cost]');
        print(tabl);

    while (np.sum(r >= epsilon * np.ones((n, 1))) != n):

        if (b_blands_rule):
            q = 0;
            while(r[q] >= 0):
                q += 1;
        else:
            q = np.argmin(r); # col #q comes into basis

        if (np.sum((tabl[0:m, [q]]) > 0) == 0):
            print('Problem unbounded, exit..');
            break;

        a = tabl[0:m, [q]] / tabl[0:m, [n]];
        p = np.argmax(a); # col #p leaves basis

        # pivoting about (p,q)-th element
        tabl = pivoting(tabl, p, q);

        if (b_print):
            print(' ');
            print('Pivoting about (%d, %d)-th element' % (p, q));
            print(' ');
            print('Updated canonical Tableau [A b; r.T cost]');
            print(tabl);

        indx_cols[p] = q;
        r = (tabl[[m], 0:n]).T;

    X = np.zeros((n, 1));
    X[indx_cols] = tabl[0:m, [n]];
    cost = C.T @ X; # cost function

    if (b_print):
        print(' ');
        print('f(X) = %0.3f' % (cost));
        print('Solution:');
        print(X);

    return (X, cost);


"""Visuals for KLD divergence between a normal and Gaussian distribution"""
import matplotlib.pyplot as plt
import numpy as np

from autoencoders.divergence.kld import KLD

##############################################################################
stds = np.array([0.25, 0.5, 1, 4, 9, 16]).reshape(2, 3)

fig, axs = plt.subplots(
    nrows=2,
    ncols=3,
    figsize=(15, 8),
    sharex=True,
    sharey=True,
    tight_layout=True,
)

kld = KLD()

for idx_row, ax_row in enumerate(axs):

    for idx_col, ax in enumerate(ax_row):

        ax.plot(kld.x, kld.prior, color="black", label="N(0,1)")

        std = stds[idx_row, idx_col]

        divergence, g = kld.to_gaussian(mu=0, std=std)

        ax.plot(kld.x, g, label=f"N(0,{std})\nKLD={divergence:.2f}")
        ax.tick_params(labelleft=False, labelbottom=False)
        ax.legend()
##############################################################################
mus = np.arange(-2, 4, 1).reshape(2, 3)

fig, axs = plt.subplots(
    nrows=2,
    ncols=3,
    figsize=(15, 8),
    sharex=True,
    sharey=True,
    tight_layout=True,
)

for idx_row, ax_row in enumerate(axs):
    for idx_col, ax in enumerate(ax_row):

        ax.plot(kld.x, kld.prior, color="black", label="N(0,1)")

        mu = mus[idx_row, idx_col]

        divergence, g = kld.to_gaussian(mu=mu, std=1)

        ax.plot(kld.x, g, label=f"N({mu}, 1)\nKLD={divergence:.2f}")
        ax.tick_params(labelleft=False, labelbottom=True)
        ax.legend()


from __future__ import division
import numpy as np
import matplotlib.pyplot as plt
from scipy.optimize import fminbound as fmin

#imports for auxiliar classes
import Spline as spl
import Interpolation as inter

'''functions, f, f(2),f(4) f(9)'''
f = lambda x: (4*np.power(x,2)+np.sin(9*x))
f2 = lambda x: (8-8*np.cos(9*x))
f4 = lambda x: 6561*np.cos(9*x)
f9 = lambda x: -386239509*np.sin(9*x)

def graphicTable(x,y,func=False):
    #Spline and Interpolating polinomial calculation
    #initiates classes
    data = spl.Spline(x,y)
    data2 = inter.Interpolation(x,y)

    spline = data.buildNormalSpline()   #calculates the spline for the dataset
    data2_x,data2_y= data2.interpolateData() #calculates the interpolating pol
    xs = np.arange(x[0],x[-1]+0.0001,0.0001)

    plt.plot(x,y,"o",label="Data")
    if func:
        #displays the actual function f
        plt.plot(xs,f(xs),label='True Value')
    plt.plot(xs,spline(xs),label="Spline Trace")
    plt.plot(data2_x,data2_y,label="Interpolation")
    plt.legend(loc='lower left',ncol=2)
    plt.grid()
    plt.show()

    return data,spline,data2

def pointSet(lower,upper,n):
    #Creates the liniar spaced values in [lower,upper]
    values = np.linspace(lower,upper,num=n,endpoint=True)
    a = []
    for val in values:
        a.append(f(val))
    return values,np.array(a)

def maxH(x):
    #Calculates the max h value in the data_set
    max_h = None
    for i in range(len(x)-1):
        h = x[i+1]-x[i]
        if not max_h:
            max_h = h
        max_h = max(max_h,h)
    return max_h

def M(lower,upper,interpolation=False):
    #Calculates the maximizer  of f4 or f9 in [lower,upper]
    if interpolation:
        return fmin(lambda x : -f9(x),lower,upper)
    return fmin(lambda x : -f4(x),lower,upper)

def interpolationError(x,value,lower,upper):
    #Calculates the major error in interpolating polinomial
    m = M(lower,upper,interpolation=True)
    m = abs(f9(m))
    print("     Max|f9(x)|="+str(m)+" ,"+str(lower)+"<=x<="+str(upper))
    n1 = len(x)+1
    b = np.arange(1,n1)
    fac = b.prod()
    mult = 1
    for i in x:
        mult *= (value-i)
    return (m/fac)*abs(mult)
def splineError(x,lower,upper):
    #Calculates the major error in the spline
    m = M(lower,upper,interpolation=False)
    m = abs(f4(m))
    h = maxH(x)
    print("     Max|f4(x)|="+str(m)+" ,"+str(lower)+"<=x<="+str(upper))
    print("     Max(h)="+str(h))
    return (5/384)*m*np.power(h,4)

def ex1():
    #exercise 1
    x = np.array([0,1,2,2.5,3,4])
    y = np.array([1.4,0.6,1.0,0.6,0.6,1.0])
    spline,natural,interpolator = graphicTable(x,y,func=False)
    print("Spline equations:")
    spline.showEquations(natural)

def ex2():
    #exercise 2
    x,y = pointSet(-1,1,9)
    spline,natural, interpolator = graphicTable(x,y,func=True)

    print("Data Table:") #Show points and images in table
    for i in range(len(x)):
        print("("+str(x[i])+","+str(y[i])+")")

    print("\n\nSpline equations:") #Show Spline equations
    spline.showEquations(natural)
    print("\n\nx=0.3: ")
    print("     S(0.3)="+str(spline.calc(0.3,spline=natural)))
    print("     |f(0.3)-S(0.3)|<="+str(splineError(x,-1,1)))

    print("\n     p(0.3)="+str(interpolator.calc(0.3)))
    print("     |f(0.3)-p(0.3)|<="+str(interpolationError(x,0.3,-1,1)))

    print("\n\nx=0.83:")
    print("     S(0.83)="+str(spline.calc(0.83,spline=natural)))
    print("     |f(0.83)-S(0.83)|<="+str(splineError(x,-1,1)))

    print("\n     p(0.83)="+str(interpolator.calc(0.83)))
    print("     |f(0.83)-p(0.83)|<="+str(interpolationError(x,0.83,-1,1)))

# Starter-----------------------------------------------------------------------
print("Exercise 1:")
ex1()
print("\nExercise2:")
ex2()


import unittest
import numpy as np

from classy_blocks.util import functions as f
from classy_blocks.util import curves as c

from tests import fixtures

class TestFixtures(fixtures.FixturedTestCase, fixtures.ExecutedTestsBase):
    def test_create_mesh(self):
        self.mesh.prepare_data()
        self.run_and_check()

class TestUtils(unittest.TestCase):
    def assert_np_equal(self, a, b, msg=None):
        return np.testing.assert_array_equal(a, b, err_msg=msg)

    def assert_np_almost_equal(self, a, b, msg=None):
        return np.testing.assert_array_almost_equal(a, b, err_msg=msg)

    def test_deg2rad(self):
        """ degrees to radians """
        deg = 30
        self.assertEqual(deg*np.pi/180, f.deg2rad(deg))

    def test_rad2deg(self):
        """ radians to degrees """
        rad = np.pi/3
        self.assertEqual(rad*180/np.pi, f.rad2deg(rad))

    def test_unit_vector(self):
        """ scale vector to magnitude 1 """
        vector = f.vector(3, 5, 7)
        unit_vector = vector/np.linalg.norm(vector)

        self.assert_np_equal(unit_vector, f.unit_vector(vector))

    def test_angle_between(self):
        """ angle between two vectors """
        v1 = f.vector(0, 0, 1)
        v2 = f.vector(0, 2, 0)

        self.assertEqual(f.angle_between(v1, v2), np.pi/2)
        self.assertEqual(f.angle_between(v1, v1), 0)

        v1 = f.vector(1, 0, 0)
        v2 = f.vector(1, 1, 0)
        self.assertAlmostEqual(f.angle_between(v1, v2), np.pi/4)

    def test_rotate(self):
        """ simple rotation around a coordinate system axis """
        v = f.vector(1, 0, 0)
        r = f.rotate(v, np.pi/2, axis='x')

        self.assertAlmostEqual(f.angle_between(r, v), 0)

        r = f.rotate(v, np.pi/4, axis='y')
        self.assertAlmostEqual(f.angle_between(r, v), np.pi/4)

    def test_rotate_exception(self):
        """ an exception must be raised if axis is anything but x, y, or z """
        with self.assertRaises(ValueError):
            f.rotate(f.vector(0, 0, 1), np.pi/2, 'a')

    def test_arbitrary_rotation_point(self):
        """ rotation of a point from another origin """
        point = f.vector(0, 2, 0)
        origin = f.vector(0, 1, 0)
        axis = f.vector(0, 0, 1)

        self.assert_np_equal(
            f.arbitrary_rotation(point, axis, -np.pi/2, origin),
            f.vector(1, 1, 0)
        )

    def test_arbitrary_rotation_axis(self):
        """ rotation of a point around arbitrary axis """
        point = f.vector(1, 0, 0)
        origin = f.vector(0, 0, 0)
        axis = f.vector(1, 1, 0)

        self.assert_np_almost_equal(
            f.arbitrary_rotation(point, axis, np.pi, origin),
            f.vector(0, 1, 0)
        )

    def test_to_polar_z_axis(self):
        """ cartesian coordinate system to polar c.s., rotation around z-axis """
        cartesian = f.vector(2, 2, 5)
        polar = f.vector(8**0.5, np.pi/4, 5)

        self.assert_np_almost_equal(polar, f.to_polar(cartesian, axis='z'))

    def test_to_polar_x_axis(self):
        """ cartesian coordinate system to polar c.s., rotation around x-axis """
        cartesian = f.vector(5, 2, 2)
        polar = f.vector(8**0.5, np.pi/4, 5)

        self.assert_np_almost_equal(polar, f.to_polar(cartesian, axis='x'))

    def test_lin_map(self):
        """ map a value """
        self.assertEqual(f.lin_map(10, 0, 100, 0, 1000), 100)

    def test_lin_map_limit(self):
        """ map a value within given limits """
        self.assertEqual(f.lin_map(200, 0, 10, 0, 100, limit=True), 100)
        self.assertEqual(f.lin_map(-5, 0, 10, 0, 100, limit=True), 0)

    def test_xy_line_intersection(self):
        """ intersection of two lines in xy-plane """
        # line 1
        p1 = f.vector(0, 0, 0)
        p2 = f.vector(1, 1, 0)

        # line 2
        p3 = f.vector(1, 0, 0)
        p4 = f.vector(0, 1, 0)

        self.assert_np_equal(
            f.xy_line_intersection(p1, p2, p3, p4),
            f.vector(0.5, 0.5, 0)
        )

    def test_extend_to_y(self):
        """ extend a line given by two points to desired y-coordinate """
        p1 = f.vector(0, 0, 0)
        p2 = f.vector(1, 1, 0)
        y = 5

        self.assert_np_equal(
            f.extend_to_y(p1, p2, y),
            f.vector(y, y, 0)
        )

    def test_to_cartesian_point(self):
        """ polar point to xyz """
        # polar point
        p = np.array([1, np.pi/2, 2])

        # cartesian versions
        # axis: z
        self.assert_np_almost_equal(
            f.to_cartesian(p, axis='z'),
            f.vector(0, 1, 2))

        self.assert_np_almost_equal(
            f.to_cartesian(p, axis='x'),
            f.vector(2, 0, 1))

        self.assert_np_almost_equal(
            f.to_cartesian(p, axis='x', direction=-1),
            f.vector(2, 0, -1))

    def test_polar2cartesian_x_axis(self):
        cp = np.array([32, 198, 95])
        pp = f.to_polar(cp, axis='x')

        self.assert_np_almost_equal(cp, f.to_cartesian(pp, axis='x'))

    def test_polar2cartesian_z_axis(self):
        cp = np.array([32, 198, 95])
        pp = f.to_polar(cp, axis='z')

        self.assert_np_almost_equal(cp, f.to_cartesian(pp, axis='z'))

    def test_to_cartesian_asserts(self):
        """ garbage input to f.to_cartesian() """
        p = np.array([1, 0, 1])

        with self.assertRaises(AssertionError):
            f.to_cartesian(p, direction=0)

        with self.assertRaises(AssertionError):
            f.to_cartesian(p, axis='a')

    def test_dilute_indexes(self):
        """ generate equally-spaced indexes """
        self.assert_np_equal(
            c.dilute_indexes(11, 6),
            np.array([0, 2, 4, 6, 8, 10])
        )

    def test_dilute_points(self):
        """ return equally-spaced elements of an array """
        points = np.array([2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22])
        diluted_points = c.dilute_points(points, 6)

        self.assert_np_equal(
            diluted_points,
            np.array([2, 6, 10, 14, 18, 22])
        )

    def test_curve_length(self):
        """ length of a curve given by a list of points """
        curve = np.array([
            f.vector(0, 0, 0),
            f.vector(1, 0, 0),
            f.vector(1, 1, 0),
            f.vector(1, 2, 0),
            f.vector(0, 2, 0),
        ])

        self.assertAlmostEqual(c.curve_length(curve), 4)

    def test_to_cartesian_curve(self):
        """ f.to_cartesian() for a list of points """
        # polar points
        points = [
            f.vector(1, 0, 0),
            f.vector(1, np.pi/2, 1),
            f.vector(2, 0, 1)
        ]

        self.assert_np_almost_equal(
            c.to_cartesian(points),
            np.array([
                f.vector(1, 0, 0),
                f.vector(0, 1, 1),
                f.vector(2, 0, 1)
            ])
        )

    def test_arc_length_3point_half(self):
        A = [0, 0, 0]
        B = [1, 1, 0]
        C = [2, 0, 0]
        self.assertAlmostEqual(f.arc_length_3point(A, B, C), np.pi)

    def test_arc_length_3point_quarter(self):
        A = [0, 0, 0]
        s2 = 2**0.5/2
        B = [1 - s2, s2, 0]
        C = [1, 1, 0]
        self.assertAlmostEqual(f.arc_length_3point(A, B, C), np.pi/2)

    def test_arc_length_3point_3quarter(self):
        A = [0, 0, 0]
        s2 = 2**0.5/2
        B = [1 + s2, s2, 0]
        C = [1, -1, 0]
        self.assertAlmostEqual(f.arc_length_3point(A, B, C), 3*np.pi/2)

    def test_arc_length_3point_full(self):
        A = [0, 0, 0]
        B = [2, 0, 0]
        C = [0, 0, 0]

        with self.assertRaises(ValueError):
            self.assertAlmostEqual(f.arc_length_3point(A, B, C), 2*np.pi)

    def test_arc_length_3point_zero(self):
        A = [0, 0, 0]
        B = [0, 0, 0]
        C = [0, 0, 0]

        with self.assertRaises(ValueError):
            self.assertAlmostEqual(f.arc_length_3point(A, B, C), 0)


import numpy as np

from utils import load_data, plot_data, computer_cost, normalize

def gradient_descent(X, Y, alpha, iteration):
    m = Y.size
    theta = np.zeros((X.shape[1], 1))
    j_list = []
    for i in range(iteration):
        v = computer_cost.computer(X, Y, theta)
        j_list.append(v)
        values = (X.dot(theta) - Y).T.dot(X).T * (alpha / m)
        theta = theta - values
    return theta, np.array(j_list)

def main():
    X_normalize, Y_normalize, Origin_X_normalize, Y_mean, Y_std, Origin_X_mean, Origin_X_std = load_data.load_and_normalize('data2.txt')

    alpha = 0.024
    iterations = 800
    theta, j_list = gradient_descent(X_normalize, Y_normalize, alpha, iterations)
    print('The linear regression formula is {0} + {1} * x1 + {2} * x2'.format(theta[0, 0], theta[1, 0], theta[2,0]))

    iteration_array = range(iterations)
    plot_data.plot(iteration_array, j_list, 'iteration', "lost",
                   {
                       'fmt': 'b-',
                       'title': 'Lost',
                       'show': True
                   })

    house_size = 1650
    br_num = 3

    house_size_normal = (house_size - Origin_X_mean[0]) / Origin_X_std[0]
    br_num = (br_num - Origin_X_mean[1]) / Origin_X_std[1]

    X_new_normalize = np.array([1, house_size_normal, br_num])
    price_normalize = X_new_normalize.dot(theta).sum()
    price = price_normalize * Y_std[0] + Y_mean[0]
    print('price is {0}'.format(price))

if __name__ == '__main__':
    main()


import time
import numpy as np
from utils.features.pca import *

def pca_preprocess(X, K):
    """Data transformation step"""
    N, D = X.shape;
    X = X - np.tile(np.mean(X, 0), (N, 1))
    P, _ = pca_naive(X, K)
    return np.dot(X, P.T).real

def p_entropy(X2, sigma):
    """Calculating perplexity entropy"""
    epsilon = 1e-15
    P = np.exp(-X2*sigma)
    P = P / np.sum(P)
    H = -np.dot(np.log(P+epsilon), P)

    return P,H

def binary_search(X, perplexity):
    """Binary search for perplexity"""
    tol = 1e-5
    goal = np.log(perplexity)
    N, D = X.shape
    sigma = np.ones((N,)).astype('float')
    P = np.zeros((N,N))

    # X2: norm2 distance matrix - NxN
    X2 = (-2*np.dot(X, X.T) + np.sum(X**2, axis=1)).T + np.sum(X**2, axis=1)

    # loop over all points
    for i in range(N):

        sigma_max = np.inf
        sigma_min = 0
        maxiter = 50

        sigma_i = sigma[i]

        for t in range(maxiter):
            X2_i = X2[i, np.concatenate((np.r_[0:i], np.r_[i+1:N]))]
            Pi, Hi = p_entropy(X2_i, sigma_i)
            # binary search for a correct sigma_i
            if abs(Hi-goal) < tol:
                break
            else:
                if Hi > goal:
                    sigma_min = sigma_i
                    if sigma_max == np.inf:
                        sigma_i *= 2
                    else:
                        sigma_i = (sigma_i + sigma_max)/2
                else:
                    sigma_max = sigma_i
                    sigma_i = (sigma_i + sigma_min)/2

        # Set Pi
        sigma[i] = sigma_i
        P[i, np.concatenate((np.r_[0:i], np.r_[i+1:N]))] = Pi

    #Return P
    print(np.mean(np.sqrt(1/sigma)))
    return P

def tsne(X, low_dim=2, perplexity=30.0):
    """
    tSNE - t-Stochastic Neighbor Embedding
    Inputs:
    - X: (float) an array of shape(N,D)
    - low_dim: (int) dimenional of output data
    - pca_dim: (int) rather than using the raw data, we can apply
                pca preprocessing to reduce the dimension to pca_dim
    - perplexity:

    Returns;
    - Y: (float) an array of shape (N,low_dim)
    """

    # X = pca_preprocess(X, low_dim)

    N,D = X.shape

    P = binary_search(X, perplexity)
    P = (P + P.T) / (2*N)
    P = np.maximum(P, 1e-12)
    P *= 4
    Y = np.random.normal(0, 1e-4, (N,low_dim))

    T = 1000
    # training parameters
    momentum = 0.5 # initial momentum
    V = np.zeros_like(Y)
    lr = 100 # initial learning rate
    beta = 0.8
    kappa = 0.2
    gamma = 0.2
    mu = np.ones_like(V)

    tic = time.time()
    for t in range(T):

        Y2 = (-2*np.dot(Y, Y.T) + np.sum(Y**2, axis=1)).T + np.sum(Y**2, axis=1)
        Q_numerator = 1/(1 + Y2)
        Q = Q_numerator
        np.fill_diagonal(Q, 0)
        Q = Q / np.sum(Q)
        Q = np.maximum(Q,1e-12)

        dY = np.zeros_like(Y)
        for i in range(N):
            dY[i,:] = 4*np.dot((P[i,:]-Q[i,:])*Q_numerator[i,:], Y-Y[i,:])

        # calculate learning rate
        dY_hat = (1-beta)*dY + beta*V
        mu[np.where(dY*dY_hat>0)] = mu[np.where(dY*dY_hat>0)] + kappa
        mu[np.where(dY*dY_hat<0)] = (1-gamma)*mu[np.where(dY*dY_hat<0)]
        # update
        if t > 250:
            momentum = 0.8
        V = momentum*V + lr*mu*dY
        Y += V

        # stop early exaggeration
        if t == 100:
            P /= 4

        # verbose: report intermediate result
        if (t+1) % 100 == 0:
            cost = np.sum(P * np.log(P / Q));
            print('The {} th loop cost: {}, computation time: {}'.format(t+1, cost, time.time()-tic))

    return Y


# -*- coding: utf-8 -*-
"""
Created on Mon Sep 21 08:33:44 2020

@author: Ashlee
"""

import numpy as np

from scipy.integrate import odeint
import matplotlib.pyplot as plt

def system_of_ODEs(numerators=[6.25,0],denominator=0, parameters=[0.15,0.6,0.1,0.2]):
    # numerators are the dependent variables
    # denominator is the independent variable
    CA = numerators[0]
    CB = numerators[1]
    t = denominator
    k1 = parameters[0]
    k2 = parameters[1]
    k3 = parameters[2]
    k4 = parameters[3]

    dCAdt = -k1*CA-k2*CA
    dCBdt = k1*CA-k3-k4*CB

#    return np.array([[dCAdt],[dCBdt]]) # column vector
    return [dCAdt,dCBdt]

tspan = np.linspace(0,10,101)
CA0 = 6.25
CB0 = 0
C0 = CA0,CB0

k1 = 0.15
k2 = 0.6
k3 = 0.1
k4 = 0.2
parameters = (k1,k2,k3,k4)

output = odeint(system_of_ODEs, C0,tspan, args = (parameters,))
print(output)
plt.plot(output)

CHEclassFa20/In Class Problem Solutions/Python/solve_system_of_ODEs.py


import numpy as np
from numpy.random import choice, rand, randint
from numba import jit, njit

######## Main Code ###########

@njit
def mcmc(spins, N, J, B, T, n_steps = 10000, out_freq = 1000):

    '''mcmc takes spin configuration and samples with given N,J,B,T
    for n_steps outputing results every out_freq'''

    confs = []

    for step in range(n_steps):

        #Pick random spin
        i, j = randint(N), randint(N)

        #Compute energy change
        z  = spins[(i+1)%N, j] + spins[(i-1)%N, j] + spins[i, (j+1)%N] + spins[i, (j-1)%N]
        dE = 2*spins[i,j]*(J*z + B)

        #Metropolis condition
        if dE <= 0 or np.exp(-dE/T) > rand():

            spins[i,j] *= -1

        #Store the spin configuration
        if step % out_freq == 0:

            confs.append(spins.copy())

    return confs

@njit
def getM(spins):

    return np.mean(spins)

@njit
def getE(spins,N,J,B):

    E = 0

    for i in range(N):
        for j in range(N):

            z = spins[(i+1)%N, j] + spins[(i-1)%N, j] +spins[i,(j+1)%N] +  spins[i,(j-1)%N]

            E += -J*z*spins[i,j]/4 # Since we overcounted interactions 4 times divide by 4.

    return E - B*np.sum(spins) #Field contribution added

@jit
def temper(configs):

    '''Randomly pick two adjacent replicas and attempt an exchange'''

    i       = np.random.randint(N_repl-1)
    j       = i+1

    deltaBeta      = 1/T[i] - 1/T[j]

    deltaEnergy    = getE(configs[i][-1],N,J,B) - getE(configs[j][-1],N,J,B)

    if deltaBeta*deltaEnergy < 0 or np.exp(-deltaBeta*deltaEnergy) > rand():

        configs[i][-1], configs[j][-1] = configs[j][-1], configs[i][-1]

    return configs

@jit
def pt_mcmc(N, J, B, T=[1, 0.1], n_exch=1000, n_steps=10000, out_freq=1000):

    N_repl     = len(T)
    configs    = [[choice([-1,1], (N,N))] for i in range(N_repl)]

    for exch_attempt in range(n_exch): #Exchange attemps

        configs = temper(configs)

        for i in range(N_repl): #mcmc in between exchange attemps

            configs_new = mcmc(configs[i][-1],N, J, B, T[i])

            configs[i].extend(configs_new)

    return configs


import pandas as pd
import random
import numpy as np

N = 10000

TotalTime=[]

T =  np.empty(shape=(N,6))

TaskTimes=[[3,5,8],
          [2,4,7],
           [3,5,9],
           [4,6,10],
           [3,5,9],
           [2,6,8]]

Lh=[]
for i in range(6):
    Lh.append((TaskTimes[i][1]-TaskTimes[i][0])/(TaskTimes[i][2]-TaskTimes[i][0]))

for p in range(N):
    for i in range(6):
        trand=random.random()
        if (trand < Lh[i]):
            T[p][i] = TaskTimes[i][0] + np.sqrt(trand*(TaskTimes[i][1]-TaskTimes[i][0])*(TaskTimes[i][2]-TaskTimes[i][0]))
        else:
            T[p][i] = TaskTimes[i][2] - np.sqrt((1-trand)*(TaskTimes[i][2]-TaskTimes[i][1])*(TaskTimes[i][2]-TaskTimes[i][0]))
    TotalTime.append( T[p][0]+ np.maximum(T[p][1],T[p][2]) + np.maximum(T[p][3],T[p][4]) + T[p][5])

Data = pd.DataFrame(T,columns=['Task1', 'Task2', 'Task3', 'Task4', 'Task5', 'Task6'])

pd.set_option('display.max_columns', None)
print(Data.describe())

hist = Data.hist(bins=10)

print("Minimum project completion time = ",np.amin(TotalTime))

print("Mean project completion time = ",np.mean(TotalTime))

print("Maximum project completion time = ",np.amax(TotalTime))


"""Mixture model using EM"""
from typing import Tuple
import numpy as np
from utils import GaussianMixture

def estep(X: np.ndarray, mixture: GaussianMixture) -> Tuple[np.ndarray, float]:
    """E-step: Softly assigns each datapoint to a gaussian component

    Args:
        X: (n, d) array holding the data
        mixture: the current gaussian mixture

    Returns:
        np.ndarray: (n, K) array holding the soft counts
            for all components for all examples
        float: log-likelihood of the assignment
    """

    n, d = X.shape
    K, _ = mixture.mu.shape
    post = np.zeros((n, K))
    dr = np.sqrt((2*np.pi*mixture.var)**d)
    exponent_nr = -0.5*(np.linalg.norm(X[:,:,None] - mixture.mu.T,axis=1)**2)
    normal_ = np.exp(exponent_nr/mixture.var)/dr
    log_likelihood = np.sum(np.log(np.sum(mixture.p*normal_,axis=1)))
    post = (mixture.p*normal_)/(np.sum(mixture.p*normal_,axis=1,keepdims=True))

    return post, log_likelihood

def mstep(X: np.ndarray, post: np.ndarray) -> GaussianMixture:
    """M-step: Updates the gaussian mixture by maximizing the log-likelihood
    of the weighted dataset

    Args:
        X: (n, d) array holding the data
        post: (n, K) array holding the soft counts
            for all components for all examples

    Returns:
        GaussianMixture: the new gaussian mixture
    """

    n, d = X.shape
    _, K = post.shape
    n_hat = post.sum(axis=0, keepdims=True)
    p = n_hat / n
    mu = np.zeros((K, d))
    var = np.zeros(K)
    mu = post.T @ X / n_hat.T
    var = np.sum(post * (np.linalg.norm(X[:,:,None] - mu.T,axis=1)**2),axis=0)/(d*n_hat)

    return GaussianMixture(mu, var[0], p[0])

def run(X: np.ndarray, mixture: GaussianMixture,
        post: np.ndarray) -> Tuple[GaussianMixture, np.ndarray, float]:
    """Runs the mixture model

    Args:
        X: (n, d) array holding the data
        post: (n, K) array holding the soft counts
            for all components for all examples

    Returns:
        GaussianMixture: the new gaussian mixture
        np.ndarray: (n, K) array holding the soft counts
            for all components for all examples
        float: log-likelihood of the current assignment
    """
    prev_cost = None
    cost = None
    while True:
        prev_cost = cost
        post, cost = estep(X, mixture)
        mixture = mstep(X, post)
        try:
            if (abs(prev_cost - cost) < (1e-6*np.abs(cost))):
                break
            else:
                continue
        except:
            pass
    return mixture, post, cost


import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
from sklearn.linear_model import LogisticRegression
from scipy import optimize as opt
import time

start = time.perf_counter()

#loading the dataset
dataset = pd.read_csv('mnist_train.csv')

#specifying predictor and response
# predictor having shape as (60000,784)
x = dataset.iloc[:, 1:].values

# reponse having shape as (60000,)
y = dataset.iloc[:, 0].values

#splitting the dataset into training and validation dataset
#taking 75% training data, 25% for test data
#taking 85% training data, 15% for test data
#taking 90% training data, 10% for test data
xtrain, xtest, ytrain, ytest = train_test_split(
		x, y, test_size = 0.1, random_state = 0)

#shape of the data
#adding constant term in x
mtrain = len(ytrain)
ones = np.ones((mtrain,1))
xtrain = np.hstack((ones, xtrain)) #add the constant
(mtrain,n) = xtrain.shape

mtest = len(ytest)
ones = np.ones((mtest,1))
xtest = np.hstack((ones, xtest)) #add the constant
(mtest,n) = xtest.shape

#vecorized implementaton of sigmoid function
def sigmoid(z):
  return 1/(1+np.exp(-z))

#cost function
def costFunctionReg(theta, X, y, lmbda):
  m = len(y)
  temp1 = np.multiply(y, np.log(sigmoid(np.dot(X, theta))))
  temp2 = np.multiply(1-y, np.log(1-sigmoid(np.dot(X, theta))))
  return np.sum(temp1 + temp2) / (-m) + np.sum(theta[1:]**2) * lmbda / (2*m)

#vectorization done for regularized gradient
def gradRegularization(theta, X, y, lmbda):
  m = len(y)
  temp = sigmoid(np.dot(X, theta)) - y
  temp = np.dot(temp.T, X).T / m + theta * lmbda / m
  temp[0] = temp[0] - theta[0] * lmbda / m
  return temp

#inital parameters
# lmbda = 0.1 gave accu_perct as 78.9, test set accuracy as 79.1
# lmbda = 0.2 gave accu_perct as 78.616, test set accuracy as 79.07
# lmbda = 0.8 gave accu_perct as 78.933 test set accuracy as 79.1
# lmbda = 0.08 gave accu_perct as 79.0, test set accuracy as 79.19
# lmbda = 0.04 gave accu_perct as 78.966, test set accuracy as 78.97
# lmbda = 0.02 gave accu_perct as 78.967, test set accuracy as 79.15
# lmbda = 0.01 gave accu_perct as 78.867, test set accuracy as 79.24 with 50 iteratons
# lmbda = 0.01 gave accu_perct as 79.1, test set accuracy as 79.37 with 100 iteratons
# lmbda = 0.01 gave accu_perct as 79.467, test set accuracy as 79.37 with 500 iteratons
lmbda = 0.09 #gave accu_perct as 79.35, test set accuracy as 79.47 with 1000 iteratons
k = 10
theta = np.zeros((k,n))

#fmin_cg from scipy library uses cost function minimization
#using a nonlinear conjugate gradient algorithm.
# f is the target of minimization
# fprime is the gradient of taget at the precise values of x and y
# args are the argument list as in the order defined in the functions
# maxiter is the maximum iterations it will run
for i in range(k):
  digit_class = i if i else 10
  theta[i] = opt.fmin_cg(f = costFunctionReg, x0 = theta[i],  fprime = gradRegularization, args = (xtrain, (ytrain == digit_class).flatten(), lmbda), maxiter = 1000)

#predicting as per the model
#using One-vs-All technique
pred = np.argmax(xtest @ theta.T, axis = 1)
pred = [e if e else 10 for e in pred]
accu_perct = np.mean(pred == ytest.flatten()) * 100

print(accu_perct)

#testing with test data from mnist_test.csv
#loading the dataset
dataset_test = pd.read_csv('mnist_test.csv')

#specifying predictor and response
# predictor having shape as (10000,784)
x_test = dataset_test.iloc[:, 1:].values

# reponse having shape as (10000,)
y_test = dataset_test.iloc[:, 0].values

#adding the constant term
m = len(y_test)
ones = np.ones((m,1))
x_test = np.hstack((ones, x_test)) #add the constant

#predicting as per the model
#using One-vs-All technique
pred_test = np.argmax(x_test @ theta.T, axis = 1)
pred_test = [e if e else 10 for e in pred_test]
accu_perct_test = np.mean(pred_test == y_test.flatten()) * 100

print(accu_perct_test)

print("Time Taken:",round(time.perf_counter(),2))


import numpy as np

def get_divisors(x):
	rez = [1]
	top_bar = x/2+1
	for i in xrange(2, x/2+1):
		if i >= top_bar:
			break
		div_whole = x / i
		if x % i == 0:
			rez.append(i)
			rez.append(div_whole)
			if div_whole < top_bar:
				top_bar = div_whole
	rez.sort()
	return rez

# print(np.array(get_divisors(220)).sum())
# print(np.array(get_divisors(284)).sum())
# exit()

total_sum = 0
found_pairs = []
for n in xrange(1, 10000):
	if n in found_pairs:
		continue
	divs1_sum = np.array(get_divisors(n)).sum()
	divs2_sum = np.array(get_divisors(divs1_sum)).sum()
	if divs2_sum == n and divs1_sum != n:
		total_sum += n
		total_sum += divs1_sum
		found_pairs.append(n)
		found_pairs.append(divs1_sum)

print(total_sum)


import numpy as np
import matplotlib.pylab  as plt
def Kendall_change_point_detection(inputdata):
    inputdata1 = np.array(inputdata)
    n=inputdata1.shape[0]
    Sk             = [0]
    UFk            = [0]
    s              =  0
    Exp_value      = [0]
    Var_value      = [0]
    for i in range(1,n):
        for j in range(i):
            if inputdata1[i] > inputdata1[j]:
                s = s+1
            else:
                s = s+0
        Sk.append(s)
        Exp_value.append((i+1)*(i+2)/4 )                     # Sk[i] mean
        Var_value.append((i+1)*i*(2*(i+1)+5)/72 )            # Sk[i] std
        UFk.append((Sk[i]-Exp_value[i])/np.sqrt(Var_value[i]))
    #Inverse sequence computing generation
    Sk2             = [0]
    # Define the reverse order statistics UBk, the length is consistent with inputdata, initial value = 0
    UBk             = [0]
    UBk2            = [0]
    # s=0
    s2              =  0
    Exp_value2      = [0]
    Var_value2      = [0]
    #
    inputdataT = list(reversed(inputdata1))
    for i in range(1,n):
        for j in range(i):
            if inputdataT[i] > inputdataT[j]:
                s2 = s2+1
            else:
                s2 = s2+0
        Sk2.append(s2)
        Exp_value2.append((i+1)*(i+2)/4 )                     # Sk[i] mean
        Var_value2.append((i+1)*i*(2*(i+1)+5)/72 )            # Sk[i] std
        UBk.append((Sk2[i]-Exp_value2[i])/np.sqrt(Var_value2[i]))
        UBk2.append(-UBk[i])
    UBkT = list(reversed(UBk2))
    diff = np.array(UFk) - np.array(UBkT)
    K    = list()
    # Find the intersection
    for k in range(1,n):
        if diff[k-1]*diff[k]<0:
            K.append(k)

 #   plt.figure(figsize=(10,5))
 #   plt.plot(range(1,n+1) ,UFk  ,label='UFk') # UFk
 #   plt.plot(range(1,n+1) ,UBkT ,label='UBk') # UBk
#    plt.ylabel('UFk-UBk')
 #   x_lim = plt.xlim()
#    plt.plot(x_lim,[-1.96,-1.96],'m--',color='r')
#    plt.plot(x_lim,[  0  ,  0  ],'m--')
#    plt.plot(x_lim,[+1.96,+1.96],'m--',color='r')
#    plt.legend(loc=2) # 图例
#     plt.savefig("./Dectection/{}_Mann-Kendall.png".format(name))
#    plt.show()
#   b=[]
#    for i in K:
#        tmp=inputdata.index[i]
#        b.append(tmp.strftime("%Y-%m-%d"))
    return K


# -*- coding: utf-8 -*-
# !/usr/bin/env python3

import numpy as np
import pandas as pd
from matplotlib import pyplot as plt

def pdf(x):
    return exp_lambda * np.exp(-exp_lambda * x)

def cdf(x):
    return 1 - np.exp(-exp_lambda * x)

def correlate(x, y):
    n = len(x)
    sx = sum(x)
    sy = sum(y)
    sx2 = sum(i ** 2 for i in x)
    sy2 = sum(i ** 2 for i in y)
    sxy = sum(x[i] * y[i] for i in range(n))
    return (n * sxy - sx * sy) / ((n * sx2 - sx * sx) * (n * sy2 - sy * sy)) ** 0.5

# Question 1(A)
exp_lambda = 57  # average confirmed cases per hour
x = np.arange(0, 0.1, 10 ** -6)
y = pdf(x)
plt.xlabel('Wait time in hours')
plt.ylabel('Probability density')
plt.plot(x, y)
plt.show()

# Question 1(B)
print("P(X < 1/60) = {:.6f}".format(cdf(1 / 60)))

# Question 1(C)
print("P(1/60 < X < 2/60) = {:.6f}".format(cdf(1 / 30) - cdf(1 / 60)))

# Question 1(D)
print("P(X > 2/60) = {:.6f}".format(1 - cdf(1 / 30)))

# Question 1(E)
exp_lambda = 114  # average confirmed cases per hour doubles
print("P(1/60 < X < 2/60) = {:.6f}".format(cdf(1 / 30) - cdf(1 / 60)))

# Question 2(A)
df = pd.read_csv('IC252_Lab8.csv')
df['Status'] = df['Status'].replace({"Hospitalized": 1, "Recovered": 2, "Dead": 3})

# Question 2(B)
r = [['Status,Population', correlate(df['Status'], df['Population'])],
     ['Status,SexRatio', correlate(df['Status'], df['SexRatio'])],
     ['Status,Literacy', correlate(df['Status'], df['Literacy'])],
     ['Status,Age', correlate(df['Status'], df['Age'])],
     ['Status,SmellTrend', correlate(df['Status'], df['SmellTrend'])],
     ['Status,Gender', correlate(df['Status'], df['Gender'])]]
print('\n'.join('r({}) = {:.6f}'.format(k[0], k[1]) for k in r))

# Question 2(C)
print(' > '.join(['r({})'.format(k[0]) for k in sorted(r, key=lambda x: x[1], reverse=True)]))


import numpy as np
import stats
import seaborn as sns
import matplotlib.pyplot as plt

# ----------- #
# Utils:
#   * scree_plot
#   * Hinton diagram
#
#
# ----------- #

def scree_plot(arr : np.ndarray, figsize=(18, 8)):
    """Generates a scree plot for a given data array.

    Parameters
    ----------
    arr : np.ndarray
        matrix in (samples x features) form
    figsize : tuple, optional
        by default (18, 8)
    """
    eigvals = np.linalg.eigs(stats.cov(arr))
    plt.figure(figsize=figsize)
    plt.title('Eigenvalue versus magnitude (Scree plot)')
    sns.barplot(x=np.arange(len(eigvals)), y=eigvals, color='blue', saturation=.3)
    plt.ylabel('Magnitude')
    plt.xlabel('Eigenvalue index')
    plt.show()

def hinton(matrix, max_weight=None, ax=None):
    """Draw Hinton diagram for visualizing a weight matrix."""
    ax = ax if ax is not None else plt.gca()

    if not max_weight:
        max_weight = 2 ** np.ceil(np.log(np.abs(matrix).max()) / np.log(2))

    ax.patch.set_facecolor('gray')
    ax.set_aspect('equal', 'box')
    ax.xaxis.set_major_locator(plt.NullLocator())
    ax.yaxis.set_major_locator(plt.NullLocator())

    for (x, y), w in np.ndenumerate(matrix):
        color = 'white' if w > 0 else 'black'
        size = np.sqrt(np.abs(w) / max_weight)
        rect = plt.Rectangle([x - size / 2, y - size / 2], size, size,
                             facecolor=color, edgecolor=color)
        ax.add_patch(rect)

    ax.autoscale_view()
    ax.invert_yaxis()

# ----------- #
# Linear Methods:
#   * pca
#
#
#
# ----------- #

def pca(data : np.ndarray, dim : int = 2, verbose : bool = False, class_identity=None, return_reconstruction=False) -> np.ndarray:
    """Principal components analysis to form a (dim) dimensional approximation
    of a given dataset.

    Parameters
    ----------
    data : np.ndarray
        matrix in (samples x features) form
    dim : int, optional
        approximating dimension, must be less than the data dimension, by default 2
    verbose : bool, optional
        whether to display reconstruction diagnostics and a scree plot
    Returns
    -------
    np.ndarray
        dim-dimensional representation of the input data
    """
    if np.sum(np.isnan(data).astype(int)) > 0:
        print("missing data detected. consider using pca_missing_data")
        raise(NotImplemented)

    reconstruction=None
    n, p = data.shape
    assert 0 < dim < p, "projection dimension must be a positive integer less than the number of data dimensions {}".format(p)

    # center the data
    de_meaned = stats.center(data)

    # optimize for high dimensionality data
    if p > n:
        print('High dimensional data detected, optimizing...')

        # X@X.T is (n x n) which we're assuming is actually smaller than (p x p) in this case
        X = de_meaned
        eigvals, X_eigvecs = np.linalg.eig(X @ X.T)

        # plop the eigenvalues into a diagonal matrix lambda
        lam = np.diag(np.real(eigvals))

        # compute the eigenvecs
        # transformed_E = X @ X_eigvecs
        lam_inv = np.linalg.inv(lam)
        eigvecs = (X.T @ X_eigvecs @ lam_inv)

    else:
        # generate the data covariance matrix
        sample_covariance = stats.cov(de_meaned)

        # extract spectrum of the covariance matrix (its set of eigenvalues and eigenvectors)
        eigvals, eigvecs = np.linalg.eig(sample_covariance)

    # arrange the eigenvectors so that they are ordered according to the magnitude (large->small) of their corresponding eigenvalue
    idx = eigvals.argsort()[::-1]
    eigvals = np.real(np.array(eigvals[idx]))
    eigvecs = np.real(np.array(eigvecs[:, idx]))
    E = eigvecs[:, :dim]

    # compute low dimensional representation.
    Y = np.array(data @ E)

    if verbose is True:
        # scree plot
        plt.figure(1, figsize=(18, 8))
        plt.title('Eigenvalue versus magnitude (Scree plot)')
        sns.barplot(x=np.arange(len(eigvals)), y=eigvals, color='blue', saturation=.3)
        plt.ylabel('Magnitude')
        plt.xlabel('Eigenvalue index')
        plt.show()

        # reconstruction
        reconstruction = Y @ E.T
        assert reconstruction.shape == data.shape
        reconstruction_params = [data, reconstruction]
        labels = ['Original data', 'Reconstruction']

        fig, axes = plt.subplots(ncols=2, nrows=1, figsize=(18, 8))
        for i, ax in zip(range(2), axes.flat):
            # extract given params
            given_params = reconstruction_params[i]
            ax.set_title(labels[i])
            sns.heatmap(given_params, cmap='Blues_r', alpha=0.65, annot=False, cbar=False, xticklabels=False, yticklabels=False, ax=ax)

        fig.tight_layout()
        plt.show()

        # print("Reconstruction error: {}".format(round(np.sum(np.linalg.norm(data - reconstruction, axis=1)), 2)))

        # plot 2D projection
        if labels:
            to_plot = Y
            if dim != 2:
                to_plot = pca(data, dim= 2, verbose=False)
            plt.figure(figsize=(10, 8))
            plt.title("2D Data Representation")
            sns.scatterplot(x=to_plot[:, 0], y=to_plot[:, 1], hue=class_identity, legend='full', \
                                palette=sns.color_palette('bright', n_colors=len(np.unique(class_identity))))
            plt.show()
        else:
            to_plot = Y
            if dim != 2:
                to_plot = pca(data, dim= 2, verbose=False)
            plt.figure(figsize=(10, 8))
            plt.title("2D Data Representation")
            sns.scatterplot(x=to_plot[:, 0], y=to_plot[:, 1])
            plt.show()

        # Hinton diagram for eigenvalue matrix (slow af though, only use for small matrices)
        if lam.shape[0] < 20:
            hinton(lam)
            plt.show()

    if return_reconstruction is True and reconstruction is not None:
        return Y, reconstruction

    elif return_reconstruction is True:
        reconstruction = Y @ E.T
        return Y, reconstruction

    else:
        return Y

def pca_missing_data(X : np.ndarray, projection_dim: int=2, max_iters: int=100, tol : float=.01, verbose : bool=False):
    assert np.sum(np.isnan(X).astype(int)) > 0, "pca_missing_data should only be called on data with missing values"

    def _fill_nan(X : np.ndarray):
        col_mean = np.nanmean(X, axis=0)

        #Find indicies that you need to replace
        inds = np.where(np.isnan(X))

        #Place column means in the indices. Align the arrays using take
        X[inds] = np.take(col_mean, inds[1])

        return X

    d, n = X.shape
    m = projection_dim
    G = np.isnan(X).astype(int)
    Y = np.zeros((m, n))

    # initialize B as the imputed basis
    X_imputed = _fill_nan(X)
    B_full, _, _ = np.linalg.svd(X_imputed)

    B = B_full[:, :m]

    old_msqerror=np.nan
    errs = []

    for iter in range(max_iters):
        for i in range(n):
            comprow= np.nonzero((G[:, i]==0).astype(int))[0] # complete rows
            Mn = B[comprow,:].T@B[comprow,:]
            cn=B[comprow,:].T@X[comprow][:, i]
            Y[:, i]=np.linalg.pinv(Mn)@cn
        for j in range(d): # update B (basis)
            goodn=np.nonzero((G[j, :]==0).astype(int))[0] #  complete columns
            Yg = Y[:, goodn]
            Fd = Yg@Yg.T
            md=Y[:,goodn]@(X[j][goodn]).T
            B[j,:]=(np.linalg.pinv(Fd)@md).T

        reconstruction = _fill_nan((X-(B@Y)))
        msqerror = np.linalg.norm(reconstruction)
        errs.append(msqerror)

        # convergence testing
        if abs(old_msqerror-msqerror) < tol:
            if verbose is True:
                print("converged after {} iterations".format(iter))
            break

        else:
            old_msqerror = msqerror

    if verbose is True:
        plt.figure(figsize=(18, 8))
        plt.title('reconstruction error versus iteration')
        plt.scatter(np.arange(len(errs)), errs)
        plt.xlabel('iteration')
        plt.ylabel('reconstruction error')
        plt.show()

    return B, Y


import numpy as np
from convexity_check import is_not_concave, is_not_convex

def convex_f(x):
    return x[:,0]**2+x[:,1]**2

def concave_f(x):
    return -x[:,0]**2-x[:,1]**2

def bohachevsky1(x):
    return x[:,0]**2+2*x[:,1]**2-0.3*np.cos(3*np.pi*x[:,0])-0.4*np.cos(4*np.pi*x[:,1])+0.7
def bohachevsky2(x):
    return x[:,0]**2+2*x[:,1]**2-0.3*np.cos(3*np.pi*x[:,0])*np.cos(4*np.pi*x[:,1])+0.3

bounds = np.array([[0,0],[1,1]])
assert is_not_concave(bohachevsky1,2,bounds)
assert is_not_convex(bohachevsky1,2,bounds)
assert is_not_concave(bohachevsky2,2,bounds)
assert is_not_convex(bohachevsky2,2,bounds)
try:
    assert not is_not_convex(convex_f,2,bounds)
except:
    pass
try:
    assert not is_not_concave(concave_f,2,bounds)
except:
    pass


from cmath import sqrt
import numpy as np
np.set_printoptions(suppress=True, linewidth=np.inf, precision=6)
def cholesky():
    S=[[0]*n for i in range(n)]
    for i in range(n):
        for j in range(i + 1):
            tổng = sum(S[k][i] * S[k][j] for k in range(j))
            if abs(tổng - A1[i][i]) == 0: break
            if i == j: S[j][i] = sqrt(A1[i][i] - tổng)
            else: S[j][i] = (A1[j][i] - tổng)/S[j][j]
        if S[i][i] == 0: break
    return S
def cholesky3(): #Hàm giải phương trình BY = E
    Y = np.zeros([n, n],dtype="complex_")
    for i in range(n-1, -1, -1):
        for j in range(i,n):
            if j == i: Y[i,j] = 1/S[i,j]
            if j>i:
                Y[i, j] = (B1[i,j]-sum(S[i][k]*Y[k,j] for k in range(i+1,n)))/S[i][i]
    return Y

#Nhập ma trận
A = np.loadtxt("test2.txt",dtype='complex', delimiter=' ') #Input

k=0
print('The input matrix is:')
print(A)
n = len(A)
B = np.zeros([n,n])
for i in range(n):
    B[i,i] = 1.0
m = 1
for i in range(n):
    for j in range(n):
        if A[i][j] == A[j][i]: continue
        else: m *= 0
if m == 1:
        A1=A
        B1=B
else:
        A1 = A.dot(np.transpose(A))
        B1 = B
        k = 1
S = np.array(cholesky())
if S[n-1][n-1] == 0:
    print('Ma trận A1 có định thức bằng 0 nên không thể khai triển Cholesky')
    quit()
Y = cholesky3()
Y = Y.dot(np.transpose(Y))
print()
print("=============================================================")
print('The inverse matrix is:')
if k == 0:
    print(Y)
if k == 1:
    Y = np.transpose(A).dot(Y)
    print(Y)
print('===============================================================')

Topic 2.5 - Matrix Inversion/Cholevsky/Cholevsky.py


#  ___________________________________________________________________________
#
# introducing the furniture factory problem
# https://www.youtube.com/watch?v=pSm1jONu2G8

# model in pygurobi
# https://www.youtube.com/watch?v=Copxhl2FShg
#  ___________________________________________________________________________

#
# Imports
#
import pyomo.environ as pyo

model = pyo.ConcreteModel()

# decision variables:
# number of chairs
# number of tables
model.c = pyo.Var(domain=pyo.NonNegativeIntegers)
model.t = pyo.Var(domain=pyo.NonNegativeIntegers)
#model.c = pyo.Var(domain=pyo.NonNegativeReals)
#model.t = pyo.Var(domain=pyo.NonNegativeReals)

# maximize the profit
model.OBJ = pyo.Objective(expr = 45*model.c + 80*model.t, sense=pyo.maximize)

# weekly material/wood constraint
model.wood = pyo.Constraint(expr = 5*model.c + 20*model.t <= 400)
# weekly labor constraint
model.labor = pyo.Constraint(expr = 10*model.c + 15*model.t <= 450)

pyo.SolverFactory('glpk').solve(model)

# model.pprint()
print("Profit = ", model.OBJ(), " per week")
print("c = ", model.c(), " chairs per week")
print("t = ", model.t(), " tables per week")


from typing import Optional
import numpy as np
import torch

def generate_origin_position(length:int, d_model:int) -> torch.Tensor:
    """
    Generate positional encoding as original paper
    """
    PE = torch.zeros((length, d_model))

    pos = torch.arange(length).unsqueeze(1)
    PE[:, 0::2] = torch.sin(
        pos / torch.pow(1000, torch.arange(0, d_model, 2, dtype=torch.float32)/d_model))
    PE[:, 1::2] = torch.cos(
        pos / torch.pow(1000, torch.arange(1, d_model, 2, dtype=torch.float32)/d_model))

    return PE

def generate_regular_position(length:int, d_model:int, period: Optional[int] = 24) -> torch.Tensor:
    PE = torch.zeros((length, d_model))

    pos = torch.arange(length, dtype=torch.float32).unsqueeze(1)
    PE = torch.sin(pos * 2 * np.pi / period)
    PE = PE.repeat((1, d_model))

    return PE