import numpy as np
import random
import matplotlib.pyplot as plt
import matplotlib
import pandas
import cv2
import tkinter
from copy import deepcopy
from scipy.io import loadmat
# matplotlib.use('TkAgg')

def isclose(a, b, rtol=1e-05, atol=1e-08):
    return np.abs(a-b) <= (atol + rtol * np.abs(b))

def sigma(z):
	return 1/(1 + np.exp(-z))

def mapFeature(X, p):
    m = len(X)
    X_poly = np.zeros((m, p))
    X_poly[:, 0] = X
    for i in range(1, p):
        X_poly[:, i] = X * X_poly[:, i-1]
    return X_poly

data=loadmat('ex5data1')
x = np.array(data['X'].flatten())
y = np.array(data['y'].flatten())

x_train = deepcopy(x)
y_train = deepcopy(y)

x_val = np.array(data['Xval'].flatten())
y_val = np.array(data['yval'].flatten())
x_test = np.array(data['Xtest'].flatten())
y_test = np.array(data['ytest'].flatten())

m = len(x)

x = mapFeature(x, 8)
x_val = mapFeature(x_val, 8)
x_test = mapFeature(x_test, 8)

x = [[(x[i][j]-x[:, j].mean())/x[:, j].std() for j in range(len(x[i]))] for i in range(len(x))]
x_val = [[(x_val[i][j]-x_val[:, j].mean())/x_val[:, j].std() for j in range(len(x_val[i]))] for i in range(len(x_val))]
x_test = [[(x_test[i][j]-x_test[:, j].mean())/x_test[:, j].std() for j in range(len(x_test[i]))] for i in range(len(x_test))]

x = np.array([np.insert(i, 0, 1) for i in x])
x_val = np.array([np.insert(i, 0, 1) for i in x_val])
x_test = np.array([np.insert(i, 0, 1) for i in x_test])

theta = np.ones(9)
lr = 0.001
alpha = 0.001
def cost_function(X, y, thetas, rl):
    m = len(y)
    a = X.dot(theta.T)
    return np.sum(np.square(a - y))/(2 * m) + rl / (2 * m) * np.sum(np.square(theta[1:]))

def gradient(x, y, theta, rl):
    a = x.dot(theta.T)
    dt = x.T.dot(a - y)/m + theta.dot(rl / m) #rl / m * np.sum(theta[1:])
    dt[0] = x.T.dot(a - y)[0]/m
    return dt

m = len(y)
costs = np.zeros(m)
val_costs = np.zeros(m)
for i in range(m):
    iterations = 5000
    theta = np.zeros(9)

    for j in range(iterations):
        new_theta = gradient(x[0:i+1, :], y[0:i+1], theta, 0)
        theta -= alpha * new_theta
    costs[i] = cost_function(x[0:i+1, :], y[0:i+1], theta, 0)
    val_costs[i] = cost_function(x_val, y_val, theta, 0)

plt.plot(np.arange(1, m+1), costs, label = "Train")
plt.plot(np.arange(1, m+1), val_costs, label = "Val")

plt.title('Learning curve for linear regression')
plt.xlabel('Number of training examples')
plt.ylabel('Error')
plt.show()
print(theta)
rls = [0, 0.001, 0.003, 0.01, 0.03, 0.1, 0.3, 1, 3, 10]
errors = []
val_errors = []
for rl in rls:
    iterations = 10000
    theta = np.zeros(9)
    for j in range(iterations):
        new_theta = gradient(x, y, theta, rl)
        theta -= alpha * new_theta
    errors.append(cost_function(x, y, theta, rl))
    val_errors.append(cost_function(x_val, y_val, theta, rl))

plt.plot(rls, errors, 'r', label = "Train")
plt.plot(rls, val_errors, 'b', label = "Train")
plt.legend(bbox_to_anchor=(1.05, 1), loc='upper left', borderaxespad=0.)
plt.title('Selecting lambda')
plt.xlabel('Lambda')
plt.ylabel('Error')
plt.show()
# rl = 100
# m = len(y)
# costs = np.zeros(m)
# val_costs = np.zeros(m)
# for i in range(m):
#     iterations = 5000
#     theta = np.zeros(9)

#     for j in range(iterations):
#         new_theta = gradient(x[0:i+1, :], y[0:i+1], theta, rl)
#         theta -= alpha * new_theta
#     costs[i] = cost_function(x[0:i+1, :], y[0:i+1], theta, rl)
#     val_costs[i] = cost_function(x_val, y_val, theta, rl)

# plt.plot(np.arange(1, m+1), costs, 'r', label = "Train")
# plt.plot(np.arange(1, m+1), val_costs, 'b', label = "Val")
# plt.legend(bbox_to_anchor=(1.05, 1), loc='upper left', borderaxespad=0.)
# plt.title('Learning curve for linear regression')
# plt.xlabel('Number of training examples')
# plt.ylabel('Error')
# plt.show()
# print(theta)
# plt.plot(x_train, y_train, 'rx')
# plt.plot(x_train, x.dot(theta.T), 'bo')
# plt.show()


import numpy as np

class NeuralNetwork(object):
    def __init__(self, input_nodes, hidden_nodes, output_nodes, learning_rate):
        # Set number of nodes in input, hidden and output layers.
        self.input_nodes  =  input_nodes
        self.hidden_nodes = hidden_nodes
        self.output_nodes = output_nodes

        # Initialize weights
        self.weights_input_to_hidden = np.random.normal(0.0, self.input_nodes**-0.5,
                                       (self.input_nodes, self.hidden_nodes))

        self.weights_hidden_to_output = np.random.normal(0.0, self.hidden_nodes**-0.5,
                                       (self.hidden_nodes, self.output_nodes))

        '''
        print ('nodes   - input, hidden, output ',                      \
               self.input_nodes, self.hidden_nodes, self.output_nodes   \
              )  # 3 2 1
        print ('weights - input_to_hidden_shape, hidden_to_output_shape: ',            \
               self.weights_input_to_hidden.shape, self.weights_hidden_to_output.shape \
              )  # (3, 2) (2, 1)
        '''

        self.lr = learning_rate

        #### TODO: Set self.activation_function to your implemented sigmoid function ####
        self.activation_function = lambda x : 1/(1 + np.exp(-x))

    def train(self, features, targets):
        #print('features.shape: ', features.shape)
        #print('targets.shape : ', targets.shape)

        ''' Train the network on a batch of features and targets.

            Arguments
            ---------

            features: 2D array, each row is one data record, each column is a feature
            targets: 1D array of target values

        '''
        n_records = features.shape[0]
        delta_weights_i_h = np.zeros(self.weights_input_to_hidden.shape)
        delta_weights_h_o = np.zeros(self.weights_hidden_to_output.shape)

        for X, y in zip(features, targets):

            final_outputs,      \
            hidden_outputs =    \
                                self.forward_pass_train(X)

            delta_weights_i_h,  \
            delta_weights_h_o = \
                                self.backpropagation(final_outputs, hidden_outputs,
                                                     X, y,
                                                     delta_weights_i_h,
                                                     delta_weights_h_o
                                                     )

        self.update_weights(delta_weights_i_h, delta_weights_h_o, n_records)

    def forward_pass_train(self, X):
        ''' Implement forward pass here

            Arguments
            ---------
            X: features batch

        '''
        ### Forward pass ###

        # signals into Hidden layer
        hidden_inputs  = np.dot(X, self.weights_input_to_hidden)

        # signals from Hidden layer
        # activation: sigmoid aka self.activation_function;
        hidden_outputs = self.activation_function(hidden_inputs)

        # signals into final Output layer
        final_inputs  = np.dot(hidden_outputs, self.weights_hidden_to_output)

        # signals from final Output layer
        # activation function: f(x) = x
        final_outputs = final_inputs

        return final_outputs, hidden_outputs

    def backpropagation(self, final_outputs, hidden_outputs,
                        X, y,
                        delta_weights_i_h, delta_weights_h_o):
        ''' Implement backpropagation

            Arguments
            ---------
            final_outputs: output from forward pass
            y: target (i.e. label) batch
            delta_weights_i_h: change in weights from input to hidden layers
            delta_weights_h_o: change in weights from hidden to output layers

        '''
        ### Backward pass ###

        ## TODO: Output error
        # Output layer error is the difference between desired target and actual output.

        error = y - final_outputs

        #print("error shape:", error.shape)                          # (1,)

        ### TODO: Backpropagated error terms ###

        ## TODO: Calculate the output layer's Backpropogated error term ##
        #  NOTICE: ACTIVATION FOR OUTPUT IS NOT SIGMOID, IT IS F(X)=X
        # output activation: f(x) = x
        # output activation_prime: 1
        # output_error_term: error * activation_prime === error * 1 === error

        output_error_term = error                                   # (1,)

        #print("output_error_term shape:", output_error_term.shape)

        ## TODO: Calculate the hidden layer's contribution to the error ##

        hidden_error = np.dot(output_error_term[:, None],           #(1,1)
                              self.weights_hidden_to_output.T       #(1,2)
                             )

        #print("hidden_error shape:", hidden_error.shape)            # (1, 2)

        ## TODO: Calculate the hidden layer's Backpropogated error term ##

        # gradient of the activation function, sigmoid is:
        # sigmoid_prime is: layer_output * (1-layer_output)
        # layer error   is: error * sigmoid_prime(layer_input)

        hidden_error_term = hidden_error * (hidden_outputs) * (1-hidden_outputs)

        #print("hidden_error_term shape:", hidden_error_term.shape)  # (1, 2)

        ### delta_weights_layer: learningRate * layer_error_weighted * layer_input_values ###

        ## Weight step (hidden to output) ##
        #delta_weights_h_o += self.lr * hidden_outputs[:, None] \
        #                             * output_error_term             #[None, :]   # ho(2,)er(1,)

        delta_weights_h_o += hidden_outputs[:, None] * output_error_term  #[None, :]   # ho(2,)er(1,)

        '''
        # These 3 versions SAME
        delta_weights_h_o_1 = self.lr * hidden_outputs[:, None] \
                                   * output_error_term            # ho(2,)er(1,)
        delta_weights_h_o_2 = self.lr * hidden_outputs[:, None] \
                                   * output_error_term[:, None]   # ho(2,)er(1,)
        delta_weights_h_o_3 = self.lr * hidden_outputs[:, None] \
                                   * output_error_term[None, :]   # ho(2,)er(1,)
        #This 1 version BAD
        delta_weights_h_o_4 = self.lr * hidden_outputs[None, :] \
                                   * output_error_term#[None, :]   # ho(??)er(??)->(1, 2)
        '''
        #print("_delta_weights_h_o shape:", delta_weights_h_o.shape)  # (2, 1)

        ## Weight step (input to hidden) ##
        #delta_weights_i_h += self.lr * X[:, None]  \
        #                             * hidden_error_term

        delta_weights_i_h += X[:, None] * hidden_error_term

        #print("delta_weights_i_h shape:", delta_weights_i_h.shape)

        return delta_weights_i_h, delta_weights_h_o

    def update_weights(self, delta_weights_i_h, delta_weights_h_o, n_records):

        ''' Update weights on gradient descent step

            Arguments
            ---------
            delta_weights_i_h: change in weights from input to hidden layers
            delta_weights_h_o: change in weights from hidden to output layers
            n_records: number of records

        '''

        #print("Batch Size: ", n_records)   # 128

        # update hidden-to-output weights with gradient descent step
        self.weights_hidden_to_output += delta_weights_h_o * self.lr / n_records

        # update input-to-hidden weights with gradient descent step
        self.weights_input_to_hidden += delta_weights_i_h * self.lr / n_records

    def run(self, features):
        ''' Run a forward pass through the network with input features

            Arguments
            ---------
            features: 1D array of feature values
        '''

        #### Implement the forward pass here ####

        # HIDDEN LAYER
        # signals into Hidden Layer
        hidden_inputs  = np.dot(features, self.weights_input_to_hidden)

        # signals from Hidden Layer
        hidden_outputs = self.activation_function(hidden_inputs)

        ## OUTPUT LAYER
        # signals into final Output Layer
        final_inputs  = np.dot(hidden_outputs, self.weights_hidden_to_output)

        # signals from final Output Layer
        final_outputs = final_inputs#self.activation_function(final_inputs)

        return final_outputs

#########################################################
# Set your hyperparameters here
##########################################################
'''
train_features shape: (15435, 56)
  val_features shape: ( 1440, 56)
 test_features shape: (  504, 56)
'''

iterations = 2400     # batch size: 128, (15435/128) = ~120.586 iterations / pass of the entire dataset (on average)
learning_rate = 1.9   # 0.015 * 128 = 1.92 as a Starting point
hidden_nodes = 6      # features: 56, hidden nodes: (n_features-n_outputs)/2, try: 28
output_nodes = 1      # I want an output of a single number: total number of rentals at each (hour)
print('params: ',iterations, learning_rate, hidden_nodes, output_nodes)

# prior to dividing the weights update step by n_records, my learning rate of .015 was good.
# Since n_records is 128, I'll start with a new learning rate set to 0.015 * 128 = 1.92

'''
# TEST RESULTS, TUNING THE PARAMETERS:
# 100, 0.1,   2, 1  :Progress: 99.0% ... Training loss: 0.952 ... Validation loss: 1.365
# 100, 0.01,  2, 1 *:Progress: 99.0% ... Training loss: 0.382 ... Validation loss: 0.468
# 100, 0.001, 2, 1  :Progress: 99.0% ... Training loss: 0.732 ... Validation loss: 1.278

# 200, 0.01,  2, 1  :Progress: 99.5% ... Training loss: 0.331 ... Validation loss: 0.479
# 200, 0.01,  3, 1  :Progress: 99.5% ... Training loss: 0.277 ... Validation loss: 0.453
# 100, 0.01,  3, 1  :Progress: 99.0% ... Training loss: 0.315 ... Validation loss: 0.500
# 150, 0.01,  2, 1  :Progress: 99.3% ... Training loss: 0.278 ... Validation loss: 0.466
# 200, 0.05,  2, 1  :Progress: 99.5% ... Training loss: 0.272 ... Validation loss: 0.434

# 100, 0.05, 27, 1  :Progress: 99.0% ... Training loss: 0.951 ... Validation loss: 1.367
# 500, 0.1,  27, 7  :Progress: 99.8% ... Training loss: 1.142 ... Validation loss: 1.367
# 240, 0.1,  26, 1  :NAN
# 100, 0.01, 26, 1  :Progress: 99.0% ... Training loss: 0.998 ... Validation loss: 1.367
# 100, 0.1,  26, 1  :Nan
# 100, 0.8,  26, 1  :Nan
# 100, 0.1,   6, 1  :Progress: 99.0% ... Training loss: 6.969 ... Validation loss: 7.359
                     Terrible Step-Function-like output,
# 100, 0.05, 20, 1  :Progress: 99.0% ... Training loss: 0.974 ... Validation loss: 1.367
                     Terrible L-like output. Flat Line
# 100, 0.1,  20, 1  :Progress: 99.0% ... Training loss: 8.073 ... Validation loss: 7.580
                     Terrible Backwards-L, and wide heartrate, way below the line
# 100, 0.01,  6, 1  :Progress: 99.0% ... Training loss: 0.326 ... Validation loss: 0.555
                     a graph offset
# 500, 0.01,  6, 1  :Progress: 99.8% ... Training loss: 0.178 ... Validation loss: 0.327
                     a graph offset
# 500, 0.05,  6, 1  :Progress: 99.8% ... Training loss: 0.973 ... Validation loss: 1.367
                     Terrible L-like output. Flat Line
# 800, 0.02,  6, 1  :Progress: 99.9% ... Training loss: 0.109 ... Validation loss: 0.192
 2400, 0.02,  6, 1  :Progress: 100.0% ... Training loss: 0.068 ... Validation loss: 0.138
 4800, 0.02?, 6?,1  :Progress: 100.0% ... Training loss: 0.064 ... Validation loss: 0.145
                     no real diff from prev.. numbers hovered without changing thru numerous epochs
 4800, 0.01, 12, 1 *:Progress: 100.0% ... Training loss: 0.057 ... Validation loss: 0.134
                     slight improvements. hand't settled yet. still slowly improving
 5600, 0.01, 24, 1  :Progress: 100.0% ... Training loss: 0.975 ... Validation loss: 1.367
                     Terrible L-like output. Flat Line. Too small LR or too many nodes
 3600, 0.04, 24, 1  :Progress: 100.0% ... Training loss: 0.901 ... Validation loss: 1.353
 7200, 0.01, 24, 1  :Progress: 100.0% ... Training loss: 0.886 ... Validation loss: 1.315
                     Terrrible, L, but interesting prediction graph
 7200, 0.01, 12, 1  :Progress: 100.0% ... Training loss: 0.066 ... Validation loss: 0.161
                     ?? not as good as 4800, .01, 12, 1 ???
 9600, 0.02, 12, 1 *:Progress: 100.0% ... Training loss: 0.511 ... Validation loss: 0.618

                     Weird. It tracks well - probably OVERFITTING
                     --EXCEPT that its offset -by like 200 maybe?
                     Subratact about 200 from the "Lows/Minimums" for prediction to match actual.
                     Highs are closer (except +/- xmas), subtract maybe 50? so P Highs == actual max.
                     22Dec - 31Dec likely won't ever match.
 9960, 0.18, 14, 1   NaN.  Seems 12 Nodes is the Max.
                     My attempt to Return 1 from sigmoid, if Nan, didn't work.
                     I could try to figure it out. But
                     1) more nodes is probably not the right solution (overcomplicated)
                     2) Time is better spent wrapping this up, and continuing onto the Next Lesson.
                     3) it's already been hinted at other ways of keeping weights in a useful range.
                     4) Leaky_abstraction
                     5) Getting into overfitting region anyway.

 9960, 0.018, 10, 1 *:Progress: 100.0% ... Training loss: 0.069 ... Validation loss: 0.132

 4800, 0.018, 10, 1 *:Progress:   0.1% ... Training loss: 1.970 ... Validation loss: 1.707
                      OOPS-WRONG STATT

  500, 0.01,   6, 1 *:Progress:  99.8% ... Training loss: 0.223 ... Validation loss: 0.332
 7200, 0.015,  8, 1 *:Progress: 100.0% ... Training loss: 0.060 ... Validation loss: 0.148
 9600, 0.015,  8, 1  :Progress: 100.0% ... Training loss: 0.058 ... Validation loss: 0.184
                      This model began increasing Validation loss about 40-45% through the training
                      Meanwhile, Training Loss held relatively steady.  Shall run again, but stop at around
                      9600*(.40) to 96*(.45) = 3840 to 4320. Let's try 4200..
 4200, 0.015,  8, 1  :Progress: 100.0% ... Training loss: 0.079 ... Validation loss: 0.204
 4200, 0.015,  6, 1**:Progress: 100.0% ... Training loss: 0.061 ... Validation loss: 0.139
 4200, 0.010,  6, 1 *:Progress: 100.0% ... Training loss: 0.062 ... Validation loss: 0.154
 4200, 0.015, 12, 1  :Progress: 100.0% ... Training loss: 0.207 ... Validation loss: 0.379
 4200, 0.015,  6, 1**:Progress: 100.0% ... Training loss: 0.059 ... Validation loss: 0.146

-----
 4200, 0.015,  6, 1  :Progress: 100.0% ... Training loss: 0.059 ... Validation loss: 0.141
 4200, 0.3,   10, 1  : NaN
 4200, 0.1,    8, 1  :Progress: 100.0% ... Training loss: 0.936 ... Validation loss: 1.355
 4200, 0.05,   8, 1  :Progress:  92.8% ... Training loss: 0.974 ... Validation loss: 1.367
                      Useless - mostly didn't change losses between iterations
 4200, 0.015,  6, 1  :Progress: 100.0% ... Training loss: 0.067 ... Validation loss: 0.163
                      Looks good, but V-Loss wasn't decreasing, so perhaps quit sooner
 3600, 0.015,  6, 1  :Progress: 100.0% ... Training loss: 0.062 ... Validation loss: 0.148
 2400, 0.02,   6, 1  :Progress: 100.0% ... Training loss: 0.098 ... Validation loss: 0.215
                      NOPE - doesn't pass. too few epochs, or lr not ideal
 3600, 0.02,   6, 1  :Progress: 100.0% ... Training loss: 0.115 ... Validation loss: 0.222
                      NOPE - VL jumped up right at the end
 2400, 0.015,  6, 1 *:Progress: 100.0% ... Training loss: 0.074 ... Validation loss: 0.150
 1000, 0.015,  6, 1  :Progress: 99.9% ... Training loss: 0.075 ... Validation loss: 0.160
 1200, 0.015,  6, 1**:Progress: 99.9% ... Training loss: 0.069 ... Validation loss: 0.161
                      SUBMITTED THIS PROJECT -- ACCEPTED !!  -- YEA!! :-D

----- AFTER CORRECTING THE WEIGHTS UPDATE PROCESS TO DIVIDE BY THE n_records IN A batch -----
                        THEN UPDATING THE LEARNING PROPORTIONALLY..
 1200, 1.92,   6, 1  :Progress: 99.9% ... Training loss: 0.107 ... Validation loss: 0.191
                      Good First Estimate!
 1280, 2.0 ,   6, 1  :Progress: 99.9% ... Training loss: 0.066 ... Validation loss: 0.152
                      EXCELLENT !
 1280, 1.0 ,   6, 1  :Progress: 99.9% ... Training loss: 0.145 ... Validation loss: 0.334
                      Terrible - As I expected.  Thought I'd try the "recommended" starting point though
 1280, 1.5 ,   6, 1  :Progress: 99.9% ... Training loss: 0.113 ... Validation loss: 0.232
                      Nope going the "recommended" direction doesn't work either. Stick with the Excellent Parmas
                      I already found ;-)
 1207, 2.0 ,   6, 1  :Progress: 99.9% ... Training loss: 0.070 ... Validation loss: 0.174
 1218, 2.0 ,   6, 1  :Progress: 99.9% ... Training loss: 0.080 ... Validation loss: 0.199
 1326, 2.0 ,   6, 1  :Progress: 99.9% ... Training loss: 0.068 ... Validation loss: 0.141
 1328, 2.0 ,   6, 1**:Progress: 99.9% ... Training loss: 0.068 ... Validation loss: 0.135
                      SUBMITTING THIS PROJECT (RE-SUBMITTING WITH THE WEIGHTS_UPDATE FUNCTION FIXED)
                      OH - CANNOT RESUBMIT, as it was "Accepted" as Correct the first Time !  :-/

------ ANOTHER UPDATE TO WHERE WEIGHTS UPDATE USES LR AND N_RECORDS - AND WHERE WEIGHTS UPDATE TAKES PLACE -------
------    THIS VERSION OF THE PROJECT IS 100% CORRECT !! IT PASSES PA AUTO_REVIEWER - SEE BELOW !! ------
1328, 2.0 ,   6, 1   :Progress: 99.9% ... Training loss: 0.074 ... Validation loss: 0.153
1328, 2.0 ,   6, 1   :Progress: 99.9% ... Training loss: 0.077 ... Validation loss: 0.158
1328, 2.1 ,   6, 1   :Progress: 99.9% ... Training loss: 0.180 ... Validation loss: 0.241
                      YEP, don't increas lr
2400, 2.0 ,   6, 1   :Progress: 100.0% ... Training loss: 0.077 ... Validation loss: 0.166
2400, 1.9 ,   6, 1   :Progress: 100.0% ... Training loss: 0.067 ... Validation loss: 0.163

'''

# Required: The training loss is below 0.09 and the validation loss is below 0.18.

''' RESULTS FROM PROJECT ASSISTENT AUTO REVIEWER
(dlnd) sherylhohman (master *) P1-BikeShare $  udacity submit
Submission includes the following files:
    my_answers.py
    Your_first_neural_network.ipynb

Uploading submission...
[=========================== 100% ===========================] 719720/719720

Waiting for results...Done!

Results:
--------

************************************************************************
                          Test Result Summary
************************************************************************

Produces good results when running the network on full data            .
The activation function is a sigmoid                                   .
The backpropagation implementation is correct                          .
The forward pass implementation is correct                             .
The learning_rate is reasonable                                        .
The number of epochs is reasonable                                     .
The number of hidden nodes is reasonable                               .
The number of output nodes is correct                                  .
The run method is correct                                              .
The update_weights implementation is correct                           .
The weights are updated correctly on training                          .

--------------------------------------------------------------------------------

Congratulations!  It looks like your network passed all of our tests.  You're ready to submit your project at
https://review.udacity.com/#!/rubrics/700/submit-zip

Details are available in first_neural_network-result-213296.json.

If you would like this version of the project to be reviewed,
submit first_neural_network-213296.zip to the reviews website.

'''


import numpy as np

class LinearRegression:
    """
    Compute linear regression on single or multiple features
    """

    def __init__(self, table,reg=False,lamda=0):
        """Initializes Class for Linear Regression

        Parameters
        ----------
        table : ndarray(n-rows,m-features + 1)
            Numerical training data, last column as training values
        reg : Boolean
            Set True to enable regularization, false by default

        """
        #regularization parameters
        self.reg = reg
        self.lamda = lamda

        self.num_training = np.shape(table)[0]
        # remove the last column from training data to extract features data
        self.X = np.delete(table, -1, 1)
        # add a column of ones in front of the training data
        self.X = np.insert(self.X, 0, np.ones(self.num_training), axis=1)
        self.num_features = np.shape(self.X)[1]
        # extract the values of the training set from the provided data
        self.y = table[:, self.num_features - 1]
        # create parameters and initialize to 1
        self.theta = np.ones(self.num_features)

    def compute_cost(self):
        """Computes cost based on the current values of the parameters using square cost function

        Returns
        -------
        cost : float
            Cost of the selection of current parameters
        """
        hypothesis = np.dot(self.X, self.theta)
        loss = hypothesis - self.y
        #regularization term
        reg = self.lamda*np.sum(np.power(self.theta,2))
        # if regularization is true, add regularization term and return cost
        if self.reg:
            return (np.sum(loss ** 2)+reg) / (2 * self.num_training)
        else:
            return np.sum(loss ** 2) / (2 * self.num_training)

    def gradient_descent(self, num_iters=1000, alpha=0.01):
        """Runs the gradient decent algorithm

        Parameters
        ----------
        num_iters : int
            The maximum number of iterations allowed to run before the algorithm terminates
        alpha : float
            The learning rate for the algorithm

        Returns
        -------
        self.theta : ndarray(self.num_features)
            Theta (parameters) values after gradient_descent

        """
        old_cost=0
        for i in range(0, num_iters):
            hypothesis = np.dot(self.X, self.theta)
            loss = hypothesis - self.y
            cost = self.compute_cost()
            #if the cost is equal for two iterations, break from the loop
            if cost==old_cost:
                break
            old_cost = cost
            #print"Iteration: %d Cost: %f" % (i, cost)
            gradient = np.dot(self.X.T, loss) / self.num_training
            #regularization term
            reg = (1 - (self.lamda*alpha)/self.num_training)
            if self.reg:
                self.theta = self.theta*reg - alpha * gradient
            else:
                self.theta = self.theta - alpha * gradient
        return self.theta

    def predict(self, data):
        """Computes the value based on current weights

        Parameters
        ----------
        data : ndarray
            Test data to score using the current weights
        Returns
        -------
        val : float
            Prediction result
        """
        # add a column of ones to the prediction data to make it compatible with parameter array
        data = np.column_stack((np.ones(data.shape[0]), data))
        return np.dot(data, self.theta)

    def accuracy(self):
        """Calculates percentage of correct predictions by the model on training data

        Returns
        -------
        accuracy : float
            Percentage of correct predictions on the features of training data
        """
        #delete extra ones column that was added
        x = np.delete(self.X, 0, 1)
        predicted = self.predict(x)
        match = float(np.sum(self.y == predicted))
        return (match / self.num_training) * 100

    def plot_data(self, feature=1):
        """Plots the training data with feature number in parameter as x axis (deafault = 1)

        Parameters
        ----------
        feature : int
                  number of the parameter to be represented in x axis
        """
        from matplotlib import pyplot as plt
        x = self.X[:, feature]
        y = self.y
        plt.scatter(x, y)
        plt.show()

    def plot_fit(self, feature=1):
        """Plots the training data with regression line. The feature number as x axis is specified in parameter

        Parameters
        ----------
        feature : int
                  number of the parameter to be represented in x axis
        """
        from matplotlib import pyplot as plt
        xi = np.arange(self.X[:, feature].min() - 1, self.X[:, feature].max() + 1)
        line = self.theta[0] + (self.theta[1] * xi)
        plt.plot(xi, line, 'r-', self.X[:, feature], self.y, 'o')
        plt.show()


import numpy as np
import matplotlib.pyplot as plt

#%%

def prob_density(x_k, sigma):
    return 1/(sigma * np.sqrt(2.0 * np.pi)) *np.exp(-1/2*(x_k**2/sigma**2))

def likelihood(x, sigmas, A):
    if(isinstance(sigmas,float)):
        likes = 1.0
        if (sigmas <=1) or (sigmas >=10):
            likes = 0
        else:
            for cosa in x:
                likes *= A * prob_density(cosa,sigmas)
    else:
        likes = np.ones(len(sigmas))
        for i in range(len(sigmas)):
            if (sigmas[i] <=1) or (sigmas[i] >=10):
                likes[i] = 0
            else:
                for cosa in x:
                    likes[i] *= A * prob_density(cosa,sigmas[i])
    return likes

def dl_ds(x, sigma, A):
    h = 0.1
    derivada = (likelihood(x, sigma+h, A) - likelihood(x, sigma-h, A))/(2.0*h)
    return derivada
def d2l_ds2(x, sigma, A):
    h = 0.1
    derivada = (dl_ds(x, sigma+h, A) - dl_ds(x, sigma-h, A))/(2.0*h)
    return derivada
def NR(x, guess, A):
    cero = guess
    while np.abs(dl_ds(x, cero, A)) > 1E-20:
        cero -= dl_ds(x, cero, A)/d2l_ds2(x, cero, A)
    return cero

data = np.loadtxt('valores.txt')

sigmas = np.linspace(1,10,1000)
likeli = likelihood(data, sigmas, 1.0)
#%%
plt.plot(sigmas,likeli)
plt.xlabel(r"$\sigma$")
plt.ylabel(r"$L\ * {\rm{prob} (\sigma)}$")
plt.savefig("like.png")
plt.close()
#%%

dLds = dl_ds(data, sigmas, 1.0)

#%%
plt.plot(sigmas, dLds)
plt.xlabel(r"$\sigma$")
plt.ylabel(r"$dL/d\sigma$")
plt.savefig("like_prime.png")
plt.close()

#%%

sigma_cero = NR(data, 4.0, 1.0)
print('sigma_cero : {}'.format(sigma_cero))

#%%

d2L = d2l_ds2(data,sigmas,1.0)

#%%

plt.plot(sigmas, d2L)
plt.xlabel(r"$\sigma$")
plt.ylabel(r"$d^2L/d\sigma^2$")
plt.savefig("like_prime_prime.png")
plt.close()

print("delta_sigma_cero= {}".format(-d2l_ds2(data,sigma_cero,1.0)))


import numpy as np
from utils import pointLocationInArray

def get_weight(p, v, a): #p, v are row vectors [x y]
        # - p, v : 1D numpy arrays. Represent locations in an image.
        # - a : Float, the alpha parameter from the paper.
	array_subtract = np.array(np.subtract(p,v))
	square_second_norm = (array_subtract[0]**2 + array_subtract[1]**2)**a
	return 1/square_second_norm #returns individual weight, Wi

def get_weighted_centroids(x, w):
        # Preconditions:
        # - x : 2D numpy array, where each row is a point. These
        #   points are either the p (original locations of handles) or
        #   q (the final location of handles).
        # - w : 1D numpy array, where each element is a weight. These
        #   weights are produced by get_weight, so they are specific
        #   to a point in the image.
        # - x and w should have the same amount of rows. In other
        #   words, x.shape[0] == w.size
        # Postconditions:
        # - returns a single centroid as a 1D numpy array.
	sigmaWiPi = 0
	sigmaWi = 0
	n = len(x)
	for i in range(n):
		sigmaWiPi += (x[i]*w[i])
		sigmaWi += w[i]
	return sigmaWiPi/sigmaWi #returns p/q star, a numpy array of length 2

def get_pq_hat(xi, xstar): #x is either pi/qi or pstar/qstar
	return np.array(np.subtract(xi,xstar)) #returns p/q hat

def least_square(phat_arr, qhat_arr, w_arr):
        # Preconditions:
        # - phat_arr : a 2D numpy array where each row is the
        #   difference between the original location of a handle and
        #   the weighted centroid of the handles.
        # - qhat_arr : A 2D numpy array similar to phat_arr, but using
        #   the final locations of handles instead of the original
        #   location.
        # - w_arr : A 1D numpy array of weights.
        # - number of points in p, points in q, and weights in w_arr
        #   should all be the same.
        #   phat_arr.shape[0] == qhat_arr.shape[0] == w_arr.shape[0]
        # Postconditions:
        # - Returns M, the 2x2 transformation matrix for the point v.
        #   v is not an argument to this function, but the weights in
        #   w_arr were calculated relative to this point.
	i = len(phat_arr)
	j = len(phat_arr)
	sigma1 = np.zeros((phat_arr.shape[1], qhat_arr.shape[1]))
	sigma2 = np.zeros((phat_arr.shape[1], qhat_arr.shape[1]))
	for temp in range(i):
                phat_i = phat_arr[temp]
                # Proper matrix computations require np.dot. Since we
                # are multiplying two row vectors to do it, np.outer
                # suffices. np.outer(u, v) = u^T * v, where * is
                # matrix multiplication.
                sigma1 += np.outer(phat_i, w_arr[temp] * phat_i)
	sigma1_inv = np.linalg.inv(sigma1)
	for temp in range(j):
		sigma2 += w_arr[temp] * np.outer(phat_arr[temp], qhat_arr[temp])
	return sigma1_inv.dot(sigma2)

def deformation(x, pstar, M, qstar):
	return (x-pstar).dot(M) + qstar

def transform(v, p, q, a=2):
        # Preconditions:
        # - p : a 2D numpy array of the original locations of the
        #   handles. Each row is a location.
        # - q : a 2D numpy array of the final locations of the
        #   handles. Each row is a location.
        # - v : a 1D numpy array representing the current point to
        #   deform.
        # - a, optional : a float parameter which tunes the values of
        #   the weights.
        # Postconditions:
        # - The return value is the moving least squares deformation
        #   of the point v with p as the original position of handles
        #   and q as the new position of handles.
        location = pointLocationInArray(v, p)
        if location != -1:
            return q[location]
        w_arr = [get_weight(handle, v, a)
                 for handle in p]
        p_centroid = get_weighted_centroids(p, w_arr)
        q_centroid = get_weighted_centroids(q, w_arr)
        phat_arr = np.array([get_pq_hat(handle, p_centroid)
                             for handle in p])
        qhat_arr = np.array([get_pq_hat(handle, q_centroid)
                             for handle in q])
        M = least_square(phat_arr, qhat_arr, w_arr)
        return deformation(v, p_centroid, M, q_centroid)


import numpy as np
from scipy.spatial.distance import cdist

class KMeans:
    def __init__(
            self,
            k: int,
            metric: str = "euclidean",
            tol: float = 1e-6,
            max_iter: int = 100):
        """
        inputs:
            k: int
                the number of centroids to use in cluster fitting
            metric: str
                the name of the distance metric to use
            tol: float
                the minimum error tolerance from previous error during optimization to quit the model fit
            max_iter: int
                the maximum number of iterations before quitting model fit
        """
        self.k = k
        self.metric = metric
        self.tol = tol
        self.max_iter = max_iter

    def fit(self, mat: np.ndarray):
        """
        This method fits the kmeans algorithm onto a provided 2D matrix.

        inputs:
            mat: np.ndarray
                A 2D matrix where the rows are observations and columns are features
        """
        self.mat = mat
        self.centroid = []

        # list to store mean squared error for each iteration
        mse_list = []

        # initialize centroids (random indices)
        random_indices = np.random.choice(self.mat.shape[0], self.k, replace=False)

        # creating a 2-D matrix of the randomly selected centroids
        for i in random_indices:
            self.centroid.append(self.mat[i])

        # initializing count object for while loop (while number of iterations has not reached max_iters)
        count = 0

        while count < self.max_iter:

            # creating empty clusters
            # list of sample indices for each cluster
            self.clusters = [[] for _ in range(self.k)]

            # creating a matrix of distances (observations to centroids)
            distance_matrix = cdist(self.mat, self.centroid, self.metric)

            #assigning minimum distances to respective clusters
            for i in range(distance_matrix.shape[0]):

                # calculating the index of the minimum value
                min_index = np.argmin(distance_matrix[i])

                # adding original observation to respective cluster
                self.clusters[min_index].append(self.mat[i])

            # WAY TO BREAK WHILE LOOP:
            # calculating the MSE!
            mean_squared_error = self._private_get_error()
            mse_list.append(mean_squared_error)

            if len(mse_list) > 1:
                difference = mean_squared_error - mse_list[-2]
                if abs(difference) < self.tol:
                    count += self.max_iter
                else:
                    count += 1
            elif len(mse_list) == 1:
                count += 1

            # figure out the new centroids
            centroids_old = self.centroid
            self.centroid = self._private_get_centroids(self.clusters)

    def predict(self, mat: np.ndarray) -> np.ndarray:
        """
        This method predicts the cluster labels for a provided 2D matrix.

        inputs:
            mat: np.ndarray
                A 2D matrix where the rows are observations and columns are features
        outputs:
            np.ndarray
                a 1D array with the cluster label for each of the observations in `mat`
        """

        self.mat = mat
        cluster_labels = []

        # calculating distance between matrix observations and centroids (from the fit model)
        distance_matrix = cdist(self.mat, self.centroid, self.metric)

        for i in range(self.mat.shape[0]):
            min_index = np.argmin(distance_matrix[i])

            cluster_labels.append(min_index)

        return cluster_labels

    def _private_get_error(self) -> float:
        """
        returns the final mean-squared error of the fit model

        outputs:
            float
                the mean-squared error of the fit model
        """
        overall_dist = 0
        mse = 0

        for i in range(self.k):
            point_to_cent = cdist([self.centroid[i]], self.clusters[i], self.metric)

            for j in point_to_cent[0]:
                overall_dist += j**2

            mse += overall_dist/len(self.clusters[i])

        return mse

    def _private_get_centroids(self, clusters) -> np.ndarray:
        """
        returns the centroid locations of the fit model

        outputs:
            np.ndarray
                a `k x m` 2D matrix representing the cluster centroids of the fit model
        """
        centroids = np.zeros((self.k, self.mat.shape[1]))

        for idx, cluster in enumerate(self.clusters):
            cluster_mean = np.mean(cluster, axis=0)
            centroids[idx] = cluster_mean

        return centroids


import numpy as np
from sympy import Matrix

class HillCipher:

    def __init__(self, message, matrix_list):
        self.alphabet = {chr(i): i - 97 for i in range(97, 123)}
        self.message = message
        self.matrix = matrix_list
        self.message_numbers = np.array([self.alphabet[x] for x in message]).reshape((-1, 2))

    def cipher_decipher(self, to_cipher):
        ciphered_message = ""
        if to_cipher:
            for vector in self.message_numbers:
                for i in (vector @ self.matrix) % 26:
                    ciphered_message += chr(i+97)
            return ciphered_message
        else:
            deciphered_message = ""
            try:
                inverse = np.array(Matrix(self.matrix).inv_mod(26))
                for vector in self.message_numbers:
                    for i in (vector @ inverse) % 26:
                        deciphered_message += chr(i + 97)
            except Exception:
                raise Exception("Not invertible matrix")
            return deciphered_message

def main():
    message = input("Insert Message:\n")
    # message = "JULY"
    matrix = []
    print("Consider the positions of the key matrix as follows: \n")
    print("(1  2")
    print(" 3  4)")
    print()
    for i in range(4):
        matrix.append(int(input("Insert Value in the position " + str(i) + " of the matrix key: \n")))
    matrix = np.array(matrix).reshape((2, 2))

    print()
    cipher = HillCipher(message.lower().replace(" ", ""), matrix)
    ciphered_message = cipher.cipher_decipher(1)
    print("Ciphered Message:", ciphered_message)

    decipher = HillCipher(ciphered_message.lower().replace(" ", ""), matrix)
    deciphered_message = decipher.cipher_decipher(0)
    print("Deciphered message:", deciphered_message)

if __name__ == '__main__':
    main()


from operator import add
from math import sqrt

from numba import jit

golden_ratio = (sqrt(5) - 1) * 0.5

@jit(cache=True)
def sumSquaresErrors(xlist, ylist, alpha):
    if len(xlist) != len(ylist):
        return None
    return reduce(add, map(lambda x, y: (x ** alpha - y) ** 2, xlist, ylist))

@jit(cache=True)
def goldenSearchMin(func, x1, x2, tol=1e-5):
    if abs(x1 - x2) < tol:
        return x1
    xl = x1 + (x2 - x1) * (1 - golden_ratio)
    xr = x1 + (x2 - x1) * golden_ratio
    fl = func(xl)
    fr = func(xr)
    return goldenSearchMin(func, x1, xr, tol=tol) if fl <= fr else goldenSearchMin(func, xl, x2, tol=tol)


import numpy as np
import pandas as pd
from matplotlib import pyplot as plt
from matplotlib import animation as animation
from mpl_toolkits.mplot3d import Axes3D
import sys

training_data = []
labels = []

mean = 0 # Mean and standard deviation of the data
stddev = 0

def readData(filenameX,filenameY): # Reads data from the given filenames
	global training_data
	global labels,mean,stddev
	training_data = pd.read_csv(filenameX,header=None)
	training_data[1] = training_data[0]
	training_data[0] = 1
	labels = pd.read_csv(filenameY,header=None)
	mean = training_data[1].mean()
	stddev = training_data[1].std()
	training_data[1] = (training_data[1]-mean)/float(stddev)

readData(sys.argv[1],sys.argv[2])

examples_count = training_data.shape[0] # Number of examples
dimensions = training_data.shape[1] # Dimensions of input space
training_data = np.array(training_data.values,dtype='f')
labels = np.array(labels.values,dtype='f')
theta = np.zeros([dimensions,1])

def error(theta): # Computes error in the linear fit parametrized by theta
	diff = np.subtract(labels,np.matmul(training_data,theta))
	x = np.matmul(np.transpose(diff),diff)/(2*examples_count)
	return(x)

def gradient(theta): # Computes the gradient of the error function at the given theta
	diff = np.subtract(labels,np.matmul(training_data,theta))
	return(np.matmul(np.transpose(training_data),diff)/(examples_count))

def distance(x1,x2): # Metric distance between two points in parameter space
	return np.linalg.norm(np.subtract(x1,x2))

def checkConvergence(theta1,theta2): # Convergence condition for gradient descent
	error1 = error(theta1)
	error2 = error(theta2)
	if abs(error1-error2)/abs(error2) < 0.001:
		return 1
	return 0

visited_theta0 = [] # For storing the theta visited by the algorithm
visited_theta1 = []
visited_errors = []

def linearRegression(learning_rate): # Performs gradient descent
	temp_theta = np.zeros([dimensions,1])
	previous_theta = np.zeros([dimensions,1])
	iteration = 0
	while 1:
		temp_theta = np.copy(np.add(temp_theta,learning_rate*gradient(temp_theta)))
		iteration = iteration+1
		to = np.ndarray.flatten(np.copy(temp_theta))
		visited_theta0.append(list(to)[0])
		visited_theta1.append(list(to)[1])
		visited_errors.append(error(temp_theta))
		if checkConvergence(temp_theta,previous_theta) == 1:
			break
		previous_theta = np.copy(temp_theta)
	global theta
	theta = np.copy(temp_theta)

##Q1
#Part A
linearRegression(float(sys.argv[3])) # Perform linear regression
print("Theta obtained : ",theta.flatten())

#Part2
x_values = []
for e in training_data:
	x_values.append(stddev*stddev*e[1]+mean)
plt.scatter(x_values,labels,label='Data')
line_x = np.linspace(min(x_values),max(x_values),100)
predicted = theta[0]+theta[1]*((line_x-mean)/stddev)
plt.plot(line_x,predicted,'r',label='Linear Fit') # Plot the linear fit
plt.legend(loc='upper right')
plt.xlabel('Acidity')
plt.ylabel('Density')
plt.title('Linear Regression')
plt.show()

#Part3
x = np.arange(-3,3,0.1)
y = np.arange(-3,3,0.1)
a,b = np.meshgrid(x,y)
z = [] # For storing error values for different thetas

for i in x:
	z.append([])
	for j in y:
		new = np.transpose(np.asarray([i,j]))
		new = np.reshape(new,(2,1))
		z[-1].append(error(new)) # Add error values
z = np.array(z)
z = np.resize(z,(x.shape[0],y.shape[0]))

fig1 = plt.figure()
ax1 = Axes3D(fig1)
surf = ax1.plot_surface(b, a, z, cmap='coolwarm') # Plot the error function
ax1.set_xlabel('theta0')
ax1.set_ylabel('theta1')
ax1.set_zlabel('Error')
ax1.view_init(60,40)

visited_theta0 = np.array(visited_theta0)
visited_theta1 = np.array(visited_theta1)
visited_errors = np.array(visited_errors)
visited_errors = np.resize(visited_errors,(visited_errors.shape[0],1))

line, = ax1.plot([], [], [], 'r-', label = 'Gradient descent', lw = 1.5) # Animated line object
point, = ax1.plot([], [], [], 'bo') # Animated point object
display_value = ax1.text(3., 3., 27.5, '', transform=ax1.transAxes)

def init(): # Initialise the graph
	line.set_data([], [])
	line.set_3d_properties([])
	point.set_data([], [])
	point.set_3d_properties([])
	return line, point,

def animate(i): # Animate Function
	line.set_data(list(visited_theta0[:i]),list(visited_theta1[:i])) # Add previous theta to show a path
	line.set_3d_properties(list(visited_errors[:i]))
	point.set_data(visited_theta0[i],visited_theta1[i]) # Add the current theta
	point.set_3d_properties(visited_errors[i])
	return line, point,

ax1.legend(loc = 1)
anim = animation.FuncAnimation(fig1, animate, init_func=init,
                               frames=len(visited_errors), interval=int(1000*float(sys.argv[4])),
                               repeat_delay=60, blit=True) # Blit the plot and call the animate every timestep
plt.show()

#Part4
fig1 = plt.figure()
ax = Axes3D(fig1)

ax.contour(b,a,z,cmap='winter',levels=np.logspace(-3,3,25)) # Plot the contours for the error function
ax.set_xlabel('theta0')
ax.set_ylabel('theta1')
ax.set_zlabel('error')
ax.view_init(60,40)

line, = ax.plot([], [], [], 'r-', label = 'Gradient descent', lw = 1.5) # Similar to the previous part
point, = ax.plot([], [], [], 'bo')
display_value = ax.text(2., 2., 27.5, '', transform=ax.transAxes)

def init_contour():
	line.set_data([], [])
	line.set_3d_properties([])
	point.set_data([], [])
	point.set_3d_properties([])
	return line, point,

def animate_contour(i):
	line.set_data(list(visited_theta0[:i]),list(visited_theta1[:i])) # Add the visited theta till this point to be plotted
	line.set_3d_properties(list(visited_errors[:i]))
	point.set_data(visited_theta0[i],visited_theta1[i]) # Current theta
	point.set_3d_properties(visited_errors[i])
	return line, point,

anim = animation.FuncAnimation(fig1, animate_contour, init_func=init_contour,
                               frames=len(visited_errors), interval=int(1000*float(sys.argv[4])),
                               repeat_delay=60, blit=True)
plt.show()


from plot import *
from cost import *
from scipy.optimize import minimize
import matplotlib.pyplot as plt
from sklearn.preprocessing import PolynomialFeatures

data = loaddata('./data1.txt', ',')

X = np.c_[np.ones((data.shape[0],1)), data[:,0:2]] # constant, f1, f2
y = np.c_[data[:,2]]

plotData(data, 'Exam 1 score', 'Exam 2 score', 'Pass', 'Fail')
plt.show()
initial_theta = np.zeros(X.shape[1])
cost = costFunction(initial_theta, X, y)
grad = gradient(initial_theta, X, y)
print('Cost: \n', cost)
print('Grad: \n', grad)

res = minimize(costFunction, initial_theta, args=(X,y), jac=gradient, options={'maxiter':400})
print(res)

sigmoid(np.array([1, 45, 85]).dot(res.x.T))
plt.scatter(45, 85, s=60, c='r', marker='v', label='(45, 85)')
plotData(data, 'Exam 1 score', 'Exam 2 score', 'Admitted', 'Not admitted')
x1_min, x1_max = X[:,1].min(), X[:,1].max(),
x2_min, x2_max = X[:,2].min(), X[:,2].max(),

# generate meshgrid
xx1, xx2 = np.meshgrid(np.linspace(x1_min, x1_max), np.linspace(x2_min, x2_max))

# res.x contains three parameter final optimized value
h = sigmoid(np.c_[np.ones((xx1.ravel().shape[0],1)), xx1.ravel(), xx2.ravel()].dot(res.x))
h = h.reshape(xx1.shape)

plt.contour(xx1, xx2, h, [0.5], linewidths=1, colors='b');
plt.show()

############################################################

data2 = loaddata('data2.txt', ',')
y = np.c_[data2[:,2]]
X = data2[:,0:2]
plotData(data2, 'Microchip Test 1', 'Microchip Test 2', 'y = 1', 'y = 0')

poly = PolynomialFeatures(6)
XX = poly.fit_transform(data2[:,0:2])
print(XX.shape)

initial_theta = np.zeros(XX.shape[1])
#costFunctionReg(initial_theta, 1, XX, y)

fig, axes = plt.subplots(1,3, sharey = True, figsize=(17,5))

for i, C in enumerate([0.0, 1.0, 100.0]):
    res2 = minimize(costFunctionReg, initial_theta, args=(C, XX, y), jac=gradientReg, options={'maxiter': 3000})

    accuracy = 100.0 * sum(predict(res2.x, XX) == y.ravel()) / y.size

    plotData(data2, 'Microchip Test 1', 'Microchip Test 2', 'y = 1', 'y = 0', axes.flatten()[i])

    x1_min, x1_max = X[:, 0].min(), X[:, 0].max(),
    x2_min, x2_max = X[:, 1].min(), X[:, 1].max(),
    xx1, xx2 = np.meshgrid(np.linspace(x1_min, x1_max), np.linspace(x2_min, x2_max))
    h = sigmoid(poly.fit_transform(np.c_[xx1.ravel(), xx2.ravel()]).dot(res2.x))
    h = h.reshape(xx1.shape)
    axes.flatten()[i].contour(xx1, xx2, h, [0.5], linewidths=1, colors='g');
    axes.flatten()[i].set_title('Train accuracy {}% with Lambda = {}'.format(np.round(accuracy, decimals=2), C))


# A marshy region used for agricultural drainage has become contaminated with a particular element.
# It has been determined that flushing the area with clean water will reduce the element for a while, but it will then begin to build up again.
# A biologist has found that the percent of the element in the soil x months after the flushing begins is given by the function below.
#
# When will the element be reduced to a minimum? What is the minimum percent?

from sympy import solve, lambdify, symbols, diff, simplify, pretty, pprint
import matplotlib.pyplot as plt
import numpy as np

def disp_fun( f ):
	pprint( '\n{0}\n\n'.format( pretty( f ) ) )

g_xlim = [ 1, 12 ]
g_ylim = [ -5, 50 ]

x = symbols('x', positive = True ) # 0 <= x
F = ( x**2 + 100 ) / ( 2*x )

lam_x = lambdify( x, F, np )

x_vals = np.linspace( g_xlim[0], g_xlim[1], 1000, endpoint=True )
y_vals = lam_x( x_vals )

plt.plot( x_vals, y_vals, label = '' )

# F'
dF = diff( F, x )

disp_fun( simplify( dF ) )

lam_dx = lambdify( x, dF, np )

dx_vals = np.linspace( g_xlim[0], g_xlim[1], 1000, endpoint=True )
dy_vals = lam_dx( x_vals )

min = solve( dF, x )[ 0 ]

plt.plot( dx_vals, dy_vals, label = '' )

F.subs( { x: min } ).evalf()

plt.show()


import csv
import numpy as np

def read_csv(path, skip_first = False):
    with open(path, 'rb') as csvfile:
        reader = csv.reader(csvfile, delimiter=',', quotechar='|')
        if skip_first:
            reader.next()
        arr = []
        for row in reader:
            arr.append(np.asarray(row, dtype=np.float))
        return np.asarray(arr, dtype=np.float)

def euclidean(lm1, lm2):
	return np.sqrt(np.sum(np.square(lm1-lm2), axis=1))

def mean_euclidean(lm1, lm2):
	return np.mean(euclidean(lm1, lm2))

def distribution(d, percentiles):
	ds = np.sort(d)
	indices = np.int(percentiles * d.shape[0])
	return ds[indices]

# Helper function for smd

def min_distance(p, set):
    mind = float('inf')
    mini = -1
    for i in xrange(set.shape[0]):
        setp = set[i, :]
        d = np.sqrt(np.sum(np.square(p-setp)))
        if d < mind:
            mind = d
            mini = i
    return (mind, mini)

# Sum of minimal distance between reference and target set of landmarks

def smd(ref, target):
    refshape = ref.shape
    acc = 0
    for i in xrange(refshape[0]):
        (d, ind) = min_distance(ref[i, :], target)
        acc = acc + d
    return acc

if __name__ == "__main__":
  ts1 = [[1, 2, 3], [4, 5, 6]]
  ts2 = [[2, 2, 3], [0, 0, 0]]

  print(euclidean(np.asarray(ts1), np.asarray(ts2)))
  print(mean_euclidean(np.asarray(ts1), np.asarray(ts2)))
  print(smd(np.asarray(ts1), np.asarray(ts2)))


"""Fit an exponential curve to input data."""
import argparse
import matplotlib.pyplot as plt
import numpy as np

import numpy
import scipy.optimize

def exp_decay(val, a, b):
    return 1-a*numpy.exp(-val**b)

def exp_exp_decay(val, a, b, c):
    return 1-a*numpy.exp(-val**b)**c

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='Fit Exponential Decay')
    parser.add_argument(
        'exponential_csv_file',
        help='Path to exponential csv')
    args = parser.parse_args()
    x_list = []
    y_list = []
    with open(args.exponential_csv_file, 'r') as exponential_file:
        for line in exponential_file:
            x, y = line.rstrip().split(',')
            x_list.append(float(x))
            y_list.append(float(y))

    fig, ax = plt.subplots()
    ax.plot(x_list, y_list, 'x', label=f'points from {args.exponential_csv_file}')

    x_sample = numpy.linspace(min(x_list), max(x_list), 50)
    p0, variant = scipy.optimize.curve_fit(
        exp_decay,
        x_list,
        y_list,
        p0=[1, -0.1],
        method='lm')
    y_exp = exp_decay(x_sample, *p0)
    exp_decay_string = f'1-{p0[0]:.5f}*numpy.exp(-x**{p0[1]:.5f})'
    ax.plot(x_sample, y_exp, '-', label=exp_decay_string)

    p0, variant = scipy.optimize.curve_fit(
        exp_exp_decay,
        x_list,
        y_list,
        p0=[1, -0.1, 1],
        method='lm')
    y_exp_exp = exp_exp_decay(x_sample, *p0)
    exp_exp_decay_string = f'1-{p0[0]:.5f}*numpy.exp(-x**{p0[1]:.5f})**{p0[2]:.5f}'
    ax.plot(x_sample, y_exp_exp, '-', label=exp_exp_decay_string)

    ax.set(xlabel='x', ylabel='y')
    legend = ax.legend(loc='upper right')
    ax.grid()
    print('**** function fit ->')
    print(exp_decay_string)
    print(exp_exp_decay_string)
    plt.show()


import numpy as np
import tensorflow as tf
from IPython.core.debugger import Tracer
debug_here = Tracer()

#  ##  numpy hilbert transform ##
def np_hilbert(xr):
    n = xr.shape[0]
    # fft over columns.
    x = np.fft.fft(xr.transpose()).transpose()
    h = np.zeros([n])
    if n > 0 and 2*np.fix(n/2) == n:
        # even and nonempty
        h[0:int(n/2+1)] = 1
        h[1:int(n/2)] = 2
    elif n > 0:
        # odd and nonempty
        h[0] = 1
        h[1:int((n+1)/2)] = 2
    if len(x.shape) == 2:
        hs = np.stack([h]*x.shape[-1], -1)
    elif len(x.shape) == 1:
        hs = h
    else:
        raise NotImplementedError
    print(hs)
    return np.fft.ifft((x*hs).transpose()).transpose()

# Xr = np.array([[1.0, 2.0], [3.0, 4.0], [5.0, 1.0], [2.0, 3.0], [4.0, 5.0]])
# Xr = np.array([1.0, 2.0, 3.0, 4.0])
Xr = np.array([0.3, 0.0])
X = np_hilbert(Xr)
print(X)

#  # tensorflow hilbert transform ##

def hilbert(xr):
    with tf.variable_scope('hilbert_transform'):
        n = tf.Tensor.get_shape(xr).as_list()[0]
        # Run the fft on the columns no the rows.
        x = tf.transpose(tf.fft(tf.transpose(xr)))
        h = np.zeros([n])
        if n > 0 and 2*np.fix(n/2) == n:
            # even and nonempty
            h[0:int(n/2+1)] = 1
            h[1:int(n/2)] = 2
        elif n > 0:
            # odd and nonempty
            h[0] = 1
            h[1:int((n+1)/2)] = 2
        tf_h = tf.constant(h, name='h', dtype=tf.float32)
        if len(x.shape) == 2:
            hs = np.stack([h]*x.shape[-1], -1)
            reps = tf.Tensor.get_shape(x).as_list()[-1]
            hs = tf.stack([tf_h]*reps, -1)
        elif len(x.shape) == 1:
            hs = tf_h
        else:
            raise NotImplementedError
        tf_hc = tf.complex(hs, tf.zeros_like(hs))
        tmp = x*tf_hc
        return tf.transpose(tf.ifft(tf.transpose(tmp)))

test_graph = tf.Graph()
with test_graph.as_default():
    xr = tf.constant(Xr, dtype=tf.float32)
    xc = tf.complex(xr, tf.zeros_like(xr))
    X = hilbert(xc)

with tf.Session(graph=test_graph):
    print(X.eval())


import os
import numpy as np
from matplotlib.pyplot import *
from mpl_toolkits.mplot3d import axes3d, Axes3D
from matplotlib import cm
import itertools
import scipy.optimize as op
import collections

FOLDER = os.path.dirname(os.path.realpath(__file__))

def load_data():
    datafile = FOLDER + '/ex2data1.txt'
    data = np.loadtxt(datafile, delimiter=',')
    y = data[:, 2:3]
    m = y.size
    X = np.hstack((np.ones((m, 1)), data[:, 0:2]))
    theta = np.zeros((3, 1))
    print('loaded %s training samples' % m)
    return X, y, theta

def plot_data(X, y):
    Xneg = np.array([xi for xi, yi in zip(X, y) if yi[0] == 0])
    Xpos = np.array([xi for xi, yi in zip(X, y) if yi[0] == 1])
    figure('fig1', figsize=(10, 6))
    plot(Xneg[:, 1], Xneg[:, 2], 'rx', markersize=10)
    plot(Xpos[:, 1], Xpos[:, 2], 'bo', markersize=10)
    grid(True)
    ylabel('Grade 2')
    xlabel('Grade 1')
    legend('Admitted', 'Not admitted')
    title('Grades vs. Approval', fontsize=24)

def plot_decision_boundary(X, theta):
    t0, t1, t2 = theta

    def y0(x):
        return -(t0 + t1 * x) / t2

    minx, maxx = min(X[:, 1]), max(X[:, 1])
    xs = [minx, maxx]
    ys = [y0(x) for x in xs]
    figure('fig1')
    plot(xs, ys, 'b--')
    pass

def sigmoid(z):
    return 1 / (1 + np.exp(-z))

def cost_function(X, y):
    m, n = X.shape
    ytrans = y.transpose()

    def _cost(theta):
        theta = theta.reshape((n, 1))
        predictions = sigmoid(X.dot(theta))
        J = -(ytrans.dot(np.log(predictions)) + (1 - ytrans).dot(np.log(1 - predictions))) / m
        return J
    return _cost

def grad_func(X, y):
    m, n = X.shape

    def _grad(theta):
        theta = theta.reshape((n, 1))
        predictions = sigmoid(X.dot(theta))
        grad = (X.transpose().dot(predictions - y)) / m
        return grad

    return _grad

def get_accuracy(X, y, theta):
    m, n = X.shape
    theta = theta.reshape(n, 1)
    predictions = X.dot(theta) >= 0
    reality = y == 1
    rights = predictions == reality
    counter = collections.Counter(rights.reshape(m))
    count_rights = counter[True]
    accuracy = float(count_rights) / m
    return accuracy

if __name__ == '__main__':
    X, y, theta = load_data()
    plot_data(X, y)
    init_theta = np.zeros((3, 1))
    _cost = cost_function(X, y)
    _grad = grad_func(X, y)
    J0 = _cost(init_theta)
    theta_opt = op.minimize(fun=cost_function(X, y),
                            x0=init_theta,
                            method='TNC',
                            jac=grad_func(X, y)).x
    Jmin = _cost(theta_opt)
    accuracy = get_accuracy(X, y, theta_opt)
    plot_decision_boundary(X, theta_opt)
    show()
    print('cost at (0,0,0) is %s' % J0[0][0])
    print('optimum theta is %s' % theta_opt)
    print('cost at theta_min = %s' % Jmin[0][0])
    print('Accuracy is %s' % accuracy)


import numpy as np

H_20 = np.array([
    [+1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1],
    [+1, +1, -1, +1, +1, -1, -1, -1, -1, +1, -1, +1, -1, +1, +1, +1, +1, -1, -1, +1],
    [+1, +1, +1, -1, +1, +1, -1, -1, -1, -1, +1, -1, +1, -1, +1, +1, +1, +1, -1, -1],
    [+1, -1, +1, +1, -1, +1, +1, -1, -1, -1, -1, +1, -1, +1, -1, +1, +1, +1, +1, -1],
    [+1, -1, -1, +1, +1, -1, +1, +1, -1, -1, -1, -1, +1, -1, +1, -1, +1, +1, +1, +1],
    [+1, +1, -1, -1, +1, +1, -1, +1, +1, -1, -1, -1, -1, +1, -1, +1, -1, +1, +1, +1],
    [+1, +1, +1, -1, -1, +1, +1, -1, +1, +1, -1, -1, -1, -1, +1, -1, +1, -1, +1, +1],
    [+1, +1, +1, +1, -1, -1, +1, +1, -1, +1, +1, -1, -1, -1, -1, +1, -1, +1, -1, +1],
    [+1, +1, +1, +1, +1, -1, -1, +1, +1, -1, +1, +1, -1, -1, -1, -1, +1, -1, +1, -1],
    [+1, -1, +1, +1, +1, +1, -1, -1, +1, +1, -1, +1, +1, -1, -1, -1, -1, +1, -1, +1],
    [+1, +1, -1, +1, +1, +1, +1, -1, -1, +1, +1, -1, +1, +1, -1, -1, -1, -1, +1, -1],
    [+1, -1, +1, -1, +1, +1, +1, +1, -1, -1, +1, +1, -1, +1, +1, -1, -1, -1, -1, +1],
    [+1, +1, -1, +1, -1, +1, +1, +1, +1, -1, -1, +1, +1, -1, +1, +1, -1, -1, -1, -1],
    [+1, -1, +1, -1, +1, -1, +1, +1, +1, +1, -1, -1, +1, +1, -1, +1, +1, -1, -1, -1],
    [+1, -1, -1, +1, -1, +1, -1, +1, +1, +1, +1, -1, -1, +1, +1, -1, +1, +1, -1, -1],
    [+1, -1, -1, -1, +1, -1, +1, -1, +1, +1, +1, +1, -1, -1, +1, +1, -1, +1, +1, -1],
    [+1, -1, -1, -1, -1, +1, -1, +1, -1, +1, +1, +1, +1, -1, -1, +1, +1, -1, +1, +1],
    [+1, +1, -1, -1, -1, -1, +1, -1, +1, -1, +1, +1, +1, +1, -1, -1, +1, +1, -1, +1],
    [+1, +1, +1, -1, -1, -1, -1, +1, -1, +1, -1, +1, +1, +1, +1, -1, -1, +1, +1, -1],
    [+1, -1, +1, +1, -1, -1, -1, -1, +1, -1, +1, -1, +1, +1, +1, +1, -1, -1, +1, +1]])

H_12=np.array([[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
               [1,-1,-1, 1,-1,-1,-1, 1, 1, 1,-1, 1],
               [1,-1, 1,-1,-1,-1, 1, 1, 1,-1, 1,-1],
               [1, 1,-1,-1,-1, 1, 1, 1,-1, 1,-1,-1],
               [1,-1,-1,-1, 1, 1, 1,-1, 1,-1,-1, 1],
               [1,-1,-1, 1, 1, 1,-1, 1,-1,-1, 1,-1],
               [1,-1, 1, 1, 1,-1, 1,-1,-1, 1,-1,-1],
               [1, 1, 1, 1,-1, 1,-1,-1, 1,-1,-1,-1],
               [1, 1, 1,-1, 1,-1,-1, 1,-1,-1,-1, 1],
               [1, 1,-1, 1,-1,-1, 1,-1,-1,-1, 1, 1],
               [1,-1, 1,-1,-1, 1,-1,-1,-1, 1, 1, 1],
               [1, 1,-1,-1, 1,-1,-1,-1, 1, 1, 1,-1]])

H_4=np.array([[1, 1, 1, 1],
              [1,-1, 1,-1],
              [1, 1,-1,-1],
              [1,-1,-1, 1]
              ])


import numpy as np
from _decimal import Decimal
from numpy import linalg
'''
Jacobi's iterative method.

Passes in Matrix A, b, uses zero vector for x as starting point.

Requires an error to be passed in, in our case 10^-8.

Requires an actual solution to be be passed in to calculate norm.

Norm is calculated using built in linalg function.

Jacobi's method in terms of matrix multiplication:
D^-1 * (L-U) * x + D^-1 * b

This function modifies the above to:
D^-1 * (b-Rx),
where R is the remainder matrix, containing lower and upper triangular matrices
'''

def jacobi(A, b, x, error, actual_soln, max_iterations=9999):
    if error and actual_soln.any():
        D = np.diag(A)
        # R is the remainder matrix, containing lower and upper triangular matrices
        R = A - np.diagflat(D)
        n = 0
        real_norm = linalg.norm(actual_soln)
        while True:
            x = (b - np.dot(R, x)) / D
            norm = linalg.norm(x)
            n += 1
            if abs(Decimal(real_norm) - Decimal(norm)) <= error or n >= max_iterations:
                break
    else:
        raise SyntaxError("Must pass in an error term and actual solution.")

    # Returns an array of the solution vector and number of iterations
    return [x, n]


"""
nitrogen.angmom
---------------

Angular momentum and spherical tensor routines.

"""

import numpy as np
import nitrogen.constants as constants
import py3nj # For Wigner-nj symbols

def wigner3j(jj1, jj2, jj3, mm1, mm2, mm3):
    """
    Calculate the Wigner 3-j symbol,

    .. math::

       \\left(\\begin{array}{ccc} j_1 & j_2 & j_3 \\\\ m_1 & m_2 & m_3 \\end{array} \\right)

    Parameters
    ----------
    jj1 : integer
        Twice the value of :math:`j_1`.
    jj2 : integer
        Twice the value of :math:`j_2`.
    jj3 : integer
        Twice the value of :math:`j_3`.
    mm1 : integer
        Twice the value of :math:`m_1`.
    mm2 : integer
        Twice the value of :math:`m_2`.
    mm3 : integer
        Twice the value of :math:`m_3`.

    Returns
    -------
    float
        The result.

    Notes
    -----
    This currently wraps the `py3nj <https://github.com/fujiisoup/py3nj>`_
    implementation. The back-end may change in the future.

    Examples
    --------
    >>> n2.angmom.wigner3j(2 * 20, 2 * 21, 2 * 22, 2 * 5, 2 * -15, 2 * 10)
    0.032597617477982975

    """

    result = py3nj.wigner3j(jj1, jj2, jj3, mm1, mm2, mm3)

    return result

def wigner6j(jj1, jj2, jj3, jj4, jj5, jj6):
    """
    Calculate the Wigner 6-j symbol,

    .. math::

       \\left\\{\\begin{array}{ccc} j_1 & j_2 & j_3 \\\\ j_4 & j_5 & j_6 \\end{array} \\right\\}

    Parameters
    ----------
    jj1 : integer
        Twice the value of :math:`j_1`.
    jj2 : integer
        Twice the value of :math:`j_2`.
    jj3 : integer
        Twice the value of :math:`j_3`.
    jj4 : integer
        Twice the value of :math:`j_4`.
    jj5 : integer
        Twice the value of :math:`j_5`.
    jj6 : integer
        Twice the value of :math:`j_6`.

    Returns
    -------
    float
        The result.

    Notes
    -----
    This currently wraps the `py3nj <https://github.com/fujiisoup/py3nj>`_
    implementation. The back-end may change in the future.

    Examples
    --------
    >>> n2.angmom.wigner6j(2 * 3, 2 * 6, 2 * 5, 2 * 4, 2 * 6, 2 * 9)
    -0.020558557070186504

    """

    result = py3nj.wigner6j(jj1, jj2, jj3, jj4, jj5, jj6)

    return result

def clebsch_gordan(jj1, jj2, jj3, mm1, mm2, mm3):
    """
    Calculate the Clebsch-Gordan coefficient,

    .. math::

       \\langle j_1\\,m_1, j_2 \\, m_2 \\vert j_3 \\, m_3 \\rangle

    Parameters
    ----------
    jj1 : integer
        Twice the value of :math:`j_1`.
    jj2 : integer
        Twice the value of :math:`j_2`.
    jj3 : integer
        Twice the value of :math:`j_3`.
    mm1 : integer
        Twice the value of :math:`m_1`.
    mm2 : integer
        Twice the value of :math:`m_2`.
    mm3 : integer
        Twice the value of :math:`m_3`.

    Returns
    -------
    float
        The result.

    Notes
    -----
    This currently wraps the `py3nj <https://github.com/fujiisoup/py3nj>`_
    implementation. The back-end may change in the future.

    Examples
    --------
    >>> n2.angmom.clebsch_gordan(2 * 6, 2 * 9, 2 * 13, 2 * -3, 2 * 4, 2 * 1)
    0.4277601867185667

    """

    result = py3nj.clebsch_gordan(jj1, jj2, jj3, mm1, mm2, mm3)

    return result

def dircos_tensor(N1,k1,m1,N2,k2,m2):
    """
    Calculate a matrix element of the
    direction cosine spherical tensor,

    ..  math::

        \\langle N_1, k_1, m_1 \\vert \\lambda_{Qq} \\vert N_2, k_2, m_2 \\rangle

    Parameters
    ----------
    N1, k1, m2, N2, k2, m2 : integer
        Angular momentum quantum numbers

    Returns
    -------
    (3,3) ndarray
        The direction cosine tensor matrix element in terms
        of spherical tensor components.

    Notes
    -----

    The basis functions are standard symmetric top rotational basis functions
    with the usual phase conventions. :math:`k` is the body-frame :math:`z`
    component with respect to "anomalous" body-frame operators.

    """

    red = (-1) ** (k1 + k2 + N1 + N2 - 1) * np.sqrt( (2*N2 + 1) / (2*N1 + 1))

    lamQq = np.zeros((3,3))

    for Q in [-1,0,1]:

        cg1 = clebsch_gordan(2*N2, 2*1, 2*N1,
                             2*m2, 2*Q, 2*m1)

        for q in [-1, 0, 1]:

            cg2 = clebsch_gordan(2*N2, 2*1, 2*N1,
                                -2*k2, 2*q,-2*k1)

            lamQq[Q,q] = red * cg1 * cg2

    return lamQq

def Jbf_cs(J):
    """
    Calculate Condon-Shortley body-fixed J operators

    Parameters
    ----------
    J : int
        Angular momentum: 0, 1, 2, ...

    Returns
    -------
    Jx,Jy,Jz : ndarray
        Body-fixed angular momentum components

    Notes
    -----
    The basis function order is :math:`k = -J, \ldots, +J`.

    """

    NJ = 2*J+1

    kI, kJ = np.meshgrid(np.arange(NJ)-J, np.arange(NJ)-J, indexing = 'ij')

    Jz = np.zeros((NJ,NJ))

    Jz[kI==kJ] = kJ[kI==kJ]

    # Calculate Jp = Jx + i*Jy
    # (this is the *lowering* operating in body-fixed frame
    #  because of the anamalous commutation sign of Jx, Jy, Jz)
    Jp = np.zeros((NJ,NJ))
    Jm = np.zeros((NJ,NJ))

    idx = (kI == kJ - 1)
    Jp[idx] = np.sqrt( J*(J+1) - kJ*(kJ-1) )[idx] # lowering operator

    idx = (kI == kJ + 1)
    Jm[idx] = np.sqrt( J*(J+1) - kJ*(kJ+1) )[idx] # raising operator

    Jx = (Jp + Jm) / 2.0
    Jy = (Jp - Jm) / (2.0 * 1j)

    return Jx, Jy, Jz

def U_wr2cs(J):
    """
    Wang transformation matrix, with additional
    phase factors for real functions.

    cs = U @ wr

    Parameters
    ----------
    J : int
        Angular momentum quantum number, 0, 1, 2, ...

    Returns
    -------
    W : ndarray
        The unitary transformation matrix

    """

    NJ = 2*J+1

    U = np.zeros((NJ,NJ), dtype = np.complex128)

    def NJKplus(J,K):
        if J % 2 == 1 and K % 2 == 1:
            return -1j * (1j)**(J+1)
        elif J % 2 == 0 and K % 2 == 0:
            return +1j * (1j)**(J+1)
        else:
            return 1 * (1j)**(J+1)
    def NJKminus(J,K):
        if J % 2 == 1 and K % 2 == 0:
            return +1j * (1j)**(J+1)
        elif J % 2 == 0 and K % 2 == 1:
            return -1j * (1j)**(J+1)
        else:
            return 1 * (1j)**(J+1)

    for i in range(J):

        U[i,i] = -1.0 / np.sqrt(2.0) * NJKminus(J,J-i)
        U[-(i+1), i] = 1.0 / np.sqrt(2.0) * NJKminus(J,J-i)

        U[i, -(i+1)] = 1.0 / np.sqrt(2.0) * NJKplus(J,J-i)
        U[-(i+1), -(i+1)] = 1.0 / np.sqrt(2.0) * NJKplus(J,J-i)

    U[J,J] = 1.0

    return U

def iJbf_wr(J):
    """
    Calculate body-fixed J operators in real, symmetrized JK basis
    ("Wang-Real")

    Parameters
    ----------
    J : int
        Angular momentum quantum number, 0, 1, 2, ...

    Returns
    -------
    iJx, iJy, iJz : ndarray
        The body-frame angular momentum operators multiplied
        by i. These are purely real, anti-symmetric matrices

    """

    Jxyz = Jbf_cs(J) # Condon-Shortley representation

    U = U_wr2cs(J)
    Uh = U.conj().T

    iJx = np.real(1j * (Uh @ Jxyz[0] @ U))
    iJy = np.real(1j * (Uh @ Jxyz[1] @ U))
    iJz = np.real(1j * (Uh @ Jxyz[2] @ U))

    # Enforce some strict selection rules
    NJ = 2*J+1
    i,j = np.meshgrid(range(NJ),range(NJ),indexing = 'ij')

    iJx[abs(i-j) != 1] = 0  # Only the diagonal +/- 1 is non-zero
    iJy[abs(i-(NJ-j-1)) != 1] = 0 # Only the anti-diagonal +/- 1 is non-zero

    iJz[i != (NJ-j-1)] = 0 # Only the anti-diagonal is non-zero
    iJz[J,J] = 0

    # Let's enforce strict anti-symmetry
    for O in (iJx, iJy, iJz):
        for i in range(NJ):
            for j in range(i+1):
                if i == j:
                    O[i,j] = 0
                else:
                    O[j,i] = -O[i,j]

    return iJx, iJy, iJz

def iJiJbf_wr(J):
    """
    Calculate the anti-commutators [iJ_a,iJ_b]_+ for
    body-fixed angular momentum components in the Wang-real
    representation.

    Parameters
    ----------
    J : int
        Total angular momentum, 0, 1, 2, ...

    Returns
    -------
    iJiJ : nested tuple of ndarrays
        iJiJ[a][b] is the [iJa, iJb]_+ anti-commutator ndarray

    """

    iJ = iJbf_wr(J)

    def ac(A,B):
        return A@B + B@A

    iJiJ = tuple(tuple(ac(iJ[a],iJ[b]) for b in range(3)) for a in range(3))

    return iJiJ

def X2I(X, mass):
    """
    Calculate the inertia tensor from Cartesian
    coordinates.

    Parameters
    ----------
    X : ndarray
        A (3*N,...) array containing the
        x, y, and z Cartesian positions of N particles.
    mass : array_like
        The masses of the N particles.

    Returns
    -------
    I : ndarray
        A (3,3,...) array containing the
        symmetric inertia tensor

    """

    if X.shape[0] % 3 != 0 :
        raise ValueError("The first dimension of X must be a multiple of 3")
    base_shape = X.shape[1:]
    N = X.shape[0] // 3 # floor division (shouldn't matter)

    X3 = X.copy()
    X3 = np.reshape(X3, (N,3) + base_shape)

    #########################################################
    # Calculate the Center-of-Mass
    COM = np.zeros((3,) + base_shape)
    for i in range(N):
        COM += mass[i] * X3[i,:]
    COM = COM / sum(mass)

    for i in range(N):
        X3[i,:] -= COM  # calculate X in COM frame
    #
    #########################################################

    ##########################################################
    # Calculate inertia tensor
    I = np.zeros((3,3) + base_shape)
    for i in range(N):
        I[0,0] += mass[i] * (X3[i,1]**2 + X3[i,2]**2) # x,x
        I[1,1] += mass[i] * (X3[i,2]**2 + X3[i,0]**2) # y,y
        I[2,2] += mass[i] * (X3[i,0]**2 + X3[i,1]**2) # z,z
        I[0,1] += -mass[i] * X3[i,0] * X3[i,1]  # x,y
        I[0,2] += -mass[i] * X3[i,0] * X3[i,2]  # x,z
        I[1,2] += -mass[i] * X3[i,1] * X3[i,2]  # y,z
    # Copy symmetric elements
    np.copyto(I[1,0:1], I[0,1:2])
    np.copyto(I[2,0:1], I[0,2:3])
    np.copyto(I[2,1:2], I[1,2:3])
    #
    ##########################################################

    return I

def X2ABC(X, mass):
    """
    Calculate rotational constants from
    Cartesian positions.

    Parameters
    ----------
    X : ndarray
        A (3*N,...) array containing the
        x, y, and z Cartesian positions of N particles.
    mass : array_like
        The masses of the N particles.

    Returns
    -------
    ABC : ndarray
        A (3,...) array containing the
        A, B, and C rotational constants
        (in energy units).

    """

    # Calculate moment of inertia tensor
    I = X2I(X, mass) # shape (3, 3, ...)
    # Move tensor indices to the last indices
    I = np.moveaxis(I, (0,1), (-2, -1)) # shape (..., 3, 3)

    # Now diagonalize
    w,_ = np.linalg.eigh(I) # w has shape (..., 3)
    w = np.moveaxis(w, -1, 0)  # move abc index to front

    # Calculate rotational constants
    # B = hbar**2 / (2 * I)
    ABC = constants.hbar**2 / (2.0 * w) # hc * cm^-1

    return ABC

def X2PAS(X, mass):
    """
    Rotate coordinates to the principal
    axis system.

    Parameters
    ----------
    X : ndarray
        A (3*N,...) array containing the
        x, y, and z Cartesian positions of N particles.
    mass : array_like
        The masses of the N particles.

    Returns
    -------
    XPAS : ndarray
        A (3*N,...) array of the positions in the
        PAS frame.
    R : ndarray
        A (3,3,...) orthogonal array containing the transformation
        matrix from the original axes to the principal
        axis system.
    COM : ndarray
        A (3,...) array of the center-of-mass position
        in the original frame.

    Notes
    -----
    The PAS coordinates are defined as

    ..  math::

        \\vec{x}_\\text{PAS} = \\mathbf{R}(\\vec{x} - \\vec{x}_\\text{COM})

    """

    # Calculate the inertia tensor
    I = X2I(X, mass)

    # Calculate the eigenvectors
    I = np.moveaxis(I, (0,1), (-2,-1))
    _,U = np.linalg.eigh(I)

    # Check that they form a right-handed axis
    # system
    #
    # Compute (a x b) . c
    #
    # If this is +1, then the system is right-handed
    # If this is -1, then the system is left-handed
    # and one vector needs to change sign (or all three)
    trip = np.inner( np.cross(U[:,0], U[:,1]), U[:,2] )
    if U.ndim == 2:
        if trip < 0:
            U *= -1
    else:
        U[:,:, trip < 0] *= -1

    # The array that transforms vectors in
    # the original frame to the PAS is
    # the transpose of U
    #
    RPAS = U.T
    RPAS = np.moveaxis(RPAS, (-2,-1), (0,1))

    # Finally, calculate the
    # PAS coordinates
    XPAS = np.zeros_like(X)

    N = X.shape[0] // 3  # Number of atoms

    # First, calculate the center of mass
    COM = 0
    for k in range(N):
        COM += mass[k] * X[3*k:3*(k+1)]
    COM /= sum(mass)

    for k in range(N):

        for i in range(3):
            for j in range(3):

                XPAS[3*k+i] += RPAS[i,j] * (X[3*k+j] - COM[j])

    return XPAS, RPAS, COM

def Nbf_matrix(N):
    """
    Calculate the body-fixed operators of :math:`\mathbf{N}` for
    a given :math:`N` quantum number,

    ..  math::
        \\langle N k' \\vert N_i \\vert N k \\rangle

    Parameters
    ----------
    N : integer
        The :math:`N` quantum number

    Returns
    -------
    Nx,Ny,Nz : ndarray
        The matrix representations.

    Notes
    -----
    The basis function order is :math:`k = 0, 1, \\ldots, N, -N, -N+1, \\ldots, -1`.
    This is different than :func:`Jbf_cs`.

    """

    #
    # k = 0, 1, 2, ..., N, -N, ..., -1
    #
    # The final ndarrays will be indexable by the signed k quantum number
    #
    krange = [i for i in range(N+1)] + [ i-N for i in range(N)]
    n = 2*N+1 # the number of functions

    kI, kJ = np.meshgrid(krange, krange, indexing = 'ij')

    Nz = np.diag(krange)

    # Calculate Np = Nx + i*Ny
    # (this is the *lowering* operating in body-fixed frame
    #  because of the anamalous commutation sign of Nx, Ny, Nz)
    Np = np.zeros((n,n))
    Nm = np.zeros((n,n))

    idx = (kI == kJ - 1)
    Np[idx] = np.sqrt( N*(N+1) - kJ*(kJ-1) )[idx] # lowering operator

    # Raising operator, Nm = Nx - i*Ny
    idx = (kI == kJ + 1)
    Nm[idx] = np.sqrt( N*(N+1) - kJ*(kJ+1) )[idx] # raising operator

    Nx = (Np + Nm) / 2
    Ny = (Np - Nm) / (2 * 1j)

    return Nx, Ny, Nz

def L_matrix(L):
    """
    Calculate the Cartesian components of a general
    angular momentum operator with normal commutation
    relations

    ..  math::
        \\langle L m' \\vert L_i \\vert L m \\rangle

    Parameters
    ----------
    L : integer
        The total angular momentum quantum number, :math:`L`.

    Returns
    -------
    LX, LY, LZ : ndarray
        The matrix representations.

    Notes
    -----
    The basis function order is :math:`m = 0, 1, \\ldots, L, -L, -L+1, \\ldots, -1`.

    """

    #
    # k = 0, 1, 2, ..., L, -L, ..., -1
    #
    # The final ndarrays will be indexable by the signed m quantum number
    #
    mrange = [i for i in range(L+1)] + [ i-L for i in range(L)]
    n = 2*L+1 # the number of functions

    mI, mJ = np.meshgrid(mrange, mrange, indexing = 'ij')

    LZ = np.diag(mrange)

    Lp = np.zeros((n,n))
    Lm = np.zeros((n,n))

    idx = (mI == mJ + 1)
    Lp[idx] = np.sqrt( L*(L+1) - mJ*(mJ + 1) )[idx] # raising operator

    idx = (mI == mJ - 1)
    Lm[idx] = np.sqrt( L*(L+1) - mJ*(mJ - 1) )[idx] # lowering operator

    LX = (Lp + Lm) / 2
    LY = (Lp - Lm) / (2 * 1j)

    return LX, LY, LZ

def caseb_multistate_N(alpha, N, k, SS1, JJ1):
    """
    Calculate the body-fixed :math:`N_i` operators
    for a multi-state case (b) basis set.

    Parameters
    ----------
    alpha : array_like
        The electronic (or other) state index.
    N : array_like
        The :math:`N` quantum number.
    k : array_like
        The signed :math:`k` quantum number.
    SS1 : array_like
        The value of :math:`2S+1`.
    JJ1 : array_like
        The value of :math:`2J+1`.

    Returns
    -------
    Nx, Ny, Nz : ndarray
        The matrix elements of the body-fixed components
        of :math:`\\mathbf{N}`.

    Notes
    -----

    The case (b) basis function is

    ..  math::
        \\vert J m_J N k S; \\alpha \\rangle = \\sum_{m_N, m_S}
        \\vert N k m_N \\rangle \\vert S m_S \\rangle \\vert \\alpha \\rangle
        \\langle N m_N, S m_S \\vert J m_J \\rangle

    The matrix elements of the body-fixed components :math:`N_i`, :math:`i = x,y,z`,
    are

    ..  math::
        \\langle J' m_J' N' k' S' ;\\alpha' \\vert N_i \\vert J m_J N k S ; \\alpha\\rangle
        = \\delta_{\\alpha\\alpha'} \\delta_{JJ'} \\delta_{m_J m_J'} \\delta_{SS'}
          \\delta_{NN'} \\langle N' k' \\vert N_i \\vert N k \\rangle
    """

    n = len(N) # The number of basis functions in the list

    Nx = np.zeros((n,n), dtype = np.complex128)
    Ny = np.zeros((n,n), dtype = np.complex128)
    Nz = np.zeros((n,n), dtype = np.complex128)

    # Calculate N operators for every possible value of N
    Nops = [ (Nbf_matrix(i) if i in N else None) for i in range(max(N)+1)]

    for i in range(n):
        for j in range(n):
            #
            # The matrix element is diagonal in
            # alpha (electronic index or other), N, S and J
            #
            if alpha[i] != alpha[j]:
                continue
            if N[i] != N[j]:
                continue
            if SS1[i] != SS1[j]:
                continue
            if JJ1[i] != JJ1[j]:
                continue

            Nx[i,j] = Nops[N[i]][0][k[i], k[j]]
            Ny[i,j] = Nops[N[i]][1][k[i], k[j]]
            Nz[i,j] = Nops[N[i]][2][k[i], k[j]]

    return Nx, Ny, Nz

def caseb_multistate_S(alpha, N, k, SS1, JJ1):
    """
    Calculate the body-fixed :math:`S_i` operators
    for a multi-state case (b) basis set.

    Parameters
    ----------
    alpha : array_like
        The electronic (or other) state index.
    N : array_like
        The :math:`N` quantum number.
    k : array_like
        The signed :math:`k` quantum number.
    SS1 : array_like
        The value of :math:`2S+1`.
    JJ1 : array_like
        The value of :math:`2J+1`.

    Returns
    -------
    Sx, Sy, Sz : ndarray
        The matrix elements of the body-fixed components
        of :math:`\\mathbf{S}`.

    Notes
    -----

    The case (b) basis function is

    ..  math::
        \\vert J m_J N k S; \\alpha \\rangle = \\sum_{m_N, m_S}
        \\vert N k m_N \\rangle \\vert S m_S \\rangle \\vert \\alpha \\rangle
        \\langle N m_N, S m_S \\vert J m_J \\rangle

    The matrix elements of the body-fixed components :math:`S_i`, :math:`i = x,y,z`,
    are calculated by first calculating the body-fixed  spherical tensor components

    ..  math::

        &\\langle J' m_J' N' k' S' ;\\alpha' \\vert S_q \\vert J m_J N k S ; \\alpha\\rangle
        = \\delta_{\\alpha\\alpha'} \\delta_{JJ'} \\delta_{m_J m_J'} \\delta_{SS'}  \\\\
        &\\qquad\\qquad \\times (-1)^{k + J + S + 1} \\sqrt{(2N+1)(2N'+1)(2S+1)S(S+1)}
          \\left(\\begin{array}{ccc} N & 1 & N' \\\\ k & -q & -k' \\end{array} \\right)
          \\left\\{\\begin{array}{ccc} N & S & J \\\\ S & N' & 1 \\end{array} \\right\\}

    and then relating

    ..  math::

        S_x &= \\frac{1}{\\sqrt{2}} ( -S_{q = +1} + S_{q = -1}) \\\\
        S_y &= \\frac{+i}{\\sqrt{2}} ( S_{q = +1} + S_{q = -1}) \\\\
        S_z &= S_{q = 0}

    """

    n = len(N) # The number of basis functions in the list

    Sq = np.zeros((3,n,n))

    for q in [-1,0,1]:
        for i in range(n):
            for j in range(n):
                #
                # The matrix element is diagonal in
                # alpha (electronic index or other), S and J
                #
                if alpha[i] != alpha[j]:
                    continue
                if SS1[i] != SS1[j]:
                    continue
                if JJ1[i] != JJ1[j]:
                    continue

                # (-1) ** (k + J + S + 1)
                coeff1 = (-1)**(k[j] + (SS1[j] + JJ1[j])/2 )
                # sqrt((2N+1) * (2N'+1) * (2S+1) * S(S+1) )
                ssp1 = (SS1[j] ** 2  - 1)/4 # the value of S(S+1)
                coeff2 = np.sqrt( (2*N[j] + 1) * (2*N[i] + 1) * SS1[j] * ssp1)

                threej = wigner3j(2*N[j], 2*1, 2*N[i],
                                  2*k[j], 2*(-q), 2*(-k[i]))

                sixj = wigner6j(2*N[j], SS1[j]-1, JJ1[j]-1,
                                SS1[j]-1, 2*N[i], 2*1)

                Sq[q,i,j] = coeff1 * coeff2 * threej * sixj
    #
    # Calculate the Cartesian components
    # from the spherical tensor components
    #
    Sx = (-Sq[1] + Sq[-1]) / np.sqrt(2)
    Sy = 1j * (Sq[1] + Sq[-1]) / np.sqrt(2)
    Sz = Sq[0].copy()

    return Sx, Sy, Sz

def caseb_multistate_L(Li_e, LiLj_ac_e, alpha, N, k, SS1, JJ1):
    """
    Calculate the body-fixed :math:`L_i` operators
    for a multi-state case (b) basis set.

    Parameters
    ----------
    Li_e : (3,NE,NE) array_like
        The pure electronic matrix elements of
        :math:`L_i`.
    LiLj_ac_e : (3,3,NE,NE) array_like
        The pure electronic matrix elements of
        the anti-commutators :math:`[L_i, L_j]_+ = L_i L_j + L_j L_i`.
    alpha : array_like
        The electronic state index, i.e. the values for
        indexing into `Li_e` and `LiLj_ac_e`.
    N : array_like
        The :math:`N` quantum number.
    k : array_like
        The signed :math:`k` quantum number.
    SS1 : array_like
        The value of :math:`2S+1`.
    JJ1 : array_like
        The value of :math:`2J+1`.

    Returns
    -------
    Li : (3,n,n) ndarray
        Li[i] is the :math:`L_i` operator in the case (b) representation.
    LiLj_ac : (3,3,n,n) ndarray
        LiLj_ac[i,j] is the :math:`[L_i, L_j]_+` anti-commutator in
        the case (b) representation.

    """

    Li_e = np.array(Li_e)
    LiLj_ac_e = np.array(LiLj_ac_e)

    # Construct Li in spin-rot-elec basis
    n = len(N) # the number of case (b) basis functions

    Li = np.zeros((3, n, n), dtype = np.complex128)
    LiLj_ac = np.zeros((3, 3, n, n), dtype = np.complex128)

    for i in range(n):
        for j in range(n):

                #
                # Diagonal in N, k, S and J
                #
                if N[i] != N[j] or k[i] != k[j] or \
                   SS1[i] != SS1[j] or \
                   JJ1[i] != JJ1[j]:
                    continue

                for a in range(3):
                    Li[a,i,j] = Li_e[a, alpha[i], alpha[j]]

                    for b in range(3):
                        LiLj_ac[a,b,i,j] = LiLj_ac_e[a,b, alpha[i], alpha[j]]

    return Li, LiLj_ac


import numpy as np

def t1(t, T1, M_0, M_inf):
    """Exponential recovery for inversion recovery and saturation recovery T1 Measurements

    Args:
        t (array_like): time series
        T_1 (float): T1 value
        M_0 (float): see equation
        M_inf (float): see equation

    Returns:
        ndarray: T1 curve

    .. math::
        f(t) = M_{\infty} - (M_{\infty} - M_0) e^{-t/T_1}
    """

    return M_inf - (M_inf - M_0) * np.exp(-1.0 * t / T1)

def t2(t, M_0, T2, p=1.0):
    """Calculate stretched or un-stretched (p=1) exponential T2 curve

    Args:
        t (array_like): time series
        M_0 (float): see equation
        T_2 (float): T2 value
        p (float): see equation

    Returns:
        ndarray: T2 curve

    .. math::
        f(t) = M_{0} e^{(-(t/T_{2})^{p}}
    """

    return M_0 * np.exp(-1.0 * (t / T2) ** p)

def general_exp(t, C1, C2, tau):
    """Calculate mono-exponential curve

    Args:
        t (array_like): time series
        C1 (float): see equation
        C2 (float): see equation
        tau (float): see equation

    Returns:
        ndarray: mono-exponential curve

    .. math::
        f(t) = C1 + C2 e^{-t/tau}
    """

    return C1 + C2 * np.exp(-1.0 * t / tau)

def general_biexp(t, C1, C2, tau1, C3, tau2):
    """Calculate bi-exponential curve

    Args:
        t (array_like): time series
        C1 (float): see equation
        C2 (float): see equation
        C3 (float): see equation
        tau1 (float): see equation
        tau2 (float): see equation

    Returns:
        ndarray: bi-exponential curve

    .. math::
        f(t) = C1 + C2 e^{-t/tau1} + C3 e^{-t/tau2}
    """

    return C1 + C2 * np.exp(-1.0 * t / tau1) + C3 * np.exp(-1.0 * t / tau2)

def buildup_function(p, E_max, p_half):
    """Calculate asymptotic buildup curve

    Args:
        p (array): power series
        E_max (float): maximum enhancement
        p_half (float): power at half saturation

    Returns:
        ndarray: buildup curve

    .. math::
        f(p) = 1 + E_{max} * p / (p_{1/2} + p)
    """

    return 1 + E_max * p / (p_half + p)

def ksigma_smax(p, E_max, p_half):
    """Calculate asymptotic buildup curve

    Args:
        p (array): power series
        E_max (float): maximum enhancement
        p_half (float): power at half saturation

    Returns:
        ndarray: buildup curve

    .. math::
        f(p) = E_{max} * p / (p_{1/2} + p)
    """

    return E_max * p / (p_half + p)

def logistic(x, c, x0, L, k):
    """Not Implemented. Placeholder for calculating asymptotic buildup curve

    Args:
        x (array): x values
        c (float): offset
        x0 (float): x-value of sigmoid's midpoint
        L (float): maximum value
        k (float): logistic growth steepness

    Returns:
        ndarray: buildup curve
    """

    # return c + L / (1.0 + np.exp(-1.0 * k * (x - x0)))
    return NotImplemented


"""
SymPy orthogonalization module. This should become part of SymPy, after better
docstrings (+doctests) as well as tests are written.

It should probably go into sympy/matrices/
"""

from sympy import sqrt, integrate, conjugate, Symbol, sympify

def l2_inner_product(a, b, lim):
    """
    Calculates the L2 inner product (a, b) over the domain lim.
    """
    return integrate(a*b, lim)

def h1_inner_product(a, b, lim):
    """
    Calculates the L2 inner product (a, b) over the domain lim.
    """
    x, _, _ = lim
    a = sympify(a)
    b = sympify(b)
    return integrate(a*b + a.diff(x)*b.diff(x), lim)

def norm_induced(v, inner_product):
    return sqrt(inner_product(v, v))

def normalize(v, inner_product):
    return v/norm_induced(v, inner_product)

def projection(v, basis, inner_product):
    """
    Projects the function v on the basis using the inner_product.
    """
    r = 0
    for b in basis:
        r += inner_product(v, b) * b
    return r

def gram_schmidt(basis, inner_product=None):
    """
    Orthonormalizes the "basis" using the Gram-Schmidt process.

    basis ........... an arbitrary set of linearly independent SymPy expressions
    inner_product ... the inner product to use for projections

    Example:
    >>> l2_gram_schmidt([1, x, x**2], (x, -1, 1)]
    [1/2*2**(1/2), x*6**(1/2)/2, -3*10**(1/2)*(1/3 - x**2)/4]

    """
    if inner_product is None:
        x = basis[-1].atoms(Symbol).pop()
        inner_product = lambda f, g: l2_inner_product(f, g, (x, -1, 1))

    r = []
    for a in basis:
        if r == []:
            v = a
        else:
            v = a - projection(a, r, inner_product)
        v = normalize(v, inner_product)
        r.append(v)
    return r


# -*- coding: utf-8 -*-
"""
Created on Wed Jun  6 16:10:42 2018

@author: lijie
"""

from numpy import *
import matplotlib.pyplot as plt

'''load data'''
def loadDataSet(filename):
    numFeat=len(open(filename).readline().split('\t'))-1
    dataMat=[]
    labelMat=[]
    fr=open(filename)
    for line in fr.readlines():
        lineArr=[]
        curLine=line.strip().split('\t')
        for i in range(numFeat):
            lineArr.append(float(curLine[i]))
        dataMat.append(lineArr)
        labelMat.append(float(curLine[-1]))
    return dataMat,labelMat

'''calculate w using OLS(最小二乘法)'''
def standRegres(xArr,yArr):
    xMat=mat(xArr)
    yMat=mat(yArr).T
    xTx=(xMat.T)*xMat
    if linalg.det(xTx)==0.0:
        print('this matrix cannot be inversed')
        return
    ws=(xTx.I)*(xMat.T)*yMat
    return ws

'''plot the fitting curve'''
def fittingcurveplot(x,y):
    x=mat(x)
    y=mat(y)
    fig=plt.figure()
    ax=fig.add_subplot(111)
    ax.scatter(x[:,1].flatten().A[0],y.T[:,0].flatten().A[0])#plot data points

    xSorted=mat(xArr) #sort x by increasing order
    xSorted.sort(0)
    yPre=xSorted*ws
    ax.plot(xSorted[:,1],yPre)
    plt.show()
    return yPre

if __name__=='__main__' :
#test
    xArr,yArr=loadDataSet('ex0.txt')
    ws=standRegres(xArr,yArr)
    print(ws)

    fittingcurveplot(xArr,yArr)
    #calc the correlation coefficient of prediction and actual number
    cc=corrcoef(yPre.T,yArr)


# __init__.py
#
# Copyright(c) Exequiel Ceasar Navarrete <esnavarrete1@up.edu.ph>
# Licensed under MIT
# Version 1.0.0

import numpy as np
from scipy.ndimage import filters

def apply_prewitt(img):
  img = img.copy()

  im_x = np.zeros(img.shape)
  filters.prewitt(img, 1, im_x, mode="nearest")

  im_y = np.zeros(img.shape)
  filters.prewitt(img, 0, im_y, mode="nearest")

  #the magnitude
  grad = np.sqrt(im_x ** 2 + im_y ** 2)

  return im_x, im_y, grad

def apply_prewitt2(img):
  img = img.copy()

  dx = np.array([[1.0, 0.0, -1.0], [1.0, 0.0, -1.0], [1.0, 0.0, -1.0],])
  dy = np.transpose(dx)

  im_x = filters.convolve(img, dx, output=np.float64, mode='nearest')
  im_y = filters.convolve(img, dy, output=np.float64, mode='nearest')

  return im_x, im_y

def apply_roberts(img):
  img = img.copy()

  # assemble the roberts kernel
  dx = np.array([[1, 0], [0, -1]])
  dy = np.array([[0, -1], [1, 0]])

  # apply the x and y kernels
  im_x = filters.convolve(img, dx, output=np.float64, mode='nearest')
  im_y = filters.convolve(img, dy, output=np.float64, mode='nearest')

  return im_x, im_y


import numpy as np

def gaussian(x, mu, sig):
    """Gaussian function.

    Args:
        x (np.ndarray): Input array for the gaussian.
        mu (float): Mean of the gaussian.
        sig (float): Standard deviation of the gaussian.

    Returns:
        np.ndarray: Gaussian output.
    """

    return np.exp(-np.power(x - mu, 2.0) / (2 * np.power(sig, 2.0)))

def gaussian_normalised(x, mu, sig):
    """Normalized gaussian function.

    Args:
        x (np.ndarray): Input array for the gaussian.
        mu (float): Mean of the gaussian.
        sig (float): Standard deviation of the gaussian.

    Returns:
        np.ndarray: Normalized gaussian output.
    """

    return np.exp(-np.power(x - mu, 2.0) / (2 * np.power(sig, 2.0))) * (
        1 / np.sqrt(2 * np.pi * np.power(sig, 2.0))
    )