import numpy as np
from scipy.stats import norm

# Acquisition functions
def probability_of_improvement(current_best, mean, std, xi):
    '''
    Thus function implements the probability of improvement acquisition function.
    It implements
        PI(x) = P(f(x) >= f(x^+))
            since we consider f(x^+) = mu^+ + epxis we have
              = Phi ( (mu - mu^+ - xi) / sigma )
    :param current_best: this is the current max of the unknown function: mu^+
    :param mean: this is the mean function from the GP over the considered set of points
    :param std: this is the std function from the GP over the considered set of points
    :param xi: small value added to avoid corner case
    :return: the value of this acquisition function for all the points
    '''
    # since std coan be 0, to avoid an error, add a small value in the denominator (like +1e-9)
    PI = norm.cdf( (mean - current_best - xi) / (std + 1e-9) ) #the norm.cdf function is your friend
    return PI

def expected_improvement(current_best, mean, std, xi):
    '''
    It implements the following function:

            | (mu - mu^+ - xi) Phi(Z) + sigma phi(Z) if sigma > 0
    EI(x) = |
            | 0                                       if sigma = 0

            where Phi is the CDF and phi the PDF of the normal distribution
            and
            Z = (mu - mu^+ - xi) / sigma

    :param current_best: this is the current max of the unknown function: mu^+
    :param mean: this is the mean function from the GP over the considered set of points
    :param std: this is the std function from the GP over the considered set of points
    :param xi: small value added to avoid corner case
    :return: the value of this acquisition function for all the points
    '''

    # start by computing the Z as we did in the probability of improvement function
    # to avoid division by 0, add a small term eg. np.spacing(1e6) to the denominator
    Z = (mean - current_best - xi) / (std + 1e-9)
    # now we have to compute the output only for the terms that have their std > 0
    EI = (mean  - current_best - xi) * norm.cdf(Z) + std * norm.pdf(Z)
    EI[std == 0] = 0

    return EI

def GP_UCB(mean, std, t, dim = 1.0, v = 1.0, delta = .1):
    '''
    Implementation of the Gaussian Process - Upper Confident Bound:
        GP-UBC(x) = mu + sqrt(v * beta) * sigma

    where we are usinv v = 1 and beta = 2 log( t^(d/2 + 2) pi^2 / 3 delta)
    as proved in Srinivas et al, 2010, to have 0 regret.

    :param mean: this is the mean function from the GP over the considered set of points
    :param std: this is the std function from the GP over the considered set of points
    :param t: iteration number
    :param dim: dimension of the input space
    :param v: hyperparameter that weights the beta for the exploration-exploitation trade-off. If v = 1 and another
              condition, it is proved we have 0 regret
    :param delta: hyperparameter used in the computation of beta
    :return: the value of this acquisition function for all the points
    '''
    beta =  2 * np.log( t**(dim/2 + 2) * np.pi**2 / (3 * delta + 1e-9)  )
    UCB = mean + np.sqrt(v * beta) * std

    return UCB

def GP_UCB_approx(mean, std, t, eps):
    '''
    Implementation of the Gaussian Process - Upper Confident Bound in a easy approximate way:
        GP-UBC(x) = mu + eps * log(t) * sigma

    we use the fact that beta ~ log(t)^2, so we have sqrt(v * log(t)^2) = log(t)*sqrt(v) ~ eps * log(t)

    :param mean: this is the mean function from the GP over the considered set of points
    :param std: this is the std function from the GP over the considered set of points
    :param t: iteration number
    :param eps: trade-off constant
    :return: the value of this acquisition function for all the points
    '''
    UCB = mean + eps * np.log(t) * std
    return UCB


# %%

######## Exercise 4 - Part 1 - Logistic Regression ########
# In this exercise, we are going to classify the breast cancer wisconsin dataset.
# This dataset contains in total 569 examples, among them 212 examples are labelled as malignant (M or 0) and 357
# examples are marked as benign (B or 1). .Each example is a vector of 30 dimensions.
# We will train a binary logistic regression model using this dataset.

# Load, normalize, split and visualize your dataset. This step has been done for you.
import numpy as np
from sklearn.datasets import load_breast_cancer
import matplotlib.pyplot as plt
import math
from sklearn.metrics import accuracy_score

# load the dataset
data = load_breast_cancer()
x = data.data
y = data.target

# split into training and test sets, namely 80 percent of examples goes for the training, 20 percent goes for the test set
N_train = int(0.8 * x.shape[0])
x_train = x[:N_train, :]
y_train = y[:N_train]
x_test = x[N_train:, :]
y_test = y[N_train:]

# scale features by removing mean and dividing by the standard deviation
x_bar = np.mean(x_train, axis=0)
x_std = np.std(x_train, axis=0)
x_train_scaled = (x_train - x_bar) / x_std
x_test_scaled = (x_test - x_bar) / x_std

print("Number of training examples: ", x_train.shape[0])
print("Number of testing examples: ", x_test.shape[0])

# visualize the dataset using histogram
labels = ['Benign', 'Malignant']
population = [np.sum(y), np.sum(y == 0)]
y_pos = np.arange(len(labels))
barlist = plt.bar(y_pos, population, align='center', width=0.3)
plt.xticks(y_pos, labels)
plt.ylabel('Number of examples')
plt.title('Breast wisconsin dataset.')
barlist[1].set_color('r')
plt.show()

# Add intercept terms and initialize parameters
# This step has been done for you.
intercept_train = np.ones((N_train, 1))
x_train_scaled = np.hstack((intercept_train, x_train_scaled))

intercept_test = np.ones((x.shape[0] - N_train, 1))
x_test_scaled = np.hstack((intercept_test, x_test_scaled))

print(x_train_scaled.shape)
print(x_test_scaled.shape)

# Step 1: Implement the sigmoid, gradient and cost functions

# this function returns the probability of y=1
# x: data matrix
# theta: model's parameters
def sigmoid(x, theta):
    return 1 / (1 + (np.exp(-(np.matmul(theta.reshape([1, -1]), x.T)))))

# logarithmic loss
# x: data matrix (2D)
# y: label (1D)
# theta: model's parameters (1D)
def compute_cost(x, y, theta):
    res = 0
    n = x.shape[0]
    for i, xrow in enumerate(x):
        res += (y[i] * math.log2(sigmoid(xrow, theta))) + ((1 - y[i]) * math.log2(1 - sigmoid(xrow, theta)))
    return -((1 / n) * res)

# def compute_gradient(x, y, theta):
#     res = 0
#     for i, xrow in enumerate(x):
#         for j, xcol in enumerate(xrow):
#             res += ((sigmoid(xrow, theta) - y[i]) * xrow[j])
#     return res

def compute_gradient(x, y, theta):
    n = x.shape[0]
    res = np.matmul(x.T, (sigmoid(x, theta)[0] - y).reshape([-1, 1]))
    # res = np.matmul(x.T, (sigmoid(x, theta) - y).T)
    return (1 / n) * res

def approximate_gradient(x, y, theta, epsilon):
    n_features = x.shape[1]
    app_grad = np.zeros(n_features)
    for i in range(n_features):
        epsilon_one_hot = np.zeros(n_features)
        epsilon_one_hot[i] = epsilon
        theta_before = theta - epsilon_one_hot
        theta_after = theta + epsilon_one_hot
        app_grad[i] = (compute_cost(x, y, theta_after) - compute_cost(x, y, theta_before)) / (2 * epsilon)
    return app_grad

theta = 0.5 * np.random.randn(x_train_scaled.shape[1])
grad = compute_gradient(x_train_scaled, y_train, theta)
epsilon = 1e-4
app_grad = approximate_gradient(x_train_scaled, y_train, theta, epsilon)
print('Sum of gradient squared error: ', np.sum((grad[:, 0] - app_grad) ** 2))

# Step 2: Update model parameters using mini-batch gradient descent algorithm

# try different values for the learning rate
learning_rate = 0.01

# number of training iterations
num_samples = x_train_scaled.shape[0]
batch_size = int(num_samples / 10)
N_iterations = int(num_samples / batch_size) * 50  # 20 times the number of training samples

# prepare to plot
plt.subplot(111)

# keep the cost value for each training step
J = np.zeros(N_iterations)

# initialize new parameters using random distribution
theta_msgd = 0.5 * np.random.randn(x_train_scaled.shape[1])

start = 0
end = 0
for step in range(N_iterations):
    if step % num_samples == 0:
        # shuffle the training data
        indices = np.random.permutation(num_samples)
        x_train_scaled = x_train_scaled[indices, :]
        y_train = y_train[indices]

    # create a mini-batch of data to train on
    end = start + batch_size
    if end >= num_samples:
        end = num_samples
    x_batch = x_train_scaled[start:end, :]
    y_batch = y_train[start:end]
    start = 0 if end >= num_samples else end

    # calculate the cost on x_step and y_step
    J[step] = compute_cost(x_batch, y_batch, theta_msgd)
    grad = compute_gradient(x_batch, y_batch, theta_msgd)
    # update theta_msgd using a x_step and y_step
    theta_msgd = theta_msgd - (learning_rate * grad[:, 0])

# calculate the loss on the whole training set
J_train = compute_cost(x_train_scaled, y_train, theta_msgd)
print('training cost: %f' % J_train)
# plot cost function
plt.plot(J)
plt.xlabel('Training step')
plt.ylabel('Cost')
plt.show()

# Step 3: Predict and draw the confusion matrix
# calculate the accuracy
from sklearn.metrics import confusion_matrix
import itertools

# this function draw the confusion matrix, you don't need to touch it.
def plot_confusion_matrix(cm, classes,
                          normalize=False,
                          title='Confusion matrix',
                          cmap=plt.cm.Blues):
    """
    This function prints and plots the confusion matrix.
    Normalization can be applied by setting `normalize=True`.
    """
    if normalize:
        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]
        print("Normalized confusion matrix")
    else:
        print('Confusion matrix, without normalization')

    print(cm)

    plt.imshow(cm, interpolation='nearest', cmap=cmap)
    plt.title(title)
    plt.colorbar()
    tick_marks = np.arange(len(classes))
    plt.xticks(tick_marks, classes, rotation=45)
    plt.yticks(tick_marks, classes)

    fmt = '.2f' if normalize else 'd'
    thresh = cm.max() / 2.
    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):
        plt.text(j, i, format(cm[i, j], fmt),
                 horizontalalignment="center",
                 color="white" if cm[i, j] > thresh else "black")

    plt.tight_layout()
    plt.ylabel('True label')
    plt.xlabel('Predicted label')

# this function computes the accuracy, which is the percentage of the correctly classified examples
# y_ground_truth: original labels (1D vector)
# y_pre: predicted labels (1D vector)
# return: accuracy in percent
def compute_accuracy(y_ground_truth, y_pred):
    return accuracy_score(y_ground_truth, y_pred)

# In logistic regression, we choose a threshold.
# If the output of our hypothesis is greater than this threshold, the example is classified as 1,
# otherwise it is given 0.

threshold = 0.5
y_pred = np.matmul(x_test_scaled, theta_msgd)
y_pred_reloaded = np.array(list(map(lambda y: 1 if y >= 0.5 else 0, y_pred)))

print("Accuracy on test set: {:.2f}".format(compute_accuracy(y_test, y_pred_reloaded)))

# calculate the cost for the test set
test_cost = compute_cost(x_test_scaled, y_test, theta_msgd)
print('Test cost: ', test_cost)

# Compute confusion matrix
cnf_matrix = confusion_matrix(y_test, y_pred_reloaded)
np.set_printoptions(precision=2)

# Plot non-normalized confusion matrix
plt.figure()
plot_confusion_matrix(cnf_matrix, classes=labels,
                      title='Confusion matrix, without normalization')
plt.show()


import math
import numpy as np
import random
from itertools import repeat

def euclidean_distance(point1, point2) -> float:
    return math.sqrt(sum(map(lambda x: math.pow((x[0]-x[1]), 2), zip(point1, point2))))

class kmeans(object):
    """ K-Means clustering.
    ----------
    Parameters
    ----------
    n_clusters : int, optional, default: 8
    The number of clusters to form as well as the number of centroids to generate.

    --------
    Examples
    --------
    import numpy as np
    from sklearn.datasets import load_iris
    iris= load_iris()
    X = iris.data
    model = kmeans(3, random_state=0)
    model = model.fit(X)
    labels = np.array(model.labels)
    print(labels)

    """

    def __init__(self, k, max_iter = 10, random_state = 0):
        self.k = k
        self.max_iter = 10
        self.random_state = 0
        self.X = None
        self.labels = None
        self.centroids = None

    def fit(self, X):
        """ Compute k-means clustering.
        ----------
        Parameters
        ----------
        X : array-like or sparse matrix, shape=(n_samples, n_features)
            Training instances to cluster.
        """
        self.X = X
        self.centroids = self._init_centroids()
        old_centroids = None
        n_iter = 0
        while n_iter <= self.max_iter and not np.array_equal(old_centroids, self.centroids):
            # stop when reach max_iter or no centroid update
            n_iter += 1
            old_centroids = np.copy(self.centroids)
            self.labels = list(map(self._assign_clusters, X)) # E step
            self.centroids = self._update_centroids() # M step
        return self

    def _init_centroids(self):
        """ Random Choose K points as initial centroids
        """
        np.random.seed(self.random_state)
        return self.X[np.random.choice(self.X.shape[0], self.k)]

    def _assign_clusters(self, point):
        """E step of the K-means EM algorithm.
        return the index of the nearest centroid as label of an instance
        """
        return np.argmin(list(map(euclidean_distance, self.centroids, repeat(point))))

    def _update_centroids(self):
        """M step of the K-means EM algorithm.
        """
        return list(map(lambda label: np.mean(self.X[list(map(lambda x: x == label, self.labels))], axis=0), sorted(set(self.labels))))

    def predict(self, X):
        """ Predict the closest cluster each sample in X belongs to.
        ----------
        Parameters
        ----------
        X : {array-like, sparse matrix} of shape (n_samples, n_features)
            New data to predict.
        """
        return list(map(self._assign_clusters, X))


import numpy as np
import time

N_darts = 1024*1024
N_trials = 1000
pis = []

t_1 = time.time()

for trial in range(0, N_trials):
    x = np.random.rand(N_darts)
    y = np.random.rand(N_darts)
    r = np.sqrt(x*x + y*y)
    count = r[ np.where(r <= 1) ]
    pi = 4.0*len(count)/N_darts
    pis.append(pi)
t_2 = time.time()
print("Time to compute", N_trials, "estimates of pi:", t_2 - t_1, "s")
print("The average value of pi was found to be: ", np.average(pis))
print("The standard deviation of pi was: ", np.std(pis))

f = open('pythonvectorized.dat', 'w')
for pi in pis:
    f.write('%f' % pi)
f.close()


from __future__ import print_function
from __future__ import unicode_literals
from __future__ import absolute_import
from __future__ import division
from math import log
from math import exp

import numpy as np

def log_one_plus_exp(z):
    """
    This function returns log(1 + exp(z)) where it rewrites the terms to reduce
    floating point errors.
    """
    if z > 0:
        return log(1 + exp(-z)) + z
    else:
        return log(1 + exp(z))

def invlogit(z):
    """
    This function return 1 / (1 + exp(-z)) where it rewrites the terms to
    reduce floating point errors.
    """
    if z > 0:
        return 1 / (1 + exp(-z))
    else:
        return exp(z) / (1 + exp(z))

invlogit_vect = np.vectorize(invlogit)
log_one_plus_exp_vect = np.vectorize(log_one_plus_exp)


import numpy as np
import matplotlib.pyplot as plt
from scipy.stats import multivariate_normal

DEBUG = True

######################################################
#  Debug output function
#  Control output by global variable DEBUG
######################################################
def debug(*args, **kwargs):
    global DEBUG
    if DEBUG:
        print(*args, **kwargs)

######################################################
# Gaussian distribution density function of the kth model
# Each i line represents the occurrence probability of the i sample in each model
# Return one-dimensional list
######################################################
def phi(Y, mu_k, cov_k):
    norm = multivariate_normal(mean=mu_k, cov=cov_k)
    return norm.pdf(Y)

######################################################
#E Step: Calculate the responsiveness of each model to the sample
#Y is a sample matrix, one row per sample, and a column vector when there is only one feature
#mu is a mean multi-dimensional array, each row represents the mean of each feature of a sample
#cov is an array of covariance matrices, alpha is an array of model responsivity
######################################################
def getExpectation(Y, mu, cov, alpha):
    #  number of samples
    N = Y.shape[0]
    #  number of models
    K = alpha.shape[0]

    assert N > 1, "There must be more than one sample!"
    assert K > 1, "There must be more than one gaussian model!"

    # Responsiveness matrix, rows correspond to samples, columns correspond to responsivity
    gamma = np.mat(np.zeros((N, K)))

    # Calculate the probability of occurrence of all samples in each model, rows correspond to samples, columns correspond to models
    prob = np.zeros((N, K))
    for k in range(K):
        prob[:, k] = phi(Y, mu[k], cov[k])
    prob = np.mat(prob)

    # Calculate the responsivity of each model to each sample
    for k in range(K):
        gamma[:, k] = alpha[k] * prob[:, k]
    for i in range(N):
        gamma[i, :] /= np.sum(gamma[i, :])
    return gamma

######################################################
# M Step: Iterate model parameters
# Y is the sample matrix, gamma is the responsivity matrix
######################################################
def maximize(Y, gamma):
    # Number of samples and features
    N, D = Y.shape
    # Number of models
    K = gamma.shape[1]

    #Initialization parameter value
    mu = np.zeros((K, D))
    cov = []
    alpha = np.zeros(K)

    # Update the parameters of each model
    for k in range(K):
        # The sum of the responsivity of the kth model to all samples
        Nk = np.sum(gamma[:, k])
        # Update mu
        # Average each feature
        mu[k, :] = np.sum(np.multiply(Y, gamma[:, k]), axis=0) / Nk
        # update cov
        cov_k = (Y - mu[k]).T * np.multiply((Y - mu[k]), gamma[:, k]) / Nk
        cov.append(cov_k)
        # upudate alpha
        alpha[k] = Nk / N
    cov = np.array(cov)
    return mu, cov, alpha

######################################################
# Data preprocessing
# Scale all data between 0 and 1
######################################################
def scale_data(Y):
    for i in range(Y.shape[1]):
        max_ = Y[:, i].max()
        min_ = Y[:, i].min()
        Y[:, i] = (Y[:, i] - min_) / (max_ - min_)
    debug("Data scaled.")
    return Y

######################################################
# Initialize model parameters
# shape is a binary representation of the sample size, (number of samples, number of features)
# K represents the number of models
######################################################
def init_params(shape, K):
    N, D = shape
    mu = np.random.rand(K, D)
    cov = np.array([np.eye(D)] * K)
    alpha = np.array([1.0 / K] * K)
    debug("Parameters initialized.")
    debug("mu:", mu, "cov:", cov, "alpha:", alpha, sep="\n")
    return mu, cov, alpha

######################################################
# Gaussian mixture model EM algorithm
# Given the sample matrix Y, calculate the model parameters
# K is the number of models
# times is the number of iterations
######################################################
def GMM_EM(Y, K, times):
    Y = scale_data(Y)
    mu, cov, alpha = init_params(Y.shape, K)
    log_likelihood = []

    for i in range(times):
        gamma = getExpectation(Y, mu, cov, alpha)
        mu, cov, alpha = maximize(Y, gamma)

        likelihood = 0

        for c in range(cov.shape[0]):
            mu_c = mu[c]
            cov_c = cov[c, :, :]
            likelihood += alpha[c] * phi(Y, mu_c, cov_c)

        log_likelihood.append(likelihood)
    debug("{sep} Result {sep}".format(sep="-" * 20))
    debug("mu:", mu, "cov:", cov, "alpha:", alpha, sep="\n")
    return mu, cov, alpha, np.array(log_likelihood)


import logging
from inspect import isfunction

import numpy as np

__all__ = [
    "NewtonIteration"
]

class NewtonIteration(object):
    """This class only contains static functions for Newton Iteration."""

    @staticmethod
    def iter_once(f, df, x, *args, **kwargs) -> float:
        return x-f(x, *args, **kwargs)/df(x, *args, **kwargs)

    @staticmethod
    def iter_full(f, df, x0, tol, maxsteps, *args, **kwargs) -> float:
        """
        A simple newton iteration method.

        Args:
            tol (:param:`Union[float, function]`): Accept float or function. For example,
                if the derivative at zero point is too small, one can change tol to::
                    tol = tol*df(x, *args, **kwargs)
                to acquire a more accurate solution.
        """
        if isfunction(tol):
            tolisfunc = True
            thistol = tol(x0, *args, **kwargs)
        else:
            tolisfunc = False
            thistol = tol
        if thistol <= 0:
            raise ValueError("TOL should be positive")

        x = x0
        a = f(x, *args, **kwargs)
        i = 0
        while np.abs(a) > thistol and i < maxsteps:
            d = df(x, *args, **kwargs)
            x = x-a/d
            a = f(x, *args, **kwargs)
            i += 1
            if tolisfunc:
                thistol = tol(x, *args, **kwargs)
                if thistol <= 0:
                    raise ValueError("TOL function should be positive")

        if i == maxsteps and np.abs(a) > thistol:
            logging.getLogger(__name__).warning(
                f"Newton method: didn't converge after {maxsteps} iterations.")
        return x


# gradient descent optimization with rmsprop for a two-dimensional test function
from math import sqrt
from numpy import asarray
from numpy.random import rand
from numpy.random import seed

# objective function
def objective(x, y):
	return x**2.0 + y**2.0

# derivative of objective function
def derivative(x, y):
	return asarray([x * 2.0, y * 2.0])

# gradient descent algorithm with rmsprop
def rmsprop(objective, derivative, bounds, n_iter, step_size, rho):
	# generate an initial point
	solution = bounds[:, 0] + rand(len(bounds)) * (bounds[:, 1] - bounds[:, 0])
	# list of the average square gradients for each variable
	sq_grad_avg = [0.0 for _ in range(bounds.shape[0])]
	# run the gradient descent
	for it in range(n_iter):
		# calculate gradient
		gradient = derivative(solution[0], solution[1])
		# update the average of the squared partial derivatives
		for i in range(gradient.shape[0]):
			# calculate the squared gradient
			sg = gradient[i]**2.0
			# update the moving average of the squared gradient
			sq_grad_avg[i] = (sq_grad_avg[i] * rho) + (sg * (1.0-rho))
		# build a solution one variable at a time
		new_solution = list()
		for i in range(solution.shape[0]):
			# calculate the step size for this variable
			alpha = step_size / (1e-8 + sqrt(sq_grad_avg[i]))
			# calculate the new position in this variable
			value = solution[i] - alpha * gradient[i]
			# store this variable
			new_solution.append(value)
		# evaluate candidate point
		solution = asarray(new_solution)
		solution_eval = objective(solution[0], solution[1])
		# report progress
		print('>%d f(%s) = %.5f' % (it, solution, solution_eval))
	return [solution, solution_eval]

# seed the pseudo random number generator
seed(1)
# define range for input
bounds = asarray([[-1.0, 1.0], [-1.0, 1.0]])
# define the total iterations
n_iter = 50
# define the step size
step_size = 0.01
# momentum for rmsprop
rho = 0.99
# perform the gradient descent search with rmsprop
best, score = rmsprop(objective, derivative, bounds, n_iter, step_size, rho)
print('Done!')
print('f(%s) = %f' % (best, score))


import numpy as np

# Setting the random seed, feel free to change it and see different solutions.
np.random.seed(42)

def stepFunction(t):
    if t >= 0:
        return 1
    return 0

def prediction(X, W, b):
    return stepFunction((np.matmul(X, W) + b)[0])

# EXERCISE: Fill in the code below to implement the perceptron trick.
# The function should receive as inputs the data X, the labels y,
# the weights W (as an array), and the bias b,
# update the weights and bias W, b, according to the perceptron algorithm,
# and return W and b.
def perceptronStep(X, y, W, b, learn_rate=0.01):
    for i, j in zip(X, y):
        y_pred = prediction(i, W, b)

        if j - y_pred == 1:  # False negative
            W[0] += i[0] * learn_rate
            W[1] += i[1] * learn_rate
            b += learn_rate

        elif j - y_pred == -1:  # False negative
            W[0] -= i[0] * learn_rate
            W[1] -= i[1] * learn_rate
            b -= learn_rate

    return W, b

# This function runs the perceptron algorithm repeatedly on the dataset,
# and returns a few of the boundary lines obtained in the iterations,
# for plotting purposes.
# Feel free to play with the learning rate and the num_epochs,
# and see your results plotted below.
def trainPerceptronAlgorithm(X, y, learn_rate=0.01, num_epochs=25):
    x_min, x_max = min(X.T[0]), max(X.T[0])
    y_min, y_max = min(X.T[1]), max(X.T[1])
    W = np.array(np.random.rand(2, 1))
    b = np.random.rand(1)[0] + x_max
    # These are the solution lines that get plotted below.
    boundary_lines = []
    for i in range(num_epochs):
        # In each epoch, we apply the perceptron step.
        W, b = perceptronStep(X, y, W, b, learn_rate)
        boundary_lines.append((-W[0] / W[1], -b / W[1]))
    return boundary_lines

if __name__ == '__main__':
    # data = pd.read_csv('B_data.csv',  names = ['X1','X2', 'y'], header = None, index_col = False)
    data = np.genfromtxt('B_data.csv', delimiter=',')
    trainPerceptronAlgorithm((data[:, 0:2]), data[:, 2])


import numpy as np
import torch
import torch.nn as nn

def dct1(x):
    """
    Discrete Cosine Transform, Type I

    :param x: the input signal
    :return: the DCT-I of the signal over the last dimension
    """
    x_shape = x.shape
    x = x.view(-1, x_shape[-1])

    return torch.rfft(torch.cat([x, x.flip([1])[:, 1:-1]], dim=1), 1)[:, :, 0].view(*x_shape)

def idct1(X):
    """
    The inverse of DCT-I, which is just a scaled DCT-I

    Our definition if idct1 is such that idct1(dct1(x)) == x

    :param X: the input signal
    :return: the inverse DCT-I of the signal over the last dimension
    """
    n = X.shape[-1]
    return dct1(X) / (2 * (n - 1))

def dct(x, norm=None):
    """
    Discrete Cosine Transform, Type II (a.k.a. the DCT)

    For the meaning of the parameter `norm`, see:
    https://docs.scipy.org/doc/scipy-0.14.0/reference/generated/scipy.fftpack.dct.html

    :param x: the input signal
    :param norm: the normalization, None or 'ortho'
    :return: the DCT-II of the signal over the last dimension
    """
    B, N = x.shape
    x = x.contiguous().view(-1, N)

    v = torch.cat([x[:, ::2], x[:, 1::2].flip([1])], dim=1)

    Vc = torch.rfft(v, 1, onesided=False)

    k = - torch.arange(N, dtype=x.dtype, device=x.device)[None, :] * np.pi / (2 * N)
    W_r = torch.cos(k)
    W_i = torch.sin(k)

    V = Vc[:, :, 0] * W_r - Vc[:, :, 1] * W_i

    if norm == 'ortho':
        V[:, 0] /= np.sqrt(N) * 2
        V[:, 1:] /= np.sqrt(N / 2) * 2

    V = 2 * V.view(B,N)

    return V

class FasterDCT(nn.Module):
    """Caches scaling variables for faster computation."""
    def __init__(self, in_features):
        super(FasterDCT, self).__init__()
        N = in_features
        k = - torch.arange(N, dtype=torch.float32)[None, :] * np.pi / (2 * N)
        self.W_r = nn.Parameter(torch.cos(k))
        self.W_i = nn.Parameter(torch.sin(k))
        self.W_r.require_grad, self.W_i.require_grad = False, False
        self.N = N
        self.pad = nn.ConstantPad1d((0,N), 0.)
        #self.reverse_idxs = torch.arange(N//2).flip(0)

    def forward(self, x, norm=None):
        B, N = x.shape
        x = x.contiguous().view(-1, N)
        v = self.pad(x)

        #x = x.view(B, N//2, 2)
        #x[:,:,1] = x[:,:,1].flip(1)
        #v = x.contiguous().view(B, N)

        #v = torch.cat([x[:,:,0], x[:,:,1].flip(1)], dim=1)

        Vc = torch.rfft(v, 1, onesided=False)[:,:self.N]

        V = Vc[:, :, 0] * self.W_r - Vc[:, :, 1] * self.W_i

        if norm == 'ortho':
            V[:, 0] /= np.sqrt(N) * 2
            V[:, 1:] /= np.sqrt(N / 2) * 2

        V = 2 * V.view(B,N)

        return V

def idct(X, norm=None):
    """
    The inverse to DCT-II, which is a scaled Discrete Cosine Transform, Type III

    Our definition of idct is that idct(dct(x)) == x

    For the meaning of the parameter `norm`, see:
    https://docs.scipy.org/doc/scipy-0.14.0/reference/generated/scipy.fftpack.dct.html

    :param X: the input signal
    :param norm: the normalization, None or 'ortho'
    :return: the inverse DCT-II of the signal over the last dimension
    """

    x_shape = X.shape
    N = x_shape[-1]

    X_v = X.contiguous().view(-1, x_shape[-1]) / 2

    if norm == 'ortho':
        X_v[:, 0] *= np.sqrt(N) * 2
        X_v[:, 1:] *= np.sqrt(N / 2) * 2

    k = torch.arange(x_shape[-1], dtype=X.dtype, device=X.device)[None, :] * np.pi / (2 * N)
    W_r = torch.cos(k)
    W_i = torch.sin(k)

    V_t_r = X_v
    V_t_i = torch.cat([X_v[:, :1] * 0, -X_v.flip([1])[:, :-1]], dim=1)

    V_r = V_t_r * W_r - V_t_i * W_i
    V_i = V_t_r * W_i + V_t_i * W_r

    V = torch.cat([V_r.unsqueeze(2), V_i.unsqueeze(2)], dim=2)

    v = torch.irfft(V, 1, onesided=False)
    x = v.new_zeros(v.shape)
    x[:, ::2] += v[:, :N - (N // 2)]
    x[:, 1::2] += v.flip([1])[:, :N // 2]

    return x.view(*x_shape)

def dct_2d(x, norm=None):
    """
    2-dimentional Discrete Cosine Transform, Type II (a.k.a. the DCT)

    For the meaning of the parameter `norm`, see:
    https://docs.scipy.org/doc/scipy-0.14.0/reference/generated/scipy.fftpack.dct.html

    :param x: the input signal
    :param norm: the normalization, None or 'ortho'
    :return: the DCT-II of the signal over the last 2 dimensions
    """
    X1 = dct(x, norm=norm)
    X2 = dct(X1.transpose(-1, -2), norm=norm)
    return X2.transpose(-1, -2)

def idct_2d(X, norm=None):
    """
    The inverse to 2D DCT-II, which is a scaled Discrete Cosine Transform, Type III

    Our definition of idct is that idct_2d(dct_2d(x)) == x

    For the meaning of the parameter `norm`, see:
    https://docs.scipy.org/doc/scipy-0.14.0/reference/generated/scipy.fftpack.dct.html

    :param X: the input signal
    :param norm: the normalization, None or 'ortho'
    :return: the DCT-II of the signal over the last 2 dimensions
    """
    x1 = idct(X, norm=norm)
    x2 = idct(x1.transpose(-1, -2), norm=norm)
    return x2.transpose(-1, -2)

def dct_3d(x, norm=None):
    """
    3-dimentional Discrete Cosine Transform, Type II (a.k.a. the DCT)

    For the meaning of the parameter `norm`, see:
    https://docs.scipy.org/doc/scipy-0.14.0/reference/generated/scipy.fftpack.dct.html

    :param x: the input signal
    :param norm: the normalization, None or 'ortho'
    :return: the DCT-II of the signal over the last 3 dimensions
    """
    X1 = dct(x, norm=norm)
    X2 = dct(X1.transpose(-1, -2), norm=norm)
    X3 = dct(X2.transpose(-1, -3), norm=norm)
    return X3.transpose(-1, -3).transpose(-1, -2)

def idct_3d(X, norm=None):
    """
    The inverse to 3D DCT-II, which is a scaled Discrete Cosine Transform, Type III

    Our definition of idct is that idct_3d(dct_3d(x)) == x

    For the meaning of the parameter `norm`, see:
    https://docs.scipy.org/doc/scipy-0.14.0/reference/generated/scipy.fftpack.dct.html

    :param X: the input signal
    :param norm: the normalization, None or 'ortho'
    :return: the DCT-II of the signal over the last 3 dimensions
    """
    x1 = idct(X, norm=norm)
    x2 = idct(x1.transpose(-1, -2), norm=norm)
    x3 = idct(x2.transpose(-1, -3), norm=norm)
    return x3.transpose(-1, -3).transpose(-1, -2)

def rfft(x):
    return torch.rfft(x, 1, onesided=False)

class DumbDCT(nn.Linear):
    """Implement any DCT as a linear layer."""
    def __init__(self, in_features, type):
        self.type = type
        self.N = in_features
        super(DumbDCT, self).__init__(in_features, in_features, bias=False)

    def reset_parameters(self):
        if self.type == 'dct':
            # initialise using dct function
            self.weight.data = dct(torch.eye(self.N)).data.t()
        elif self.type == 'idct':
            self.weight.data = idct(torch.eye(self.N)).data.t()
        self.weight.require_grad = False # don't learn this!

if __name__ == '__main__':
    x = torch.Tensor(1000,4096)
    x.normal_(0,1)
    dumb_dct = DumbDCT(4096, 'dct')
    assert torch.abs(dct(x) - dumb_dct(x)).mean() < 1e-3
    dumb_idct = DumbDCT(4096, 'idct')
    assert torch.abs(idct(x) - dumb_idct(x)).mean() < 1e-3

    import timeit
    print("CPU Speed (100 executions):")
    setup = "from __main__ import %s; import torch; x = torch.Tensor(1000,4096); x.normal_(0,1)"
    for d in ["dct1", "dct", "idct", "rfft"]:
        print("  %s: "%d, timeit.timeit("_ = %s(x)"%d, setup=setup%d, number=100))
    setup = "from __main__ import FasterDCT; import torch; x = torch.Tensor(1000,4096);  model = %s; model = model.eval(); x.normal_(0,1)"
    print("  FasterDCT: ", timeit.timeit("_ = model(x)", setup=setup%"FasterDCT(4096)", number=100))
    setup = "import torch; x = torch.Tensor(1000,4096);  model = %s; model = model.eval(); x.normal_(0,1)"
    print("  Linear: ", timeit.timeit("_ = model(x)", setup=setup%"torch.nn.Linear(4096,4096)", number=100))
    print("GPU speed (100 executions):")
    setup = "from __main__ import %s; import torch; x = torch.Tensor(1000,4096); x = x.to('cuda'); x.normal_(0,1)"
    for d in ["dct1", "dct", "idct", "rfft"]:
        print("  %s: "%d, timeit.timeit("_ = %s(x)"%d, setup=setup%d, number=100))
    setup = "from __main__ import FasterDCT; import torch; x = torch.Tensor(1000,4096);  model = %s; model = model.to('cuda').eval(); x = x.to('cuda'); x.normal_(0,1)"
    print("  FasterDCT: ", timeit.timeit("_ = model(x)", setup=setup%"FasterDCT(4096)", number=100))
    setup = "import torch; x = torch.Tensor(1000,4096);  model = %s; model = model.to('cuda').eval(); x = x.to('cuda'); x.normal_(0,1)"
    print("  Linear: ", timeit.timeit("_ = model(x)", setup=setup%"torch.nn.Linear(4096,4096)", number=100))
    setup = "from __main__ import DumbDCT; import torch; x = torch.Tensor(1000,4096);  model = %s; model = model.to('cuda').eval(); x = x.to('cuda'); x.normal_(0,1)"
    print("  Dense DCT: ", timeit.timeit("_ = model(x)", setup=setup%"DumbDCT(4096, 'dct')", number=100))


import numpy as np
import theano.tensor as T
from theano import function

# ALGEBRA
x = T.dmatrix('x')
y = T.dmatrix('y')
z = x + y
f = function([x, y], z)

# print(f(2, 3))
# print(numpy.allclose(f(16.3, 12.1), 28.4))
print(f([[1, 2], [3, 4]], [[10, 20], [30, 40]]))

# exercise
import theano
a = T.vector()                                  # declare variable
b = T.vector()                                  # declare variable
out = a ** 2 + b ** 2 + 2 * a * b               # build symbolic expression
f = function([a, b], out)                       # compile function
print(f([1, 2], [4, 5]))

###################################################
# OTHER EXAMPLES

# logistic function
x = T.dmatrix('x')
logistic_eq = 1 / (1 + T.exp(-x))
logistic = function([x], logistic_eq)
print(logistic([[0, 1], [-1, -2]]))

# multiple things calculation
a, b = T.dmatrices('a', 'b')
diff = a - b
abs_diff = abs(diff)
diff_squared = diff**2
f = function([a, b], [diff, abs_diff, diff_squared])
print(f([[1, 1], [1, 1]], [[0, 1], [2, 3]]))

# default value
c = T.matrix('c')
c = a + b
f = function([a, theano.In(b, value=[[1, 1], [1, 1]])], c)
print(f([[1, 1], [1, 1]]))

# accumulator
state = theano.shared([[0, 0], [0, 0]])
print("accumulator")
print(state.get_value())

state = theano.shared(np.matrix('0 0; 0 0', dtype=np.int32))
print(type(np.matrix('0 0; 0 0', dtype=np.int64)))
print(type(np.matrix('0 1; 2 3', dtype=np.int64)))
inc = T.imatrix('inc')
expression = state+inc
print(type(expression))
accumulator = function([inc], state, updates=[(state, state+inc)])

accumulator(np.matrix('1 2; 3 4', dtype=np.int32))
print(state.get_value())
accumulator(np.matrix('1 1; 1 1', dtype=np.int32))
print(state.get_value())

# function copy
print("function copy")
new_state = theano.shared(np.matrix('0 0; 0 0', dtype=np.int32))
new_accumulator = accumulator.copy(swap={state: new_state})
new_accumulator(np.matrix('1 2; 3 4', dtype=np.int32))
print(new_state.get_value())
print(state.get_value())

# random numbers
# POSSIBLE THAT THIS DOES NOT WORK ON GPU
print("random numbers")
srng = T.shared_randomstreams.RandomStreams(seed=234)
rv_u = srng.uniform((2, 2))
rv_n = srng.normal((2, 2))
f = function([], rv_u)
g = function([], rv_n, no_default_updates=True)     # Not updating rv_n.rng
nearly_zeros = function([], rv_u + rv_u - 2 * rv_u)

print(f())
print(f())

print(g())
print(g())

print("sharing streams between functions")
state_after_v0 = rv_u.rng.get_value().get_state()
# nearly_zeros()       # this affects rv_u's generator
v1 = f()
rng = rv_u.rng.get_value(borrow=True)
rng.set_state(state_after_v0)
rv_u.rng.set_value(rng, borrow=True)
v2 = f()             # v2 != v1
v3 = f()             # v3 == v1

print(v1)
print(v2)
print(v3)


"""
分段线性插值

"""
import sympy as sp
from sympy import Rational as r

def piecewise_linear_inter(X, Y):
    """
    分段线性插值
    :param X: 一系列x的一维向量
    :param Y: 一系列y的一维向量
    :return: 分段线性插值多项式
    """
    x = sp.Symbol('x')
    for i in range(len(X) - 1):  # i代表当前第几段
        print('在[{}, {}]上的线性插值为：'.format(X[i], X[i + 1]))
        S = (x - X[i + 1]) / (X[i] - X[i + 1]) * Y[i] + (x - X[i]) / (X[i + 1] - X[i]) * Y[i + 1]
        print(S)

if __name__ == '__main__':
    # 注意根号用sp.sqrt()来表示，如sp.sqrt(2)表示 根号2
    # 分数用r(..)表示，如r(1 / 2)表示分式 1 / 2
    X = [r(30), r(45), r(60), r(90)]
    Y = [r(0.5), sp.sqrt(2) / 2, sp.sqrt(3) / 2, r(1)]
    piecewise_linear_inter(X, Y)


import numpy as np

def hms2dec(h,m,s):
  return 15*(h + (m/60) + (s/3600))

def dms2dec(d,m,s):
  if d>=0:
    return (d + (m/60) + (s/3600))
  return (d - (m/60) - (s/3600))

def angular_dist(r1, d1, r2, d2):
  r1= np.radians(r1)
  r2= np.radians(r2)
  d1= np.radians(d1)
  d2= np.radians(d2)
  a = (np.sin(np.abs(d1 - d2)/2))**2
  b = np.cos(d1)*np.cos(d2)*np.sin(np.abs(r1 - r2)/2)**2
  d = np.degrees(2*np.arcsin(np.sqrt(a + b)))
  return d

def import_bss():
  cat = np.loadtxt('bss.dat', usecols=range(1, 7))
  loaded = []
  count = 1
  for i in cat:
    loaded.append((count, hms2dec(i[0],i[1],i[2]), dms2dec(i[3],i[4],i[5])))
    count+=1
  return loaded

def import_super():
  cat = np.loadtxt('super.csv', delimiter=',', skiprows=1, usecols=[0, 1])
  loaded = []
  count = 1
  for i in cat:
    loaded.append((count, i[0], i[1]))
    count+=1
  return loaded

def find_closest(catalogue,target_ra, target_dec):
  min_distance = 9999999
  ID = 0
  for i in catalogue:
    distance = angular_dist(i[1], i[2], target_ra, target_dec)
    if distance<min_distance:
      min_distance = distance
      ID = i[0]
  return (ID,min_distance)

if __name__ == '__main__':
  cat = import_bss()
  print(find_closest(cat, 175.3, -32.5))
  print(find_closest(cat, 32.2, 40.7))


"""
  elementary ballistic problem
"""

import numpy as np
from scipy import constants
import helper

### initial conditions ###

# initial position
x0 = 0
y0 = 125
s0 = (x0,y0)
# scalar inital speed
v0s = 12
# angle
theta = np.pi/6

# define acceleration vector - acceleration is constant here
g = constants.physical_constants['standard acceleration of gravity'][0]
a = np.array([0, -g ])

### end initial conditions ###

# from initial condition get vector form of speed
v0 = helper.polarToCartesian(np.array([v0s, theta]))

print "vo"
print v0

print "a:  "+str(a)
print "v0: "+str(v0)
print "s0: "+str(s0)

"""
  speed in function of time

  @arg t: time
  @return v: speed
"""
def v(t):
  return a*t +v0

"""
  position in function of time

  @arg t: time
  @return s: position
"""
def s(t):
  return .5*a*(t**2) + v0*t + s0

"""
  find time in function of x
  @return t
  @param x: position
"""
def tFromX(x):
  return (x-s0[0])/v0[0]

"""
  get position (x and y) from x
"""
def sfromX(x):
  return s(tFromX(x))

"""
  find time in function of y
  @return t
  @param y: position
"""
def tFromY(y):
  s = helper.solve2ndDegEq(.5*a[1], v0[1], s0[1]-y) # solve 2nd deg equation
  f = max(s) # take max of two results (most likely is the other negative)
  return f.real # and return real part to avoid confussion

"""
  find t that maximizes height
"""
tMaxHeight = -v0[1]/a[1]

"""
  computes associated position
"""
sMaxHeight = s(tMaxHeight)
print "Missile reaches highest point at time "+str(tMaxHeight)+" and position "+str(sMaxHeight)

"""
  find t when y=0
"""

tZeroHeight = tFromY(0)
sZeroHeight = s(tZeroHeight)

print "Missile reaches floor at time "+str(tZeroHeight)+" and position "+str(sZeroHeight)

#print tFromX(6.3561497)
#print sfromX(6.3561497)
#print tFromY(126.83486239)

import matplotlib.pyplot as plt
t = np.arange(0, tZeroHeight, .0002)
t = t.reshape(len(t),-1).transpose().transpose()

labels = ['x trajectory', 'y trajectory']

print s(t)

for y, label in zip(s(t).transpose(), labels):
  plt.plot(t, y, label = label)

plt.xlabel('time [s]')
plt.ylabel('trajectory [m]')
plt.legend()
plt.savefig("ballistic.png")
plt.show()

#print s


"""Implementation of K-Means Clustering."""

import numpy as np

from .base import Estimator
from .util import validate_data, normalize

class KMeans(Estimator):
    """Perform K-Means Clustering."""

    def __init__(self, k=2, epsilon=(2 ** -23), seed=42):
        """Initialize KMeans.

        Parameters
        ----------
        k : int
            Number of clusters to find.
        epsilon : float
            Tolerance for cluster center changes between iterations. Once
            centers move less than this, model fitting ceases.
        seed : int
            Random number generator seed.
        """
        self._k = k
        self._epsilon = epsilon
        self._seed = seed
        self._centers = None
        self._iterations = None

    def fit(self, X, y=None):
        """Fit KMeans on a feature matrix.

        Parameters
        ----------
        X : array-like {n_samples, n_features}
            A feature matrix.
        y : array-like {n_samples, }
            A target vector.
        """
        np.random.seed(self._seed)
        validate_data(X)
        X = normalize(X)

        init_centers_indx = np.random.choice(list(range(X.shape[0])), self._k)
        init_centers = X[init_centers_indx, :]
        self._centers = self._find_centers(X, init_centers)

        return self

    def _find_centers(self, X, centers, iteration=1):
        """Determine the k centers for the dataset, recursively.

        Parameters
        ----------
        X : array-like {n_samples, n_features}
            Feature matrix.
        centers : array-like {k, }
            Centers of each cluster.
        iteration : int
            Current iteration count.

        Returns
        -------
        Centers of each cluster.
        """
        # Mark each sample as belonging to a cluster
        labels = np.array([
            self._closest_center(sample, centers)
            for sample in iter(X)
        ])
        # Find new centers from the samples' labels
        new_centers = np.array([
            np.mean(X[labels == label], axis=0)
            for label in range(self._k)
        ])
        # Check if the centers have moved significantly from the last iteration
        center_deltas = np.linalg.norm(new_centers - centers, axis=0)

        if all(center_deltas < self._epsilon):
            self._iterations = iteration
            return new_centers
        else:
            return self._find_centers(X, new_centers, iteration + 1)

    @staticmethod
    def _closest_center(sample, centers):
        """Find the center that is closest to a sample from the dataset.

        Parameters
        ----------
        sample : array-like {n_features, }
            A single sample from the dataset.
        centers : array-like {k, }
            The current k centers of each cluster.

        Returns
        -------
        Index of the closest center.
        """
        # Find distance from the sample to each center
        distances = [np.linalg.norm(sample - center) for center in centers]
        return distances.index(min(distances))  # Pick the closest center

    def predict(self, X):
        """Predict labels for the feature matrix based on the training data.

        Parameters
        ----------
        X : array-like {n_samples, n_features}
            A feature matrix.

        Return
        ------
        Predicted classes.
        """
        if self._centers is None:
            raise ValueError('KMeans must be fit first.')

        validate_data(X)
        X = normalize(X)

        return np.array([
            self._closest_center(sample, self._centers)
            for sample in iter(X)
        ])


import numpy as np
from scipy.misc import factorial as ft
from scipy.misc import comb as nCk

#=====================================================================================================

def calc_bez_coeffs(T, p0, p1):
    # Bezier curve interpolation
    # T-conversion factor, dp/dt = dp/ds*1/T
    # p0, p1 are start/end points: x, xd, xdd, xddd, xdddd
    d = 4 # number of derivatives of p0, p1
    n = 9 # degree of curve
    Cout = np.zeros((n+1,3))

    # note: interpolation is done in [0,1], then converted to [0,T]
    for i in range(3):
        # cal x,y,z dimensions seperately
        CL = np.zeros((d+1,(n+1)/2))
        CR = np.zeros((d+1,(n+1)/2))

        x0 = np.array([p0[0,i],p0[1,i]*T,p0[2,i]*T**2,p0[3,i]*T**3,p0[4,i]*T**4])
        x1 = np.array([p1[0,i],p1[1,i]*T,p1[2,i]*T**2,p1[3,i]*T**3,p1[4,i]*T**4])

        CL[:,0] = np.multiply(ft(n-np.arange(d+1))/ft(n),x0)
        for j in range(d):
            CL[0:d-j,j+1] = CL[0:d-j,j] + CL[1:d-j+1,j]

        CR[:,-1] = np.multiply(ft(n-np.arange(d+1))/ft(n),x1)
        for j in range(d):
            CR[0:d-j,-2-j] = CR[0:d-j,-2-j+1] - CR[1:d-j+1,-2-j+1]

        Cout[:,i] = np.transpose(np.hstack((CL[0,:],CR[0,:])))
    return Cout

#=====================================================================================================

def state_from_bez(T, t, Cin):
    # Extract nominal state from Bezier control points
    d = 4 # derivative degree of Cin
    n = 9 # degree of curve
    tn = min(t/T,1) # normalized time, [0,1]
    Ctemp = np.zeros((d+1,n+1))
    phat  = np.zeros((d+1,3))

    for i in range(3):
        # cal 3 dimensional nominal states
        Ctemp[0,:] = np.transpose(Cin[:,i])
        for j in range(d):
            if j == 0:
                Ctemp[  1,   0:-1] = Ctemp[j,   1:] - Ctemp[j,   0:-1]
            else:
                Ctemp[j+1, 0:-1-j] = Ctemp[j, 1:-j] - Ctemp[j, 0:-1-j]
        # cal 0-4th order derivatives
        for k in range(d+1):
            temp = 0
            for kk in range(n-k+1):
                Btemp = nCk(n-k,kk)*(tn**kk)*((1-tn)**(n-k-kk))
                temp  = temp + ft(n)/ft(n-k)*Ctemp[k,kk]*Btemp
            phat[k,i] = temp/T**k
    return phat


import numpy as np

class MinMaxScaler:
    """Transform features by scaling each feature to a given range.
    This estimator scales and translates each feature individually such
    that it is in the given range on the training set, e.g. between
    zero and one.
    The transformation is given by::
        X_std = (X - X.min(axis=0)) / (X.max(axis=0) - X.min(axis=0))
        X_scaled = X_std * (max - min) + min
    where min, max = feature_range.
    The transformation is calculated as::
        X_scaled = scale * X + min - X.min(axis=0) * scale
        where scale = (max - min) / (X.max(axis=0) - X.min(axis=0))
    This transformation is often used as an alternative to zero mean,
    unit variance scaling.

    Parameters
    ----------
    feature_range : tuple (min, max), default=(0, 1)
        Desired range of transformed data.
    """
    def __init__(self, feature_range=(0, 1)):
        self.feature_range = feature_range

    def fit(self, X):
        """Compute the minimum and maximum to be used for later scaling.

        Parameters
        ----------
        X : array-like of shape (n_samples, n_features)
            The data used to compute the per-feature minimum and maximum
            used for later scaling along the features axis.
        """
        data_min = np.nanmin(X, axis=0)
        data_max = np.nanmax(X, axis=0)

        data_range = (data_max - data_min)

        self.scale_ = (self.feature_range[1] - self.feature_range[0]) / data_range
        self.min_ = self.feature_range[0] - data_min * self.scale_
        self.data_range_ = data_range
        self.data_min_ = data_min
        self.data_max_ = data_max

    def transform(self, X):
        """Scale features of X according to feature_range.

        Parameters
        ----------
        X : array-like of shape (n_samples, n_features)
            Input data that will be transformed.

        Returns
        -------
        Xt : array-like of shape (n_samples, n_features)
            Transformed data.
        """
        X = np.asarray(X, dtype=np.float64)
        X *= self.scale_
        X += self.min_
        return X

    def fit_transform(self, X):
        """Compute the minimum and maximum and Scale features of X according to feature_range.

        Parameters
        ----------
        X : array-like of shape (n_samples, n_features)
            Input data that will be transformed.

        Returns
        -------
        Xt : array-like of shape (n_samples, n_features)
            Transformed data.
        """
        X = np.asarray(X, dtype=np.float64)
        self.fit(X)
        return self.transform(X)

    def inverse_transform(self, X):
        """Undo the scaling of X according to feature_range.

        Parameters
        ----------
        X : array-like of shape (n_samples, n_features)
            Input data that will be transformed.

        Returns
        -------
        Xt : array-like of shape (n_samples, n_features)
            Transformed data.
        """
        X = np.asarray(X, dtype=np.float64)
        X -= self.min_
        X /= self.scale_
        return X

class RobustScaler:
    """Scale features using statistics that are robust to outliers.
    This Scaler removes the median and scales the data according to
    the quantile range (defaults to IQR: Interquartile Range).
    The IQR is the range between the 1st quartile (25th quantile)
    and the 3rd quartile (75th quantile).
    Centering and scaling happen independently on each feature by
    computing the relevant statistics on the samples in the training
    set. Median and interquartile range are then stored to be used on
    later data using the ``transform`` method.

    Parameters
    ----------
    with_centering : boolean, True by default
        If True, center the data before scaling.
    with_scaling : boolean, True by default
        If True, scale the data to interquartile range.
    quantile_range : tuple (q_min, q_max), 0.0 < q_min < q_max < 100.0
        Default: (25.0, 75.0) = (1st quantile, 3rd quantile) = IQR
        Quantile range used to calculate ``scale_``.
    """
    def __init__(self, with_centering=True, with_scaling=True, quantile_range=(25.0, 75.0)):
        self.with_centering = with_centering
        self.with_scaling = with_scaling
        self.quantile_range = quantile_range

    def fit(self, X):
        """Compute the median and quantiles to be used for scaling.
        Parameters
        ----------
        X : array-like, shape [n_samples, n_features]
            The data used to compute the median and quantiles
            used for later scaling along the features axis.
        """
        q_min, q_max = self.quantile_range
        self.center_ = np.nanmedian(X, axis=0) if self.with_centering else None

        if self.with_scaling:
            quantiles = []
            for feature_idx in range(X.shape[1]):
                column_data = X[:, feature_idx]
                quantiles.append(np.nanpercentile(column_data, self.quantile_range))

            quantiles = np.transpose(quantiles)
            self.scale_ = quantiles[1] - quantiles[0]
        else:
            self.scale_ = None

        return self

    def transform(self, X):
        """Center and scale the data.
        Parameters
        ----------
        X : array-like
            The data used to scale along the specified axis.
        """
        X = np.asarray(X, dtype=np.float64)

        if self.with_centering:
            X -= self.center_
        if self.with_scaling:
            X /= self.scale_

        return X

    def fit_transform(self, X):
        """Compute the median and quantiles to be used for scaling and center and scale the data.

        Parameters
        ----------
        X : array-like
            The data used to scale along the specified axis.
        """
        X = np.asarray(X, dtype=np.float64)

        self.fit(X)
        return self.transform(X)

    def inverse_transform(self, X):
        """Scale back the data to the original representation

        Parameters
        ----------
        X : array-like
            The data used to scale along the specified axis.
        """
        X = np.asarray(X, dtype=np.float64)

        if self.with_scaling:
            X *= self.scale_
        if self.with_centering:
            X += self.center_

        return X


from __future__ import division
import numpy as np
from LinearSolve import LinearSolver
from utils import Grapher

class Regressor():
    def __init__(self, x, y, dg):
        self.x = x
        self.y = y
        self.N = x.size
        self.dg = dg
        self.c = None
        self.cod = None
        self.f = None

    def regress(self):
        m = self.dg + 1
        sigma_x = np.array([np.sum(self.x**i) for i in range(m * 2 + 1)])
        print sigma_x
        a, b = np.ndarray(shape=(m, m)), np.ndarray(shape=(m))
        for i in range(m):
            a[i] = sigma_x[i:i + m]
            b[i] = np.sum((self.x**i) * self.y)

        solver = LinearSolver(a, b)
        solver.GE_pivoted()
        self.c = solver.x

        def func(pt):
            return np.sum(self.c * np.array([pt**i for i in range(m)]))
        self.f = func

        ''' coefficient of determination '''
        y_mean = np.mean(self.y)
        st = np.sum((self.y - y_mean)**2)
        sr = np.sum((self.y - map(func, self.x))**2)
        self.cod = (st - sr) / st

def main():

    degree = int(raw_input('Enter the degree of Polynomial:'))

    with open("q2.txt", "r") as f:
        n = int(f.readline())
        a = np.array([map(float, f.readline().split()) for i in range(n)])
        x, y = a[..., 0], a[..., 1]

    func = Regressor(x, y, degree)
    func.regress()
    Grapher.plot(func.f, np.amin(func.x), np.amax(func.x), x=func.x, y=func.y)

    with open("out.txt", "w") as fout:
        if func.c is not None:
            np.savetxt(fout, func.c, fmt='%f', header='Coefficients of the polynomial(degree %d):\n' %
                       degree, footer='\n\n', comments='', newline=' ')
            fout.write('Coefficient of determination= %f\n' % func.cod)

if __name__ == '__main__':
    main()


import scipy.stats as ss
import numpy as np
import statsmodels.stats.api as sms
from math import sqrt

# import data

def getdata(path):
    datalist =[]
    data = open(path, "r")
    for line in data:
        line = line.rstrip()
        line = float(line)
        datalist.append(line)
    data.close()
    return datalist

#inputs für data

path_a = input("Data A? (Example: data_A.txt): ")
path_b = input("Data B? (Example: data_B.txt): ")

#hardcoded für tests zum zeitsparen
#path_a = "data_A.txt"
#path_b = "data_B.txt"

data_a = getdata(path_a)
data_b = getdata(path_b)

#data_a = [0.03,0.91,0.64,0.99,0.64,0.16,0.16,0.91,0.16,0.27]
#data_b = [0.64,0.08,0.16,0.27,0.02,0.01,0.16,0.03,0.03,0.64]
#alpha = 0.05
check = False
while check == False:
    alpha = float(input("Which Alpha? (Value between 0 and 1): "))
    if alpha >0 and alpha < 1:
        check = True
    else:
        print("Alpha out of bounds. Please try again.")

#print(path_a)
#print(path_b)

#print(alpha)

#######################################################################
#sort data:
data_a_sorted = sorted(data_a)
data_b_sorted = sorted(data_b)

#print data:
print("*****************Data*****************")
print("Datapoints A:", data_a)
print("Datapoints B:", data_b)
print("")

#print sorted data:
print("*****************Sorted Data*****************")
print("Sorted Data A:", data_a_sorted)
print("Sorted Data B:", data_b_sorted)
print("")

#######################################################################

def ranks(a, b):
    ranks_a = []
    ranks_b = []

    sequence_a = sorted(a)
    sequence_b = sorted(b)

    sequence_both = sorted(a+b)
    ranks_both = ss.rankdata(sequence_both)

    for i in range(len(sequence_a)):
        for j in range(len(sequence_both)):
            if sequence_a[i] == sequence_both[j]:
                ranks_a.append(ranks_both[j])
                break
        continue

    for i in range(len(sequence_b)):
        for j in range(len(sequence_both)):
            if sequence_b[i] == sequence_both[j]:
                ranks_b.append(ranks_both[j])
                break
        continue

    return ranks_a, ranks_b

ranksA, ranksB = ranks(data_a,data_b)
print("***************Ranks per datalist***************")
print("a:", data_a_sorted)
print("ranks a:", list(ss.rankdata(sorted(data_a))))
print("")
print("b:", data_b_sorted)
print("ranks b:", list(ss.rankdata(sorted(data_b))))
print("")
print("*****************Ranks combined*****************")
print("a:", data_a_sorted)
print("ranks a:", ranksA)
print("")
print("b:", data_b_sorted)
print("ranks b:", ranksB)
print("")

#######################################################################
#Tests:

def test_all(a, b, alpha = 0.05):
    mean_a = np.mean(a)
    mean_b = np.mean(b)

    std_dev_a = np.std(a, ddof=1)
    std_dev_b = np.std(b, ddof=1)

    percentile2_5_a = np.percentile(a,2.5)
    if len(a)*0.025 < 1:
        percentile2_5_a = sorted(a)[0]
    percentile25_a = np.percentile(a,25)
    percentile50_a = np.percentile(a,50)
    percentile75_a = np.percentile(a,75)
    percentile97_5_a = np.percentile(a,97.5)
    if len(a)*0.975 > len(a)-1:
        percentile97_5_a = sorted(a)[len(a)-1]

    percentile2_5_b = np.percentile(b,2.5)
    if len(b)*0.025 < 1:
        percentile2_5_b = sorted(b)[0]
    percentile25_b = np.percentile(b ,25)
    percentile50_b = np.percentile(b,50)
    percentile75_b = np.percentile(b,75)
    percentile97_5_b = np.percentile(b,97.5)
    if len(b)*0.975 > len(b)-1:
        percentile97_5_b = sorted(b)[len(b)-1]

    ci_lower_a, ci_upper_a = sms.DescrStatsW(a).tconfint_mean(alpha)
    ci_lower_b, ci_upper_b = sms.DescrStatsW(b).tconfint_mean(alpha)

    #ts ci intervalls:
    rankedA = list(ss.rankdata(sorted(a)))
    rankedB = list(ss.rankdata(sorted(b)))
    ci_lower_a_rank, ci_upper_a_rank = sms.DescrStatsW(rankedA).tconfint_mean(alpha)
    ci_upper_a_rank = int(round(ci_upper_a_rank))
    ci_lower_a_rank = int(round(ci_lower_a_rank))
    ci_lower_a_rank_value = sorted(a)[ci_lower_a_rank-1]
    ci_upper_a_rank_value = sorted(a)[ci_upper_a_rank-1]
    ci_lower_b_rank, ci_upper_b_rank = sms.DescrStatsW(rankedB).tconfint_mean(alpha)
    ci_upper_b_rank = int(round(ci_upper_b_rank))
    ci_lower_b_rank = int(round(ci_lower_b_rank))
    ci_lower_b_rank_value = sorted(b)[ci_lower_b_rank-1]
    ci_upper_b_rank_value = sorted(b)[ci_upper_b_rank-1]

    #sharpio-wilk-test:
    w_value_a, w_p_value_a = ss.shapiro(a)
    w_value_b, w_p_value_b = ss.shapiro(b)
    #Kolmogorov–Smirnov-Test(normality):
    ksscore_a, ks_p_value_a = ss.kstest(a,"norm")
    ksscore_b, ks_p_value_b = ss.kstest(b, "norm")
    #Anderson–Darling-Test:
    anderson_a ,_, _ = ss.anderson(a)
    anderson_b, _, _ = ss.anderson(b)

    ###interset tests

    #ttest norm same sigma:
    tscore_norm_same, pvalue_norm_same = ss.ttest_ind(a,b)
    tscore_norm_diff, pvalue_norm_diff = ss.ttest_ind(a, b,equal_var=False)

    #manwhitneyu test
    u_value, pvalue_mwu = ss.mannwhitneyu(a,b)
    pvalue_mwu = pvalue_mwu*2

    #Kolmogorov–Smirnov-Test:
    ksscore, ks_p_value = ss.ks_2samp(a,b)

    #cohens d:
    cohens_d = (mean_a - mean_b) / (sqrt((std_dev_a ** 2 + std_dev_b ** 2) / 2))

    #hedges g:
    hedges_g = cohens_d*(1-(3/(4*(len(a)+len(b))-9)))

    #glass's delta:
    if len(a) > len(b):
        glass_delta = (mean_b-mean_a)/(std_dev_a)
    else:
        glass_delta = (mean_a-mean_b)/(std_dev_b)

    return (mean_a, mean_b, std_dev_a, std_dev_b, percentile2_5_a, percentile25_a, percentile50_a,
            percentile75_a, percentile97_5_a, percentile2_5_b, percentile25_b, percentile50_b,
            percentile75_b, percentile97_5_b, ci_lower_a, ci_upper_a, ci_lower_b, ci_upper_b,
            tscore_norm_same,pvalue_norm_same,tscore_norm_diff,pvalue_norm_diff, u_value,pvalue_mwu,
            w_value_a, w_p_value_a, w_value_b, w_p_value_b,ksscore,ks_p_value,ksscore_a,ksscore_b,
            ks_p_value_a,ks_p_value_b,anderson_a,anderson_b,cohens_d,hedges_g, glass_delta,
            ci_lower_a_rank, ci_upper_a_rank, ci_lower_b_rank, ci_upper_b_rank, ci_lower_a_rank_value,
            ci_upper_a_rank_value, ci_lower_b_rank_value, ci_upper_b_rank_value)

#Rankedbased Tests:

def test_all2(a, b, alpha = 0.05):
    rankA, rankB = ranks(a,b)

    mean_a = np.mean(rankA)
    mean_b = np.mean(rankB)

    std_dev_a = np.std(rankA, ddof=1)
    std_dev_b = np.std(rankB, ddof=1)

    percentile2_5_a = np.percentile(rankA,2.5)
    if len(rankA)*0.025 < 1:
        percentile2_5_a = rankA[0]
    percentile25_a = np.percentile(rankA,25)
    percentile50_a = np.percentile(rankA,50)
    percentile75_a = np.percentile(rankA,75)
    percentile97_5_a = np.percentile(rankA,97.5)
    if len(rankA)*0.975 > len(rankA)-1:
        percentile97_5_a = rankA[len(rankA)-1]

    percentile2_5_b = np.percentile(rankB,2.5)
    if len(rankB)*0.025 < 1:
        percentile2_5_b = rankB[0]
    percentile25_b = np.percentile(rankB,25)
    percentile50_b = np.percentile(rankB,50)
    percentile75_b = np.percentile(rankB,75)
    percentile97_5_b = np.percentile(rankB,97.5)
    if len(rankB)*0.975 > len(rankB)-1:
        percentile97_5_b = rankB[len(rankB)-1]

    ci_lower_a, ci_upper_a = sms.DescrStatsW(rankA).tconfint_mean(alpha)
    ci_lower_b, ci_upper_b = sms.DescrStatsW(rankB).tconfint_mean(alpha)

    #ts ci intervalls:
    rankedA = list(ss.rankdata(sorted(rankA)))
    rankedB = list(ss.rankdata(sorted(rankB)))
    ci_lower_a_rank, ci_upper_a_rank = sms.DescrStatsW(rankedA).tconfint_mean(alpha)
    ci_upper_a_rank = int(round(ci_upper_a_rank))
    ci_lower_a_rank = int(round(ci_lower_a_rank))
    ci_lower_a_rank_value = sorted(rankA)[ci_lower_a_rank-1]
    ci_upper_a_rank_value = sorted(rankA)[ci_upper_a_rank-1]
    ci_lower_b_rank, ci_upper_b_rank = sms.DescrStatsW(rankedB).tconfint_mean(alpha)
    ci_upper_b_rank = int(round(ci_upper_b_rank))
    ci_lower_b_rank = int(round(ci_lower_b_rank))
    ci_lower_b_rank_value = sorted(rankB)[ci_lower_b_rank-1]
    ci_upper_b_rank_value = sorted(rankB)[ci_upper_b_rank-1]

    #sharpio-wilk-test:
    w_value_a, w_p_value_a = ss.shapiro(rankA)
    w_value_b, w_p_value_b = ss.shapiro(rankB)
    #Kolmogorov–Smirnov-Test(normality):
    ksscore_a, ks_p_value_a = ss.kstest(rankA,"norm")
    ksscore_b, ks_p_value_b = ss.kstest(rankB, "norm")
    #Anderson–Darling-Test:
    anderson_a ,_, _ = ss.anderson(a)
    anderson_b, _, _ = ss.anderson(b)

    ###interset tests:

    #ttest norm same sigma:
    tscore_norm_same, pvalue_norm_same = ss.ttest_ind(rankA,rankB)
    tscore_norm_diff, pvalue_norm_diff = ss.ttest_ind(rankA, rankB, equal_var=False)

    #manwhitneyu test
    u_value, pvalue_mwu = ss.mannwhitneyu(rankA,rankB)
    pvalue_mwu = pvalue_mwu*2

    #Kolmogorov–Smirnov-Test:
    ksscore, ks_p_value = ss.ks_2samp(rankA,rankB)

    #cohens d:
    cohens_d = (mean_a - mean_b) / (sqrt((std_dev_a ** 2 + std_dev_b ** 2) / 2))

    #hedges g:
    hedges_g = cohens_d*(1-(3/(4*(len(rankA)+len(rankB))-9)))

    #glass's delta:
    if len(a) > len(b):
        glass_delta = (mean_b-mean_a)/(std_dev_a)
    else:
        glass_delta = (mean_a-mean_b)/(std_dev_b)

    return (mean_a, mean_b, std_dev_a, std_dev_b, percentile2_5_a, percentile25_a, percentile50_a,
            percentile75_a, percentile97_5_a, percentile2_5_b, percentile25_b, percentile50_b,
            percentile75_b, percentile97_5_b, ci_lower_a, ci_upper_a, ci_lower_b, ci_upper_b,
            tscore_norm_same,pvalue_norm_same,tscore_norm_diff,pvalue_norm_diff, u_value,pvalue_mwu,
            w_value_a,w_p_value_a,w_value_b,w_p_value_b,ksscore,ks_p_value,ksscore_a,ksscore_b,
            ks_p_value_a,ks_p_value_b,anderson_a,anderson_b,cohens_d,hedges_g, glass_delta,
            ci_lower_a_rank, ci_upper_a_rank, ci_lower_b_rank, ci_upper_b_rank, ci_lower_a_rank_value,
            ci_upper_a_rank_value, ci_lower_b_rank_value, ci_upper_b_rank_value)

#print results

def print_results_raw(alpha,mean_a,mean_b,std_dev_a, std_dev_b, percentile2_5_a, percentile25_a,
                      percentile50_a, percentile75_a, percentile97_5_a, percentile2_5_b,
                      percentile25_b, percentile50_b, percentile75_b, percentile97_5_b, ci_lower_a,
                      ci_upper_a, ci_lower_b, ci_upper_b, tscore_norm_same,pvalue_norm_same,
                      tscore_norm_diff, pvalue_norm_diff, u_value,pvalue_mwu,w_value_a,w_p_value_a,
                      w_value_b,w_p_value_b,ksscore,ks_p_value,ksscore_a,ksscore_b,
                      ks_p_value_a,ks_p_value_b,anderson_a,anderson_b,cohens_d,hedges_g, glass_delta,
                      ci_lower_a_rank, ci_upper_a_rank, ci_lower_b_rank, ci_upper_b_rank, ci_lower_a_rank_value,
                      ci_upper_a_rank_value, ci_lower_b_rank_value, ci_upper_b_rank_value):
    print("*****************Full analysis of raw data*****************")
    print("Alpha:\t\t\t\t\t\t\t", alpha)
    # Sequence A:
    print("")
    print("***Sequence A:")
    print("")
    print("Mean:\t\t\t\t\t\t\t", round(mean_a, 4))
    print("Std. Deviation:\t\t\t\t\t\t", round(std_dev_a, 4))
    print("2.5th Percentile:\t\t\t\t\t", round(percentile2_5_a,4))
    print("25th Percentile 25%:\t\t\t\t\t", round(percentile25_a,4))
    print("50th Percentile 50%:\t\t\t\t\t", round(percentile50_a,4))
    print("75th Percentile 75%:\t\t\t\t\t", round(percentile75_a,4))
    print("97.5th Percentile 97,5%:\t\t\t\t", round(percentile97_5_a,4))
    print("CI Lower:\t\t\t\t\t\t", round(ci_lower_a,4))
    print("CI Upper:\t\t\t\t\t\t", round(ci_upper_a,4))
    print("CI Thompson Savur Lower Rank:\t\t\t\t", ci_lower_a_rank)
    print("CI Thompson Savur Lower Value:\t\t\t\t", round(ci_lower_a_rank_value,4))
    print("CI Thompson Savur Upper Rank:\t\t\t\t", ci_upper_a_rank)
    print("CI Thompson Savur Upper Value:\t\t\t\t", round(ci_upper_a_rank_value,4))
    print("")
    print("###Normality Tests:")
    print("#sharpio-wilk-test:")
    print("p-value:\t\t\t\t\t\t", round(w_p_value_a,4))
    print("W:\t\t\t\t\t\t\t", round(w_value_a,4))
    print("#Kolmogorov Smirnov Test:")
    print("p-value:\t\t\t\t\t\t",round(ks_p_value_a,4))
    print("ksscore:\t\t\t\t\t\t",round(ksscore_a,4))
    print("#Anderson–Darling-Test:")
    print("A:\t\t\t\t\t\t\t", round(anderson_a,4))

    # Sequence B:
    print("")
    print("***Sequence B:")
    print("")
    print("Mean:\t\t\t\t\t\t\t", round(mean_b, 4))
    print("Std. Deviation:\t\t\t\t\t\t", round(std_dev_b, 4))
    print("2.5th Percentile:\t\t\t\t\t",round(percentile2_5_b,4))
    print("25th Percentile 25%:\t\t\t\t\t", round(percentile25_b,4))
    print("50th Percentile 50%:\t\t\t\t\t", round(percentile50_b,4))
    print("75th Percentile 75%:\t\t\t\t\t", round(percentile75_b,4))
    print("97.5th Percentile 97,5%:\t\t\t\t",round(percentile97_5_b,4))
    print("CI Lower:\t\t\t\t\t\t", round(ci_lower_b,4))
    print("CI Upper:\t\t\t\t\t\t", round(ci_upper_b,4))
    print("CI Thompson Savur Lower Rank:\t\t\t\t", ci_lower_b_rank)
    print("CI Thompson Savur Lower Value:\t\t\t\t", round(ci_lower_b_rank_value,4))
    print("CI Thompson Savur Upper Rank:\t\t\t\t", ci_upper_b_rank)
    print("CI Thompson Savur Upper Value:\t\t\t\t", round(ci_upper_b_rank_value,4))
    print("")
    print("###Normality Tests:")
    print("#sharpio-wilk-test:")
    print("p-value:\t\t\t\t\t\t", round(w_p_value_b,4))
    print("W:\t\t\t\t\t\t\t", round(w_value_b,4))
    print("#Kolmogorov Smirnov Test:")
    print("p-value:\t\t\t\t\t\t",round(ks_p_value_b,4))
    print("ksscore:\t\t\t\t\t\t",round(ksscore_b,4))
    print("#Anderson–Darling-Test:")
    print("A:\t\t\t\t\t\t\t", round(anderson_b,4))

    print("")
    print("***Interset Tests")

    print("")
    print("T-test und p-Value:")
    print("asume same sigma:")
    print("t-score:\t\t\t\t\t\t", round(tscore_norm_same,4))
    print("p-value:\t\t\t\t\t\t", round(pvalue_norm_same,4))
    print("asume diffrent sigma:")
    print("t-score:\t\t\t\t\t\t", round(tscore_norm_diff,4))
    print("p-value:\t\t\t\t\t\t", round(pvalue_norm_diff,4))

    print("")
    print("Manwhitney-U-Test:")
    print("p-value:\t\t\t\t\t\t",round(pvalue_mwu,4))
    print("U:\t\t\t\t\t\t\t",round(u_value,4))

    print("")
    print("Kolmogorov Smirnov Test:")
    print("p-value:\t\t\t\t\t\t", round(ks_p_value,4))
    print("ks score:\t\t\t\t\t\t", round(ksscore,4))

    print("")
    print("Cohens-D:")
    print("d measure:\t\t\t\t\t\t", round(cohens_d,4))

    print("")
    print("Hedge's g:")
    print("g measure:\t\t\t\t\t\t", round(hedges_g,4))

    print("")
    print("Glass's delta:")
    print("delta:\t\t\t\t\t\t\t", round(glass_delta,4))

    print("")
    print("***************Full analysis of raw data done***************")
    print("")
    print("")

def print_results_ranks(alpha, mean_a, mean_b, std_dev_a, std_dev_b, percentile2_5_a, percentile25_a,
                        percentile50_a, percentile75_a, percentile97_5_a, percentile2_5_b,
                        percentile25_b, percentile50_b, percentile75_b, percentile97_5_b,
                        ci_lower_a, ci_upper_a, ci_lower_b, ci_upper_b, tscore_norm_same,
                        pvalue_norm_same,tscore_norm_diff,pvalue_norm_diff, u_value,pvalue_mwu,
                        w_value_a,w_p_value_a,w_value_b,w_p_value_b,ksscore,ks_p_value,
                        ksscore_a, ksscore_b, ks_p_value_a, ks_p_value_b,anderson_a,anderson_b,
                        cohens_d,hedges_g, glass_delta,ci_lower_a_rank, ci_upper_a_rank,
                        ci_lower_b_rank, ci_upper_b_rank, ci_lower_a_rank_value,
                        ci_upper_a_rank_value, ci_lower_b_rank_value, ci_upper_b_rank_value):
    print("*****************Full analysis of rank data*****************")
    print("Alpha:\t\t\t\t\t\t\t", alpha)

    #Sequence A:
    print("")
    print("***Sequence A:")
    print("")
    print("Mean:\t\t\t\t\t\t\t",round(mean_a, 4))
    print("Std. Deviation:\t\t\t\t\t\t", round(std_dev_a, 4))
    print("2.5th Percentile:\t\t\t\t\t", round(percentile2_5_a,4))
    print("25th Percentile 25%:\t\t\t\t\t", round(percentile25_a,4))
    print("50th Percentile 50%:\t\t\t\t\t", round(percentile50_a,4))
    print("75th Percentile 75%:\t\t\t\t\t", round(percentile75_a,4))
    print("97.5th Percentile 97,5%:\t\t\t\t", round(percentile97_5_a,4))
    print("CI Lower:\t\t\t\t\t\t", round(ci_lower_a,4))
    print("CI Upper:\t\t\t\t\t\t", round(ci_upper_a,4))
    print("CI Thompson Savur Lower Rank:\t\t\t\t", ci_lower_a_rank)
    print("CI Thompson Savur Lower Value:\t\t\t\t", round(ci_lower_a_rank_value,4))
    print("CI Thompson Savur Upper Rank:\t\t\t\t", ci_upper_a_rank)
    print("CI Thompson Savur Upper Value:\t\t\t\t", round(ci_upper_a_rank_value,4))
    print("")
    print("###Normality Tests:")
    print("#sharpio-wilk-test:")
    print("p-value:\t\t\t\t\t\t", round(w_p_value_a,4))
    print("W:\t\t\t\t\t\t\t", round(w_value_a,4))
    print("#Kolmogorov Smirnov Test:")
    print("p-value:\t\t\t\t\t\t",round(ks_p_value_a,4))
    print("ksscore:\t\t\t\t\t\t",round(ksscore_a,4))
    print("#Anderson–Darling-Test:")
    print("A:\t\t\t\t\t\t\t", round(anderson_a,4))

    #Sequence B:
    print("")
    print("***Sequence B:")
    print("")
    print("Mean:\t\t\t\t\t\t\t",round(mean_b, 4))
    print("Std. Deviation:\t\t\t\t\t\t", round(std_dev_b, 4))
    print("2.5th Percentile:\t\t\t\t\t", round(percentile2_5_b,4))
    print("25th Percentile 25%:\t\t\t\t\t", round(percentile25_b,4))
    print("50th Percentile 50%:\t\t\t\t\t", round(percentile50_b,4))
    print("75th Percentile 75%:\t\t\t\t\t", round(percentile75_b,4))
    print("97.5th Percentile 97,5%:\t\t\t\t", round(percentile97_5_b,4))
    print("CI Lower:\t\t\t\t\t\t", round(ci_lower_b,4))
    print("CI Upper:\t\t\t\t\t\t", round(ci_upper_b,4))
    print("CI Thompson Savur Lower Rank:\t\t\t\t", ci_lower_b_rank)
    print("CI Thompson Savur Lower Value:\t\t\t\t", round(ci_lower_b_rank_value,4))
    print("CI Thompson Savur Upper Rank:\t\t\t\t", ci_upper_b_rank)
    print("CI Thompson Savur Upper Value:\t\t\t\t", round(ci_upper_b_rank_value,4))
    print("")
    print("###Normality Tests:")
    print("#sharpio-wilk-test:")
    print("p-value:\t\t\t\t\t\t", round(w_p_value_b,4))
    print("W:\t\t\t\t\t\t\t", round(w_value_b,4))
    print("#Kolmogorov Smirnov Test:")
    print("p-value:\t\t\t\t\t\t",round(ks_p_value_b,4))
    print("ksscore:\t\t\t\t\t\t",round(ksscore_b,4))
    print("#Anderson–Darling-Test:")
    print("A:\t\t\t\t\t\t\t", round(anderson_b,4))

    print("")
    print("***Interset Tests")

    print("")
    print("T-test und p-Value:")
    print("asume same sigma:")
    print("t-score:\t\t\t\t\t\t", round(tscore_norm_same,4))
    print("p-value:\t\t\t\t\t\t", round(pvalue_norm_same,4))
    print("asume diffrent sigma:")
    print("t-score:\t\t\t\t\t\t", round(tscore_norm_diff,4))
    print("p-value:\t\t\t\t\t\t", round(pvalue_norm_diff,4))

    print("")
    print("Manwhitney-U-Test:")
    print("p-value:\t\t\t\t\t\t",round(pvalue_mwu,4))
    print("U:\t\t\t\t\t\t\t",round(u_value,4))

    print("")
    print("Kolmogorov Smirnov Test:")
    print("p-value:\t\t\t\t\t\t", round(ks_p_value, 4))
    print("ks score:\t\t\t\t\t\t", round(ksscore, 4))

    print("")
    print("Cohens-D:")
    print("d measure:\t\t\t\t\t\t", round(cohens_d,4))

    print("")
    print("Hedge's g:")
    print("g measure:\t\t\t\t\t\t", round(hedges_g,4))

    print("")
    print("Glass's delta:")
    print("delta:\t\t\t\t\t\t\t", round(glass_delta,4))

    print("")
    print("***************Full analysis of rank data done***************")
    print("")
    print("")

(mean_a, mean_b, std_dev_a, std_dev_b, percentile2_5_a, percentile25_a, percentile50_a,
 percentile75_a, percentile97_5_a, percentile2_5_b, percentile25_b, percentile50_b,
 percentile75_b, percentile97_5_b, ci_lower_a, ci_upper_a, ci_lower_b, ci_upper_b,
 tscore_norm_same,pvalue_norm_same,tscore_norm_diff,pvalue_norm_diff, u_value,
 pvalue_mwu,w_value_a,w_p_value_a,w_value_b,w_p_value_b,ksscore,ks_p_value,ksscore_a,
 ksscore_b,ks_p_value_a,ks_p_value_b,anderson_a,anderson_b,cohens_d,hedges_g, glass_delta,
 ci_lower_a_rank, ci_upper_a_rank, ci_lower_b_rank, ci_upper_b_rank, ci_lower_a_rank_value,
 ci_upper_a_rank_value, ci_lower_b_rank_value, ci_upper_b_rank_value) = test_all(data_a, data_b, alpha)

print_results_raw(alpha, mean_a, mean_b, std_dev_a, std_dev_b, percentile2_5_a, percentile25_a,
                  percentile50_a, percentile75_a, percentile97_5_a, percentile2_5_b,
                  percentile25_b, percentile50_b, percentile75_b, percentile97_5_b,
                  ci_lower_a, ci_upper_a, ci_lower_b, ci_upper_b, tscore_norm_same,
                  pvalue_norm_same,tscore_norm_diff,pvalue_norm_diff, u_value,pvalue_mwu,
                  w_value_a, w_p_value_a, w_value_b, w_p_value_b,ksscore,ks_p_value,ksscore_a,
                  ksscore_b,ks_p_value_a,ks_p_value_b,anderson_a,anderson_b,cohens_d,hedges_g,
                  glass_delta, ci_lower_a_rank, ci_upper_a_rank, ci_lower_b_rank, ci_upper_b_rank,
                  ci_lower_a_rank_value,ci_upper_a_rank_value, ci_lower_b_rank_value, ci_upper_b_rank_value)

(mean_a, mean_b, std_dev_a, std_dev_b, percentile2_5_a, percentile25_a, percentile50_a,
 percentile75_a, percentile97_5_a, percentile2_5_b, percentile25_b, percentile50_b,
 percentile75_b, percentile97_5_b, ci_lower_a, ci_upper_a, ci_lower_b, ci_upper_b,
 tscore_norm_same,pvalue_norm_same,tscore_norm_diff,pvalue_norm_diff, u_value,
 pvalue_mwu,w_value_a,w_p_value_a,w_value_b,w_p_value_b,ksscore,ks_p_value,ksscore_a,
 ksscore_b,ks_p_value_a,ks_p_value_b,anderson_a,anderson_b,cohens_d,hedges_g, glass_delta,
 ci_lower_a_rank, ci_upper_a_rank, ci_lower_b_rank, ci_upper_b_rank, ci_lower_a_rank_value,
 ci_upper_a_rank_value, ci_lower_b_rank_value, ci_upper_b_rank_value) = test_all2(data_a, data_b, alpha)

print_results_ranks(alpha, mean_a, mean_b, std_dev_a, std_dev_b, percentile2_5_a, percentile25_a,
                    percentile50_a, percentile75_a, percentile97_5_a, percentile2_5_b,
                    percentile25_b, percentile50_b, percentile75_b, percentile97_5_b,
                    ci_lower_a, ci_upper_a, ci_lower_b, ci_upper_b, tscore_norm_same,
                    pvalue_norm_same,tscore_norm_diff,pvalue_norm_diff, u_value,pvalue_mwu,
                    w_value_a, w_p_value_a, w_value_b, w_p_value_b,ksscore,ks_p_value,ksscore_a,
                    ksscore_b,ks_p_value_a,ks_p_value_b,anderson_a,anderson_b,cohens_d,hedges_g,
                    glass_delta,ci_lower_a_rank, ci_upper_a_rank, ci_lower_b_rank, ci_upper_b_rank,
                    ci_lower_a_rank_value,ci_upper_a_rank_value, ci_lower_b_rank_value, ci_upper_b_rank_value)

import numpy,import scipy,import statsmodels


import numpy
from bblr.models.Model import Model

class HopfieldModel(Model):
    def __init__(self, properties, seedAddition):
        self.trainingRule = properties.get('trainingRule')

        # Validating the configuration.

        if self.trainingRule not in ('hebbian', 'storkey'):
            raise Exception('Unknown training rule "' + self.trainingRule + '". Only "hebbian" and "storkey" are supported')

    # Public methods. A model must implement these methods in order to use it in Main.py

    def train(self, patternDataSet, patternDataSetProperties):
        self.patternSize = len(patternDataSet[0])
        hopfieldDomainPatternDataSet = map(lambda pattern: self.toHopfieldDomain(pattern), patternDataSet)

        if self.trainingRule == 'hebbian':
            self.trainHebbian(hopfieldDomainPatternDataSet)
        else:
            self.trainStorkey(hopfieldDomainPatternDataSet)

        return {'trainingEpochs': 1}

    def recall(self, inputVector):
        result = self.toHopfieldDomain(inputVector)
        iterations = 0
        changed = True

        while (changed):
            changed = self.updateNeurons(result)
            iterations += 1

        return self.toStandardDomain(result), iterations

    # Private methods.

    def toHopfieldDomain(self, vector):
        return numpy.array(map(lambda x: 1.0 if x == 1 else -1.0, vector))

    def toStandardDomain(self, vector):
        return tuple(map(lambda x: int(x != -1.0), vector))

    def trainHebbian(self, patternDataSet):
        self.weights = numpy.zeros((self.patternSize, self.patternSize))

        for pattern in patternDataSet:
            self.weights += numpy.outer(pattern, pattern)

        self.weights[numpy.diag_indices(self.patternSize)] = 0
        self.weights /= self.patternSize

    def trainStorkey(self, patternDataSet):
        self.weights = numpy.zeros((self.patternSize, self.patternSize))
        n = self.patternSize

        for v in xrange(len(patternDataSet)):
            lastWeights = self.weights.copy()

            for i in range(0, self.patternSize):
                for j in range(i + 1, self.patternSize):
                    weight = lastWeights[i][j]
                    weight += patternDataSet[v][i] * patternDataSet[v][j] / n
                    weight -= patternDataSet[v][i] * self.h(v, j, i, lastWeights, patternDataSet) / n
                    weight -= patternDataSet[v][j] * self.h(v, i, j, lastWeights, patternDataSet) / n

                    self.weights[i][j] = weight
                    self.weights[j][i] = weight

    def h(self, v, i, j, lastWeights, patternDataSet):
        s = 0.0

        for k in xrange(self.patternSize):
            if k != i and k != j:
                s += lastWeights[i][k] * patternDataSet[v][k]

        return s

    def updateNeurons(self, inputVector):
        changed = False

        for neuron in xrange(self.patternSize):
            neuronOutput = self.calculateNeuronOutput(neuron, inputVector)

            if neuronOutput != inputVector[neuron]:
                inputVector[neuron] = neuronOutput
                changed = True

        return changed

    def calculateNeuronOutput(self, neuron, inputVector):
        return 1.0 if numpy.sum(numpy.multiply(self.weights[neuron,:], inputVector)) > 0.0 else -1.0


"""
Basic Inner Products
====================

Inner products between two scalar or vector quantities represents the most
basic class of inner products. For this class of inner products, we demonstrate:

    - How to construct the inner product matrix
    - How to use inner product matricies to approximate the inner product
    - How to construct the inverse of the inner product matrix.

For scalar quantities :math:`\\psi` and :math:`\\phi`, the
inner product is given by:

.. math::
    (\\psi , \\phi ) = \\int_\\Omega \\psi \\, \\phi \\, dv

And for vector quantities :math:`\\vec{u}` and :math:`\\vec{v}`, the
inner product is given by:

.. math::
    (\\vec{u}, \\vec{v}) = \\int_\\Omega \\vec{u} \\cdot \\vec{v} \\, dv

In discretized form, we can approximate the aforementioned inner-products as:

.. math::
    (\\psi , \\phi) \\approx \\mathbf{\\psi^T \\, M \\, \\phi}

and

.. math::
    (\\vec{u}, \\vec{v}) \\approx \\mathbf{u^T \\, M \\, v}

where :math:`\\mathbf{M}` in either equation represents an
*inner-product matrix*. :math:`\\mathbf{\\psi}`, :math:`\\mathbf{\\phi}`,
:math:`\\mathbf{u}` and :math:`\\mathbf{v}` are discrete variables that live
on the mesh. It is important to note a few things about the
inner-product matrix in this case:

    1. It depends on the dimensions and discretization of the mesh
    2. It depends on where the discrete variables live; e.g. edges, faces, nodes, centers

For this simple class of inner products, the inner product matricies for
discrete quantities living on various parts of the mesh have the form:

.. math::
    \\textrm{Centers:} \\; \\mathbf{M_c} &= \\textrm{diag} (\\mathbf{v} ) \n
    \\textrm{Nodes:} \\; \\mathbf{M_n} &= \\frac{1}{2^{2k}} \\mathbf{P_n^T } \\textrm{diag} (\\mathbf{v} ) \\mathbf{P_n} \n
    \\textrm{Faces:} \\; \\mathbf{M_f} &= \\frac{1}{4} \\mathbf{P_f^T } \\textrm{diag} (\\mathbf{I_k \\otimes v} ) \\mathbf{P_f} \n
    \\textrm{Edges:} \\; \\mathbf{M_e} &= \\frac{1}{4^{k-1}} \\mathbf{P_e^T } \\textrm{diag} (\\mathbf{I_k \\otimes v}) \\mathbf{P_e}

where :math:`k = 1,2,3`, :math:`\\mathbf{I_k}` is the identity matrix and
:math:`\\otimes` is the kronecker product. :math:`\\mathbf{P}` are projection
matricies that map quantities from one part of the cell (nodes, faces, edges)
to cell centers.

"""

####################################################
#
# Import Packages
# ---------------
#
# Here we import the packages required for this tutorial
#

from discretize.utils import sdiag
from discretize import TensorMesh
import matplotlib.pyplot as plt
import numpy as np

# sphinx_gallery_thumbnail_number = 2

#####################################################
# Scalars
# -------
#
# It is natural for scalar quantities to live at cell centers or nodes. Here
# we will define a scalar function (a Gaussian distribution in this case):
#
# .. math::
#     \phi(x) = \frac{1}{\sqrt{2 \pi \sigma^2}} \, e^{- \frac{(x- \mu )^2}{2 \sigma^2}}
#
#
# We will then evaluate the following inner product:
#
# .. math::
#     (\phi , \phi) = \int_\Omega \phi^2 \, dx = \frac{1}{2\sigma \sqrt{\pi}}
#
#
# according to the mid-point rule using inner-product matricies. Next we
# compare the numerical approximation of the inner product with the analytic
# solution. *Note that the method for evaluating inner products here can be
# extended to variables in 2D and 3D*.
#

# Define the Gaussian function
def fcn_gaussian(x, mu, sig):

    return (1 / np.sqrt(2 * np.pi * sig ** 2)) * np.exp(-0.5 * (x - mu) ** 2 / sig ** 2)

# Create a tensor mesh that is sufficiently large
h = 0.1 * np.ones(100)
mesh = TensorMesh([h], "C")

# Define center point and standard deviation
mu = 0
sig = 1.5

# Evaluate at cell centers and nodes
phi_c = fcn_gaussian(mesh.gridCC, mu, sig)
phi_n = fcn_gaussian(mesh.gridN, mu, sig)

# Define inner-product matricies
Mc = sdiag(mesh.vol)  # cell-centered
# Mn = mesh.getNodalInnerProduct()  # on nodes (*functionality pending*)

# Compute the inner product
ipt = 1 / (2 * sig * np.sqrt(np.pi))  # true value of (f, f)
ipc = np.dot(phi_c, (Mc * phi_c))
# ipn = np.dot(phi_n, (Mn*phi_n)) (*functionality pending*)

fig = plt.figure(figsize=(5, 5))
ax = fig.add_subplot(111)
ax.plot(mesh.gridCC, phi_c)
ax.set_title("phi at cell centers")

# Verify accuracy
print("ACCURACY")
print("Analytic solution:    ", ipt)
print("Cell-centered approx.:", ipc)
# print('Nodal approx.:        ', ipn)

#####################################################
# Vectors
# -------
#
# To preserve the natural boundary conditions for each cell, it is standard
# practice to define fields on cell edges and fluxes on cell faces. Here we
# will define a 2D vector quantity:
#
# .. math::
#     \vec{v}(x,y) = \Bigg [ \frac{-y}{r} \hat{x} + \frac{x}{r} \hat{y} \Bigg ]
#     \, e^{-\frac{x^2+y^2}{2\sigma^2}}
#
# We will then evaluate the following inner product:
#
# .. math::
#     (\vec{v}, \vec{v}) = \int_\Omega \vec{v} \cdot \vec{v} \, da
#     = 2 \pi \sigma^2
#
# using inner-product matricies. Next we compare the numerical evaluation
# of the inner products with the analytic solution. *Note that the method for
# evaluating inner products here can be extended to variables in 3D*.
#

# Define components of the function
def fcn_x(xy, sig):
    return (-xy[:, 1] / np.sqrt(np.sum(xy ** 2, axis=1))) * np.exp(
        -0.5 * np.sum(xy ** 2, axis=1) / sig ** 2
    )

def fcn_y(xy, sig):
    return (xy[:, 0] / np.sqrt(np.sum(xy ** 2, axis=1))) * np.exp(
        -0.5 * np.sum(xy ** 2, axis=1) / sig ** 2
    )

# Create a tensor mesh that is sufficiently large
h = 0.1 * np.ones(100)
mesh = TensorMesh([h, h], "CC")

# Define center point and standard deviation
sig = 1.5

# Evaluate inner-product using edge-defined discrete variables
vx = fcn_x(mesh.gridEx, sig)
vy = fcn_y(mesh.gridEy, sig)
v = np.r_[vx, vy]

Me = mesh.getEdgeInnerProduct()  # Edge inner product matrix

ipe = np.dot(v, Me * v)

# Evaluate inner-product using face-defined discrete variables
vx = fcn_x(mesh.gridFx, sig)
vy = fcn_y(mesh.gridFy, sig)
v = np.r_[vx, vy]

Mf = mesh.getFaceInnerProduct()  # Edge inner product matrix

ipf = np.dot(v, Mf * v)

# The analytic solution of (v, v)
ipt = np.pi * sig ** 2

# Plot the vector function
fig = plt.figure(figsize=(5, 5))
ax = fig.add_subplot(111)
mesh.plotImage(
    v, ax=ax, v_type="F", view="vec", stream_opts={"color": "w", "density": 1.0}
)
ax.set_title("v at cell faces")

fig.show()

# Verify accuracy
print("ACCURACY")
print("Analytic solution:    ", ipt)
print("Edge variable approx.:", ipe)
print("Face variable approx.:", ipf)

##############################################
# Inverse of Inner Product Matricies
# ----------------------------------
#
# The final discretized system using the finite volume method may contain
# the inverse of an inner-product matrix. Here we show how the inverse of
# the inner product matrix can be explicitly constructed. We validate its
# accuracy for cell-centers, nodes, edges and faces by computing the folling
# L2-norm for each:
#
# .. math::
#     \| \mathbf{v - M^{-1} M v} \|^2
#
#

# Create a tensor mesh
h = 0.1 * np.ones(100)
mesh = TensorMesh([h, h], "CC")

# Cell centered for scalar quantities
Mc = sdiag(mesh.vol)
Mc_inv = sdiag(1 / mesh.vol)

# Nodes for scalar quantities  (*functionality pending*)
# Mn = mesh.getNodalInnerProduct()
# Mn_inv = mesh.getNodalInnerProduct(invMat=True)

# Edges for vector quantities
Me = mesh.getEdgeInnerProduct()
Me_inv = mesh.getEdgeInnerProduct(invMat=True)

# Faces for vector quantities
Mf = mesh.getFaceInnerProduct()
Mf_inv = mesh.getFaceInnerProduct(invMat=True)

# Generate some random vectors
phi_c = np.random.rand(mesh.nC)
phi_n = np.random.rand(mesh.nN)
vec_e = np.random.rand(mesh.nE)
vec_f = np.random.rand(mesh.nF)

# Generate some random vectors
norm_c = np.linalg.norm(phi_c - Mc_inv.dot(Mc.dot(phi_c)))
# norm_n = np.linalg.norm(phi_n - Mn_inv*Mn*phi_n)
norm_e = np.linalg.norm(vec_e - Me_inv * Me * vec_e)
norm_f = np.linalg.norm(vec_f - Mf_inv * Mf * vec_f)

# Verify accuracy
print("ACCURACY")
print("Norm for centers:", norm_c)
# print('Norm for nodes:  ', norm_n)
print("Norm for edges:  ", norm_e)
print("Norm for faces:  ", norm_f)


## exercise 2.1.6
from ex2_1_1 import *
import matplotlib.pyplot as plt
from scipy.linalg import svd

r = np.arange(1,X.shape[1]+1)
plt.bar(r, np.std(X,0))
plt.xticks(r, attributeNames)
plt.ylabel('Standard deviation')
plt.xlabel('Attributes')
plt.title('NanoNose: attribute standard deviations')

## Investigate how standardization affects PCA

# Try this *later* (for last), and explain the effect
#X_s = X.copy() # Make a to be "scaled" version of X
#X_s[:, 2] = 100*X_s[:, 2] # Scale/multiply attribute C with a factor 100
# Use X_s instead of X to in the script below to see the difference.
# Does it affect the two columns in the plot equally?

# Subtract the mean from the data
Y1 = X - np.ones((N, 1))*X.mean(0)

# Subtract the mean from the data and divide by the attribute standard
# deviation to obtain a standardized dataset:
Y2 = X - np.ones((N, 1))*X.mean(0)
Y2 = Y2*(1/np.std(Y2,0))
# Here were utilizing the broadcasting of a row vector to fit the dimensions
# of Y2

# Store the two in a cell, so we can just loop over them:
Ys = [Y1, Y2]
titles = ['Zero-mean', 'Zero-mean and unit variance']
threshold = 0.9
# Choose two PCs to plot (the projection)
i = 0
j = 1

# Make the plot
plt.figure(figsize=(10,15))
plt.subplots_adjust(hspace=.4)
plt.title('NanoNose: Effect of standardization')
nrows=3
ncols=2
for k in range(2):
    # Obtain the PCA solution by calculate the SVD of either Y1 or Y2
    U,S,Vh = svd(Ys[k],full_matrices=False)
    V=Vh.T # For the direction of V to fit the convention in the course we transpose
    # For visualization purposes, we flip the directionality of the
    # principal directions such that the directions match for Y1 and Y2.
    if k==1: V = -V; U = -U;

    # Compute variance explained
    rho = (S*S) / (S*S).sum()

    # Compute the projection onto the principal components
    Z = U*S;

    # Plot projection
    plt.subplot(nrows, ncols, 1+k)
    C = len(classNames)
    for c in range(C):
        plt.plot(Z[y==c,i], Z[y==c,j], '.', alpha=.5)
    plt.xlabel('PC'+str(i+1))
    plt.xlabel('PC'+str(j+1))
    plt.title(titles[k] + '\n' + 'Projection' )
    plt.legend(classNames)
    plt.axis('equal')

    # Plot attribute coefficients in principal component space
    plt.subplot(nrows, ncols,  3+k)
    for att in range(V.shape[1]):
        plt.arrow(0,0, V[att,i], V[att,j])
        plt.text(V[att,i], V[att,j], attributeNames[att])
    plt.xlim([-1,1])
    plt.ylim([-1,1])
    plt.xlabel('PC'+str(i+1))
    plt.ylabel('PC'+str(j+1))
    plt.grid()
    # Add a unit circle
    plt.plot(np.cos(np.arange(0, 2*np.pi, 0.01)),
         np.sin(np.arange(0, 2*np.pi, 0.01)));
    plt.title(titles[k] +'\n'+'Attribute coefficients')
    plt.axis('equal')

    # Plot cumulative variance explained
    plt.subplot(nrows, ncols,  5+k);
    plt.plot(range(1,len(rho)+1),rho,'x-')
    plt.plot(range(1,len(rho)+1),np.cumsum(rho),'o-')
    plt.plot([1,len(rho)],[threshold, threshold],'k--')
    plt.title('Variance explained by principal components');
    plt.xlabel('Principal component');
    plt.ylabel('Variance explained');
    plt.legend(['Individual','Cumulative','Threshold'])
    plt.grid()
    plt.title(titles[k]+'\n'+'Variance explained')

plt.show()


import numpy as np
import math

def k_prac(x1,x2,l):
    '''
    Radial Basis Kernel

    args:
        x1: 1xd vector
        x2: 1xd vector
        l: lenghtscale
    return:
        int: Similarity measure
    '''
    exponent = - 1/(2*l**2) * np.linalg.norm(x1-x2)**2
    return np.exp(exponent)

def k_homework(x1, x2, l, P=3):
    exponent_1 = - 1/(2*l**2) * np.linalg.norm(x1-x2)**2
    exponent_2 = - 2/(l**2) * math.sin(math.pi * np.linalg.norm(x1-x2) / P )**2
    return np.exp(exponent_1) * np.exp(exponent_2)

def k_q2(x1,x2,c,d):
    return (x1*x2 +c)**d


import math
import matplotlib.pyplot as plt
import numpy as np

iteration=45

#deal with a_0
a_0=math.pi/4

#deal with a_n,b_n
a_n=[]
b_n=[]
for i in range(1,iteration+1):
    a_n.append((1-(-1)**i)/(i**2)/math.pi)
    b_n.append(1/i)

print(a_n)
print(b_n)

def fourier(a_0,a_n,b_n,iteration,x):
    value=a_0
    for i in range(1,iteration):
        value+=a_n[i-1]*math.cos(i*x)+b_n[i-1]*math.sin(i*x)
    return value

#visualize the fourier transform
discrete_num=1000
x=np.linspace(-math.pi,math.pi,discrete_num)
y=[]
for j in range(discrete_num):
    y.append(fourier(a_0,a_n,b_n,iteration,x[j]))

plt.plot(x,y)
plt.title("realize the fourier transformation (number of summations: %d)" %(iteration))
plt.xlabel("x")
plt.ylabel("y")
plt.text(-3,1,"f(x)=0, for x=-π~0")
plt.text(-3,0.5,"f(x)=π-x, for x=0~π")
plt.show()