import numpy

from base.polynomial import Polynomial
from newtons_method import get_roots

class Legendre(Polynomial):
    """
    Legendre Polynomials are a set of polynomials of increasing order that
    have various use cases in numerical mathematics.

    These Polynomials are defined on the interval [-1,1] and the roots of
    the polynomial are used in the numerical approximation of definite integrals.

    """

    # Define a list of the roots of the Legendre Polynomials
    roots = []

    def __init__(self, degree) -> None:
        Polynomial.__init__(self, self.generate_recursive(degree))
        if degree > 1:
            self.roots = get_roots(Polynomial(self.co_array))
        else:
            roots = [0]

    def __str__(self) -> str:
        sret = "{}\n".format(self.degree)
        sret += format(self.co_array)
        return sret

    def __repr__(self):
        return super().__repr__()

    # Generate Legendre polynomial(co-array) up to the n-th degree.
    #   - Uses recursive method
    def generate_recursive(self, order):
        # Pn(x) = (2n + 1)*x/(n+1) * Pn-1(x) - (n)/(n+1) * Pn-2(x)
        #   - see https://en.wikipedia.org/wiki/Legendre_polynomials
        #     for details on recurrence relations.
        if order == 0:
            return numpy.array([1])
        elif order == 1:
            return numpy.array([0, 1])
        else:
            coefficient_array = numpy.zeros(order + 1)

            # Generate First term
            Pn_minOne = self.generate_recursive(order - 1)
            Pn_minOne = Pn_minOne*(2*order - 1)/(order)
            Pn_minOne = numpy.insert(Pn_minOne, [0], 0, axis=0)
            #print(f"Pn_min1:{Pn_minOne}")

            # Generate Second Term
            Pn_minTwo = self.generate_recursive(order - 2)
            Pn_minTwo = Pn_minTwo*-(order - 1)/(order)
            Pn_minTwo = numpy.append(Pn_minTwo, [0, 0])
            #print(f"Pn_min2:{Pn_minTwo}")

            # Combine Terms
            coefficient_array += Pn_minOne + Pn_minTwo
            #print(f"coeff_array:{coefficient_array}")

        return coefficient_array

# Define the Test functions and methods
def main():

    # Generate Legendre polynomials for various degrees
    for n in range(11):
        # Test generation for degree i
        legendre = Legendre(n)
        print(f"Test{n}|_Legendre:{repr(legendre)}, roots:{legendre.roots}")

if __name__ == "__main__":
    main()


import numpy as np
import matplotlib.pyplot as plt

from utils import sigmoid

# visualize sigmoid function

def sigmoid_visual():
    z = np.arange(-7, 7, 0.1)
    phi_z = sigmoid(z)
    plt.plot(z, phi_z)
    plt.axvline(0.0, color='k')
    plt.ylim(-0.1, 1.1)
    plt.xlabel('z')
    plt.ylabel('$\phi (z)$')

    plt.yticks([0.0, 0.5, 1.0])
    ax = plt.gca()
    ax.yaxis.grid(True)
    plt.tight_layout()
    plt.show()


from common import traces, textins, num_traces, num_points, hypothetical_power_usage

# ANCHOR: additional_imports
import numpy as np
# ANCHOR_END: additional_imports

# ANCHOR: manual_analysis
# Pearson correlation
########################################
# ANCHOR: pearson_basic
def covariance(X, Y):
    if len(X) != len(Y):
        print("Lengths are unequal, quiting...")
        quit()

    n = len(X)
    mean_x = np.mean(X, dtype=np.float64)
    mean_y = np.mean(Y, dtype=np.float64)

    return np.sum((X - mean_x) * (Y - mean_y)) / n

def standard_deviation(X):
    n = len(X)
    mean_x = np.mean(X, dtype=np.float64)

    return np.sqrt( np.sum( np.power( (X - mean_x), 2 ) ) / n )

def pearson_correlation_coefficient(X, Y):
    cov = covariance(X, Y)
    sd_x = standard_deviation(X)
    sd_y = standard_deviation(Y)

    return cov / ( sd_x * sd_y )
# ANCHOR_END: pearson_basic
########################################

# Define a function to calculate the Correlation Coefficients for a byte in a
# subkey.
########################################
# ANCHOR: calc_correlations
def calculate_correlation_coefficients(subkey, subkey_index):
    # Declare a numpy for the hypothetical power usage
    hypothetical_power = np.zeros(num_traces)

    for trace_index in range(0, num_traces):
        hypothetical_power[trace_index] = hypothetical_power_usage(
            subkey,
            textins[trace_index][subkey_index]
        )

    # We are going to the determine correlations between each trace point
    # and the hypothetical power usage. This will save all those coefficients
    point_correlation = np.zeros(num_points)

    # Loop through all points and determine their correlation coefficients
    for point_index in range(0, num_points):
        point_correlation[point_index] = pearson_correlation_coefficient(
            hypothetical_power,

            # Look at the individual traces points for every trace
            traces[:, point_index]
        )

    return point_correlation
# ANCHOR_END: calc_correlations
########################################
# ANCHOR: manual_analysis


# -*- coding: utf-8 -*-
"""
Created on Mon Dec 07 13:43:24 2015

@author: hehu
"""

import matplotlib as mpl
mpl.use('PDF')
import matplotlib.pyplot as plt
import numpy as np
from matplotlib import rc

def gaussian(x, mu, sig):
    multiplier = (1/np.sqrt(2*np.pi*sig**2.))
    return multiplier * np.exp(-np.power(x - mu, 2.) / (2 * np.power(sig, 2.)))

if __name__ == "__main__":

    rc('font',**{'family':'sans-serif'}) #,'sans-serif':['Helvetica']})
    ## for Palatino and other serif fonts use:
    #rc('font',**{'family':'serif','serif':['Palatino']})
    rc('text', usetex=True)

    x = np.linspace(-3, 9, 100)
    y = gaussian(x, 1, 1)

    ax = plt.subplot(211)
    plt.plot(x, y, linewidth = 2, label = "Gaussian with $\mu=1$")

    y = gaussian(x, 0, 1)
    plt.plot(x, y, linewidth = 2, label = "Gaussian with $\mu=0$")
    plt.legend(loc = "best")

    plt.plot([2], [0], 'go', clip_on=False)

    ax.annotate("Where did this come from?",
                 [2,0],
                 xytext=[6,0.15],
                 size=13,
                 bbox=dict(boxstyle="round4", fc="w", ec = "g"),
                 arrowprops=dict(arrowstyle="simple",
                                 connectionstyle="arc3,rad=-0.2",
                                 shrinkA = 0,
                                 shrinkB = 8,
                                 fc = "g",
                                 ec = "g"),
                 horizontalalignment='center',
                 verticalalignment='center')

    ax.set_ylim([0,0.5])

    plt.savefig("../images/two_gaussians.pdf", bbox_inches = "tight")


#!/usr/bin/env python
"""
Written by: CAPT Stu Blair
Purpose: eigenvalue finding / Fourier Expansion process in Python
"""

import numpy as np
import matplotlib.pyplot as plt
from scipy import optimize
import scipy.integrate as integrate

# define eigenfunctions
def ef(alpha_n,x):
    return np.sin(alpha_n*x);

# construct Fourier expansion (this is crap)
def FF_exp(x,a_n,ev_n,N):
    ret_val = 0.
    for i in range(N):
        ret_val += a_n[i]*ef(ev_n[i],x); # this really sucks

    return ret_val;

# define function with which to determine eigenvalues
def f(x):
    return x + np.tan(x);

# function that I want to expand in eigenfunctions
def f_exp(x):
    return np.exp(-x);

Nx= 1000;
# # create a plot of f(x) to see what it looks like
# a = 0.; b = 20.;
# X = np.linspace(a,b,Nx,endpoint=False)

# # plot the function to help develop algorithm for finding eigenvalues
# plt.plot(X,f(X))
# plt.axis([a,b,-20,20]);
# plt.grid(True);
# plt.xlabel('X',fontsize=16,fontweight='bold');
# plt.ylabel('f(X)',fontsize=16,fontweight='bold');
# plt.title('Non-linear function for eigenvalues',fontsize=16,fontweight='bold');
# plt.show()

N_evs = 35
roots = np.empty([N_evs,1]);

a = np.pi/2.+ 1e-6;
b = 3.*np.pi/2.-1e-6;
for i in range(N_evs):
    roots[i] = optimize.brentq(f,a,b);
    a = a + np.pi # reset lower bound
    b = b + np.pi # reset upper bound

# np.set_printoptions(precision=4);
# print('The first %d roots:'%N_evs);
# print(roots);

# evs = roots**2.;
# print('The first %d eigenvalues:'%N_evs);
# print(evs);

#########################################################

# construct a Fourier Approximation using the eigenfunctions ef
a = 0; b = 1;
a_n = np.zeros_like(roots);

for i in range(N_evs):
    # get the next eigenfunction
    y_n = lambda x: ef(roots[i],x);

    # compute the Fourier Coefficient
    num, err = integrate.quad(lambda x: f_exp(x)*y_n(x),a,b)
    denom, err = integrate.quad(lambda x: y_n(x)**2,a,b)
    a_n[i] = num/denom;

FF = lambda x: FF_exp(x,a_n,roots,N_evs);

X = np.linspace(a,b,Nx)
plt.plot(X,f_exp(X),'-b',linewidth=2.0,label='f(x)')
plt.plot(X,FF(X),'--r',linewidth=2.0,label='FF(x)')
plt.grid(True);
plt.xlabel('X',fontsize=16,fontweight='bold');
plt.ylabel('f(X)',fontsize=16,fontweight='bold')
plt.title('Simple Fourier Expansion',fontsize=18,fontweight='bold')
plt.legend(loc='best');
plt.show();


"""
  Name     : c15_03_4moments.py
  Book     : Python for Finance (2nd ed.)
  Publisher: Packt Publishing Ltd.
  Author   : Yuxing Yan
  Date     : 6/6/2017
  email    : yany@canisius.edu
             paulyxy@hotmail.com
"""
import numpy as np
from scipy import stats, random
#
random.seed(12345)
ret=random.normal(0,1,50000)
print('mean =',np.mean(ret))
print('std =',np.std(ret))
print('skewness=',stats.skew(ret))
print('kurtosis=',stats.kurtosis(ret))


#!/usr/bin/env python3

"""
Functions used to evaluate statistical significance of changes.
cmdoret, 20200403
"""
import numpy as np
import scipy.stats as ss

def vals_to_percentiles(vals: np.ndarray, dist: np.ndarray) -> np.ndarray:
    """
    Return the percentiles corresponding to values in a distribution.

    Parameters
    ----------
    vals : np.ndarray of floats
        The values for which percentiles should be returned.
    dist : np.ndarray of floats
        The distribution to use when computing the percentiles.

    Returns
    -------
    np.ndarray of floats :
        The array of percentiles corresponding to input values
    """
    # Get indices corresponding to vals in the distribution
    sorted_dist = np.sort(dist)
    percentiles = np.searchsorted(sorted_dist, vals) / len(sorted_dist)
    return percentiles

def vec_ttest(arr1, arr2):
    """
    Given 2 numpy arrays containing data for 2 different conditions, where
    each array has shape shape rxd where r is the number of replicates and N
    is the number of independent tests to perform. N must be equal in both array,
    however the number of replicates can differ.
    """
    if not (len(arr1.shape) == len(arr2.shape) == 2):
        raise ValueError("Both arrays must be 2-dimensional.")
    if arr1.shape[0] != arr2.shape[0]:
        raise ValueError(
            "Both arrays must have the same number of test datasets"
        )
    # Number of sample in each test (both conditions)
    n_samples = arr1.shape[1] + arr2.shape[1]
    # Compute mean and variance for each test independently
    means_1, means_2 = arr1.mean(axis=1), arr2.mean(axis=1)
    var1, var2 = arr1.var(axis=1, ddof=1), arr2.var(axis=1, ddof=1)
    stds = np.sqrt((var1 + var2) / 2)
    stds[stds == 0] = np.nan
    # alt - control
    t_stats = (means_2 - means_1) / (stds * np.sqrt(1 / n_samples))
    t_stats[np.isnan(t_stats)] = 0

    return t_stats

# TODO: Write a vectorized linear model function for cases with more than
# 2 (quantitative) conditions
def vec_lm():
    ...

def tvals_to_pvals(tvals, n_samples):
    """Compute p-values from t-values
    Given an array of test statistics from multiple t-tests and the (fixed)
    number of samples used for each test, compute p-values for all tests
    """
    # get p-values from test statistics using the cumulative distribution
    # function of the student distribution
    return 1 - ss.t.cdf(np.abs(tvals), df=n_samples - 1)

def pval_to_tval(pval, n_samples):
    """Conversion of a single p-value to corresponding t
    Given a desired p-value and sample size, return the corresponding t-value.
    The absolute value for the 2 sided hypothesis is returned.
    """
    return abs(ss.t.ppf(pval / 2, df=n_samples - 1))


import numpy as np
import matplotlib.pyplot as plt

def generatePoints(num, mu=0, sigma2=1):
	return np.sqrt(sigma2) * np.random.randn(num,2) + mu

def generateCosts(d1,d2, m):
	for i, p1 in enumerate(d1):
		for j, p2 in enumerate(d2):
			m[i,j] = np.linalg.norm(p1-p2)**2
	return m

def sinkhorn(t, scale_row=True, err=1.0, i=0, timeout=100, stopThres=1e-9):
	while (abs(err) > stopThres and i < timeout):
		i+=1
		if scale_row:
			sums = np.sum(t, axis=1)
			err = np.sum(sums) - m
			scalar = 1.0/sums
			t = (t.transpose() * scalar).transpose()
			print(t)
			print("new sum {} by rows".format(np.sum(t, axis=1)))
			scale_row = not scale_row
		else:
			sums = np.sum(t, axis=0)
			err = np.sum(sums) - n
			scalar = 1.0/sums
			t =  t * scalar.transpose()
			scale_row =  not scale_row
			print(t)
			print("new sum {} by cols".format(np.sum(t, axis=0)))
		print("matrix: {}, error {}".format(t, err))
	return t

# init
m, n = 5,3
iters = 3
a = 0.05
scale = 1.0 /(float(m)*float(n))
costs = np.zeros((n,m))

# start and end distribution
start = generatePoints(n, mu=5)
end = generatePoints(m)
res = start
costs =  generateCosts(start,end,costs)
costs =  generateCosts(res,end,costs)
t = np.exp(-costs * a)
G = sinkhorn(t)

# color
xs = start
xt = end
thr =1e-8
mx = G.max()

# Plotting mapping
plt.figure(1)
plt.plot(start.transpose()[0], start.transpose()[1],'ro')
plt.plot(end.transpose()[0], end.transpose()[1], 'bo')
print("G {}".format(G))
for i in range(xs.shape[0]):
    for j in range(xt.shape[0]):
        if G[i, j] / mx > thr:
			print("{} {}".format([xs[i, 0], xt[j, 0]], [xs[i, 1], xt[j, 1]] ))
			print("alpha {}, G ij {} ".format(G[i, j] / mx, G[i,j]))
			# plt.plot([xs[i, 0], xt[j, 0]], [xs[i, 1], xt[j, 1]], alpha=G[i, j] / mx)
			plt.plot([xs[i, 0], xt[j, 0]], [xs[i, 1], xt[j, 1]], alpha=G[i, j])

plt.figure(2)
plt.imshow(G, interpolation='nearest') # mapping gamma weights
plt.show()


import matplotlib.pyplot as plt
import numpy as np
import random

def g(x):
    return x ** 4 + x ** 3 + 2 * x ** 2 + 2 * x

def g_prime(x):
    return (4*x**3) + (3*x**2) + (4*x) + 2

def printStatus(iteration: int, weight: float):
  print("=================")
  print("Iteration: {}".format(iteration))
  print("Weight: {}".format(weight))
  print("Derivative Value of Weight: {}".format(g_prime(weight)))
  print("=================")

def plotFuncAndWgt(weight: float):
  ## make plot ##
  plt.scatter(weight, g(weight), c="green")
  x = np.arange(-10, 10, 0.1)
  plt.plot(x,g(x), "r")
  plt.xlim(-5, 5)
  plt.ylim(-3, 5)
  plt.show()

# Gradient descent
STUFF = []
max_iter = 100  # you choose
step_size = .02  # you choose
epsilon = .001  # when the difference is so small
conv = False
weight = float(random.randint(-500, 500)/100.0)  # choose starting point
last_wt = weight
i = 0

while i < max_iter and not conv:
  try:
    last_wt = weight
    plt.scatter(weight, g(weight))
    weight = weight - (step_size*g_prime(weight))
  except OverflowError:
    step_size *= .5

  if g(weight) > g(last_wt):
    step_size *= .5
  if abs(g_prime(last_wt) - g_prime(weight)) < epsilon:
    break
  elif g_prime(weight) == 0:
    conv = True

  if i % 50 == 0:
    printStatus(i, weight)
  STUFF.append([i,g_prime(weight)])
  i += 1

printStatus(i, weight)
plotFuncAndWgt(weight)


#============Visulizing the dataset

#这一次将使用三部分的数据集training、validation、test
import numpy as np
import matplotlib.pyplot as plt
from scipy.io import loadmat
import scipy.optimize as opt
#load data
path = 'E:\lessons\ML wu.n.g\coursera-ml-py-master\coursera-ml-py-master\machine-learning-ex5\ex5\ex5data1.mat'
data = loadmat(path)
#Training set
X, y = data['X'], data['y']
#validation set
Xval, yval = data['Xval'], data['yval']
#Test set
Xtest, ytest = data['Xtest'], data['ytest']
#Insert one column
X = np.insert(X, 0, 1, axis=1)
Xval = np.insert(Xval, 0, 1, axis=1)
Xtest = np.insert(Xtest, 0, 1, axis=1)
'''
print('X={},y={}'.format(X.shape, y.shape))
print('Xval={},yval={}'.format(Xval.shape, yval.shape))
print('Xtest={},ytest={}'.format(Xtest.shape, ytest.shape))

X=(12, 2),y=(12, 1)
Xval=(21, 2),yval=(21, 1)
Xtest=(21, 2),ytest=(21, 1)
'''
def plotData():
    plt.figure(figsize=(8, 5))
    plt.scatter(X[:,1:], y, c='r', marker='x')
    plt.xlabel('Change in water level(X)')
    plt.ylabel('water flowing out of the dam(y)')
    plt.grid(True)

#plotData()

#=====================Regularized linear regression cost function

def costReg(theta, X, y, l):
    '''
    X (12 ,2) theta(2,) y(12, 1)
    '''
    cost = ((X @ theta - y.flatten()) ** 2).sum()
    regterm = l * (theta[1:] @ theta[1:])
    return (cost  + regterm )/ (2 * len(X))

theta = np.ones(X.shape[1])
#print(costReg(theta, X, y, 1))

#====================Regularized linear regression gradient

def gradientReg(theta, X, y, l):
    #grad has same shape with theta (2,)
    grad = X.T @ (X @ theta - y.flatten())
    regterm = l * theta
    regterm[0] = 0
    return (grad + regterm) / len(X)
print(gradientReg(theta, X, y, 1))
#[-15.30301567 598.25074417]

#===================fitting

def train(X, y, l):
    theta = np.zeros(X.shape[1])
    res = opt.minimize(fun = costReg, x0 = theta, args=(X, y, l), method = 'TNC', jac = gradientReg)
    return res.x
fit_theta = train(X, y, 0)
plotData()
#plt.plot(X[:,1], X @ fit_theta)
#plt.show()

#===================Learning curves

def plot_learning_curve(X, y, Xval, yval, l):
#学习曲线即交叉验证误差和训练误差
    xx = range(1, len(X)+1)
    training_cost, cv_cost = [],[]
    for i in xx:
        res = train(X[: i], y[:i], l)
        training_cost_i = costReg(res, X[:i], y[:i], 0)
        cv_cost_i = costReg(res, Xval, yval, 0)
        training_cost.append(training_cost_i)
        cv_cost.append(cv_cost_i)

    plt.figure(figsize=(8, 5))
    plt.plot(xx, training_cost, label = 'training cost')
    plt.plot(xx, cv_cost, label = 'cv cost')
    plt.legend()
    plt.xlabel('Number of training examples')
    plt.ylabel('Error')
    plt.title('Learning curve for linear regression')
    plt.grid(True)
#plot_learning_curve(X, y, Xval, yval, 0)
#plt.show()

#====================Learnign polynomial Regression

def genPolyFeatures(X, power):
    #增加多项式特征，每次在array的最后一列插入第二列的i+2次方，2次方开始
    Xpoly = X.copy()
    for i in range(2, power + 1):
        Xpoly = np.insert(Xpoly, Xpoly.shape[1], np.power(Xpoly[:,1],i), axis = 1)
    return Xpoly

def get_means_std(X):
    #获取训练集的均值和误差，来标准化数据
    means = np.mean(X, axis=0)
    #ddof=1 表示样本标准差使用np.std()时，将ddof=1则是样本标准差，默认=0是总体标准差。而pandas默认计算样本标准差。
    stds= np.std(X, axis=0, ddof=1)
    return means, stds

def featureNormalize(myX, means, stds):
    #standarlize
    X_norm = myX.copy()
    X_norm[:,1:] = X_norm[:, 1:] - means[1:]
    X_norm[:,1:] = X_norm[:, 1:] / stds[1:]
    return X_norm

#print(genPolyFeatures(X, 6).shape, genPolyFeatures(X, 6))　（12, 7)
train_means, train_stds = get_means_std(genPolyFeatures(X, 6))
#print(train_means.shape, train_stds.shape) (7, ) (7, )
X_norm = featureNormalize(genPolyFeatures(X, 6), train_means, train_stds)
Xval_norm = featureNormalize(genPolyFeatures(Xval, 6), train_means, train_stds)
Xtest_norm = featureNormalize(genPolyFeatures(Xtest, 6), train_means, train_stds)

def plot_polofit(means, stds, l):
    theta = train(X_norm, y, l) #（7, ）
    x = np.linspace(-75,55,50) #50个元素
    xmat = x.reshape(-1,1)
    xmat = np.insert(xmat, 0 ,1, axis=1)
    Xmat = genPolyFeatures(xmat, 6)
    Xmat_norm = featureNormalize(Xmat, means, stds)
    print(Xmat.shape, Xmat_norm.shape)
    yy = Xmat_norm @ theta
   # print(yy.shape)
    plotData()
    plt.plot(x, yy)
#plot_polofit(train_means, train_stds, 0)
#plot_learning_curve(X_norm, y, Xval_norm, yval, 0)
#plt.show()

#===================adjusting lambda(the regularization parameter)

#lambada = 0 overfitting
#try : lambda = 1
'''
plot_polofit(train_means, train_stds, 1)
plot_learning_curve(X_norm, y, Xval_norm, yval, 1)
plt.show()
'''

#==============selecting lambda using a cv set
lambdas = [0, 0.001, 0.003, 0.01, 0.03, 0.1, 0.3, 1, 3, 10]
errors_train, errors_val = [], []
for l in lambdas:
    theta = train(X_norm, y, l)
    errors_train.append(costReg(theta, X_norm, y, 0))
    errors_val.append(costReg(theta, Xval_norm, yval, 0))

plt.figure(figsize=(8,5))
plt.plot(lambdas, errors_train,label = 'Train')
plt.plot(lambdas, errors_val, label = 'Validation')
plt.legend()
plt.xlabel('lambda')
plt.ylabel('Error')
plt.grid(True)
#plt.show()
print(lambdas[np.argmin(errors_val)])

plot_polofit(train_means, train_stds, 3)
plot_learning_curve(X_norm, y, Xval_norm, yval, 3)
plt.show()

theta = train(X_norm, y, 3)
print('test cost(lambda = {}) = {}'.format(3, costReg(theta, Xtest_norm, ytest, 0)))


import numpy as np
import gc

class CustomSGDLinearRegressor:
    """
    A custom class implementation of Stochastic Gradient Descent for Linear Regression.

    Attributes:

        w (array(n_features)): The weights associated with each feature to be learnt using train data points.

        X (array(n_samples, n_features)): The training data points used for finding optimal weights and bias.

        y (array(n_samples)): The real values associated with the train data points which are to be predicted.

        learning_rate - learning_rate (float): The amount of gradient to be used to converge to the solution.

        k - batch_size (int): The # random batch points to be taken for calculating gradient.

        epochs - epochs (int): The # times whole data to be passed through the model.
    """

    def __init__(self, learning_rate=1e-3, epochs=512, batch_size=64):
        """
        The constructor for the class CustomSGDLinearRegressor.

        Parameters:

            learning_rate (float): The rate of how fast to converge to the solution.

            epochs (int): The number of times whole data to be passed for convergence.

            batch_size (int): The number of random points to be taken for calculating gradient.
        """

        # Initializing the attributes of the class.
        self.w = None
        self.X = None
        self.y = None
        self.iters = epochs
        self.k = batch_size
        self.learning_rate = learning_rate

    def fit(self, DTrain, y):
        """
        The method used to fit to the data provided.

        Parameters:

            DTrain (array(n_samples, n_features)): The data points for fitting the model.

            y (array(n_samples)): The real values to be predicted and for fitting the model.
        """

        # Preparing the Train data.
        self.X = np.array(DTrain)
        self.X = np.hstack(( self.X, np.ones(( self.X.shape[0], 1 )) ))

        self.y = np.array(y).reshape(-1)

        self.w = np.random.normal(size = self.X.shape[1])

        # Iterating through the complete data, basically number of epochs.
        for _ in range( self.iters ):
            self.findBestWeights()

        self.DeAllocateMemory()

    def DeAllocateMemory(self):
        self.X = None
        self.y = None
        gc.collect()

    def findBestWeights(self):
        """
        The method used for updating the weights based on learning rate.

        Parameters:
            None
        """

        opt_weights = self.w

        r = self.learning_rate

        # Selecting the K random batch points and getting the gradient value using the old weights.
        for _ in range( self.iters // self.k ):
            opt_weights = opt_weights + ( r * 2 * self.selectKPointsforWeights( opt_weights ) ) / self.k

            # Reducing the learning rate to avoid oscillation and jump over optimal solution.
            r /= 2

        self.w = opt_weights

    def selectKPointsforWeights(self, weights):
        """
        The method to get the gradient of weights for k random points.

        Parameters:
            weights (array(n_features)): The weights associated with each feature.

        Returns:
            float: The value of gradient at old weight.
        """

        # Generating K random batch indices.
        idxs = np.random.randint(0, self.X.shape[0], size = self.k)

        return np.matmul(self.y[idxs] - np.matmul(self.X[idxs], weights), self.X[idxs])

    def predict(self, DTest):
        """
        The method used to predict the real value given an unseen data point.

        Parameters:
            DTest (array(n_samples, n_features)): The data points for the model has to predict real value.

        Returns:
            float: The predicted real value using the optimal weights found by SGD.
        """

        # Preparing the Test data.
        DTest = np.array(DTest)
        DTest = np.hstack(( DTest, np.ones(( DTest.shape[0], 1 )) ))

        # Predicting the real value for the test data points.
        return np.round((self.w * DTest).sum(axis=1), 1)


import numpy as np

def logistic(x, a, b, r):

    return a / (1 + (b * np.exp(-1 * r * x)))

def exponential(x, a, k, b):
    return a*np.exp(x*k) + b

def logisticDistribution(x, a, b, r):

    return (a * r * b * np.exp(-1 * r * x)) / ((1 + (b * np.exp(-1 * r * x))) ** 2)


import numpy as np

def gradient_descent(x, y):
    m_curr = b_curr = 0
    iterations = 1250
    n = len(x)
    learning_rate = 0.08

    for i in range(iterations):
        y_predicted = m_curr * x + b_curr
        cost = (1 / n) * sum([val ** 2 for val in (y - y_predicted)])
        md = -(2 / n) * sum(x * (y - y_predicted))
        bd = -(2 / n) * sum(y - y_predicted)
        m_curr = m_curr - learning_rate * md
        b_curr = b_curr - learning_rate * bd
        print("m {}, b {}, cost {} iteration {}".format(m_curr, b_curr, cost, i))

x1 = np.array([1, 2, 3, 4, 5])
y1 = np.array([5, 7, 9, 11, 13])

gradient_descent(x1, y1)

Python/Machine Learning/Code Basics/ML Algorithm/4 Gradient Descent and Cost Function.py


# -*- coding: utf-8 -*-
"""
-------------------------------------------------
   File Name :      short_pad_attack
   Description :
   Author :         x3nny
   date :           2021/7/23
-------------------------------------------------
   Change Activity:
                    2021/7/23: Init
-------------------------------------------------
"""

# This file was *autogenerated* from the file /Users/x3nny/tools/pypackages/xenny/xenny/ctf/crypto/modern/asymmetric/rsa/copper_smith/short_pad_attack.sage
from sage.all_cmdline import *   # import sage library

_sage_const_0 = Integer(0); _sage_const_1 = Integer(1); _sage_const_30 = Integer(30); _sage_const_2 = Integer(2); _sage_const_0p5 = RealNumber('0.5'); _sage_const_25 = Integer(25)
__author__ = 'x3nny'

# Franklin-Reiter attack against RSA.
# If two messages differ only by a known fixed difference between the two messages
# and are RSA encrypted under the same RSA modulus N
# then it is possible to recover both of them.

# Inputs are modulus, known difference, ciphertext 1, ciphertext2.
# Ciphertext 1 corresponds to smaller of the two plaintexts. (The one without the fixed difference added to it)

def franklinReiter(n,e,r,c1,c2):
    R = Zmod(n)['X']; (X,) = R._first_ngens(1)
    f1 = X**e - c1
    f2 = (X + r)**e - c2
    # coefficient 0 = -m, which is what we wanted!
    return Integer(n-(compositeModulusGCD(f1,f2)).coefficients()[_sage_const_0 ])

# GCD is not implemented for rings over composite modulus in Sage
# so we do our own implementation. Its the exact same as standard GCD, but with
# the polynomials monic representation
def compositeModulusGCD(a, b):
    if(b == _sage_const_0 ):
        return a.monic()
    else:
        return compositeModulusGCD(b, a % b)

def CoppersmithShortPadAttack(e,n,C1,C2,eps=_sage_const_1 /_sage_const_30 ):
    """
    Coppersmith's Shortpad attack!
    Figured out from: https://en.wikipedia.org/wiki/Coppersmith's_attack#Coppersmith.E2.80.99s_short-pad_attack
    """
    P = PolynomialRing(ZZ, names=('x', 'y',)); (x, y,) = P._first_ngens(2)
    ZmodN = Zmod(n)
    g1 = x**e - C1
    g2 = (x+y)**e - C2
    res = g1.resultant(g2)
    P = PolynomialRing(ZmodN, names=('y',)); (y,) = P._first_ngens(1)
    # Convert Multivariate Polynomial Ring to Univariate Polynomial Ring
    rres = _sage_const_0
    for i in range(len(res.coefficients())):
        rres += res.coefficients()[i]*(y**(res.exponents()[i][_sage_const_1 ]))

    diff = rres.small_roots(epsilon=eps)
    if diff:
        return diff[_sage_const_0 ]
    return None

def another_get_diff(e, n, c1, c2):
    PRxy = PolynomialRing(Zmod(n), names=('x', 'y',)); (x, y,) = PRxy._first_ngens(2)
    PRx = PolynomialRing(Zmod(n), names=('xn',)); (xn,) = PRx._first_ngens(1)
    PRZZ = PolynomialRing(Zmod(n), names=('xz', 'yz',)); (xz, yz,) = PRZZ._first_ngens(2)

    g1 = x ** e - c1
    g2 = (x + y) ** e - c2
    q1 = g1.change_ring(PRZZ)
    q2 = g2.change_ring(PRZZ)
    h = q2.resultant(q1)
    h = h.univariate_polynomial()
    h = h.change_ring(PRx).subs(y=xn)
    h = h.monic()
    kbits = n.bit_length() // (_sage_const_2  * e * e)
    diff = h.small_roots(X=_sage_const_2 **kbits, beta=_sage_const_0p5 )[_sage_const_0 ]  # find root < 2^kbits with factor >= n^0.5
    return diff

def getdiff(e, n, c1, c2, eps):
    diff = CoppersmithShortPadAttack(e, n, c1, c2, eps)
    if diff:
        return diff
    return another_get_diff(e, n, c1, c2)

def attack(e, n, c1, c2, r=None, eps=_sage_const_1 /_sage_const_25 ):
    if r is None:
        r = getdiff(e, n, c1, c2, eps)
    m = franklinReiter(n,e,r,c1,c2)
    return m


# coding: utf-8

# In[1]:

#import packages
import time
import numpy as np
from matplotlib import pyplot as plt
import seaborn as sns
from pandas import Series, DataFrame
import pandas as pd
get_ipython().magic(u'matplotlib inline')

# ### 1. Import Data

# In[3]:

#Read in data from a data file to data_df in DateFrame format

## Import data from poly_data.csv to data_df
data_df = pd.read_csv("poly_data.csv")

print(data_df.head(6))

# ### 2. Observe Data

# In[4]:

#joint plot (or scatter plot) of X1 and y
sns.jointplot(data_df['X1'], data_df['y'])

# In[5]:

#joint plot (or scatter plot) of X2 and y
sns.jointplot(data_df['X2'], data_df['y'])

# In[6]:

#joint plot (or scatter plot) of X1 and X2
sns.jointplot(data_df['X1'], data_df['X2'])

# ### Based on observing the above 3 diagrams and the p-values displayed, we found both X1 and X2 have close correlation with y. X1 and X2 are independent from each other.

# ### 3. Split the Data

# In[7]:

# split the data into training and testing datasets
# the percentage of training data is 75%

#split point
percentage_for_training = 0.75

#number of training data
number_of_training_data = int(data_df.shape[0]*percentage_for_training)

#create training and testing datasets
train_df  = data_df[0:number_of_training_data]
test_df = data_df[number_of_training_data:]
print(train_df.shape)
print(test_df.shape)

# ### 4. Create Polynomial Features

# In[8]:

from sklearn.preprocessing import PolynomialFeatures
from sklearn.linear_model import LinearRegression

polynomial_features = PolynomialFeatures(degree=3)

# In[9]:

X_poly = polynomial_features.fit_transform(data_df[['X1','X2']])

X_train = X_poly[0:number_of_training_data]
X_test = X_poly[number_of_training_data:]

# ### 5. Create and Train a Linear Regression Model

# In[10]:

# mse() calculates mean square error of a model on given X and y
def mse(X, y, model):
    return  ((y-model.predict(X))**2).sum()/y.shape[0]

# In[11]:

# use all the features to train the linear model
lm = LinearRegression()
lm.fit(X_train, train_df['y'])
train_mse = mse(X_train, train_df['y'], lm)
print("Training Data Set's MSE is: \t", train_mse)
test_mse = mse(X_test, test_df['y'], lm)
print("Testing Data Set's MSE is : \t", test_mse)

# # REFLECTION

# ### 6. Use Lasso in Linear Regression to Penalize Large Number of Features

# In[12]:

from sklearn.linear_model import Lasso

#Train the model, try different alpha values.
Lasso_model = Lasso(alpha=0.15,normalize=True, max_iter=1e5, )
Lasso_model.fit(X_train, train_df['y'])

# In[13]:

#see the trained parameters. Zero means the feature can be removed from the model
Lasso_model.coef_

# In[14]:

#let's see the train_mse and test_mse from Lasso when
#alpha = 0.15

train_mse = mse(X_train, train_df['y'], Lasso_model)
print("Training Data Set's MSE is: \t", train_mse)
test_mse = mse(X_test, test_df['y'], Lasso_model)
print("Testing Data Set's MSE is : \t", test_mse)

# In[15]:

#let's try a large range of values for alpha first
#create 50 alphas from 100 to 0.00001 in logspace
alphas = np.logspace(2, -5, base=10, num=50)
alphas

# In[16]:

#use arrays to keep track of the MSE of each alpha used.
train_mse_array =[]
test_mse_array=[]

#try each alpha
for alpha in alphas:

    #create Lasso model using alpha
    Lasso_model = Lasso(alpha=alpha,normalize=True, max_iter=1e5, )
    Lasso_model.fit(X_train, train_df['y'])

    #Calculate MSEs of train and test datasets
    train_mse = mse(X_train, train_df['y'], Lasso_model)
    test_mse = mse(X_test, test_df['y'], Lasso_model)

    #add the MSEs to the arrays
    train_mse_array.append(train_mse)
    test_mse_array.append(test_mse)

# In[17]:

#plot the MSEs based on alpha values
#blue line is for training data
#red line is for the testing data
plt.plot(np.log10(alphas), train_mse_array)
plt.plot(np.log10(alphas), test_mse_array, color='r')

# ### There is something interesting between 0 and 1 in the above diagram. 0 mean 10^0=1 While 1 means 10^1 = 10  so, we will look closely within this range to find the optimal alpha value
#

# In[18]:

# We can try a smaller search space now (a line space between 1 and 10)
alphas = np.linspace(1, 10, 1000)
train_mse_array =[]
test_mse_array=[]
alp = []

i = 0
for alpha in alphas:

    #create Lasso model using alpha
    Lasso_model = Lasso(alpha=alpha,normalize=True, max_iter=1e5, )
    Lasso_model.fit(X_train, train_df['y'])

    #Calculate MSEs of train and test datasets
    train_mse = mse(X_train, train_df['y'], Lasso_model)
    test_mse = mse(X_test, test_df['y'], Lasso_model)

    #add the MSEs to the arrays
    train_mse_array.append(train_mse)
    test_mse_array.append(test_mse)
    alp.append(alpha)

    diff = train_mse - test_mse

    if (diff > 0 and i != 1):
        opt_alpha = alpha
        tr_mse = train_mse
        te_mse = test_mse
        i = 1

print("The optimal alpha is", opt_alpha)
print("Train MSE is", tr_mse)
print("Test MSE is", te_mse)

plt.plot(alp, train_mse_array)
plt.plot(alp, test_mse_array, color='r')

# ### By observing a smaller range of alpha, we can clearly see how the MSEs change as we change the model and features. Use the diagram to explain the trends of the two lines and summarize what you learned so far.

# In[20]:

print("From the above diagram, I found that the optimized alpha value is 3.567. This means that, at this alpha value, we can prevent overfitting and underfitting. Lasso model with alpha value below 3.567 is underfitting and above 3.567 is overfitting the data. Instead of using linear model, I used Lasso linear model which is used to penalize number of features with low variance or correlation. These features can be found by looking at the coefficients of the Lasso linear model.\n")

print("Overall, from the dataset, I found the polynomial that best fits the data. I checked my model by dividing the dataset into training data and testing data. I generated the model using the training data of 75% and then test the model using the testing data of 25%. To prevent overfitting and underfitting, I generated Lasso linear model with the optimized alpha value to penalize the features and retain only the required features. Thus, I reached the best alpha value by minimizing the mean square error between training data and testing data.")

# In[21]:

##Thus, above cells fulfill the assignment.


# -*- coding: utf-8 -*-
"""
Created on Wed Mar  6 21:47:41 2019

@author: Ham

HackerRanch Challenge: Shape and Reshape

shape

The shape tool gives a tuple of array dimensions
and can be used to change the dimensions of an array.

(a). Using shape to get array dimensions

import numpy

my_1D_array = numpy.array([1, 2, 3, 4, 5])
print my_1D_array.shape     #(5,) -> 5 rows and 0 columns

my_2D_array = numpy.array([[1, 2],[3, 4],[6,5]])
print my_2D_array.shape     #(3, 2) -> 3 rows and 2 columns

(b). Using shape to change array dimensions

import numpy

change_array = numpy.array([1,2,3,4,5,6])
change_array.shape = (3, 2)
print change_array

#Output
[[1 2]
[3 4]
[5 6]]

reshape

The reshape tool gives a new shape to an array without changing its data.
It creates a new array and does not modify the original array itself.

import numpy

my_array = numpy.array([1,2,3,4,5,6])
print numpy.reshape(my_array,(3,2))

#Output
[[1 2]
[3 4]
[5 6]]

Task

You are given a space separated list of nine integers.
Your task is to convert this list into a 3x3 NumPy array.

Input Format

A single line of input containing 9 space separated integers.

Output Format

Print the 3x3 NumPy array.

Sample Input

1 2 3 4 5 6 7 8 9

Sample Output

[[1 2 3]
 [4 5 6]
 [7 8 9]]

"""

import numpy

#arr = numpy.array(list(map(int, input().strip().split())))
print(numpy.reshape(numpy.array(list(map(int, input().strip().split()))), (3, 3)))


import numpy as np
from numpy import linalg as LA

class LinearRegression:
    """
    線形回帰を行うモデル

    Parameters
    ----------
    intercept: boolean(default=True)
        切片要素を入れるかどうか

    Examples
    ---------
    >>> from load_data import load_boston
    >>> (X_train, y_train), (X_test, y_test) = load_boston()
    >>> reg = LinearRegression()
    >>> reg.fit(X_train, y_train)
    >>> reg.b
    array([ 4.02936706e+01, -1.19997513e-01,  5.70003304e-02,  3.98379660e-03,
            4.12698187e+00, -2.05002963e+01,  3.38024903e+00,  7.56807584e-03,
           -1.71189793e+00,  3.34747537e-01, -1.17797225e-02, -9.02318039e-01,
            8.71912756e-03, -5.55842510e-01])
    >>> reg.score(X_test, y_test)
    23.19559925642053
    """
    def __init__(self, intercept=True):
        self.intercept = intercept

    def fit(self, X, y):
        """
        学習データにフィットさせる

        Parameters
        ----------
        X: array, shape=(samples, colunms)
            説明変数の行列

        y: vector, len=(samples)
            目的変数のベクトル

        Attributes
        ----------
        b: vector
            係数のベクトル

        _error: float
            最適な係数の誤差
        """
        if self.intercept:
            self.b = self._solve(np.hstack((np.ones((X.shape[0], 1)), X)), y)
        else:
            self.b = self._solve(X, y)

        self._error = self._culc_error(y, self.predict(X))

    def predict(self, X):
        """
        fitメソッドで算出した係数ベクトルを用いて予測を行う

        Parameters
        ----------
        X: array, shape=(samples, columns)
            予測したいデータの説明変数

        Returns
        -------
        y: vector, len=(samples)
            予測された目的変数
        """
        if self.intercept:
            y = np.hstack((np.ones((X.shape[0], 1)), X)).dot(self.b)
        else:
            y = X.dot(self.b)

        return y

    def _solve(self, X, y):
        return LA.inv(X.T.dot(X)).dot(X.T.dot(y))

    def _culc_error(self, y, y_pred):
        return LA.norm(y - y_pred)**2 / y.shape[0]

    def score(self, X, y):
        """
        モデルの平均二乗誤差を求める

        Parameters
        ----------
        X: array, shape=(samples, columns)
            説明変数の行列

        y: vector, len=(samples)
            目的変数のベクトル

        Returns
        -------
        error: float
            誤差
        """
        return self._culc_error(y, self.predict(X))

class KNNRegression:
    """
    K近傍法(k-nearest neighbor)による回帰を行う

    Parameters
    ----------
    k: int (default=5)
        考慮する最近傍データの数

    weights: str (default="same")
        重み付けの有無(デフォルトは重み付け無し)

        距離に応じた重みを考慮するときは"distance"を指定

    practice: int (default=2)
        距離計算方法

        * 1:  マンハッタン距離
        * 2:  ユークリッド距離
        * <3: 任意の次元のミンコフスキー距離

    Examples
    --------
    >>> from load_data import load_boston
    >>> (X_train, y_train), (X_test, y_test) = load_boston()
    >>> reg = KNNRegression()
    >>> reg.fit(X_train, y_train)
    >>> reg.score(X_test, y_test)
    36.969313725490196
    """
    def __init__(self, k=5, weight="same", practice=2):
        if type(k) is not int:
            raise TypeError(
                    "k should be int.")
        if weight not in ["same", "distance"]:
            raise ValueError(
                    "weight not recognized: should be 'same' or 'distance'.")
        if type(practice) is not int:
            raise TypeError(
                    "practice should be int.")

        self.k = k
        self.weight = weight
        self.practice = practice

    def fit(self, X, y):
        """
        学習データをインプットする

        Parameters
        ----------
        X: array, shape=(samples, columns)
            説明変数の行列

        y: vector, len=(samples)
            目的変数のベクトル
        """
        self.X = X
        self.y = y

    def _culc_distance(self, sample):
        """
        あるsampleについてトレーニングデータとの距離を求める

        Parameters
        ----------
        sample: vector
            サンプルの特徴量を並べたベクトル

        Returns
        -------
        distance: vector
            各トレーニングデータとの距離
        """
        distance = np.abs(self.X - sample)**self.practice
        return np.sum(distance, axis=1)

    def predict(self, samples):
        """
        複数のsampleについて予測を行う

        Parameters
        ----------
        samples: array, shape=(samples, columns)
            予測したいサンプルの行列

        Returns
        -------
        y: vector, len=(samples)
            予測されたカテゴリ
        """
        y = np.zeros(samples.shape[0])
        for i, sample in enumerate(samples):
            y[i] = self._predict_one(sample)
        return y

    def _predict_one(self, sample):
        """
        １つのサンプルがどのカテゴリに入っているかを確認する

        Parameters
        ----------
        sample: vector
            サンプルの特徴量を並べたベクトル

        Returns
        -------
        result: int
            予測されたカテゴリ番号
        """
        dis = self._culc_distance(sample)
        index = np.arange(self.X.shape[0])
        index = index[np.argsort(dis, axis=0)]
        result = 0

        if self.weight == "same":
            for i in range(self.k):
                result += self.y[index[i]]
            result /= self.k
        elif self.weight == "distance":
            distance = 0
            for i in range(self.k):
                result += self.y[index[i]] * dis[index[i]]
                distance += dis[index[i]]
            result /= distance

        return result

    def score(self, X, y):
        """
        モデルの平均二乗誤差を求める

        Parameters
        ----------
        X: array, shape=(samples, columns)
            説明変数の行列

        y: vector, len=(samples)
            目的変数のベクトル

        Returns
        -------
        error: float
            誤差
        """
        return self._culc_error(y, self.predict(X))

    def _culc_error(self, y, y_pred):
        return LA.norm(y - y_pred)**2 / y.shape[0]


# exercise 2.1.3
# (requires data structures from ex. 2.2.1)
from ex2_1_1 import *

import matplotlib.pyplot as plt
from scipy.linalg import svd

# Subtract mean value from data
Y = X - np.ones((N,1))*X.mean(axis=0)

# PCA by computing SVD of Y
U,S,V = svd(Y,full_matrices=False)

# Compute variance explained by principal components
rho = (S*S) / (S*S).sum()

threshold = 0.9

# Plot variance explained
plt.figure()
plt.plot(range(1,len(rho)+1),rho,'x-')
plt.plot(range(1,len(rho)+1),np.cumsum(rho),'o-')
plt.plot([1,len(rho)],[threshold, threshold],'k--')
plt.title('Variance explained by principal components');
plt.xlabel('Principal component');
plt.ylabel('Variance explained');
plt.legend(['Individual','Cumulative','Threshold'])
plt.grid()
plt.show()

print('Ran Exercise 2.1.3')


import numpy as np
import scipy.stats as st

y = np.array([-67, -48, 6, 8, 14, 16, 23, 24, 28, 29, 41, 49, 56, 60, 75])
model_num = 2
gamma = np.zeros((y.size, model_num), dtype=float)
alpha = [0.5, 0.5]
miu = [1, 0]
theta = [10, 10]

while True:
    alpha_ori = alpha.copy()
    miu_ori = miu.copy()
    theta_ori = theta.copy()
    for j in range(y.size):
        pdf_j = []
        for k in range(model_num):
            pdf_j.append(alpha[k] * st.norm.pdf(y[j], loc=miu[k], scale=theta[k]))

        pdf_sum_j = np.sum(pdf_j)
        for k in range(model_num):
            gamma[j, k] = pdf_j[k] / pdf_sum_j
    for k in range(model_num):
        theta[k] = np.sqrt(np.sum((y - miu[k]) ** 2 * gamma[:, k]) / np.sum(gamma[:, k]))
        alpha[k] = np.sum(gamma[:, k]) / y.size
        miu[k] = np.sum(y * gamma[:, k]) / (np.sum(gamma[:, k]))

    print("miu", miu)
    print("theta", theta[0] ** 2, theta[1] ** 2)
    print("alpha", alpha)
    if np.linalg.norm(np.array(alpha + miu + theta) - np.array(alpha_ori + miu_ori + theta_ori)) < 0.5:
        break


# Adapted from https://github.com/Pyomo/PyomoGettingStarted,
# by William and Becca Hart

from pyomo.environ import *

model = AbstractModel()

model.m = Param(within=NonNegativeIntegers)
model.n = Param(within=NonNegativeIntegers)

model.I = RangeSet(1, model.m)
model.J = RangeSet(1, model.n)

model.a = Param(model.I, model.J)
model.b = Param(model.I)
model.c = Param(model.J)

# the next line declares a variable indexed by the set J
model.x = Var(model.J, domain=NonNegativeReals)

def obj_expression(model):
    return summation(model.c, model.x)

model.OBJ = Objective(rule=obj_expression)

def ax_constraint_rule(model, i):
    # return the expression for the constraint for i
    return sum(model.a[i,j] * model.x[j] for j in model.J) >= model.b[i]

# the next line creates one constraint for each member of the set model.I
model.AxbConstraint = Constraint(model.I, rule=ax_constraint_rule)


import numpy as np
from scipy.signal import gaussian

def gaussian_filter(y_size, x_size, sigma):
    ret =  np.outer(gaussian(y_size, sigma), gaussian(x_size, sigma))
    return ret / np.sum(ret)

def mexican_hat(x_size,y_size,width):
    '''generate a 2D-mexican hat filter'''
    x_linear=np.arange(-np.floor(x_size/2.0),np.ceil(x_size/2.0))
    y_linear=np.arange(-np.floor(y_size/2.0),np.ceil(y_size/2.0))
    x_grid, y_grid = np.meshgrid(x_linear,y_linear)
    t = np.sqrt(x_grid**2 + y_grid**2)

    mexican = 2.0 / np.sqrt(3.0) / width / np.pi**(1/4) * (1-1/2.0*t**2/width**2) \
            * np.exp(-t**2/2/width**2)
    return np.real(mexican)


import sys, os
sys.path.insert(0, os.path.join(os.pardir, os.pardir, 'vib', 'src-vib'))
import numpy as np
from devito import Dimension, TimeFunction, Eq, solve, Operator, Constant

def solver_v1(I, w, dt, T):
    """
    Solve u'=v, v' = - w**2*u for t in (0,T], u(0)=I and v(0)=0,
    by a central finite difference method with time step dt.
    """
    dt = float(dt)
    Nt = int(round(T/dt))

    t = Dimension('t', spacing=Constant('h_t'))
    u = TimeFunction(name='u', dimensions=(t,), shape=(Nt+1,), space_order=2)
    v = TimeFunction(name='v', dimensions=(t,), shape=(Nt+1,), space_order=2)

    u.data[:] = I
    v.data[:] = 0 - 0.5*dt*w**2*u.data[:]

    eq_u = Eq(u.dt, v)
    eq_v = Eq(v.dt, -(w**2)*u.forward)

    stencil_u = solve(eq_u, u.forward)
    stencil_v = solve(eq_v, v.forward)

    update_u = Eq(u.forward, stencil_u)
    update_v = Eq(v.forward, stencil_v)

    op = Operator([update_u, update_v])
    op.apply(h_t=dt, t_M=Nt-1)

    t_mesh = np.linspace(0, Nt*dt, Nt+1)    # mesh for u
    t_v_mesh = (t_mesh + dt/2)[:-1]         # mesh for v

    return u.data, t_mesh, v.data, t_v_mesh

class HalfInt:
    """
    Class for allowing to write n+half and mean n,
    while n-half is n-1. Used for nice notation in staggered
    meshes.
    """
    def __radd__(self, other):
        return other

    def __rsub__(self, other):
        return other - 1

half = HalfInt()  # singleton object

def solver(I, w, dt, T):
    """
    Solve u'=v, v' = - w**2*u for t in (0,T], u(0)=I and v(0)=0,
    by a central finite difference method with time step dt on
    a staggered mesh with v as unknown at (i+1/2)*dt time points.
    """
    dt = float(dt)
    Nt = int(round(T/dt))
    u = np.zeros(Nt+1)
    v = np.zeros(Nt+1)
    t = np.linspace(0, Nt*dt, Nt+1)  # mesh for u
    t_v = t + dt/2                # mesh for v

    u[0] = I
    v[0+half] = 0 - 0.5*dt*w**2*u[0]
    for n in range(1, Nt+1):
        u[n] = u[n-1] + dt*v[n-half]
        v[n+half] = v[n-half] - dt*w**2*u[n]
    return u, t, v[:-1], t_v[:-1]

def test_staggered():
    I = 1.2; w = 2.0; T = 5; dt = 2/w
    u, t, v, t_v = solver(I, w, dt, T)
    from vib_undamped import solver as solver2
    u2, t2 = solver2(I, w, dt, T)
    error = abs(u - u2).max()
    tol = 1E-14
    assert error < tol

def test_convergence():
    """Verify 2nd-order convergence."""
    from vib_undamped import convergence_rates
    def wrapped_solver(I, w, dt, T):
        u, t, v, t_v = solver(I, w, dt, T)
        return u, t

    r = convergence_rates(8, wrapped_solver, 8)
    print(r)
    assert abs(r[-1] - 2) < 1E-5

if __name__ == '__main__':
    test_staggered()
    test_convergence()


#!/usr/bin/env python

__author__ = "bt3"

import random

''' The simplest way...'''
def quickSelect(seq, k):
    # this part is the same as quick sort
    len_seq = len(seq)
    if len_seq < 2: return seq

    # we could use a random choice here doing
    #pivot = random.choice(seq)
    ipivot = len_seq // 2
    pivot = seq[ipivot]

    # O(n)
    smallerList = [x for i,x in enumerate(seq) if x <= pivot and  i != ipivot]
    largerList = [x for i,x in enumerate(seq) if x > pivot and  i != ipivot]

    # here starts the different part
    m = len(smallerList)
    if k == m:
        return pivot
    elif k < m:
        return quickSelect(smallerList, k)
    else:
        return quickSelect(largerList, k-m-1)

''' If you don't want to use pythons feature at all and
    also select pivot randomly'''

def swap(seq, x, y):
    tmp = seq[x]
    seq[x] = seq[y]
    seq[y] = tmp

def quickSelectHard(seq, k, left=None, right=None):
    left = left or 0
    right = right or len(seq) - 1
    #ipivot = random.randint(left, right)
    ipivot = len(seq)//2
    pivot = seq[ipivot]

    # Move pivot out of the sorting range
    swap(seq, ipivot, right)
    swapIndex, i = left, left
    while i < right:
        if seq[i] < pivot:
            swap(seq, i, swapIndex)
            swapIndex += 1
        i += 1

    # Move pivot to final position
    swap(seq, right, swapIndex)

    # Check if pivot matches, else recurse on the correct half
    rank = len(seq) - swapIndex

    if k == rank:
        return seq[swapIndex]
    elif k < rank:
        return quickSelectHard(seq, k, swapIndex+1, right)
    else:
        return quickSelectHard(seq, k, left, swapIndex-1)

if __name__ == '__main__':
    # Checking the Answer
    seq = [10, 60, 100, 50, 60, 75, 31, 50, 30, 20, 120, 170, 200]
    #seq = [3, 7, 2, 1, 4, 6, 5, 10, 9, 11]

    # we want the middle element
    k = len(seq) // 2

    # Note that this only work for odd arrays, since median in
    # even arrays is the mean of the two middle elements
    print(quickSelect(seq, k))
    print(quickSelectHard(seq, k))
    import numpy
    print numpy.median(seq)


import numpy as np
from scipy import linalg

def norm_of_columns(A, p=2):
    """Vector p-norm of each column of a matrix.

    Parameters
    ----------
    A : array_like
        Input matrix.
    p : int, optional
        p-th norm.

    Returns
    -------
    array_like
        p-norm of each column of A.
    """
    _, N = A.shape
    return np.asarray([linalg.norm(A[:, j], ord=p) for j in range(N)])

def coherence_of_columns(A):
    """Mutual coherence of columns of A.

    Parameters
    ----------
    A : array_like
        Input matrix.
    p : int, optional
        p-th norm.

    Returns
    -------
    array_like
        Mutual coherence of columns of A.
    """
    A = np.asmatrix(A)
    _, N = A.shape
    A = A * np.asmatrix(np.diag(1/norm_of_columns(A)))
    Gram_A = A.H*A
    for j in range(N):
        Gram_A[j, j] = 0
    return np.max(np.abs(Gram_A))

def asarray_1d(a, **kwargs):
    """Squeeze the input and check if the result is one-dimensional.

    Returns *a* converted to a `numpy.ndarray` and stripped of
    all singleton dimensions.  Scalars are "upgraded" to 1D arrays.
    The result must have exactly one dimension.
    If not, an error is raised.

    """
    result = np.squeeze(np.asarray(a, **kwargs))
    if result.ndim == 0:
        result = result.reshape((1,))
    elif result.ndim > 1:
        raise ValueError("array must be one-dimensional")
    return result

def matdiagmul(A, b):
    """Efficient multiplication of  matrix and diagonal matrix .

    Returns the multiplication of a matrix *A* and a diagonal matrix. The
    diagonal matrix is given by the vector *b* containing its elements on
    the main diagonal.  If *b* is a matrix, it is treated as a stack of vectors
    residing in the last index and broadcast accordingly.

    Parameters
    ----------
    A : array_like
        Input matrix.
    b : array_like
        Main diagonal elements or stack of main diagonal elements.

    Returns
    -------
    array_like
        Result of matrix multiplication.
    """
    if len(b.shape) == 1:
        b = b[np.newaxis, :]
    K, N = b.shape
    M, N = A.shape

    C = np.zeros([K, M, N], dtype=A.dtype)
    for k in range(K):
        C[k, :, :] = A * b[k, :]
    return C

def db(x, power=False):
    """Convert *x* to decibel.

    Parameters
    ----------
    x : array_like
        Input data.  Values of 0 lead to negative infinity.
    power : bool, optional
        If ``power=False`` (the default), *x* is squared before
        conversion.

    """
    with np.errstate(divide='ignore'):
        return 10 if power else 20 * np.log10(np.abs(x))


import itertools
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
from scipy.stats import bernoulli, binom, expon, poisson, norm, skewnorm
import seaborn as sns
from statsmodels.distributions.empirical_distribution import ECDF
from statsmodels.tsa.seasonal import seasonal_decompose

def _non_symmetric_data():
    """Generate non-symmetric data for plots"""
    # generate data
    np.random.seed(0)
    return pd.Series(
        np.random.gamma(7, 5, size=1000) * np.random.choice([-2.2, -1.85, 0, -0.4, 1.33], size=1000), name='x'
    )

def anscombes_quartet():
    """Plot Anscombe's Quartet along with summary statistics."""

    # get data
    anscombe = sns.load_dataset('anscombe').groupby('dataset')

    # define subplots and titles
    fig, axes = plt.subplots(2, 2, figsize=(12, 12))
    axes = axes.flatten()
    titles = ['linear', 'non-linear', 'linear with outlier', 'vertical with outlier']

    for ax, (group_name, group_data), title in zip(axes, anscombe, titles):
        # get x, y
        x, y = group_data.x, group_data.y

        # make a scatter plot
        ax.scatter(x, y)

        # add title and labels
        ax.set_title(f'{group_name} - {title}')
        ax.set_xlabel('x')
        ax.set_ylabel('y')

        # align the axes limits
        ax.set_xlim((3, 19.5))
        ax.set_ylim((2, 13))

        # plot the regression line
        m, b = np.polyfit(x, y, 1)
        reg_x = np.append([0, 20], x)
        reg_y = [m*num + b for num in reg_x]
        ax.plot(reg_x, reg_y, 'r--')

        # annotate the summary statistics
        ax.annotate(
            f"""ρ = {np.corrcoef(x,y)[0][1]:.2}\ny = {m:.2}x + {b:.2}\n\n{
                r'$μ_x$'
            } = {np.mean(x):2} | {
                r'$σ_x$'
            } = {np.std(x):.2}\n{
                r'$μ_y$'
            } = {np.mean(y):.2} | {r'$σ_y$'} = {np.std(y):.2}""", xy=(13, 2.5)
        )

    # give the plots a title
    plt.suptitle("Anscombe's Quartet", fontsize=16, y=0.95)

    return axes

def cdf_example():
    """Subplots to understand CDF."""
    data = _non_symmetric_data()
    ecdf = ECDF(data)

    fig, axes = plt.subplots(1, 3, figsize=(15, 3))

    for ax in axes:
        ax.plot(ecdf.x, ecdf.y)
        ax.set_xlabel('x')
        ax.set_ylabel('F(x)')

    # less than or equal to 50
    axes[0].fill_between(ecdf.x[ecdf.x <= 50], ecdf.y[ecdf.x <= 50], 0, alpha=0.5)
    axes[0].axhline(0.93, xmax=0.76, linestyle='dashed')
    axes[0].set_title(r'$P(X \leq 50) \approx 93\%$')

    # equal to 50
    axes[1].fill_between(ecdf.x[ecdf.x == 50], ecdf.y[ecdf.x == 50], 0, alpha=0.5)
    axes[1].set_title(r'$P(X = 50) = 0\%$')

    # greater than 50
    axes[2].fill_between(ecdf.x[ecdf.x >= 50], ecdf.y[ecdf.x >= 50], 0, alpha=0.5)
    axes[2].axhline(0.93, xmax=0.76, linestyle='dashed')
    axes[2].set_title(r'$P(X > 50) = 1 - P(X \leq 50) \approx 7\%$')

    plt.suptitle('Understanding the CDF', y=1.1)

    return axes

def common_dists():
    """Show some commonly used distributions."""
    # prep the subplots
    fig, axes = plt.subplots(2, 3, figsize=(15, 10))
    axes = axes.flatten()

    # gaussian
    mu, sigma = 0, 1
    x = np.linspace(mu - 3*sigma, mu + 3*sigma, 100)
    axes[0].plot(x, norm.pdf(x, mu, sigma))
    axes[0].set_title('Gaussian PDF')
    axes[0].set_ylabel('density')
    axes[0].set_xlabel('x')
    axes[0].annotate(r'$\mu$', xy=(mu, 0.4), xytext=(mu - 0.09, 0.3), arrowprops=dict(arrowstyle='->'))
    axes[0].annotate(
        '', xy=(mu - sigma, 0.25), xytext=(mu + sigma, 0.25),
        arrowprops=dict(arrowstyle='|-|, widthB=0.5, widthA=0.5')
    )
    axes[0].annotate(r'$2\sigma$', xy=(mu - 0.15, 0.22))

    # uniform distribution defined by min (a) and max (b)
    a, b = 0, 1
    peak = 1 / (b - a)
    axes[1].plot([a, a, b, b], [0, peak, peak, 0])
    axes[1].set_title('Uniform PDF')
    axes[1].set_ylabel('density')
    axes[1].set_xlabel('x')
    axes[1].annotate('min', xy=(a, peak), xytext=(a + 0.2, peak - 0.2), arrowprops=dict(arrowstyle='->'))
    axes[1].annotate('max', xy=(b, peak), xytext=(b - 0.3, peak - 0.2), arrowprops=dict(arrowstyle='->'))
    axes[1].set_ylim(0, 1.5)

    # exponential
    x = np.linspace(0, 5, 100)
    axes[2].plot(x, expon.pdf(x, scale=1/3))
    axes[2].set_title('Exponential PDF')
    axes[2].set_ylabel('density')
    axes[2].set_xlabel('x')
    axes[2].annotate(r'$\lambda$ = 3', xy=(0, 3), xytext=(0.5, 2.8), arrowprops=dict(arrowstyle='->'))

    # Bernoulli of coin toss
    axes[3].bar(['heads', 'tails'], bernoulli.pmf([0, 1], p=0.5))
    axes[3].set_title('Bernoulli with fair coin toss (p = 0.5)')
    axes[3].set_ylabel('probability')
    axes[3].set_xlabel('coin toss result')
    axes[3].set_ylim(0, 1)

    # Binomial of tossing a fair coin many times
    x = np.arange(0, 10)
    axes[4].plot(x, binom.pmf(x, n=x.shape, p=0.5), linestyle='--', marker='o')
    axes[4].set_title('Binomial PMF - many Bernoulli trials')
    axes[4].set_ylabel('probability')
    axes[4].set_xlabel('number of heads')

    # Poisson PMF (probability mass function) because this is a discrete random variable
    x = np.arange(0, 10)
    axes[5].plot(x, poisson.pmf(x, mu=3), linestyle='--', marker='o')
    axes[5].set_title('Poisson PMF')
    axes[5].set_ylabel('mass')
    axes[5].set_xlabel('x')
    axes[5].annotate(r'$\lambda$ = 3', xy=(3, 0.225), xytext=(1.9, 0.2), arrowprops=dict(arrowstyle='->'))

    # add a title
    plt.suptitle('Some commonly used distributions', fontsize=15, y=0.95)

    return axes

def correlation_coefficient_examples():
    """Show some examples of scatter plots with correlation coefficients."""
    # starting data
    np.random.seed(0)
    x = np.random.normal(size=100)
    y = np.random.normal(size=100)

    # make subplots
    fig, axes = plt.subplots(1, 4, figsize=(16, 3))

    # no correlation
    axes[0].scatter(x, y)
    axes[0].set_title(f'ρ = {np.round(np.corrcoef(x, y)[0][1], 2)}')

    # weak negative correlation
    a, b = x, (x + y*2)*-1
    axes[1].scatter(a, b)
    axes[1].set_title(f'ρ = {np.round(np.corrcoef(a, b)[0][1], 2)}')

    # strong positive correlation
    s, t = x, (x - np.random.uniform(1, 3, size=100))
    axes[2].scatter(s, t)
    axes[2].set_title(f'ρ = {np.round(np.corrcoef(s, t)[0][1], 2)}')

    # perfect negative correlation
    c, d = x, (x - y*.1) * -1
    axes[3].scatter(c, d)
    axes[3].set_title(f'ρ = {np.round(np.corrcoef(c, d)[0][1], 2)}')

    for ax in axes:
        ax.set_xlabel('x')
        ax.set_ylabel('y')

    return axes

def different_modal_plots():
    """Show unimodal, bimodal, and multimodal example distributions."""

    # distribution details
    x = np.linspace(-4, 4, 500)
    loc1, scale1, size1 = (-2, 0.75, 150)
    loc2, scale2, size2 = (3, 0.5, 50)
    loc3, scale3, size3 = (0.4, 1, 150)

    # make subplots
    fig, axes = plt.subplots(1, 3, figsize=(15, 3))

    # plot unimodal
    axes[0].plot(x, norm.pdf(x))

    # plot bimodal
    bimodal_pdf = norm.pdf(x, loc=loc1, scale=scale1) * float(size1) / (size1 + size2) + \
       norm.pdf(x, loc=loc2, scale=scale2) * float(size2) / (size1 + size2)
    axes[1].plot(x, bimodal_pdf)

    # plot multimodal
    multimodal_pdf = bimodal_pdf + norm.pdf(x, loc=loc3, scale=scale3) * float(size3) / (size1 + size2)
    axes[2].plot(x, multimodal_pdf)

    # label everything and format
    for ax, title in zip(axes, ['unimodal', 'bimodal', 'multimodal']):
        ax.set_ylim(0, 0.45)
        ax.set_xlabel('x')
        ax.set_ylabel('density')
        ax.set_title(title)

    return axes

def effect_of_std_dev():
    """Display two normal distributions with different standard deviations."""
    np.random.seed(0)
    data = pd.DataFrame({
        'σ = 0.5': np.random.normal(scale=0.5, size=1000),
        'σ = 2': np.random.normal(scale=2, size=1000)
    })

    ax = data.plot(kind='density', title='Different Population Standard Deviations', figsize=(5, 2))
    plt.xlabel('x')

    return ax

def example_boxplot():
    """Generate an example box plot."""
    non_symmetric = _non_symmetric_data()

    # find the quartiles and iqr
    q1_y, median_y, q3_y = non_symmetric.quantile([0.25, 0.5, 0.75])
    iqr = q3_y - q1_y

    # make the boxplot
    ax = non_symmetric.plot(kind='box', figsize=(6, 6), title='Box plot')

    # label the box
    ax.annotate('median', xy=(0.945, median_y + 2))
    ax.annotate(r'$Q_3$', xy=(1, q3_y), xytext=(1.08, q3_y - 5))
    ax.annotate(r'$Q_1$', xy=(1, q1_y), xytext=(1.08, q1_y))
    ax.annotate(
        'IQR', xy=(0.9, (q3_y + q1_y)/2), xytext=(0.8, (q3_y + q1_y)/2 - 2.85),
        arrowprops=dict(arrowstyle='-[, widthB=3.3, lengthB=0.5')
    )

    # label the whiskers
    ax.annotate(r'$Q_3 + 1.5 * IQR$', xy=(1.05, q3_y + 1.5 * iqr - 7))
    ax.annotate(r'$Q_1 - 1.5 * IQR$', xy=(1.05, q1_y - 1.5 * iqr - 2))

    # label the outliers
    ax.annotate(
        'outlier', xy=(0.99, non_symmetric.min()), xytext=(0.8, non_symmetric.min() - 2.1),
        arrowprops=dict(arrowstyle='->')
    )

    for i, val in enumerate(non_symmetric[non_symmetric > q3_y + 1.5*iqr]):
        if not i:
            text = 'outliers'
            x, y = 0.75, 102
        else:
            text = ''
            x, y = 0.87, 103
        ax.annotate(
            text, xy=(0.99, val), xytext=(x, y),
            arrowprops=dict(facecolor='black', arrowstyle='-|>')
        )

    return ax

def example_histogram():
    """Generate an example histogram."""
    non_symmetric = _non_symmetric_data()

    # get the bins
    bins = np.histogram_bin_edges(non_symmetric)

    # plot the data
    ax = non_symmetric.plot(
        kind='hist', legend=False, figsize=(15, 3),
        title=f'Histogram with 10 bins (each of width {bins[1] - bins[0]:.2f})'
    )
    ax.set_xlabel('x')

    # annotate measures of central tendency
    x_mode, x_mean, x_median = non_symmetric.mode(), non_symmetric.mean(), non_symmetric.median()
    ax.annotate(
        f'mode ({x_mode.iat[0]:.0f})', xy=(x_mode, 210), xytext=(x_mode + 5, 250), arrowprops=dict(arrowstyle='->')
    )
    ax.annotate(
        f'mean ({x_mean:.0f})', xy=(x_mean, 180), xytext=(x_mean - 20, 220), arrowprops=dict(arrowstyle='->')
    )
    ax.annotate(
        f'median ({x_median:.0f})', xy=(x_median, 180), xytext=(x_median - 12, 280), arrowprops=dict(arrowstyle='->')
    )
    plt.ylim((0, 320))

    return ax

def example_kde():
    """Generate an example KDE."""
    non_symmetric = _non_symmetric_data()

    # plot the data
    ax = non_symmetric.plot(
        kind='kde', legend=False, figsize=(15, 3),
        title='Kernel density estimate', bw_method=0.1, ylim=(0, 0.02)
    )
    ax.set_xlabel('x')

    # find measures of central tendency
    x_mode, x_mean, x_median = non_symmetric.mode(), non_symmetric.mean(), non_symmetric.median()

    # mark measures of central tendency with vertical dashed lines
    ax.axvline(x_mean, ymax=0.2, color='orange', linestyle='dashed')
    ax.axvline(x_median, ymax=0.5, color='orange', linestyle='dashed')
    ax.axvline(x_mode.iat[0], ymax=0.87, color='orange', linestyle='dashed')

    # annotate measures of central tendency
    ax.annotate('mode', xy=(x_mode - 11, 0.0178))
    ax.annotate('mean', xy=(x_mean, 0.0015), xytext=(x_mean - 70, 0.001), arrowprops=dict(arrowstyle='->'))
    ax.annotate('median', xy=(x_median, 0.01), xytext=(x_median - 50, 0.013), arrowprops=dict(arrowstyle='->'))

    return ax

def example_regression():
    """Show example regression."""
    # generate data
    np.random.seed(0)
    ice_cream_sales = pd.DataFrame({
        'temps': np.linspace(20, 40, num=30),
        'sales': np.abs(np.append(np.arange(2, 22), np.arange(22, 32)) + np.random.randint(-10, 10, size=30))
    })

    # make the scatter plot
    ax = ice_cream_sales.plot(
        kind='scatter', x='temps', y='sales', xlim=(15, 45), ylim=(0, 40), figsize=(12, 5),
        title='Using regression to predict ice cream sales'
    )

    # plot regression line
    m, b = np.polyfit(ice_cream_sales.temps, ice_cream_sales.sales, 1)
    reg_x = ice_cream_sales.temps
    reg_y = [m*num + b for num in reg_x]
    ax.plot(reg_x, reg_y, 'r-', label='regression line')

    # note the equation of the regression line
    ax.annotate(f'y = {m:.2f}x + {b:.2f}', xy=(15.5, 32))

    # extend the line to show extrapolation
    ax.plot([-b/m, 20], [0, 20*m + b], 'r:', label='extrapolated regression line')
    ax.plot([40, 45], [m*x + b for x in [40, 45]], 'r:')

    # labeling
    plt.legend()
    plt.xlabel('temperature in °C')
    plt.ylabel('ice cream sales')

    return ax

def example_scatter_plot():
    """Show example scatter plot."""
    # generate data
    np.random.seed(0)
    ice_cream_sales = pd.DataFrame({
        'temps': np.linspace(20, 40, num=30),
        'sales': np.abs(np.append(np.arange(2, 22), np.arange(22, 32)) + np.random.randint(-10, 10, size=30))
    })

    # make the scatter plot
    ax = ice_cream_sales.plot(
        kind='scatter', x='temps', y='sales', xlim=(15, 45), ylim=(0, 40), figsize=(12, 5),
        title='ice cream sales at a given temperature'
    )

    # labeling
    plt.xlabel('temperature in °C')
    plt.ylabel('ice cream sales')

    return ax

def hist_and_kde():
    """Show histogram with KDE."""
    # get data
    data = _non_symmetric_data()

    # plot histogram and KDE
    ax = data.plot(kind='hist', density=True, bins=12, alpha=0.5, title='Estimating the distribution', figsize=(15, 3))
    data.plot(kind='kde', ax=ax, color='blue').set_xlabel('x')

    return ax

def non_linear_relationships():
    """Plot logarithmic and exponential data along with correlation coefficients."""
    # make subplots
    fig, axes = plt.subplots(1, 2, figsize=(12, 3))

    # plot logarithmic
    log_x = np.linspace(0.01, 10)
    log_y = np.log(log_x)
    axes[0].scatter(log_x, log_y)
    axes[0].set_title(f'ρ = {np.round(np.corrcoef(log_x, log_y)[0][1], 2):.2f}')

    # plot exponential
    exp_x = np.linspace(0, 10)
    exp_y = np.exp(exp_x)
    axes[1].scatter(exp_x, exp_y)
    axes[1].set_title(f'ρ = {np.round(np.corrcoef(exp_x, exp_y)[0][1], 2):.2f}')

    # labels
    for ax in axes:
        ax.set_xlabel('x')
        ax.set_ylabel('y')

    return axes

def skew_examples():
    """Visualize left, right, and no skew distributions."""

    # create subplots
    fig, ax = plt.subplots(1, 3, figsize=(20, 4))

    # determine skew
    a = 4

    # find stats for annotation
    mean_skew_val = skewnorm.mean(a)
    median_skew_val = skewnorm.median(a)

    # get x data where PDF has value
    x = np.linspace(skewnorm.ppf(0.001, a), skewnorm.ppf(0.999, a), 100)

    # plot left skew
    ax[0].plot(x * -1, skewnorm.pdf(x, a))
    ax[0].set_title('Left/Negative Skewed')

    # annotate left skew's mode
    ax[0].axvline(-0.42, 0.72, 0.925, color='orange')
    ax[0].text(s='mode', x=-0.49, y=0.5, rotation=90)
    ax[0].axvline(-0.42, 0, 0.53, color='orange')

    # annotate left skew's median
    ax[0].axvline(median_skew_val * -1, 0.52, 0.83, color='orange')
    ax[0].text(s='median', x=-0.74, y=0.35, rotation=90)
    ax[0].axvline(median_skew_val * -1, 0, 0.3, color='orange')

    # annotate left skew's mean
    ax[0].axvline(mean_skew_val * -1, 0.26, 0.77, color='orange')
    ax[0].text(s='mean', x=-0.84, y=0.16, rotation=90)
    ax[0].axvline(mean_skew_val * -1, 0, 0.09, color='orange')

    # plot no skew normal
    ax[1].plot(x, norm.pdf(x, loc=x.mean(), scale=0.56))
    ax[1].set_title('No Skew')

    # annotate mean, median, and mode
    ax[1].text(s='  mean\nmedian\n  mode', x=x.mean() - 0.25, y=0.25)
    ax[1].axvline(x.mean(), 0.5, 0.94, color='orange')
    ax[1].axvline(x.mean(), 0, 0.3, color='orange')

    # plot right skew
    ax[2].plot(x, skewnorm.pdf(x, a))
    ax[2].set_title('Right/Positive Skewed')

    # annotate right skew's mode
    ax[2].axvline(0.42, 0.72, 0.925, color='orange')
    ax[2].text(s='mode', x=0.35, y=0.5, rotation=90)
    ax[2].axvline(0.42, 0, 0.53, color='orange')

    # annotate right skew's median
    ax[2].axvline(median_skew_val, 0.52, 0.83, color='orange')
    ax[2].text(s='median', x=0.6, y=0.35, rotation=90)
    ax[2].axvline(median_skew_val, 0, 0.3, color='orange')

    # annotate right skew's mean
    ax[2].axvline(mean_skew_val, 0.26, 0.77, color='orange')
    ax[2].text(s='mean', x=0.72, y=0.16, rotation=90)
    ax[2].axvline(mean_skew_val, 0, 0.09, color='orange')

    # label axes and set y-axis limits
    for axes in ax:
        axes.set_xlabel('x')
        axes.set_ylabel('f(x)')
        axes.set_ylim(0, 0.75)

    return ax

def time_series_decomposition_example():
    """Show an example of time series decomposition."""
    # generate a random time series
    np.random.seed(0)
    ts = pd.DataFrame({'timestamp' : pd.date_range('2018-01-01', periods=365, freq='D')})
    for i, drift, seasonality, noise in zip(
        ts.index,
        np.linspace(0, 1, num=365),
        itertools.cycle(np.append(np.linspace(0, np.pi, num=25), np.linspace(np.pi, 0, num=25, endpoint=False))),
        np.random.uniform(-10, 10, size=365)
    ):
        if i in [0]:
            ts.loc[i, 'value'] = i
        else:
            ts.loc[i, 'value'] = ts.loc[i - 1, 'value'] + drift + np.sin(seasonality) + noise

    # plot the result
    plt.rcParams['figure.figsize'] = [10, 6]
    result = seasonal_decompose(ts.set_index('timestamp').value, freq=50)
    plot = result.plot()
    plt.suptitle('Time Series Decomposition', y=1)
    plt.rcdefaults()

import numpy,from scipy,from statsmodels


# -*- coding: utf-8 -*-

from numpy import *

def f(x):
	return(x**5-2*x+1);

X=linspace(-2,3,6)
n=len(X)
Y=matrix([f(X[i]) for i in range(n)])
#Y.reshape(3,2)
Y=transpose(Y)
#print(Y)

V=ones((n,n))
V=vander(X,increasing=True) # génère une matrice de vandermonde ...

"""
# Une alternative
if n>1:
	for j in range(1,n,1):
		for i in range(n):
			V[i][j]=(X[i])**j

"""
print(V)

a=linalg.solve(V,Y)
a=linalg.inv(V)*Y
coef=[float(a[i]) for i in range(n)]
#print(coef)
#print(poly([0,1])) # par défaut Polynôme dont les racines sont 0 et 1
#print(poly1d([0,1],true)) # par défaut Polynôme dont les racines sont 0 et 1
p=poly1d([float(a[n-i-1]) for i in range(n)])
print(p)
print(p.order)
print(p.coeffs)
print(p.roots)
print(roots(p))
print(p(0))
print(polyval(p,0))

from  numpy.polynomial  import  Polynomial as poly
p=poly(coef)# Polynôme p à coefficients coef
#print(p)

print("%f"%(p.coef[0],),end=" ")
if (len(p)>1):
	if (p.coef[1]>0):
		print("+%f*x" %(p.coef[1],),end=" ")
	elif (p.coef[1]<0):
		print("%f*x" %(p.coef[1],),end=" ")

i=2
while (i<len(p)-1):
	if (p.coef[i]>0):
		print("+%f*x^%d" %(p.coef[i],i,),end=" ")
	elif (p.coef[i]<0):
		print("%f*x^%d" %(p.coef[i],i,),end=" ")
	i=i+1

if (len(p)>1):
	if (p.coef[len(p)-1]>0):
		print("+%f*x^%d" %(p.coef[len(p)-1],len(p)-1,),end=" ")
	elif (p.coef[len(p)-1]<0):
		print("%f*x^%d" %(p.coef[len(p)-1],len(p)-1,),end=" ")


import numpy as np

class Softmax(object):

  def __init__(self, dims=[10, 3073]):
    self.init_weights(dims=dims)

  def init_weights(self, dims):
    """
    Initializes the weight matrix of the Softmax classifier.
    Note that it has shape (C, D) where C is the number of
    classes and D is the feature size.
    """
    self.W = np.random.normal(size=dims) * 0.0001

  def loss(self, X, y):
    """
    Calculates the softmax loss.

    Inputs have dimension D, there are C classes, and we operate on minibatches
    of N examples.

    Inputs:
    - X: A numpy array of shape (N, D) containing a minibatch of data.
    - y: A numpy array of shape (N,) containing training labels; y[i] = c means
      that X[i] has label c, where 0 <= c < C.

    Returns a tuple of:
    - loss as single float
    """

    # Initialize the loss to zero.
    loss = 0.0

    # ================================================================ #
    # YOUR CODE HERE:
    #   Calculate the normalized softmax loss.  Store it as the variable loss.
    #   (That is, calculate the sum of the losses of all the training
    #   set margins, and then normalize the loss by the number of
    #   training examples.)
    # ================================================================ #
    N = X.shape[0]
    C = self.W.shape[0]
    WTX = self.W@X.T
    eWTX = np.exp(WTX)
    for i in range(N):
      loss = loss + np.log(np.ones((1,C))@eWTX[:,i]) - WTX[y[i],i]
    loss = loss/N
    # ================================================================ #
    # END YOUR CODE HERE
    # ================================================================ #

    return loss

  def loss_and_grad(self, X, y):
    """
    Same as self.loss(X, y), except that it also returns the gradient.

    Output: grad -- a matrix of the same dimensions as W containing
      the gradient of the loss with respect to W.
    """

    # Initialize the loss and gradient to zero.
    loss = 0.0
    grad = np.zeros_like(self.W)

    # ================================================================ #
    # YOUR CODE HERE:
    #   Calculate the softmax loss and the gradient. Store the gradient
    #   as the variable grad.
    # ================================================================ #
    N = X.shape[0]
    C = self.W.shape[0]
    WTX = self.W@X.T
    eWTX = np.exp(WTX)
    # Calculate Loss
    for i in range(N):
      loss = loss + np.log(np.ones((1,C))@eWTX[:,i]) - WTX[y[i],i]
    loss = loss/N
    # Calculate Gradient
    for j in range(C):
      for i in range(N):
        grad[j,:] = grad[j,:] + ( (1/(np.ones((1,C))@eWTX[:,i]))*(eWTX[j,i]) - (y[i]==j) )*X[i,:].T
    grad = grad/N

    # ================================================================ #
    # END YOUR CODE HERE
    # ================================================================ #

    return loss, grad

  def grad_check_sparse(self, X, y, your_grad, num_checks=10, h=1e-5):
    """
    sample a few random elements and only return numerical
    in these dimensions.
    """

    for i in np.arange(num_checks):
      ix = tuple([np.random.randint(m) for m in self.W.shape])

      oldval = self.W[ix]
      self.W[ix] = oldval + h # increment by h
      fxph = self.loss(X, y)
      self.W[ix] = oldval - h # decrement by h
      fxmh = self.loss(X,y) # evaluate f(x - h)
      self.W[ix] = oldval # reset

      grad_numerical = (fxph - fxmh) / (2 * h)
      grad_analytic = your_grad[ix]
      rel_error = abs(grad_numerical - grad_analytic) / (abs(grad_numerical) + abs(grad_analytic))
      print('numerical: %f analytic: %f, relative error: %e' % (grad_numerical, grad_analytic, rel_error))

  def fast_loss_and_grad(self, X, y):
    """
    A vectorized implementation of loss_and_grad. It shares the same
    inputs and ouptuts as loss_and_grad.
    """
    loss = 0.0
    grad = np.zeros(self.W.shape) # initialize the gradient as zero

    # ================================================================ #
    # YOUR CODE HERE:
    #   Calculate the softmax loss and gradient WITHOUT any for loops.
    # ================================================================ #
    N = X.shape[0]
    C = self.W.shape[0]
    WTX = self.W@X.T
    eWTX = np.exp(WTX)
    loss = np.ones((1,N))@(np.log(eWTX.T@np.ones((C,))) - WTX[y,np.arange(N)])/N
    scaling = eWTX/(eWTX.T@np.ones((C,)))
    scaling[y,np.arange(N)] -= 1
    grad = scaling@X/N
    # END YOUR CODE HERE
    # ================================================================ #

    return loss, grad

  def train(self, X, y, learning_rate=1e-3, num_iters=100,
            batch_size=200, verbose=False):
    """
    Train this linear classifier using stochastic gradient descent.

    Inputs:
    - X: A numpy array of shape (N, D) containing training data; there are N
      training samples each of dimension D.
    - y: A numpy array of shape (N,) containing training labels; y[i] = c
      means that X[i] has label 0 <= c < C for C classes.
    - learning_rate: (float) learning rate for optimization.
    - num_iters: (integer) number of steps to take when optimizing
    - batch_size: (integer) number of training examples to use at each step.
    - verbose: (boolean) If true, print progress during optimization.

    Outputs:
    A list containing the value of the loss function at each training iteration.
    """
    num_train, dim = X.shape
    num_classes = np.max(y) + 1 # assume y takes values 0...K-1 where K is number of classes

    self.init_weights(dims=[np.max(y) + 1, X.shape[1]])	# initializes the weights of self.W

    # Run stochastic gradient descent to optimize W
    loss_history = []

    for it in np.arange(num_iters):
      X_batch = None
      y_batch = None

      # ================================================================ #
      # YOUR CODE HERE:
      #   Sample batch_size elements from the training data for use in
      #     gradient descent.  After sampling,
      #     - X_batch should have shape: (dim, batch_size)
      #     - y_batch should have shape: (batch_size,)
      #   The indices should be randomly generated to reduce correlations
      #   in the dataset.  Use np.random.choice.  It's okay to sample with
      #   replacement.
      # ================================================================ #
      indicies = (num_train*np.random.rand(batch_size,1)).astype(int)
      X_batch = X[indicies].reshape(batch_size,dim)
      y_batch = y[indicies].reshape(batch_size,)

      # ================================================================ #
      # END YOUR CODE HERE
      # ================================================================ #

      # evaluate loss and gradient
      loss, grad = self.fast_loss_and_grad(X_batch, y_batch)
      loss_history.append(loss)

      # ================================================================ #
      # YOUR CODE HERE:
      #   Update the parameters, self.W, with a gradient step
      # ================================================================ #
      self.W = self.W -learning_rate*grad
      # ================================================================ #
      # END YOUR CODE HERE
      # ================================================================ #

      if verbose and it % 100 == 0:
        print('iteration {} / {}: loss {}'.format(it, num_iters, loss))

    return loss_history

  def predict(self, X):
    """
    Inputs:
    - X: N x D array of training data. Each row is a D-dimensional point.

    Returns:
    - y_pred: Predicted labels for the data in X. y_pred is a 1-dimensional
      array of length N, and each element is an integer giving the predicted
      class.
    """
    y_pred = np.zeros(X.shape[1])
    # ================================================================ #
    # YOUR CODE HERE:
    #   Predict the labels given the training data.
    # ================================================================ #
    y_pred = np.argmax(self.W@X.T,axis =0)
    # ================================================================ #
    # END YOUR CODE HERE
    # ================================================================ #

    return y_pred


import numpy as np
from scipy.integrate import odeint
import matplotlib.pyplot as plt
import math_models_animate as anim

class CompartmentModelBase:
    def __init__(self, initialConditions, labels, colors):
        self.numCompartments = len(initialConditions)
        self.initialConditions = initialConditions
        self.labels = labels
        self.colors = colors

    def hasLabel(self, label):
        return label in self.labels

    def hasColor(self, color):
        return color in self.colors

    def labelColor(self, label):
        return self.colors[self.labels.index(label)]

class SIRModel(CompartmentModelBase):
    def __init__(self, transmitRate=3.5, removeRate=0.5, sir0=(0.99, 0.01, 0.0)):
        super().__init__(sir0, ['Susceptible', 'Infected', 'Removed'], ['b', 'r', 'g'])
        self.transmitRate = transmitRate
        self.removeRate = removeRate

    def __str__(self):
        return 'SIR: Transmit={} Remove={}'.format(self.transmitRate, self.removeRate)

    def __repr__(self):
        return 'SIR({}, {})'.format(self.transmitRate, self.removeRate)

    def __call__(self, sir, t):
        # S'(t) = - transmitRate * S(t) * I(t)
        # I'(t) = transmitRate * S(t) * I(t) - removeRate * I(t)
        # R'(t) = removeRate * I(t)

        transmitted = self.transmitRate * sir[0] * sir[1]
        removed = self.removeRate * sir[1]
        dS = - transmitted
        dI = transmitted - removed
        dR = removed
        return dS, dI, dR

class SEIRModel(CompartmentModelBase):
    def __init__(self,
                 transmitRate=3.5,
                 reducedEIRate=0.0,
                 infectRate=1.0,
                 removeRate=0.5,
                 seir0=(0.99, 0.01, 0.0, 0.0)):
        super().__init__(seir0,
                         ['Susceptible', 'Exposed', 'Infected', 'Removed'],
                         ['b', 'c', 'r', 'g'])
        self.transmitRate = transmitRate
        self.reducedEIRate = reducedEIRate
        self.infectRate = infectRate
        self.removeRate = removeRate

    def __str__(self):
        return 'SEIR: Transmit={} ReduceEI={} Infect={} Remove={}'.format(
            self.transmitRate,
            self.reducedEIRate,
            self.infectRate,
            self.removeRate)

    def __repr__(self):
        return 'SIR({}, {}, {}, {})'.format(
            self.transmitRate,
            self.reducedEIRate,
            self.infectRate,
            self.removeRate)

    def __call__(self, seir, t):
        # S'(t) = - transmitRate * S(t) * (I(t) + reducedEIRate * E(t))
        # E'(t) = transmitRate * S(t) * (I(t) + reducedEIRate * E(t)) - infectRate * E(t)
        # I'(t) = infectRate * E(t) - removeRate * I(t)
        # R'(t) = removeRate * I(t)

        transmitted = self.transmitRate * seir[0] * (seir[2] + self.reducedEIRate * seir[1])
        infected = self.infectRate * seir[1]
        removed = self.removeRate * seir[2]
        dS = - transmitted
        dE = transmitted - infected
        dI = infected - removed
        dR = removed
        return dS, dE, dI, dR

def solve(model=SIRModel(), maxTime=10, timeSteps=100):
    t = np.linspace(0, maxTime, timeSteps)
    sir = odeint(model, model.initialConditions, t)
    return t, sir

def plotFinish_(title, legend=None):
    plt.xlabel('Time')
    plt.ylabel('Population')
    plt.title(title)
    if legend is not None:
        plt.legend(loc=legend)
    plt.grid()
    plt.show()

def plot(model, t, ys):
    for i in range(model.numCompartments):
        plt.plot(t, ys[:, i], color=model.colors[i], label=model.labels[i])

    plotFinish_('{}'.format(model), legend='best')

def animate(model, t, ys, legend=None):
    animation = anim.Animate(t,
                             [ys[:, i] for i in range(model.numCompartments)],
                             labels=model.labels,
                             colors=model.colors)
    animation.run(interval=25)
    plotFinish_('{}'.format(model), legend=legend)


import numpy as np
import time

def kmeans(x, k, max_it=20):
    '''
    KMEANS K-Means clustering algorithm

        Input:  x - data point features, n-by-p maxtirx.
                k - the number of clusters

        OUTPUT: idx  - cluster label
                ctrs - cluster centers, K-by-p matrix.
                iter_ctrs - cluster centers of each iteration, (iter, k, p)
                        3D matrix.
                err  - total square distance
    '''
    # YOUR CODE HERE

    # begin answer
    np.random.seed(int(time.time()))
    N = x.shape[0]
    p = x.shape[1]
    ctrs = x[np.random.randint(N, size=k)]
    iter_ctrs = np.zeros((max_it, k, p))
    idx = np.zeros(N, dtype=np.int64)
    dist = np.zeros((N, k))
    cnt = 0
    for i in range(max_it):
        # print(i)
        for j, ctr in enumerate(ctrs):
            dist[:,j] = np.linalg.norm(x-ctr, axis=1)
        # some dist may be NAN, so must use nanargmin to ignore these values.
        iidx = np.nanargmin(dist, axis=1)
        if((iidx == idx).all()):
            break
        idx = iidx
        for j in range(k):
            ctrs[j,:] = np.mean(x[idx == j], 0)
        iter_ctrs[i] = ctrs
        cnt += 1
    for j, ctr in enumerate(ctrs):
        dist[:, j] = np.linalg.norm(x - ctr, axis=1)
    err = np.sum(np.min(dist, axis=1))
    iter_ctrs = iter_ctrs[:cnt,:]
    # end answer
    return idx, ctrs, iter_ctrs, err


""""""
import numpy as np

def cartesian_to_polar(arr, axis=-1):
    """ Transform cartesian to polar coordinates in two dimensions.

    Parameters
    ----------
    arr : array_like
        Input array.
    axis : int, default -1
        Axis of input array representing x and y in cartesian coordinates.
        Must be of length 2.

    Returns
    -------
    arr_polar : array_like
        Output array.
    """
    if arr.shape[axis] != 2:
        raise ValueError(
            f"Expected length of axis {axis} to be 2, got {arr.shape[axis]} "
            f"instead."
        )

    r = np.linalg.norm(arr, axis=axis)
    phi = np.arctan2(np.take(arr, 1, axis=axis), np.take(arr, 0, axis=axis))
    return np.stack((r, phi), axis=axis)

def polar_to_cartesian(arr, axis=-1):
    """ Transform polar to cartesian coordinates in two dimensions.

    Parameters
    ----------
    arr : array_like
        Input array.
    axis : int, default -1
        Axis of input array representing r and phi in polar coordinates.
        Must be of length 2.

    Returns
    -------
    arr_cartesian : array_like
        Output array.
    """
    if arr.shape[axis] != 2:
        raise ValueError(
            f"Expected length of axis {axis} to be 2, got {arr.shape[axis]} "
            f"instead."
        )

    x = np.take(arr, 0, axis=axis) * np.cos(np.take(arr, 1, axis=axis))
    y = np.take(arr, 0, axis=axis) * np.sin(np.take(arr, 1, axis=axis))
    return np.stack((x, y), axis=axis)

def cartesian_to_spherical(arr, axis=-1):
    """ Transform cartesian to spherical coordinates in three dimensions.

    The spherical coordinate system is defined according to ISO 80000-2.

    Parameters
    ----------
    arr : array_like
        Input array.
    axis : int, default -1
        Axis of input array representing x, y and z in cartesian coordinates.
        Must be of length 3.

    Returns
    -------
    arr_spherical : array_like
        Output array.
    """
    if arr.shape[axis] != 3:
        raise ValueError(
            f"Expected length of axis {axis} to be 3, got {arr.shape[axis]} "
            f"instead."
        )

    r = np.linalg.norm(arr, axis=axis)
    theta = np.arccos(np.take(arr, 2, axis=axis) / r)
    phi = np.arctan2(np.take(arr, 1, axis=axis), np.take(arr, 0, axis=axis))
    return np.stack((r, theta, phi), axis=axis)

def spherical_to_cartesian(arr, axis=-1):
    """ Transform spherical to cartesian coordinates in three dimensions.

    The spherical coordinate system is defined according to ISO 80000-2.

    Parameters
    ----------
    arr : array_like
        Input array.
    axis : int, default -1
        Axis of input array representing r, theta and phi in spherical
        coordinates. Must be of length 3.

    Returns
    -------
    arr_cartesian : array_like
        Output array.
    """
    if arr.shape[axis] != 3:
        raise ValueError(
            f"Expected length of axis {axis} to be 3, got {arr.shape[axis]} "
            f"instead."
        )

    x = (
        np.take(arr, 0, axis=axis)
        * np.sin(np.take(arr, 1, axis=axis))
        * np.cos(np.take(arr, 2, axis=axis))
    )
    y = (
        np.take(arr, 0, axis=axis)
        * np.sin(np.take(arr, 1, axis=axis))
        * np.sin(np.take(arr, 2, axis=axis))
    )
    z = np.take(arr, 0, axis=axis) * np.cos(np.take(arr, 1, axis=axis))
    return np.stack((x, y, z), axis=axis)


import csv
import math
import matplotlib.pyplot as plt
import matplotlib.animation as animation
import time
import numpy as np

# read the data
csvfile=open("linearX.csv", 'r')
x = list(csv.reader(csvfile))
csvfile=open("linearY.csv", 'r')
y = list(csv.reader(csvfile))

m=len(x)
n=1
x3=[]
y2=[]
for i in range(m):
    x3.append(float(x[i][0]))
    y2.append(float(y[i][0]))

# # a different dataset
# from sklearn import datasets, linear_model
#
# diabetes = datasets.load_diabetes()
# diabetes_X = diabetes.data[:, np.newaxis, 2]
# data = diabetes_X[:-20]
# y_data = diabetes.target[:-20]
#
# x3=data
# y2=y_data
# m=len(data)
# print(len(data))
# print(len(y_data))

# normalise the data
meanx=sum(x3)/m
v=0
for i in range(m):
    temp=x3[i]-meanx
    v+=temp*temp
v/=m
v=math.sqrt(v)
for i in range(m):
    x3[i]=(x3[i]-meanx)/v

x2=[]
for i in range(m):
    x2.append(np.array([1,x3[i]]))

X=np.array(x2)
Y=np.array(y2)

xvalues=np.arange(min(x3)-1,max(x3)+1,0.1)

plt.ion()
fig=plt.figure()
ax1 = fig.add_subplot(1,1,1)

# plots hypothesis funtion and training data
def pl(th):
    ax1.clear()
    ax1.scatter(x3, y2, label= "Training Data", color= "r",
                marker= ".", s=10)

    the=list(th)
    yvalues=the[1]*xvalues+the[0]

    ax1.plot(xvalues, yvalues, label="Hypothesis function learned",color ='b')

    plt.xlabel('Acidity')
    plt.ylabel('Density')
    plt.legend()
    plt.title('Q1 (b)')
    plt.show()
    plt.pause(0.001)

# return the value of J(theta)
def findcost(th):
    ans=0
    for i in range(m):
        temp=(Y[i]-np.sum(np.dot(th,X[i])))
        ans+=(temp*temp)
    ans=float(ans/2)
    return ans

def find_convergence(th1,th2):
    return abs(findcost(th1)-findcost(th2))

# Part (a)&(b)
epsilon=0.0000001
# epsilon=5
eta=0.001
theta=np.array([0,0])
t=0

pl(theta)

while(True):

    hthetas=[]
    for i in range(m):
        hthetas.append(np.sum(np.dot(theta,X[i])))

# calculate -ve of gradient
    gradient=np.zeros((n+1))
    for i in range(m):
        gradient+=(Y[i]-hthetas[i])*X[i]

# update theta
    thetaold=theta
    theta=theta+eta*gradient

    t+=1

    print("theta={}\n".format(theta))

    pl(theta)

# check if converged
    convergence=find_convergence(thetaold,theta)
    print("convergence={}".format(convergence))
    if(convergence<epsilon):
        break

print("Number of iterations=",t)
plt.ioff()
pl(theta)

# part (c)
from mpl_toolkits.mplot3d import Axes3D
import matplotlib.pyplot as plt
from matplotlib import cm
from matplotlib.ticker import LinearLocator, FormatStrFormatter
import numpy as np

plt.ion()

fig = plt.figure()
ax = fig.add_subplot(111, projection='3d')

# plot J(theta)
xvalues = np.linspace(0, 2, 70)
yvalues = np.linspace(-1, 1, 70)
xvalues, yvalues = np.meshgrid(xvalues, yvalues)
Zs = np.array([findcost(np.array([x,y])) for x,y in zip(np.ravel(xvalues), np.ravel(yvalues))])
Z = Zs.reshape(xvalues.shape)

surf = ax.plot_surface(xvalues, yvalues, Z, cmap=cm.coolwarm,linewidth=0,alpha=0.7 ,antialiased=False,label='J(theta)')
plt.show()
plt.pause(0.001)

# plots point in 3d mesh
def plot_2(th):

    ax.scatter(th[0],th[1],findcost(th),c="black",depthshade=False,s=6)

    plt.xlabel('theta0')
    plt.ylabel('theta1')
    plt.title('Q1 (c)')
    plt.show()
    plt.pause(0.2)
    # plt.pause(0.001)

theta=np.array([0,0])
t=0
levels=[] # stores the values of J(theta) at each iteration
levels.append(findcost(theta))
thetas=[]
thetas.append(theta)
while(True):
    hthetas=[]
    for i in range(m):
        hthetas.append(np.sum(np.dot(theta,X[i])))

    gradient=np.zeros((n+1))
    for i in range(m):
        gradient+=(Y[i]-hthetas[i])*X[i]
    thetaold=theta
    theta=theta+eta*gradient

    levels.append(findcost(theta))
    thetas.append(theta)

    t+=1

    print("theta={}\n".format(theta))

    plot_2(theta)

    convergence=find_convergence(thetaold,theta)
    print("convergence=",convergence)
    if(convergence<epsilon):
        break

print("Number of iterations=",t)
plt.ioff()
plot_2(theta)

# part (d)&(e)
# plots contours at each iteration

plt.ion()
plt.figure()

plt.xlabel('theta0')
plt.ylabel('theta1')
plt.title('eta={}'.format(eta))
for i in range(len(levels)):
    print(i)
    plt.contour(xvalues, yvalues, Z,levels[i])
    plt.scatter(thetas[i][0],thetas[i][1])

    plt.show()
    plt.pause(0.2)

plt.ioff()
plt.show()


import numpy as np
from hilbert_tensor import make_hilbert_tensor
from tensor_operations import make_tensor_train_with_conventional_method, build_dense_tensor_from_tensor_train
import matplotlib.pyplot as plt

d = 5

# T_shape = [21,22,23,24,25]
# T_shape = [31,32,33,34,35]
T_shape = [41,42,43,44,45]
T = make_hilbert_tensor(T_shape)
rr = 2+np.arange(18)
errs = np.zeros(len(rr))
for k in range(len(rr)):
    r = rr[k]
    CC = make_tensor_train_with_conventional_method(T, tol=1e-20, max_rank=r)
    T2 = build_dense_tensor_from_tensor_train(CC)
    tt_err = np.linalg.norm(T2 - T) / np.linalg.norm(T)
    errs[k] = tt_err
    print('r=', r, ', tt_err=', tt_err,)

plt.figure()
plt.semilogy(rr.reshape(-1), errs.reshape(-1), color='k')
plt.legend(['TT-RSVD'])
plt.title('Hilbert tensor: approximation error vs. rank',fontsize=14)
plt.xlabel('Tensor train rank, $r$',fontsize=12)
plt.ylabel(r'$\frac{||T - \widetilde{T}||}{||T||}$', rotation=0, fontsize=16, labelpad=30)
plt.xlim([2,19])


import numpy as np

#Input array
# X=np.array([[1,0,1,0],[1,0,1,1],[0,1,0,1]])
X=np.array([[1,0,1,0]])
#Output
# y=np.array([[1,0],[1,0],[0,1]])
y=np.array([[1,0]])
#Sigmoid Function
def sigmoid (x):
    return 1/(1 + np.exp(-x))

#Derivative of Sigmoid Function
def derivatives_sigmoid(x):
    return x * (1 - x)

#Variable initialization
epoch=5000 #Setting training iterations
lr=0.2 #Setting learning rate
inputlayer_neurons = X.shape[1] #number of features in data set
hiddenlayer_neurons = 3 #number of hidden layers neurons
output_neurons = 2 #number of neurons at output layer

#weight and bias initialization
wh=np.random.uniform(size=(inputlayer_neurons,hiddenlayer_neurons))

bh=np.random.uniform(size=(1,hiddenlayer_neurons))

wout=np.random.uniform(size=(hiddenlayer_neurons,output_neurons))
bout=np.random.uniform(size=(1,output_neurons))

for i in range(10000):

    #Forward Propogation
    hidden_layer_input1=np.dot(X,wh)
    hidden_layer_input=hidden_layer_input1 + bh
    hiddenlayer_activations = sigmoid(hidden_layer_input)
    output_layer_input1=np.dot(hiddenlayer_activations,wout)
    output_layer_input= output_layer_input1+ bout
    output = sigmoid(output_layer_input)

    #Backpropagation
    E = y-output
    slope_output_layer = derivatives_sigmoid(output)
    slope_hidden_layer = derivatives_sigmoid(hiddenlayer_activations)
    d_output = E * slope_output_layer
    Error_at_hidden_layer = d_output.dot(wout.T)
    d_hiddenlayer = Error_at_hidden_layer * slope_hidden_layer
    wout += hiddenlayer_activations.T.dot(d_output) *lr
    bout += np.sum(d_output, axis=0,keepdims=True) *lr
    wh += X.T.dot(d_hiddenlayer) *lr
    bh += np.sum(d_hiddenlayer, axis=0,keepdims=True) *lr

print output


import numpy as np
import scipy.stats as sps
from scipy.spatial import distance
# from sklearn.metrics.pairwise import distance_metrics
import math

class PoloMath():

    @staticmethod
    def entropy(s1):
        return sps.entropy(s1, base=2)

    @staticmethod
    def cosine_sim(x, y):
        """Cosine similarity between two vectors"""
        return np.dot(x, y) / (np.sqrt(np.dot(x, x)) * np.sqrt(np.dot(y, y)))
        # return 1 - cosine(x, y)

    @staticmethod
    def cosine_dist(x, y):
        """Cosine distance between two vectors. 1 - cosine similarity."""
        return distance.cosine(x, y)

    @staticmethod
    def js_divergence(p1, p2):
        """Jennsen-Shannon Divergence."""
        p = sps.entropy(p1, p2, 2)
        q = sps.entropy(p2, p1, 2)
        jsd = (p + q) / 2
        return jsd

    @staticmethod
    def js_dist(x, y):
        """Jennsen-Shannon Distance. Square root of the convergence."""
        return distance.jensenshannon(x, y)

    @staticmethod
    def jaccard_dist(x, y):
        """Jaccard Distance. Square root of the convergence."""
        return distance.jaccard(x, y)

    @staticmethod
    def euclidean_dist(x, y):
        """Simple Euclidean distance"""
        # return math.sqrt(((s1 - s2)**2).sum())
        return distance.euclidean(x, y)

    @staticmethod
    def standard_euclidean_dist(x, y, V):
        return distance.seuclidean(x, y, V)

    @staticmethod
    def chebyshev_dist(x, y):
        return distance.chebyshev(x, y)

    @staticmethod
    def manhattan_dist(x, y):
        return distance.cityblock(x, y)

    @staticmethod
    def pwmi(p_a, p_b, p_ab, norm=.000001):
        """Computes the adjusted point-wise mutual information of two items (a and b)
        that appear in container vectors of some kind, e.g. items in a shopping
        basket."""
        # if p_ab == 0: p_ab = .000001  # To pr event craziness in prob calcs
        p_ab += norm
        i_ab = math.log2(p_ab / (p_a * p_b))  # Raw
        try:
            i_ab = i_ab / (math.log2(p_ab) * -1)  # Adjusted
        except ZeroDivisionError:
            i_ab = 0
        return i_ab

    @staticmethod
    def jscore(s1, s2, thresh = 0):
        """Computes the Jaccard score (aka distance) for two vectors (series). Series passed must
        share an index. This condition will be met for an unstacked matrix of weights or counts,
        where the two series belong to the matrix."""
        A = set(s1[s1 > thresh].index)
        B = set(s2[s2 > thresh].index)
        if len(A | B) > 0:
            return 1 - (len(A & B) / len(A | B))
        else:
            return -1  # Is this correct?

    @staticmethod
    def kl_distance(s1, s2):
        """Kullback-Leibler distance"""
        return sps.entropy(s1, s2, 2)

    @staticmethod
    def softmax(x):
        """Computes softmax values for each sets of scores in x."""
        e_x = np.exp(x - np.max(x))
        return e_x / e_x.sum(axis=0)