import numpy as np
import pandas
from ggplot import *

"""
In this question, you need to:
1) implement the compute_cost() and gradient_descent() procedures
2) Select features (in the predictions procedure) and make predictions.

"""

def normalize_features(df):
    """
    Normalize the features in the data set.
    """
    mu = df.mean()
    sigma = df.std()

    if (sigma == 0).any():
        raise Exception("One or more features had the same value for all samples, and thus could " + \
                         "not be normalized. Please do not include features with only a single value " + \
                         "in your model.")
    df_normalized = (df - df.mean()) / df.std()

    return df_normalized, mu, sigma

def compute_cost(features, values, theta):
    """
    Compute the cost function given a set of features / values,
    and the values for our thetas.

    This can be the same code as the compute_cost function in the lesson #3 exercises,
    but feel free to implement your own.
    """
    m = len(values)
    sum_of_square_errors = np.square(np.dot(features, theta) - values).sum()
    cost = sum_of_square_errors / (2*m)
    return cost

def gradient_descent(features, values, theta, alpha, num_iterations):
    """
    Perform gradient descent given a data set with an arbitrary number of features.

    This can be the same gradient descent code as in the lesson #3 exercises,
    but feel free to implement your own.
    """

    m = len(values)
    cost_history = []

    for i in range(num_iterations):
        cost = compute_cost(features,values,theta)
        cost_history.append(cost)
        theta = theta + (alpha/m)*np.dot((values - np.dot(features,theta)),features)

    return theta, pandas.Series(cost_history)

def predictions(dataframe):
    '''
    The NYC turnstile data is stored in a pandas dataframe called weather_turnstile.
    Your prediction should have a R^2 value of 0.40 or better.
    You need to experiment using various input features contained in the dataframe.
    '''
    # print dataframe # see which fields we can use

    # Select Features (try different features!)
    features = dataframe[['rain', 'precipi', 'Hour', 'meantempi']]

    # Add UNIT to features using dummy variables
    dummy_units = pandas.get_dummies(dataframe['UNIT'], prefix='unit')
    features = features.join(dummy_units)

    # Values
    values = dataframe['ENTRIESn_hourly']
    m = len(values)

    features, mu, sigma = normalize_features(features)
    features['ones'] = np.ones(m) # Add a column of 1s (y intercept)

    # Convert features and values to numpy arrays
    features_array = np.array(features)
    values_array = np.array(values)

    # Set values for alpha, number of iterations.
    alpha = 0.1 # please feel free to change this value
    num_iterations = 75 # please feel free to change this value

    # Initialize theta, perform gradient descent
    theta_gradient_descent = np.zeros(len(features.columns))
    theta_gradient_descent, cost_history = gradient_descent(features_array,
                                                            values_array,
                                                            theta_gradient_descent,
                                                            alpha,
                                                            num_iterations)

    plot = None

    predictions = np.dot(features_array, theta_gradient_descent)
    return predictions, plot

def plot_cost_history(alpha, cost_history):
   """This function is for viewing the plot of your cost history.
   You can run it by uncommenting this

       plot_cost_history(alpha, cost_history)

   call in predictions.

   If you want to run this locally, you should print the return value
   from this function.
   """
   cost_df = pandas.DataFrame({
      'Cost_History': cost_history,
      'Iteration': range(len(cost_history))
   })
   return ggplot(cost_df, aes('Iteration', 'Cost_History')) + \
      geom_point() + ggtitle('Cost History for alpha = %.3f' % alpha )


from typing import Tuple, Optional

import numpy as np

__all__ = [
    "sigmoid",
    "tanh",
    "relu",
    "softmax",
]

def sigmoid(x: np.ndarray) -> np.ndarray:
    """
    Applies the element-wise sigmoid function.

    Parameters:
        x (np.ndarray): Input array.

    Returns:
        Function output.
    """
    return 1 / (1 + np.exp(-x))

def tanh(x: np.ndarray) -> np.ndarray:
    """
    Applies the element-wise hyperbolic tangent function. Note that this function is a
    simple wrapper around `numpy.tanh()`.

    Parameters:
        x (np.ndarray): Input array.

    Returns:
        Function output.
    """
    return np.tanh(x)

def relu(x: np.ndarray) -> np.ndarray:
    """
    Applies the element-wise rectified linear unit function.

    Parameters:
        x (np.ndarray): Input array.

    Returns:
        Function output.
    """
    return x * (x > 0)

def softmax(x: np.ndarray, axis: Optional[int] = None) -> np.ndarray:
    """
    Applies a softmax function along the argument dimension `axis`.

    Parameters:
        x (np.ndarray): Input array.
        axis (Optional[int]): A dimension along which softmax will be computed.

    Returns:
        Function output.
    """
    e_x = np.exp(x - np.max(x))
    return e_x / e_x.sum(axis=axis, keepdims=True)


import numpy as np
import sys

'''
v0.3 Dec. 17, 2017
    - refactor > rename functions by adding "_rad"
    - refactor > add function comment
v0.2 Dec. 10, 2017
    - fix bug: calc_gamma_rad() was too complicated
        _+ use arctan2()
    - add Test_group_run_random()
v0.1 Dec. 09, 2017
    - add Test_group_run_axes()
    - add calc_gamma_rad()
    - add calc_beta_rad()
'''

def calc_beta_rad(pvec):
    '''
    polar angle [0, pi]
    '''
    return np.arccos(pvec[2])  # arccos:[0, pi]

def calc_gamma_rad(pvec):
    '''
    azimuth angle [0, 2pi]
    '''
    gamma = np.arctan2(pvec[1], pvec[0])
    if gamma < 0.0:
        gamma += 2 * np.pi
    return gamma

def Test_group_run_axes():
    pvecs = [
        [0, 0, 1],  # z direction
        [0, 0, -1],  # z direction
        [0, 1, 0],  # y direction
        [0, -1, 0],  # y direction
        [1, 0, 0],  # x direction
        [-1, 0, 0],  # x direction
        ]
    for elem in pvecs:
        beta_rad = calc_beta_rad(elem)
        gamma_rad = calc_gamma_rad(elem)

        fmt = "{0} \tbeta:{1:.5f} gamma:{2:.5f}"
        msg = fmt.format(elem, beta_rad, gamma_rad)
        print(msg)

def Test_calc_xyz(beta_rad, gamma_rad):
    rad = 1.0  # radius
    resx = rad * np.cos(gamma_rad) * np.sin(beta_rad)
    resy = rad * np.sin(gamma_rad) * np.sin(beta_rad)
    resz = rad * np.cos(beta_rad)
    return resx, resy, resz

def Test_group_run_random():
    bt_stp = np.pi / 4.0  # beta step
    gm_stp = np.pi / 4.0  # gamma step
    for bt_in in np.arange(0.0, np.pi, bt_stp):
        for gm_in in np.arange(0.0, 2 * np.pi, gm_stp):
            # print(bt_in, gm_in)
            wrk = Test_calc_xyz(bt_in, gm_in)
            bt_out = calc_beta_rad(wrk)
            gm_out = calc_gamma_rad(wrk)
            if abs(gm_in - gm_out) > sys.float_info.epsilon:
                print("%+.5e" % (gm_in - gm_out), end=' ')
                print("NG:", end=" ")
            else:
                continue
            msg = ','.join("%+.5f" % elem for elem in wrk)
            print(msg, end=' -- ')
            fmt = "b{0:.5f} b{1:.5f} g{2:.5f} g{3:.5f}"
            msg = fmt.format(bt_in, bt_out, gm_in, gm_out)
            print(msg)

if __name__ == '__main__':
    Test_group_run_random()
    #Test_group_run_axes()


from lega.shen_basis import shen_function, mass_matrix, stiffness_matrix,\
    shen_basis
from scipy.linalg import eigh
from sympy.plotting import plot
from sympy import Symbol

# Visualize the eigenfunctions of -u'' = lmnda u in (-1, 1) with u(-1)=u(1)=0
# Are they in some sense similar to sines(k*pi*x) and cos(k*pi/2*x) which solve
# the problem?

n = 10
basis = shen_basis(10)

# Solve the eigenvalue problem to get coeffs of the new basis
A = stiffness_matrix(n)
M = mass_matrix(n)
lmbdas, V = eigh(A.toarray(), M.toarray())

# Make the new basis
Abasis = [shen_function(v) for v in V.T]

print 'eigenvalues', lmbdas

# Plot the basis for comparison
x = Symbol('x')
f_fA = iter(zip(basis, Abasis))

f, fA = next(f_fA)
p = plot(f, (x, -1, 1), show=False)
p[0].line_color = 'red'
p_ = plot(fA, (x, -1, 1), show=False)
p_[0].line_color = 'blue'
p.append(p_[0])

for f, fA in f_fA:
    p_ = plot(f, (x, -1, 1), show=False)
    p_[0].line_color = 'red'
    p.append(p_[0])

    p_ = plot(fA, (x, -1, 1), show=False)
    p_[0].line_color = 'blue'
    p.append(p_[0])

p.show()

# Some questions, what are the approximation properties of Abasis? And how does
# the transformation between function and its series look? And, is there a
# clever way to get the eigenvalues and eigenvectors


from numpy import conj, dot
# tool for evaluating operators

class OperatorChecker():
    def check_hermitian(operator):
        '''
        check_hermitian(operator)

        Description:
            - this function takes a 2 by 2 operator matrix and checks to see if it is hermitian (equal to its transposed conjugate)
            - this is useful because all qbit operators corresponding to quantum logic gates must be hermitian

        Parameters:
            - operator <type 'list'>: matrix representing the quantum operator

        Returns:
            - hermitian <type 'bool'>: boolean value storing if the passed matrix is hermitian

        Example:
            >>> oc = qonic_misc.OperatorChecker
            >>> pauli_z = [[1, 0], [0, -1]] # pauli z gate
            >>> print(oc.check_hermitian(pauli_z)) # check to see if the pauli z gate is hermitian
            True

        '''

        # check all 4 of the elements of the matrix to test for hermitianarity (hermitiary, hermitianary, i dont know...)
        element00 = (operator[0][0]) == conj(operator[0][0])
        element01 = (operator[0][1]) == conj(operator[1][0])
        element10 = (operator[1][0]) == conj(operator[0][1])
        element11 = (operator[1][1]) == conj(operator[1][1])

        # if all 4 of the above conditions are true, then the operator matrix is hermitian
        return (element00 and element01 and element10 and element11)

    def check_unitary(operator):
        '''
        check_unitary(operator)

        Description:
            - this function takes a 2 by 2 operator matrix and checks to see if it is unitary (produces the identity matrix when multiplied by its transposed conjugate)
            - this is useful because all qbit operators corresponding to quantum logic gates must be unitary

        Parameters:
            - operator <type 'list'>: matrix representing the quantum operator

        Returns:
            - unitary <type 'bool'>: boolean value storing if the passed matrix is unitary

        Example:
            >>> oc = qonic_misc.OperatorChecker
            >>> pauli_z = [[1, 0], [0, -1]] # pauli z gate
            >>> print(oc.check_unitary(pauli_z) # check to see if the pauli z gate is unitary
            True

        '''

        # define the transposed conjugate of the passed operator
        trans_conj = [[conj(operator[0][0]), conj(operator[1][0])], [conj(operator[0][1]), conj(operator[1][1])]]

        # get the product of the two matricies
        product = dot(operator, trans_conj)

        # round to 10 decimal points to avoid small errors causing the function to return false
        for i in range(2):
            for j in range(2):
                product[i][j] = round(product[i][j], 10)

        # check to see if the product is equal to the identity matrix, and return the result
        return product.tolist() == [[1, 0], [0, 1]]


from scipy import eye, dot

def rowswap(n, j, k):
    """Swaps two rows
        INPUTS: n -> matrix size
        j, k -> the two rows to swap"""
    out = eye(n)
    out[j,j]=0
    out[k,k]=0
    out[j,k]=1
    out[k,j]=1
    return out

def cmult(n, j, const):
    """Multiplies a row by a constant
    INPUTS: n -> array size
            j -> row
            const -> constant"""
    out = eye(n)
    out[j,j]=const
    return out

def cmultadd(n, j, k, const):
    """Multiplies a row (k) by a constant and adds the result to another row (j)"""
    out = eye(n)
    out[j,k] = const
    return out

def ref(A):
    """Performs a naive row reduction on A"""

    n = min(A.shape)
    ref = A.astype(float).copy()
    for a in xrange(n):
        for b in xrange(a,n):
            if ref[a,a] != 0:
                ref = dot(cmultadd(n, b, a, -ref[b, a]/ref[a, a]), ref)
            else: continue
        ref = dot(cmult(n, a, 1.0/ref[a,a]), ref)

    return ref

def LU(A):
    rows, cols = A.shape
    U = A.copy()
    L = eye(rows, rows)
    for i in range(rows):
        for j in range(i+1,rows):
            E = cmultadd(rows,j,i,-U[j,i]/U[i,i])
            F = cmultadd(rows,j,i,U[j,i]/U[i,i])
            U = dot(E,U)
            L = dot(L,F)
    return (L,U)

def LU_det(A):
    """Find the determinant of a matrix using the LU factorization"""
    U = LU(A)[1]
    det = 1
    for i in range(U.shape[0]):
        det *= U[i,i]
    return det


import numpy as np
import matplotlib.pyplot as plt

ts = np.arange(100)
tis = np.array([25, 50, 75])
lr_list = []
lr0 = 1
gamma = 0.9
for t in ts:
    passed_num_thresholds = sum(t > tis)
    lr = lr0 * np.power(gamma, passed_num_thresholds)
    lr_list.append(lr)

plt.figure()
plt.plot(lr_list)
plt.title('piecewise constant')
plt.savefig('../figures/lr_piecewise_constant.pdf', dpi=300)
plt.show()

ts = np.arange(100)
lam = 0.999
lr0 = 1
lr_list = lr0 * np.exp(-lam*ts)

plt.figure()
plt.plot(lr_list)
plt.title('exponential decay')
plt.savefig('../figures/lr_exp_decay.pdf', dpi=300)
plt.show()

ts = np.arange(100)
alpha = 0.5
beta = 1
lr0 = 1
lr_list = lr0 * np.power(beta*ts + 1, -alpha)

plt.figure()
plt.plot(lr_list)
plt.title('polynomial decay')
plt.savefig('../figures/lr_poly_decay.pdf', dpi=300)
plt.show()


from scipy.misc import derivative

f = lambda x: 0.1*x**5 - 0.2*x**3 + 0.1*x - 0.2

dx = 0.01       # Step size, equal differences of x, ∆x
# dx = 0.001    # Smaller step size gives less error
x0 = 0.1        # Find derivatives of f(x) at point x

# Derivative Function:
y1 = derivative(f, x0, dx, 1)   # n = 1 for 1st derivative
y2 = derivative(f, x0, dx, 2)   # n = 2 for 2nd derivative

print('derivative(f, ', x0,', ', dx,', 1) = %f' % y1, sep='')
print('derivative(f, ', x0,', ', dx,', 2) = %f' % y2, sep='')

# ~~~~~~~~~~~~~~~~~~~~~~~~~ Plotting the function ~~~~~~~~~~~~~~~~~~~~~~~~~~

from numpy import linspace
from matplotlib.pyplot import plot, xlabel, ylabel, legend, grid, show

X = linspace(-1, 1)             # Divides from -1 to 1 into 50 points

# Central Differences Approximation :
DFC1 = derivative(f, X, dx, 1)
DFC2 = derivative(f, X, dx, 2)

# pyplot.
plot(X,f(X),'-k', X,DFC1,'--b', X,DFC2,'-.r')
xlabel('x')
ylabel('y, y\', y\'\'')
legend(['f(x)', 'f\'(x)', 'f\'\'(x)'])
grid()
show()

3. Numerical Differentiation/0. Numerical Differentiation function of SciPy.py


import numpy as np
from numba import jit

@jit(nopython=True)
def interp1d(grid, vals, x):
    """
    Linearly interpolate (grid, vals) to evaluate at x.

    Parameters
    ----------
        grid and vals are numpy arrays, x is a float

    Returns
    -------
        a float, the interpolated value

    """

    a, b, G = np.min(grid), np.max(grid), len(grid)

    s = (x - a) / (b - a)

    q_0 = max(min(int(s * (G - 1)), (G - 2)), 0)
    v_0 = vals[q_0]
    v_1 = vals[q_0 + 1]

    λ = s * (G - 1) - q_0

    return (1 - λ) * v_0 + λ * v_1

@jit(nopython=True)
def interp1d_vectorized(grid, vals, x_vec):
    """
    Linearly interpolate (grid, vals) to evaluate at x_vec.

    All inputs are numpy arrays.

    Return value is a numpy array of length len(x_vec).
    """

    out = np.empty_like(x_vec)

    for i, x in enumerate(x_vec):
        out[i] = interp1d(grid, vals, x)

    return out


import numpy as np
#-------------------------------------------------------------------------
'''
    Problem 3: PageRank algorithm (version 1)
    In this problem, we implement a simplified version of the pagerank algorithm, which doesn't consider about sink node problem or sink region problem.
    You could test the correctness of your code by typing `nosetests -v test3.py` in the terminal.
'''

#--------------------------
def compute_P(A):
    '''
        compute the transition matrix P from addjacency matrix A. P[j][i] represents the probability of moving from node i to node j.
        Input:
                A: adjacency matrix, a (n by n) numpy matrix of binary values. If there is a link from node i to node j, A[j][i] =1. Otherwise A[j][i]=0 if there is no link.
        Output:
                P: transition matrix, a (n by n) numpy matrix of float values.  P[j][i] represents the probability of moving from node i to node j.
    The values in each column of matrix P should sum to 1.
    '''
    #########################################
    ## INSER YOUR CODE HERE

    np.seterr(divide='ignore', invalid='ignore')
    # create a diagonal matrix
    np.fill_diagonal(A, 0)

    # sum of each column of A
    colsums = A.sum(axis=0)

    # normalize each column of A
    P = A / colsums

    #########################################
    return P

#--------------------------
def random_walk_one_step(P, x_i):
    '''
        compute the result of one step random walk.
        Input:
                P: transition matrix, a (n by n) numpy matrix of float values.  P[j][i] represents the probability of moving from node i to node j.
                x_i: pagerank scores before the i-th step of random walk. a numpy vector of shape (n by 1).
        Output:
                x_i_plus_1: pagerank scores after the i-th step of random walk. a numpy vector of shape (n by 1).
    '''
    #########################################
    ## INSERT YOUR CODE HERE
    x_i_plus_1 = np.dot(P, x_i)

    #########################################
    return x_i_plus_1

#--------------------------
def random_walk(P, x_0, max_steps=10000):
    '''
        compute the result of multiple-step random walk. The random walk should stop if the score vector x no longer change (converge) after one step of random walk, or the number of iteration reached max_steps.
        Input:
                P: transition matrix, a (n by n) numpy matrix of float values.  P[j][i] represents the probability of moving from node i to node j.
                x_0: the initial pagerank scores. a numpy vector of shape (n by 1).
                max_steps: the maximium number of random walk steps. an integer value.
        Output:
                x: the final pagerank scores after multiple steps of random walk. a numpy vector of shape (n by 1).
                n_steps: the number of steps actually used (for example, if the vector x no longer changes after 3 steps of random walk, return the value 3.
        Hint: you could use np.allclose(x, previous_x) function to determine when to stop the random walk iterations.
    '''
    #########################################
    ## INSERT YOUR CODE HERE

    x = x_0

    for n_steps in range(1, max_steps+1):
        previous_x = x
        x = random_walk_one_step(P, previous_x)
        if np.allclose(x, previous_x):
            break

    #########################################

    return x, n_steps

#--------------------------
def pagerank_v1(A):
    '''
        A simplified version of PageRank algorithm.
        Given an adjacency matrix A, compute the pagerank score of all the nodes in the network.
        Here we ignore the issues of sink nodes and sink regions in the network.
        Input:
                A: adjacency matrix, a numpy matrix of binary values. If there is a link from node i to node j, A[j][i] =1. Otherwise A[j][i]=0 if there is no link.
        Output:
                x: the ranking scores, a numpy vector of float values, such as np.array([[.3], [.5], [.7]])
    '''

    # compute the transition matrix from adjacency matrix
    P = compute_P(A)

    # initialize the score vector with all one values
    num_nodes, _ = A.shape # get the number of nodes (n)
    x_0 =  np.ones((num_nodes,1)) # create an all-one vector of shape (n by 1)

    # random walk
    x, n_steps = random_walk(P, x_0)

    return x


'''
@author: Professor
'''
# import RootFinders as R

from numpy import ceil, log2
from math import fabs

# -------------------------------------------

def bisection(f,a,b,tol=1e-9):
	fa, fb = f(a), f(b)
	N = int( ceil( log2((b-a)/tol ) ) )

	x = []  # empty list

	for i in range(N):
		c = (b+a)/2
		x.append(c)
		fc = f(c)
		if fa*fc < 0:  # x* is in [a,c]
			b, fb = c, fc
		else:
			a, fa = c, fc

	x.append((b+a)/2)
	return x

# ----------------------------------------------

def newton(f,fp,x0,tol=1e-10,maxIt=50):
	dx, fx = 100, 100
	iter = 0
	xa = []
	xa.append(x0)
	x = x0

	while fabs(dx)>tol or fabs(fx)>tol and iter<=maxIt:
		fx, fpx = f(x), fp(x)
		dx = -fx/fpx
		print(dx)
		x += dx
		iter += 1
		xa.append(x)

	return xa

# ----------------------------------------------

from pylab import *

def newtonSystem(f,J,x0,tol=1e-10,maxIt=50):

	N = len(x0)
	dx, fx = ones((N,)), ones((N,))
	iter = 0
	xa = []
	xa.append(x0)
	x = x0

	while norm(dx,inf)>tol or norm(fx,inf)>tol and iter<=maxIt:
		fx, fpx = f(x), J(x)
#		dx = -fx/fpx
		dx = solve(fpx,-fx)
		x += dx
		iter += 1
		xa.append(x)

	return xa


# -*- coding: utf-8 -*-
import numpy as np
# PCA
# X: 观测矩阵
# newX: 待转换的新样本
X = np.array([[1,2,3],[2,3,5],[5,6,7],[2,3,2]])  # 4个变量,3个特征
newX = np.array([[2,1,3]])

# 首先使用sklearn做PCA转换, 与下面进行对比
from sklearn.decomposition import PCA
pca = PCA(n_components=3)
pca.fit(X)
print pca.transform(newX)
radio = pca.explained_variance_ratio_
print(radio)

print ("================")
#  自己实现
# sklearn api均已行为一个样本, 下面计算使用列作为一个样本
X = X.T
##(1)计算协方差矩阵 :
 #   先把观测矩阵化为平均偏差形式: B = X - Xbar
 #   在计算平均偏差下的协方差矩阵: varX= B * B.T *(1/n).
Xbar = (1.0/4)*np.sum(X,axis=1,keepdims=True)
print Xbar
B = X-Xbar # 平均偏差形式
varX = np.dot(B,B.T)*(1.0/4)  # 协方差矩阵 = (1/n)* B*B.T

##(2)计算协方差矩阵的特征向量,后从大到小排列,得到解
print "协方差矩阵的特征值:"
eigVal = np.linalg.eig(varX)[0]  # 特征值 , 已按照从大到小排列 [  7.47698648e+00   1.56366740e-16   7.10513523e-01]
# eigVal = eigVal.reshape(3,1)
print(eigVal)
print("第一个主成分占总方差的: %s" % (eigVal[0]/np.sum(eigVal)))  # 对比radio[0]
p = np.linalg.eig(varX)[1]  # 特征向量矩阵
index = np.argsort(-eigVal) # 特征值从大到小排序的序号
print "协方差矩阵特征向量: "
p = p[:,index]  # 安札特征值从大到小排序后的特征向量重组矩阵
print p
# 如下为p的每一列,即协方差矩阵对应的特征向量, 每个特征向量都是原观测矩阵的Feature的一个线性组合
# sklearn中n_components属性为取前n特征向量, 对feature线性组合后的结果
# [[ -5.29168529e-01  -4.69020967e-01  -7.07106781e-01]
#  [ -5.29168529e-01  -4.69020967e-01   7.07106781e-01]
# [ -6.63295813e-01   7.48357311e-01  -1.26760092e-16]]

print ("my methods :")   # 取得的值与sklearn一致
print np.dot(p.T,(Xbar-newX.T))
# [[ 2.41662535]
#  [-1.41421356]
#  [ 0.47161626]]


import numpy as np

class Utils(object):
    @staticmethod
    def sigmoid(array):
        return 1 / (1 + np.exp(array))

    @staticmethod
    def sigmoid_derivative(array):
        sigmoid_value = 1 / (1 + np.exp(array))
        return np.multiply(sigmoid_value, 1 - sigmoid_value)

def softmax(x):
    """
    对输入x的每一行计算softmax。
    该函数对于输入是向量（将向量视为单独的行）或者矩阵（M x N）均适用。
    代码利用softmax函数的性质: softmax(x) = softmax(x + c)
    参数:
    x -- 一个N维向量，或者M x N维numpy矩阵.
    返回值:
    x -- 在函数内部处理后的x
    """
    orig_shape = x.shape

    # 根据输入类型是矩阵还是向量分别计算softmax
    if len(x.shape) > 1:
        # 矩阵
        tmp = np.max(x, axis=1)  # 得到每行的最大值，用于缩放每行的元素，避免溢出
        x -= tmp.reshape((x.shape[0], 1))  # 利用性质缩放元素
        x = np.exp(x)  # 计算所有值的指数
        tmp = np.sum(x, axis=1)  # 每行求和
        x /= tmp.reshape((x.shape[0], 1))  # 求softmax
    else:
        # 向量
        tmp = np.max(x)  # 得到最大值
        x -= tmp  # 利用最大值缩放数据
        x = np.exp(x)  # 对所有元素求指数
        tmp = np.sum(x)  # 求元素和
        x /= tmp  # 求somftmax
    return x

# def softmax(x):
#     max = np.max(x)
#     return np.exp(x - max) / sum(np.exp(x - max))

def der_softmax_cross_entropy(act_array, pre_array):
    # y_act = onehot(num_class,label_array)
    y_act = act_array
    y_hat = pre_array
    return y_hat - y_act

def sigmoid(x):
    """sigmoid函数"""
    if x.any() >= 0:  # 对sigmoid函数的优化，避免了出现极大的数据溢出
        return 1.0 / (1 + np.exp(-x))
    else:
        return np.exp(x) / (1 + np.exp(x))

def der_sigmoid(x):
    """sigmoid函数的导数"""
    return sigmoid(x) * (1 - sigmoid(x))

def tanh(x):
    """tanh函数"""
    return ((np.exp(x) - np.exp(-x)) / (np.exp(x) + np.exp(-x)))

def der_tanh(x):
    """tanh函数的导数"""
    return 1 - tanh(x) * tanh(x)

def relu(x):
    """relu函数"""
    temp = np.zeros_like(x)
    if_bigger_zero = (x > temp)
    return x * if_bigger_zero

def der_relu(x):
    """relu函数的导数"""
    temp = np.zeros_like(x)
    if_bigger_equal_zero = (x >= temp)
    return if_bigger_equal_zero * np.ones_like(x)

def onehot(num_class, label_array):
    return np.eye(num_class)[label_array]


import numpy as np
import nanalysis as na
import support
import matplotlib.pyplot as plt
from support import BaseFunction

class Lotka_volterra( BaseFunction ):
    """
    :param r1, r2 : float : intrinsic rate of natural increase
    :param K1, K2 : float : carrying capacity
    :param a      : float :
    :param b      : float :
    :return dx/dt, dy/dt : float
    """
    def __init__( self, r1 = 1, r2 = 2,  K1 = 80, K2 = 70, a = 0.7, b = 1.1 ):

        self.r1 = r1
        self.r2 = r2
        self.K1 = K1
        self.K2 = K2
        self.a = a
        self.b = b

    def get_values( self, x, y, t ):

        return self.r1 * x * ( 1 - ( x + self.a * y ) /self.K1 ), self.r2 * y * ( 1 - (self.b *  x + y ) / self.K2  )

    def get_linelist( self ):

        return  [[ -1 / self.a, self.K1 / self.a ], [ -self.b, self.K2 ] ]

class Fujita_model( BaseFunction ):

    def __init__( self,  r = 0.1, a = 0.3, b = 0.1, c = 0.3 ):

        self.r = r
        self.a = a
        self.b = b
        self.c = c

    def get_values( self, x, y, t ):

        return self.r * x * ( 1 - self.a * x - self.b * y), -self.c * y + x

    def get_linelist( self ):

        return [ [ - self.a / self.b, 1 / self.b ], [ 1 / self.c, 0 ] ]

if __name__ == '__main__':

    ps = support.get_base_args()
    args = ps.parse_args()

    graph_n = args.equation_number
    ad = args.argdict

    sp   = args.spoint
    ep   = args.epoint

    hl   = args.hline
    vl   = args.vline
    llist  =  args.line
    pp   =  args.pointplot
    initial_value = args.initial_value

    saveoff = support.offsaving( args.savefigoff )

    graph = { 1: Lotka_volterra , 4: Fujita_model}

    if ad:
        function = graph[ graph_n ](**ad)
    else:
        function = graph[ graph_n ]()
    linelist =  function.get_linelist()

    if llist:
        linelist.append(llist)

    t, xpoints, ypoints = function.get_values_rungekutta( sp = sp, ep = ep, initial_value = initial_value )

    na.graph_plot( t, xpoints  ,ypoints , chapter = 2, function = function, hlines = hl, vlines = vl, linelist = linelist , savefigOn = saveoff, graph_n = graph_n, N = 1000, pointplot = pp)


"""
Support library for computing nonlinearities and derivatives.

Requires numpy.
"""

import numpy as np

def sigmoid(X):
    """
    INPUT:
    X: a numpy matrix

    OUTPUT:
    A numpy matrix after element-wise application of sigmoid
    """
    Y = 1.0 / (1.0 + np.exp(-X))
    return Y

def deriv_sigmoid(X):
    """
    INPUT:
    X: a numpy matrix

    OUTPUT:
    A numpy matrix giving the element-wise derivation of sigmoid(X) wrt X
    """
    Y = (1.0 - sigmoid(X)) * sigmoid(X)
    return Y

def positivecosine(X):
    """
    INPUT:
    X: a numpy matrix

    OUTPUT:
    A numpy matrix after element-wise application of cosine
    """
    Y = 1.0 + np.cos(X)
    return Y

def deriv_positivecosine(X):
    """
    INPUT:
    X: a numpy matrix

    OUTPUT:
    A numpy matrix giving the element-wise derivation of cosine(X) wrt X
    """
    Y = -np.sin(X)
    return Y

def positivesine(X):
    """
    INPUT:
    X: a numpy matrix

    OUTPUT:
    A numpy matrix after element-wise application of sine
    """
    Y = 1.0 + np.sin(X)
    return Y

def deriv_positivesine(X):
    """
    INPUT:
    X: a numpy matrix

    OUTPUT:
    A numpy matrix giving the element-wise derivation of sin(X) wrt X
    """
    Y = np.cos(X)
    return Y

def negexp(X):
    """
    INPUT:
    X: a numpy matrix

    OUTPUT:
    A numpy matrix after element-wise application of sine
    """
    Y = np.exp(-X)
    return Y

def deriv_negexp(X):
    """
    INPUT:
    X: a numpy matrix

    OUTPUT:
    A numpy matrix giving the element-wise derivation of sin(X) wrt X
    """
    Y = -np.exp(-X)
    return Y

def softReLU(X,epsilon=0):
    """
    INPUT:
    X: a numpy matrix

    OUTPUT:
    A numpy matrix after element-wise application of rectifier linear unit
    """
    Y = X.copy()
    Y[Y<0] = epsilon
    return Y

def deriv_softReLU(X,epsilon=0):
    """
    INPUT:
    X: a numpy matrix

    OUTPUT:
    A numpy matrix giving the element-wise derivation of softReLU(X) wrt X
    """
    Y = np.ones(X.shape)
    Y[X<0] = 0
    return Y

def softabsolute(X, epsilon=1e-8):
    """
    INPUT:
    X: a numpy matrix
    epsilon: epsilon machine to compute soft-absolute value

    OUTPUT:
    A numpy matrix after element-wise application of soft-absolute value
    """
    Y = np.sqrt(X**2 + epsilon)
    return Y

def deriv_softabsolute(X, epsilon=1e-8):
    """
    INPUT:
    X: a numpy matrix
    epsilon: epsilon machine to compute soft-absolute value

    OUTPUT:
    A numpy matrix giving the element-wise derivation of softabsolute(X) wrt X
    """
    Y = X / softabsolute(X, epsilon)
    return Y


import numpy as np

def dichoto(function, p0, max_depth=10,  eps=1e-10):
    a,b=p0
    i=0
    while i < max_depth:
        c=0.5*(a+b)
        if np.abs(function(c))<=eps:
            return c
        if(function(c)<0):
            a=c
        if(function(c)>0):
            b=c
        i=i+1
    return c

# Find the the golden ratio
f = lambda x : x**2-1-x
x = dichoto(f, (1, 2))
print("The golden ratio is : {}".format(x))

# Find the the solution
f = lambda x : np.tan(x)-1
x = dichoto(f, (0.5, 3.15/4), )
print("The solution to tan(x)=1 is : {}".format(x))

# Find the the solution
f = lambda x : (x-2)**2
x = dichoto(f, (1, 3), )
print("The solution to (x-2)^2=0 is : {}".format(x))


import numpy as np
#computes the logistic regression log-likelihood for data z and parameter th
#input: z = N x D numpy array, th = length D numpy array
#output: length N numpy array of log_likelihoods
def log_likelihood(z, th):
  if len(z.shape) == 1:
    m = -(th*z).sum()
    if m < 100:
      m = -np.log1p(np.exp(m))
    else:
      m = -m
    return m
  else:
    m = -(th*z).sum(axis=1)
    idcs = m < 100
    m[idcs] = -np.log1p(np.exp(m[idcs]))
    m[np.logical_not(idcs)] = -m[np.logical_not(idcs)]
    return m

#computes the gradient of the logistic regression log-likelihood
# z is data, one row per vector; th is the parameter
#input: z = N x D numpy array, th = length D numpy array, idx = optional gradient component index
#output: (if idx = None): N x D array of gradients  (if idx = integer) N x 1 array of gradient components
def grad_log_likelihood(z, th, idx=None):
  if len(z.shape) == 1:
    m = -(th*z).sum()
    if m < 100:
      m = np.exp(m)/(1.+np.exp(m))
    else:
      m = 1.
    return m*z
  else:
    m = -(th*z).sum(axis=1)
    idcs = m < 100
    m[idcs] = np.exp(m[idcs])/(1.+np.exp(m[idcs]))
    m[np.logical_not(idcs)] = 1.
    if idx is None:
      return m[:, np.newaxis]*z
    return m*z[:, idx]

#computes the log prior for parameter th
#input: th = length D numpy array
#output: log prior density value, scalar
def log_prior(th):
  return -0.5*th.shape[0]*np.log(2.*np.pi) - 0.5*(th**2).sum()

#computes the log prior gradient for parameter th
#input: th = length D numpy array
#output: length D numpy gradient array
def grad_log_prior(th):
  return -th

#computes the log joint probability for data z and parameter th, where the data are weighted by wts
#input: Z = N x D numpy array, th = length D numpy array, wts = length N numpy array of nonnegative values
#output: weighted log joint, scalar
def log_joint(Z, th, wts):
  return (wts*log_likelihood(Z, th)).sum() + log_prior(th)

#same as above; outputs length D numpy array gradient
def grad_log_joint(z, th, wts):
  return grad_log_prior(th) + (wts[:, np.newaxis]*grad_log_likelihood(z, th)).sum(axis=0)

#same as above; outputs D x D numpy array Hessian
def hess_log_joint(z, th):
  es = np.exp(-(th*z).sum(axis=1))
  H_log_like = -(z.T).dot((es/(1.+es)**2)[:, np.newaxis]*z)
  H_log_prior = -np.eye(th.shape[0])
  return H_log_like + H_log_prior


import numpy as np

x = np.array([1,2,3])
y = np.array([4,5,6])
print(x,"\n", y)

#Different ways to get the dot product

#1
import numpy as np

x = np.array([1,2,3])
y = np.array([4,5,6])
dot_Product = 0

for i in range (3):
  dot_Product += x[i]*y[i]
print(dot_Product)

#2
dot = 0
for i , j in zip(x,y):
  dot += (i*j)
print("\nDot Product = ", dot)

#3
print("\nDot Product = ", sum(x*y))

#4
print("\nDot Product = ", np.dot(x,y))

#calculating The angle between two vectors:

# dot product = magnitue * magnitude * cos of angle

#2
cosangle = np.dot(x,y) / (np.linalg.norm(x) * np.linalg.norm(y))
print("\nangle =",np.arccos(cosangle))

#Creating an ideneity Matrix
print(np.eye((3)))


import numpy as np

def Brownian(seed, N):
    """Generates Brownian Motion.
    :param seed:   random seed to use
    :param N:      increments
    :return W:     brownian motion
    :return b:     brownian increments
    """
    np.random.seed(seed)
    dt = 1./N
    b = np.random.normal(0., 1., int(N))*np.sqrt(dt)
    W = np.cumsum(b)
    return W, b

def GBM(So, mu, sigma, W, N):
    """Generates Geometric Brownian Motion.
    :param So:       starting price
    :param mu:       drift coefficient
    :param sigma:    diffusion coefficient
    :param W:        Brownian Motion
    :param N:        increments to predict - **N = len(W)**
    :return S:       Geometric Brownian Motion (S(t))
    :return t:       all time-steps
    """
    t = np.linspace(0.,1.,N+1)
    S = []
    S.append(So)
    for i in range(1,int(N+1)):
        drift = (mu - 0.5 * sigma**2) * t[i]
        diffusion = sigma * W[i-1]
        S_temp = So*np.exp(drift + diffusion)
        S.append(S_temp)
    return S, t

def GBM_step(mu, S, N, sigma, W):
    """Calculates dS over time T
    :param mu: drift
    :param S:  true price - an iterable?
    :param N:
    :param sigma: diffusion
    :param W: brownian
    :return dS: GBM step
    """
    dt = 1./N
    dS = []
    for i in range(1,int(N)):
        dS.append(mu*S[i]*dt + sigma*S[i]*W[i])
    S = np.cumsum(dS)
    return S

def sentiment_GBM(P, S, mu, sigma, alpha, W, N):
    """Applies weighted sentiment to GBM
    :param :
    :return :
    """
    dt = 1./N
    dP = []
    for i in range(int(N)):
        dP.append(mu*P[i]*dt + sigma*P[i]*W[i] + alpha*S[i])
    P = np.cumsum(dP)
    return P

def daily_returns(close, N):
    """Calculates daily returns, then drift and diffusion from those returns.
    :param close:       iterable of closing prices as [n, price]
    :param N:           increments to predict
    :return returns:    daily returns
    :return mu:         drift
    :return sigma:      diffusion
    """
    returns = []
    for i in range(0, len(close)-1):
        today = close[i+1]
        yesterday = close[i]
        r = (today - yesterday) / yesterday
        returns.append(r)
    mu = np.mean(returns)*(N*1.) # drift
    sigma = np.std(returns)*np.sqrt(N*1.)#diffusion
    return mu, sigma


import sys
sys.path.append('../../utils')

import numpy as np
import matplotlib.pyplot as plt
import scipy.optimize as opt
import data_preprocessing as Preprocessing
from pandas.io.parsers import read_csv

def load_csv(file_name):
    values = read_csv(file_name, header=None).values

    return values.astype(float)

def gradient(thetas, XX, Y):
    H = h(thetas, XX)
    grad = (1/len(Y)) * np.dot(XX.T, H-Y)

    return grad

def cost(thetas, X, Y):
    m = np.shape(X)[0]
    H = h(thetas, X)

    c = (-1/(len(X))) * (np.dot(Y.T, np.log(H)) + np.dot((1-Y).T, np.log(1-H)))

    return c

def sigmoid(Z):
    return 1/(1 + np.e**(-Z))

def h(thetas, X):
    return np.c_[sigmoid(np.dot(X, thetas))]

def show_border(thetas, X, Y):
    x1_min, x1_max = X[:, 1].min(), X[:, 1].max()
    x2_min, x2_max = X[:, 2].min(), X[:, 2].max()

    xx1, xx2 = np.meshgrid(np.linspace(x1_min, x1_max),
    np.linspace(x2_min, x2_max))

    H = sigmoid(np.c_[np.ones((xx1.ravel().shape[0], 1)),
    xx1.ravel(),
    xx2.ravel()].dot(thetas))

    H = H.reshape(xx1.shape)

    plt.figure()

    positives = np.where(Y == 1)
    negatives = np.where(Y == 0)
    plt.scatter(X[positives, 1], X[positives, 2], marker='+', color='blue')
    plt.scatter(X[negatives, 1], X[negatives, 2], color='red')

    plt.contour(xx1, xx2, H, [0.5], linewidths=1, colors='b')
    plt.savefig("images/regresion_logistic_border.png")
    plt.show()
    plt.close()

def evaluate(thetas, X, Y):
    result = h(thetas, X)
    passed_missed = np.logical_and((result >= 0.5), (Y == 0)).sum()
    failed_missed = np.logical_and((result < 0.5), (Y == 1)).sum()

    errors = (passed_missed + failed_missed)

    return (result.shape[0] - errors) / (result.shape[0])

def train(X, Y, verbose = True):
    XOnes = Preprocessing.addInitialOnes(X)
    thetas = np.zeros((XOnes.shape[1], 1), dtype=float)
    result = opt.fmin_tnc(func=cost, x0=thetas, fprime=gradient, args=(XOnes, Y), disp = 5 if verbose else 0)
    thetas = result[0]

    return thetas

def main():
    # DATA PREPROCESSING
    data = load_csv("data/ex2data1.csv")
    X, Y, m, n = Preprocessing.separate_data(data)
    thetas = train(X, Y)

    print("Accuracy: ", evaluate(thetas, X, Y)*100, "%")
    show_border(thetas, X, Y)

if __name__ == "__main__":
    main()


import matplotlib.pyplot as mplt
import numpy as np

def f(x):
    return x**2.0

def oneDGradient(vals, dx):

    n = len(vals)
    grad = [0.0] * n

    #forward difference for outer bounds
    grad[0] = (vals[1] - vals[0]) / dx
    for i in range(1, n - 1):
        #central difference now that data is available
        grad[i] = (vals[i + 1] - vals[i - 1]) / (2.0 * dx)
    #forward difference for outer bounds
    grad[n - 1] = (vals[n - 1] - vals[n - 2]) / dx

    return grad

#########################################

#dimensions
Nx = 21;       # Number of X-grid points
dx = 1.0;      # x coordinate grid spacing

########## sanity check against numpy ##

#establish X axis
X = np.arange(0, Nx, dx)
#compute function LUT
fx = X**2.0
#compute derivative in 1D
dFdX = np.gradient(fx, dx)

########## my implementation ##########

testData = [0.0] * Nx

x = 0
for i in range(Nx):
    #function LUT
    testData[i] = f(x)
    #x axis
    x += dx

#computer gradient
gradientData = oneDGradient( testData, dx )

#########################################

fig, (ax1, ax2) = mplt.subplots(2)

ax1.plot(testData, label = 'my implementation')
ax1.plot(gradientData)
ax1.legend( bbox_to_anchor = (1.0, 1), loc = 'upper right' )

ax2.plot(fx, label = 'numpy implementation')
ax2.plot(dFdX)
ax2.legend( bbox_to_anchor = (1.0, 1), loc = 'upper right' )

mplt.show()


# Takes in a list of Y values
# Returns a list of 0's and 1's
# 1 means the matching value in Y is an outlier
import numpy as np

def zscore(points, threshold=3):
    dataset = [y for x, y in points]
    mean = np.mean(dataset)
    std = np.std(dataset)
    outliers = []
    for y in dataset:
        z_score = (y - mean) / std
        if np.abs(z_score) > threshold:
            outliers.append(1)
        else:
            outliers.append(0)
    return outliers

# inter quartile range
def IQR(points):
    dataset = [y for x, y in points]
    outliers = []
    sorted_data = sorted(dataset)
    q1, q3 = np.percentile(sorted_data, [25, 75])
    iqr = q3 - q1
    lower_bound = q1 - (1.5 * iqr)
    upper_bound = q3 + (1.5 * iqr)

    for data in dataset:
        if data > upper_bound or data < lower_bound:
            outliers.append(1)
        else:
            outliers.append(0)
    return outliers

def _seperate(data, outlier_list):
    output = ([], [])
    for e, outlier in zip(data, outlier_list):
        if outlier == 1:
            output[1].append(e)
        else:
            output[0].append(e)
    return output

# Take points as a list of pairs
# returns a pair of lists with ([good points],[outliers])
def calc_outliers(points, method):
    if "zscore" in method:
        return _seperate(points, zscore(points))
    elif "IQR" in method:
        return _seperate(points, IQR(points))
    return (points, [])


#! /usr/bin/env python
import numpy as np

def multi_variate_normal(x, mu, sigma=None, log=True, inv_sigma=None):
	"""
	Multivariatve normal distribution PDF

	:param x: data, np.array([nb_samples, nb_dim])
	:param mu: mean, np.array([nb_dim])
	:param sigma: covariance matrix, np.array([nb_dim, nb_dim])
	:param log: compute the loglikelihood if true
	:return: pdf of the data for the given Gaussian distribution
	"""
	dx = x - mu
	if sigma.ndim == 1:
		sigma = sigma[:, None]
		dx = dx[:, None]
		inv_sigma = np.linalg.inv(sigma) if inv_sigma is None else inv_sigma
		log_lik = -0.5 * np.sum(np.dot(dx, inv_sigma) * dx, axis=1) - 0.5 * np.log(np.linalg.det(2 * np.pi * sigma))
	else:
		inv_sigma = np.linalg.inv(sigma) if inv_sigma is None else inv_sigma
		log_lik = -0.5 * np.einsum('...j,...j', dx, np.einsum('...jk,...j->...k', inv_sigma, dx)) - 0.5 * np.log(np.linalg.det(2 * np.pi * sigma))

	return log_lik if log else np.exp(log_lik)


# -*- coding: utf-8 -*-
"""
This problem was asked by Facebook.

There is an N by M matrix of zeroes. Given N and M, write a function to count the number of ways of starting at the top-left corner and getting to the bottom-right corner. You can only move right or down.

For example, given a 2 by 2 matrix, you should return 2, since there are two ways to get to the bottom-right:

Right, then down
Down, then right
Given a 5 by 5 matrix, there are 70 ways to get to the bottom-right.
"""

from scipy.special import comb

def f(m,n):
    k=min(m-1,n-1) #since matrix is transposable, can also use max w/ same result.
    n=(m-1)+(n-1)
    print(comb(n,k))

#test:

f(2,2)
#2
f(5,5)
#70

#test transposition
f(10,4)
#220

f(4,10)
#220

#solve time : 5 minutes


# Daniel J. Rodriguez
# https://github.com/danieljoserodriguez

import numpy as np

# A straight line function where activation is proportional to input
# ( which is the weighted sum from neuron ).

# In mathematics, an identity function, also called an identity relation or
# identity map or identity transformation, is a function that always returns
# the same value that was used as its argument.
#
# https://en.wikipedia.org/wiki/Identity_function
def identity(x):
    return x

# derivative identity
def d_identity(x):
    return 1.0

# bent identity
#
# The Heaviside step function, or the unit step function, usually denoted by
# H or θ (but sometimes u, 1 or 𝟙), is a discontinuous function, named after
# Oliver Heaviside (1850–1925), whose value is zero for negative arguments
# and one for positive arguments
def bent_identity(x):
    return ((np.sqrt(((x ** 2.0) + 1.0)) - 1.0) / 2.0) + x

# derivative bent identity
def d_bent_identity(x):
    return (x / (2.0 * np.sqrt((x ** 2.0) + 1.0))) + 1.0

# also called heaviside step
#
# https://en.wikipedia.org/wiki/Heaviside_step_function
def binary_step(x):
    return 1.0 if x >= 0.0 else 0.0

def perceptron(weights, bias, inputs):
    return 1.0 if (np.dot(weights, inputs) + bias) > 0.0 else 0.0

def smooth_perceptron(weights, bias, inputs):
    return np.dot(weights, inputs) + bias

# Sigmoid takes a real value as input and outputs another value between 0 and 1.
# It’s easy to work with and has all the nice properties of activation functions:
# it’s non-linear, continuously differentiable, monotonic, and has a fixed output range.
#
# https://en.wikipedia.org/wiki/Logistic_function
def sigmoid(x):
    return 1.0 / (1.0 + np.exp(-x))

# derivative sigmoid
def d_sigmoid(x):
    s = sigmoid(x)
    return s * (1.0 - s)

# Tanh squashes a real-valued number to the range [-1, 1]. It’s non-linear. But unlike
# Sigmoid, its output is zero-centered. Therefore, in practice the tanh non-linearity
# is always preferred to the sigmoid non-linearity.
#
# https://en.wikipedia.org/wiki/Hyperbolic_function#Hyperbolic_tangent
def tanh(x):
    return np.tanh(x)

# derivative hyperbolic tangent
def d_tanh(x):
    return 1.0 - tanh(x) ** 2.0

# arc tangent
# https://en.wikipedia.org/wiki/Inverse_trigonometric_functions
def arc_tan(x):
    return np.arctan(x)

# derivative arc tangent
def d_arc_tan(x):
    return 1.0 / ((x ** 2.0) + 1.0)

# Inverse hyperbolic sine
# https://en.wikipedia.org/wiki/Inverse_hyperbolic_functions#Inverse_hyperbolic_sine
def ar_sinh(x):
    return np.log(x + np.sqrt((x ** 2.0) + 1.0))

# derivative of Inverse hyperbolic sine
def d_ar_sinh(x):
    return 1.0 / (np.sqrt((x ** 2.0) + 1.0))

# elliot sig also called soft sign
# https://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.46.7204
def elliot_sig(x):
    return x / (1.0 + abs(x))

# derivative elliot sig / soft sign
def d_elliot_sig(x):
    return 1.0 / ((1.0 + abs(x)) ** 2.0)

# inverse square root unit
# https://arxiv.org/abs/1710.09967
def isru(x, alpha):
    return x / np.sqrt((x + (alpha * (x ** 2.0))))

# derivative inverse square root unit
def d_isru(x, alpha):
    return x / (np.sqrt(1.0 + (alpha * (x ** 2.0))) ** 3.0)

# inverse square root linear unit
# https://arxiv.org/abs/1710.09967
def isrlu(x, alpha):
    return x / (np.sqrt(1.0 + (alpha * (x ** 2.0)))) if x < 0.0 else x

# derivative inverse square root linear unit
def d_isrlu(x, alpha):
    return ((1.0 / np.sqrt(1.0 + alpha * (x ** 2.0))) ** 3.0) if x < 0.0 else 1.0

# square non-linearity
# https://ieeexplore.ieee.org/document/8489043
def sqnl(x):
    if x > 2.0:
        return 1.0
    elif 0.0 <= x <= 2.0:
        return x - ((x ** 2.0) / 4.0)
    elif -2.0 <= x < 0.0:
        return x + ((x ** 2.0) / 4.0)
    elif x < -2.0:
        return -1.0

# derivative square non-linearity
def d_sqnl(x):
    return (1.0 - (x / 2.0)), (1.0 + (x / 2.0))

# A recent invention which stands for Rectified Linear Units. The formula is deceptively
# simple: max(0,z). Despite its name and appearance, it’s not linear and provides the
# same benefits as Sigmoid but with better performance.
#
# https://en.wikipedia.org/wiki/Rectifier_(neural_networks)
def relu(x):
    return x if x >= 0.0 else 0.0

# derivative rectified linear units
def d_relu(x):
    return 1.0 if x >= 0.0 else 0.0

# bipolar relu
# https://arxiv.org/abs/1709.04054
def brelu(x, index):
    return relu(x) if index % 2.0 == 0.0 else -relu(-x)

# derivative bipolar relu
def d_brelu(x, index):
    return d_relu(x) if index % 2.0 == 0.0 else -d_relu(-x)

# LeakyRelu is a variant of ReLU. Instead of being 0 when z<0, a leaky ReLU allows a
# small, non-zero, constant gradient α (Normally, α=0.01). However, the consistency
# of the benefit across tasks is presently unclear.
#
# https://pdfs.semanticscholar.org/367f/2c63a6f6a10b3b64b8729d601e69337ee3cc.pdf
def leaky_relu(x):
    return x if x >= 0.0 else 0.01 * x

# derivative leaky relu
def d_leaky_relu(x):
    return 1.0 if x >= 0.0 else 0.01

# parametric relu makes the coefficient of leakage into a parameter that is
# learned along with other neural network parameters - alpha
#
# https://arxiv.org/abs/1502.01852
def prelu(x, alpha):
    return x if x >= 0.0 else alpha * x

# derivative parametric relu
def d_prelu(x, alpha):
    return 1.0 if x >= 0.0 else alpha

# randomized leaky relu
# https://arxiv.org/abs/1505.00853
def rrelu(x, alpha):
    return x if x >= 0.0 else alpha * x

# derivative randomized leaky relu
def d_rrelu(x, alpha):
    return 1.0 if x >= 0.0 else alpha

# Exponential Linear Unit or its widely known name ELU is a function that tend to
# converge cost to zero faster and produce more accurate results. Different to other
# activation functions, ELU has a extra alpha constant which should be positive number
#
# ELU is very similar to RELU except negative inputs. They are both in identity
# function form for non-negative inputs. On the other hand, ELU becomes smooth
# slowly until its output equal to -α whereas RELU sharply smooths.
#
# https://arxiv.org/abs/1511.07289
def elu(x, alpha):
    return x if x > 0.0 else alpha * ((np.e ** 2.0) - 1.0)

# derivative exponential linear unit
def d_elu(x, alpha):
    return 1.0 if x > 0.0 else elu(x, alpha) + alpha

# scaled exponential linear unit
# https://en.wikipedia.org/wiki/Activation_function#cite_note-20
def selu(x, alpha):
    # d = 1.0507
    # if x >= 0:
    #     return x * d
    # else:
    #     return d * alpha * ((np.e ** 2) - 1)
    pass

# derivative scaled exponential linear unit
def d_selu(x, alpha):
    pass

# s-shaped relu
# https://arxiv.org/abs/1512.07030
def srelu():
    pass

# derivative s-shaped relu
def d_srelu():
    pass

# adaptive piecewise linear
# https://arxiv.org/abs/1412.6830
def apl(x, alpha):
    # return np.maximum(0, x) + np.sum(alpha * np.max(0, -x + ))
    pass

# derivative apl
def d_apl():
    pass

# gaussian error linear units
# https://arxiv.org/abs/1606.08415
def gelu(x):
    # return (x * (1 + (x / np.sqrt(2)))) / 2
    pass

# derivative gaussian error linear units
def d_gelu():
    pass

# Softmax function calculates the probabilities distribution of the event over ‘n’ different
# events. In general way of saying, this function will calculate the probabilities of each
# target class over all possible target classes. Later the calculated probabilities will be
# helpful for determining the target class for the given inputs.
#
# https://en.wikipedia.org/wiki/Softmax_function
def soft_max(x):
    return np.exp(x) / np.sum(np.exp(x), axis=0)

# derivative soft max
def d_soft_max(output, trues):
    temp = output - trues
    return temp / len(trues)

# soft plus
# http://proceedings.mlr.press/v15/glorot11a/glorot11a.pdf
def soft_plus(x):
    return np.log(1.0 + (np.e ** x))

# derivative soft plus
def d_soft_plus(x):
    return 1.0 / (1.0 + (np.e ** -x))

# soft exponential
# https://arxiv.org/abs/1602.01321
def soft_exponential(x, alpha):
    if alpha < 0.0:
        return -((np.log(1.0 - alpha * (x + alpha))) / alpha)
    elif alpha > 0.0:
        return ((np.e ** (alpha * x)) / alpha) + alpha
    elif alpha == 0.0:
        return x

# derivative soft exponential
def d_soft_exponential(x, alpha):
    return 1.0 / (1.0 - alpha * (alpha + x)) if alpha < 0.0 else np.e ** (alpha * x)

# soft clipping
# https://arxiv.org/abs/1810.11509
def soft_clipping(x, alpha):
    return (1.0 / alpha) * (np.log10((1.0 + (np.e ** (alpha * x))) / (1.0 + (np.e ** (alpha * (x - 1.0))))))

# derivative soft clipping
def d_soft_clipping(x, p):
    a = np.cosh((p * x) / 2.0) ** (-1.0)
    b = np.cosh((p / 2.0) * (1.0 - x)) ** (-1.0)
    return 0.5 * np.sinh(p / 2.0) * a * b

def gaussian_radial_basis(x, c, s):
    return np.exp(-1.0 / (2.0 ** (s ** 2.0)) * (x - c) ** 2.0)

# improvement over relu for deeper networks
def swish(x):
    # return x * (1 + np.exp(-x)) ** -1
    return x / ((np.e ** -x) + 1.0)

# derivative swish
def d_swish(x):
    return ((np.e ** x) * ((np.e ** x) + x + 1.0)) / (((np.e ** x) + 1.0) ** 2.0)

# sinusoid
# https://arxiv.org/abs/1405.2262
def sinusoid(x):
    return np.sin(x)

# derivative sinusoid
def d_sinusoid(x):
    return np.cos(x)

# sinc
# https://en.wikipedia.org/wiki/Sinc_function
def sinc(x):
    return 1.0 if x == 0.0 else np.sin(x) / x

# derivative sinc
def d_sinc(x):
    return 0.0 if x == 0.0 else (np.cos(x) / x) - (np.sin(x) / (x ** 2.0))

# gaussian
# https://en.wikipedia.org/wiki/Gaussian_function
def gaussian(x):
    return np.e ** (-x ** 2.0)

# derivative gaussian
def d_gaussian(x):
    return -2.0 * x * np.e ** (-x ** 2.0)

# gradient relu
def gradient_relu(a, x):
    _relu = relu(x)
    return np.multiply(a, np.int64(_relu > 0.0))


from sympy import *
import math
from numpy import arange

class FixedPointIteration:

    num_of_iteration = 0
    root

    def __init__(self, function_formula, initial_x, max_iterations, precision, x=Symbol('x')):
        self.function_formula = function_formula
        self.initial_x = initial_x
        self.X = x
        self.max_iterations = max_iterations
        if max_iterations == 0:
            self.max_iterations = 50
        self.precision = precision
        if precision == 0:
            self.precision = 0.0001

    def compute_root(self):

        try:
            # create the table
            table = [['i', 'Xi', 'relative_error']]
            row = [0, self.initial_x, None]
            table.append(row)

            i = 0
            # if the initial guess is the root
            if self.function_formula.evalf(subs={self.X: self.initial_x}) < 1e-10 \
                    and self.function_formula.evalf(subs={self.X: self.initial_x}) > -1e-10:
                FixedPointIteration.root = self.initial_x
                return [table], self.initial_x, true

            while True:

                i = i + 1

                iterative_x = float(self.function_formula.evalf(subs={self.X: self.initial_x}))

                # if the root is zero
                if iterative_x == 0.0:
                    if self.function_formula.evalf(subs={self.X: iterative_x}) < 1e-10 \
                            and self.function_formula.evalf(subs={self.X: iterative_x}) > -1e-10:

                        return [table], iterative_x, true
                    else:
                        relative_error = 1.0
                else:
                    relative_error = (iterative_x - self.initial_x) / iterative_x

                # add Row to the table
                row = [i, iterative_x, math.fabs(relative_error)]
                table.append(row)

                # break when reach max iteration or precision
                if (math.fabs(relative_error) <= self.precision) | (i >= self.max_iterations):
                    break

                self.initial_x = iterative_x

                if iterative_x < 1e-10 and iterative_x > -1e-10:
                    break

            final_table = [table]
            print (final_table)
            FixedPointIteration.root = iterative_x
            return final_table, iterative_x, true
        except:
            return [[[]]], 0.0, false

    # to do call to check if root
    def is_root(self):
        try:
            if self.function_formula.evalf(subs={self.X: FixedPointIteration.root}) < 1e-1 \
                    and self.function_formula.evalf(subs={self.X: FixedPointIteration.root}) > -1e-1:
                return true
            else:
                return false
        except:
            return false

    def get_x_y(self):

        a = []
        b = []

        for x in range(-50, 50, 1):
            y = self.function_formula.evalf(subs={self.X: x})
            a.append(x)
            b.append(y)
        return a, b


# variance calculator python
# https://www.google.com/search?q=variance+calculator+python&oq=variance+calculator+py&aqs=chrome.1.69i57j0j0i22i30.3719j0j7&sourceid=chrome&ie=UTF-8

import numpy as np
from statistics import variance, stdev

results = [-14.82381293, -0.29423447, -13.56067979, -1.6288903, -0.31632439,
           0.53459687, -1.34069996, -1.61042692, -4.03220519, -0.24332097]
# # way 1
# print(np.var(results, ddof=1))
# print(np.std(results, ddof=1))
a = np.array([[1, 2], [3, 4]])
# print(a)
# print(np.var(a))
# print(np.var(a, ddof=1))
# print(np.var(a, axis=0))
# print(np.var(a, axis=1))

# way 2
m = sum(results) / len(results)
var_res = sum((xi-m)**2 for xi in results)/(len(results)-1)
# print(var_res)

# way 3: not need ddof
# print(variance(results))
# print(stdev(results))


# --------------
import pandas as pd
import numpy as np
import math

#Code starts here
class complex_numbers:
    '''
    The Complex Number Class

    Attributes:
    real: real part of complex number
    imag: imaginary part of complex number
    '''

    def __init__(self,real,imag):
        self.real=real
        self.imag=imag

    def __repr__(self):
        if self.real == 0.0 and self.imag == 0.0:
            return "0.00"
        if self.real == 0:
            return "%.2fi" % self.imag
        if self.imag == 0:
            return "%.2f" % self.real
        return "%.2f %s %.2fi" % (self.real, "+" if self.imag >= 0 else "-", abs(self.imag))

    def __add__(self,other):
        '''
        '+' operator overloading
        complex number addition

        params:
        other:- other complex number for addition

        return:
        result of addition
        '''
        a=self.real+other.real
        b=self.imag+other.imag
        return complex_numbers(a,b)

    def __sub__(self,other):
        '''
        '-' operator overloading
        complex number subtraction

        params:
        other:- other complex number for subtraction

        return:
        result of subtraction
        '''
        a=self.real-other.real
        b=self.imag-other.imag
        return complex_numbers(a,b)

    def __mul__(self,other):
        '''
        '*' operator overloading
        complex number multiplication

        params:
        other:- other complex number for multiplication

        return:
        result of multiplication
        '''
        a=self.real*other.real - self.imag*other.imag
        b=self.real*other.imag + self.imag*other.real
        return complex_numbers(a,b)

    def __truediv__(self,other):
        '''
        '/' operator overloading
        complex number division

        params:
        other:- other complex number for division

        return:
        result of division
        '''
        a=(self.real*other.real+self.imag*other.imag)/(other.real*other.real+other.imag*other.imag)
        b=(self.imag*other.real-self.real*other.imag)/(other.real*other.real+other.imag*other.imag)
        return complex_numbers(a,b)

    def absolute(self):
        '''
        Absolute value of Complex number

        return:
        absolute value
        '''
        return np.sqrt((self.real**2)+(self.imag**2))

    def argument(self):
        '''
        argument value of Complex number

        return:
        argument value
        '''
        return math.degrees(math.atan(self.imag/self.real))

    def conjugate(self):
        '''
        conjugate value of Complex number

        return:
        conjugate value
        '''
        a,b=self.real,self.imag*-1
        return complex_numbers(a,b)

comp_1=complex_numbers(3,5)
print(comp_1)
comp_2=complex_numbers(4,4)
print(comp_2)
comp_sum=comp_1+comp_2
print(comp_sum)
comp_diff=comp_1-comp_2
print(comp_diff)
comp_prod=comp_1*comp_2
print(comp_prod)
comp_quot=comp_1/comp_2
print(comp_quot)
comp_abs=comp_1.absolute()
print(comp_abs)
comp_conj=comp_1.conjugate()
print(comp_conj)
comp_arg=comp_1.argument()
print(comp_arg)


import numpy as np
import pandas as pd
from scipy.optimize import minimize

class F_class:
    def __init__(self, a, A, B):
        self.a = a
        self.A = A
        self.B = B

    def calc(self, x):
        res = np.zeros(x.shape)

        bigger_idx = x > self.a
        smaller_idx = x <= self.a

        res[bigger_idx] = np.power(x[bigger_idx], 2) * self.A
        res[smaller_idx] = np.power(x[smaller_idx], 2) * self.B

        return res

a = 0
A = 1
B = -1

f = F_class(a, A, B)
X = np.arange(-5, 5)
Y = f.calc(X)
D = pd.DataFrame(columns=['X', 'Y'], data=np.array([X, Y]).T)

def sum_of_squares(params, X, Y):
    a, A, B = params
    model = F_class(a, A, B)
    y_pred = model.calc(X)
    obj = np.sqrt(((y_pred - Y) ** 2).sum())
    return obj

# perform fit to find optimal parameters
# initial value is a guess
initial_guess = [0., 0., 0.]  # a, A, B
res = minimize(sum_of_squares, x0=initial_guess, args=(X, Y), tol=1e-5, method="Powell")
a_pred, A_pred, B_pred = res.x

print("Estimated values:")
print(f"a = {a_pred:>.3f}")
print(f"A = {A_pred:>.3f}")
print(f"B = {B_pred:>.3f}")

model = F_class(a_pred, A_pred, B_pred)
Y_pred = model.calc(X)
MSE = np.sqrt(((Y_pred - Y) ** 2).sum())
print(f"MSE: {MSE}")


from sympy import *

def lagrange_interpolate(points: list, simplify_result=True, verbose=False):
    """拉格朗日插值

    Args:
        points: list, [(x1, y1), (x2, y2), ..., (xn, yn)]
        simplify_result: bool, 化简最终结果, default True
        verbose: bool, 输出每一步的结果, default False

    Returns:
        L: sympy object of Symbol('x'), 插值多项式 $L(x)$
    """
    x = Symbol('x')
    L = 0  # 插值多项式
    for i, point in enumerate(points):
        xi, yi = point
        li = 1
        for j in range(len(points)):
            if j == i:
                continue
            xj, yj = points[j]
            li *= (x - xj) / (xi - xj)
        L += yi * li
        if verbose:
            print(f"l_{i}(x) = ", simplify(yi * li))

    if simplify_result:
        L = simplify(L)
    return L


""" Rosembrock function """

import numpy as np

def function(x: np.array, n: int = 100) -> float:
    """ Compute the evaluation for Extended Rosembrock function with n=100
        Args:
        x: Array of length=n with x's parameters
        n: Rosembrock, n = 100

        Returns:
            Evaluation of f(X)
    """
    ans = 0.0
    for i in range(n-1):
        ans += 100 * (x[i+1] - x[i]**2)**2 + (1 - x[i])**2
    return ans

def gradient(x: np.array, n: int = 100) -> np.array:
    """ Compute the gradient evaluation for Extended Rosembrock function with n=2
        Args:
        x: Array of length=n with x's parameters
        n: Rosembrock, n = 100

        Returns:
            Gradient of f(x1, ..., xn), array with lenght=n
    """
    # grad = np.zeros(n, dtype=np.float64)
    # for i in range(n-1):
    #     grad[i] = -400 * x[i+1] * x[i] + 400 * x[i]**3 + 2 * x[i] -2
    # grad[n-1] = 200 * (x[n-1] - x[n-2]**2)
    # return grad
    grad = np.array([-400*(x[1]-x[0]**2)*x[0]-2*(1-x[0])])

    for i in range(1, n-1):
        grad = np.append(grad, [200*(x[i]-x[i-1]**2)-400*(x[i+1]-x[i]**2)*x[i]-2*(1-x[i])])

    grad = np.append(grad, [200*(x[99] - x[98]**2)])

    return grad

def hessian(x: np.array, n: int = 100) -> np.array:
    """ Compute the Hessian evaluation for Extended Rosembrock function with n=2
        Args:
        x: Array of length=n with x's parameters

        Returns:
            Hessian of f(x1, ..., xn), Matrix with size=nxn
    """
    hess = np.zeros((n, n), dtype=np.float64)
    for i in range(n-1):
        hess[i][i] = -400 * x[i+1] + 1200 * x[i]**2 + 2
        hess[i][i] += 200 if i != 0 else 0
        hess[i][i+1] = hess[i+1][i] = -400 * x[i]
    hess[n-1][n-1] = 200.0
    return hess


#!/usr/bin/python
# -*- coding: utf-8 -*-
##########################################################################################
# author: Nikolas Schnellbaecher
# contact: khx0@posteo.net
# date: 2021-04-11
# file: create_samples.py
# tested with python 3.7.6
##########################################################################################
# description:
# Creates (1) fully correlated and (2) fully independent normally distributed
# random samples using inverse transform sampling (ITS).
##########################################################################################

import os
import datetime
import numpy as np
import scipy
from scipy.stats import norm

today = datetime.datetime.now().strftime("%Y-%m-%d")

BASEDIR = os.path.dirname(os.path.abspath(__file__))
RAWDIR = os.path.join(BASEDIR, 'raw')
OUTDIR = os.path.join(BASEDIR, 'out')

os.makedirs(RAWDIR, exist_ok = True)

def inverseTransformSamplingJoint(n_samples, mu1, sigma1, mu2, sigma2):
    u = np.random.uniform(size = n_samples)
    x1 = norm.ppf(u, loc = mu1, scale = sigma1)
    x2 = norm.ppf(u, loc = mu2, scale = sigma2)
    return x1, x2

if __name__ == '__main__':

    print("using np.__version__ =", np.__version__)
    print("using scipy.__version__ =", scipy.__version__)

    # set parameters
    n_samples = 20000

    mu1, sigma1 = 87.25, 8.124
    mu2, sigma2 = 125.75, 11.25

    seed_value = 987654321

    ######################################################################################
    # 01 - Create fully correlated Gaussian samples using the inverse transform method
    samples = np.zeros((n_samples, 2))

    np.random.seed(seed_value)

    samples[:, 0], samples[:, 1] = \
        inverseTransformSamplingJoint(n_samples, mu1, sigma1, mu2, sigma2)
    outname = f'GaussianSamples_correlated_seed_{seed_value:d}.txt'
    np.savetxt(os.path.join(RAWDIR, outname), samples, fmt = '%.8f')

    ######################################################################################
    # 02 - Create two independent Gaussian random realizations

    np.random.seed(seed_value) # reset the seed value

    samples = np.zeros((n_samples, 2))
    samples[:, 0] = norm.rvs(loc = mu1, scale = sigma1, size = n_samples)
    samples[:, 1] = norm.rvs(loc = mu2, scale = sigma2, size = n_samples)
    outname = f'GaussianSamples_uncorrelated_seed_{seed_value:d}.txt'
    np.savetxt(os.path.join(RAWDIR, outname), samples, fmt = '%.8f')


import numpy as np
from problem3 import compute_P,random_walk

#-------------------------------------------------------------------------
'''
    Problem 4: Solving sink-node problem in PageRank
    In this problem, we implement the pagerank algorithm which can solve the sink node problem.
    You could test the correctness of your code by typing `nosetests test4.py` in the terminal.
'''

#--------------------------
def compute_S(A):
    '''
        compute the transition matrix S from addjacency matrix A, which solves sink node problem by filling the all-zero columns in A.
        S[j][i] represents the probability of moving from node i to node j.
        If node i is a sink node, S[j][i] = 1/n.
        Input:
                A: adjacency matrix, a (n by n) numpy matrix of binary values. If there is a link from node i to node j, A[j][i] =1. Otherwise A[j][i]=0 if there is no link.
        Output:
                S: transition matrix, a (n by n) numpy matrix of float values.  S[j][i] represents the probability of moving from node i to node j.
    The values in each column of matrix S should sum to 1.
    '''
    #########################################
    ## INSERT YOUR CODE HERE

    shape = set(np.shape(A))
    assert len(shape) == 1
    n = shape.pop()

    P = compute_P(A)
    colsums = P.sum(axis=0)
    sink_nodes = np.asarray(np.isnan(colsums))
    sink_nodes = sink_nodes.reshape((n,))

    from copy import deepcopy
    S = deepcopy(P)

    if sink_nodes.any():
        S[:, sink_nodes] = 1/n

    #########################################
    return S

#--------------------------
def pagerank_v2(A):
    '''
        A simplified version of PageRank algorithm, which solves the sink node problem.
        Given an adjacency matrix A, compute the pagerank score of all the nodes in the network.
        Input:
                A: adjacency matrix, a numpy matrix of binary values. If there is a link from node i to node j, A[j][i] =1. Otherwise A[j][i]=0 if there is no link.
        Output:
                x: the ranking scores, a numpy vector of float values, such as np.array([[.3], [.5], [.7]])
    '''

    # Initialize the score vector with all one values
    num_nodes, _ = A.shape
    x_0 =  np.asmatrix(np.ones((num_nodes,1)))

    # compute the transition matrix from adjacency matrix
    S = compute_S(A)

    # random walk
    x, n_steps = random_walk(S,x_0)

    return x


#!/usr/local/bin/python3

import numpy as np
import matplotlib.pyplot as plt
from scipy.spatial import distance as sd

class Clusterers:
    def __init__(self, data_path):
        self._data_path = data_path
        self._points = list()
        self._clusters = list()
        self._centroids = list()
        self.load_data()

    def load_data(self):
        with open(self._data_path, 'r') as f:
            lines = f.readlines()
            for line in lines:
                point_str = line.strip("\n").split("\t")
                self._points.append([float(point_str[0]), float(point_str[1])])
            self._points = np.array(self._points)
        # print(np.average(self._points, axis=0))

    ''' ################# Following is implementation for K-means clustering ################# '''
    def k_means(self, init_centroids):
        cent = init_centroids
        # print(self.mink_dist(cent[0], cent[1], 3))
        print("=============  (a) Euclidean  ==============")
        self.k_means_train(init_centroids, self.eucl_dist)
        print("=============  (b) Cityblock  ==============")
        self.k_means_train(init_centroids, self.city_dist)
        print("=============  (c) Minkovski  ==============")
        self.k_means_train(init_centroids, self.mink_dist)
        return

    def k_means_train(self, init_cents, dist_m):
        # num_clusters = len(init_cents)
        self._centroids = list(init_cents)
        self._clusters.clear()

        loss = 1000
        epoch = 0
        while loss > 0:
            epoch += 1
            print("=============  epoch", epoch, "==============")
            pre_obj = self.k_means_obj_func(dist_m)
            # assign points
            self.print_centroids()
            self.k_means_assign(dist_m)
            # update centroids
            self.k_means_update()
            self.print_clusters()
            obj = self.k_means_obj_func(dist_m)
            print("result of pre_obj func:", pre_obj)
            print("result of obj func:", obj)
            loss = abs(pre_obj - obj)
            print("loss:", loss)
        print("amount of epoch:", epoch)

        return self._clusters

    def k_means_assign(self, dist_m):
        cents = self._centroids
        self._clusters = [np.zeros([0,2]) for i in range(len(cents))]
        # for cent in cents:
        #     self._clusters.append(np.array([cent]))
        for point in self._points:
            min_dist = 1000
            min_ind = 0
            for cent_ind in range(len(cents)):
                dist = dist_m(point, cents[cent_ind])
                if dist < min_dist:
                    min_dist = dist
                    min_ind = cent_ind
            # print(point, self._clusters[min_ind][0], min_dist)
            self._clusters[min_ind] = np.append(self._clusters[min_ind], np.array([point]), axis=0)
        return self._clusters

    def k_means_update(self):
        self._centroids.clear()
        for clst in self._clusters:
            # print(clst[1:])
            self._centroids.append(np.average(clst[:], axis = 0))
            # print(self._centroids)
        return self._centroids

    def k_means_obj_func(self, dist_m):
        # res = 0
        # for clst_i in range(len(self._clusters)):
        #     cent = self._clusters[clst_i][0]
        #     res += sum(dist_m(cent, self._clusters[clst_i][i]) for i in range(1, len(self._clusters[clst_i])))
        # return res
        return sum( sum(dist_m(self._clusters[clst_i][0], self._clusters[clst_i][i]) for i in range(1, len(self._clusters[clst_i]))) for clst_i in range(len(self._clusters)))

    ''' ################# Following is implementation for Hierarchical clustering ################# '''
    def hierarchical(self):
        self._clusters.clear()
        # print pair-wise proximity matrix for original points
        print("======================= a. initial proximity matrix & clusters =======================")
        pdist = self.hier_pdist()
        self.hier_show_prox_mat(self._points, pdist, "[Point-Point Proximities]", True, "p")
        dists = self.hier_init_clusters()
        self.print_clusters()
        self._clusters.clear()

        # complete linkage
        print("======================= b. complete linkage  =======================")
        dists = self.hier_init_clusters(self.hier_dist_complete)
        self.hier_show_prox_mat(self._clusters, dists, "[Cluster-Cluster Proximities]", True)
        while len(self._clusters)>1:
            dists = self.hier_update(dists, self.hier_dist_complete)
            self.hier_show_prox_mat(self._clusters, dists, "[Cluster-Cluster Proximities]", True)
        self._clusters.clear()

        # single linkage
        print("======================= c. single linkage  =======================")
        dists = self.hier_init_clusters(self.hier_dist_single)
        self.hier_show_prox_mat(self._clusters, dists, "[Cluster-Cluster Proximities]", True)
        while len(self._clusters)>1:
            dists = self.hier_update(dists, self.hier_dist_single)
            self.hier_show_prox_mat(self._clusters, dists, "[Cluster-Cluster Proximities]", True)
        self._clusters.clear()

        # group average
        print("======================= d. group average =======================")
        dists = self.hier_init_clusters(self.hier_dist_group)
        self.hier_show_prox_mat(self._clusters, dists, "[Cluster-Cluster Proximities]", True)
        while len(self._clusters)>1:
            dists = self.hier_update(dists, self.hier_dist_group)
            self.hier_show_prox_mat(self._clusters, dists, "[Cluster-Cluster Proximities]", True)
        self._clusters.clear()

        return

    def hier_init_clusters(self, dist_m = None):
        # initialize 5 separations (0-3, 4-7, 8-11, 12-15, 16-19)
        # self._clusters = [np.zeros([0,2]) for i in range(5)]
        # for clst_i in range(len(self._clusters)):
        #     for p_i in range(clst_i * 4, clst_i * 4 + 4):
        #         self._clusters[clst_i] = np.append(self._clusters[clst_i], np.array([self._points[p_i]]), axis=0)
        #         # print(np.array([self._points[p_i]]))

        # initialize 5 separations using single-linkage
        for point in self._points:
            self._clusters.append(np.array([point]))
        pdist = self.hier_pdist()
        dists = self.hier_update(pdist, self.hier_dist_single)
        while len(self._clusters) != 5:
            dists = self.hier_update(dists, self.hier_dist_single)
        if dist_m:
            # dists = dist_m(pdist)
            dists = dist_m()
            return dists
        return

    def hier_update(self, dists, merge_m):
        self.hier_merge(dists)
        # return merge_m(dists)
        return merge_m()

    def hier_merge(self, dists):
        # merge clusters based on a certain linkage method
        min_ind = np.argmin(dists)
        # self.print_clusters()
        num_clst = len(self._clusters)
        for clst_1 in range(num_clst-1):
            for clst_2 in range(clst_1+1, num_clst):
                if min_ind == self.hier_rcl(clst_1, clst_2, num_clst):
                    print("[Notes] 'c%02d'" % clst_2, "will be merged into 'c%02d'" % clst_1)
                    # found the closest cluster pair
                    # print(clst_1, clst_2)
                    clusters = list()
                    for clst_i in range(num_clst):
                        if clst_i == clst_2:
                            clusters[clst_1] = np.append(clusters[clst_1], self._clusters[clst_2], axis=0)
                            continue
                        clusters.append(self._clusters[clst_i])
                    self._clusters = clusters
                    break
        # self.print_clusters()
        return self._clusters

    def hier_pdist(self):
        return sd.pdist(self._points)

    # def hier_dist_single(self, pdist):
    def hier_dist_single(self):
        clusters = self._clusters
        dists = np.zeros([0])
        for clst_1 in range(len(clusters)-1):
            for clst_2 in range(clst_1 + 1, len(clusters)):
                min_dist = np.min(sd.cdist(clusters[clst_1], clusters[clst_2]))
                dists = np.append(dists, np.array([min_dist]))
                # print(clst_1, clst_2, min_dist)
        return dists

    # def hier_dist_complete(self, pdist):
    def hier_dist_complete(self):
        clusters = self._clusters
        dists = np.zeros([0])
        for clst_1 in range(len(clusters)-1):
            for clst_2 in range(clst_1 + 1, len(clusters)):
                max_dist = np.max(sd.cdist(clusters[clst_1], clusters[clst_2]))
                dists = np.append(dists, np.array([max_dist]))
        return dists

    # def hier_dist_group(self, pdist):
    def hier_dist_group(self):
        clusters = self._clusters
        dists = np.zeros([0])
        for clst_1 in range(len(clusters)-1):
            for clst_2 in range(clst_1 + 1, len(clusters)):
                avg_dist = np.average(sd.cdist(clusters[clst_1], clusters[clst_2]))
                dists = np.append(dists, np.array([avg_dist]))
        return dists

    def hier_show_prox_mat(self, clusters, pdist, title, if_print = False, label = 'c'):
        if if_print:
            num_p = len(clusters)
            # print table header
            print(title)
            if label == "p":
                print("             ", end="")
            else:
                print("    ", end="")
            for i in range(num_p):
                print(" %s%02d " % (label, i), end="")
            print()
            # print matrix
            row = 0
            for i in sd.squareform(pdist):
                col = 0
                if label == "p":
                    print("%s%02d[%.1f,%.1f] " % (label, row, clusters[row][0], clusters[row][1]), end="")
                else:
                    print("%s%02d " % (label, row), end="")
                for j in i:
                    if row == col:
                        print("0.00 ", end="")
                    else:
                        print("%.2f " % pdist[self.hier_rcl(row,col,num_p)], end="")
                        # if j == pdist[self.hier_rcl(row,col,num_p)]:
                        #     print('true ', end="")
                    col += 1
                row += 1
                print()
        return pdist

    # get dist from condensed pdist, based on #row, #col and len
    def hier_rcl(self, i, j, l):
        if i < j:
            res = (i*(l-2)+j-i*(i-1)/2-1)
        elif i > j:
            res = (j*(l-2)+i-j*(j-1)/2-1)
        # print(res)
        return int(res)

    ''' ################# Following is implementation for SOM clustering ################# '''
    def som(self, init_centroids, a = 0.3, a_nb = 0.2, size_nb = 3, loss_threshold = 1e-16, learning_rate_threshold = 1e-10):
        # self.display_points(init_centroids)

        self._clusters.clear()
        centroids = init_centroids
        print("[b] before training:\n", centroids)
        epoch = 0
        while True:
            epoch += 1
            pre_centroids = np.array(centroids)
            centroids = self.som_train(self._points, centroids, a, a_nb, size_nb, epoch)
            loss = np.linalg.norm(pre_centroids - centroids)
            print("loss: ", loss)
            if loss <= loss_threshold:
                break
        print("after training:\n", centroids, "\nround: ", epoch)
        self._clusters = self.som_assign(self._points, centroids)
        self.print_clusters()

        print("======================================")

        centroids = init_centroids
        print("[c] before training:\n", centroids)
        epoch = 0
        pre_loss = 0
        while True:
            epoch += 1
            pre_centroids = np.array(centroids)
            # TODO: epoch from 0 or 1?
            a -= 0.02
            a_nb -= 0.02
            centroids = self.som_train(self._points, centroids, a, a_nb, size_nb, epoch)
            loss = np.linalg.norm(pre_centroids - centroids)
            pre_loss = loss
            print("loss: ", loss)
            if loss <= loss_threshold or a <= learning_rate_threshold or a_nb <= learning_rate_threshold:
                break
        print("after training:\n", centroids, "\nround: ", epoch, ", a=", a, ", a_nb=", a_nb)
        self._clusters = self.som_assign(self._points, centroids)
        self.print_clusters()

        return self._clusters

    def som_train(self, examples, centroids, a, a_nb, size_nb, cur_round):
        # print("round: ", cur_round)
        for example in examples:
            # select winner centroid
            min_dist = 1000
            closest_cent = 0
            for ind in range(len(centroids)):
                dist = self.eucl_dist(centroids[ind], example)
                if dist < min_dist:
                    min_dist = dist
                    closest_cent = ind
            # print(closest_cent, centroids[closest_cent], min_dist)
            # look for winner's neighbours
            nb_inds = list()
            for nb_i in range(len(centroids)):
                if nb_i != closest_cent:
                    nb_inds.append(nb_i)
            # update winner
            centroids[closest_cent] += a * ( example - centroids[closest_cent] )
            # update winner's neighbours
            for i in range(size_nb):
                centroids[nb_inds[i]] += a_nb * (example - centroids[nb_inds[i]])
        return centroids

    def som_assign(self, examples, centroids):
        self._clusters = list()
        # create clusters
        for ind in range(len(centroids)):
            self._clusters.append(np.array([centroids[ind]]))
        for example in examples:
            min_dist = 1000
            closest_cent = 0
            for ind in range(len(centroids)):
                dist = self.eucl_dist(centroids[ind], example)
                if dist < min_dist:
                    min_dist = dist
                    closest_cent = ind
            # print(closest_cent, min_dist)
            self._clusters[closest_cent] = np.append(self._clusters[closest_cent], np.array([example]), axis=0)
        return self._clusters

    # a special case of minkovski-distance with 2 as lambda
    def eucl_dist(self, vec1, vec2):
        # return self.mink_dist(vec1, vec2, 2)
        return np.linalg.norm(vec1 - vec2)

    # a special case of minkovski-distance with 1 as lambda
    def city_dist(self, vec1, vec2):
        # return self.mink_dist(vec1, vec2, 1)
        return sum(abs(vec1[dim] - vec2[dim]) for dim in range(len(vec1)))

    def mink_dist(self, vec1, vec2, lmd = 3):
        return (sum((abs(vec1[dim] - vec2[dim]) ** lmd) for dim in range(len(vec1))) ** (1 / lmd))

    def display_points(self, points):
        plt.plot(points)
        plt.show()

    def display_clusters(self):
        # plot self._clusters
        return self._clusters

    def print_clusters(self):
        cluster_num = 0
        for cluster in self._clusters:
            cluster_num += 1
            print("cluster", cluster_num)
            print(cluster)

    def print_centroids(self):
        cent_num = 0
        for cent in self._centroids:
            cent_num += 1
            print("centroid", cent_num, ":", cent)

def main():
    # configuration
    data_path = "./data_points"
    # create my clusterers
    my_clusterers = Clusterers(data_path)

    # run k-means
    init_centroids_kmeans = np.array([[1.8, 2.3], [2.3, 1.4]])
    lambda_kmeans_mink = 3
    my_clusterers.k_means(init_centroids_kmeans)

    # run hierarchical
    my_clusterers.hierarchical()

    # run som
    init_centroids_som = np.array([[1, 3.1], [2, 2.2], [1.5, 2.1], [3.1, 1.1]])
    my_clusterers.som(init_centroids_som)

if __name__ == "__main__":
    main()


import numpy as np
import matplotlib.pylab as plt

def funcion(x):
    return x**5 - 2*(x**4) - 10*(x**3) + 20*(x**2) + 9*x - 18

xx=np.linspace(-3.5,3.5,100)

plt.figure()
plt.plot(xx,funcion(xx))
plt.xlabel('X')
plt.ylabel('Y')
plt.title('Gráfica de la función')
plt.grid()
plt.savefig('NRpoli.pdf')
plt.close()

def NR(x,funcion,precision=1*(10**-5)):
    N=0
    dx=3**-1
    for i in range(10000):
        F=funcion(x)
        if(abs(F)<precision):
            break
        df= (funcion(x+(dx/2))-funcion(x-(dx/2)))/dx
        dx=-F/df
        x+=dx
        N+=1
    return x,N

# x_guess = -3
print('')

xmenos3,Nmenos3=NR(-3,funcion)

print('Con un x_guess inicial de -3 el valor de la raíz X0r da: ',xmenos3,'y el valor de f(X0r) da: ',funcion(xmenos3))

# x_guess = -1
print('')

xmenos1,Nmenos1=NR(-1,funcion)

print('Con un x_guess inicial de -1 el valor de la raíz X1r da: ',xmenos1,'y el valor de f(X1r) da: ',funcion(xmenos1))

# Número de iteraciones

xx2=np.linspace(-4,4,1000)
Ns=np.zeros(len(xx2))
raices=np.zeros(len(xx2))

for i in range(len(xx2)):
    raiz,N=NR(xx2[i],funcion,precision=1*(10**-10))
    Ns[i]=N
    raices[i]=raiz

plt.figure()
plt.scatter(xx2,Ns,s=15)
plt.xlabel('X_guess')
plt.ylabel('No. Iteraciones')
plt.title('Número de iteraciones')
plt.grid()
plt.savefig('NR_itera.pdf')
plt.close()

plt.figure()
plt.scatter(xx2,raices,s=15)
plt.xlabel('X_guess')
plt.ylabel('Raíz')
plt.title('Raíces encontradas')
plt.grid()
plt.savefig('NRxguess.pdf')
plt.close()

print('')
print('Se puede observar claramente que los puntos de X_guess en los que el método necesita una mayor cantidad de iteraciones (Gráfica NR_itera.pdf) está correlacionado con las partes de la función (Gráfica NRpoli.pdf) en las que su derivada es cercana a cero. Esto se debe a que el método busca puntos en el eje x que se crucen con la pendiente de la funcion de x0, lo que significa que esos nuevos puntos de x serán muy lejanos del x0 y el método necesitará mucho más tiempo para encontrar una raíz en la que la función de 0.')
print('')
print('Además, se puede observar que también hay una correlación con los tramos de X_guess (Gráfica NRxguess.pdf) en los que la raiz encontrada es variable. Esto se da por la misma razón de que la distancia entre nuevos puntos de x y el x anterior es muy grande y los ceros encontrados son de una parte lejana de la función.')


# %%

#!/anaconda3/bin/python

import matplotlib.pyplot as plt
import pandas as pd
import numpy as np
from mpl_toolkits.mplot3d import Axes3D  # for surface plot

# %%
def featureNormaliz(X):
    X_norm = X
    mu = np.mean(X, axis=0)
    sigma = np.std(X, axis=0, ddof=1)  # ddof=1 除法使用N-1
    X_norm = (X - mu) / sigma
    return X_norm, mu, sigma

def gradientDescentMulti(X, y, theta, alpha, num_iters):
    m = len(y)
    n = np.size(X, axis=1)
    J_history = np.zeros([num_iters, 1])
    for iter in np.arange(1, num_iters + 1):
        delta = 1 / m * (X.T.dot(X).dot(theta) - X.T.dot(y).reshape(n, 1))
        theta = theta - alpha * delta
        J_history[iter - 1] = computeCostMulti(X, y, theta)
    return theta, J_history

def computeCostMulti(X, y, theta):
    m = len(y)
    J = sum((X.dot(theta) - y.reshape(m, 1))**2) / 2 / m
    return J

def normalEqn(X,y):
    theta = np.linalg.pinv(X.T.dot(X)).dot(X.T).dot(y) # pinv(X' * X) * X' * y
    return theta

data = pd.read_csv('ex1data2.txt', header=-1)
data = np.asarray(data)
X = data[:, (0, 1)]
y = data[:, 2]
m = len(y)

#   %% ================ Part 1: Feature Normalization ================

#   对所有特征做归一化

X, mu, sigma = featureNormaliz(X)

#   在第一行添加1，用于thete0的计算
X = np.append(np.ones((m, 1)), X, axis=1)

#   %% ================ Part 2: Gradient Descent ================
alpha = 0.1
num_iters = 50
theta = np.zeros((3, 1))  #  如果要扩展至任意维度也是可以的，这里已经有提示了n

theta, J_history0_01 = gradientDescentMulti(X, y, theta, alpha, num_iters)

alpha = 0.3
num_iters = 50
theta = np.zeros((3, 1))  #  如果要扩展至任意维度也是可以的，这里已经有提示了n

theta, J_history0_03 = gradientDescentMulti(X, y, theta, alpha, num_iters)

alpha =  1
num_iters = 50

theta = np.zeros((3, 1))
theta, J_history0_1 = gradientDescentMulti(X, y, theta, alpha, num_iters)

fig, ax = plt.subplots()

ax.plot(J_history0_01, 'r-')
ax.hold
ax.plot(J_history0_03, 'g-')
ax.hold
ax.plot(J_history0_1, 'b-')

ax.set(xlabel='Number of iterations', ylabel='Cost J')
plt.legend(('alpha = 0.01','alpha = 0.03','alpha = 0.1'))

plt.show()

# %% ================ Part 3: Normal Equations ================

data = pd.read_csv('ex1data2.txt', header=-1)
data = np.asarray(data)
X = data[:, (0, 1)]
y = data[:, 2]
m = len(y)

X = np.append(np.ones((m, 1)), X, axis=1)

theta = normalEqn(X,y)

print(theta)

# %%


# solutions.py
"""Volume 1A: Linear Systems. Solutions file."""

import numpy as np
from time import time
from scipy import sparse
from scipy import linalg as la
from scipy.sparse import linalg as spla
from matplotlib import pyplot as plt

# Problem 1
def ref(A):
    """Reduce the square matrix A to REF. You may assume that A is invertible
    and that a 0 will never appear on the main diagonal. Avoid operating on
    entries that you know will be 0 before and after a row operation.
    """
    A = np.array(A, dtype=np.float, copy=True)
    m,n = A.shape
    for j in xrange(n):
        for i in xrange(j+1, m):
            A[i,j:] -= A[j,j:] * A[i,j] / A[j,j]
    return A

# Problem 2
def lu(A):
    """Compute the LU decomposition of the square matrix A. You may assume the
    decomposition exists and requires no row swaps.

    Returns:
        L ((n,n) ndarray): The lower-triangular part of the decomposition.
        U ((n,n) ndarray): The upper-triangular part of the decomposition.
    """
    m, n = A.shape
    U = np.array(A, dtype=np.float, copy=True)
    L = np.eye(n)
    for j in xrange(n):
        for i in xrange(j+1, m):
            L[i,j] = U[i,j]/U[j,j]
            U[i,j:] -= L[i,j]*U[j,j:]
    return L,U

# Problem 3
def solve(A, b):
    """Use the LU decomposition and back substitution to solve the linear
    system Ax = b. You may assume that A is invertible (hence square).
    """
    m, n = A.shape
    L, U = lu(A)

    # First solve Ly = Pb (assume P = I).
    y = np.zeros(n)
    for k in xrange(n):
        y[k] = b[k] - np.dot(L[k,:k], y[:k])

    # Now solve Ux = y.
    x = np.zeros(n)
    for k in reversed(xrange(n)):
        x[k] = (y[k] - np.dot(U[k,k:], x[k:])) / U[k,k]

    return x

# Problem 4
def prob4(N=11):
    """Time different scipy.linalg functions for solving square linear systems.
    Plot the system size versus the execution times. Use log scales if needed.
    """
    domain = 2**np.arange(1,N+1)
    inv, solve, lu_factor, lu_solve = [], [], [], []

    for n in domain:
        A = np.random.random((n,n))
        b = np.random.random(n)

        start = time()
        la.inv(A).dot(b)
        inv.append(time()-start)

        start = time()
        la.solve(A, b)
        solve.append(time()-start)

        start = time()
        x = la.lu_factor(A)
        la.lu_solve(x, b)
        lu_factor.append(time()-start)

        start = time()
        la.lu_solve(x, b)
        lu_solve.append(time()-start)

    plt.subplot(121)
    plt.plot(domain, inv, '.-', lw=2, label="la.inv()")
    plt.plot(domain, solve, '.-', lw=2, label="la.solve()")
    plt.plot(domain, lu_factor, '.-', lw=2,
                                    label="la.lu_factor() and la.lu_solve()")
    plt.plot(domain, lu_solve, '.-', lw=2, label="la.lu_solve() alone")
    plt.xlabel("n"); plt.ylabel("Seconds")
    plt.legend(loc="upper left")

    plt.subplot(122)
    plt.loglog(domain, inv, '.-', basex=2, basey=2, lw=2)
    plt.loglog(domain, solve, '.-', basex=2, basey=2, lw=2)
    plt.loglog(domain, lu_factor, '.-', basex=2, basey=2, lw=2)
    plt.loglog(domain, lu_solve, '.-', basex=2, basey=2, lw=2)
    plt.xlabel("n")

    plt.suptitle("Problem 4 Solution")
    plt.show()

# Problem 5
def prob5(n):
    """Return a sparse n x n tridiagonal matrix with 2's along the main
    diagonal and -1's along the first sub- and super-diagonals.
    """
    return sparse.diags([2,-1,2], [-1,0,1], shape=(n,n))

# Problem 6
def prob6(N=10):
    """Time regular and sparse linear system solvers. Plot the system size
    versus the execution times. As always, use log scales where appropriate.
    """
    domain = 2**np.arange(2,N+1)
    solve, spsolve = [], []

    for n in domain:
        A = prob5(n).tocsr()
        b = np.random.random(n)

        start = time()
        spla.spsolve(A, b)
        spsolve.append(time()-start)

        A = A.toarray()
        start = time()
        la.solve(A, b)
        solve.append(time()-start)

    plt.subplot(121)
    plt.plot(domain, spsolve, '.-', lw=2, label="spla.spsolve()")
    plt.plot(domain, solve, '.-', lw=2, label="la.solve()")
    plt.xlabel("n"); plt.ylabel("Seconds")
    plt.legend(loc="upper left")

    plt.subplot(122)
    plt.loglog(domain, spsolve, '.-', basex=2, basey=2, lw=2)
    plt.loglog(domain, solve, '.-', basex=2, basey=2, lw=2)
    plt.xlabel("n")

    plt.suptitle("Problem 6 Solution")
    plt.show()

# Additional Material =========================================================

def ref_fast(A):
    """Alternate REF using an outer product. Fast, but not very intuitive."""
    for i in xrange(A.shape[0]):
        A[i+1:,i:] -= np.outer(A[i+1:,i]/A[i,i], A[i,i:])

def lu2_fast(A):
    """Alternative LU decomposition using an outer product."""
    U = A.copy()
    L = np.eye(A.shape[0])
    for i in xrange(A.shape[0]-1):
        L[i+1:,i] = U[i+1:,i] / U[i,i]
        U[i+1:,i:] -= np.outer(L[i+1:,i], U[i,i:])
    return L, U

def lu_inplace(A):
    """Compute the LU decomposition of the square matrix A *IN PLACE*."""
    for j in xrange(A.shape[0]-1):
        for i in xrange(j+1, A.shape[0]):
            A[i,j] /= A[j,j]
            A[i,j+1:] -= A[i,j] * A[j,j+1:]

def lu_det(A):
    """Compute det(A) using the LU decomposition (via la.lu_factor())."""
    lu, piv = la.lu_factor(A)

    # Determine if there were an even or odd number of row swaps.
    s = (piv != np.arange(A.shape[0])).sum() % 2
    return ((-1)**s) * lu.diagonal().prod()

def cholesky(A):
    L = np.zeros_like(A)
    for i in xrange(A.shape[0]):
        for j in xrange(i):
            L[i,j]=(A[i,j] - np.inner(L[i,:j], L[j,:j])) / L[j,j]
        sl = L[i,:i]
        L[i,i] = sqrt(A[i,i] - np.inner(sl, sl))
    return L

def cholesky_inplace(A):
    for i in xrange(A.shape[0]):
        A[i,i+1:] = 0.
        for j in range(i):
            A[i,j] = (A[i,j] - np.inner(A[i,:j],A[j,:j])) / A[j,j]
        sl = A[i,:i]
        A[i,i] = sqrt(A[i,i] - np.inner(sl, sl))

def cholesky_solve(A, B):
    for j in xrange(A.shape[0]):
        B[j] /= A[j,j]
        for i in xrange(j+1, A.shape[0]):
            B[i] -= A[i,j] * B[j]
    for j in xrange(A.shape[0]-1, -1, -1):
        B[j] /= A[j,j]
        for i in xrange(j):
            B[i] -= A[j,i] * B[j]


from numpy import *
from scipy import *
from pylab import *

import numpy.random as random

import pdb

ion()

i = 1j

def my_slow_fft(f):
    # a slow ifft that **CAN'T** interpolate!
    N = len(f)
    F = zeros(N, dtype=complex)
    for k in range(N):
        for n in range(N):
            F[k] += f[n]*e**(-2.0*pi*i*k*n/N)
    return F

def my_slow_ifft(F):
    # a slow ifft that **CAN'T** interpolate!
    N = len(F)
    f = zeros(N, dtype=complex)
    for n in range(N):
        for k in range(N):
            f[n] += F[k]*e**(2.0*pi*i*k*n/N)
    return (1.0/N)*f

def my_slow_ifft_interp(F,t):
    # a slow ifft that **CAN'T** interpolate!
    N = len(F)
    f = zeros(len(t), dtype=complex)
    for n in range(len(t)):
        for k in range(N):
            f[n] += F[k]*e**(2.0*pi*i*k*t[n])
    return (1.0/N)*f

def my_correct_slow_ifft_interp(F,t):
    # an ifft that **CAN** interpolate
    N = len(F)
    f = zeros(len(t), dtype=complex)
    for n in range(len(t)):
        for k in range(int(N/2)):
            f[n] += F[k]*e**(2.0*pi*i*k*t[n])
        for k in range(int(N/2),N):
            f[n] += F[k]*e**(-2.0*pi*i*(N-k)*t[n])
    return (1.0/N)*f

if __name__ == "__main__":
    # Sample measurements
    ts = linspace(0,1,num=21)
    f = zeros(len(ts))
    hf = int(len(ts)/2)
    f[0:hf] = ts[0:hf]
    #f[hf:] = e**-ts[hf:] + 0.5 - e**-0.5
    f[hf:] = 1.0/ts[hf:] - 1.5
    #f = ts*(ts-1)
    #f = sin(ts*(2.*pi)) + cos(ts*(2.*pi))#-1.0*ts**2#exp(ts)

    F = fftshift(fft(f))
    F_unshifted = fft(f)
    ff = fftshift(ifftshift(f))

    # plot where we are so far
    # this includes F^-1{F(f)} for now
    figure(1)
    plot(ts,f, 'bo-',markerfacecolor='none', mec='blue')
    #ylim(-0.6,0.6)
    title("Original Function")

    figure(2)
    plot(ts,F.real,ts,F.imag)
    legend(('real', 'imaginary'))
    title("Shifted Fourier Transform")

    figure(22)
    plot(ts,F_unshifted.real,ts,F_unshifted.imag)
    legend(('real', 'imaginary'))
    title("Unshifted Fourier Transform")

    figure(3)
    plot(ts,ff)
    #ylim(-0.6,0.6)
    title("Getting Back The Original")

    # now interpolate
    # new timestamps, m samples between former samples.
    Nf = len(ts)

    t = linspace(0,1,num=len(ts)*3)

    Fs = my_slow_fft(f)
    fm = my_slow_ifft(Fs)

    # using the Fourier Coefficients "sas-is"
    fwi = my_slow_ifft_interp(F_unshifted,t)
    fci = my_correct_slow_ifft_interp(F_unshifted,t)

    figure(4)
    plot(ts,Fs.real,ts,Fs.imag)
    legend(('real', 'imaginary'))
    title("My Unshifted Fourier Transform")

    figure(5)
    plot(ts,f, 'bo-',markerfacecolor='none', mec='blue')
    plot(ts,fm,'g^-',markerfacecolor='none', mec='green')
    #ylim(-0.6,0.6)
    title("Original Using My Inverse")

    figure(500)
    plot(ts,f, 'bo-',markerfacecolor='none', mec='blue')
    plot(ts[:-1],my_correct_slow_ifft_interp(fft(f[:-1]),ts[:-1]),'g^-',markerfacecolor='none', mec='green')
    #ylim(-0.6,0.6)
    title("Original Using My Inverse Interp ")

    figure(6)
    plot(ts,f, 'bo-',markerfacecolor='none', mec='blue')
    plot(t,fwi,'go-',markerfacecolor='none', mec='green')
    #ylim(-0.6,0.6)
    legend(('original', 'interpolatation'))
    title("Interpolating with Fourier Coefficients, No Change")

    figure(7)
    plot(ts,f,'bo-',markerfacecolor='none', mec='blue')
    plot(t[:-1],my_correct_slow_ifft_interp(fft(f[:-1]),t[:-1]),'g^-',markerfacecolor='none', mec='green')
    #ylim(-0.6,0.6)
    legend(('original', 'interpolatation'))
    title("Interpolating with Fourier Coefficients Accounting for Aliasing")

    show()