"""
This code shows an implementation of polinomial logistic regression, normalization,
automatic alpha tuning, convergence of training algorithm, regularization to avoid
overfitting.

We changed from np.matrix to np.array to be faster and more flexible, so
instead of using * as matrix dot product now we need to use @
"""
polyorder=6 # here you decide what polinomial order your system will have
lambd = 0.001 # that's the regularization term, higher values make the boundary shape "rounder" (bias), lower values make it "crispier" (overfitting). Reasonable values to try here would be [30, 10, 3, 1, 0.3, 0.1, 0.03, 0.01, ..., 0]
example=2 # here you choose the example to be used

# suppress the warnings when logarithm calculates infinity
import warnings
warnings.simplefilter("ignore", RuntimeWarning)

import numpy as np
print('Welcome to Machine Learning with Python!')
print('Lesson 3: Polinomial Logistic regression')
print('\n'+40*'=')

print('The task now is to create a model which would separate 2 classes of objects based on 2 inputs.\n')
print('But a simple line is not able to perform a good separation because the ', end='')
print('classes are mixed into each other')
# data representing the measurements and category 1 and 2 (or 'o' and 'x')
data1 =\
[(10, 12, 1), (15, 14, 1), (6, 18, 1), (12, 10, 1),\
(17, 13, 1), (18, 18, 1), (11, 15, 1), (3, 17, 1),\
(5, 15, 1), (16, 9, 2), (16, 16, 1), (14, 7, 2),\
(22, 18, 1), (25, 19, 1), (27, 17, 1), (26, 10, 1),\
(18, 12, 1), (24, 7, 1), (23, 12, 1), (28, 9, 1),\
(7, 2, 2), (4, 12, 2), (2, 3, 2), (14, 1, 2),\
(7, 5, 2), (20, 1, 1), (2, 9, 2), (10, 4, 2),\
(2, 7, 2), (4, 1, 2), (13, 2, 2), (10, 1, 2),\
(4, 5, 2), (17, 3, 2), (3, 13, 2), (25, 3, 1),\
(5, 3, 2), (14, 4, 2), (23, 2, 1), (17, 1, 2)]

# second data set so we can choose which example to use
data2 =\
[(15, 10, 1), (7, 9, 1), (10, 12, 2), (18, 8, 1),\
(19, 11, 1), (20, 13, 1), (15, 7, 1), (13, 17, 1), (18, 18, 1),\
(3, 9, 2), (28, 6, 2), (27, 12, 2), (5, 5, 2),\
(10, 4, 2), (26, 4, 2), (8, 16, 2), (22, 16, 2)]

if example==1: data = data1
else: data = data2

dataX1 = np.array(data, dtype=np.float64)[:,0] #float64 is necessary to avoid "wrap around" phenomena in higher orders (>7)
dataX2 = np.array(data, dtype=np.float64)[:,1]
y = 2 - np.array(data, dtype=np.float64)[:,2][np.newaxis].T #is float64 so everything is in the same basis and calculations are faster

'''
Extending the Logistic regression model, we want to build a high order
polinomial model able to separate 2 classes of objects based on two inputs

X matrix will hold the inputs - upscaled to a polinomial degree
X has each data point as a row, and has m rows.
The columns are [ constant, X2, X1, X1*X2, X2^2, X1^2] for order 2
[cte, X2, X1, X1*X2, X1*X2^2, X2^2, X1^2, X1^2*X2, X2^3, X1^3 ] for order 3
and so on.

The higher the order the more complex can the classification boundary be, but
computational effort also sky rocks.
'''
def polyX(dataX1, dataX2, pord):
	pord+=1
	X = np.ones((dataX1.shape[0], round(pord/2*(pord+1))), dtype=np.float64)
	k=1
	for i in range(1,pord):
		X[:,k]=dataX2**i
		k+=1
		for j in range(pord-i):
			X[:,k]=(dataX1**i*dataX2**j)
			k+=1
	return X

def normalize(X):
	''' the normalizer puts all X inputs in about the same scale, so
			gradient descent can work better. Only the first collumn
			(the constant) remains at 1'''
	reg=np.ones((2, X.shape[1]))
	reg[0,:]=X.mean(axis=0)
	reg[1,:]=X.std(axis=0)
	reg[0,0]=0
	reg[1,0]=1
	return (X-reg[0])/reg[1], reg # return the adjustment also, so we can readjust further inputs when predicting a value

X = polyX(dataX1, dataX2, polyorder)

print('So we need to choose our degree of polinomial extension, in this case it is {}'.format(polyorder))
print('this means our new X input that had a size of 2 (or 3 with the constant) now has {} elements for each example\n'.format(X.shape[1]))
print('X[0] (for the first point) = '+np.array_str(X[0], precision=0, suppress_small=True))
print('as inputs numbers get orders of magnitute to each other (3 compared to 6546778) a normalization of the input data is necessary. This can be done using average and standard deviation, so every new X has an average around zero and a span of about +/- 3')
[X, Xnor] = normalize(X)
print('X[0] after normalization   = '+np.array_str(X[0], precision=3, suppress_small=True))

'''
Below is the show() function, which will use the ASCII to graphically "plot" the charts
'''

def show(arr, p):
	for j in range(np.size(arr,1)-1, -1, -1):
		print('{:>2}'.format(j if j%5 == 0 else ' ')+' '+chr(179+(j%5==0)), end='')
		for i in range(np.size(arr,0)):
			print(p[arr[i][j]], end='')
		print()
	print('   '+chr(192)+(chr(194)+chr(196)*4)*(np.size(arr,0)//5))
	print('	', end='')
	print(*list(('{:<4}'.format(i*5) for i in range(np.size(arr,0)//5))))
	print()

# t and p are the plot variables
t = np.zeros((30,20), np.int8)
p = [' ', '#', 'o', 'x']

def sigmoid(z):
	return 1 / (1 + np.exp(-z))

def J(X, y, theta, lambd):
	# X is m samples x j features, y is m samples x k outputs, theta is j features x k outputs
	# all three arguments must be given as np.array already in the right shape
	m = y.shape[0] # m is the number of datapoints
	predictions = sigmoid(X@theta) # stores the sigmoid predictions, translated into "binary" classes
	l0 = np.log(predictions)
	l1 = np.log(1-predictions)
	l0[l0==-np.inf] = -100
	l0[l0==np.inf] = -100
	l1[l1==-np.inf] = -100
	l1[l1==np.inf] = -100
	# here we included lambda*sum(theta) as the regularization term,
		# that penalizes the error with increasing values of theta.
	return 1/m * (((-y).T@l0) - ((1-y).T@l1) + (lambd/2*np.sum(theta[1:,:])))[0,0], 1/m * (X.T @ (predictions-y)) # the second parameter is the gradient

'''
The optimal theta vector will be needed for our model to predict with the smallest possible error.
Assuming that J is a cost function, this is again an optimization problem - we need to find the minimum of J.
The gradient descent function looks somewhat as the logistic regression.
This time, however, we have automatic alpha, regularization (to keep theta values
low and avoid overfitting) and some calculation enhancements so the code runs faster.
'''

# gradient descent function will iteratively update theta by a small fraction alpha (also called the learning rate) for a number of iterations
def gradient(X, y, theta, iters, lambd):
	iters+=1
	m = len(y) # m is the number of datapoints
	alpha = 0.01 # initial learning rate, will be adjusted further in the loop
	J_history = np.zeros(iters, dtype=np.float64) # will store historical values of J for each iteration
	reg = np.ones((len(theta),1), dtype=np.float64)
	[J_history[0], delta] = J(X, y, theta, lambd) # get the first data set
	for i in range(1,iters):
		oldtheta = theta # remember last values
		olddelta = delta
		reg.fill(1-alpha*lambd/m) # this is the regularization term
		reg[0] = 1  # no regularization for the constant
		theta = (theta * reg) - (alpha * delta) # update theta by learning rate times gradient
		[J_history[i], delta] = J(X, y, theta, lambd) # save the cost J of a particular iteration and get the slopes for adjusting in the next loop
		if J_history[i] <= J_history[i-1]: #  It should drop in the next iteration, if not alpha is too big and must be lowered
			alpha *= 1.05 # increase alpha slowly to get a faster optimum
		else:
			alpha *= 0.3 # decrease alpha rapidly, we tried to be too fast and J increased
			theta = oldtheta # restore the old values for gradient descend track from there
			delta = olddelta
			J_history[i]=J_history[i-1]
			if (alpha*np.max(np.abs(delta/theta))<5e-4): # local minimum has been reached because no significant change will happend in the next iteration - increase the number if "Time limit exceeded"
				print('converged after {} loops.'.format(i), end='')
				break
	return J_history, theta # return the history of cost J (for debugging) plus the optimal theta

print('\n'+40*'=')
print('Using a higher order polynomial input can lead to overfitting, that might reflect negatively in the model prediction ability with new unknowen data. To try to avoid this we introduce a regularization term Lambda that tries to keep theta values low. This has an "rounding" effect on the decision boundary')
theta = np.zeros((X.shape[1],1), dtype=np.float64)
iters = 50000 # number of maximum iterations - reduce if "Time limit exceeded"
print('\n== Model summary ==\nMax. iterations: {}\nLambda: {}\nInitial theta:\n{}\nInitial J: {:.12f}\n'.format(iters, lambd, theta.T, J(X,y,theta,lambd)[0].item()))

print('Training the model... ', end='')
J_history, theta_min = gradient(X, y, theta, iters, lambd)
print('Done.')

print('\nFinal theta:\n'+np.array_str(theta_min.T, precision=3, suppress_small=True)+'\nFinal J: {:.12f}\n'.format(J(X,y,theta_min,lambd)[0].item()))

'''
Now that we have the model trained, let's check if it works. We will plot the chart with the decision boundary calculated by the model and on top of that, we will plot the real datapoints (our training set). To do that, we will iteratively cast a 'predict' function (defined below) on every possible measurement pair.
'''

def predict(a,b,nor,porder,theta_):
	a=np.array([[a]], dtype=np.float64)
	b=np.array([[b]], dtype=np.float64)
	Xpred=(polyX(a,b,porder)-nor[0])/nor[1]#here we are normalizing the input using the same values as in the beginning
	return np.round(sigmoid(Xpred@theta_))
# round does the differenciation that if h0>0,5 then prediction is 1, otherwise it is 0

print('\n'+40*'=')

# db will store the chart for displaying the decision boundary
db = np.zeros((30,20), dtype=np.int8)
for j in range(np.size(db, 1)):
	for i in range(np.size(db, 0)):
		db[i][j] = predict(i,j,Xnor,polyorder,theta_min)[0,0]

# Populating the plot array from the measurement data
for i in data:
	db[i[0]][i[1]] = i[2]+1

# let's show it now
print('\nDoing predictions...\nEverything in the dotted area will be predicted by the model as a "o" element, while datapoints belonging to the blank area will be predicted as "x" ones.')
show(db, [' ', '.', 'o', 'x'])

def score(X, y, theta):
	return np.mean(np.round(sigmoid((X)@theta))==y), np.sum(np.round(sigmoid((X)@theta))==y)

print('Model accuracy {:.1%} ({}/{} points)'.format(score(X, y, theta_min)[0], score(X, y, theta_min)[1], len(y)))

print('Feel free to modify the datapoints, lambda or the polinomial depth and see what happens! :)')


#!/usr/bin/env python
"""Kinematic function skeleton code for Lab 3 prelab.

Course: EE 106A, Fall 2020
Originally written by: Aaron Bestick, 9/10/14
Adapted for Fall 2020 by: Amay Saxena, 9/10/20

This Python file is a code skeleton for Lab 3 prelab. You should fill in
the body of the five empty methods below so that they implement the kinematic
functions described in the assignment.

When you think you have the methods implemented correctly, you can test your
code by running "python kinematic.py at the command line.
"""

import numpy as np

np.set_printoptions(precision=4,suppress=True)

#-----------------------------2D Examples---------------------------------------
#--(you don't need to modify anything here but you should take a look at them)--

def rotation_2d(theta):
    """
    Computes a 2D rotation matrix given the angle of rotation.

    Args:
    theta: the angle of rotation

    Returns:
    rot - (2,2) ndarray: the resulting rotation matrix
    """

    rot = np.zeros((2,2))
    rot[0,0] = np.cos(theta)
    rot[1,1] = np.cos(theta)
    rot[0,1] = -np.sin(theta)
    rot[1,0] = np.sin(theta)

    return rot

def hat_2d(xi):
    """
    Converts a 2D twist to its corresponding 3x3 matrix representation

    Args:
    xi - (3,) ndarray: the 2D twist

    Returns:
    xi_hat - (3,3) ndarray: the resulting 3x3 matrix
    """
    if not xi.shape == (3,):
        raise TypeError('omega must be a 3-vector')

    xi_hat = np.zeros((3,3))
    xi_hat[0,1] = -xi[2]
    xi_hat[1,0] =  xi[2]
    xi_hat[0:2,2] = xi[0:2]

    return xi_hat

def homog_2d(xi, theta):
    """
    Computes a 3x3 homogeneous transformation matrix given a 2D twist and a
    joint displacement

    Args:
    xi - (3,) ndarray: the 2D twist
    theta: the joint displacement

    Returns:
    g - (3,3) ndarray: the resulting homogeneous transformation matrix
    """
    if not xi.shape == (3,):
        raise TypeError('xi must be a 3-vector')

    g = np.zeros((3,3))
    wtheta = xi[2]*theta
    R = rotation_2d(wtheta)
    p = np.dot(np.dot( \
        [[1 - np.cos(wtheta), np.sin(wtheta)],
        [-np.sin(wtheta), 1 - np.cos(wtheta)]], \
        [[0,-1],[1,0]]), \
        [[xi[0]/xi[2]],[xi[1]/xi[2]]])

    g[0:2,0:2] = R
    g[0:2,2:3] = p[0:2]
    g[2,2] = 1

    return g

#-----------------------------3D Functions--------------------------------------
#-------------(These are the functions you need to complete)--------------------

def skew_3d(omega):
    """
    Prelab Part (a)

    Converts a rotation vector in 3D to its corresponding skew-symmetric matrix.

    Args:
    omega - (3,) ndarray: the rotation vector

    Returns:
    omega_hat - (3,3) ndarray: the corresponding skew symmetric matrix
    """
    if not omega.shape == (3,):
        raise TypeError('omega must be a 3-vector')

    omega_hat = np.zeros((3, 3))
    omega_hat[1, 0] =  omega[2]
    omega_hat[2, 0] = -omega[1]
    omega_hat[0, 1] = -omega[2]
    omega_hat[2, 1] =  omega[0]
    omega_hat[0, 2] =  omega[1]
    omega_hat[1, 2] = -omega[0]

    return omega_hat

def rotation_3d(omega, theta):
    """
    Prelab Part (b)

    Computes a 3D rotation matrix given a rotation axis and angle of rotation.

    Args:
    omega - (3,) ndarray: the axis of rotation
    theta: the angle of rotation

    Returns:
    rot - (3,3) ndarray: the resulting rotation matrix
    """
    if not omega.shape == (3,):
        raise TypeError('omega must be a 3-vector')

    #import scipy
    #rot = scipy.linalg.expm(np.dot(skew_3d(omega), theta))
    omega_hat = skew_3d(omega)
    omega_norm = np.linalg.norm(omega)
    rot = np.eye(3) + omega_hat / omega_norm * np.sin(omega_norm * theta) + \
        omega_hat.dot(omega_hat) / omega_norm**2 * (1 - np.cos(omega_norm * theta))

    return rot

def hat_3d(xi):
    """
    Prelab Part (c)

    Converts a 3D twist to its corresponding 4x4 matrix representation

    Args:
    xi - (6,) ndarray: the 3D twist

    Returns:
    xi_hat - (4,4) ndarray: the corresponding 4x4 matrix
    """
    if not xi.shape == (6,):
        raise TypeError('xi must be a 6-vector')

    xi_hat = np.zeros((4, 4))
    xi_hat[0:3, 0:3] = skew_3d(xi[3:])
    xi_hat[0:3, 3] = xi[:3]

    return xi_hat

def homog_3d(xi, theta):
    """
    Prelab Part (d)

    Computes a 4x4 homogeneous transformation matrix given a 3D twist and a
    joint displacement.

    Args:
    xi - (6,) ndarray: the 3D twist
    theta: the joint displacement
    Returns:
    g - (4,4) ndarary: the resulting homogeneous transformation matrix
    """
    if not xi.shape == (6,):
        raise TypeError('xi must be a 6-vector')

    v = xi[:3]
    w = xi[3:]
    I = np.eye(3)
    if np.allclose(w, 0):
        # Pure translation
        R = I
        p = v * theta
    else:
        # Translation and rotation
        R = rotation_3d(w, theta)
        w_vec = np.expand_dims(w, axis=1)
        v_vec = np.expand_dims(v, axis=1)
        p = 1 / np.linalg.norm(w)**2 * ((I - R).dot(skew_3d(w).dot(v_vec)) +
                                        w_vec.dot(w_vec.T).dot(v_vec) * theta)
    g = np.eye(4)
    g[:3, :3] = R
    g[:3, 3] = np.squeeze(p)

    return g

def prod_exp(xi, theta):
    """
    Prelab Part (e)

    Computes the product of exponentials for a kinematic chain, given
    the twists and displacements for each joint.

    Args:
    xi - (6, N) ndarray: the twists for each joint
    theta - (N,) ndarray: the displacement of each joint

    Returns:
    g - (4,4) ndarray: the resulting homogeneous transformation matrix
    """
    if not xi.shape[0] == 6:
        raise TypeError('xi must be a 6xN')
    if not xi.shape[1] == theta.shape[0]:
        raise TypeError('there must be the same number of twists as joint angles.')

    g = np.eye(4)
    for idx in range(len(theta)):
        g = g.dot(homog_3d(xi[:, idx], theta[idx]))

    return g

#-----------------------------Testing code--------------------------------------
#-------------(you shouldn't need to modify anything below here)----------------

def array_func_test(func_name, args, ret_desired):
    ret_value = func_name(*args)
    if not isinstance(ret_value, np.ndarray):
        print('[FAIL] ' + func_name.__name__ + '() returned something other than a NumPy ndarray')
    elif ret_value.shape != ret_desired.shape:
        print('[FAIL] ' + func_name.__name__ + '() returned an ndarray with incorrect dimensions')
    elif not np.allclose(ret_value, ret_desired, rtol=1e-3):
        print('[FAIL] ' + func_name.__name__ + '() returned an incorrect value')
    else:
        print('[PASS] ' + func_name.__name__ + '() returned the correct value!')

if __name__ == "__main__":
    print('Testing...')

    #Test skew_3d()
    arg1 = np.array([1.0, 2, 3])
    func_args = (arg1,)
    ret_desired = np.array([[ 0., -3.,  2.],
                            [ 3., -0., -1.],
                            [-2.,  1.,  0.]])
    array_func_test(skew_3d, func_args, ret_desired)

    #Test rotation_3d()
    arg1 = np.array([2.0, 1, 3])
    arg2 = 0.587
    func_args = (arg1,arg2)
    ret_desired = np.array([[-0.1325, -0.4234,  0.8962],
                            [ 0.8765, -0.4723, -0.0935],
                            [ 0.4629,  0.7731,  0.4337]])
    array_func_test(rotation_3d, func_args, ret_desired)

    #Test hat_3d()
    arg1 = np.array([2.0, 1, 3, 5, 4, 2])
    func_args = (arg1,)
    ret_desired = np.array([[ 0., -2.,  4.,  2.],
                            [ 2., -0., -5.,  1.],
                            [-4.,  5.,  0.,  3.],
                            [ 0.,  0.,  0.,  0.]])
    array_func_test(hat_3d, func_args, ret_desired)

    #Test homog_3d()
    arg1 = np.array([2.0, 1, 3, 5, 4, 2])
    arg2 = 0.658
    func_args = (arg1,arg2)
    ret_desired = np.array([[ 0.4249,  0.8601, -0.2824,  1.7814],
                            [ 0.2901,  0.1661,  0.9425,  0.9643],
                            [ 0.8575, -0.4824, -0.179 ,  0.1978],
                            [ 0.    ,  0.    ,  0.    ,  1.    ]])
    array_func_test(homog_3d, func_args, ret_desired)

    #Test prod_exp()
    arg1 = np.array([[2.0, 1, 3, 5, 4, 6], [5, 3, 1, 1, 3, 2], [1, 3, 4, 5, 2, 4]]).T
    arg2 = np.array([0.658, 0.234, 1.345])
    func_args = (arg1,arg2)
    ret_desired = np.array([[ 0.4392,  0.4998,  0.7466,  7.6936],
                            [ 0.6599, -0.7434,  0.1095,  2.8849],
                            [ 0.6097,  0.4446, -0.6562,  3.3598],
                            [ 0.    ,  0.    ,  0.    ,  1.    ]])
    array_func_test(prod_exp, func_args, ret_desired)

    print('Done!')


'''
numpy最终要的特点就是Ｎ维数组对象(ndarray),他是一个快速而领会的大数据集容器
可以对数据执行一些数学运算,其语法和标量元素相同.
'''
from numpy.random import randn
import numpy as np
data = randn(2, 3) # 创建随机数2*3维
print(data)
print(data*10)
print(data + data)
'''
output:
data
[[ 0.26474196  0.39938213  0.00644464]
 [-0.89070803  1.20568504  0.77317617]]

data * 10
[[  2.64741962   3.99382133   0.06444643]
 [ -8.90708032  12.05685039   7.73176167]]

data + data
[[ 0.52948392  0.79876427  0.01288929]
 [-1.78141606  2.41137008  1.54635233]]
'''

'''
需要注意的是ndarray是一个通用的同构数据多维容器, 也就是说**所有元素的类型必须相同**
同时,每个数组都有一个shape(各维大小的元组)属性和一个dtype(用于说明数组数据类型的对象)
'''
arr = [1, '2', 'a', 'c']
d = np.array(arr) # 会把数组中不同类型的元素转换成相同类型
print(d)
'''
output:
['1' '2' 'a' 'c']
'''
print(d.shape, d.dtype)
'''
output:
(4,) <U21
'''

data2 = [[1, 2, 3, 4], [5, 6, 7, 8]] # 创建多维数组
arr2 = np.array(data2) # 转换成多维
print(arr2.ndim, arr2.shape)
'''
output:
2 (2, 4)
'''

np.zeros(10) # 创建一维10个数据都为0的数组
np.zeros((3, 6)) # 创建一个3*6数据都为0的数组
np.empty((2, 3, 2)) # 创建2*3*2的数组

'''
创建数组函数
array 		将输入数据转换为ndarray.

asarray 	将输入数据转换为ndarray.

arange		类似内置range.返回ndarray

ones,		根据指定的形状和dtype创建一个全1的数组.
ones_like	ones_like以另一个数组为参数, 并根据器形状和dtype创建一个全1的数组

zeros
zeros_like 	类似上一个

empty
empty_like 	同上

eye,
identity	穿件一个N*N单位矩阵(对角线为1, 其余为0)
'''

# ndarray的type可以显示转换
array = np.array([1, 2, 3, 4, 5])
float_arr = array.astype(np.float64)	# 整形转换为浮点型
print(float_arr.dtype)
'''
output:
float64
'''

array = np.array([3.7, -1.2, -2.6, 0.5, 12.9, 10.1])
array = array.astype(np.int32) # 浮点型转换为整型(小数部分被截断)
print(array)
'''
output:
[ 3 -1 -2  0 12 10]
'''

'''
### 数组与标量之间的运算
数组可以不用编写循环就可以对数据进行批量运算(也就是矢量化).
大小相同的数组运算会将运算应用到元素级
'''
arr = np.array([[1., 2., 3.], [4., 5., 6.]])
print(arr)
'''
output:
[[ 1.  2.  3.]
 [ 4.  5.  6.]]
'''
print(arr * arr)
'''
output:
[[  1.   4.   9.]
 [ 16.  25.  36.]]
'''

# 同样的,数组与标量间的计算也会传播到元素级(不同大小的数组运算叫广播)
print(1 / arr)
'''
output:
[[ 1.          0.5         0.33333333]
 [ 0.25        0.2         0.16666667]]
'''

'''
## 基本的切片与索引
类似于python内置的list和tuple

### 花式索引
花式索引是numpy的术语, 他是利用整数数组进行索引
'''
arr = np.empty((8, 4))
for i in range(8):
	arr[i] = i
print(arr)
'''
output:
[[ 0.  0.  0.  0.]
 [ 1.  1.  1.  1.]
 [ 2.  2.  2.  2.]
 [ 3.  3.  3.  3.]
 [ 4.  4.  4.  4.]
 [ 5.  5.  5.  5.]
 [ 6.  6.  6.  6.]
 [ 7.  7.  7.  7.]]
'''
print(arr[[4, 3, 0, 6, 5]]) # 选取索引为4, 3, 0, 6, 5的行
'''
output:
[[ 4.  4.  4.  4.]
 [ 3.  3.  3.  3.]
 [ 0.  0.  0.  0.]
 [ 6.  6.  6.  6.]
 [ 5.  5.  5.  5.]]
'''
print(arr[[-1, -5]])
'''
output:
[[ 7.  7.  7.  7.]
 [ 3.  3.  3.  3.]]
'''

# 一次性传入多个索引会返回一个一维数组
arr = np.empty((8, 4))
for i in range(1, 9):
	for j in range(1, 5):
		arr[i-1][j-1] = i * j
print(arr)
'''
output:
[[  1.   2.   3.   4.]
 [  2.   4.   6.   8.]
 [  3.   6.   9.  12.]
 [  4.   8.  12.  16.]
 [  5.  10.  15.  20.]
 [  6.  12.  18.  24.]
 [  7.  14.  21.  28.]
 [  8.  16.  24.  32.]]
 '''
print(arr[[1, 5, 7, 2], [0, 3, 1, 2]]) # 返回(1, 0), (5, 3), (7,1), (2, 2)这四个元素
# output: [  2.  24.  16.   9.]

print(arr[[1, 5, 7, 2]][:, [0, 3, 1, 2]]) # 以返回(1, 0), (1, 3), (1, 1), (1, 2)
											#	 (5, 0), ...
											#	 (7, 0), ...
											#	 (2, 0), ...
											# 4*4的数组
'''
output:
[[  2.   8.   4.   6.]
 [  6.  24.  12.  18.]
 [  8.  32.  16.  24.]
 [  3.  12.   6.   9.]]
'''
arr_ix = arr[np.ix_([1, 5, 7, 2], [0, 3, 1, 2])]
print(arr_ix)

'''
output:
[[  2.   8.   4.   6.]
 [  6.  24.  12.  18.]
 [  8.  32.  16.  24.]
 [  3.  12.   6.   9.]]
'''

'''
numpy可以简化数据处理,避免了python写循环,用数组表达式代替循环的做法一般叫矢量化
一般来说,矢量化的运算要比纯Python的循环快上1-2个数量级
在一组树枝上计算(sqrt(x^2 + y^2))
'''
'''
from matplotlib.pyplot import imshow, title
import matplotlib.pyplot as plt
points = np.arange(-5, 5, 0.01)
xs, ys = np.meshgrid(points, points) # np.meshgrid函数接受2个一维5数组,并产生2个二维矩阵(分别对应x和y)
z = np.sqrt(xs ** 2 + ys ** 2)
plt.imshow(z, cmap=plt.cm.gray)
plt.colorbar()
'''
# plt.show()

# 将条件逻辑表述为数组运算
# numpy.where函数是三元表达式x if condition else y的矢量版本
xarr = np.array([1.1, 1.2, 1.3, 1.4, 1.5])
yarr = np.array([2.1, 2.2, 2.3, 2.4, 2.5])
cond = np.array([True, False, True, True, False])
# 需要判断cond中的值为True时选取xarr, 为False时选取yarr
result = [(x if c else y) for x, y, c in zip(xarr, yarr, cond)] # python版本
result = np.where(cond, xarr, yarr)
print(result)
'''
output:
[ 1.1  2.2  1.3  1.4  2.5]
'''
# np.where的第二个和第三个参数不一定是数组,它们可以是标量值.
# 在数据分析中,where通常用于根据一个数组产生另一个数组.

# 假设有一个由随机数数据组成的矩阵,希望把所有大于0的值替换为2, 小于0的值替换为-2
arr = randn(4, 4)
np.where(arr > 0, 2, -2) # 大于0为2, 小于0为-2
np.where(arr > 0, 2, arr)# 大于0为2, 小于0不变
# 传给where的数组大小可以不相等,甚至可以是标量
'''
result = []
for i in range(n):
	if cond1[i] and cond2[i]:
		result.append(0)
	elif cond1[i]:
		result.append(1)
	elif cond2[i]:
		result.append(2)
	else:
		result.append(3)
# python原始实现
np.where(cond1 & cond2, 0, p.where(cond2, np.where(2, 3)))
resutlt = 1 * cond1 + 2 * cond2 + 3 * -(cond1 | cond2)
'''
# 数学和统计方法
# sum, mean以及标准差std等聚合计算可以当所数组的方法使用,也可以当做numpy函数使用
arr = np.random.randn(5, 4) # 正态分布的数据
print(arr)
print(arr.mean())
print(arr.mean() * 20)
print(arr.sum())
'''
output:
-0.391861236317
-7.83722472633
-7.83722472633
'''
arr = np.zeros((2, 6))
print(arr)
print(arr.mean(axis=0), '\n') # 0求每一列平均
print(arr.mean(axis=1), '\n') # 1求每一行平均
#print(arr.mean(axis=2), '\n')

print(arr.sum(0))

arr = np.array([[0, 1, 2],
				[3, 4, 5],
				[6, 7, 8]])
print(arr.cumsum(0))
print(arr.cumprod(1))

# 排序
'''
numpy也可以用sort方法来排序

# 唯一化以及其他的几个逻辑
numpy提供了针对以为数组的基本集合运算,
np.unique 用于找出数组中的唯一值并返回已排序的结果
'''
names = np.array(['Bob', 'Joe', 'Will', 'Bob',
					'Will', 'Joe', 'Joe'])
print(np.unique(names))

ints = np.array([3, 3, 3, 2, 2, 1, 1, 4, 4])
print(np.unique(ints))

'''
数组的集合运算

unique(x)			计算x中唯一元素,并返回有序结果
intersect1(x, y)	计算x和y中的公共元素,并返回有序结果
union(x, y)			x和y的并集,并返回有序结果
in1d(x, y)			得到一个表示"x的元素是否包含在y"的bool型数组
setdiff1d(x, y) 	x和y的集合差
setxor1d(x, y)		x和y对称差(只存在与一个集合中的元素集合)
'''

'''
# 线性代数(如矩阵乘法, 矩阵分解, 行列式以及其他方阵数学)的计算, numpy提供了dot函数
'''
x = np.array([[1., 2., 3.],
			  [4., 5., 6.]])
y = np.array([[6., 23.],
			  [-1., 7.],
			  [8., 9]])
x1 = np.array([[1., -1.],
			   [1., 1.]])
print(x)
print(y)
print(x.dot(y))
print(np.dot(x, np.ones(3)))
print(np.linalg.det(x1))

'''
常用的numpy.linalg函数

diag		以一维数组的形式返回方阵的对角线(或非对角线)元素,或将一位数组转换为方阵(非对角线元素为0)
dot 		矩阵乘法
trace		计算对角线元素的和
det 		计算矩阵行列式
eig			计算方阵的本真值和本真向量
inv			方阵的逆
pinv		计算方阵的Moore-Penrose伪逆
qr 			计算QR分解
svd			计算奇异值分解
solve		解线性方程Ax = b, 其中A为一个方阵
lstsq 		计算Ax = b的最小二乘解
'''

# 随机数生成
'''
numpy.random模块对python内置的random进行了补充,
增加了一些用于高级生成多种概率分布的样本值函数.
例如,用normal生成一个标准正态分布的4*4样本数组
'''
samples = np.random.normal(size=(4, 4))
print(samples)
'''
部分numpy.random函数

seed 			确定随机数生成器的种子
permutation		返回一个序列的随机排序或返回一个随机排序范围
shuffle			对一个对列就地随机排序
rand 			产生均匀分布的样本值
randint			从给定的上下限范围内随机选取整数
randn 			产生正态分布(平均值为0, 标准差为1)的样本值
uniform 		产生在[0, 1)中均匀分布的样本值
'''

'''
通过模拟随机漫步来说明如何运用数组运算

简单例子: 从0开始, 步长为1和-1出现的概率相等.
通过内置的random模块以及纯python的方式实现1000步长的随机漫步
'''
import random
import matplotlib.pyplot as plt
position = 0
walk = [position]
steps = 1000
for i in range(steps):
	step = 1 if random.randint(0, 1) else -1
	position += step
	walk.append(position)
plt.plot(range(1000), walk[:1000])
# plt.show()

nsteps = 1000
draws = np.random.randint(0, 2, size=nsteps)
steps = np.where(draws > 0, 1, -1)
walk = steps.cumsum()

print(walk.min())
print(walk.max())
plt.plot(range(1000), walk[:1000])
plt.show()

nwalks = 5000
nsteps = 1000
draws = np.random.randint(0, 2, size=(nwalks, nsteps)) # 传入一个二元元组产生一个二维数组
steps = np.where(draws > 0, 1, -1)
walks = steps.cumsum(1)
print(walks)


import numpy as np

# WARMUPEXERCISE Example function
#   A = WARMUPEXERCISE() is an example function that returns the 5x5 identity matrix

# ============= YOUR CODE HERE ==============
# Instructions: Return the 5x5 identity matrix
#
def warm_up_exercise():
    return

# PLOTDATA Plots the data points x and y into a new figure
#   PLOTDATA(x,y) plots the data points and gives the figure axes labels of
#   population and profit.

# ====================== YOUR CODE HERE ======================
# Instructions: Plot the training data into a figure using the
#               "figure" and "plot" commands. Set the axes labels using
#               the "xlabel" and "ylabel" commands. Assume the
#               population and revenue data have been passed in
#               as the x and y arguments of this function.
#
# Hint: You can use the 'rx' option with plot to have the markers
#       appear as red crosses. Furthermore, you can make the
#       markers larger by using plot(..., 'rx', 'MarkerSize', 10);
def plot_data(x, y):
    return

# COMPUTECOST Compute cost for linear regression
#   J = COMPUTECOST(X, y, theta) computes the cost of using theta as the
#   parameter for linear regression to fit the data points in X and y
#
def compute_cost(X, y, theta):
    m = y.size     # number of training examples

    # You need to return the following variables correctly
    J = 0;

    # ====================== YOUR CODE HERE ======================
    # Instructions: Compute the cost of a particular choice of theta
    #               You should set J to the cost.

    # for i=1:size(X, 1);
    #   J = J + (1 / (2 * m) * (X(i,:)*theta - y(i))^2);
    # end;

    # =========================================================================
    return J

# GRADIENTDESCENT Performs gradient descent to learn theta
#   theta = GRADIENTDESCENT(X, y, theta, alpha, num_iters) updates theta by
#   taking num_iters gradient steps with learning rate alpha
def gradient_descent(X, y, theta, alpha, num_iters):
    # Initialize some useful values
    m = y.size    # number of training examples
    J_history = np.zeros((num_iters, 1))
    for iter in range(num_iters):
        # ====================== YOUR CODE HERE =======================
        # Instructions: Perform a single gradient step on the parameter vector
        # theta.
        #
        # Hint: While debugging, it can be useful to print out the values
        # of the cost function(computeCost) and gradient here.
        #

        # for i=1:size(X, 2)
        #   theta(i) = theta(i) - alpha/m*(X(:,i)'*delta);
        # end;

        # ============================================================

        # Save the cost J in every iteration
        J_history[iter] = compute_cost(X, y, theta)

    return theta, J_history


import numpy as np
import random

def distribution1(x, batch_size=512):
    # Distribution defined as (x, U(0,1)). Can be used for question 3
    while True:
        yield(np.array([(x, random.uniform(0, 1)) for _ in range(batch_size)]))

def distribution2(batch_size=512):
    # High dimension uniform distribution
    while True:
        yield(np.random.uniform(0, 1, (batch_size, 2)))

def distribution3(batch_size=512):
    # 1D gaussian distribution
    while True:
        yield(np.random.normal(0, 1, (batch_size, 1)))

e = lambda x: np.exp(x)
tanh = lambda x: (e(x) - e(-x)) / (e(x)+e(-x))
def distribution4(batch_size=1):
    # arbitrary sampler
    f = lambda x: tanh(x*2+1) + x*0.75
    while True:
        yield(f(np.random.normal(0, 1, (batch_size, 1))))

if __name__ == '__main__':
    # Example of usage
    dist = iter(distribution1(0, 100))
    samples = next(dist)


# import pandas and numpy, and load the nls data
import pandas as pd
import numpy as np
pd.set_option('display.width', 200)
pd.set_option('display.max_columns', 35)
pd.set_option('display.max_rows', 200)
pd.options.display.float_format = '{:,.1f}'.format
nls97 = pd.read_csv("data/nls97b.csv")
nls97.set_index("personid", inplace=True)

# create a function for calculating interquartile range
def iqr(x):
  return x.quantile(0.75) - x.quantile(0.25)

# run the interquartile range function
aggdict = {'weeksworked06':['count', 'mean', iqr], 'childathome':['count', 'mean', iqr]}
nls97.groupby(['highestdegree']).agg(aggdict)

# define a function to return the summary statistics as a series
def gettots(x):
  out = {}
  out['qr1'] = x.quantile(0.25)
  out['med'] = x.median()
  out['qr3'] = x.quantile(0.75)
  out['count'] = x.count()
  return pd.Series(out)

# use apply to run the function
pd.options.display.float_format = '{:,.0f}'.format
nls97.groupby(['highestdegree'])['weeksworked06'].apply(gettots)

# chain reset_index to set the default index
nls97.groupby(['highestdegree'])['weeksworked06'].apply(gettots).reset_index()

# allow the index to be created
nlssums = nls97.groupby(['highestdegree'])['weeksworked06'].apply(gettots).unstack()
nlssums
nlssums.info()

# run the groupby without creating an index and create a data frame
nls97.groupby(['highestdegree'], as_index=False)['weeksworked06'].apply(gettots)

07 - Fixing Messy Data When Aggregating/5. groupby_udf.py


import numpy as np
from scipy.ndimage import convolve, maximum_filter

def gauss2d(sigma, fsize):
    """ Create a 2D Gaussian filter

    Args:
        sigma: width of the Gaussian filter
        fsize: (w, h) dimensions of the filter
    Returns:
        *normalized* Gaussian filter as (h, w) np.array
    """
    m, n = fsize
    x = np.arange(-m / 2 + 0.5, m / 2)
    y = np.arange(-n / 2 + 0.5, n / 2)
    xx, yy = np.meshgrid(x, y, sparse=True)
    g = np.exp(-(xx ** 2 + yy ** 2) / (2 * sigma ** 2))
    return g / np.sum(g)

def derivative_filters():
    """ Create derivative filters for x and y direction

    Returns:
        fx: derivative filter in x direction
        fy: derivative filter in y direction
    """
    fx = np.array([[0.5, 0, -0.5]])
    fy = fx.transpose()
    return fx, fy

def compute_hessian(img, gauss, fx, fy):
    """ Compute elements of the Hessian matrix

    Args:
        img:
        gauss: Gaussian filter
        fx: derivative filter in x direction
        fy: derivative filter in y direction

    Returns:
        I_xx: (h, w) np.array of 2nd derivatives in x direction
        I_yy: (h, w) np.array of 2nd derivatives in y direction
        I_xy: (h, w) np.array of 2nd derivatives in x-y direction
    """

    #
    # You code here
    #

    # set mode
    mode = "mirror"

    # smooth image
    img = convolve(img, gauss, mode = mode, cval=0)

    # first derivatives
    I_x = convolve(img, fx, mode = mode, cval=0)
    I_y = convolve(img, fy, mode = mode, cval=0)

    # second derivatives
    I_xx = convolve(I_x, fx, mode = mode, cval=0)
    I_xy = convolve(I_x, fy, mode = mode, cval=0)
    I_yy = convolve(I_y, fy, mode = mode, cval=0)

    return I_xx,I_yy,I_xy

def compute_criterion(I_xx, I_yy, I_xy, sigma):
    """ Compute criterion function

    Args:
        I_xx: (h, w) np.array of 2nd derivatives in x direction
        I_yy: (h, w) np.array of 2nd derivatives in y direction
        I_xy: (h, w) np.array of 2nd derivatives in x-y direction
        sigma: scaling factor

    Returns:
        criterion: (h, w) np.array of scaled determinant of Hessian matrix
    """

    #
    # You code here
    #

    det = I_xx * I_yy - I_xy ** 2
    return sigma ** 4 * det

def nonmaxsuppression(criterion, threshold):
    """ Apply non-maximum suppression to criterion values
        and return Hessian interest points

        Args:
            criterion: (h, w) np.array of criterion function values
            threshold: criterion threshold
        Returns:
            rows: (n,) np.array with y-positions of interest points
            cols: (n,) np.array with x-positions of interest points
    """

    #
    # You code here
    #

    criterion_max = maximum_filter(criterion, (5,5), mode= "mirror")
    criterion_thresh = np.logical_and(criterion_max > threshold, criterion >= criterion_max)

    mask = np.zeros_like(criterion_thresh)
    mask[5:-5, 5:-5] = criterion_thresh[5:-5, 5:-5]
    rows, cols = np.nonzero(mask)
    return rows, cols


import numpy as np
from sklearn.cluster import KMeans

class GaussianDistribution:

    def __init__(self, n_clusters, n_epochs):
        self.n_clusters = n_clusters
        self.n_epochs = n_epochs

    def gaussian(self, X, mu, cov):
        ''' here we implement the Gaussian Density function '''
        n = X.shape[1]
        diff = (X - mu).T
        return np.diagonal(1 / ((2 * np.pi) ** (n / 2) * np.linalg.det(cov) ** 0.5) * np.exp(-0.5 * np.dot(np.dot(diff.T, np.linalg.inv(cov)), diff))).reshape(-1, 1)

    #Step 1: (Intialization)
    def initialize_clusters(self, X):

        ''' This is the initialization step of the GMM. At this point, we must initialise our parameters  μk, πk and Σk. Here we'll be using results of KMeans as an initial value for μk , πk to one over the number of clusters and Σk to identity matrix.
        NOTE: We could also use random numbers for everything, but using a sensible initialisation procedure will help the algorithm achieve better results.
         '''

        clusters = []
        idx = np.arange(X.shape[0])

        # We use the KMeans centroids to initialise the GMM

        kmeans = KMeans(self.n_clusters).fit(X)
        mu_k = kmeans.cluster_centers_

        for i in range(self.n_clusters):
            clusters.append({
                'pi_k': 1.0 / self.n_clusters,
                'mu_k': mu_k[i],
                'cov_k': np.identity(X.shape[1], dtype=np.float64)
            })

        return clusters

    #Step 2 (Expectation step)
    def expectation_step(self, X, clusters):

        ''' Here we calculate the value of ⲅ.
            For simplicity, we just calculate the denominator as a sum over all terms in the numerator, and then assign it to a variable named totals
         '''

        totals = np.zeros((X.shape[0], 1), dtype=np.float64)

        for cluster in clusters:
            pi_k = cluster['pi_k']
            mu_k = cluster['mu_k']
            cov_k = cluster['cov_k']

            gamma_nk = (pi_k * self.gaussian(X, mu_k, cov_k)).astype(np.float64)

            for i in range(X.shape[0]):
                totals[i] += gamma_nk[i]

            cluster['gamma_nk'] = gamma_nk
            cluster['totals'] = totals

        for cluster in clusters:
            cluster['gamma_nk'] /= cluster['totals']

    #Step 3 (Maximization step)
    def maximization_step(self, X, clusters):

        ''' Here the value of parameters μk, πk and Σk are updated '''

        N = float(X.shape[0])

        for cluster in clusters:
            gamma_nk = cluster['gamma_nk']
            cov_k = np.zeros((X.shape[1], X.shape[1]))

            N_k = np.sum(gamma_nk, axis=0)

            pi_k = N_k / N
            mu_k = np.sum(gamma_nk * X, axis=0) / N_k

            for j in range(X.shape[0]):
                diff = (X[j] - mu_k).reshape(-1, 1)
                cov_k += gamma_nk[j] * np.dot(diff, diff.T)

            cov_k /= N_k

            cluster['pi_k'] = pi_k
            cluster['mu_k'] = mu_k
            cluster['cov_k'] = cov_k

    #Let us now determine the log-likelihood of the model.
    def get_likelihood(self, X, clusters):
        sample_likelihoods = np.log(np.array([cluster['totals'] for cluster in clusters]))
        return np.sum(sample_likelihoods)

    #Putting everything together
    # 1. Initialise the parameters by using the initialise_clusters function
    # 2. perform several expectation-maximization steps
    def train_gmm(self, X):
        clusters = self.initialize_clusters(X)
        likelihoods = np.zeros((self.n_epochs, ))
        scores = np.zeros((X.shape[0], self.n_clusters))

        for i in range(self.n_epochs):

            self.expectation_step(X, clusters)
            self.maximization_step(X, clusters)

            likelihood = self.get_likelihood(X, clusters)
            likelihoods[i] = likelihood

            print('Epoch: ', i + 1, 'Likelihood: ', likelihood)

        for i, cluster in enumerate(clusters):
            scores[:, i] = np.log(cluster['gamma_nk']).reshape(-1)

        return likelihoods


import numpy as np
class Solution:
    def findTheCity(self, n: int, edges: [[int]], distanceThreshold: int) -> int:
        # dp[i][j] distance of city i to j
        dp = np.zeros((n, n), int)
        dp.fill(-1)
        for i in range(n):
            dp[i][i] = 0
        # initializer
        for edge in edges:
            dp[edge[0]][edge[1]] = edge[2]
            dp[edge[1]][edge[0]] = edge[2]
        for k in range(n):
            for i in range(n):
                for j in range(n):
                    if dp[i][k] > -1 and dp[j][k] > -1 and (dp[i][j] > dp[i][k] + dp[k][j] or dp[i][j] == -1):
                        dp[i][j] = dp[i][k] + dp[k][j]
        minc = n
        pset = set()
        for i in range(n):
            cnt = 0
            for j in range(n):
                if dp[i][j] > 0 and dp[i][j] <= distanceThreshold:
                    cnt += 1
            if minc >= cnt:
                pset.clear()
                minc = cnt
                pset.add(i)

        return pset.pop()

n = 4
edges = [[0,1,3],[1,2,1],[1,3,4],[2,3,1]]
distanceThreshold = 4

n = 5
edges = [[0,1,2],[0,4,8],[1,2,3],[1,4,2],[2,3,1],[3,4,1]]
distanceThreshold = 2

sol = Solution()
res = sol.findTheCity(n, edges, distanceThreshold)
print(res)


# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Contains the sigmoid activation function"""

import numpy as np
from .activation_function import ActivationFunction

class Sigmoid(ActivationFunction):
    """Sigmoid activation function.

    The sigmoid activation function is defined as:
    f(x) = 1 / (1 + e^(-x))

    It returns a value between 0 and 1.
    """

    def __init__(self):
        super().__init__(name='sigmoid')

    @staticmethod
    def forward(x: np.ndarray) -> np.ndarray:
        """Computes the sigmoid activation function."""
        return 1 / (1 + np.exp(-x))

    @staticmethod
    def gradient(z: np.ndarray) -> np.ndarray:
        """Computes the gradient of the sigmoid activation function."""
        sigmoid = Sigmoid.forward(z)
        return sigmoid * (1 - sigmoid)


#!/usr/bin/env python
# -*- coding: utf-8 -*-

import numpy as np
import numpy.linalg as npl
import numpy.random as npr
import unittest
import matplotlib.pyplot as mp
import sys
from householder_transforms import *

def qr_decomposition(A, mode = "", graph_mode = False):
	""" Décomposition QR d'une matrice
		Entrée:
			A matrice quelconque.
		Sortie:
			Tuple (Q, R) des matrices de la décomposition.
	"""
	if graph_mode:
		it = []
		decomp_quality = []
	A = np.matrix(A, dtype='f')
	N, M = A.shape
	Q = np.eye(N, N, dtype='f')
	R = np.matrix(A)
	for i in range(min(N, M)):
		X = R[i:, i]
		z = np.zeros([len(X), 1])
		if X[0, 0] > 0:
			z[0, 0] = npl.norm(X)
		else:
			z[0, 0] = -npl.norm(X)
		Q[:, i:] = householder_product_right(X, z, Q[:, i:])
		R[i:, :] = householder_product_left(X, z, R[i:, :])
		if graph_mode:
			it.append(i)
			Z = np.dot(Q, R)
			decomp_quality.append(npl.norm(Z - A))
	if graph_mode:
		return it, decomp_quality
	return Q, R

def qr_decomposition_bidiag(A, mode = "", graph_mode = False):
	""" Décomposition QR d'une matrice bidiagonale.
		Entrée:
			A matrice bidiagonale.
		Sortie:
			Tuple (Q, R) des matrices de la décomposition.
	"""
	A = np.matrix(A, dtype='f')
	N, M = A.shape
	Q = np.eye(N, dtype='f')
	R = A
	for i in range(min(N, M)-1):
		X = R[i:i+2, i]
		z = np.zeros([2, 1])
		if X[0, 0] > 0:
			z[0, 0] = npl.norm(X)
		else:
			z[0, 0] = -npl.norm(X)
		Q[:, i:i+2] = householder_product_right(X, z, Q[:, i:i+2])
		R[i:i+2, i:i+2] = householder_product_left(X, z, R[i:i+2, i:i+2])
	return Q, R

def toSVD(BD, iterations = 10, qr = npl.qr, graph_mode = False):
	""" Transformation SVD d'une matrice diagonale
		Entrée:
			BD matrice tridiagonale.
		Sortie:
			Un triplet correspondant aux matrices de la décomposition.
	"""
	if graph_mode:
		it = []
		decomp_quality = []
		convergence = []
	N, M = BD.shape
	U = np.eye(N)
	V = np.eye(M)
	S = np.matrix(BD)
	for i in range(iterations):
		Q1, R1 = qr(S.T, 'complete')
		Q2, R2 = qr(R1.T, 'complete')
		S = R2
		U = np.dot(U, Q2)
		V = np.dot(Q1.T, V)
		if graph_mode:
			it.append(i)
			Z = np.dot(np.dot(U, S), V)
			decomp_quality.append(npl.norm(Z - BD))
			convergence.append(npl.norm(np.diag(np.diag(S)) - S[:, :N]))
	if graph_mode:
		return it, decomp_quality, convergence
	for i in range(min(N, M)):
		if S[i,i] < 0:
			S[:, i] = -S[:, i]
			U[:, i] = -U[:, i]
	return U, S, V

if __name__ == '__main__':
	if "graphs" in sys.argv:
		# toSVD
		# BD = np.array([[1, 0, 0, 0], [2, -3, 0, 0], [0, -10, 5, 0]])
		BD = np.zeros([5, 7])
		for i in range(len(BD)):
			for j in range(len(BD[i])):
				if j <= i+1 and j >= i:
					BD[i, j] = npr.random()*100
		it, decomp_quality, convergence = toSVD(BD, 1000, qr_decomposition, True)
		mp.xlabel("Nombre d'iterations")
		mp.ylabel("Egalite B = U*S*V")
		mp.title("Verification de l'egalite BD = U*S*V")
		mp.plot(it, decomp_quality)
		mp.show()
		it, decomp_quality, convergence = toSVD(BD, 15, qr_decomposition_bidiag, True)
		mp.xlabel("Nombre d'iterations")
		mp.ylabel("Distance de S aux matrices diagonales")
		mp.title("Convergence de S vers une matrice diagonale")
		mp.plot(it[1:], convergence[1:])
		mp.show()

		mp.clf()

		# qr_decomp
		BD = np.zeros([10, 12])
		for i in range(len(BD)):
			for j in range(len(BD[i])):
				BD[i, j] = npr.random()*10
		it, decomp_quality = qr_decomposition(BD, 'complete', True)
		mp.xlabel("Nombre d'iterations")
		mp.ylabel("Egalite A = Q*R")
		mp.title("Verification de l'egalite A = Q*R")
		mp.plot(it, decomp_quality)
		mp.show()
	else:
		print "Usage: ./" + os.path.basename(__file__) + " graphs"


# Supplementary classes and functions for ENGSCI233 notebook quality_control.ipynb
# author: David Dempsey

# module imports
import numpy as np

def harmonic_mean(xs):
	''' Compute the harmonic mean of XS

		Parameters:
		-----------
		xs : array-like
			list of values

		Returns:
		--------
		xharm : float
			harmonic mean

		Notes:
		------
		xs cannot be empty or contain zero values

	'''
	# check preconditions
	assert not any([xi == 0. for xi in xs]), 'cannot have zero-values... idiot'
	assert len(xs) > 0, 'xs has zero-length'

	# compute harmonic mean
	xharm = 1./(np.sum([1./xi for xi in xs]))

	# check number is defined
	if xharm != xharm:
		raise ValueError('Computation returns NaN')

	return xharm

def geometric_mean(xs):
	''' Compute the geometric mean of XS

		Parameters:
		-----------
		xs : array-like
			list of values

		Returns:
		--------
		xgeom : float
			geometric mean

		Notes:
		------
		xs cannot be empty

	'''
	# check preconditions
	assert len(xs) > 0, 'xs has zero-length'

	# compute geometric mean
	xgeom = np.exp(np.mean([np.log(xi) for xi in xs]))

	# check number is defined
	if xgeom != xgeom:
		raise ValueError('Computation returns NaN')

	return xgeom


import pandas as pd
import numpy as np
import matplotlib.pyplot as plt

from sklearn.datasets import (load_boston,
                              load_iris,
                              load_digits,
                              load_diabetes,
                              load_wine,
                              load_breast_cancer,
                              fetch_olivetti_faces,
                              make_friedman1,
                              make_classification,
                              make_blobs)

from sklearn.cluster import (KMeans,
                             MiniBatchKMeans,
                             DBSCAN,
                             FeatureAgglomeration,
                             AgglomerativeClustering,
                             AffinityPropagation,
                             MeanShift,
                             estimate_bandwidth,

Birch,
SpectralBiclustering,
SpectralClustering,
SpectralCoclustering,
affinity_propagation,
affinity_propagation_,
bicluster,
estimate_bandwidth,
get_bin_seeds,
hierarchical?,
linkage_tree?,
ward_tree?)

from sklearn.decomposition import (PCA,
                                   IncrementalPCA,
                                   FactorAnalysis,
                                   FastICA,
                                   KernelPCA,
                                   DictionaryLearning,
                                   LatentDirichletAllocation,
                                   MiniBatchDictionaryLearning,
                                   MiniBatchSparsePCA,
                                   NMF,
                                   PCA,
                                   RandomizedPCA,
                                   SparseCoder,
                                   SparsePCA,
                                   TruncatedSVD,)

"""
K-Means needs implicit assumption around data, otherwise, leads to
underdesirable clusters
- incorrect k
- unequal variance
- Anisotropy distribution
    property of being directionally dependent, which implies different
    properties in different directions, as opposed to isotropy.
    Isotropy is uniformity in all orientations
- unevenly distributed clusters (majority vs minority, so long segregated it works)

make_blobs?
Signature: make_blobs(n_samples=100, n_features=2, centers=3, cluster_std=1.0,
                      center_box=(-10.0, 10.0), shuffle=True, random_state=None)
Docstring:
Generate isotropic Gaussian blobs for clustering.

KMeans?
Init signature: KMeans(n_clusters=8, init='k-means++', n_init=10, max_iter=300,
                       tol=0.0001, precompute_distances='auto', verbose=0,
                       random_state=None, copy_x=True, n_jobs=1, algorithm='auto')

Parameters
----------

n_clusters : int, optional, default: 8
    The number of clusters to form as well as the number of
    centroids to generate.

init : {'k-means++', 'random' or an ndarray}
    Method for initialization, defaults to 'k-means++':

    'k-means++' : selects initial cluster centers for k-mean
    clustering in a smart way to speed up convergence. See section
    Notes in k_init for more details.

    'random': choose k observations (rows) at random from data for
    the initial centroids.

    If an ndarray is passed, it should be of shape (n_clusters, n_features)
    and gives the initial centers.

n_init : int, default: 10
    Number of time the k-means algorithm will be run with different
    centroid seeds. The final results will be the best output of
    n_init consecutive runs in terms of inertia.

max_iter : int, default: 300
    Maximum number of iterations of the k-means algorithm for a
    single run.

tol : float, default: 1e-4
    Relative tolerance with regards to inertia to declare convergence

precompute_distances : {'auto', True, False}
    Precompute distances (faster but takes more memory).

    'auto' : do not precompute distances if n_samples * n_clusters > 12
    million. This corresponds to about 100MB overhead per job using
    double precision.

    True : always precompute distances

    False : never precompute distances

algorithm : "auto", "full" or "elkan", default="auto"
    K-means algorithm to use. The classical EM-style algorithm is "full".
    The "elkan" variation is more efficient by using the triangle
    inequality, but currently doesn't support sparse data. "auto" chooses
    "elkan" for dense data and "full" for sparse data.

"""
plt.figure(figsize=(12, 12))

n_samples = 1500
random_state = 170
X, y = make_blobs(n_samples=n_samples, random_state=random_state, centers=3)

# Incorrect number of clusters
y_pred = KMeans(n_clusters=2, # <========
                random_state=random_state).fit_predict(X)

plt.subplot(221) # 221 means 2x2 grid 1st plot ==> plt.subplot(2,2,1)
plt.scatter(X[:, 0], X[:, 1], c=y_pred)
plt.title("Incorrect Number of Blobs")

# Anisotropicly distributed data
transformation = [[0.60834549, -0.63667341], [-0.40887718, 0.85253229]]# <========
X_aniso = np.dot(X, transformation)
y_pred = KMeans(n_clusters=3, random_state=random_state).fit_predict(X_aniso)

plt.subplot(222)
plt.scatter(X_aniso[:, 0], X_aniso[:, 1], c=y_pred)
plt.title("Anisotropicly Distributed Blobs")

# Different variance
X_varied, y_varied = make_blobs(n_samples=n_samples,
                                cluster_std=[1.0, 2.5, 0.5], # <========
                                random_state=random_state)
y_pred = KMeans(n_clusters=3, random_state=random_state).fit_predict(X_varied)

plt.subplot(223)
plt.scatter(X_varied[:, 0], X_varied[:, 1], c=y_pred)
plt.title("Unequal Variance")

# Unevenly sized blobs
X_filtered = np.vstack((X[y == 0][:500], X[y == 1][:100], X[y == 2][:10]))
y_pred = KMeans(n_clusters=3,
                random_state=random_state).fit_predict(X_filtered)

plt.subplot(224)
plt.scatter(X_filtered[:, 0], X_filtered[:, 1], c=y_pred)
plt.title("Unevenly Sized Blobs")

plt.show()

X,y = make_blobs(n_samples=3000, centers=5, n_features=2,
                 cluster_std=[1,1,1,1,1])
kmeans = KMeans(n_clusters=5).fit(X)

pca  = PCA(n_components = len(set(y)) ).fit(X)
pca.components_
pca.explained_variance_
pca.explained_variance_ratio_
kmeans = KMeans(n_clusters=5, init=pca.components_).fit(X)

kmeans.labels_
kmeans.cluster_centers_
plt.scatter(X[:,0],X[:,1],c=kmeans.fit_predict(X))
plt.scatter(X[:,0],X[:,1],c=y) # real label

"""
MiniBatchKMeans?
Init signature: MiniBatchKMeans(n_clusters=8, init='k-means++', max_iter=100,
                                batch_size=100, verbose=0, compute_labels=True,
                                random_state=None, tol=0.0, max_no_improvement=10,
                                init_size=None, n_init=3, reassignment_ratio=0.01)
MiniBatchKMeans
    Alternative online implementation that does incremental updates
    of the centers positions using mini-batches.
    For large scale learning (say n_samples > 10k) MiniBatchKMeans is
    probably much faster than the default batch implementation.

Parameters
----------

batch_size : int, optional, default: 100
    Size of the mini batches.

tol : float, default: 0.0
    Control early stopping based on the relative center changes as
    measured by a smoothed, variance-normalized of the mean center
    squared position changes. This early stopping heuristics is
    closer to the one used for the batch variant of the algorithms
    but induces a slight computational and memory overhead over the
    inertia heuristic.

    To disable convergence detection based on normalized center
    change, set tol to 0.0 (default).

max_no_improvement : int, default: 10
    Control early stopping based on the consecutive number of mini
    batches that does not yield an improvement on the smoothed inertia.

    To disable convergence detection based on inertia, set
    max_no_improvement to None.

init_size : int, optional, default: 3 * batch_size
    Number of samples to randomly sample for speeding up the
    initialization (sometimes at the expense of accuracy): the
    only algorithm is initialized by running a batch KMeans on a
    random subset of the data. This needs to be larger than n_clusters.

n_init : int, default=3
    Number of random initializations that are tried.
    In contrast to KMeans, the algorithm is only run once, using the
    best of the ``n_init`` initializations as measured by inertia.

reassignment_ratio : float, default: 0.01
    Control the fraction of the maximum number of counts for a
    center to be reassigned. A higher value means that low count
    centers are more easily reassigned, which means that the
    model will take longer to converge, but should converge in a
    better clustering.

"""
X,y = make_blobs(n_samples=3000, centers=5, n_features=2,
                 cluster_std=[1,1,1,1,1], random_state=1)
kmeans = MiniBatchKMeans(n_clusters=5, batch_size=300,
                         max_iter=500, max_no_improvement=100,
                         verbose=1).fit(X)

kmeans.labels_
kmeans.cluster_centers_

plt.figure(figsize=(24,12))
plt.subplot(121)
plt.scatter(X[:,0],X[:,1],c=kmeans.fit_predict(X))
plt.subplot(122)
plt.scatter(X[:,0],X[:,1],c=y) # real label
plt.show()

"""
AffinityPropagation?
Similar to DBSCAN. Group points based on proximity.
it chooses the number of clusters based on the data provided.
suitable for small - mid sized data

two important parameters are the preference, which controls how
many exemplars are used, and the damping factor which damps the
responsibility and availability messages to
avoid numerical oscillations when updating these messages.

Drawback: high time complexity n_samples^2 * n_iter
            high memory complexity, esp for dense matrix

signature: AffinityPropagation(damping=0.5, max_iter=200, convergence_iter=15,
                               copy=True, preference=None, affinity='euclidean',
                               verbose=False)

Parameters
----------
damping : float, optional, default: 0.5
    Damping factor (between 0.5 and 1) is the extent to
    which the current value is maintained relative to
    incoming values (weighted 1 - damping). This in order
    to avoid numerical oscillations when updating these
    values (messages).

max_iter : int, optional, default: 200
    Maximum number of iterations.

convergence_iter : int, optional, default: 15
    Number of iterations with no change in the number
    of estimated clusters that stops the convergence.

copy : boolean, optional, default: True
    Make a copy of input data.

preference : array-like, shape (n_samples,) or float, optional
    Preferences for each point - points with larger values of
    preferences are more likely to be chosen as exemplars. The number
    of exemplars, ie of clusters, is influenced by the input
    preferences value. If the preferences are not passed as arguments,
    they will be set to the median of the input similarities.

affinity : string, optional, default=``euclidean``
    Which affinity to use. At the moment ``precomputed`` and
    ``euclidean`` are supported. ``euclidean`` uses the
    negative squared euclidean distance between points.

verbose : boolean, optional, default: False
    Whether to be verbose.

Attributes
----------
cluster_centers_indices_ : array, shape (n_clusters,)
    Indices of cluster centers

cluster_centers_ : array, shape (n_clusters, n_features)
    Cluster centers (if affinity != ``precomputed``).

labels_ : array, shape (n_samples,)
    Labels of each point

affinity_matrix_ : array, shape (n_samples, n_samples)
    Stores the affinity matrix used in ``fit``.

n_iter_ : int
    Number of iterations taken to converge.

"""

"""
MeanShift

- works well with tight clusters, small std dev

"""

# #############################################################################
# Generate sample data
X, y = make_blobs(n_samples=10000, centers=5,
                  cluster_std=0.6, random_state=1)

# #############################################################################
# Compute clustering with MeanShift

# The following bandwidth can be automatically detected using
bandwidth = estimate_bandwidth(X, quantile=0.2, n_samples=500)

ms = MeanShift(bandwidth=bandwidth, bin_seeding=True)
ms.fit(X)
labels = ms.labels_
cluster_centers = ms.cluster_centers_

labels_unique = np.unique(labels)
n_clusters_ = len(labels_unique)

print("number of estimated clusters : %d" % n_clusters_)

# #############################################################################
# Plot result
import matplotlib.pyplot as plt
from itertools import cycle

plt.figure(1)
plt.clf()

colors = cycle('bgrcmykbgrcmykbgrcmykbgrcmyk')
for k, col in zip(range(n_clusters_), colors):
    my_members = labels == k
    cluster_center = cluster_centers[k]
    plt.plot(X[my_members, 0], X[my_members, 1], col + '.')
    plt.plot(cluster_center[0], cluster_center[1], 'o', markerfacecolor=col,
             markeredgecolor='k', markersize=14)
plt.title('Estimated number of clusters: %d' % n_clusters_)
plt.show()


import math

def pipe_Do_Di_WT(*, Do=None, Di=None, WT=None):
    """Calculate pipe wall thickness / outer diameter / inner diameter.
    """
    if Do is not None and Di is not None and WT is not None:
        assert Do==Di+2*WT, f"pipe_D_WT: inconsistent pipe dimensions Do={Do} Di={Di} WT={WT}."
    elif WT is None:
        WT = (Do - Di) / 2
    elif Di is None:
        Di = Do - 2 * WT
    elif Do is None:
        Do = Di + 2 * WT
    else:
        return False
    return Do, Di, WT

def pipe_Do(Di, WT):
    """Calculate pipe outer diameter, given the pipe inner diamater and wall thickness.
    """
    Do = Di + 2 * WT
    return Do

def pipe_Di(Do, WT):
    """Calculate pipe inner diameter, given the pipe outer diamater and wall thickness.
    """
    Di = Do - 2 * WT
    return Di

def pipe_WT(Do, Di):
    """Calculate pipe wall thickness, given the pipe outer diamater and inner diameter.
    """
    WT = (Do - Di) / 2
    return WT

def pipe_CSA(Do, Di):
    """Calculate pipe cross sectional area, given the pipe outer diamater and inner diameter.

    :param Do: pipe outer diameter :math:`(D_o)`
    :type p_d: float
    :param Di: pipe inner diameter :math:`(D_i)`
    :type p_d: float
    :returns: pipe cross sectional area
    :rtype: float

    .. math::
        A =  \frac{\pi}{4} \left( D_o^2 - D_i^2   \right)

    .. doctest::

        >>> pipe_CSA(0.6656, 0.6172)
        0.19522098526377624
    """
    CSA = math.pi * (Do*Do - Di*Di)
    return CSA

# ================================================================

#import inspect
import logging

import numpy as np

# __all__ = [
#     "pipe_dimensions",
#     "pipe_weight",
# ]

logger = logging.getLogger(__name__)

def calc_pipe_Di(Do, WT) -> "Di":
    """Calculate pipe inner diameter from Do and WT.
    """
    return Do - 2*WT

def pipe_D_WT(*, Do=None, Di=None, WT=None, _round=False) -> "('Do', 'Di', 'WT')":
    """Resolve pipe radial dimensions. Given two of three dimensional arguments
    outer diameter Do; inner diameter Di; wall thickness WT;
    calculate the missing dimension and return a tuple (Do, Di, WT).

    If all arguments are specified a consistency check is carried
    out, and an AssertionError is raised if the dimensions are inconsistent.

    :param Do: pipe outer diameter :math:`(D_o)`
    :type p_d: float
    :param Di: pipe inner diameter :math:`(D_i)`
    :type p_d: float
    :param WT: pipe wall thickness :math:`(WT)`
    :type WT: float
    :returns: outer diameter, inner diameter, wall thickness
    :rtype: tuple

    .. doctest::

        >>> lpipe_D_WT(Do=0.660, WT=0.05)
        (0.66, 0.56, 0.05)
    """
    if _round and isinstance(_round, (int, bool)):
        Do = round(Do, _round)
        Di = round(Di, _round)
        WT = round(WT, _round)
    if all([isinstance(x, (float, int)) for x in [Do, Di, WT]]):
        try:
            assert Di==Do-2*WT, f"inconsistent pipe dimensions Do={Do} Di={Di}, WT={WT}."
            #return True
        except AssertionError as err:
            logger.error("lpipe_D_WT:  %s" % (err,))
            return False
    elif Do is None and all([isinstance(x, (float, int)) for x in [Di, WT]]):
        Do = Di + 2*WT
    elif Di is None and all([isinstance(x, (float, int)) for x in [Do, WT]]):
        Di = Do - 2*WT
    elif WT is None and all([isinstance(x, (float, int)) for x in [Do, Di]]):
        WT = (Do - Di)/2
    else:
        logger.error("pipe_D_WT: arguments incorrectly specified: Do=%s, Di=%s, WT=%s" % (Do,Di,WT))
    return Do, Di, WT

def pipe_CSA(*, Do=None, Di=None, WT=None):
    """Calculate the area of a circular ring,
    by specifying at least
    two of the three dimensional arguments outer diameter Do,
    inner diameter Di, thickness t.

    :param Do: pipe outer diameter :math:`(D_o)`
    :type p_d: float
    :param Di: pipe inner diameter :math:`(D_i)`
    :type p_d: float
    :param WT: pipe wall thickness :math:`(WT)`
    :type WT: float
    :returns: pipe cross sectional area
    :rtype: float

    .. math::
        A =  \frac{\pi}{4} \left( D_o^2 - D_i^2   \right)

    .. doctest::

        >>> pipe_CSA(Do=0.660, WT=0.05)
        0.09581857593448868
    """
    _Do, _Di, _WT = pipe_D_WT(Do=Do, Di=Di, WT=WT)
    #_csa = np.pi/4 * (_Do**2 - _Di**2)
    _csa = np.pi/4 * (np.power(_Do, 2) - np.power(_Di, 2))
    return _csa

def pipe_mass(ρ, *, Do=None, Di=None, WT=None, CSA=None, length=None):
    """Calculate pipe mass (per unit length)
    """
    if CSA is None:
        CSA = pipe_CSA(Do=Do, Di=Di, WT=WT)
    mass = CSA * ρ
    if length:
        mass *= length
    return mass

def pipe_layers_equivalent(layers, *, Di_ref=None, Do_ref=None):
    """Calculate equivalent properties for pipe coating layers.
    """
    if len([n for n in (Di_ref, Do_ref) if n is None]) != 1:
        logger.error("pipe_layers_equivalent: arguments incorrectly specified: Di_ref=%s, Do_ref=%s" % (Di_ref, Do_ref))
        raise ValueError(f"arguments not correctly specified: Di_ref={Di_ref}, Do_ref={Do_ref}")
    _do = _di = Di_ref if Di_ref else Do_ref
    mass_total = 0
    wt_total = 0
    csa_total = 0
    for (wt, ρ, *args) in layers:
        if Di_ref:
            layer_Di = _do
            _do = layer_Do = _do + (2 * wt)
        else:
            layer_Do = _di
            _di = layer_Do = _di - (2 * wt)
        wt_total += wt
        layer_csa = pipe_CSA(Do=layer_Do, Di=layer_Di)
        #_mass = _csa * ρ
        mass_total += layer_csa * ρ
        csa_total += layer_csa
    ρ_equiv = mass_total / csa_total
    return {
        "Di": _di,
        "Do": _do,
        "ρ": ρ_equiv,
        "mass": mass_total
    }

class Cylinder:

    def __init__(self, length=None, Do=None, Di=None, WT=None):
        (self.Do, self.Di, self.WT) = pipe_D_WT(Do=Do, Di=Di, WT=WT)
        self.CSA = pipe_CSA(Do=self.Do, WT=self.WT)
        if length:
            self.volume = self.CSA * length

class LinePipe(Cylinder):

    def __init__(self, length, ρ, Do=None, Di=None, WT=None,
        coat_layers=None):
        super().__init__(length=length, Do=Do, Di=Di, WT=WT)
        #(self.pipe_Do, self.pipe_Di, self.pipe_WT) = pipe_D_WT(Do=Do, Di=Di, WT=WT)
        self.mass = self.volume * ρ
        self.Doa = self.Do
        self.status = {
            "external_coat": False,
            "internal_coat": False,
            "internal_clad": False,
            "internal_liner": False,
        }
        if coat_layers:
            self.external_coat = {
                "layers": coat_layers,
                "equivalent": pipe_layers_equivalent(coat_layers,
                    Di_ref=self.Do)
            }
            self.Doa = self.external_coat["equivalent"]["Do"]
            self.status["external_coat"] = True

# old versions of functions below -----------------------------------

def pipe_dimensions(D_o=None, D_i=None, WT=None) -> "(D_o, D_i, WT)":
    # if funcname is None:
    #     funcname = inspect.currentframe().f_code.co_name
    if all([isinstance(x, (float, int)) for x in [D_o, D_i, WT]]):
        assert D_i==D_o-2*WT, f"pipe_basic_dims: inconsistent pipe dimensions D_o={D_o} D_i={D_i}, WT={WT}."
    elif D_o is None and all([isinstance(x, (float, int)) for x in [D_i, WT]]):
        D_o = D_i + 2*WT
    elif D_i is None and all([isinstance(x, (float, int)) for x in [D_o, WT]]):
        D_i = D_o - 2*WT
    elif WT is None and all([isinstance(x, (float, int)) for x in [D_o, D_i]]):
        WT = (D_o - D_i)/2
    else:
        logger.error("pipe_basic_dims: incorrectly specified arg: D_o=%s,D_i=%s,WT=%s" % (D_o,D_i,WT))
        return False
    return D_o, D_i, WT

def pipe_cross_sectional_area(D_o=None, D_i=None, WT=None) -> "pipe_CSA":
    D_o, D_i, WT = pipe_dimensions(D_o, D_i, WT)
    _CSA = np.pi/4 * (D_o**2 - D_i**2)
    return _CSA

def pipe_physical_props(rho_pipe, Do=None, Di=None, WT=None, length=None, rho_cont=None,
        coating=None, clad=None, liner=None, rho_water=None, g=9.81) -> "{}":
    Do, Di, WT = pipe_dimensions(Do, Di, WT)
    _CSA = pipe_cross_sectional_area(Do, Di, WT)
    _mass_len = _CSA * rho_pipe
    pp = {
        "pipe_Do": Do,
        "pipe_Di": Di,
        "pipe_WT": WT,
        "pipe_CSA": _CSA,
        "pipe_mass": _mass_len,
        "pj_mass": _mass_len,
        "D_buoy": Do,
        "D_cont": Di,
    }
    # if length:
    #     pp["pipe_len"] = length
    #     pp["pipe_mass"] = _mass_len * length
    if coating is not None:
        _coat_Di = pp["pipe_Do"]
        _coat_mass = 0
        if not isinstance(coating[0], (tuple,list)):
            coating = [coating]
        for ii, (_thick, _density, *args) in enumerate(coating):
            _Do, _Di, _thick = pipe_dimensions(None, _coat_Di, _thick)
            pp[f"coat{ii}_Do"] = _Do
            pp[f"coat{ii}_WT"] = _thick
            pp[f"coat{ii}_mass"] = np.pi/4 * (_Do**2 - _Di**2) * _density
            _coat_mass += pp[f"coat{ii}_mass"]
            _coat_Di +=  _thick * 2
        _mass_len += _coat_mass
        pp["coat_Do"] = _Do
        pp["coat_WT"] = (_Do - pp["pipe_Do"]) / 2
        pp["coat_mass"] = _coat_mass
        pp["coat_rho_eqv"] = _coat_mass / (np.pi/4 * (_Do**2 - pp["pipe_Do"]**2) )
        pp["D_buoy"] = _Do
        pp["pj_mass"] += _coat_mass
    if clad is not None:
        _clad_Do = pp["pipe_Di"]
        _clad_mass = 0
        if not isinstance(clad[0], (tuple,list)):
            clad = [clad]
        for ii, (_thick, _density, *args) in enumerate(clad):
            _Do, _Di, _thick = pipe_dimensions(_clad_Do, None, _thick)
            pp[f"clad{ii}_Di"] = _Di
            pp[f"clad{ii}_WT"] = _thick
            pp[f"clad{ii}_mass"] = np.pi/4 * (_Do**2 - _Di**2) * _density
            _clad_mass += pp[f"clad{ii}_mass"]
            _clad_Do -=  _thick * 2
        pp["D_cont"] = _Di
        pp["pj_mass"] += _clad_mass
    if liner is not None:
        _liner_Do = pp["D_cont"]
        _liner_mass = 0
        if not isinstance(liner[0], (tuple,list)):
            liner = [liner]
        for _thick, _density, *args in liner:
            _Do, _Di, _thick = pipe_dimensions(_liner_Do, None, _thick)
            pp[f"liner{ii}_Di"] = _Di
            pp[f"liner{ii}_WT"] = _thick
            pp[f"liner{ii}_mass"] = np.pi/4 * (_Do**2 - _Di**2) * _density
            _liner_mass += pp[f"liner{ii}_mass"]
            _liner_Do -=  _thick * 2
        pp["D_cont"] = _Di
        pp["pj_mass"] += _liner_mass
    if rho_cont is not None:
        pp["pj_cont_mass"] = pp["pj_mass"] + np.pi/4 * pp["D_cont"]**2 * rho_cont
    if rho_water is not None:
        pp["pj_flood_mass"] = pp["pj_mass"] + np.pi/4 * pp["D_cont"]**2 * rho_water
        pp["pj_buoyf"] = rho_water * g * np.pi/4 * pp["D_buoy"]**2
        pp["pj_subw"] = pp["pj_mass"] * g - pp["pj_buoyf"]
        if rho_cont is not None:
            pp["pj_cont_subw"] = pp["pj_cont_mass"] * g - pp["pj_buoyf"]
        pp["pj_flood_subw"] = pp["pj_flood_mass"] * g - pp["pj_buoyf"]
    return pp

#def pipe_weight(rho_pipe, D_o=None, D_i=None, WT=None, length=None, rho_cont=None, D_buoy=None):

if __name__ == "__main__":
    parameters = {
        "Do": 0.3229,
        "WT": 0.0185,
        "rho_pipe": 7850,
        "clad": (0.003, 8000),
        "coating": [
            (0.0003, 1300),
            (0.0003, 900),
            (0.0094, 900),
            (0.031, 620),
            (0.003, 900),
            (0.031, 620),
            (0.004, 900),
        ],
        "rho_water": 1027,
        "rho_cont": np.array([10, 860]),
    }
    pp = pipe_physical_props(**parameters)


import numpy as np
import matplotlib.pyplot as plt

def reward(prev_state, next_state):
  if 100 <= next_state:
    return 1.0
  else:
    return 0.0

def compute_value(state, action, state_values, transition_probability):
  gamma = 1.0
  pnext = state + action
  nnext = state - action
  assert 0 <= nnext and pnext <= 100

  return transition_probability * (reward(state, pnext) + gamma * state_values[pnext]) \
      + (1.0 - transition_probability) * (reward(state, nnext) + gamma * state_values[nnext])

def value_iteration(transition_probability):
  state_values = np.zeros(101)
  threshold = 1e-20
  max_sweep = 32
  sweep = 0

  while True:
    delta = 0.0
    for state in range(len(state_values)):
      if state == 0 or state == 100:
        continue
      max_state_value = 0.0
      for action in range(min(state, 100 - state)):
        action = action + 1
        # print("state: " + str(state) + " action: " + str(action))
        state_value = compute_value(state, action, state_values, transition_probability)
        if max_state_value < state_value:
          max_state_value = state_value
      prev_value = state_values[state]
      state_values[state] = max_state_value
      delta = max(abs(prev_value - max_state_value), delta)
    sweep = sweep + 1
    if max_sweep < sweep:
      break
  return state_values

def compute_policy(state_values, transition_probability):
  policy = np.zeros(len(state_values))
  for state in range(len(state_values)):
    if state == 0 or state == 100:
        continue
    max_state_value = 0.0
    for action in range(min(state, 100 - state)):
      action = action + 1
      state_value = compute_value(state, action, state_values, transition_probability)
      if 1e-5 < abs(max_state_value - state_value) and max_state_value < state_value:
        max_state_value = state_value
        policy[state] = action
  return policy

def state_values_and_policy_for(transition_probability):
  state_values = value_iteration(transition_probability)
  policy = compute_policy(state_values, transition_probability)
  return state_values, policy

def main():
  probabilities = [ 0.4, 0.25, 0.55 ]
  value_function = []
  policy_function = []

  for probability in probabilities:
    values, policy = state_values_and_policy_for(probability)
    value_function.append(values)
    policy_function.append(policy)

  index = 1
  rows = 2
  columns = len(probabilities)
  for probability, values, policy in zip(probabilities, value_function, policy_function):
    # Value function
    plt.subplot(rows, columns, index)
    plt.plot(range(1, len(values) - 1), values[1:100])
    plt.title("value_func prob: " + str(probability))

    # Policy function
    plt.subplot(rows, columns, index + columns)
    plt.step(range(1, len(policy) - 1), policy[1:100])
    plt.title("policy_func prob: " + str(probability))

    index += 1

  plt.show()

if __name__=='__main__':
  main()


import sys
import numpy as np
from sklearn.cluster import KMeans
from sklearn import manifold
from time import time
from matplotlib import offsetbox
import matplotlib.pyplot as plt
import seaborn as sns
#### plot embedding

def plot_embedding(X, label, title=None):
    x_min, x_max = np.min(X, 0), np.max(X, 0)
    X = (X - x_min) / (x_max - x_min)
    plt.figure()
    ax = plt.subplot(111)
    for i in range(X.shape[0]):
        plt.text(X[i, 0], X[i, 1], str(label[i]),
                 color=plt.cm.Set1((label[i]+2) / 10.),
                 fontdict={'weight': 'bold', 'size': 9})
    if hasattr(offsetbox, 'AnnotationBbox'):
        # only print thumbnails with matplotlib > 1.0
        shown_images = np.array([[1., 1.]])  # just something big
        for i in range(X.shape[0]):
            dist = np.sum((X[i] - shown_images) ** 2, 1)
            if np.min(dist) < 4e-3:
                # don't show points that are too close
                continue
            shown_images = np.r_[shown_images, [X[i]]]

    plt.xticks([]), plt.yticks([])
    if title is not None:
        plt.title(title)

## Data loader
np.set_printoptions(threshold=sys.maxsize)
# Option
mode = 'Training'
num_cluster =10
eps = 1e-10
height = 125
width = 110
channel = 1
sampleN = 4123 #746
spec = np.genfromtxt('spec.csv',delimiter=' ')
spec_train = np.reshape(spec, (-1, 1, height, width))
label = np.genfromtxt('fake_label.csv',delimiter=' ')
train_labels = np.asarray(label, dtype=np.int32)
image_data = spec_train
image_label = train_labels

##### normalize

print('min:', np.min(image_data[84,::]), 'max:', np.max(image_data[84,::]))
for i in range(0, sampleN):
  current_min = np.min(image_data[i,::])
  current_max = np.max(image_data[i,::])
  image_data[i,::] = (current_max - image_data[i,::]) / (current_max - current_min)

#### plot ion image
print(np.shape(image_data))
plt.imshow(np.reshape(image_data[84,::],(125,110)))

############K-means

X=np.reshape(image_data, (-1, height*width))

kmeans = KMeans(n_clusters=10, init='k-means++', max_iter=500, n_init=10, random_state=2)
km_labels = kmeans.fit_predict(X)

########### t-sne embeddings
tsne = manifold.TSNE(n_components=2, init='pca', random_state=0)
t0 = time()
X_tsne = tsne.fit_transform(X)
print('plot embedding')
plot_embedding(X_tsne, label = km_labels,
               title = "t-SNE embedding of the digits (time %.2fs)" %
               (time() - t0))
plt.show()

plt.figure(figsize=(5,3.5))
sns.scatterplot(
    x=X_tsne[:,0], y=X_tsne[:,1],
    hue=km_labels,
    palette=sns.color_palette("hls", num_cluster),#3
    legend="full"
)
plt.xlabel('TSNE 1')
plt.ylabel('TSNE 2')


import numpy as np

def feature_normalize(X):
    # You need to set these values correctly
    n = X.shape[1]  # the number of features
    X_norm = X
    mu = np.zeros(n)
    sigma = np.zeros(n)

    # ===================== Your Code Here =====================
    # Instructions : First, for each feature dimension, compute the mean
    #                of the feature and subtract it from the dataset,
    #                storing the mean value in mu. Next, compute the
    #                standard deviation of each feature and divide
    #                each feature by its standard deviation, storing
    #                the standard deviation in sigma
    #
    #                Note that X is a 2D array where each column is a
    #                feature and each row is an example. You need
    #                to perform the normalization separately for
    #                each feature.
    #
    # Hint: You might find the 'np.mean' and 'np.std' functions useful.
    #       To get the same result as Octave 'std', use np.std(X, 0, ddof=1)
    #

    mu = np.mean(X,axis=0)
    sigma = np.std(X, axis=0, ddof=1)

    X_norm = X_norm - mu

    X_norm = X_norm / sigma

    # ===========================================================

    return X_norm, mu, sigma


import tensorflow as tf
import matplotlib.pyplot as plt
import numpy as np
import sys
from sklearn import linear_model

def scale(t):
    tMin = t.min(axis=0)
    tMax = t.max(axis=0)
    return (t-tMin)/(tMax-tMin)

class Problem1Base():
    def __init__(self):
        tf.reset_default_graph()

    def create_dataset(self):
        n_samples = 30
        self.x_point = np.linspace(0,20,n_samples)

        self.y_point = 3.7 * self.x_point + 14 + 4 * np.random.randn(n_samples)

        plt.plot(self.x_point, self.y_point,'o')
        plt.savefig("lesson5.png")

class Problem1SK(Problem1Base):
    def __init__(self):
        self.RANDOM_SEED = 42
        np.random.seed(self.RANDOM_SEED)

    def compute_regression(self):
        linreg = linear_model.LinearRegression()
        self.x_point = np.reshape(self.x_point,(len(self.x_point),1))
        linreg.fit(self.x_point,self.y_point,self.y_point)
        return linreg.coef_,linreg.intercept_

class Problem1TF(Problem1Base):
    def __init__(self):
        self.graph = tf.Graph()
        self.RANDOM_SEED = 42
        tf.set_random_seed(self.RANDOM_SEED)
        np.random.seed(self.RANDOM_SEED)

    def build(self):
        self.X = tf.placeholder(tf.float32)
        self.Y = tf.placeholder(tf.float32)
        self.LR = tf.placeholder(tf.float32)
        self.slope = tf.Variable(tf.random_uniform([1],-1.0,1.0))
        self.intercept = tf.Variable(tf.zeros([1]))
        self.response = self.slope*self.X + self.intercept
        self.cost = tf.reduce_mean(tf.square(self.response - self.Y))
        self.optimizer = tf.train.GradientDescentOptimizer(self.LR).minimize(self.cost)

    def compute_regression(self,epochs,learning_rate):
        init = tf.global_variables_initializer()
        with tf.Session() as session:
            session.run(init)
            for epoch in range(epochs):
                session.run(
                        self.optimizer,feed_dict = {
                            self.LR: learning_rate,
                            self.X : self.x_point,
                            self.Y : self.y_point})
                if (epoch % (epochs/20)) == 0:
                    c = session.run(
                            self.cost, feed_dict = {
                                self.LR: learning_rate,
                                self.X : self.x_point,
                                self.Y : self.y_point})
                    print("Epoch:", epoch , "Cost:", c, "Slope:",
                            session.run(self.slope),
                            "Intercept:",
                            session.run(self.intercept))
            c = session.run(self.cost, feed_dict = {
                self.LR: learning_rate,
                self.X : self.x_point,
                self.Y : self.y_point})
            print("Epoch:", epoch , "Cost:", c, "Slope:",
                    session.run(self.slope),
                    "Intercept:",
                    session.run(self.intercept))
            return c,session.run(self.slope),session.run(self.intercept)

class Problem2Base():
    def __init__(self):
        tf.reset_default_graph()

    def create_dataset(self):
        f = open("00 kc_house_data.csv")
        f.readline()
        dataset = np.genfromtxt(fname = f, delimiter = ',',usecols=(2,3,5))
        self.predictors = dataset[:,1:3]
        predictorsMin = self.predictors.min(axis=0)
        predictorsMax = self.predictors.max(axis=0)
        self.predictors_scaled = scale(self.predictors)
        self.response = dataset[:,0:1]
        responseMin = self.response.min(axis=0)
        responseMax = self.response.max(axis=0)
        self.response_scaled = scale(self.response)

class Problem2SK(Problem2Base):
    def __init__(self):
        self.RANDOM_SEED = 42
        np.random.seed(self.RANDOM_SEED)

    def compute_regression(self):
        linreg = linear_model.LinearRegression()
        linreg.fit(self.predictors_scaled,self.response_scaled)
        return linreg.coef_,linreg.intercept_

class Problem2TF(Problem2Base):
    def __init__(self):
        self.RANDOM_SEED = 42
        tf.set_random_seed(self.RANDOM_SEED)
        np.random.seed(self.RANDOM_SEED)

    def build(self):
        self.X1 = tf.placeholder(tf.float32)
        self.X2 = tf.placeholder(tf.float32)
        self.Y = tf.placeholder(tf.float32)

        self.LR = tf.placeholder(tf.float32)

        self.slope1 = tf.Variable([0],dtype=tf.float32,name="weight1")
        self.slope2 = tf.Variable([0],dtype=tf.float32,name="weight2")
        self.intercept = tf.Variable(tf.zeros([1]),dtype=tf.float32,name="bias")
        self.response = self.slope1*self.X1 + self.slope2*self.X2 + self.intercept
        self.cost = tf.reduce_mean(tf.square(self.response - self.Y))
        self.optimizer = tf.train.GradientDescentOptimizer(self.LR).minimize(self.cost)

    def compute_regression(self,epochs,learning_rate):
        init = tf.global_variables_initializer()
        with tf.Session() as session:
            session.run(init)
            for epoch in range(epochs):
                session.run(
                        self.optimizer,feed_dict = {
                            self.LR: learning_rate,
                            self.X1:self.predictors_scaled[:,0:1],
                            self.X2:self.predictors_scaled[:,1:2],
                            self.Y:self.response_scaled})
                if not epoch % 100000:
                    c = session.run(
                            self.cost, feed_dict = {
                                self.LR:learning_rate,
                                self.X1:self.predictors_scaled[:,0:1],
                                self.X2:self.predictors_scaled[:,1:2],
                                self.Y:self.response_scaled})
                    print("Epoch:", epoch , "Cost:", c, "Slope1:",
                            session.run(self.slope1),
                            "Slope2:",
                            session.run(self.slope2),
                            "Intercept:",
                            session.run(self.intercept))
            c = session.run(
                    self.cost, feed_dict = {
                        self.LR: learning_rate,
                        self.X1 : self.predictors_scaled[:,0:1],
                        self.X2 : self.predictors_scaled[:,1:2],
                        self.Y : self.response_scaled} )
            print("Epoch:", epoch , "Cost:", c, "Slope1:",
                    session.run(self.slope1),
                    "Slope2:",
                    session.run(self.slope2),
                    "Intercept:",
                    session.run(self.intercept))

            return c,session.run(self.slope1),session.run(self.slope2),session.run(self.intercept)

def main():

    print("Problem 1 - SKLearn")
    print("====================")
    problem1 = Problem1SK()
    problem1.create_dataset()
    slope,intercept = problem1.compute_regression()
    print(slope,intercept)
    assert ("%.8f" % slope[0]) == "3.54942311"
    assert ("%.15f" % intercept) == "14.841075232789862"

    print("\nProblem 1, solution 1 - Tensorflow")
    print("====================================")
    print("   Epoch 50000, Learning rate 0.0001")
    problem1 = Problem1TF()
    problem1.create_dataset()
    problem1.build()
    cost,slope,intercept = problem1.compute_regression(50000,0.0001)
    print(cost,slope,intercept)
    assert ("%.5f" % cost) == "11.36642"
    assert ("%.7f" % slope[0]) == "3.5775077"
    assert ("%.6f" % intercept[0]) == "14.184714"

    print("\nProblem 1, solution 2 - Tensorflow")
    print("====================================")
    print("   Epoch 5000, Learning rate 0.001")
    problem1 = Problem1TF()
    problem1.create_dataset()
    problem1.build()
    cost,slope,intercept = problem1.compute_regression(5000,0.001)
    print(cost,slope,intercept)
    assert ("%.6f" % cost) == "11.365601"
    assert ("%.7f" % slope[0]) == "3.5774026"
    assert ("%.6f" % intercept[0]) == "14.186133"

    print("Problem 2 - SKLearn")
    print("====================")
    problem2 = Problem2SK()
    problem2.create_dataset()
    slope,intercept = problem2.compute_regression()
    print(slope,intercept)

    print("Problem 2 - Tensorflow")
    print("======================")
    print("   Epoch 1000000, Learning rate 0.01")
    problem2 = Problem2TF()
    problem2.create_dataset()
    problem2.build()
    cost,slope1,slope2,intercept = problem2.compute_regression(1000000,0.01)

if __name__ == "__main__":
    main()


import pandas as pd
import matplotlib.pyplot as plt
from statsmodels.graphics.tsaplots import plot_acf, plot_pacf
from statsmodels.graphics.gofplots import qqplot
from statsmodels.stats.diagnostic import acorr_ljungbox
from statsmodels.stats.stattools import jarque_bera

def evaluate_ts_data(data):
    """
    Function takes time series data and evaluates with various tests whether the data is
    normally distributed and checks for autocorrelation between given and lagged entries

    Tests for normality:
        * Q-Q plot
        * Jarque Bera test
    Test for autocorrelation:
        * Autocorrelation function
        * Partial autocorrelation function
        * Ljung-Box test

    """
    # Tests
    jb_score, jb_pvalue, jb_skew, jb_kurtosis = jarque_bera(data)
    jb_s = pd.DataFrame({'Score': [jb_score], 'p value': [jb_pvalue], "skew": [jb_skew], "kurtosis":[jb_kurtosis]})
    print("\nJarque Bera test (tests for normality of data)")
    print(jb_s)

    lb_result = acorr_ljungbox(data, lags=[30], boxpierce=False, return_df=True)
    print("\nLjung-Box test ( tests autocorrelation)")
    print(lb_result)

    # Plot graphs
    fig, ((ax1, ax2), (ax3, ax4)) = plt.subplots(2, 2, figsize=(10,5))

    plot_acf(data,ax=ax1)
    ax1.set_title('Autocorrelation function')

    plot_pacf(data,ax=ax2)
    ax2.set_title('Partial autocorrelation function')

    qqplot(data,ax=ax3,line="s")
    ax3.set_title('Q-Q plot')

    ax4.hist(data,bins=50)
    ax4.set_title('Data histogram')

    plt.tight_layout()


import numpy as np;
import scipy.linalg as spla;
from gauss_elimination import gauss_elimination;

def finite_difference(yhist, h, n=1, customge=False):
    '''
    Forward, Central, Backwards finite difference calculation of derivative
    This function uses the Central Finite Difference method to compute the
    derivative for the given data vector. At the ends of the array, central
    difference doesn't work, so the forward difference method and backwards
    difference methods are used instead.

    Note that this function decides for you to use forward and bakwards
    differencing functions at either end of the dataset. This cannot be
    turned off or changed.

    @arg
    yhist    - M x N numpy.ndarray
               Function value time history, where N is the length of the
               dataset and M is the number different things to take the
               derivative of. M is usually 1. N corresponds to the number
               of samples in the time history of the dataset.
    h        - double
               Time step
    n        - double (optional)
               Order of finite difference
    customge - bool (optional)
               Specify to use the custom gauss elimination function.
               True to use the function, False use numpy's built in function.
               False by default.

    @return
    ydothist - N x M numpy.ndarray
               Finite difference derivative time history

    @dependencies
    python 3.6.0
    numpy
    scipy

    @author: Matt Marti
    @date: 2019-06-14
    '''

    # Raise Type Errors on bad input
    if type(yhist) != np.ndarray:
        raise TypeError("Argument 'yhist' is not of type numpy.ndarray");
    #
    if type(h) not in [int, float]:
        raise TypeError("Argument 'h' is not of type [int, float]");
    elif type(h) == int:
        h = h + 0.0;
    #
    if type(n) != int:
        raise TypeError("Argument 'n' is not of type int");
    #

    # Check yhist shape
    assert n > 0, "Argument 'n' is not greater than zero";
    if yhist.shape.__len__() == 1:
        M = 1;
        yhistold = yhist.copy();
        yhist = np.ndarray((1, yhistold.shape[0]));
        for i in range(0, yhistold.shape[0]):
            yhist[0,i] = yhistold[i];
        #
    else:
        M = yhist.shape[0];
    #
    N = yhist.shape[1];
    assert N >= n+1, 'Not enough data points for given order';

    # Preallocate output
    ydothist = np.zeros(yhist.shape, np.float64);
    hmat = np.zeros((n,n), np.float64);

    # Forward difference method
    for i in range(0, n):

        # Delta t matrix
        for j in range(1, n+1):
            d = 1.0;
            hj = h*j;
            for k in range(1, n+1):
                d = d*k;
                hmat[j-1,k-1] = (hj**k)/d;
            #
        #

        # Forward finite difference
        k = 0;
        yfvec = np.ndarray((M, n));
        for j in range(i+1, i+n+1):
            yfvec[:,k] = yhist[:,j] - yhist[:,i];
            k += 1;
        #

        # Compute derivative
        if customge:
            ydoti = gauss_elimination(hmat, yfvec.transpose());
        else:
            ydoti = spla.solve(hmat, yfvec.transpose());
        #

        # Assign output
        for j in range(ydothist.shape[0]):
            ydothist[j,i] = ydoti[0,j];
        #
    #

    # Central difference method
    for i in range( n, N-n ):

        # Delta t matrix
        for j in range(1, n+1):
            d = 1.0;
            hj = h*j;
            for k in range(1, n+1):
                d = d*k;
                hmat[j-1,k-1] = (hj**k)/d;
            #
        #

        # Forward finite difference
        k = 0;
        yfvec = np.ndarray((M, n));
        for j in range(i+1, i+n+1):
            yfvec[:,k] = yhist[:,j] - yhist[:,i];
            k += 1;
        #

        # Backward finite difference
        k = 0;
        ybvec = np.ndarray((M, n));
        for j in range(i-1, i-n-1, -1):
            ybvec[:,k] = yhist[:,i] - yhist[:,j];
            k += 1;
        #

        # Compute derivative
        if customge:
            ydoti_f = gauss_elimination(hmat, yfvec.transpose());
            ydoti_b = gauss_elimination(hmat, ybvec.transpose());
            ydoti = 0.5 * (ydoti_f + ydoti_b);
        else:
            ydoti_f = spla.solve(hmat, yfvec.transpose());
            ydoti_b = spla.solve(hmat, ybvec.transpose());
            ydoti = 0.5 * (ydoti_f + ydoti_b);
        #

        # Assign output
        for j in range(ydothist.shape[0]):
            ydothist[j,i] = ydoti[0,j];
        #
    #

    # Backwards difference method
    for i in range(N-n, N):

        # Delta t matrix
        for j in range(1, n+1):
            d = 1.0;
            hj = h*j;
            for k in range(1, n+1):
                d = d*k;
                hmat[j-1,k-1] = (hj**k)/d;
            #
        #

        # Backward finite difference
        k = 0;
        ybvec = np.ndarray((M, n));
        for j in range(i-1, i-n-1, -1):
            ybvec[:,k] = yhist[:,i] - yhist[:,j];
            k += 1;
        #

        # Compute derivative
        if customge:
            ydoti = gauss_elimination(hmat, ybvec.transpose());
        else:
            ydoti = spla.solve(hmat, ybvec.transpose());
        #

        # Assign output
        for j in range(ydothist.shape[0]):
            ydothist[j,i] = ydoti[0,j];
        #
    #
    return ydothist;
#


import numpy as np

def sigmoid(z: np.ndarray) -> np.ndarray:
    return 1 / (1 + np.exp(-z))

def relu(z: np.ndarray) -> np.ndarray:
    return np.maximum(0, z)

def leaky_relu(z: np.ndarray, rate: float = .01) -> np.ndarray:
    return np.maximum(z * rate, z)

# def sigmoid(z):
#     return 1 / (1 + np.exp(-z))

# def relu(z):
#     return np.maximum(0, z)

# def leaky_relu(z, rate=.01):
#     return np.maximum(z * rate, z)

# def sigmoid_prime(z):
#     return 1 / (1 + np.exp(-z))

def relu_prime(z):
    return (z > 0) * 1

def sigmoid_prime(z):
    return sigmoid(z) * (1 - sigmoid(z))

# def leaky_relu(z, rate=.01):
#     return np.maximum(z * rate, z)


# -*- coding: utf-8 -*-
"""
Created on Fri Oct 19, 2018

@author: Timofey Golubev

This contains an implementation of the Thomas algorithm for solving a tridiagonal matrix equation.
"""
import numpy as np
import time
from numba import jit

@jit(nopython=True, parallel = True)
def thomas_solve(diagonal, upper, lower, rhs):
    '''
    Solves a tridiagonal matrix equation using the Thomas algorithm [1].
    Inputs: The matrix is passed in terms of 3 NumPy arrays corresponding to the upper, lower, and main
        diagonal.
        rhs: array for the right hand side

    Reference:
        [1] https://en.wikipedia.org/wiki/Tridiagonal_matrix_algorithm
    '''

    num_elements = len(diagonal) - 1

    #start = time.clock()

    diagonal = np.copy(diagonal)  #do a copy  to ensure the original diag is not changed

    x = np.empty(num_elements+2)

    # Forward substitution
    for i in range(2, num_elements + 1): # recall range uses [ ) ]
        cdiag_ratio = lower[i-1]/diagonal[i-1]
        diagonal[i] -= cdiag_ratio * upper[i-1]
        rhs[i] -= cdiag_ratio * rhs[i-1]

    # Backward substitution
    x[num_elements] = rhs[num_elements]/diagonal[num_elements] #lin eqn. corresponding to last row
    for i in range(num_elements, 1, -1):
        x[i-1] = (rhs[i-1] - x[i]*upper[i-1])/diagonal[i-1]

    #end = time.clock()

    return x


from numpy import random
from numpy.core.fromnumeric import size

# Generate a random integer from 0 to 100:

x = random.randint(100)
print(x)

# Generate a random float from 0 to 1:

x = random.rand()
print(x)

# Generate a 1-D array containing 5 random integers from 0 to 100:

x = random.randint(100, size=(5))
print(x)

# Generate a 2-D array with 3 rows, each row containing 5 random integers from 0 to 100:

x = random.randint(100, size=(3, 5))
print(x)

# Generate a 1-D array containing 5 random floats:

x = random.rand(5)
print(x)

# Generate a 2-D array with 3 rows, each row containing 5 random numbers:

x = random.rand(3, 5)
print(x)

###    Generate Random Number From Array

x = random.choice([1, 2, 3, 4, 5, 6])
print(x)

# Generate a 2-D array that consists of the values in the array parameter (3, 5, 7, and 9):

x = random.choice([3, 5, 7, 9], size=(3, 5))
print(x)