import torch
import numpy as np

def my_softmax(x):
    """
    softmax原理：
    输入任意实数x，输出[0, 1]代表概率，且概率之和为1,
    本质上是在做0-1概率分布的归一化
    这里利用了 非负实数集合/sum 的值域为[0,1]来做计算
    这样不仅能限定值域为[0, 1]，还能让输出值之和为1,
    而sigmoid只能做到前者

    为什么softmax需要exp？softmax和max的关系？
    要判断x属于哪个类别，首先能想到的是x属于哪个类的概率最高，我就认为它属于哪个类，概率可以用[0,1]表示，
    所以我遍历一下概率向量，直接把最大的概率置为1，其他置为0即可，这是很直接的max想法，但max函数是不可
    导的，不能求梯度做反向传播来学习。怎么办呢？我们还是再回顾一下目标
    1. 希望总的概率和为1
    2. 希望只有一个概率为1，其他都为0
    3. 函数要可导

    第一个目标很好实现，做一个归一化即可，即把x变成非负实数，再除以sum，即可得到。
    比较简单的让x平移使得最小值变为0，满足非负实数的条件，再除以sum： (x - min(x)) / sum((x - min(x)))
    假如x是[0.3, 0.8, 0.9]，那么计算后得到[0, 0.456, 0.544],
    问题来了，0.456和0.544比较接近，我们能不能让0.544变得更接近1， 而0.456更接近0？
    也就是说，我们怎么能让微小的变化变得更大？我们应该马上就能想到指数函数可以做到！
    这样，用极限的思维来看，最大的概率会扩大得比其他概率大得多，其他概率相当于无穷小，
    再除以sum，最大的概率就会接近1，其他相对无穷小的概率就会接近0。
    而用e做底数的指数函数，在结合交叉熵做反向传播时，很方便算导数，因此使用了exp，
    这样一来，我们达到了上面3个目标，使得max值尽量接近1，而且是连续可导的，相比max
    会很平滑，所以是soft的
    """

    # 由于对任意实数x取指数函数很容易导致类型上溢，
    # 而且softmax(x) 和 softmax(x + c)是等价的（softmax主要是用x向量内各个值的相对差来做计算，所以整体平移相对差不变，结果也不变）
    # 因此先整体减去一个最大值做预处理，让x变成(-∞, 0]
    x = x - np.max(x)

    # 指数函数可以把x变成正实数，此时值域变成了(0, 1]，和不为1
    e = np.exp(x)
    # 除以sum后结果还是(0, 1]，但是和为1
    # 虽然概率永远不可能为0，但是可以无限接近0，我们认为这样就够了
    # 这也是exp的小弊端
    return e / np.sum(e)

def my_softmax_grad(y, loss_value):
    # softmax要分n类（这里我们分10类）的求导有两种情况
    # 第一种情况：
    #    假如有一个yj，代表x分到第j类的概率，那么yj对yj的偏导等于yj - yj ** 2
    # 第二种情况：
    #    假如有一个yj，代表x分到第j类的概率，那么除了第j个y值，其他n-1个y值，对这个yj的偏导等于 0 -yi * yj    (i从0到n-1，i等于j除外)

    # n个y值，生成对角矩阵，第j行第j列，代表x被分到第j类的概率yj
    diag_matrix = np.diag(y)
    # np.outer用来生成双层for循环，生成n*n的矩阵，矩阵的i,j处位置代表yi * yj的结果
    multi_matrix = np.outer(y, y)
    # 对角矩阵 - 相乘矩阵 = 偏导矩阵，在对角部分等于yj - yj * yj, 在非对角部分等于0 - yi * yj
    partial_matrix = diag_matrix - multi_matrix

    # partial_matrix的第j行，代表x的分类概率向量（y，n个值）中每一个分量（概率）对x被分在第j类的偏导
    # loss_value有n个值，代表x被分为这n类的误差,所以-loss_value * partial_matrix[j]等于
    # x被分到第j类的梯度
    return np.dot(partial_matrix, loss_value)

if __name__ == '__main__':
    class_num = 10  # 分10类
    input = np.random.rand(class_num)  # 1x10 vector
    loss = np.random.rand(class_num)

    print("torch的softmax:")
    print(torch.softmax(torch.tensor(input), dim=0).numpy())
    print("自己写的softmax:")
    print(my_softmax(input))

    t = torch.tensor(input, requires_grad=True)
    s = torch.softmax(t, dim=0)
    s.backward(torch.tensor(loss))
    print("torch的softmax grad:")
    print(t.grad.numpy())
    print("自己写的softmax grad:")
    print(my_softmax_grad(my_softmax(input), loss))


import numpy as np
from data import io

'''
flatten(imageset)
    - converts an image to a column vector
'''
def flatten(imageset):
    flat = imageset.reshape(imageset.shape[ 0 ], -1).T
    return flat

'''
weight(U, dataset, shi)
    - calculate weight of each train sample
    - using first k pca
'''
def weight(U, dataset, shi):
    return np.dot(U.T, dataset - shi)

'''
pca(k, trainset):
    - returns the first k pca
'''
def pca(k, trainset):
    shi = np.asmatrix(trainset.mean(axis = 1)).T
    phi = trainset - shi
    covariance = np.dot(phi, phi.T)
    eig_vals, eig_vecs = np.linalg.eig(covariance)

    i = eig_vals.argsort()[-k:][::-1]
    eig_vals = eig_vals[i]
    component = eig_vecs[:, i]

    return component, shi

'''
train_pca(n, k, image_size, directory)
    - calculate pca, weight
'''
def train_pca(n, k, image_size, directory):
    trainset = io.load(n, image_size, directory)
    print("trainset shape:", trainset.shape)

    trainset = flatten(trainset)
    print("trainset shape after flattening:", trainset.shape)

    U, shi = pca(k, trainset)
    print("Principle components:", U.shape)
    print("Average Column Vector:", shi.shape)

    train_weight = weight(U, trainset, shi)
    print("train weight data shape:", train_weight.shape)

    return U, shi, train_weight

'''
test_pca(n, image_size, directory, U, shi, train_weight)
    - maps the testset with pca
    - calculates test weights
'''
def test_pca(n, image_size, directory, U, shi, train_weight):
    testset = io.load(n, image_size, directory)
    print("testset shape:", testset.shape)

    testset = flatten(testset)
    print("testset shape after flattening::", testset.shape)

    test_weight = weight(U, testset, shi)
    return test_weight


import pdb
import numpy as np
import matplotlib.pyplot as plt
import math

######################################################################
#
# Learning algorithms and helpers
#
######################################################################

# Ordinary least squares regression
# X is an n x d matrix
# Y is an n x 1 matrix
# returns weights: d x 1 matrix
def ols(X, y):
    return (X.T * X).I * X.T * y

# "Ridge" regression with lambda = l; otherwise like ols
# Penalizes the offset weight, which is sometimes undesirable
def olsr(X, y, l = 0):
    d = X.shape[1]
    return (X.T * X + l * np.identity(d)).I * X.T * y

# Given an order for a polynomial feature space, d
# return a *function* that maps a single value into a vector of d+1 features
def polynomialFeatures(d):
    def p(xi):
        return np.matrix([xi[0,0]**i for i in range(d+1)])
    return p

# Given X, an array of training examples, and b, a bandwidth,
# return a *function* that maps a single value into a vector of features
def RBFs(X, b):
    def g(x1, x2, b):
        return np.exp(-((np.linalg.norm(x1 - x2) / b)**2))
    return lambda xi: np.matrix([g(xi, x, b) for x in X] + [1.0])

# Given a function, such as the one generated by polynomialFeatures,
# which takes a d-dimensional vector and returns a D-dimensional
# vector, and an n x d matrix X, return an n x D matrix of feature
# vectors
def applyFeatureFun(phi, X):
    return np.vstack([phi(X[i,:]) for i in range(X.shape[0])])

# Given a D x 1 weight vector and a "feature function" that takes a
# d-dimensional vector into a D x 1 vector, return a **function** that
# takes a d-dimensional vector into a scalar regression value, but
# mapping the input into the high-dimensional feature space and then
# taking the dot product with the weights.
def makeRegressor(w, phi):
    def r(x):
        return (phi(x)*w)[0,0]
    return r

# Given an n x d matrix X of input values and an n x 1 matrix y of
# target values, and a function that maps a row of X into a
# prediction, return the root mean squared error of the predictor
# applied to the X values.
def rmse(X, y, predictor):
    return np.sqrt(sse(X, y, predictor)/len(y))

# As for rmse, but just return the sum squared error.
def sse(X, y, predictor):
    p = np.matrix([predictor(X[i,:]) for i in range(X.shape[0])]).T
    assert p.shape == y.shape
    return np.sum(np.square(p - y))

# Generic gradient descent:
# - f is a function mapping a vector of parameters to a score
#     (not strictly necessary)
# - df is a function mapping a vector of parameters to the gradient at
#     that point
# - x0 is an initial parameter vector
# - step_size is the step size
#
#
# - terminates after max_iter iterations, or when parameters have not changed
#    by more than eps from one iteration to the next
#   in every iteration updates the current param vector by the gradient
# Returns: the final parameter vector, a list of scores (one per iteration)
#   and a list of parameter vectors (one per iteration)
def gd(f, df, x0, step_size = .01, max_iter = 1000, eps = .00001):
    diff = 1
    iterCount = 0
    fs = []
    xs = []

    fs.append(f(x0))
    xs.append(x0)
    while (diff > eps and iterCount < max_iter):
        x = x0 - step_size * df(x0)
        diff = np.linalg.norm(x - x0)
        fs.append(f(x))
        xs.append(x0)

        iterCount += 1
        x0 = x
    # your code here

    return x, fs, xs

# Linear regression using gradient descent.
# X and y are as for ordinary least squares;
# w0 is the initial weight vector
# max_iter is as for gd
# l currently unused, but can be used to do ridge regression
# Returns: final weight vector, a list of scores, list of weight vectors

def gdLinReg(X, y, l = 0, step_size = 0.01, w0 = None, max_iter = 1000):
    # w is d by 1; X is n by d
    # return result is d by 1
    n = float(len(y))
    def df(w):
        return 2 * np.sum(np.multiply(X*w - y, X), axis = 0).T + l * w
    def f(w):
        return np.sum(np.square(X*w - y)) + l * float(w.T * w)

    if w0 is None: w0 = np.matrix(np.ones(X.shape[1])).T
    return gd(f, df, w0, step_size = step_size, max_iter = max_iter)

# Stochastic gradient descent
# X is an n x d matrix of input examples
# y is an n x 1 matrix of outputs (we could combine X and Y if we wanted to)
# f is a function taking a parameter vector, a 1 x d input example,
#     and a 1 x 1 output value, and returning an error; we want to
#     minimize this error in expectation over the whole data set
# df has the same inputs as f, but returns a gradient vector of the same
#     dimension as w0
# if earlyTermination is true, then terminate if params move by less
# than epsilon; otherwise, terminate after max_iter iterations.

# Returns last parameter vector, list of scores, list of param
# vectors, the iteration on which termination happened, and a Boolean
# indicating whether termination was early or not.

def sgd(X, y, f, df, w0, step_size = .01, max_iter = 20000, eps = .0000001,
            earlyTermination = False):
    n = y.shape[0]
    prev_w = w0
    fs = []; ws = [prev_w]
    for i in range(max_iter):
        j = np.random.randint(n)
        Xj = X[j]; yj = y[j]
        w = prev_w - step_size * df(prev_w, Xj, yj)
        if earlyTermination and np.all(abs(w - prev_w) < eps):
            return w, fs, ws, i, False
        fs.append(f(prev_w, Xj, yj)); ws.append(w)
        prev_w = w
    return w, fs, ws, i, True

# Linear regression computed by stochastic gradient descent.
# Parameters are as for gdLinReg.

# Returns last parameter vector, list of scores, list of param
# vectors, the iteration on which termination happened, and a Boolean
# indicating whether termination was early or not.

def sgdLinReg(X, y, l = 0, step_size = 0.01, w0 = None, max_iter = 20000):
    def df(w, Xj, yj):
        return 2 * ((Xj * w - yj) * Xj).T

    def f(w, Xj, yj):
        return np.square(Xj * w - yj)

    if w0 is None: w0 = np.matrix(np.ones(X.shape[1])).T
    return sgd(X, y, f, df, w0, step_size = step_size, max_iter = max_iter)

######################################################################
#
# Data
#
######################################################################

# Uses data set from Chapter 1 of Bishop, stored in file
# "curvefitting.txt".  They are 10 points drawn from (x, sin(2 pi x))
# with noise added (but I'm not sure how much.)

# If random is not False, it should be an integer, and instead of
# returning data from the file, we will generate a new random data set
# of that size, with 0 mean, 0.2 stdev Gaussian noise.

# if addOnes is true, return: n x 1 matrix X, n x 2 matrix F (with
# column of 1's added) and n x 1 matrix Y.

def getCurveData(addOnes = False, random = False):
    if random:
        X = np.matrix([[i / float(random)] for i in range(random + 1)])
        noise = np.random.normal(scale = 0.2, size = (random+1, 1))
        y = np.matrix([[np.sin(2 * np.pi * X[i,0])] for i in range(X.shape[0])]) + noise
    else:
        data = np.loadtxt('curvefitting.txt')
        X, y = np.matrix(data[0]).T, np.matrix(data[1]).T
    if addOnes:
        F = np.append(np.ones_like(X), X, 1)
        return X, F, y
    else:
        return X, y

# Get the Blog data

def getBlogData(n):
    dir = '/home/binder/entwurf6/teaching/2017/ai_2017/wek1/code/'
    X = np.matrix(np.genfromtxt(dir+'x_train.csv',
                                    delimiter = ','))
    if n == None:
        n = X.shape[0]
    else:
        X = X[:n]

    y = np.matrix(np.genfromtxt(dir+'y_train.csv',
                                    delimiter = ',')).T[:n]
    XTest = np.matrix(np.genfromtxt(dir+'x_test.csv',
                                    delimiter = ','))
    yTest = np.matrix(np.genfromtxt(dir+'y_test.csv',
                                    delimiter = ',')).T
    return X, y, XTest, yTest

######################################################################
#
# Plotting stuff
#
######################################################################

def tidyPlot(xmin, xmax, ymin, ymax, center = False, title = None,
                 xlabel = None, ylabel = None):
    plt.ion()
    plt.figure(facecolor="white")
    ax = plt.subplot()
    if center:
        ax.spines['left'].set_position('zero')
        ax.spines['right'].set_color('none')
        ax.spines['bottom'].set_position('zero')
        ax.spines['top'].set_color('none')
        ax.spines['left'].set_smart_bounds(True)
        ax.spines['bottom'].set_smart_bounds(True)
        ax.xaxis.set_ticks_position('bottom')
        ax.yaxis.set_ticks_position('left')
    else:
        ax.spines["top"].set_visible(False)
        ax.spines["right"].set_visible(False)
        ax.get_xaxis().tick_bottom()
        ax.get_yaxis().tick_left()
    eps = .05
    plt.xlim(xmin-eps, xmax+eps)
    plt.ylim(ymin-eps, ymax+eps)
    if title: ax.set_title(title)
    if xlabel: ax.set_xlabel(xlabel)
    if ylabel: ax.set_ylabel(ylabel)
    return ax

def plotData(ax, x, y, style = 'ro', label = None):
    if style is None:
        ax.plot(x, y, label = label)
    else:
        ax.plot(x, y, style, label = label)
    plt.show()

# w is a (1 x 2) matrix
def plotLine(ax, w, xmin, xmax, nPts = 100):
    b = float(w[0])
    m = float(w[1])
    plotFun(ax, lambda x: m*x + b, xmin, xmax, nPts)

def plotFun(ax, f, xmin, xmax, nPts = 100, label = None):
    x = np.linspace(xmin, xmax, nPts)
    y = np.vstack([f(np.matrix([[xi]])) for xi in x])
    ax.plot(x, y, label = label)
    plt.show()

def smooth(n, vals):
    # Run a box filter of size n
    x = sum(vals[0:n])
    result = [x]
    for i in range(n, len(vals)):
        x = x - vals[i-n] + vals[i]
        result.append(x)
    return result

######################################################################
#
# Tests
#
######################################################################

######################################################################
#
# Ordinary least squares in 2D (one constant input dimension)

def t1():
    X, F, y = getCurveData(True)

    w = ols(F, y)

    print ('w', w.T)
    xmin, xmax = float(min(X)), float(max(X))
    ymin, ymax = float(min(y)), float(max(y))
    ax = tidyPlot(xmin, xmax, ymin, ymax, xlabel = 'x', ylabel = 'y')
    plotData(ax, X, y)
    plotLine(ax, w, xmin, xmax)

######################################################################
#
# Ordinary least squares in polynomial feature spaces
# Plots predictors

def t2(ds = range(1, 10)):
    X, y = getCurveData()
    xmin, xmax = float(min(X)), float(max(X))
    ymin, ymax = float(min(y)), float(max(y))
    ax = tidyPlot(xmin, xmax, ymin-1, ymax+1, xlabel = 'x', ylabel = 'y')

    plotData(ax, X, y)

    for d in ds:
        phi = polynomialFeatures(d)
        phiD = applyFeatureFun(phi, X)
        w = ols(phiD, y)
        predictor = makeRegressor(w, phi)

        plotFun(ax, predictor, xmin, xmax, label = str(d))
        print ('Order', d, 'Training RMSE', rmse(X, y, predictor))
        print ('     w', w.T)
    ax.legend(loc="upper left", bbox_to_anchor=(1,1))

######################################################################
#
# Ordinary least squares in polynomial feature spaces
# Plots train and test error versus order of polynomial basis

def t3(ds = range(1, 10)):
    X, y = getCurveData()
    XTest, yTest = getCurveData(random = 10)
    xmin, xmax = float(min(X)), float(max(X))
    ymin, ymax = float(min(y)), float(max(y))
    ax = tidyPlot(xmin, xmax, ymin-1, ymax+1, xlabel = 'x', ylabel = 'y')
    plotData(ax, X, y)
    plotData(ax, XTest, yTest, style = 'go')
    trainErr = []
    testErr = []

    for d in ds:
        phi = polynomialFeatures(d)
        phiD = applyFeatureFun(phi, X)
        w = ols(phiD, y)
        predictor = makeRegressor(w, phi)
        plotFun(ax, predictor, xmin, xmax)
        trainErr.append(rmse(X, y, predictor))
        testErr.append(rmse(XTest, yTest, predictor))
        print ('Order', d, 'Training RMSE', trainErr[-1])
        print ('        Test RMSE', testErr[-1])

    ax = tidyPlot(0, len(ds), 0, max(max(testErr), max(trainErr)), center =True,
                      xlabel = 'Polynomial order', ylabel = 'RMSE')
    ax.plot(ds, trainErr, label = 'train err')
    ax.plot(ds, testErr, label = 'test err')
    ax.legend()

######################################################################
#
# Ordinary least squares in RBF feature spaces
# Plots train and test error versus bandwidth

# LPK: these are good b values (.01, .05, .1, .5, 1, 2, 4)

def tRBF(bs = []):
    X, y = getCurveData()
    XTest, yTest = getCurveData(random = 10)
    xmin, xmax = float(min(X)), float(max(X))
    ymin, ymax = float(min(y)), float(max(y))
    ax = tidyPlot(xmin, xmax, ymin-1, ymax+1, xlabel = 'x', ylabel = 'y')
    plotData(ax, X, y)
    trainErr = []
    testErr = []

    for b in bs:
        phi = RBFs(X, b)
        phiD = applyFeatureFun(phi, X)
        w = olsr(phiD, y, 1e-5)
        predictor = makeRegressor(w, phi)
        plotFun(ax, predictor, xmin, xmax, label = str(b))
        trainErr.append(rmse(X, y, predictor))
        testErr.append(rmse(XTest, yTest, predictor))
        print ('Bandwidth', b, 'Training RMSE', trainErr[-1])
        print ('        Test RMSE', testErr[-1])
        print ('    w', w.T)
    ax.legend(loc = 'best')

    if len(bs) > 1:
        ax = tidyPlot(min(bs), max(bs), 0, max(max(testErr), max(trainErr)),
                        xlabel = 'Bandwidth', ylabel = 'RMSE')
        ax.plot(bs, trainErr, label = 'train err')
        ax.plot(bs, testErr, label = 'test err')
        ax.legend(loc = 'best')

######################################################################
#
# Ordinary least squares in polynomial feature spaces
# Plots train and test error versus size of training set

def t4(trainSizes = (10, 15, 20, 30, 50, 100), showData = False):
    XTest, yTest = getCurveData(random = 100)
    xmin, xmax = float(np.min(XTest)), float(np.max(XTest))
    ymin, ymax = float(np.min(yTest)), float(np.max(yTest))
    trainErr = []
    testErr = []
    phi = polynomialFeatures(9)

    if not showData:
        ax = tidyPlot(xmin, xmax, ymin-1, ymax+1, xlabel = 'x', ylabel = 'y',
                          title = 'Predictors for different train sizes')

    for ts in trainSizes:
        X, y = getCurveData(random = ts-1)
        if showData:
            ax = tidyPlot(xmin, xmax, ymin-1, ymax+1, xlabel = 'x', ylabel = 'y',
                          title = 'Train size = ' + str(ts))
            plotData(ax, X, y)
        phiD = applyFeatureFun(phi, X)
        w = ols(phiD, y)
        predictor = makeRegressor(w, phi)
        plotFun(ax, predictor, xmin, xmax, label=str(ts))
        trainErr.append(rmse(X, y, predictor))
        testErr.append(rmse(XTest, yTest, predictor))
        print ('Train size', ts, 'Training RMSE', trainErr[-1])
        print ('        Test RMSE', testErr[-1])
        print ('     w', w.T)
    ax.legend(loc="upper left", bbox_to_anchor=(1,1))
    if len(trainSizes) > 1:
        ax = tidyPlot(0, max(trainSizes), 0, max(testErr), center = True,
                      xlabel = 'Training set size', ylabel = 'RMSE')
        ax.plot(trainSizes, trainErr, label = 'train err')
        ax.plot(trainSizes, testErr, label = 'test err')
        ax.legend(loc="upper right")

######################################################################
#
# Ridge regression in polynomial feature spaces
# Plot train and test error versus ridge parameter lambda

def t5(logLambdaValues = (-50, -40, -30, -20, -15, -10, -1, 0, 1, 10)):
    XTest, yTest = getCurveData(random = 100)
    xmin, xmax = float(np.min(XTest)), float(np.max(XTest))
    ymin, ymax = float(np.min(yTest)), float(np.max(yTest))
    trainErr = []
    testErr = []
    phi = polynomialFeatures(9)

    ax = tidyPlot(xmin, xmax, ymin-1, ymax+1,
                      xlabel = 'x', ylabel = 'y',
                          title = 'Predictors for different lambda values')
    X, y = getCurveData()
    phiD = applyFeatureFun(phi, X)
    plotData(ax, X, y)
    for llv in logLambdaValues:
        w = olsr(phiD, y, np.exp(llv))
        predictor = makeRegressor(w, phi)
        plotFun(ax, predictor, xmin, xmax, label=str(llv))
        trainErr.append(rmse(X, y, predictor))
        testErr.append(rmse(XTest, yTest, predictor))
        print ('Log lambda', llv, 'Training RMSE', trainErr[-1])
        print ('        Test RMSE', testErr[-1])
        print (w.T)
    ax.legend(loc="upper left", bbox_to_anchor=(1,1))

    if len(logLambdaValues) > 1:
        ax = tidyPlot(min(logLambdaValues), max(logLambdaValues),
                        0, max(testErr), center = True,
                        xlabel = 'Log lambda', ylabel = 'RMSE')
        ax.plot(logLambdaValues, trainErr, label = 'train err')
        ax.plot(logLambdaValues, testErr, label = 'test err')
        ax.legend(loc="best")

######################################################################
#
# Batch gradient descent in 2D feature space
# Plots error as a function of iteration number for different step sizes
# Individual plots of trajectory of w during optimization

def t6(learning_rates = (.0001, .001, .01, .05, .07, .075, .1)):
    X, F, y = getCurveData(True)
    ax = tidyPlot(0, 1000, 2, 8,
                      xlabel = 'iteration', ylabel = 'err')
    for lr in learning_rates:
        w, scores, vals = gdLinReg(F, y, step_size = lr)

        plotData(ax, range(len(scores)), scores, style = None,
                     label = str(lr))
        print ('lr', lr, 'w', w.T, 'err', scores[-1])
        dmin = min(-1.5, np.min(vals)); dmax = max(1.5, np.max(vals))
        nax = tidyPlot(dmin, dmax, dmin, dmax,
                       xlabel = 'w0', ylabel = 'w1',
                       title = 'step size = '+str(lr), center = True)
        plotData(nax, [float(xd) for (xd, yd) in vals],
                      [float(yd) for (xd, yd) in vals],
                      style = 'bo-')
    ax.legend(loc="upper left", bbox_to_anchor=(1,1))

######################################################################
#
# Stochastic  gradient descent in 2D feature space
# Plots error as a function of iteration number for different step sizes
# Individual plots of trajectory of w during optimization

def t7(learning_rates = (.0001, .001, .01, .1, .5, 1)):
    X, F, y = getCurveData(True)
    ax = tidyPlot(0, 1000, 0, 100,
                      xlabel = 'iteration', ylabel = 'err')
    for lr in learning_rates:
        w, scores, vals, iters, failed = sgdLinReg(F, y, step_size = lr)

        smoothScores = smooth(len(y)*4, [float(s) for s in scores])
        plotData(ax, range(len(smoothScores)), smoothScores,
                     style = None, label = str(lr))
        print ('lr', lr, 'w', w.T, 'err', smoothScores[-1])
        dmin = min(-1.5, np.min(vals)); dmax = max(1.5, np.max(vals))
        nax = tidyPlot(dmin, dmax, dmin, dmax,
                       xlabel = 'w0', ylabel = 'w1',
                       title = 'step size = '+str(lr), center = True)
        plotData(nax, [float(xd) for (xd, yd) in vals],
                      [float(yd) for (xd, yd) in vals],
                      style = 'bo-')
    ax.legend(loc="best")

######################################################################
#
# Batch gradient descent in polynomial feature space
# Plots train and test error as a function of iteration number

def t8(order = 9):
    goodw = np.matrix(\
      [[  3.49512436e-01,   2.32326298e+02,  -5.32086773e+03,   4.85596769e+04,
         -2.31598783e+05,   6.39931981e+05,  -1.06162004e+06,   1.04222515e+06,
         -5.57590377e+05,   1.25180846e+05]]).T
    zeros = np.matrix(np.zeros([order+1, 1]))
    medw = np.matrix(\
      [[  3.0e-01,   2.0e+02,  -5.0e+03,   5.0e+04,
         -2.0e+05,   6.0e+05,  -1.0e+06,   1.0e+06,
         -5.0e+05,   1.0e+05]]).T

    X, y = getCurveData()
    xmin, xmax = float(min(X)), float(max(X))
    ymin, ymax = float(min(y)), float(max(y))
    XTest, yTest = getCurveData(random = 100)

    phi = polynomialFeatures(order)
    phiD = applyFeatureFun(phi, X)
    lr = 0.005
    w, scores, vals = gdLinReg(phiD, y, step_size = .01, w0 = zeros,
                                   max_iter = 10000)
    print ('w', w.T, 's', scores[-1], 'iter', len(scores))
    iVals = []; trainVals = []; testVals = []
    axPred = tidyPlot(xmin, xmax, ymin-1, ymax+1, xlabel = 'x', ylabel = 'y')
    for i in range(0, len(vals), 100):
        predictor = makeRegressor(vals[i], phi)
        plotFun(axPred, predictor, xmin, xmax, label = 'iters = '+str(i))
        iVals.append(i)
        trainVals.append(rmse(X, y, predictor))
        testVals.append(rmse(XTest, yTest, predictor))
    plotData(axPred, X, y)
    axPred.legend(loc="upper left", bbox_to_anchor=(1,1))

    ax = tidyPlot(0, np.max(iVals), 0, 1, #np.max(testVals),
                   xlabel = 'iteration', ylabel = 'rmse')
    plotData(ax, iVals, trainVals, label = 'training error', style = 'bo-')
    plotData(ax, iVals, testVals, label = 'testing error', style = 'ro-')
    ax.legend(loc="best")

def t9(order = 9):
    zeros = np.matrix(np.zeros([order+1, 1]))
    goodw = np.matrix(\
      [[  3.49512436e-01,   2.32326298e+02,  -5.32086773e+03,   4.85596769e+04,
         -2.31598783e+05,   6.39931981e+05,  -1.06162004e+06,   1.04222515e+06,
         -5.57590377e+05,   1.25180846e+05]]).T
    medw = np.matrix(\
      [[  3.0e-01,   2.0e+02,  -5.0e+03,   5.0e+04,
         -2.0e+05,   6.0e+05,  -1.0e+06,   1.0e+06,
         -5.0e+05,   1.0e+05]]).T
    randw = np.matrix(np.random.randn(order+1,1))

    X, y = getCurveData()
    xmin, xmax = float(min(X)), float(max(X))
    ymin, ymax = float(min(y)), float(max(y))
    XTest, yTest = getCurveData(random = 100)

    phi = polynomialFeatures(order)
    phiD = applyFeatureFun(phi, X)
    lr = 0.05
    w, scores, vals, iters, failed = sgdLinReg(phiD, y, step_size = lr,
                                                   max_iter = 100000,
                                                   w0 = randw)
    print ('w', w.T, 's', scores[-1], 'iter', len(scores))
    iVals = []; trainVals = []; testVals = []
    axPred = tidyPlot(xmin, xmax, ymin-1, ymax+1, xlabel = 'x', ylabel = 'y')
    for i in range(0, len(vals), 5000):
        predictor = makeRegressor(vals[i], phi)
        plotFun(axPred, predictor, xmin, xmax, label = 'iters = '+str(i))
        iVals.append(i)
        trainVals.append(rmse(X, y, predictor))
        testVals.append(rmse(XTest, yTest, predictor))
    plotData(axPred, X, y)

    ax = tidyPlot(0, np.max(iVals), 0, 1, #np.max(testVals),
                   xlabel = 'iteration', ylabel = 'rmse')
    plotData(ax, iVals, trainVals, label = 'training error', style = 'bo-')
    plotData(ax, iVals, testVals, label = 'testing error', style = 'ro-')
    ax.legend(loc="upper left", bbox_to_anchor=(1,1))

# Blog feedback data, with analytic ridge regression
def t10(n = None,
        logLambdaValues = [-14, -12, -10, -8, -6, -4, -2, 0, 2, 4, 5, 6, 7,
                               8, 9, 10, 12, 14, 16, 18, 20]):
    X, y, XTest, yTest = getBlogData(n = n)
    print (X.shape, y.shape)
    iVals = []; trainVals = []; testVals = []
    for llv in logLambdaValues:
        w = olsr(X, y, np.exp(llv))
        predictor = lambda x: x * w
        iVals.append(llv)
        trainVals.append(rmse(X, y, predictor))
        testVals.append(rmse(XTest, yTest, predictor))
        print (llv, trainVals[-1], testVals[-1])

    ax = tidyPlot(np.min(iVals), np.max(iVals), 28, 40, #np.max(testVals),
                   xlabel = 'Log lambda', ylabel = 'rmse',
                   title = 'Training data size = '+str(n))
    plotData(ax, iVals, trainVals, label = 'training error', style = 'bo-')
    plotData(ax, iVals, testVals, label = 'testing error', style = 'ro-')
    ax.legend(loc="upper left", bbox_to_anchor=(1,1))

# Blog feedback data, with gradient regression.
# Add ridge!

# lambda = 80; rmse = 31.8; mse = 1024
# lambda = ? ; mse = 1015   (seems untrustworthy)
# log lambda = 2.8
# lambda 100, MSE 894

# lr = .000000005 good for whole data set (or divide by 5)

def t11(lr = .000000005, n = None):
    X, y, XTest, yTest = getBlogData(n = n)
    print (X.shape, y.shape)
    iVals = []; trainVals = []; testVals = []
    #w, scores, vals = gdLinReg(X, y, step_size = lr, max_iter = 20000)
    w, scores, vals, iters, failed = \
                            sgdLinReg(X, y,
                                          step_size = lr, max_iter = 2000000)
    print ('s', scores[-1], 'iter', len(scores))
    iVals = []; trainVals = []; testVals = []
    for i in range(0, len(vals), 50000):
        predictor = lambda x: float(x * vals[i])
        iVals.append(i)
        trainVals.append(rmse(X, y, predictor))
        testVals.append(rmse(XTest, yTest, predictor))

    print (trainVals)
    print (testVals)

    ax = tidyPlot(0, np.max(iVals), 0, 400, #np.max(testVals),
                   xlabel = 'iteration', ylabel = 'rmse')
    plotData(ax, iVals, trainVals, label = 'training error', style = 'bo-')
    plotData(ax, iVals, testVals, label = 'testing error', style = 'ro-')
    ax.legend(loc="upper left", bbox_to_anchor=(1,1))

def tGD(step_sizes = []):
    def f(x):
        return (2 * x + 3)**2
    def df(x):
        return 2 * 2 * (2 * x + 3)
    x0 = 0
    for ss in step_sizes:
        x, fs, xs = gd(f, df, x0, step_size = ss)
        print ('ss', ss, 'x', x)
        nax = tidyPlot(-4, 1, 0, 15,
                       xlabel = 'x', ylabel = 'f(x)',
                       title = 'step size = '+str(ss), center = True)
        plotFun(nax, f, -4, 1)
        plotData(nax, xs, fs, style = 'ro-')

def tGD2(x0 = 0):
    def f(x):
        return (x - 2) * (x - 3) * (x + 3) * (x + 1)
    def df(x):
        return 9 - (22 * x) - (3 * x**2) + (4 * x**3)
    ss = 0.01
    x, fs, xs = gd(f, df, x0, step_size = ss)
    nax = tidyPlot(-4, 4, -25, 25,
                       xlabel = 'x', ylabel = 'f(x)',
                       title = 'step size = '+str(ss), center = True)
    plotFun(nax, f, -4, 4)
    plotData(nax, xs, fs, style = 'ro-')

if __name__ == '__main__':
    print ('Loaded learn.py')
    # tGD2()
    stepSizes = [0.01]
    tGD(stepSizes)
    plt.show(block=True)


import numpy as np

def CalcVonMiesesEquivalentStress(sigma_A, sigma_B ):
    """
    Calculate the von Mises Equivalent stress for a planar (2D) situations
    using the principal stresses sigma_A and sigma_B, with sigma_A > sigma_B
    :param sigma_A:  the first principal stress
    :param sigma_B:  the second principal stress:
    :return: the von Mises equivalent stress
    """
    sigma_eq = sigma_A**2 - sigma_A * sigma_B + sigma_B**2
    sigma_eq = np.sqrt(sigma_eq)
    return sigma_eq

def CalcVonMiesesFoS(sigma_A, sigma_B, Sy):
    vonMisesEq = CalcVonMiesesEquivalentStress(sigma_A, sigma_B)
    if vonMisesEq == 0.0:
        vonMisesEq = 0.00000001
    FoS_vonMisesEq = Sy / vonMisesEq

    return [ vonMisesEq, FoS_vonMisesEq ]

def CalcTrescaEquicalentStress(sigma_A, sigma_B ):
    """
    \
    :param sigma_A:
    :param sigma_B:
    :return:
    """
    s1 = sigma_A
    s2 = sigma_B
    if sigma_B > sigma_A:
        s2 = sigma_A
        s1 = sigma_B

    stress = (s1 - s2)/2

    return stress

def CalcTrescaFoS(sigma_A, sigma_B, Sy ):

    s1 = sigma_A
    s2 = sigma_B
    if sigma_B > sigma_A:
        s2 = sigma_A
        s1 = sigma_B

    stress = CalcTrescaEquicalentStress(s1, s2)

    if stress == 0.0:
        stress = 0.00000001

    if s1 == 0.0:
        s1 = 0.00000001

    if s2 == 0.0:
        s2 = 0.00000001

    #FoS_TrescaEqStress = 0
    if s1 >= s2 and s1 >= 0 and s2 >= 0:
        FoS_TrescaEqStress = Sy / s1
    elif s1 >= 0 and s2 < 0:
        FoS_TrescaEqStress = Sy / 2 / stress
    elif s1 < 0 and s2  < s1:
        FoS_TrescaEqStress = -Sy / s2

    #FoS_TrescaEqStress = Sy/(stress)

    return [stress, FoS_TrescaEqStress]

def CalcPrincipalStressFos(sigma_A, sigma_B, Sy ):
    if sigma_A == 0.0:
        sigma_A = 0.00000001

    if sigma_B == 0.0:
        sigma_B = 0.00000001

    n1 = Sy / sigma_A
    n2 = Sy / sigma_B

    return min(n1,n2)

def CalcMNSTStressFoS(s1, s3, Sut, Suc):

    if s1 < 0.000001 and s1 > -0.000001:
        s1 = 0.0001

    if s3 < 0.000001 and s3 > -0.000001:
        s3 = 0.0001

    case = 0
    stress = 0
    n = 0
    if s1 >= 0 and s3 >= 0 and s1 >= s3:
        case = 1
        n = Sut / s1
        stress = s1
    elif s1 >= 0 and s3 >= 0 and s3 > s1:
        case = 2
        n = Sut / s3
        stress = s3
    elif s1 < 0 and s3 >= 0 and -Sut / Suc > s3 / s1:
        case = 3
        n = Sut / s3
        stress = s3
    elif s1 < 0 and s3 >= 0 and -Sut/Suc <= s3/s1:
        case = 4
        n = Suc / s1
        stress = s1
    elif s1 < 0 and s3 < 0 and -Suc/-Suc >= s3/s1:
        case = 5
        n = Suc / s1
        stress = s1
    elif s1 < 0 and s3 < 0 and -Suc/-Suc < s3/s1:
        case = 6
        n = Suc / s3
        stress = s3
    elif s1 >= 0 and s3 < 0 and -Suc / Sut > s3 / s1:
        case = 7
        n = Suc / s3
        stress = s3
    elif s1 >= 0 and s3 < 0 and -Suc / Sut < s3 / s1:
        case = 8
        n = Sut / s1
        stress = s1

   # print( -Suc / Sut, "  ",  s3 / s1, " ", case)

    return [stress, abs(round(n,2))]

def CalcMMStressFoS(s1, s3, Sut, Suc):

    # swap s1 and s3 so that s1 > s3
    if s1 < s3:
        t = s1
        s1 = s3
        s3  = t

    if s1 < 0.000001 and s1 > -0.000001:
        s1 = 0.0001

    if s3 < 0.000001 and s3 > -0.000001:
        s3 = 0.0001

    case  = 0
    s = 0
    FoS = 0
    if s1 >= s3 and s3 >= 0:
        FoS = Sut / s1
        s = s1
        case = 1
    elif s1 >= 0 and s3 < 0 and abs(s3/s1) <= 1.0:
        FoS = Sut / s1
        s = s1
        case = 2
    elif  s1 >= 0 and s3 < 0 and abs(s3/s1) > 1.0:
        n = ((Suc - Sut) * s1)/(Suc* Sut) - s3/Suc
        FoS = 1/n
        s = np.sqrt(s1**2 + s3**2)
        case = 3
    elif s1 <=0 and s3 < s1:
        FoS = -Suc/s3
        s = s3
        case = 4

    return [s, abs(round(FoS, 2))]

def CalcBCMStressFoS(s1, s3, Sut, Suc):
    # swap s1 and s3 so that s1 > s3
    if s1 < s3:
        t = s1
        s1 = s3
        s3 = t

    if s1 < 0.000001 and s1 > -0.000001:
        s1 = 0.0001

    if s3 < 0.000001 and s3 > -0.000001:
        s3 = 0.0001

    case = 0
    s = 0
    FoS = 0

    if s1 >= s3 and s3 >= 0:
        FoS = Sut/s1
        s = s1
    elif  s1 >= 0 and s3 <= 0:
        n = s1/Sut - s3/Suc
        FoS = 1/n
        s = np.sqrt(s1**2 + s3**2)
    elif s1 <= 0 and s3 <= 0:
        FoS = -Suc /s3
        s = s3

    return [s, abs(round(FoS, 2))]


# -*- coding: utf-8 -*-
# <nbformat>3.0</nbformat>

# <markdowncell>

# # Linear Regression with Gradient Descent Algorithm
#
# This notebook demonstrates the implementation of linear regression with gradient descent algorithm.
#
# Consider the following implementation of the gradient descent loop with NumPy arrays based upon [1]:

# <codecell>

def gradient_descent_numpy(X, Y, theta, alpha, num_iters):
    m = Y.shape[0]

    theta_x = 0.0
    theta_y = 0.0

    for i in range(num_iters):
        predict = theta_x + theta_y * X
        err_x = (predict - Y)
        err_y = (predict - Y) * X
        theta_x = theta_x - alpha * (1.0 / m) * err_x.sum()
        theta_y = theta_y - alpha * (1.0 / m) * err_y.sum()

    theta[0] = theta_x
    theta[1] = theta_y

# <markdowncell>

# To speedup this implementation with Numba, we need to add the `@jit` decorator to annotate the function signature.  Then, we need to expand the NumPy array expressions into a loop.  The resulting code is shown below:

# <codecell>

from numba import autojit, jit, f8, int32, void

@jit(void(f8[:], f8[:], f8[:], f8, int32))
def gradient_descent_numba(X, Y, theta, alpha, num_iters):
    m = Y.shape[0]

    theta_x = 0.0
    theta_y = 0.0

    for i in range(num_iters):
        err_acc_x = 0.0
        err_acc_y = 0.0
        for j in range(X.shape[0]):
            predict = theta_x + theta_y * X[j]
            err_acc_x += predict - Y[j]
            err_acc_y += (predict - Y[j]) * X[j]
        theta_x = theta_x - alpha * (1.0 / m) * err_acc_x
        theta_y = theta_y - alpha * (1.0 / m) * err_acc_y

    theta[0] = theta_x
    theta[1] = theta_y

# <markdowncell>

# The rest of the code generates some artificial data to test our linear regression algorithm.

# <codecell>

import numpy as np
import pylab
from timeit import default_timer as timer

def populate_data(N):
    noise = np.random.random(N).astype(np.float64)
    X = np.arange(N, dtype=np.float64)
    slope = 3
    Y = noise * (slope * X)
    return X, Y

def run(gradient_descent, X, Y, iterations=1000, alpha=1e-6):
    theta = np.empty(2, dtype=X.dtype)

    ts = timer()
    gradient_descent(X, Y, theta, alpha, iterations)
    te = timer()

    timing = te - ts

    print "x-offset = {}    slope = {}".format(*theta)
    print "time elapsed: {} s".format(timing)

    return theta, timing

def plot(X, theta, c='r'):
    result = theta[0] + theta[1] * X
    pylab.plot(X, result, c=c, linewidth=2)

# <markdowncell>

# We will a benchmark with 50 elements to compare the pure python version against the numba version.

# <codecell>

N = 50
X, Y = populate_data(N)
pylab.scatter(X, Y, marker='o', c='b')
pylab.title('Linear Regression')

print 'Python'.center(30, '-')
theta_python, time_python = run(gradient_descent_numpy, X, Y)

print 'Numba'.center(30, '-')
theta_numba, time_numba  = run(gradient_descent_numba, X, Y)

# make sure all method yields the same result
assert np.allclose(theta_python, theta_numba)

print 'Summary'.center(30, '=')
print 'Numba speedup %.1fx' % (time_python / time_numba)

plot(X, theta_numba, c='r')
pylab.show()

# <markdowncell>

#
# ## References
#
# [1] http://aimotion.blogspot.com/2011/10/machine-learning-with-python-linear.html


import numpy as np

def conv_nested(image, kernel):
    """A naive implementation of convolution filter.

    This is a naive implementation of convolution using 4 nested for-loops.
    This function computes convolution of an image with a kernel and outputs
    the result that has the same shape as the input image.

    Args:
        image: numpy array of shape (Hi, Wi)
        kernel: numpy array of shape (Hk, Wk)

    Returns:
        out: numpy array of shape (Hi, Wi)
    """
    Hi, Wi = image.shape
    Hk, Wk = kernel.shape
    out = np.zeros((Hi, Wi))

    # YOUR CODE HERE
    for i in range(Hi - 1):
        for j in range(Wi - 1):
            for m in range(Hk):
                for n in range(Wk):
                    out[i, j] = out[i, j] + \
                        image[i - m + 1, j - n + 1] * kernel[m, n]
    # END YOUR CODE

    return out

def zero_pad(image, pad_height, pad_width):
    """ Zero-pad an image.

    Ex: a 1x1 image [[1]] with pad_height = 1, pad_width = 2 becomes:

        [[0, 0, 0, 0, 0],
         [0, 0, 1, 0, 0],
         [0, 0, 0, 0, 0]]         of shape (3, 5)

    Args:
        image: numpy array of shape (H, W)
        pad_width: width of the zero padding (left and right padding)
        pad_height: height of the zero padding (bottom and top padding)

    Returns:
        out: numpy array of shape (H+2*pad_height, W+2*pad_width)
    """

    H, W = image.shape
    #out = None

    # YOUR CODE HERE
    out = np.zeros((pad_height * 2 + H, pad_width * 2 + W))
    for i in range(H):
        for j in range(W):
            out[i + pad_height, j + pad_width] = image[i, j]
    # END YOUR CODE
    return out

def conv_fast(image, kernel):
    """ An efficient implementation of convolution filter.

    This function uses element-wise multiplication and np.sum()
    to efficiently compute weighted sum of neighborhood at each
    pixel.

    Hints:
        - Use the zero_pad function you implemented above
        - There should be two nested for-loops
        - You may find np.flip() and np.sum() useful

    Args:
        image: numpy array of shape (Hi, Wi)
        kernel: numpy array of shape (Hk, Wk)

    Returns:
        out: numpy array of shape (Hi, Wi)
    """
    Hi, Wi = image.shape
    Hk, Wk = kernel.shape
    out = np.zeros((Hi, Wi))

    # YOUR CODE HERE
    image = zero_pad(image, Hk, Wk)
    kernel = np.flip(np.flip(kernel, 1), 0)
    for i in range(Hi):
        for j in range(Wi):
            out[i, j] = sum(
                sum(kernel * image[i + 2:i + 2 + Hk, j + 2:j + 2 + Wk]))
    # END YOUR CODE

    return out

def conv_faster(image, kernel):
    """
    Args:
        image: numpy array of shape (Hi, Wi)
        kernel: numpy array of shape (Hk, Wk)

    Returns:
        out: numpy array of shape (Hi, Wi)
    """
    Hi, Wi = image.shape
    Hk, Wk = kernel.shape
    out = np.zeros((Hi, Wi))

    # YOUR CODE HERE
    pass
    # END YOUR CODE

    return out

def cross_correlation(f, g):
    """ Cross-correlation of f and g

    Hint: use the conv_fast function defined above.

    Args:
        f: numpy array of shape (Hf, Wf)
        g: numpy array of shape (Hg, Wg)

    Returns:
        out: numpy array of shape (Hf, Wf)
    """

    out = None
    # YOUR CODE HERE
    g = np.flip(np.flip(g, 0), 1)
    out = conv_fast(f, g)

    # END YOUR CODE

    return out

def zero_mean_cross_correlation(f, g):
    """ Zero-mean cross-correlation of f and g

    Subtract the mean of g from g so that its mean becomes zero

    Args:
        f: numpy array of shape (Hf, Wf)
        g: numpy array of shape (Hg, Wg)

    Returns:
        out: numpy array of shape (Hf, Wf)
    """
    # YOUR CODE HERE
    tot = sum(sum(g))
    ave = tot / (g.shape[0] * g.shape[1])
    g = g - ave
    out = cross_correlation(f, g)
    # END YOUR CODE

    return out

def normalized_cross_correlation(f, g):
    """ Normalized cross-correlation of f and g

    Normalize the subimage of f and the template g at each step
    before computing the weighted sum of the two.

    Args:
        f: numpy array of shape (Hf, Wf)
        g: numpy array of shape (Hg, Wg)

    Returns:
        out: numpy array of shape (Hf, Wf)
    """

    # YOUR CODE HERE
#    print(np.shape(f));
#    print(np.shape(g));

#    sumf=sum(sum(f));
#    avef=sumf/(f.shape[0]*f.shape[1]);
#    sigf=sum(sum(np.abs(f-avef)))/(f.shape[0]*f.shape[1]);
#    nomf=(f-avef)/sigf;
    Hi, Wi = f.shape
    Hk, Wk = g.shape

    aveg = sum(sum(g))/ (Hk*Wk)
    sigg = sum(sum(np.abs(g - aveg))) / (Hk*Wk)
    kernel = (g - aveg) / sigg

    out = np.zeros((Hi, Wi))
    image = zero_pad(f, Hk, Wk)
    for i in range(Hi):
        for j in range(Wi):
            avef = sum(
                sum(image[i + 2:i + 2 + Hk, j + 2:j + 2 + Wk])) / (Hk * Wk)
            sigf = sum(
                sum(np.abs(image[i + 2:i + 2 + Hk, j + 2:j + 2 + Wk] - avef))) / (Hk * Wk)
            out[i, j] = sum(
                sum(kernel * (image[i + 2:i + 2 + Hk, j + 2:j + 2 + Wk] - avef) / sigf))

    # END YOUR CODE

    return out


import numpy as np

from typing import Tuple

from icecream import ic

ic.configureOutput(prefix="", outputFunction=print)

def bin_data(np_array_in: np.ndarray, nbr_bins: int) -> Tuple[np.ndarray, np.ndarray, np.ndarray]:
    """Get bins for binning the data, large enough that no edge effect issues - will always be possible
    to find right bin using np.searchsorted on the output bin edges.

    Arguments:
        np_array_in: the data to bin
        nbr_bins: the nbr of bins to use

    Returns:
        bin_limits: the limits of the bins; will be nbr_bins+1 long
        bin_mids: the middle of the bins; will be nbr_bins long
        array_bin_indexes: which bin each of the entries in np_array_in should got to; will be same
            length as np_array_in

    example of use:

    ```
    nbr_bins = 20

    bin_limits, bin_mids, array_bin_indexes = get_bins(some_array, nbr_bins)

    dict_entries_in_their_bin = {}
    for crrt_bin in range(nbr_bins):
        dict_entries_in_their_bin[crrt_bin] = []

    for crrt_bin_index, crrt_entry in enumerate(array_bin_indexes, some_array):
        dict_entries_in_their_bin[crrt_bin_index].append(crrt_entry)
    ```

    NOTES:
        could possibly be (partly) rewritten with np.digitize
        we already use np.searchsorted for speed
    """

    assert len(np_array_in.shape) == 1, "only implemented for input arrays of size 1"
    assert nbr_bins > 0, "need a positive number of bins"

    val_min = np.min(np_array_in)
    val_max = np.max(np_array_in)

    # need a bit wider bins
    if val_min == 0:
        val_min = -val_max * 0.001
    elif val_min >= 0:
        val_min = val_min * 0.99
    else:
        val_min = val_min * 1.01

    if val_max == 0:
        val_max = -val_min * 0.001
    elif val_max >= 0:
        val_max = val_max * 1.01
    else:
        val_max = val_max * 0.99

    bin_width = (val_max - val_min) / nbr_bins
    bin_limits = np.arange(val_min, val_max+bin_width/2.0, bin_width)
    bin_mids = (bin_limits[:-1] + bin_limits[1:]) / 2.0

    array_bin_indexes = np.searchsorted(bin_limits, np_array_in, side="left")

    return (bin_limits, bin_mids, array_bin_indexes)

array_test = np.array([34.5, 2.0, 23.3, 9.5, 11, 12, 14, 13, 13.9, 27.3])
nbr_bins = 4
bin_limits, bin_mids, array_bin_indexes = bin_data(array_test, nbr_bins)

ic(array_test)
ic(nbr_bins)
ic(bin_limits)
ic(bin_mids)
ic(array_bin_indexes)


'''
N_sim: number of simulations
T: horizon
dt: step length in years
sigma: volatility per year
mu: drift terms (moving average or long-term mean for stock returns)
S0: initial stock price
'''

from numpy.random import standard_normal
from numpy import array, zeros, sqrt, shape
from pylab import *

S0 = 10.222

T =1
dt =0.0002
sigma = 0.4
mu = 1
N_Sim = 10

Steps=round(T/dt); #Steps in years
S = zeros([N_Sim, Steps], dtype=float)
x = range(0, int(Steps), 1)

for j in range(0, N_Sim, 1):
        S[j,0]= S0
        for i in x[:-1]:
                S[j,i+1]=S[j,i]+S[j,i]*(mu-0.5*pow(sigma,2))*dt+sigma*S[j,i]*sqrt(dt)*standard_normal();
        plot(x, S[j])

title('Simulations %d Steps %d Sigma %.6f Mu %.6f S0 %.6f' % (int(N_Sim), int(Steps), sigma, mu, S0))
xlabel('steps')
ylabel('stock price')
show()


# Find the minimum value of the average cost for the given cost function on the given intervals.
# C(x) = x**3 + 29*x + 432

from sympy import solve, lambdify, symbols, diff, pprint, pretty
import matplotlib.pyplot as plt
import numpy as np

x = symbols('x', positive = True ) # 0 <= x
C = x**3 + 29*x + 432
avgC = C / x

domain_end = 20

g_xlim = [ 10, 20 ]
g_ylim = [-5, 70]

lam_x = lambdify( x, avgC, np )

x_vals = np.linspace( g_xlim[0], g_xlim[1], 1000, endpoint=True )
y_vals = lam_x( x_vals )

plt.plot( x_vals, y_vals, label = '' )

# The minimum value of the average cost over the interval 1 <= x <=10

daC = diff( avgC, x, 1 )

min = solve( daC, x )[ 0 ]

# Global Min
avgC.subs( { x: min } )

# Plotted Area Min
y_vals.min()

plt.show()


''' NOTE: The Pagerank Algorithm is essentially not a part of our defination. It has been used to illustrate the fact steady state is actually,
# what we intended to find through Page Rank Algorithm as well. The page rank algorithm has been imported from Wikepedia.'''

import numpy as np

def pagerank(M, num_iterations: int = 4500, d: float = 0.85):
    """PageRank: The trillion dollar algorithm."""
    N = M.shape[1]
    v = np.random.rand(N, 1)
    v = v / np.linalg.norm(v, 1)
    M_hat = (d * M + (1 - d) / N)
    for i in range(num_iterations):
        v = M_hat @ v
    return v

n = int(input("Enter the number of rows:"))
m = int(input("Enter the number of columns:"))
print("Enter the entries rowise (separated by space): ")
matrix=[]
for i in range(n):
    a=[]
    for j in range(m):
        a.append(float(input()))
    matrix.append(a)

M=np.array(matrix)

# v = pagerank(M, 100, 0.86)
v_absolute=pagerank(M, 100, 1)
print("Pagerank of given Network: ")
print(v_absolute)
# print("Real time Value: ")
# print(v)


#!/usr/bin/env python

if __name__ == '__main__':

    from argparse import ArgumentParser
    from math import exp, sqrt, sin, cos, atan2, pi
    import numpy as np

    min_val = -1.0
    max_val = 1.0
    default_points = 10
    arg_parser = ArgumentParser(description='produce a 3D vector field')
    arg_parser.add_argument('-x', type=int, default=default_points,
                            help='number of x points')
    arg_parser.add_argument('-y', type=int, default=default_points,
                            help='number of y points')
    arg_parser.add_argument('-z', type=int, default=default_points,
                            help='number of z points')
    options = arg_parser.parse_args()
    fmt_str = '{x:.4f},{y:.4f},{z:.4f},{v_x:.8f},{v_y:.8f},{v_z:.8f}'
    print 'x,y,z,v_x,v_y,v_z'
    for z in np.linspace(0.0, max_val, num=options.z, endpoint=True):
        for x in np.linspace(min_val, max_val, num=options.x, endpoint=True):
            for y in np.linspace(min_val, max_val, num=options.y, endpoint=True):
                d = sqrt(x**2 + y**2 + z**2)
                v = (1.0 - exp(-d))*exp(-d)
                theta = atan2(y, x)
                v_x = v*sin(-theta)
                v_y = v*cos(theta)
                v_z = 0.1*exp(-3.0*z)
                print fmt_str.format(x=x, y=y, z=z,
                                     v_x=v_x, v_y=v_y,v_z=v_z)


# Linear Regression
#%%
import numpy as np
import random
from matplotlib import pyplot as plt

#%%
class LinearRegression():
    def __init__(self):
        self.w = random.random()
        self.b = random.random()

    def train(self, X, y, print_params = False):
        """ X is M x 1 matrix and y is M x 1 matrix """
        alpha = 0.001
        thresh = 1.1
        i = 1
        m = y.shape[0]
        while(True):
            y_pred = self.w * X + self.b
            J_train = np.sum((y_pred - y)**2) / m
            dw = 2 * np.sum((y_pred - y) * X) / m
            db = 2 * np.sum(y_pred - y) / m
            self.w = self.w - alpha * dw
            self.b = self.b - alpha * db
            if print_params:
                print("Step #{0}".format(i))
                print("J_train:", J_train)
                print("w:", self.w, "b:", self.b)
                print()
            i += 1
            if J_train < thresh:
                print("=> Training is complete.")
                break

    def predict(self, x_test):
        """ Returns the prediction for all x_test """
        return self.w * x_test + self.b
#%%
# Generating random dataset
# Let's assume we have a straight line: y = 4 * x + 10,
# We need to create a dataset that approcimately represents this line.
x = np.random.randn(1000, 1) + 4
rand_bias = np.random.randn(1000, 1)
y = 4 * x + 10 + rand_bias
print("x is", x.shape)
print("y is", y.shape)

#%%
# Splitting dataset into training and test sets
x_train = x[:800,:]
y_train = y[:800,:]
x_test = x[800:,:]
y_test = y[800:,:]
print("x_train:", x_train.shape)
print("y_train:", y_train.shape)
print("x_test:", x_test.shape)
print("y_test:", y_test.shape)
#%%
# Plot test set examples
# plt.figure(1)
# plt.plot(x_test, y_test, "r*")
# plt.show()
#%%
# Create and traing a linear regression model
model = LinearRegression()
model.train(x_train, y_train)

#%%
# Generating predictions for test set
y_pred = model.predict(x_test)
J_test = np.sum((y_pred - y_test)**2) / y_pred.shape[0]
print("J_test:", J_test)

#%%
# Plotting the model against the data examples
xline = np.array([[np.min(x_train)], [np.max(x_train)]])
yline = model.predict(xline)
plt.figure(0)

plt.plot(x, y, "r*")
plt.plot(xline, yline)
plt.show()


import numpy as np
import matplotlib.pyplot as plt
from itertools import product

class BinomialTree:
    def __init__(self, S0, r, sigma, T, n):
        self.S0 = S0
        self.r = r
        self.sigma = sigma
        self.T = T
        self.n = n
        self.dt = self.T / self.n
        self.u = np.exp(self.sigma * np.sqrt(self.dt))
        self.d = 1./self.u
        self.p = (np.exp(self.r *self.dt) - self.d) / (self.u - self.d)

    def buildtree(self):
        S = np.zeros((self.n + 1, self.n + 1))
        S[0, 0] = self.S0
        for i in range(1, self.n + 1):
            S[i, 0] = S[i - 1, 0] * self.u
            for j in range(1, i + 1):
                S[i, j] = S[i - 1, j - 1] * self.d
        return S

    def tree2matrix(self):
        a = np.array(list(product((0, 1), repeat = self.n)))
        a = np.c_[[0] * 2 ** self.n, a]
        a = a.cumsum(axis = 1)
        return np.choose(a, self.buildtree().T)

    def putprice(self, K):
        putvalue = np.zeros((self.n + 1, self.n + 1))
        tree = self.buildtree()
        putvalue[self.n, :] = np.maximum(0, K - tree[self.n, :])
        for i in range(self.n - 1, -1, -1):
            for j in range(i + 1):
                putvalue[i, j] = max(0, K - tree[i, j], np.exp(-self.r * self.dt)*(self.p*putvalue[i+1, j] + (1 - self.p)*putvalue[i+1, j+1]))
        return putvalue

if __name__ == '__main__':
    print('This is a code for generate binomial tree and covert the paths to matrix')
    print("Let's see a basic example: ")
    S0, r, sigma, T, n = 36, 0.06, 0.2, 2, 15
    model = BinomialTree(S0, r, sigma, T, n)
    print('Now we are going to build a binomail tree:')
    tree = model.buildtree()
    print('Now we are going to convert to trajectories')
    trajectories = model.tree2matrix()
    plt.plot(trajectories.T)
    plt.show()
    print("let's calculate the american put option price from binomial tree")
    print('The American Put Option price is: ', model.putprice(40)[0,0])


# This is an implementation from scratch of the logistic regression

# Algorithm of logistic regression
# The main goal of logistic regression is to find the best coefficients, and intercept
# to make the prediction accurate
import numpy as np
from exam import hours_studied, passed_exam

# For some reason np array that start with an int will only be an int
coefficients = np.array([[1.1]])
intercept = np.array([1.0])
learning_rate = 0.0001

def log_odds(features, coefficients, intercept):
  """Return the log odds for all the features"""
  return np.dot(features,coefficients) + intercept

def sigmoid(z):
    """Simulate the sigmoid function and return a result"""
    denominator = 1 + np.exp(-z)
    return 1/denominator

def log_loss(probabilities, actual_class, Xi):
    """Return a summed log loss of all values in the feature dataset"""
    return np.sum(-(1/actual_class.shape[0])*(actual_class*np.log(probabilities) + (1-actual_class)*np.log(1-probabilities))*Xi)

def get_error_rate(predicted, actual):
    """Return the average error rate"""
    error = 0
    for i in range(len(predicted)):
        error += abs(predicted[i][0] - actual[i][0])
    return error/len(predicted)

def train(features, labels):
    n = len(features)
    error_rate = 1
    i = 0
    print("Training Started!")
    while error_rate > 0:
        print(f"Iteration={i}")
        # First get the probabilities
        calculated_log_odds = log_odds(features, coefficients, intercept)
        probabilities = sigmoid(calculated_log_odds)
        # Calculate log loss
        logloss = log_loss(probabilities, labels, features)
        # Apply gradient descent algorithm
        derivative_sigmoid = (1/n) * logloss
        coefficients[0][0] = coefficients[0][0] - learning_rate*derivative_sigmoid
        intercept[0] = intercept[0] - learning_rate*derivative_sigmoid
        print(f"coefficient={coefficients},intercept={intercept}")
        # Calculate error rate
        result = np.where(probabilities >= 0.5, 1, 0)
        old_error_rate = error_rate
        error_rate = get_error_rate(result,labels)
        print(f"[debug]error_rate:{error_rate}")
        i += 1

# Create predict_class() function
def predict(features, threshold):
  calculated_log_odds = log_odds(features, coefficients, intercept)
  probabilities = sigmoid(calculated_log_odds)
  result = np.where(probabilities >= threshold, 1, 0)
  return result

# First, train the model
train(hours_studied, passed_exam)

# Make final classifications on Codecademy University data here
final_results = predict(hours_studied, 0.5)
predicted = final_results.tolist()
print(f"predicted: {predicted}")
actual = passed_exam.tolist()
print(f"realY_val: {actual}")


import numpy as np

def sigmoid(x):
    return 1/(1+np.exp(-x))

def sigmoid_prime(x):
    return x*(1-x)

def relu(x):
    return x if x > 0 else 0

def relu_prime(x):
    return 1 if x > 0 else 0

class NeuralNet(object):
    def __init__(self, num_input, num_hidden_layers, num_neurons_per_layer, num_output, seed=-1):
        if seed != -1:
            np.random.seed(seed)
        self.weights = []

        self.weights.append(2*np.random.random((num_input, num_neurons_per_layer)) - 1)

        for i in xrange(num_hidden_layers-1):
            self.weights.append(2*np.random.random((num_neurons_per_layer, num_neurons_per_layer)) - 1)

        self.weights.append(2*np.random.random((num_neurons_per_layer, num_output)) - 1)

    def forward(self, inpt):
        past_input = [inpt]
        for weight in self.weights:
            past_input.append(sigmoid(np.dot(past_input[-1], weight)))

        return past_input

    def learn(self, inputs, outputs, rate):
        results = self.forward(inputs)

        deltas = []

        output_error = outputs - results[-1]
        deltas.append(output_error * sigmoid_prime(results[-1]))

        for index,result in enumerate(reversed(results[:-1])):
            error = np.dot(deltas[-1], self.weights[-index-1].T)
            deltas.append(error * sigmoid_prime(result))

        deltas.reverse()

        for index,weight in enumerate(self.weights):
            weight += np.dot(results[index].T, deltas[index+1]) * rate

    def error(self, inputs, outputs):
        results = self.forward(inputs)

        output_error = outputs - results[-1]

        return np.mean(np.abs(output_error))

    def output_dot(self, error=None):
        output = []
        output.append("digraph {")
        for index,weight in enumerate(self.weights):
            if index == 0:
                prefix = "I"
                nextprefix = "H"
            elif index == len(self.weights)-1:
                prefix = "H"
                nextprefix = "O"
            else:
                prefix = "H"
                nextprefix = "H"
            for row in xrange(weight.shape[0]):
                for col in xrange(weight.shape[1]):
                    output.append("{}{} -> {}{} [color=\"{},{},0.6\"]".format(
                        prefix, row,
                        nextprefix, col,
                        0.0 if np.sign(weight[row][col]) == -1 else 0.7,
                        round((sigmoid(np.abs(weight[row][col]))-0.5)*2, 4),
                    ))
        if error is not None:
            output.append("error [label=\"Error:\n{}%\", shape=plaintext]".format(round(self.error(error[0], error[1])*100, 2)))
            output.append("O0 -> error [style=invis]")
            output.append("O{} -> error [style=invis]".format(self.weights[-1].shape[1]-1))
        output.append("}")
        return "\n".join(output)


#T# linear equations in one variable contain only one variable and its exponent is 1, said variable can be repeated accross several different terms

#T# to simplify linear equations in one variable, the sympy package is used
import sympy

#T# the simplify function is used to simplify linear equations in one variable
x = sympy.Symbol('x')              # x
eq1 = sympy.Eq(6*x - 11, -3*x + 7) # Eq(6*x - 11, 7 - 3*x)
eq2 = sympy.simplify(eq1)          # Eq(x, 2)
#| when simplifying a linear equation in one variable, the simplification incidentally solves for the variable, the solution in this case is x = 2


#!/usr/bin/env python3
# vim: set ts=4:
#
# Assignment:	Image Enhancement and Filtering
# Course:		SCC0251
# Semester:		First of 2020
# Group:
#	<name>					<nUSP>
# 	***REMOVED***                       ***REMOVED***
#	***REMOVED***                       ***REMOVED***
#

import imageio
import math
import numpy as np

def pad_image(image, amount):
	h, w = image.shape						  # height, width
	nh, nw = h + amount * 2, w + amount * 2   # new values
	padded = np.zeros([nh, nw])
	padded[amount:amount + h, amount:amount + w] += image
	return padded

######################
## Bilateral filter ##
######################

def G(x, sigma):
	"""
Calculates a gaussian kernel"""
	return ((1 / (2 * math.pi * pow(sigma, 2))) *
			math.exp(-(pow(x, 2) / (2 * pow(sigma, 2)))))

def spatial_component(n, sigma):
	"""
Calculates the spatial gaussian kernel
Arguments:
	n	  -- filter size
	sigma -- variance"""
	center = int((n - 1) / 2) # index
	def dist(x, y):
		return math.sqrt(pow(x - center, 2) + pow(y - center, 2))
	matrix = np.zeros([n, n])
	for i in range(n):
		for j in range(n):
			matrix[i, j] = G(dist(i, j), sigma)
	return matrix

def range_component(region, n, sigma):
	"""
Calculates the range gaussian kernel of a region of the image
Arguments:
	region -- region of the image to use
	n	   -- filter size
	sigma  -- variance"""
	center = int((n - 1) / 2)
	pixel = region[center, center]
	matrix = np.zeros([n, n])
	for i in range(n):
		for j in range(n):
			Ii = region[i, j]
			matrix[i, j] = G(Ii - pixel, sigma)
	return matrix

def bilateral(image, n, sigma_s, sigma_r):
	"""
Bilateral Filter
Arguments:
	image -- input image
	n	  -- size of the filter
	σs	  -- space variance
	σr	  -- range variance"""

	gs = spatial_component(n, sigma_s)

	padding_needed = math.floor(n / 2)

	result = np.empty_like(image)
	working_area = pad_image(image, padding_needed)

	for i in range(image.shape[0]):
		for j in range(image.shape[1]):
			region = working_area[i:i + n, j:j + n]
			gr = range_component(region, n, sigma_r)
			w = gr * gs
			Wp = np.sum(w)
			If = np.sum(w * region)
			result[i, j] = If / Wp

	return result

####################
## Unsharp filter ##
####################

kernels = [
	np.array([[ 0, -1,	0], [-1,  4, -1], [ 0, -1,	0]]),
	np.array([[-1, -1, -1], [-1,  8, -1], [-1, -1, -1]])
]

def normalize(image):
	return ((image - np.min(image)) * 255) / (np.max(image) - np.min(image))

def unsharp(image, c, k):
	"""
Unsharp mask using the Laplacian Filter
Arguments:
	iamge -- input image
	c	  -- scaling factor
	k	  -- kernel to use"""
	n = 3	# kernels are n x n

	kernel = kernels[k - 1]

	working_area = pad_image(image, math.floor(n / 2))
	filtered = np.empty_like(image)

	# 1. Convolve original image with kernel, storing into `filtered`
	for i in range(image.shape[0]):
		for j in range(image.shape[1]):
			region = working_area[i:i + n, j:j + n]
			filtered[i, j] = np.sum(region * kernel)

	# 2. Scale the filtered image
	filtered = normalize(filtered)

	# 3. Add the filtered image, multiplied by c, to the original image
	result = filtered * c + image

	# 4. Scale the final image
	return normalize(result)

#####################
## Vignette filter ##
#####################

def gauss(x, sig):
	"""
	Samples the value at a single point from the Gaussian Curve, parameterized
	using the sig value as its sigma.
		x	- Relative positioning along the curve, which is centered at x = 0.
		sig - Sigma parameter.
	"""
	from math import pi as PI;
	from math import exp;
	return exp(-(x ** 2 / 2 * sig ** 2)) / (2 * PI * sig ** 2)

def vinagrette(img, sigma_row, sigma_col):
	"""
	Applies a vignette filter to a desired source image, returning the result.
		img		  - Source image.
		sigma_row - Sigma parameter for the curve along the vertical axis.
		sigma_col - Sigma parameter for the curve along the horizontal axis.
	"""
	width  = len(img[0])
	height = len(img)
	target = img.astype("float")
	maxval = None
	minval = None

	# Generate the combined gaussian matrix for the image and apply it as we go.
	# We also gather the minimum and maximum resulting values so that we can
	# normalize the results later on back to the 0-255 range.
	for i in range(height):
		for j in range(width):
			a = gauss(int(j - width  / 2), 1 / sigma_col)
			b = gauss(int(i - height / 2), 1 / sigma_row)
			target[i][j] *= a * b

			if maxval is None or target[i][j] > maxval: maxval = target[i][j]
			if minval is None or target[i][j] < minval: minval = target[i][j]

	# Normalize the image
	for i in range(height):
		for j in range(width):
			a = minval
			b = maxval
			c = 0
			d = 255
			target[i][j] = (target[i][j] - a) * ((d - c) / (b - a)) + c

	return target

###########################
## General functionality ##
###########################

def RSE(m, r):
	return math.sqrt(np.square(m - r).sum())

if __name__ == "__main__":
	filename = str(input()).rstrip()
	img = imageio.imread(filename).astype(np.float)
	method = int(input())
	save = int(input())

	if method == 1:
		n = int(input())
		sigma_s = float(input())
		sigma_r = float(input())
		output_img = bilateral(img, n, sigma_s, sigma_r)
	elif method == 2:
		c = float(input())
		k = int(input())
		output_img = unsharp(img, c, k)
	elif method == 3:
		sigma_row = float(input())
		sigma_col = float(input())
		output_img = vinagrette(img, sigma_row, sigma_col)
	else:
		raise Exception("Method " + str(method) + " unimplemented")

	output_img = output_img.astype(np.uint8)

	print("{0:.4f}".format(RSE(output_img, img)))

	if save == 1:
		imageio.imwrite("output_img.png", output_img)


import numpy as np
import iqplot
import bokeh.io
import bokeh
import scipy
import bebi103
import scipy.stats as st

def __ecdf(x, data):
    """Give the value of an ECDF at arbitrary points x

    Parameters
    __________
    x : array
    points to calculate ECDF for

    data : array
    input data to generate the ECDF based on

    Returns
    __________
    output : array of ECDF values for each point in x
    """
    y = np.arange(len(data) + 1) / len(data)
    return y[np.searchsorted(np.sort(data), x, side="right")]

def draw_bs_sample(data):
    '''Draw a boostrap sample from a 1D data set
    Parameters
    __________
    data : array
        input data to sample from

    Returns
    __________
    output: array
        new dataset drawn from the original dataset
    '''
    return np.random.choice(data, size = len(data))

def draw_bs_reps(data, stat_fun, size=1):
    '''Draw boostrap replicates computed with stat_fun from 1D dataset
    Parameters
    __________
    data : array
        input data to sample from
    stat_fun : function
        statistical function to calculate on each dataset replicate
    size : int, optional, default = 1
        number of dataset replicates to generate

    Returns
    __________
    output: array
        test statistic for each of size bootstrap replicate(s)
    '''
    return np.array([stat_fun(draw_bs_sample(data)) for _ in range (size)])

def draw_bs_reps_mean(data, size=1):
    '''Draw boostrap replicates of the mean from 1D data set
    Parameters
    __________
    data : array
        input data to sample from
    size : int, default = 1
        number of datasets to draw

    Returns
    __________
    output: array
        mean of new dataset drawn from the original dataset
    '''
    out = np.empty(size)
    for i in range(size):
        out[i] = np.mean(draw_bs_sample(data))
    return out

def draw_bs_reps_test_stat(x, y, size=1):
    """
    Generate bootstrap replicates with Kolmogrov Smirnov
    as the test statistic
    Parameters
    __________
    x : array
        input dataset 1 to sample from
    y : array
        input dataset 2

    Returns
    __________
    output: array
        Kolmogorov Smirnov test statistic for each trial
    """
    out = np.empty(size)
    for i in range(size):
        out[i] = st.ks_2samp(draw_bs_sample(x), draw_bs_sample(y))[0]

    return out

def __L(x, epsilon, data):
    '''Calculates L(x) for a given x value, epsilon, and data set

    Parameters
    ___________
    x : array
        points to calculate L(x) for
    epsilon : float
        epsilon value to use in equation
    data : array
        raw data

    Returns
    ___________
    output : array
        value of L(x) for each point in x
    '''
    ecdf_vals = __ecdf(x, data)

    return [max(0, val - epsilon) for val in ecdf_vals]

def __U(x, epsilon, data):
    '''Calculates U(x) for a given x value, epsilon, and data set
    x : array
        points to calculate U(x) for
    epsilon : float
        epsilon value to use in equation
    data : array
        raw data

    Returns
    ___________
    output : array
        value of U(x) for each point in x
    '''
    ecdf_vals = __ecdf(x, data)

    return [min(1, val + epsilon) for val in ecdf_vals]

def plot_conf_int(data, title, xlabel, color = 'green', palette = ["limegreen"]):
    ''' plots an ECDF with confidence intervals
    data : array
        contains raw data points from experiment
    title : string
        title for output graph
    xlabel : string
        x axis label for output graph
    color : string (optional)
        if given, color for upper and lower confidence interval bounds

    Returns
    ________
    output : bokeh figure
    '''
    x = np.linspace(0, max(data), 100)
    epsilon = np.sqrt(np.log(2/0.05)/(2 * len(data)))
    p = bokeh.plotting.figure(title=title, x_axis_label=xlabel,
                   y_axis_label='ECDF', width = 400, height = 400)
    l = __L(x, epsilon, data)
    u = __U(x, epsilon, data)
    p.line(x, l, color = color)
    p.line(x, u, color = color)
    # overlay with experimental ECDF
    iqplot.ecdf(data, p = p, conf_int = True, palette = palette)
    return p

def viz_compare_conf_int(data, data2, fun = np.mean, xlabel = "mean", title = "Comparison of Confidence Intervals",
                         label1 = "sample1", label2 = "sample2", size = 1000, color = "dimgray"):
    """Function to visualize overlap between two confidence intervals
    Parameters
    _________

    data : array
        first dataset
    data2 : array
        second dataset
    fun : statistical function (optional), default is mean
        if given, statistical function to calculate for each dataset
    label : string (optional), default is "mean"
        if given, used as axis label for confidence intervals graph
    size : int (optional), default = 1000
        number of samples for each dataset
    color : string (optional), default = "dimgray"
        if given, color of intervals in visualization

    Returns
    _________
    output : bokeh figure containing the visualization

    """
    bs_reps = draw_bs_reps(data, fun, size = size)
    bs_reps2 = draw_bs_reps(data2, fun, size = size)
    mean = np.mean(bs_reps)
    mean2 = np.mean(bs_reps2)
    conf_int = np.percentile(bs_reps, [2.5, 97.5])
    conf_int2 = np.percentile(bs_reps2, [2.5, 97.5])
    dict1 = {"estimate": mean, "conf_int": conf_int, "label" : label1}
    dict2 = {"estimate":mean2, "conf_int":conf_int2, "label" : label2}
    p = bebi103.viz.confints([dict1, dict2],
                             marker_kwargs = {"color":color},
                            line_kwargs = {"color":color})
    p.xaxis.axis_label = xlabel
    p.title.text = title
    return p


import numpy as np

def function_2(x):
    return x[0]**2 + x[1]**2

def numerical_gradient(f, x):
    h = 1e-4
    grad = np.zeros_like(x)

    for idx in range(x.size):
        tmp_val = x[idx]

        x[idx] = tmp_val + h
        fxh1 = f(x)

        x[idx] = tmp_val - h
        fxh2 = f(x)

        grad[idx] = (fxh1 - fxh2) / (2*h)
        x[idx] = tmp_val

    return grad

def gradient_descent(f, init_x, lr=0.01, step_num=1000):
    x = init_x

    for i in range(step_num):
        grad = numerical_gradient(f, x)
        x -= lr * grad

    return x

init_x = np.array([-3.,4.])
print(function_2(init_x))
print(numerical_gradient(function_2,init_x))
print(gradient_descent(function_2, init_x))


import numpy as np
import matplotlib.pyplot as plt

N = 14
n = np.arange(N)
fn=(-1/2)**n
hn1=np.pad(fn, (0,2), 'constant', constant_values=(0))
hn2=np.pad(fn, (2,0), 'constant', constant_values=(0))
h = hn1+hn2

xtemp=np.array([1.0,2.0,3.0,4.0,2.0,1.0])
x=np.pad(xtemp, (0,8), 'constant', constant_values=(0))

X = np.zeros(N) + 1j*np.zeros(N)
for k in range(0,N):
	for n in range(0,N):
		X[k]+=x[n]*np.exp(-1j*2*np.pi*n*k/N)
H = np.zeros(N) + 1j*np.zeros(N)
for k in range(0,N):
	for n in range(0,N):
		H[k]+=h[n]*np.exp(-1j*2*np.pi*n*k/N)

Y = np.zeros(N) + 1j*np.zeros(N)
for k in range(0,N):
	Y[k] = X[k]*H[k]

y = np.zeros(N) + 1j*np.zeros(N)
for k in range(0,N):
	for n in range(0,N):
		y[k]+=Y[n]*np.exp(1j*2*np.pi*n*k/N)

#print(X)
y = np.real(y)/N
#plots
plt.stem(range(0,N),y)
plt.title('Filter Output using DFT')
plt.xlabel('$n$')
plt.ylabel('$y(n)$')
plt.grid()# minor

plt.savefig('yndft.png')


import math
from typing import List, Tuple

import numpy as np
from scipy import fftpack as sff
from scipy.special import erfc as erfc

class SpectralTest:

    @staticmethod
    def spectral_test(binary_data:str, verbose=False) -> List[Tuple[str, float, bool]]:
        """
        Note that this description is taken from the NIST documentation [1]
        [1] http://csrc.nist.gov/publications/nistpubs/800-22-rev1a/SP800-22rev1a.pdf
        The focus of this test is the peak heights in the Discrete Fourier Transform of the sequence. The purpose of
        this test is to detect periodic features (i.e., repetitive patterns that are near each other) in the tested
        sequence that would indicate a deviation from the assumption of randomness. The intention is to detect whether
        the number of peaks exceeding the 95 % threshold is significantly different than 5 %.

        :param      binary_data:        The seuqnce of bit being tested
        :param      verbose             True to display the debug messgae, False to turn off debug message
        :return:    (p_value, bool)     A tuple which contain the p_value and result of frequency_test(True or False)
        """
        length_of_binary_data = len(binary_data)
        plus_one_minus_one = []

        # Step 1 - The zeros and ones of the input sequence (ε) are converted to values of –1 and +1
        # to create the sequence X = x1, x2, …, xn, where xi = 2εi – 1.
        for char in binary_data:
            if char == '0':
                plus_one_minus_one.append(-1)
            elif char == '1':
                plus_one_minus_one.append(1)

        # Step 2 - Apply a Discrete Fourier transform (DFT) on X to produce: S = DFT(X).
        # A sequence of complex variables is produced which represents periodic
        # components of the sequence of bits at different frequencies
        spectral = sff.fft(plus_one_minus_one)

        # Step 3 - Calculate M = modulus(S´) ≡ |S'|, where S´ is the substring consisting of the first n/2
        # elements in S, and the modulus function produces a sequence of peak heights.
        slice = math.floor(length_of_binary_data / 2)
        modulus = abs(spectral[0:slice])

        # Step 4 - Compute T = sqrt(log(1 / 0.05) * length_of_string) the 95 % peak height threshold value.
        # Under an assumption of randomness, 95 % of the values obtained from the test should not exceed T.
        tau = math.sqrt(math.log(1 / 0.05) * length_of_binary_data)

        # Step 5 - Compute N0 = .95n/2. N0 is the expected theoretical (95 %) number of peaks
        # (under the assumption of randomness) that are less than T.
        n0 = 0.95 * (length_of_binary_data / 2)

        # Step 6 - Compute N1 = the actual observed number of peaks in M that are less than T.
        n1 = len(np.where(modulus < tau)[0])

        # Step 7 - Compute d = (n_1 - n_0) / sqrt (length_of_string * (0.95) * (0.05) / 4)
        d = (n1 - n0) / math.sqrt(length_of_binary_data * (0.95) * (0.05) / 4)

        # Step 8 - Compute p_value = erfc(abs(d)/sqrt(2))
        p_value = erfc(math.fabs(d) / math.sqrt(2))

        if verbose:
            print("Discrete Fourier Transform (Spectral) Test:")
            print("  {:<40}{:>20}".format("Length of Binary Data:", length_of_binary_data))
            print("  {:<40}{:>20}".format("Value of T:", tau))
            print("  {:<40}{:>20}".format("Value of n1:", n1))
            print("  {:<40}{:>20}".format("Value of n0:", n0))
            print("  {:<40}{:>20}".format("Value of d:", d))
            print("  {:<40}{:>20}".format("P-Value:", p_value))

        return [("spectral_test", p_value, (p_value >= 0.01))]


import numpy as np
import matplotlib.pyplot as plt
from matplotlib.animation import FuncAnimation, writers

# Set up formatting for the movie files
Writer = writers['ffmpeg']
writer = Writer(fps=20, metadata=dict(artist='Llew'), bitrate=1800)

xmin,xmax = -3,3
ymin,ymax = -20,20
fig, ax = plt.subplots(figsize=(10,8))
ydata= ydata2 = np.linspace(ymin,ymax, 1000)
ln, = plt.plot([], [], 'r.',
                    markersize=1.5,
                    animated=True,
                    label=r'x-nullcline: $x = \frac{y}{(\mu+y^2)}$')
ln2, = plt.plot([],[],
                    'b:',
                    lw=.5,
                    animated=True, label=r'y-nullcline: $x =  -\mu y + y^3}$')
ax.set_facecolor(plt.cm.gray(.95))
ax.set_title('Animation of Hopf Bifurcation for $\mu = \{-1.5 ... 3\}$')
ax.set_xlabel(r'$x$', size=14)
ax.set_ylabel(r'$y$', size=14)

def init():
    ax.set_xlim(xmin, xmax)
    ax.set_ylim(ymin, ymax)
    return ln, ln2,

def update(frame):
    xdata1 = ydata/(frame+ydata**2)
    xdata2 = -frame*ydata + ydata**3
    ln.set_data(ydata, xdata1)
    ln2.set_data(ydata,xdata2)
    return ln, ln2,

ani = FuncAnimation(fig, update, frames=np.linspace(-1.5,3, 500),
                    init_func=init, interval=60, blit=True)
plt.grid(True)
plt.legend()
#ani.save('Hopf_8_3_2.mp4', writer=writer)
plt.show()


## 2. ReLU Activation Function ##

import matplotlib.pyplot as plt

import numpy as np
x = np.linspace(-2, 2, 20)

def relu(x):
    return np.maximum(0,x)
relu_y = relu(x)
print(x)
print(relu_y)
plt.plot(x,relu_y)
plt.show()

## 3. Trigonometric Functions ##

x = np.linspace(-2*np.pi, 2*np.pi, 100)

tan_y =np.tan(x)
print(x, tan_y)
plt.plot(x, tan_y)

## 5. Hyperbolic Tangent Function ##

x = np.linspace(-40, 40, 100)
tanh_y = np.tanh(x)
print(x)
print(tanh_y)

plt.plot(x, tanh_y)

7. Machine Learning Intermediate/deep-learning-fundamentals/Nonlinear Activation Functions-242.py


import numpy as np
from numbers import Number

from ya_pca.rank_selection.noise_estimates import estimate_noise
from ya_pca.linalg_utils import svd_wrapper

def select_rank_rmt_threshold(X, thresh_method='dg',
                              noise_est='mp',
                              noise_est_kwargs={},
                              UDV=None):
    """
    Selects the PCA rank using one of the random matrix theory based thresholds.

    Parameters
    ----------
    X: array-like, (n_samples, n_features)
        The data matrix.

    thresh_method: str
        Which thesholding method to use. Must be one of
        'dg': Donoho-Gavish method

        'mpe': Marcenko Pastur edge method

    noise_est: float, str
        Either the noise estimate or a method to estimate the noise.

    noise_est_kwargs: dict
        Key word arguments for noise estimation method.

    UDV: None, tuple
        Precomputed SVD.

    Output
    ------
    rank_est, out

    rank_est: int
        Estimated rank

    out: dict
        All output from rank selection algorithm.
    """

    if UDV is None:
        UDV = svd_wrapper(X)

    if not isinstance(noise_est, Number):
        noise_est = estimate_noise(X, method=noise_est,
                                   UDV=UDV, **noise_est_kwargs)

    assert thresh_method in ['mpe', 'dg']

    if thresh_method == 'dg':
        thresh = donoho_gavish_threshold(shape=X.shape,
                                         sigma=noise_est)

    elif thresh_method == 'mpe':
        thresh = marcenko_pastur_edge_threshold(shape=X.shape,
                                                sigma=noise_est)

    svals = UDV[1]
    rank_est = sum(np.array(svals) > thresh)

    return rank_est, {'svals': svals,
                      'shape': X.shape,
                      'thresh': thresh,
                      'noise_estimate': noise_est}

def marcenko_pastur_edge_threshold(shape, sigma):
    """
    See (Gavish and Donoho, 2014).

    Parameters
    ----------
    shape: tuple (n_samples, n_features)
        Shape of the data matrix.

    sigma: float
        Estiamte of the noise standard deviation.

    Output
    ------
    sigular_value_threshold: float

    """
    n_samples, n_features = shape
    beta = n_features / n_samples
    mult = n_samples

    if beta > 1:
        beta = 1 / beta
        mult = n_features

    return (1 + np.sqrt(beta)) * np.sqrt(mult) * sigma

def dg_threshold(beta):
    return np.sqrt(2 * (beta + 1) + 8 * beta / (beta + 1 + np.sqrt(beta ** 2 + 14 * beta + 1)))

def donoho_gavish_threshold(shape, sigma):
    """
    (Gavish and Donoho, 2014)

    Parameters
    ----------
    shape: tuple (n_samples, n_features)
        Shape of the data matrix.

    sigma: float
        Estiamte of the noise standard deviation.

    Output
    ------
    sigular_value_threshold: float

    """
    n_samples, n_features = shape
    beta = n_features / n_samples
    mult = n_samples

    # TODO: is this what we want to do?
    if beta > 1:
        beta = 1 / beta
        mult = n_features

    if n_samples == n_features:
        lambd = 4 / np.sqrt(3)
    else:
        lambd = dg_threshold(beta)

    return lambd * np.sqrt(mult) * sigma


from pydeep.core import Function
import numpy as np

class Loss(Function):

    def backward(self):

        gx = self.cache["local_gx"]
        return gx

class CrossEntropyLoss(Loss):

    def forward(self, x, y):
        """
        Descr.: calculate the mean loss of the batch

        :param x:   final layer of the network (batch_size, n_dim),
                    no need for softmax as we calculate it here
        :param y:   truth values (batch_size, 1), each entry in the
                    array is the an integer indicating the class
                    of the data point
        :return:    the mean cross entropy loss of the batch
        """

        exp_x = np.exp(x)
        probs = exp_x / np.sum(exp_x, axis=1, keepdims=True)
        logprobs = -np.log([probs[i, y[i]] for i in range(len(probs))])
        crossentropy_loss = np.mean(logprobs)

        self.cache["probs"] = probs
        self.cache['y'] = y

        return crossentropy_loss

    def local_grad(self, x, y):
        """
        Descr.:     cache the local gradient for backprop.

        :param x:   final layer of the network (batch_size, n_dim),
                    no need for softmax as we calculate it here
        :param y:   truth values (batch_size, 1), each entry in the
                    array is the an integer indicating the class
                    of the data point
        """

        probs = self.cache["probs"]
        truth = np.zeros_like(probs)
        for i, j in enumerate(y):
            truth[i, j] = 1.0

        self.cache["local_gx"] = (probs - truth) / float(len(x))

class MSELoss(Loss):

    def forward(self, x, y):
        """
        Descr.:     calculate mean loss of the batch, the avg. loss of the current batch

        :param x:   predicted values, (batch_size, 1)
        :param y:   truth values, (batch_sze, 1)
        :return:    scalar value, the mse of the batch
        """
        loss = ((x - y)**2).mean()
        return loss

    def local_grad(self, x, y):
        """
        Descr.: calulate dL/dx, and cache it for backprop.

        :param x: predicted values, (batch_size, 1)
        :param y: truth values, (batch_sze, 1)
        """
        self.cache["local_gx"] = 2 * (x - y) / x.shape[0]


"""Rotating cartesian points that live on the unit sphere
around a random or specified axis by a random or specified amount"""
from numpy.random import rand, randn
from numpy import zeros, cos, sin, pi, multiply, matmul
from numpy.linalg import norm

# pylint: disable=C0103

def rotate(axis, angle, xyz):
    """Rotates xyz around axis by angle."""
    rot = rotationmatrix(axis, angle)
    if len(xyz.shape) == 1:  # just one point
        if not len(xyz) == 3:
            raise ValueError("Points have to either be of shape (3,) or (n,3). ")
        return multiply(rot, xyz)
    if len(xyz.shape) == 2:
        # we assume xyz to be of shape n x 3
        if not xyz.shape[1] == 3:
            raise ValueError("Points have to either be of shape (3,) or (n,3). ")

        return matmul(rot, xyz.T).T

    raise ValueError("Points have to either be of shape (3,) or (n,3). ")

def randomaxisrotate(angle, xyz):
    """Rotates xyz around a random axis by angle."""

    # get random point on unit sphere
    axis = randn(3)
    axis = axis / norm(axis)
    return rotate(axis, angle, xyz)

def randomanglerotate(axis, xyz):
    """Rotates xyz around axis by a random angle."""
    angle = 2 * pi * rand()
    return rotate(axis, angle, xyz)

def randomrotate(xyz):
    """Rotates xyz around a random axis by a random angle."""
    # get random point on unit sphere
    axis = randn(3)
    axis = axis / norm(axis)
    angle = 2 * pi * rand()
    return rotate(axis, angle, xyz)

def rotationmatrix(axis, angle):
    """Returns the rotation matrix of size 3x3 that rotates a point
    around axis by angle.
    See: https://en.wikipedia.org/wiki/Rotation_matrix
    """
    ux = axis[0]
    uy = axis[1]
    uz = axis[2]

    costheta = cos(angle)
    sintheta = sin(angle)
    rot = zeros((3, 3))

    rot[0, 0] = ux * ux * (1 - costheta) + costheta
    rot[0, 1] = ux * uy * (1 - costheta) - uz * sintheta
    rot[0, 2] = ux * uz * (1 - costheta) + uy * sintheta

    rot[1, 0] = uy * ux * (1 - costheta) + uz * sintheta
    rot[1, 1] = uy * uy * (1 - costheta) + costheta
    rot[1, 2] = uy * uz * (1 - costheta) - ux * sintheta

    rot[2, 0] = uz * ux * (1 - costheta) - uy * sintheta
    rot[2, 1] = uz * uy * (1 - costheta) + ux * sintheta
    rot[2, 2] = uz * uz * (1 - costheta) + costheta

    return rot


#
# line segment intersection using vectors
# see Computer Graphics by F.S. Hill
#
from numpy import *
import sys

def perp( a ) :
    b = empty_like(a)
    b[0] = -a[1]
    b[1] = a[0]
    return b

# line segment a given by endpoints a1, a2
# line segment b given by endpoints b1, b2
# return
def intersectPoint( a1,a2, b1,b2 ) :
    a1 = array(a1)
    a2 = array(a2)
    b1 = array(b1)
    b2 = array(b2)
    da = a2-a1
    db = b2-b1
    dp = a1-b1
    dap = perp(da)
    denom = dot( dap, db)
    if not denom : return None
    num = dot( dap, dp )
    return (num / denom.astype(float))*db + b1

def isBetween( a, b, c):
    a = array(a)
    b = array(b)
    c = array(c)
    cma = c-a
    bma = b-a
    crossproduct = cma[1]*bma[0] - cma[0]*bma[1]
    if abs(crossproduct) > finfo(float32).eps:
        #print 'crossFail', abs(crossproduct), finfo(float32).eps
        return False

    dotproduct = cma[0]*bma[0] + cma[1]*bma[1]
    if dotproduct < 0:
        #print 'dotFail', dotProduct
        return False
    squaredLength = pow( bma[0], 2) + pow( bma[1], 2 )
    if dotproduct > squaredLength:
        #print 'squareFail',dotproduct,  squaredLength
        return False

    return True

def intersect( a1,a2, b1,b2 ) :
    point = intersectPoint( a1,a2, b1,b2 )
    #print 'point', point
    if point is None: return None

    #print 'a', isBetween(a1, a2, point)
    #print 'b', isBetween(b1, b2, point)
    if not isBetween(a1, a2, point): return None
    if not isBetween(b1, b2, point): return None

    return point

def dist( a,b, c ) :
    a = array(a)
    b = array(b)
    c = array(c)

    lineA = a-b
    lineADist = sqrt( pow(lineA[0], 2) + pow(lineA[1], 2) )
    lineB = a-c
    lineBDist = sqrt( pow(lineB[0], 2) + pow(lineB[1], 2) )

    print 'dists', lineADist, lineBDist
    return lineBDist/lineADist


"""
Specific useful functions.
"""

from __future__ import division
from ..utils.py3 import textstring

import numpy as np

### Kernels ###

def gaussian_k(x, sigma=1., height=None):
    """
    Gaussian kernel function.

    Normalized by the area if `height` is ``None``, otherwise `height` is the value at 0.
    """
    if height is None:
        return 1./np.sqrt(2*np.pi*sigma**2)*np.exp(-(x**2)/(2.*sigma**2))
    else:
        return np.exp(-x**2/(2.*sigma**2))*height

def rectangle_k(x, width=1., height=None):
    """"
    Symmetric rectangle kernel function.

    Normalized by the area if `height` is ``None``, otherwise `height` is the value at 0.
    """
    if height is None:
        return (abs(x)<width)/(2.*width)
    else:
        return (abs(x)<width)*height

def lorentzian_k(x, gamma=1., height=None):
    """
    Lorentzian kernel function

    Normalized by the area if `height` is ``None``, otherwise `height` is the value at 0.
    """
    if (height is None):
        return gamma/(x**2+(gamma/2.)**2)/np.pi
    else:
        return (gamma/2.)**2/(x**2+(gamma/2.)**2)*height

def complex_lorentzian_k(x, gamma=1., amplitude=1.j):
    """
    Complex Lorentzian kernel function.
    """
    return (gamma/2.)/(gamma/2.+1j*x)*amplitude

def exp_decay_k(x, width=1., height=None, mode="causal"):
    """
    Exponential decay kernel function

    Normalized by area if ``height=None`` (if possible), otherwise `height` is the value at 0.

    Mode determines value for ``x<0``:
        - ``'causal'`` - it's 0 for ``x<0``;
        - ``'step'`` - it's constant for ``x<=0``;
        - ``'continue'`` - it's a continuous decaying exponent;
        - ``'mirror'`` - function is symmetric: ``exp(-|x|/width)``.
    """
    if mode=="causal":
        if height is None:
            return (x>=0)*np.exp(-x/float(width))/width
        else:
            return (x>=0)*np.exp(-x/float(width))*height
    elif mode=="step":
        if height is None:
            return ((x>=0)*np.exp(-x/float(width))+(x<0)*1.)/width
        else:
            return ((x>=0)*np.exp(-x/float(width))+(x<0)*1.)*height
    elif mode=="continue":
        if height is None:
            return np.exp(-x/float(width))/width
        else:
            return np.exp(-x/float(width))*height
    elif mode=="mirror":
        if height is None:
            return np.exp(-np.abs(x)/float(width))/width
        else:
            return np.exp(-np.abs(x)/float(width))*height

_kernel_functions={"gaussian":gaussian_k, "rectangle":rectangle_k, "lorentzian":lorentzian_k, "exp_decay":exp_decay_k,
                   "complex_lorentzian":complex_lorentzian_k}
def get_kernel_func(kernel):
    """
    Get a kernel function by its name.

    Available functions are: ``'gaussian'``, ``'rectangle'``, ``'lorentzian'``, ``'exp_decay'``, ``'complex_lorentzian'``.
    """
    if isinstance(kernel,textstring):
        try:
            return _kernel_functions[kernel]
        except KeyError:
            raise ValueError("unrecognized kernel name: "+kernel)
    else:
        return kernel

### Windows ###
# TODO: different compensations (total power, peak height, etc.?)
def rectangle_w(x, N, ft_compensated=False):
    """
    Rectangle FT window function.
    """
    return x*0+1

def gen_hamming_w(x, N, alpha, beta, ft_compensated=False):
    """
    Generalized Hamming FT window function.

    If ``ft_compensated==True``, multiply the window function by a compensating factor to preserve power in the spectrum.
    """
    if ft_compensated:
        return (alpha-beta*np.cos(2*np.pi*x/(N-1)))/np.sqrt(alpha**2+beta**2/2.)
    else:
        return alpha-beta*np.cos(2*np.pi*x/(N-1))

def hann_w(x, N, ft_compensated=False):
    """
    Hann FT window function.

    If ``ft_compensated==True``, multiply the window function by a compensating factor to preserve power in the spectrum.
    """
    return gen_hamming_w(x,N,0.5,0.5,ft_compensated=ft_compensated)

def hamming_w(x, N, ft_compensated=False):
    """
    Specific Hamming FT window function.

    If ``ft_compensated==True``, multiply the window function by a compensating factor to preserve power in the spectrum.
    """
    return gen_hamming_w(x,N,0.54,0.46,ft_compensated=ft_compensated)

_window_functions={"hamming":hamming_w, "rectangle":rectangle_w, "hann":hann_w}
def get_window_func(window):
    """
    Get a window function by its name.

    Available functions are: ``'hamming'``, ``'rectangle'``, ``'hann'``.
    """
    if isinstance(window,textstring):
        try:
            return _window_functions[window]
        except KeyError:
            raise ValueError("unrecognized window name: "+window)
    else:
        return window

def gen_hamming_w_ft(f, t, alpha, beta):
    """
    Get Fourier Transform of a generalized Hamming FT window function.

    `f` is the argument, `t` is the total window size.
    """
    lim=beta/(2.*alpha)
    gen=np.sinc(f*t)*(1-(f*t)**2 *(alpha-beta)/alpha)/(1-(f*t)**2)
    if isinstance(gen,np.ndarray):
        return np.where(f*t==1,lim,gen)
    return lim if f*t==1 else gen

def rectangle_w_ft(f, t):
    """
    Get Fourier Transform of the rectangle FT window function.

    `f` is the argument, `t` is the total window size.
    """
    return gen_hamming_w_ft(f,t,1.,0.)

def hann_w_ft(f, t):
    """
    Get Fourier Transform of the Hann FT window function.

    `f` is the argument, `t` is the total window size.
    """
    return gen_hamming_w_ft(f,t,.5,.5)

def hamming_w_ft(f, t):
    """
    Get Fourier Transform of the specific Hamming FT window function.

    `f` is the argument, `t` is the total window size.
    """
    return gen_hamming_w_ft(f,t,.54,.46)

_window_ft_functions={"hamming":hamming_w_ft, "rectangle":rectangle_w_ft, "hann":hann_w_ft}
def get_window_ft_func(window):
    """
    Get a Fourier Transform of a window function by its name.

    Available functions are: ``'hamming'``, ``'rectangle'``, ``'hann'``.
    """
    if isinstance(window,textstring):
        try:
            return _window_ft_functions[window]
        except KeyError:
            raise ValueError("unrecognized window name: "+window)
    else:
        return window


from numpy import (
    array,  # for numpy arrays
    dot,  # for matrix multiplication
    exp,  # used to find sigmoid function
    random  # for random number generation
)

class SimpleNeuralNetwork:
    def __init__(self, dimensions):
        """ initializing weights based on dimensions count """
        weights = list()
        for i in range(dimensions):
            weights.append([random.random()])
        self.weights = array(weights)

    @staticmethod
    def sigmoid_func(val):
        """ Sigmoid Function """
        return 1 / (1 + exp(-val))

    @staticmethod
    def sigmod_func_derivative(prediction):
        """ derivative of Sigmoid Function """
        return prediction * (1 - prediction)

    def predict(self, pred_vals):
        """ sigmoid(sum(each point * its weight)) """
        return self.sigmoid_func(dot(pred_vals, self.weights))

    def train(self, input_x, input_y, iterations):
        """ predicting, finding error, updating weights, start over """
        for _ in range(iterations):
            # predicting with current weights
            prediction = self.predict(input_x)

            # difference in actual output and current output
            error = input_y - prediction

            # updating weights with gradient based on error value
            self.weights += dot(input_x.T, error * self.sigmod_func_derivative(prediction))

if __name__ == '__main__':
    """ simple example usecase for above module """
    # independent variables for training
    train_x = array([
        [0, 0],
        [1, 1],
        [1, 0],
        [0, 1]
    ])
    # dependent variables for training
    train_y = array([
        [0],
        [1],
        [1],
        [0]
    ])

    # initialise neural network model with random weights
    nn = SimpleNeuralNetwork(dimensions=2)
    # train the models (update weights based on independent & dependent variables)
    nn.train(train_x, train_y, 100)

    test_x = array([1, 0])  # independent variables for testing
    # predicting dependent variables from independent variables
    test_y = nn.predict(test_x)
    print(test_x, '=>', test_y)