"""This file contains code used in "Think Stats",
by Allen B. Downey, available from greenteapress.com

Copyright 2014 Allen B. Downey
License: GNU GPLv3 http://www.gnu.org/licenses/gpl.html
"""

from __future__ import print_function, division

import thinkbayes2
import thinkplot

import numpy

"""
Bayesian solution to the Lincoln index, described in a blog
article at Probably Overthinking It.

http://tinyurl.com/lincoln14

Last year my occasional correspondent John D. Cook wrote an excellent
blog post about the Lincoln index, which is a way to estimate the
number of errors in a document (or program) by comparing results from
two independent testers.

http://www.johndcook.com/blog/2010/07/13/lincoln-index/

Here's his presentation of the problem:

   "Suppose you have a tester who finds 20 bugs in your program. You
want to estimate how many bugs are really in the program. You know
there are at least 20 bugs, and if you have supreme confidence in your
tester, you may suppose there are around 20 bugs. But maybe your
tester isn't very good. Maybe there are hundreds of bugs. How can you
have any idea how many bugs there are? There's no way to know with one
tester. But if you have two testers, you can get a good idea, even if
you don't know how skilled the testers are."

Then he presents the Lincoln index, an estimator "described by
Frederick Charles Lincoln in 1930," where Wikpedia's use of
"described" is a hint that the index is another example of Stigler's
law of eponymy.

    "Suppose two testers independently search for bugs. Let k1 be the
number of errors the first tester finds and k2 the number of errors
the second tester finds. Let c be the number of errors both testers
find. The Lincoln Index estimates the total number of errors as k1 k2
/ c [I changed his notation to be consistent with mine]."

So if the first tester finds 20 bugs, the second finds 15, and they
find 3 in common, we estimate that there are about 100 bugs.

Of course, whenever I see something like this, the idea that pops into
my head is that there must be a (better) Bayesian solution! And there
is.

"""

def choose(n, k, d={}):
    """The binomial coefficient "n choose k".

    Args:
      n: number of trials
      k: number of successes
      d: map from (n,k) tuples to cached results

    Returns:
      int
    """
    if k == 0:
        return 1
    if n == 0:
        return 0

    try:
        return d[n, k]
    except KeyError:
        res = choose(n-1, k) + choose(n-1, k-1)
        d[n, k] = res
        return res

def binom(k, n, p):
    """Computes the rest of the binomial PMF.

    k: number of hits
    n: number of attempts
    p: probability of a hit
    """
    return p**k * (1-p)**(n-k)

class Lincoln(thinkbayes2.Suite, thinkbayes2.Joint):
    """Represents hypotheses about the number of errors."""

    def Likelihood(self, data, hypo):
        """Computes the likelihood of the data under the hypothesis.

        hypo: n, p1, p2
        data: k1, k2, c
        """
        n, p1, p2 = hypo
        k1, k2, c = data

        part1 = choose(n, k1) * binom(k1, n, p1)
        part2 = choose(k1, c) * choose(n-k1, k2-c) * binom(k2, n, p2)
        return part1 * part2

def main():

    data = 20, 15, 3
    probs = numpy.linspace(0, 1, 31)
    hypos = []
    for n in range(32, 350):
        for p1 in probs:
            for p2 in probs:
                hypos.append((n, p1, p2))

    suite = Lincoln(hypos)
    suite.Update(data)

    n_marginal = suite.Marginal(0)

    thinkplot.Pmf(n_marginal, label='n')
    thinkplot.Save(root='lincoln1',
                   xlabel='number of bugs',
                   ylabel='PMF',
                   formats=['pdf', 'png'])

    print('post mean n', n_marginal.Mean())
    print('MAP n', n_marginal.MaximumLikelihood())

    p1_marginal = suite.Marginal(1, label='p1')
    p2_marginal = suite.Marginal(2, label='p2')

    thinkplot.Pdf(p1_marginal)
    thinkplot.Pdf(p2_marginal)
    thinkplot.Show()

    print('post mean p1', p1_marginal.Mean())
    print('MAP p1', p1_marginal.MaximumLikelihood())

    print('post mean p2', p2_marginal.Mean())
    print('MAP p2', p2_marginal.MaximumLikelihood())

    print('p1 > p2', p1_marginal > p2_marginal)
    print('p1 < p2', p1_marginal < p2_marginal)

if __name__ == '__main__':
    main()


#!/usr/bin/env python
""" LMS adaptive filters """
from __future__ import print_function, division
__author__ = 'Richard Lindsley'

import sys
import numpy as np
import matplotlib.pyplot as plt
import convm

def lms(x, d, p, mu, w0=None):
    """ Finds the LMS adaptive filter w based on the inputs

    Adapted from the Hayes MATLAB code.

    Inputs:
        x: input data to the filter, Nx1 vector
        d: desired output signal, Nx1 vector
        p: filter order
        mu: step size, scalar or Nx1 vector
        w0: initial guess (optional)

    Outputs:
        w: filter coefficient matrix: w[n,p] where
            n is the time index and
            p is the filter tap index
        e: error sequence e(n): Nx1 vector

    """
    # Initialize
    N = len(x)
    X = convm.convm(x, p)
    if not w0:
        w0 = np.zeros(p)
    # Promote mu to be a vector if it's not already
    if not isinstance(mu, np.ndarray):
        mu = np.ones(N) * mu
    w = np.zeros((N,p))
    e = np.zeros(N)
    w[0,:] = w0

    # Run the filter
    for n in xrange(N-1):
        y = np.dot(w[n,:], X[n,:])
        e[n] = d[n] - y
        w[n+1,:] = w[n,:] + mu[n] * e[n] * X[n,:].conj()

    return w, e

def nlms(x, d, p, beta, w0=None, verbose=False):
    """ Finds the NLMS adaptive filter w based on the inputs

    Adapted from the Hayes MATLAB code.

    Inputs:
        x: input data to the filter, Nx1 vector
        d: desired output signal, Nx1 vector
        p: filter order
        beta: normalized step size (0 < beta < 2)
        w0: initial guess (optional)
        verbose: verbosity flag

    Outputs:
        w: filter coefficient matrix: w[n,p] where
            n is the time index and
            p is the filter tap index
        e: error sequence e(n): Nx1 vector

    """
    # Initialize
    N = len(x)
    eps = 1e-4 # epsilon for denominator
    X = convm.convm(x, p)
    if not w0:
        w0 = np.zeros(p)
    w = np.zeros((N,p))
    e = np.zeros(N)
    w[0,:] = w0
    if verbose:
        print(' ')

    # Run the filter
    for n in xrange(N-1):
        if verbose and n % 10000 == 0:
            print(' {}/{}\r'.format(n,N-1), end='')
            sys.stdout.flush()
        y = np.dot(w[n,:], X[n,:])
        e[n] = d[n] - y
        norm = eps + np.dot(X[n,:], X[n,:].T.conj())
        w[n+1,:] = w[n,:] + beta / norm * e[n] * X[n,:].conj()

    if verbose:
        print(' ')
    return w, e

def plot_traj(w, w_true, fname=None):
    """ Plot the trajectory of the filter coefficients

    Inputs:
        w: matrix of filter weights versus time
        w_true: vector of true filter weights
        fname: what filename to export the figure to. If None, then doesn't
        export

    """
    plt.figure()
    n = np.arange(w.shape[0])
    n_ones = np.ones(w.shape[0])
    plt.hold(True)
    # NOTE: This construction places a limit of 4 on the filter order
    plt_colors = ['b', 'r', 'g', 'k']
    for p in xrange(w.shape[1]):
        plt.plot(n, w[:,p], '{}-'.format(plt_colors[p]), label='w({})'.format(p))
        plt.plot(n, w_true[p] * n_ones, '{}--'.format(plt_colors[p]))
    plt.xlabel('Iteration')
    plt.ylabel('Coefficients')
    plt.legend()
    if fname:
        plt.savefig(fname)
        plt.close()

def plot_error(e, fname=None):
    """ Plot the squared error versus time

    Inputs:
        e: vector of the error versus time
        fname: what filename to export the figure to. If None, then doesn't
        export

    """
    plt.figure()
    n = np.arange(len(e))
    e2 = np.power(e, 2)
    plt.plot(n, e2)
    #plt.semilogy(n, e2)
    plt.xlabel('Iteration')
    plt.ylabel('Squared error')
    if fname:
        plt.savefig(fname)
        plt.close()


"""
levenberg_marquardt_algorithm.py

Returns the minimizer of the function
X0 - initial guess
func_residual- anonimous function residual
func_jacobian- anonimous function jacobian
"""

import numpy as np

def levenberg_marquardt_algorithm(X0, func_residual, func_jacobian, options):

    reg_coeff = 1;
    report = {};
    N_iter_max = options['N_iter_max'];
    tolerance_x = options['tolerance_x'];
    tolerance_y = options['tolerance_y'];
    X_lower = options['x_lower'];
    X_upper = options['x_upper'];
    progress_x = np.zeros((X0.size, N_iter_max + 1));
    progress_y = np.zeros((1, N_iter_max + 1));
    progress_x[:, [0]] = X0;
    progress_y[0, [0]] = np.sum(func_residual(X0));
    X_old = X0;

    for iter_no in range(1, N_iter_max + 1):
        jacobian = func_jacobian(X_old);
        residuals = func_residual(X_old);

        #d = np.linalg.solve(jacobian, residuals); # Newton's algorithm
        #d = np.linalg.solve(jacobian.T @ jacobian, jacobian.T @ residuals); # Newton's algorithm
        d = np.linalg.solve(jacobian.T @ jacobian + reg_coeff * np.eye(X_old.size, X_old.size), jacobian.T @ residuals); # with Levenberg-Marquardt modification
        X = X_old + d.reshape(X_old.shape);

        # Projection onto constrained parameter space
        indx_limits = (X < X_lower);
        X[indx_limits] = X_lower[indx_limits];
        indx_limits = (X > X_upper);
        X[indx_limits] = X_upper[indx_limits];

        progress_x[:, [iter_no]] = X;
        progress_y[0, [iter_no]] = np.sum(func_residual(X));

        if (np.linalg.norm(X - X_old) < tolerance_x * np.linalg.norm(X_old)):
            print('Tolerance in X is reached in %d iterations, exit..' % (iter_no));
            break;

        if (np.abs(progress_y[0, [iter_no]] - progress_y[0, [iter_no - 1]]) < tolerance_y * np.abs(progress_y[0, [iter_no - 1]])):
            print('Tolerance in Y is reached in %d iterations, exit..' % (iter_no));
            break;

        X_old = X;

    report = {'N_iter_max' : N_iter_max, 'iter_no' : iter_no, 'X0' : X0, 'X' : X, 'progress_x' : progress_x, 'progress_y' : progress_y};
    return (X, report);


import numpy as np
import pandas as pd
import matplotlib
matplotlib.use('TkAgg')
import matplotlib.pyplot as plt
import ui
import scipy.io as sio
import scipy.optimize as opt
from scipy.io import loadmat
import seaborn as sb

print('PART 1 异常检测')
ui.split_line2()
print('用高斯模型检测数据是否异常')
print('载入数据')
data = loadmat('data/ex8data1.mat')
X = data['X']
print(X.shape)
print('查看数据分布')
fig, ax = plt.subplots(figsize=(12,8))
ax.scatter(X[:,0], X[:,1])
plt.show()
print('定义高斯函数')
def estimate_gaussian(X):
    mu = X.mean(axis=0)
    sigma = X.var(axis=0)

    return mu, sigma
print('进行高斯预估')
mu, sigma = estimate_gaussian(X)
print(mu, sigma)
print('确定验证集')
Xval = data['Xval']
yval = data['yval']
print(Xval.shape, yval.shape)
print('计算数据点属于正态分布的概率')
from scipy import stats
dist = stats.norm(mu[0], sigma[0])
print(dist.pdf(15))
print('每个点的概率密度')
print(dist.pdf(X[:,0])[0:50])
print('计算数据集中每个点的概率密度')
p = np.zeros((X.shape[0], X.shape[1]))
p[:,0] = stats.norm(mu[0], sigma[0]).pdf(X[:,0])
p[:,1] = stats.norm(mu[1], sigma[1]).pdf(X[:,1])
print(p.shape)
print('计算验证集中每个点的概率密度')
pval = np.zeros((Xval.shape[0], Xval.shape[1]))
pval[:,0] = stats.norm(mu[0], sigma[0]).pdf(Xval[:,0])
pval[:,1] = stats.norm(mu[1], sigma[1]).pdf(Xval[:,1])
print(pval.shape)
print('找到判断的最佳阈值')
def select_threshold(pval, yval):
    best_epsilon = 0
    best_f1 = 0
    f1 = 0

    step = (pval.max() - pval.min()) / 1000

    for epsilon in np.arange(pval.min(), pval.max(), step):
        preds = pval < epsilon

        tp = np.sum(np.logical_and(preds == 1, yval == 1)).astype(float)
        fp = np.sum(np.logical_and(preds == 1, yval == 0)).astype(float)
        fn = np.sum(np.logical_and(preds == 0, yval == 1)).astype(float)

        precision = tp / (tp + fp)
        recall = tp / (tp + fn)
        f1 = (2 * precision * recall) / (precision + recall)

        if f1 > best_f1:
            best_f1 = f1
            best_epsilon = epsilon

    return best_epsilon, best_f1

epsilon, f1 = select_threshold(pval, yval)
print(epsilon, f1)
print('将该阈值应用到数据集，并可视化结果')
# indexes of the values considered to be outliers
outliers = np.where(p < epsilon)
print(outliers)

fig, ax = plt.subplots(figsize=(12,8))
ax.scatter(X[:,0], X[:,1])
ax.scatter(X[outliers[0],0], X[outliers[0],1], s=50, color='r', marker='o')
plt.show()

ui.split_line1()
print('PART 2 协同过滤之电影推荐')
print('载入数据')
data = loadmat('data/ex8_movies.mat')
print(data)
print('Y是包含从1到5的等级的（数量的电影x数量的用户）数组.R是包含指示用户是否给电影评分的二进制值的“指示符”数组')
print('两者应该具有相同的维度。')
Y = data['Y']
R = data['R']
print(Y.shape, R.shape)
print('电影的平均评级')
print(Y[1,np.where(R[1,:]==1)[0]].mean())
print('可视化')
fig, ax = plt.subplots(figsize=(12,12))
ax.imshow(Y)
ax.set_xlabel('Users')
ax.set_ylabel('Movies')
fig.tight_layout()
plt.show()

print('定义代价函数')
def cost(params, Y, R, num_features):
    Y = np.matrix(Y)  # (1682, 943)
    R = np.matrix(R)  # (1682, 943)
    num_movies = Y.shape[0]
    num_users = Y.shape[1]

    # reshape the parameter array into parameter matrices
    X = np.matrix(np.reshape(params[:num_movies * num_features], (num_movies, num_features)))  # (1682, 10)
    Theta = np.matrix(np.reshape(params[num_movies * num_features:], (num_users, num_features)))  # (943, 10)

    # initializations
    J = 0

    # compute the cost
    error = np.multiply((X * Theta.T) - Y, R)  # (1682, 943)
    squared_error = np.power(error, 2)  # (1682, 943)
    J = (1. / 2) * np.sum(squared_error)

    return J

print('评估一小段数据')
params_data = loadmat('data/ex8_movieParams.mat')
X = params_data['X']
Theta = params_data['Theta']
X.shape, Theta.shape

users = 4
movies = 5
features = 3

X_sub = X[:movies, :features]
Theta_sub = Theta[:users, :features]
Y_sub = Y[:movies, :users]
R_sub = R[:movies, :users]

params = np.concatenate((np.ravel(X_sub), np.ravel(Theta_sub)))
print('计算 Cost')
print(cost(params, Y_sub, R_sub, features))

print('定义梯度计算')
def cost0(params, Y, R, num_features):
    Y = np.matrix(Y)  # (1682, 943)
    R = np.matrix(R)  # (1682, 943)
    num_movies = Y.shape[0]
    num_users = Y.shape[1]

    # reshape the parameter array into parameter matrices
    X = np.matrix(np.reshape(params[:num_movies * num_features], (num_movies, num_features)))  # (1682, 10)
    Theta = np.matrix(np.reshape(params[num_movies * num_features:], (num_users, num_features)))  # (943, 10)

    # initializations
    J = 0
    X_grad = np.zeros(X.shape)  # (1682, 10)
    Theta_grad = np.zeros(Theta.shape)  # (943, 10)

    # compute the cost
    error = np.multiply((X * Theta.T) - Y, R)  # (1682, 943)
    squared_error = np.power(error, 2)  # (1682, 943)
    J = (1. / 2) * np.sum(squared_error)

    # calculate the gradients
    X_grad = error * Theta
    Theta_grad = error.T * X

    # unravel the gradient matrices into a single array
    grad = np.concatenate((np.ravel(X_grad), np.ravel(Theta_grad)))

    return J, grad

print('计算梯度')
J, grad = cost0(params, Y_sub, R_sub, features)
print(J, grad)
print('定义正则化 cost')
def cost1(params, Y, R, num_features, learning_rate):
    Y = np.matrix(Y)  # (1682, 943)
    R = np.matrix(R)  # (1682, 943)
    num_movies = Y.shape[0]
    num_users = Y.shape[1]

    # reshape the parameter array into parameter matrices
    X = np.matrix(np.reshape(params[:num_movies * num_features], (num_movies, num_features)))  # (1682, 10)
    Theta = np.matrix(np.reshape(params[num_movies * num_features:], (num_users, num_features)))  # (943, 10)

    # initializations
    J = 0
    X_grad = np.zeros(X.shape)  # (1682, 10)
    Theta_grad = np.zeros(Theta.shape)  # (943, 10)

    # compute the cost
    error = np.multiply((X * Theta.T) - Y, R)  # (1682, 943)
    squared_error = np.power(error, 2)  # (1682, 943)
    J = (1. / 2) * np.sum(squared_error)

    # add the cost regularization
    J = J + ((learning_rate / 2) * np.sum(np.power(Theta, 2)))
    J = J + ((learning_rate / 2) * np.sum(np.power(X, 2)))

    # calculate the gradients with regularization
    X_grad = (error * Theta) + (learning_rate * X)
    Theta_grad = (error.T * X) + (learning_rate * Theta)

    # unravel the gradient matrices into a single array
    grad = np.concatenate((np.ravel(X_grad), np.ravel(Theta_grad)))

    return J, grad

print('计算梯度')
J, grad = cost1(params, Y_sub, R_sub, features, 1.5)
print(J, grad)

print('加载字典')
movie_idx = {}
f = open('data/movie_ids.txt',encoding= 'gbk')
for line in f:
    tokens = line.split(' ')
    tokens[-1] = tokens[-1][:-1]
    movie_idx[int(tokens[0]) - 1] = ' '.join(tokens[1:])
print('查看第一条')
print(movie_idx[0])

print('进行评分')
ratings = np.zeros((1682, 1))

ratings[0] = 4
ratings[6] = 3
ratings[11] = 5
ratings[53] = 4
ratings[63] = 5
ratings[65] = 3
ratings[68] = 5
ratings[97] = 2
ratings[182] = 4
ratings[225] = 5
ratings[354] = 5

print('Rated {0} with {1} stars.'.format(movie_idx[0], str(int(ratings[0]))))
print('Rated {0} with {1} stars.'.format(movie_idx[6], str(int(ratings[6]))))
print('Rated {0} with {1} stars.'.format(movie_idx[11], str(int(ratings[11]))))
print('Rated {0} with {1} stars.'.format(movie_idx[53], str(int(ratings[53]))))
print('Rated {0} with {1} stars.'.format(movie_idx[63], str(int(ratings[63]))))
print('Rated {0} with {1} stars.'.format(movie_idx[65], str(int(ratings[65]))))
print('Rated {0} with {1} stars.'.format(movie_idx[68], str(int(ratings[68]))))
print('Rated {0} with {1} stars.'.format(movie_idx[97], str(int(ratings[97]))))
print('Rated {0} with {1} stars.'.format(movie_idx[182], str(int(ratings[182]))))
print('Rated {0} with {1} stars.'.format(movie_idx[225], str(int(ratings[225]))))
print('Rated {0} with {1} stars.'.format(movie_idx[354], str(int(ratings[354]))))

print('将评分添加到已有数据集中')
R = data['R']
Y = data['Y']

Y = np.append(Y, ratings, axis=1)
R = np.append(R, ratings != 0, axis=1)

print(Y.shape, R.shape, ratings.shape)
print('对变量进行归一化')
movies = Y.shape[0]  # 1682
users = Y.shape[1]  # 944
features = 10
learning_rate = 10.

X = np.random.random(size=(movies, features))
Theta = np.random.random(size=(users, features))
params = np.concatenate((np.ravel(X), np.ravel(Theta)))

print(X.shape, Theta.shape, params.shape)

Ymean = np.zeros((movies, 1))
Ynorm = np.zeros((movies, users))

for i in range(movies):
    idx = np.where(R[i,:] == 1)[0]
    Ymean[i] = Y[i,idx].mean()
    Ynorm[i,idx] = Y[i,idx] - Ymean[i]

print(Ynorm.mean())

print('进行模型训练与优化')
from scipy.optimize import minimize

fmin = minimize(fun=cost1, x0=params, args=(Ynorm, R, features, learning_rate),
                method='CG', jac=True, options={'maxiter': 100})
print(fmin)

X = np.matrix(np.reshape(fmin.x[:movies * features], (movies, features)))
Theta = np.matrix(np.reshape(fmin.x[movies * features:], (users, features)))

print(X.shape, Theta.shape)
print('进行预测')
predictions = X * Theta.T
my_preds = predictions[:, -1] + Ymean
print(my_preds.shape)
print('排序预测')
idx = np.argsort(my_preds, axis=0)[::-1]
print(idx)
print("Top 10 movie predictions:")
for i in range(10):
    j = int(idx[i])
    print('Predicted rating of {0} for movie {1}.'.format(str(float(my_preds[j])), movie_idx[j]))


import numpy as np
import matplotlib.pyplot as plt
from numpy.linalg import inv

N = 25
X = np.reshape(np.linspace(0, 0.9, N), (N, 1))
Y_train = np.cos(10*X**2) + 0.1 * np.sin(100*X)
#K = 12  # order of the func
listTrainning = []
for i in np.linspace(0, 0.9, N):
    listTrainning.append(np.cos(10*i**2) + 0.1 * np.sin(100*i))

def Phi(K):
    Psii = np.zeros((N, K + 1))
    for i in range(N):
        for j in range(0, K+1):
            Psii[i][j] = X[i][0]**j

    return Psii

def lml(alpha, beta, Phi, Y):
    """
    4 marks

    :param alpha: float
    :param beta: float
    :param Phi: array of shape (N, M)
    :param Y: array of shape (N, 1)
    :return: the log marginal likelihood, a scalar
    """
    N = len(Phi)
    M = len(Phi[0])
    part1 = (-N*0.5)*np.log(2*np.pi)
    wholePhi = np.dot(np.dot(Phi, alpha*np.identity(M)),Phi.T)
    wholeBeta = beta*np.identity(N)
    part2 = - 0.5*np.log(np.linalg.det( wholePhi + wholeBeta))

    part3 = -0.5*np.dot(np.dot(Y.T, inv((wholePhi + wholeBeta))),Y)
    logFunc  = part1 + part2 + part3

    return logFunc[0][0]

def grad_lml(alpha, beta, Phi, Y):
    """
    8 marks (4 for each component)

    :param alpha: float
    :param beta: float
    :param Phi: array of shape (N, M)
    :param Y: array of shape (N, 1)
    :return: array of shape (2,). The components of this array are the gradients
    (d_lml_d_alpha, d_lml_d_beta), the gradients of lml with respect to alpha and beta respectively.

    """
    N = len(Phi)
    M = len(Phi[0])
    X = np.dot(np.dot(alpha, Phi), Phi.T) + np.dot(beta, np.identity(N))
    common = 0.5 * np.dot(np.dot(inv(X), np.dot(Y, Y.T) - X), inv(X))
    d_alpha = np.trace(np.dot(common.T, np.dot(Phi, Phi.T)))
    d_bete = np.trace(common.T)

    return np.array([d_alpha, d_bete])

#order = 1
Phi = Phi(1)
# From calculation, it is expected that the local minimum occurs at x=9/4

cur_x = np.array([0.5, 0.65]) # The algorithm starts at x=1
gamma = 0.01 # step size multiplier

max_iters = 20000 # maximum number of iterations
iters = 0 #iteration counter

x_gd = []
y_gd = []
z_gd = []
while (iters < max_iters):
    x_gd.append(cur_x[0])
    y_gd.append(cur_x[1])
    z_gd.append(lml(cur_x[0],cur_x[1], Phi, Y_train))

    prev_x = np.array([cur_x[0], cur_x[1]])
    #cur_x = prev_x + gamma * grad_f2(prev_x)
    cur_x = prev_x + gamma * grad_lml(prev_x[0], prev_x[1], Phi, Y_train)
    iters += 1

print("The local maximum occurs at", cur_x)
print("the maximum is ", lml(cur_x[0], cur_x[1], Phi, Y_train))

xlist = np.linspace(0.2, 1.0, 50)
ylist = np.linspace(0.3, 0.7, 50)

X, Y = np.meshgrid(xlist, ylist)
# new an array with all 0 inside
Z = np.zeros((50, 50))

for i in range(50):
    for j in range(50):
        #Z[i][j] = f2(np.array([xlist[i], ylist[j]]))
        Z[i][j] = lml(X[i,j], Y[i,j], Phi, Y_train)

# print(Z)
plt.contour(X, Y, Z, 100, cmap='jet')
plt.colorbar()

plt.plot(x_gd, y_gd, color='green', marker='v', linewidth=2, markersize=0, label='Gradient descent of order 1')
plt.legend(loc='best')
plt.ylabel('beta')
plt.xlabel('alpha')
plt.title('this 1 b')
plt.show()
#print(theta)


import numpy as np

class TriangularMemoizer(object):
    """Specific class for triangular recurrence sequences of the kind
    s(0, 0) = s(1, 1) = 0, s(a, 0) = 0 for a > 0, s(a, b) = 0 for b > a,
    s(a, b) = f(a, b) * s(a - 1, b) + g(a, b) s(a - 1, b - 1).

    For instance, Stirling numbers of the first kind are recovered setting
    f(a, b) = (a - 1) and g(a, b) = 1
    """

    def __init__(self, first_term_multiplier, second_term_multiplier):
        self.memo = [np.array([1])]
        self.max_a = 0
        self.first_term_multiplier = first_term_multiplier
        self.second_term_multipier = second_term_multiplier

    def __call__(self, a, b):
        if a == b == 0:
            return 1

        if b == 0:
            return 0

        if b > a:
            return 0

        if a > self.max_a:
            self.add_rows(a)

        return self.memo[a][b]

    def add_rows(self, new_max_a):
        old_max_a = self.max_a
        self.max_a = new_max_a
        for a in range(old_max_a + 1, new_max_a + 1):
            self.memo.append(np.zeros(a + 1))
            self.memo[a][0] = 0
            for b in range(1, a + 1):
                res = self.first_term_multiplier(a, b) * self.__call__(a - 1, b) + \
                    self.second_term_multipier(a, b) * self.__call__(a - 1, b - 1)
                self.memo[a][b] = res

stirling = TriangularMemoizer(lambda a, b: a - 1, lambda a, b: 1)

class generalized_factorial_memoizer(TriangularMemoizer):
    """Computes the Generalized Factorial numbers and stores the results
    in a memoizer.
    The generalized factorial with parameter sigma in [0, 1) satisfy the
    triangular recurrence relation
    s(n, k) = (n - 1 - sigma * k) s(n - 1, k) + sigma * s(n - 1, k-1)
    """

    def __init__(self, sigma):
        def first_arg(n, k): return n-1 - sigma * k
        def second_arg(n, k): return sigma
        super().__init__(first_arg, second_arg)


import numpy as np
import matplotlib.pyplot as plt
from scipy.spatial import distance_matrix

# Squared exponential covariance function (for stationary GP).
def sqexp_cov_fn(d, rho, alpha):
    return alpha ** 2 * np.exp(-0.5 * (d / rho) ** 2)

# Function to create gp prediction function.
def gp_predict_maker(y, x, x_new, cov_fn):
    N = x.shape[0]
    N_new = x_new.shape[0]
    M = N + N_new
    xx = np.concatenate((x_new, x)).reshape(M, 1)
    D = distance_matrix(xx, xx)

    # Function which takes parameters of covariance function
    # and predicts at new locations.
    def gp_predict(rho, alpha, sigma):
        K = cov_fn(D, rho, alpha) + np.eye(M) * np.power(sigma, 2)
        K_new_old = K[:N_new, N_new:]
        K_old_inv = np.linalg.inv(K[N_new:, N_new:])
        C = K_new_old.dot(K_old_inv)
        mu = C.dot(y)
        S = K[:N_new, :N_new] - C.dot(K_new_old.T)
        return np.random.multivariate_normal(mu, S)

    return gp_predict

# Function for plotting parameter posterior.
def plot_post(samples, key, bins=None, suffix=""):
    plt.hist(samples[key], density=True, bins=bins)
    plt.xlabel(key)
    plt.ylabel('density')
    if suffix != "":
        suffix = "({})".format(suffix)

    plt.title("{} {}".format(key, suffix));

# Function for making all plots.
def make_plots(samples, x, y, x_grid, f, sigma_true, cov_fn=sqexp_cov_fn,
               n_new=100, figsize=(12,4), figsize_f=(12, 4), suffix="",
               x_min=-3.5, x_max=3.5, ci=95):
    # Create new locations for prediction.
    x_new = np.linspace(x_min, x_max, n_new)

    # Create gp predict function.
    gp_predict = gp_predict_maker(y, x, x_new, cov_fn)

    # Number of posterior samples.
    nsamples = len(samples['alpha'])

    # Make predictions at new locations.
    preds = np.stack([gp_predict(alpha=samples['alpha'][b],
                                 rho=samples['rho'][b],
                                 sigma=samples['sigma'][b])
                      for b in range(nsamples)])

    # Plot parameters posterior.
    plt.figure(figsize=figsize)
    plt.subplot(1, 3, 1)
    plot_post(samples, 'alpha', bins=30, suffix=suffix)
    plt.subplot(1, 3, 2)
    plot_post(samples, 'rho', bins=30, suffix=suffix)
    plt.subplot(1, 3, 3)
    plot_post(samples, 'sigma', bins=30, suffix=suffix)
    plt.axvline(sigma_true, ls=":", color='red', label="truth")
    plt.legend()
    plt.tight_layout()

    # Summarize function posterior.
    ci_lower = (100 - ci) / 2
    ci_upper = (100 + ci) / 2
    preds_mean = preds.mean(0)
    preds_lower = np.percentile(preds, ci_lower, axis=0)
    preds_upper = np.percentile(preds, ci_upper, axis=0)

    # Make suffix
    if suffix != "":
        suffix = "({})".format(suffix)

    # Plot function posterior.
    plt.figure(figsize=figsize_f)
    plt.scatter(x, y, c='black', zorder=3, label='data')
    plt.fill_between(x_new, preds_upper, preds_lower, alpha=.3, label='95% CI')
    plt.plot(x_new, preds.mean(0), lw=2, label="mean fn.", color="blue")
    plt.plot(x_grid, f, label="truth", lw=2, c='red', ls=':')
    plt.title("GP Posterior Predictive with 95% CI {}".format(suffix))
    plt.legend()
    plt.tight_layout();


'''
datecreated: 190926
objective: first practice with naive bayes classification
source: https://www.youtube.com/watch?v=CPqOCI0ahss
KJG191119: use this to prep for bayes with iris dataset

'''

import numpy as np
# import matplotlib.pyplot as plt

# pseudo dataset: probability to play golf, given variables
# outlook: sunny,overcast,rainy = 0,1,2
# temperture: cool, mild, hot = 0,1,2
# humidity: normal,high = 0,1
# windy: false,true = 0,1
# play: no, yes = 0,1

dat=np.array(
# outlook | temperature | humidity | windy | play
[[0,2,1,0,0],
[0,2,1,1,0],
[1,2,1,0,1],
[2,1,1,0,1],
[2,0,0,0,1],
[2,0,0,1,0],
[1,0,0,1,1],
[0,1,1,0,0],
[0,0,0,0,1],
[2,1,0,0,1],
[0,1,0,1,1],
[1,1,1,1,1],
[1,2,0,0,1],
[2,1,1,1,0]
])

def filt(arr,column,condition):
    return arr[arr[:,column]==condition,:]

def p(arr,col0,cond0,col1,cond1):
    ''' What is the probability col0 is at cond0, given col1 is at cond1?
    Ex: What is the probability the weather outlook is
        sunny, given that we will play golf? '''
    n0 = len(filt(filt(arr,col0,cond0),col1,cond1)) # ex: P(sunny|yes)
    nT = len(filt(arr,col1,cond1)) # ex: P(yes)
    return n0/nT
# first, get p(yes) and p(no):
ntot = dat.shape[0]
nyes = dat[:,-1].sum()
nno  = ntot-nyes # binary

p_yes = nyes/ntot
p_no  = 1-p_yes # binary outcome

print(p_yes,p_no)

# look at probabilities that we can play given all variable combinations

# outlook table
# should have as seen in temp.png
print('outlook:')
print('sunny & yes:',p(dat,0,0,-1,1))
print('ocast & yes:',p(dat,0,1,-1,1))
print('rainy & yes:',p(dat,0,2,-1,1))
print('sunny &  no:',p(dat,0,0,-1,0))
print('ocast &  no:',p(dat,0,1,-1,0))
print('rainy &  no:',p(dat,0,2,-1,0))

print('example set of factors.')
print('sunny, cool, humid, windy')

p_x_yes=(p(dat,0,0,-1,1)* # outlook
         p(dat,1,0,-1,1)* # temp
         p(dat,2,1,-1,1)* # humidity
         p(dat,3,1,-1,1)* # wind
         1)
p_x_no =(p(dat,0,0,-1,0)* # outlook
         p(dat,1,0,-1,0)* # temp
         p(dat,2,1,-1,0)* # humidity
         p(dat,3,1,-1,0)* # wind
         1)
p_x = ( len(filt(dat,0,0))/ntot*
        len(filt(dat,1,0))/ntot*
        len(filt(dat,2,1))/ntot*
        len(filt(dat,3,1))/ntot*
        1)
print('P(x) = ',p_x)
print('P(play|factors):   ',p_x_yes*p_yes/p_x)
print('P(noplay|factors): ',p_x_no*p_no/p_x)

# print('ocast & yes:',p(dat,0,0,-1,1))
# print('ocast & yes:',p(dat,0,0,-1,1))
# print('ocast & yes:',p(dat,0,0,-1,1))
# sunny    2,3,2/9,3/5 -- p(sunny|golf)
# overcast 4,0,4/9,0/5
# rainy    3,2,3/9,2/5

# temperature table
# hot   2,2,2/9,2/5
# mild  4,2,4/9,2/5
# cold  3,1,3/9,1/5

if(__name__=='__main__'):
    print('done')

# eof


"""
2017 A. R. Malipeddi
A simple 2D geometric multigrid solver for the homogeneous Dirichlet Poisson problem on Cartesian grids and unit square. Cell centered 5-point finite difference operator.

Thanks Dr. Malpeddi!
"""

import numpy as np
from scipy.sparse.linalg import LinearOperator

def Jacrelax(level,nx,ny,u,f,iters=1,pre=False,r=0):
  '''
  under-relaxed Jacobi method smoothing
  '''

  dx=1.0/nx; dy=1.0/ny
  Ax=1.0/dx**2; Ay=1.0/dy**2
  Ap=1.0/(2.0*(Ax+Ay))

  #Dirichlet BC
  u[ 0,:] = -u[ 1,:]
  u[-1,:] = -u[-2,:]
  u[:, 0] = -u[:, 1]
  u[:,-1] = -u[:,-2]
  #
  # if(r):
  #     u[int(nx/2),int(ny/2)] = 1

  T = u.copy()
  for it in range(iters):
    for i in range(1,nx+1):
      for j in range(1,ny+1):
         u[i,j]= Ap*( Ax*(u[i+1,j]+u[i-1,j])
                     + Ay*(u[i,j+1]+u[i,j-1]) - f[i,j])

    u[ 0,:] = -u[ 1,:]
    u[-1,:] = -u[-2,:]
    u[:, 0] = -u[:, 1]
    u[:,-1] = -u[:,-2]

#  if(not pre):
#    return u,None

  res=np.zeros([nx+2,ny+2])
  for i in range(1,nx+1):
    for j in range(1,ny+1):
      res[i,j]=f[i,j] - ((Ax*(u[i+1,j]+u[i-1,j])+Ay*(u[i,j+1]+u[i,j-1]) - 2.0*(Ax+Ay)*u[i,j]))

  # if(r):
  #     res[int(nx/2),int(ny/2)] = 0

  return u,res

def restrict(nx,ny,v):
  '''
  restrict 'v' to the coarser grid
  '''
  v_c=np.zeros([nx+2,ny+2])

  for i in range(1,nx+1):
    for j in range(1,ny+1):
      v_c[i,j]=0.25*(v[2*i-1,2*j-1]+v[2*i,2*j-1]+v[2*i-1,2*j]+v[2*i,2*j])

  return v_c

def prolong(nx,ny,v):
  '''
  interpolate 'v' to the fine grid
  '''
  v_f=np.zeros([2*nx+2,2*ny+2])

  for i in range(1,nx+1):
      for j in range(1,ny+1):
        v_f[2*i-1,2*j-1] = 0.5625*v[i,j]+0.1875*(v[i-1,j]+v[i,j-1])+0.0625*v[i-1,j-1]
        v_f[2*i  ,2*j-1] = 0.5625*v[i,j]+0.1875*(v[i+1,j]+v[i,j-1])+0.0625*v[i+1,j-1]
        v_f[2*i-1,2*j  ] = 0.5625*v[i,j]+0.1875*(v[i-1,j]+v[i,j+1])+0.0625*v[i-1,j+1]
        v_f[2*i  ,2*j  ] = 0.5625*v[i,j]+0.1875*(v[i+1,j]+v[i,j+1])+0.0625*v[i+1,j+1]

  return v_f

def V_cycle(nx,ny,num_levels,u,f,level=1):
  '''
  V cycle
  '''
  if(level==num_levels):#bottom solve
    u,res=Jacrelax(level,nx,ny,u,f,iters=50,pre=True)
    return u,res

  #Step 1: Relax Au=f on this grid
  u,res=Jacrelax(level,nx,ny,u,f,iters=2,pre=True,r=1)

  #Step 2: Restrict residual to coarse grid
  res_c=restrict(nx//2,ny//2,res)

  #Step 3:Solve A e_c=res_c on the coarse grid. (Recursively)
  e_c=np.zeros_like(res_c)
  e_c,res_c=V_cycle(nx//2,ny//2,num_levels,e_c,res_c,level+1)

  #Step 4: Interpolate(prolong) e_c to fine grid and add to u
  u+=prolong(nx//2,ny//2,e_c)

  #Step 5: Relax Au=f on this grid
  u,res=Jacrelax(level,nx,ny,u,f,iters=1,pre=False,r=1)
  return u,res

def FMG(nx,ny,num_levels,f,nv=1,level=1):

  if(level==num_levels):#bottom solve
    u=np.zeros([nx+2,ny+2])
    u,res=Jacrelax(level,nx,ny,u,f,iters=50,pre=True)
    return u,res

  #Step 1: Restrict the rhs to a coarse grid
  f_c=restrict(nx//2,ny//2,f)

  #Step 2: Solve the coarse grid problem using FMG
  u_c,_=FMG(nx//2,ny//2,num_levels,f_c,nv,level+1)

  #Step 3: Interpolate u_c to the fine grid
  u=prolong(nx//2,ny//2,u_c)

  #step 4: Execute 'nv' V-cycles
  for _ in range(nv):
    u,res=V_cycle(nx,ny,num_levels-level,u,f)
  return u,res


# Highly divisible triagular number
#
# The sequence of triangle numbers is generated by adding the natural numbers.
# So the 7th triangle number would be 1 + 2 + 3 + 4 + 5 + 6 + 7 = 28.
# The first ten terms would be:
#
#   1, 3, 6, 10, 15, 21, 28, 36, 45, 55, ...
#
# Lets us list the factors of the first seven triangle number:
#
#  1: 1
#  3: 1, 3
#  6: 1, 2, 3, 6
# 10: 1, 2, 5, 10
# 15: 1, 3, 5, 15
# 21: 1, 3, 7, 21
# 28: 1, 2, 4, 7, 14, 28
#
# We can see that 29 is the first triangle number to have over five divisors.
#
# What is the value of the first triangle number to have over five hundred divisors?
import numpy as np

# input: number to be factored
# output: N/A
# return: array of factors
def factor(factor_number):
    factors = 0
#    factors = np.array([])
#    factors = []
    for i in range(1,factor_number+1):
        if factor_number % i == 0:
            factors += 1
#            factors = np.append(factors,[i])
#            factors.append(i)

    return factors

# input: number to triangle
# output: N/A
# return: triangular number
def triangular(triangular_number):
    triangular_value = 0
    for x in range(1,triangular_number+1):
        triangular_value += x

    return triangular_value

#
#   Main Program Execution
#
if __name__ == "__main__":
    count = 0
    t_number = 0
    num_factors = 0
#    for x in range(1,100):
#        print("{}, value {}".format(x,triangular(x)))
#        print("Factors: {}".format(factor(triangular(x))))

    while num_factors < 500:
        count += 1
        t_number = triangular(count)
#        factors = factor(t_number)
#        print(factors.size)
        num_factors = factor(t_number)
#        print(t_number)
        print(num_factors)
    print(t_number)


'''
Power Iteration method of finding dominant eigenvector.
	source : https://www.wikiwand.com/en/Power_iteration
'''
import numpy as np
from numpy.linalg import norm

def rayleigh_quotient( A, b ):
	bt = conjugate_transpose( b )
	return ( np.dot( np.dot( bt, A ), b ) )/( np.dot( bt, b ) )

def conjugate_transpose( b ):
	return np.conj( b.T )

def __test():
	# same A as in power_iteration_eigenvector.py
	A = np.array( [ [ 0.5, 0.5 ], [ 0.2, 0.8 ] ] )

	# value estimated by test case in power_iteration_eigenvector.py
	b = np.array([0.70710679, 0.70710677])
	return rayleigh_quotient( A, b )#, np.linalg.eig( A )

if __name__ == '__main__':
	print __test()


from autodiff import operations
from autodiff.structures import Number
from autodiff.structures import Array
import numpy as np
from copy import deepcopy

def newtons_method(func, initial_guess, iterations=100,tolerance = 10**-7,verbose = False, show_fxn=False):

    """Use Newton's method to find the root of the function
    Args:
        func: the function that the user wants to find root for
        initial_guess: a number object for the initial guess
        iterations: number of maximum iterations
        show_fxn: if true, return function value at the final xstar

    Returns:
        xn: the x value of the root
        jacobians: the jacobian at each step of the root_finding
        fxn: func(xn). If root_finding is successful, this value should be 0
    """
    if isinstance(initial_guess,Number):
        #scalar case
        jacobians = []

        x0 = initial_guess

        fxn = func(initial_guess)

        fpxn = fxn.jacobian(initial_guess)

        x1 = x0 - fxn/fpxn

        jacobians.append(fpxn)

        for i in range(iterations):
            if abs(fxn.val) > 1e-7:
                x0 = x1

                fxn = func(x0)

                fpxn = fxn.jacobian(x0)

                jacobians.append(fpxn)

                x1 = x0- fxn / fpxn

        if show_fxn:
            return x1, jacobians,fxn
        else:
            return x1, jacobians
    elif isinstance(initial_guess,Array):
        jacobians = []
        if isinstance(func(initial_guess),tuple):
            for i in range(iterations):

                if i == 0:
                    x0= initial_guess
                else:
                    x0 = x1

                fxn = func(x0)
                fpxn = []
                if verbose:
                    print(i,x0,fxn)

                vector = []
                for n in range(len(fxn)):
                    vector.append(fxn[n].val)

                if np.linalg.norm(vector)< tolerance:
                    break

                for k in range(len(fxn)):
                    fpxn_row = []
                    for j in range(len(x0)):
                        fpxn_row.append(fxn[k].jacobian(x0[j]))
                    fpxn.append(fpxn_row)
                jacobians.append(fpxn)
                x1 = x0 - np.dot(np.linalg.inv(fpxn),fxn)

            if show_fxn:
                return x1, jacobians,fxn
            else:
                return x1, jacobians

        else:
            for i in range(iterations):

                if i == 0:
                    x0= initial_guess
                else:
                    x0 = x1

                fxn = func(x0)
                fpxn = []
                if verbose:
                    print(i,x0,fxn)

                if abs(fxn.val)< tolerance:
                    break

                for j in range(len(x0)):
                    fpxn.append(fxn.jacobian(x0[j]))
                jacobians.append(fpxn)
                print(fpxn)
                x1 = x0 - np.dot(np.reciprocal(fpxn),fxn)

            if show_fxn:
                return x1, jacobians,fxn
            else:
                return x1, jacobians


"""
Use the Gram Schmidt procedure to reorient a set of points such that one point is at the origin, another point lies
on one axis (x1, 0, ..., 0) and the next point lies on a plane spanned by the two axes  (x2, x3, 0, ..., 0), and
so on. This constrains the minimization function that looks for the points of best fit, as rotations and
translations are made redundant.

This can also take into effect the noise parameter as we do not constrain where on the first axis, the second
point lies.
"""
import numpy as np

def choose_basis_vectors(M):
    """
    Given a matrix of points M (m by n), where m is the number of dimensions and n is the number of points,
    choose m vectors that are linearly independent from each other.

    Since the chosen submatrix would be m by m, a simple way to do this is by calculating the determinant.
    If the determinant is 0, the vectors are not linearly independent
    """
    # first check if M has full rank
    n_dim, n_points = M.shape
    if np.linalg.matrix_rank(M) < n_dim:
        raise ValueError("Matrix is rank-deficient. Points lie in a subspace of rank smaller than {} dim".format(n_dim))
    submatrix = M[:, 0:n_dim]
    if np.linalg.det(submatrix) == 0:
        return NotImplementedError("Edge case not handled. Points will need reordering if other vectors"
                                   "are selected.")
    return submatrix

def gram_schmidt(B):
    """
    Take in an m by m matrix A and return a matrix with orthonormal vectors that span the same space as A
    In this case, we assume that the input matrix A is square and has linearly independent vectors.
    """
    G = np.zeros(B.shape)
    n_row, n_col = B.shape
    for j in range(n_col):
        vector_j = B[:, j]
        for k in range(0, j):
            # subtract the projection of j onto the previous vector from vector j
            scalar = (np.dot(G[:, k], B[:, j]) / np.dot(G[:, k], G[:, k]))
            projection = G[:, k] * scalar
            vector_j = vector_j - projection
        G[:, j] = vector_j
    norms = np.linalg.norm(G, axis=0)
    G = G / norms
    return G

def anchor_points(points):
    """
    points: 37 by d matrix
    Given a set of 37 points, find the rotation needed using Gram Schmidt and then
    transform the  coordinates to the format needed - one point is zero and the rest are partly in a triangular form
    """
    # make the first point the origin (translate all by first vector)
    points = points - points[0, :]
    # transpose the matrix so that each point is a column
    points = points.T
    # choose d independent vectors
    subset_points = choose_basis_vectors(points[:, 1:])
    Q = gram_schmidt(subset_points)
    # transform the 37 points
    R = Q.T @ points
    # put rotated points in the correct format
    return R.T


import numpy as np

class Optimizer:

    def __init__(self, precision=1e-6) -> None:
        super().__init__()
        self.has_converged = False
        self.precision = precision

    def next(self, X, dX):
        _X = self._next(X, dX)

        if np.abs(_X - X).mean() < self.precision:
            self.has_converged = True

        return _X

class GradientDescent(Optimizer):

    def __init__(self, learning_rate=0.01, **kwargs) -> None:
        super().__init__(**kwargs)
        self.learning_rate = learning_rate

    def _next(self, X, dX):
        return X - self.learning_rate * dX

class Adam(Optimizer):

    def __init__(self, alpha=0.01, beta_1=0.9, beta_2=0.999, epsilon=1e-8, **kwargs) -> None:
        super().__init__(**kwargs)

        self.alpha = alpha
        self.beta_1 = beta_1
        self.beta_2 = beta_2
        self.epsilon = epsilon

        self.m_t = 0
        self.v_t = 0
        self.t = 0

    def _next(self, X, dX):
        self.t += 1
        beta_1, beta_2 = self.beta_1, self.beta_2

        # update moving average of gradient and squared gradient
        self.m_t = beta_1 * self.m_t + (1 - beta_1) * dX
        self.v_t = beta_2 * self.v_t + (1 - beta_2) * (dX * dX)

        # calculates the bias-corrected estimates
        m_cap = self.m_t / (1 - (beta_1 ** self.t))
        v_cap = self.v_t / (1 - (beta_2 ** self.t))

        # do the gradient update
        _X = X - (self.alpha * m_cap) / (np.sqrt(v_cap) + self.epsilon)

        return _X


try:
    from scipy.special import gamma as _gamma, gammainc as _gammainc, digamma as _digamma
    import numpy as _np
    from math import sqrt as _sqrt, log as _log, exp as _exp
    from typing import Union, Dict, List
    from univariate._base import SemiInfinite
except Exception as e:
    print(f"some modules are missing {e}")

class Gamma(SemiInfinite):
    """
    This class contains methods concerning a variant of Gamma distribution [#]_.

    .. math::
        \\text{Gamma}(x;a,b) = \\frac{1}{b^a \\Gamma(a)} \\ x^{a-1} e^{\\frac{-x}{b}}

    Args:

        shape(float): shape parameter (:math:`a`) where shape > 0
        scale(float): scale parameter (:math:`b`) where scale > 0
        x(float): random variable where x > 0

    References:
        .. [#] Matlab(2020). Gamma Distribution. https://www.mathworks.com/help/stats/gamma-distribution.html
    """

    def __init__(self, shape: float, b: float, x: float):
        if shape < 0:
            raise ValueError('shape should be greater than 0.')
        if b < 0:
            raise ValueError('scale should be greater than 0.')
        self.shape = shape
        self.scale = b

    def pdf(self, x: Union[List[float], _np.ndarray, float]) -> Union[float, _np.ndarray]:
        """
        Args:
            x (Union[List[float], numpy.ndarray, float]): random variable(s)

        Raises:
            ValueError: when there exist a value of x that is less than 0

        Returns:
            Union[float, numpy.ndarray]: evaluation of pdf at x
        """
        shape = self.shape
        scale = self.scale

        if isinstance(x, (_np.ndarray, List)):
            if not type(x) is _np.ndarray:
                x = _np.array(x)
            if _np.any(x < 0):
                raise ValueError('random variable should be greater than 0.')
            return (1 / (pow(scale, shape) * _gamma(shape))) * _np.log(x, shape - 1) * _np.exp(-x / scale)

        if x < 0:
            raise ValueError('random variable should be greater than 0.')
        return (1 / (pow(scale, shape) * _gamma(shape))) * _log(x, shape - 1) * _exp(-x / scale)

    def cdf(self, x: Union[List[float], _np.ndarray, float]) -> Union[float, _np.ndarray]:
        """
        Args:
            x (Union[List[float], numpy.ndarray, float]): data point(s) of interest

        Returns:
            Union[float, numpy.ndarray]: evaluation of cdf at x
        """
        shape = self.shape
        scale = self.scale

        # there is no apparent explanation for reversing gammainc's parameter, but it works quite perfectly in my prototype
        def __generator(shape: float, b: float, x: Union[float, _np.ndarray]) -> Union[float, _np.ndarray]:
            return 1 - _gammainc(shape, x / b)

        if isinstance(x, (_np.ndarray, List)):
            if not type(x) is _np.ndarray:
                x = _np.array(x)
            return __generator(shape, scale, x)
        return __generator(shape, scale, x)

    def mean(self) -> Union[float, int]:
        """
        Returns: Mean of the Gamma distribution
        """
        return self.shape * self.scale

    def median(self) -> str:
        """
        Returns: Median of the Gamma distribution.
        """
        return "No simple closed form."

    def mode(self) -> Union[float, int]:
        """
        Returns: Mode of the Gamma distribution
        """
        return (self.shape - 1) * self.scale

    def var(self) -> Union[float, int]:
        """
        Returns: Variance of the Gamma distribution
        """
        return self.shape * pow(self.scale, 2)

    def std(self) -> float:
        """
        Returns: Standard deviation of the Gamma distribution
        """
        return _sqrt(self.var())

    def skewness(self) -> float:
        """
        Returns: Skewness of the Gamma distribution
        """
        return 2 / _sqrt(self.shape)

    def kurtosis(self) -> float:
        """
        Returns: Kurtosis of the Gamma distribution
        """
        return 6 / self.shape

    def entropy(self) -> float:
        """
        Returns: differential entropy of the Gamma distribution

        Reference: Park, S.Y. & Bera, A.K.(2009). Maximum entropy autoregressive conditional heteroskedasticity model. Elsivier.
        link: http://wise.xmu.edu.cn/uploadfiles/paper-masterdownload/2009519932327055475115776.pdf
        """
        k = self.shape
        theta = self.scale
        return k + _log(theta)+_log(_gamma(k))-(1-k)*_digamma(k)

    def summary(self) -> Dict[str, Union[float, int, str]]:
        """
        Returns:
            Dictionary of Gamma distirbution moments. This includes standard deviation.
        """
        return {
            'mean': self.mean(), 'median': self.median(), 'mode': self.mode(),
            'var': self.var(), 'std': self.std(), 'skewness': self.skewness(), 'kurtosis': self.kurtosis()
        }


from __future__ import print_function

import numpy as np

def meanclip(indata, clipsig=3.0, maxiter=5, converge_num=0.02, verbose=1, return_array=0, return_median=0):
    """Computes an iteratively sigma-clipped mean on a data set.

    Clipping is done about median, but mean is returned by default
    (use return_median to return the clipped median value).

    Parameters:
        indata : array_like
            Input data.

        clipsig : float
            Number of sigma at which to clip.

        maxiter : int
            Ceiling on number of clipping iterations.

        converge_num : float
            If the proportion of rejected pixels is less than
            this fraction, the iterations stop.

        verbose : {0, 1}
            Print messages to screen?

        return_array : {0, 1}
            Return the final array indices that were used to compute
            statistics.

        return_median : {0, 1}
            Return the median if 1. Return the mean if 0.

    Returns:
        val : float
            The N-sigma clipped mean or median.
        sigma : float
            The standard deviation of remaining pixels.

    Outputs:
        Prints to screen mean or median statistics.

    History:
        * 21/10/1998 Written by RSH, RITSS
        * 20/01/1999 Added SUBS, fixed misplaced paren on float call, improved doc. RSH
        * 24/11/2009 Converted to Python. PLL.
        * 08/01/2013 Added option to return the array indices of non-clipped pixels. DMH
        * Added option to return median of the clipped array. DMH

    Notes:
        This is based on MYMEANCLIP routine from ACS library.

    Examples:

    >>> mean, sigma = meanclip(indata)
    """
    # Flatten array
    skpix = indata.reshape( indata.size, )

    # initialize array to store indices of pixels used to compute stats
    arrind = np.arange(0,skpix.size)

    ct = indata.size
    iter = 0; c1 = 1.0 ; c2 = 0.0

    while (c1 >= c2) and (iter < maxiter):
        lastct = ct
        medval = np.median(skpix)
        sig = np.std(skpix)
        wsm = np.where( abs(skpix-medval) < clipsig*sig )
        ct = len(wsm[0])
        if ct > 0:
            skpix = skpix[wsm]
            arrind = arrind[wsm]

        c1 = abs(ct - lastct)
        c2 = converge_num * lastct
        iter += 1

    if return_median:
        val = np.median(skpix)
        val_type = 'median'
    else:
        val  = np.mean( skpix )
        val_type = 'mean'
    sigma = np.std( skpix )

    if verbose:
        if return_median:
            prf = 'MEDIANCLIP:'
            print('{} {}.1f-sigma clipped median'.format(prf, clipsig))
            print('{} Median computed in {} iterations'.format(prf, iter))
            print('P{} Median = {}, sigma = {}'.format(prf, val, sigma))

        else:
            prf = 'MEANCLIP:'
            print('{} {}.1f-sigma clipped mean'.format(prf, clipsig))
            print('{} Mean computed in {} iterations'.format(prf, iter))
            print('{} Mean = {}, sigma = {}'.format(prf, val, sigma))

    if return_array:
        return np.copy(arrind)
    else:
        return val, sigma


# https://www.hackerrank.com/challenges/np-mean-var-and-std/problem

# Variance: Mean of all the squared deviations from mean of the set of numbers
# Standard Deviation: Square root of variance

import numpy as np

n, m = list(map(int, input().split(' ')))

myList = []
for _ in range(n):
    myList += [input().split(' ')]
myArr = np.array(myList, int)
myArr.shape = (n, m)

print(np.mean(myArr, axis=1))
print(np.var(myArr, axis=0))
print(round(float(np.std(myArr)), 11))  # Rounded for compatibility in the challenge


import numpy as np

def projsplx(y):
    """projsplx projects a vector to a simplex
    by the algorithm presented in
    (Chen an Ye, "Projection Onto A Simplex", 2011)"""
    assert len(y.shape) == 1
    N = y.shape[0]
    y_flipsort = np.flipud(np.sort(y))
    cumsum = np.cumsum(y_flipsort)
    t = (cumsum - 1) / np.arange(1,N+1).astype('float')

    t_iter = t[:-1]
    t_last = t[-1]
    y_iter = y_flipsort[1:]

    if np.all((t_iter - y_iter) < 0):
        t_hat = t_last
    else:
        # find i such that t>=y
        eq_idx = np.searchsorted(t_iter - y_iter, 0, side='left')
        t_hat = t_iter[eq_idx]

    x = y - t_hat
    # there may be a numerical error such that the constraints are not exactly met.
    x[x<0.] = 0.
    x[x>1.] = 1.
    assert np.abs(x.sum() - 1.) <= 1e-5
    assert np.all(x >= 0) and np.all(x <= 1.)
    return x


import sys
from operator import add
from pyspark.sql import SparkSession
from pyspark import SparkContext
import pyspark
from pyspark.ml.linalg import Vectors
import numpy as np
from sklearn.linear_model import LinearRegression
from pyspark.sql.types import *
from pyspark.sql import functions as func
from pyspark.sql.functions import *
from pyspark.sql import SQLContext
import matplotlib.pyplot as plt
import time
from pandas import Series,DataFrame
import pandas as pd
import re
from collections import Counter
from sklearn.linear_model import LinearRegression
from pyspark.ml.classification import LogisticRegression
from pyspark.ml.evaluation import MulticlassClassificationEvaluator
from pyspark.ml.classification import LinearSVC

# building functions

def isfloat(value):
	try:
		float(value)
		return True
	except:
		return False

def correctRows(p):
	if isfloat(p[3]) and isfloat(p[4]) and isfloat(p[6]) and isfloat(p[7]) and isfloat(p[9]):
		return p

def to_list(a):
	return [a]

def addToList(x, y):
	x.append(y)
	return x

def extend(x,y):
	x.extend(y)
	return x

if __name__ == "__main__":
	if len(sys.argv) != 3:
		print("Usage: wordcount <file> <output> ", file=sys.stderr)
		exit(-1)

	spark = SparkSession.builder.master("local[*]").getOrCreate()
	sc = SparkContext.getOrCreate()
	sqlContext = SQLContext(sc)

	# load data set
	lines2 = sc.textFile("Google-Playstore.csv")

	# generate test case file
	df = pd.read_csv("Google-Playstore.csv")
	test_case = df.sample(n = 10000)
	test_case.to_csv('Google-Playstore_test.csv', index=False)

	"""

	Simple Linear Regression

	"""

	print("##### Finding Simple Linear Regression Equation #####")

	# data pre-processing

	correctLine = lines2.map(lambda x: x.split(','))
	cleaned = correctLine.filter(correctRows)

	max_install = cleaned.map(lambda p: (float(p[7])))
	rating = cleaned.map(lambda p: (float(p[3])))

	# apply linear regression
	x = np.array(max_install.collect())
	y = np.array(rating.collect())

	X = np.stack([x], axis = 1)

	reg = LinearRegression(fit_intercept=True).fit(X, y)

	print("The m (coefficient) =",reg.coef_)
	print("The b (y-intercept) =",reg.intercept_)
	print("The equation is: y = "+str(reg.coef_[0])+"X + "+str(reg.intercept_))

	"""

	Gradient Descent for parameters

	"""

	print("##### Finding the parameters using gradient descent #####")

	start1 = time.time()
	df = np.stack([y, x], axis=1)
	dff = map(lambda x: (float(x[0]), Vectors.dense(x[1:])), df)
	mydf = spark.createDataFrame(dff, schema=["Money", "Distance"])
	myRDD=mydf.rdd.map(tuple).map(lambda x: (float(x[0]), np.array(x[1]) ))

	learningRate = 0.00001
	num_iteration = 100
	size = float(len(y))
	beta = np.array([0.1])
	costs = []

	for i in range(num_iteration):
		gradientCost=myRDD.map(lambda x: (x[1], (x[0] - x[1] * beta) ))\
							   .map(lambda x: (x[0]*x[1], x[1]**2 )).reduce(lambda x, y: (x[0] +y[0], x[1]+y[1] ))
		cost= gradientCost[1]
		gradient=(-1/float(size))* gradientCost[0]
		print(i, "Beta", beta, " Cost", cost)
		beta = beta - learningRate * gradient
		costs.append(cost[0])

	end1 = time.time()

	print(f"Computation time of BGD is {(end1 - start1)/60} Minutes")

	# making plot
	xValues = [i for i in range(len(costs))]
	plt.plot(xValues, costs, 'o', markersize=2)
	plt.xlabel("Number of Iteration")
	plt.ylabel("Cost")
	plt.title("Cost with the number of iteration")
	plt.show()

	"""

	Multi-Linear Regression

	"""

	print("##### Finding the parameters of multi-linear regression using gradient descent #####")

	start2 = time.time()

	rating = cleaned.map(lambda p: (p[0], p[3]))
	rating_count = cleaned.map(lambda p: (p[0], p[4]))
	min_install = cleaned.map(lambda p: (p[0], p[6]))
	max_install = cleaned.map(lambda p: (p[0], p[7]))
	price = cleaned.map(lambda p: (p[0], p[9]))

	rating = rating.combineByKey(to_list, addToList, extend)
	rating = rating.collect()
	rating_count = rating_count.combineByKey(to_list, addToList, extend)
	rating_count = rating_count.collect()
	min_install = min_install.combineByKey(to_list, addToList, extend)
	min_install = min_install.collect()
	max_install = max_install.combineByKey(to_list, addToList, extend)
	max_install = max_install.collect()
	price = price.combineByKey(to_list, addToList, extend)
	price = price.collect()

	ratingKey = []
	ratingValue = []
	for i in range(len(rating)):
		ratingKey.append(rating[i][0])

		rate = 0
		total = 0
		for j in [float(i) for i in rating[i][1]]:
			rate += j
			total += 1

		ratingValue.append(rate/total)

	ratingCountKey = []
	ratingCountValue = []
	for i in range(len(rating_count)):
		ratingCountKey.append(rating_count[i][0])

		rate = 0
		for j in [float(i) for i in rating_count[i][1]]:
			rate += j

		ratingCountValue.append(rate)

	min_installKey = []
	min_installValue = []
	for i in range(len(min_install)):
		min_installKey.append(min_install[i][0])

		count = 0
		for j in [float(i) for i in min_install[i][1]]:
			if j != 0:
				count += 1

		min_installValue.append(count)

	max_installKey = []
	max_installValue = []
	for i in range(len(max_install)):
		min_installKey.append(max_install[i][0])

		count = 0
		for j in [float(i) for i in max_install[i][1]]:
			if j != 0:
				count += 1

		max_installValue.append(count)

	priceKey = []
	priceValue = []
	for i in range(len(price)):
		priceKey.append(price[i][0])

		amount = 0
		for j in [float(i) for i in price[i][1]]:
			amount += j

		priceValue.append(amount)

	app = ratingKey
	rating = ratingValue
	countOfRating = ratingCountValue
	mi_install = min_installValue
	Price = priceValue

	ma_install = max_installValue

	x = []
	y = []
	for i in range(len(app)):
		x.append([float(rating[i]), float(countOfRating[i]), float(mi_install[i]), float(Price[i])])
		y.append(float(ma_install[i]))

	learningRate = 0.000001
	num_iteration = 100
	size = len(y)
	costs = []
	beta = np.array([0.1, 0.1, 0.1, 0.1])

	data = {'y':y, 'x':x}
	df = DataFrame(data)
	spark_df_from_pandas = spark.createDataFrame(df, schema=['x', 'y'])
	myRDD=spark_df_from_pandas.rdd.map(lambda x: (float(x[0]), np.array(x[1])))

	for i in range(num_iteration):
		gradientCost=myRDD.map(lambda x: (x[1], (x[0] - x[1] * beta)))\
							   .map(lambda x: (x[0]*x[1], x[1]**2 )).reduce(lambda x, y: (x[0] +y[0], x[1]+y[1] ))

		cost = 0
		for j in gradientCost[1]:
			cost += j

		gradient=(-1/float(size))* gradientCost[0]
		print(i, "Beta", beta, " Cost", cost)
		beta = beta - learningRate * gradient

		costs.append(cost)

	end2 = time.time()

	print(f"Computation time of multi-linear regression by BGD is {(end2 - start2)/60} minutes")

	xValues = [i for i in range(len(costs))]

	plt.plot(xValues, costs, 'o', markersize=2)
	plt.xlabel("Number of Iteration")
	plt.ylabel("Cost")
	plt.title("Cost with the number of iteration")
	plt.show()

	"""

	Logistic Regression

	"""

	print("##### gradient descent algorithm to learn a logistic regression model #####")

	start3 = time.time()

	total_install = cleaned.map(lambda p: (p[0], p[7]))
	tuples = total_install.collect()

	appWords = []

	for i in app:
		words = i.split(" ")
		for j in words:
			j = re.sub('[^A-Za-z0-9]+', '', j)
			appWords.append(j)

	appWords = ' '.join(appWords).split()

	allWords = sc.parallelize(appWords)
	allCount = allWords.map(lambda x: (x, 1)).reduceByKey(add)
	topWords = allCount.top(20000, lambda x: x[1])

	topWordsK = sc.parallelize(range(20000))
	dictionary = topWordsK.map(lambda x: (topWords[x][0], x))

	def TF(words_list, top_words):
		words_dict = dict(Counter(words_list))
		tf_vector = []
		for word in top_words:
			if word in words_dict.keys():
				tf = words_dict[word]
				tf_vector.append(tf)
			else:
				tf_vector.append(0)
		return tf_vector

	key_id = [i for i in range(len(tuples))]
	key_values = {tuples[i][0]: i for i in key_id}
	topWordsBC = sc.broadcast(dictionary.keys().collect())
	feat = cleaned.map(lambda x: (key_values[x[0]], TF(x[1], topWordsBC.value)))
	labels = cleaned.map(lambda x: (key_values[x[0]], int(x[0][0] == 'A' and x[0][1] == 'U')))
	trainRDD = feat.join(labels)

	learningRate = 0.0003
	num_iteration = 5
	lambda_cof = 0.01
	size = len(tuples)

	loss_list = list()

	parameter_vector = np.random.normal(0, 0.1, (dictionary.count(), 1))

	def sigmoid(x):
		return 1.0 / (1 + np.exp(-x))

	def loss_func(feat_line, y, parameter_vector):
		feat_line = np.array(feat_line)
		pred = sigmoid(np.dot(feat_line, parameter_vector))
		return -y * np.log(pred + 1e-12) - (1 - y) * np.log((1 - pred) + 1e-12)

	def accuracy_score(feat_line, y, parameter_vector):
		feat_line = np.array(feat_line)
		pred = sigmoid(np.dot(feat_line, parameter_vector))
		pred = 1 if pred >= 0.5 else 0
		acc = int(pred == y)
		return acc

	def grad_func(feat_line, y, parameter_vector):
		feat_line = np.array(feat_line)
		pred = sigmoid(np.dot(feat_line, parameter_vector))
		grad = (pred - y) @ feat_line[None, :]
		return grad

	acc_his = []
	loss_his = []
	grad_his = []
	prev_param_norm = 0

	for i in tqdm.trange(num_iteration):
		parameter_vector_BC = sc.broadcast(parameter_vector)

		loss = trainRDD.map(lambda x: loss_func(x[1][0], x[1][1], parameter_vector_BC.value)).reduce(add) / size
		acc = trainRDD.map(lambda x: accuracy_score(x[1][0], x[1][1], parameter_vector_BC.value)).reduce(add) / size
		grad = trainRDD.map(lambda x: grad_func(x[1][0], x[1][1], parameter_vector_BC.value)).reduce(add) / size

		parameter_vector = parameter_vector - learningRate * grad[:, None]

		if np.abs(np.linalg.norm(parameter_vector) - prev_param_norm) < 1e-7:
			print('Break')
			break

		prev_param_norm = np.linalg.norm(parameter_vector)
		# L2
		parameter_vector = parameter_vector - 2 * lambda_cof * parameter_vector

		acc_his.append(acc)
		loss_his.append(loss)
		grad_his.append(np.linalg.norm(grad))

	end3 = time.time()

	print(f"Computation time of multi-linear regression by BGD is {(end3 - start3)/60} minutes")

	fig, ax = plt.subplots(3, figsize=(13, 13))
	ax[0].set_title('Accurary')
	ax[0].plot(acc_his)
	ax[1].set_title('Loss')
	ax[1].plot(loss_his)
	ax[2].set_title('GradNorm')
	ax[2].plot(grad_his)
	plt.savefig("TrainingProcess.png")
	plt.show()

	print('The five words with the largest coefficients',
		  np.array(topWordsBC.value)[np.argsort(parameter_vector[:, 0])[-5:]])

	"""

	Logistic Regression Model Evaluation

	"""

	print("##### model evaluation #####")

	start4 = time.time()

	t_tuples = total_install.collect()
	key_id = [i for i in range(len(tuples))]
	key_values = {tuples[i][0]: i for i in key_id}
	topWordsBC = sc.broadcast(dictionary.keys().collect())
	feat = cleaned.map(lambda x: (key_values[x[0]], TF(x[1], topWordsBC.value)))
	labels = cleaned.map(lambda x: (key_values[x[0]], int(x[0][0] == 'A' and x[0][1] == 'U')))
	testRDD = feat.join(labels)

	# val
	def TP_func(feat_line, y, parameter_vector):
		feat_line = np.array(feat_line)
		pred = sigmoid(np.dot(feat_line, parameter_vector))
		pred = 1 if pred >= 0.5 else 0
		TP = int(pred == 1 and y == 1)
		return TP

	def FP_func(feat_line, y, parameter_vector):
		feat_line = np.array(feat_line)
		pred = sigmoid(np.dot(feat_line, parameter_vector))
		pred = 1 if pred >= 0.5 else 0
		FP = int(pred == 1 and y != 1)
		return FP

	def FN_func(feat_line, y, parameter_vector):
		feat_line = np.array(feat_line)
		pred = sigmoid(np.dot(feat_line, parameter_vector))
		pred = 1 if pred >= 0.5 else 0
		FN = int(pred != 1 and y == 1)
		return FN

	def TN_func(feat_line, y, parameter_vector):
		feat_line = np.array(feat_line)
		pred = sigmoid(np.dot(feat_line, parameter_vector))
		pred = 1 if pred >= 0.5 else 0
		TN = int(pred != 1 and y != 1)
		return TN

	parameter_vector_BC = sc.broadcast(parameter_vector)
	acc = testRDD.map(lambda x: accuracy_score(x[1][0], x[1][1], parameter_vector_BC.value)).reduce(add) / size
	TP = testRDD.map(lambda x: TP_func(x[1][0], x[1][1], parameter_vector_BC.value)).reduce(add)
	FP = testRDD.map(lambda x: FP_func(x[1][0], x[1][1], parameter_vector_BC.value)).reduce(add)
	FN = testRDD.map(lambda x: FN_func(x[1][0], x[1][1], parameter_vector_BC.value)).reduce(add)
	TN = testRDD.map(lambda x: TN_func(x[1][0], x[1][1], parameter_vector_BC.value)).reduce(add)

	F1 = 2 * TP / (2 * TP + FN + FP)
	print('The Acc of Test: ', acc)
	print('The F1 score of Test: ', F1)

	print(f"Computation time of multi-linear regression by BGD is {(end4 - start4)/60} minutes")

	"""

	SVM Model

	"""

	print("##### SVM model #####")

	start5 = time.time()

	d_corpus = sc.textFile("Google-Playstore.csv")
	d_keyAndText = d_corpus.map(lambda x: (x[x.index('id="') + 4: x.index('" url=')], x[x.index('">') + 2:][:-6]))
	regex = re.compile('[^a-zA-Z]')

	d_keyAndListOfWords = d_keyAndText.map(lambda x: (str(x[0]), regex.sub(' ', x[1]).lower().split())).sortByKey(False)

	tuples = d_keyAndListOfWords.collect()
	allWordsList = []

	for i in range(len(tuples)):
		for j in tuples[i][1]:
			allWordsList.append(j)

	allWords = sc.parallelize(allWordsList)
	allCount = allWords.map(lambda x: (x, 1)).reduceByKey(add)
	topWords = allCount.top(20000, lambda x: x[1])

	topWordsK = sc.parallelize(range(20000))
	dictionary = topWordsK.map(lambda x: (topWords[x][0], x))

	def TF(words_list, top_words):
		words_dict = dict(Counter(words_list))
		tf_vector = []
		for word in top_words:
			if word in words_dict.keys():
				tf = words_dict[word]
				tf_vector.append(tf)
			else:
				tf_vector.append(0)
		return Vectors.dense(tf_vector)

	key_id = [i for i in range(len(tuples))]
	key_values = {tuples[i][0]: i for i in key_id}
	topWordsBC = sc.broadcast(dictionary.keys().collect())
	feat = d_keyAndListOfWords.map(lambda x: (key_values[x[0]], TF(x[1], topWordsBC.value)))
	labels = d_keyAndListOfWords.map(lambda x: (key_values[x[0]], int(x[0][0] == 'A' and x[0][1] == 'U')))
	train_feat_df = sqlContext.createDataFrame(feat, ['ind', 'features'])
	train_labels_df = sqlContext.createDataFrame(labels, ['ind', 'labels'])
	train_df = train_feat_df.join(train_labels_df, on=['ind']).sort(['ind'])
	train_df.cache()

	svc = LinearSVC(labelCol='labels')
	svg_model = svc.fit(train_df)

	st_test_read = time.time()
	t_corpus = sc.textFile('Google-Playstore_test.csv')
	t_keyAndText = t_corpus.map(lambda x: (x[x.index('id="') + 4: x.index('" url=')], x[x.index('">') + 2:][:-6]))
	regex = re.compile('[^a-zA-Z]')
	t_keyAndListOfWords = t_keyAndText.map(lambda x: (str(x[0]), regex.sub(' ', x[1]).lower().split())).sortByKey(False)
	t_tuples = t_keyAndListOfWords.collect()
	t_key_id = [i for i in range(len(t_tuples))]
	t_key_values = {t_tuples[i][0]: i for i in t_key_id}
	t_feat = t_keyAndListOfWords.map(lambda x: (t_key_values[x[0]], TF(x[1], topWordsBC.value)))
	t_labels = t_keyAndListOfWords.map(
		lambda x: (t_key_values[x[0]], int(x[0][0] == 'A' and x[0][1] == 'U')))
	test_feat_df = sqlContext.createDataFrame(t_feat, ['ind', 'features'])
	test_labels_df = sqlContext.createDataFrame(t_labels, ['ind', 'labels'])
	test_df = test_feat_df.join(test_labels_df, on=['ind']).sort(['ind'])
	test_df.cache()

	st_test = time.time()
	test_pred = svg_model.evaluate(test_df).predictions
	evaluator = MulticlassClassificationEvaluator(labelCol='labels')

	print('Acc of Test: ', evaluator.evaluate(test_pred, {evaluator.metricName: "accuracy"}))
	print('F1 of Test:', evaluator.evaluate(test_pred, {evaluator.metricName: "f1"}))

	end5 = time.time()

	print(f"Computation time of SVM regression by BGD is {(end5 - start5)/60} minutes")


# coding: utf-8
import numpy as np
import matplotlib.pyplot as plt

X = 2 * np.random.rand(100, 1) # pass in desired shape. In this case, (100,1)
y = 4 + 3 * X + np.random.randn(100, 1)
# NOTE: I'm using two different random functions. rand() gives a uniform distribution, randn() gives a Gaussian distribution
plt.scatter(X, y, s=20)
plt.axis(ymin=0.0)
plt.show()

# Linear Regression

X_b = np.c_[np.ones((100,1)), X] # add the bias term to every instance
print(X.shape)
print(X_b.shape)

# Cost Function
#
# Before we start training our model, we need to define an error function to minimize.
# One of the most common cost functions used in regression is the Mean Squared Error, or MSE.

def MSE_error(theta):
    err = 0
    m = X.shape[0]
    for i in range(m):
        y_pred = theta.T.dot(X_b[i])[0]
        err += (y_pred - y[i])**2
    return err/m

# # Training Algorithm: Gradient Descent
#
# * https://en.wikipedia.org/wiki/Gradient_descent
# * Generic Optimization Algorithm - used in many domains outside of ML
#
# A common analogy to explain Gradient Descent is to image that you are standing on hilly terrain covered in fog.
# You want to find the deepest valley (minimum), but can only see your immediate surroundings.
# Your best strategy is to look at the steepness of the terrain at your feet. Find the direction of steepest descent,
# walk some distance in that direction, and repeat. Eventually you will find yourself in a valley.
#
# We are trying to find the parameter values that minimize our cost function (MSE). To do this, we approximate the derivative of
# the cost function with respect to our model parameters. When we have a multivariate (more than one input variable) function,
# we need to compute the *Gradient* of the function. The Gradient is the multivariable generalization of the derivative.
# It is a vector containing *partial derivatives.*
# A partial derivative is obtained by holding all other input variables constant, and differentiating our function with respect to just one variable.
# The Gradient, then, is a vector containing all the partial derivatives. In this case, our gradient vector will have two components.
#

learn_rate = 0.1
n_iter = 20
m = X.shape[0]

MSE_list = np.zeros(shape=(n_iter,))

theta = np.random.randn(2,1) # random initialization of theta
print('starting value of theta:\n', theta)

plt.scatter(X, y, s=20)
for iter in range(n_iter):
    gradients = 2/m * X_b.T.dot(X_b.dot(theta) - y) # compute the gradient
    theta = theta - learn_rate * gradients # update theta

    y_0 = theta.T.dot((1,0)) # should be roughly 4
    y_2 = theta.T.dot((1,2)) # should be roughly 10
    plt.plot([0,2], [y_0,y_2])
    MSE_list[iter] = MSE_error(theta)

plt.show()

print('optimized value of theta:\n', theta)

print('MSE error at each iteration:')
plt.plot(np.arange(n_iter), MSE_list)
plt.show()

# In[6]:

y_0 = theta.T.dot((1,0)) # should be roughly 4
y_2 = theta.T.dot((1,2)) # should be roughly 10

plt.scatter(X, y, s=20)
plt.plot([0,2], [y_0,y_2])
plt.show()


from math import ceil, sqrt

import numpy as np

def hoeffding_n_given_t_and_p_one_sided(t:np.double, p:np.double, C=0.5) -> int:
    """
    Return n such that with probability at least p, P(E[X] < \bar X_n + t).

    Where \bar X_n is the mean of n samples.

    Parameters
    ----------
    t : double
        one sided confidence interval width
    p : double
        probability of bound holding
    C : double
        Width of sample support domain. E.g. 0.5 if all samples fall in
            [0.5, 1.0]

    Returns
    -------

    """
    return int(ceil(C ** 2 * np.log(1 - p) / (-2 * t ** 2)))

def hoeffding_n_given_t_and_p_two_sided(t:np.double, p:np.double, C=0.5) -> int:
    """
    Return n such that with probability at least p, P(|E[X] - \bar X_n| <= t).

    Where \bar X_n is the mean of n samples.

    Parameters
    ----------
    t : double
        two sided confidence interval width
    p : double
        probability of bound holding
    C : double
        Width of sample support domain. E.g. 0.5 if all samples fall in
            [0.5, 1.0]

    Returns
    -------

    """
    return int(ceil(C ** 2 * np.log( 0.5*(1 - p) ) / (-2 * t ** 2)))

def chebyshev_k_from_upper_bound_prob(p_bound_holds:np.double) -> int:
    """
    Return k such with with probability at least p_bound_holds X will be < mu + k*sigma

    Parameters
    ----------
    p_bound_holds : double

    Returns
    -------

    """
    p_bound_violated = 1 - p_bound_holds
    return int(ceil(sqrt(1 / p_bound_violated)))

def accuracy_to_statistical_distance(accuracy):
    return (accuracy - 0.5) * 2

def statistical_distance_to_accuracy(statistical_distance):
    return 0.5 + 0.5 * statistical_distance


import numpy as np
from numpy.linalg import *

def leverage(X):
    """ compute leverage of a design matrix

    Parameters: X (nd-array) - the design matrix (with intercept)

    Returns: (array) - the leverage
    """
    # the orthogonal projection onto the column space of X
    H = np.dot(np.dot(X, np.linalg.inv(np.dot(X.T, X))), X.T)
    # diagonal of hat matrix, aka, leverage
    H_diag = H.diagonal()

    return H_diag

def internally_studentized(residuals, h, p, n):
    """ compute the Internally Studentized Residuals

    Parameters: residual (array-like) - the raw residuals
                h (array-like) - leverae of the design matrix
                p (int) - coefficients dimensions
                n (int) - total numbers of observations

    Returns: (array) - the studentized residual
    """
    sigma = np.sqrt((1/(n-p))*np.dot(residuals.T, residuals))
    standardized_residuals = residuals / (sigma*np.sqrt(1-h))
    return standardized_residuals

def cooks_distance(standard_residuals, h, p):
    """ compute cook's distance

    Parameters: standard_residuals (array) - studentized residauls
                h (array) - leverage
                p (int) - coefficients dimensions

    Returns: (array) - an array of Cook's distance
    """
    cooks = (h / (1-h)) * (standard_residuals**2 / p)
    return cooks

################### for t test ###################
def coef_standard_error(residuals, X):
    """ find standard errors of regression coefficients

    Parameters: residuals (array) - raw residuals
                X (nd-array) - design matrix, including intercept

    Returns: (nd-array) - the standard errors of regression coefficients
    """
    sse = np.dot(residuals.T, residuals) / (X.shape[0] - X.shape[1])
    se = np.sqrt(np.diagonal(sse * np.linalg.inv(np.dot(X.T, X))))
    return se

################### r2 & adj r2 ###################
def r_squared(y, residuals, p):
    """ compute r-squared and adjusted r-squared

    Parameters: y (array) - raw response
                residuals (array) - raw residuals
                p (int) - coefficients dimensions, including intercept

    Returns: r2, adj_r2 (float), (float)
    """
    n = len(y)
    sse = np.sum(residuals**2)
    sst = np.sum((y - np.mean(y))**2)
    r2 = 1 - (sse / sst)
    adj_r2 = 1 - ((sse / (n-p)) / (sst/(n-1)))
    return r2, adj_r2

################### for f test ###################
def f_stat(residuals, fitted_values, y, p):
    """ compute f statistics for testing regression model slope

    Parameters: residuals (array) - raw residuals
                fitted_values (array) - the model fitted values
                y (array) - raw response
                p (int) - coefficients dimesnion, including intercept

    Returns: (float) - the f statistics for the model
    """
    n = len(y)
    sse = np.sum(residuals**2)
    ssr = np.sum((fitted_values - np.mean(y))**2)
    f_stat = (ssr / (p-1)) / (sse / (n-p))
    return f_stat


import sympy as sym
# Mesh: |--------|-------|
#       0      pi/2      pi
#
# Basis functions:
#
#   phi_0   phi_1   phi_2
#     \      /\      /
#      \    /  \    /
#       \  /    \  /
#        \/      \/
#     |-------|-------|
#     0      pi/2     pi

x = sym.Symbol('x')
A = sym.zeros(3,3)
f = sym.sin

phi_0 = 1 - (2*x)/sym.pi
phi_1l = 2*x/sym.pi          # left part of phi_1
phi_1r = 2 - (2*x)/sym.pi    # right part of phi_1
phi_2 = x/(sym.pi/2) - 1
node_0 = 0
node_1 = sym.pi/2
node_2 = sym.pi

# Diagonal terms
A[0,0] = sym.integrate(phi_0**2,  (x, node_0, node_1))
A[1,1] = sym.integrate(phi_1l**2, (x, node_0, node_1)) + \
         sym.integrate(phi_1r**2, (x, node_1, node_2))
A[2,2] = sym.integrate(phi_2**2,  (x, node_1, node_2))

# Off-diagonal terms
A[0,1] = sym.integrate(phi_0*phi_1l, (x, node_0, node_1))
A[1,0] = A[0,1]

A[1,2] = sym.integrate(phi_1r*phi_2, (x, node_1, node_2))
A[2,1] = A[1,2]

print('A:\n', A)  # Can compare with general matrix, h=pi/2

b = sym.zeros(3,1)

b[0] = sym.integrate(phi_0*f(x),  (x, node_0, node_1))
b[1] = sym.integrate(phi_1l*f(x), (x, node_0, node_1)) + \
       sym.integrate(phi_1r*f(x), (x, node_1, node_2))
b[2] = sym.integrate(phi_2*f(x),  (x, node_1, node_2))

print('b:\n', b)

c = A.LUsolve(b)
print('c:\n', c)

for i in range(len(c)):
    print('c[%d]=%g' % (i, c[i].evalf()))
print('u(pi/2)=%g' % c[1])

# For reports
print(sym.latex(A))
print(sym.latex(b))
print(sym.latex(c))


from math import exp

import numpy as np

class Utils:
    @staticmethod
    def cast_int(s):
        try:
            return int(s)
        except:
            return None

    @staticmethod
    def cast_float(s):
        try:
            return float(s)
        except:
            return None

    @staticmethod
    def softmax(x):
        """
            https://medium.com/data-science-bootcamp/understand-the-softmax-function-in-minutes-f3a59641e86d
            Compute softmax values for each sets of scores in x.
        """
        return np.exp(x) / np.sum(np.exp(x), axis=0)

    @staticmethod
    def sigmoid(x):
        """
            https://machinelearningmastery.com/choose-an-activation-function-for-deep-learning/
            :param x:
            :return:
        """
        return 1.0 / (1.0 + exp(-x))

    @staticmethod
    def tanh(x):
        """
            https://machinelearningmastery.com/choose-an-activation-function-for-deep-learning/
            :param x:
            :return:
        """
        return (exp(x) - exp(-x)) / (exp(x) + exp(-x))


"""
  *** Hussain Salih Mahdi ***
  _________Shubbair__________

TODO Gradiant Descent find (m,b) from student degree @m = coefficient , @b = Intercept

inputs : csv file of student degree
output : predict the value should be close to the m an b

"""

import numpy as np
import pandas as pd
from sklearn import linear_model
import math

def sklearn_predict():
    data = pd.read_csv('input.csv')
    reg = linear_model.LinearRegression()
    reg.fit(data[['math']], data.cs)
    return reg.coef_, reg.intercept_

def gradiant_descent(x, y):
    m_curr = b_curr = 0
    iterations = 1000000
    learning_rate = 0.0002
    cost_previous = 0

    for i in range(iterations):
        py = m_curr * x + b_curr
        cost = (1 / len(x)) * sum([value ** 2 for value in (y - py)])
        md = -(2 / len(x)) * sum(x * (y - py))
        bd = -(2 / len(x)) * sum(y - py)
        m_curr = m_curr - learning_rate * md
        b_curr = b_curr - learning_rate * bd
        if math.isclose(cost, cost_previous, rel_tol=1e-20):
            break
        cost_previous = cost
        print("m {}, b {}, cost {}, iteration {}".format(m_curr, b_curr, cost, i))
    return m_curr, b_curr

if __name__ == "__main__":
    degree = pd.read_csv('input.csv')
    x = np.array(degree.math)
    y = np.array(degree.cs)

    m, b = gradiant_descent(x, y)
    print('Using Gradiant Descent : Coefficient {} , Intercept {}'.format(m, b))

    sklearn_m, sklearn_b = sklearn_predict()
    print('Using Sklearn : Coefficient {} , Intercept {}'.format(sklearn_m, sklearn_b))


import numpy as np
# DO NOT add any library

def train_transformation(X, Y,R, train_steps=100, learning_rate=0.0003):
    #please write your code here
    # find best R using Gradient Descent
    #WRITE your code here
    m = len(Y)
    prediction = X.dot(R)
    for _ in range(train_steps):
        R = R - np.transpose((learning_rate / m * (np.transpose(prediction - Y))).dot(X))

    #END of your code
    return R

def nearest_neighbor(v, candidates, k=1):
    # find k best similar vectors index. please sort them in order max to min and return index
    # for your similarity function please use cosine similarity
    similarity_l = []
    #WRITE your code here
    for candidate in candidates:
        ss = v.dot(np.transpose(candidate)) / np.sqrt(v.dot(np.transpose(v)) * candidate.dot(np.transpose(candidate)))
        similarity_l = np.append(similarity_l, ss)
    #END of your code
    sorted_ids = np.argsort(similarity_l)

    k_idx = sorted_ids[-k:]
    return k_idx

# This is how this code is called
# X, y = np.load('X_train.npy'), np.load('Y_train.npy')
# print(train_transformation(X, y, np.random.randn(300,300)))


import numpy as np
from math import tau
from scipy.special import binom

import matplotlib.pyplot as plt

def perimeter_bessel(a, b=1, k=10):
    h = (a-b)**2 / (a+b)**2
    return np.pi * (a + b) * [(binom(0.5, range(k)) ** 2 * v**np.arange(k)).sum() for v in h]

def perimeter_parker(a, b=1):
    return 3 * tau * (a / 5 + b / 8)

def perimeter_jcop(a, b=1):
    return tau * (7/45 * a + 27/41 * np.sqrt((a**2 + b**2)/2) + 7/37 * b)

def main():
    space = np.linspace(1, 75, 500)

    parker = perimeter_parker(space)
    bessel = perimeter_bessel(space)
    jcop = perimeter_jcop(space)

    err_parker = np.abs(1 - parker / bessel) * 100
    err_jcop = np.abs(1 - jcop / bessel) * 100

    plt.figure(figsize=(12, 4))
    plt.subplot(121)
    plt.plot(space[:40], err_parker[:40], "k-", label="parker")
    plt.plot(space[:40], err_jcop[:40], "m-", label="jcop")
    plt.legend()

    plt.subplot(122)
    plt.plot(space, err_parker, "k-", label="parker")
    plt.plot(space, err_jcop, "m-", label="jcop")
    plt.legend()

    plt.tight_layout()
    plt.show()

if __name__ == "__main__":
    main()


#########################
##                     ##
## Irving Gomez Mendez ##
##  February 12, 2021  ##
##                     ##
#########################

import numpy as np
import matplotlib.pyplot as plt
from scipy.stats import t

# The general formula of a ellipse is given by
# x^2 + By^2 + Cxy + Dx + Ey + F = 0
# i.e. By^2 + Cxy + Dx + Ey + F = -x^2

# We can see this problem as finding the function
# f(x,y) = x^2 + By^2 + Cxy + Dx + Ey + F
# which minimizes the data:
# x  | y  | f(x,y)
# x1 | y1 | 0
# x2 | y2 | 0

points = np.array([[0,2],[2,1],[1,-1],[-1,-2],[-3,1],[-3,-2]])

points_x = points[:,0]
points_y = points[:,1]
points_x2 = points_x**2
points_y2 = points_y**2
points_xy = points_x*points_y

n = points.shape[0]
yy = -points_x2
XX = np.vstack([points_y2, points_xy, points_x, points_y, np.ones(len(points_x))]).T
p = XX.shape[1]

# We get the coeff.
B, C, D, E, F = np.linalg.lstsq(XX, yy, rcond=None)[0]
SSE = np.linalg.lstsq(XX, yy, rcond=None)[1]

# We plot the ellipse
x = np.linspace(-4.5, 2.5, 50)
y = np.linspace(-2.5, 2.5, 50)
x, y = np.meshgrid(x, y)

plt.plot(points_x, points_y, 'o', markersize=7)
CS = plt.contour(x, y, (x**2+B*y**2+C*x*y+D*x+E*y+F), [0], colors='purple')
plt.title('Best fit ellipse by least squares')

# These are the points on the ellipse
points_ellipse = CS.allsegs[0][0]

ellipse_x = points_ellipse[:,0]
ellipse_y = points_ellipse[:,1]
ellipse_y2 = ellipse_y**2
ellipse_x2 = ellipse_x**2
ellipse_xy = ellipse_x*ellipse_y

ellipse_matrix = np.vstack([ellipse_y2, ellipse_xy, ellipse_x, ellipse_y, np.ones(len(ellipse_x))]).T

# Confidence interval
# Remember that the points on the adjusted ellipse satisfied: \hat{f}(x,y)=0
# So the interval is just given by +- t_{alpha/2,n-p}\sqrt{SSE/(n-p)*X_0(X^TX)^{-1}X_0^T}
# which represent the level on which the conic surface should be cut.

alpha = 0.05
aux_t = np.sqrt(SSE/(n-p)*(np.diag(ellipse_matrix @ np.linalg.inv(XX.T @ XX) @ ellipse_matrix.T)))
upp_contour = np.mean(t.ppf(1-alpha/2,1)*aux_t)
low_contour = np.mean(-t.ppf(1-alpha/2,1)*aux_t)

x = np.linspace(-5, 3, 50)
y = np.linspace(-3, 3, 50)
x, y = np.meshgrid(x, y)

plt.plot(points_x, points_y, 'o', markersize=7)
CS = plt.contour(x, y, (x**2+B*y**2+C*x*y+D*x+E*y+F), [low_contour, 0, upp_contour], colors=['red', 'purple', 'green'])
plt.title('Best fit ellipse by least squares')


# For this part of the assignment, please implement your own code for all computations,
# Do not use inbuilt functions like fft from either numpy, opencv or other libraries
import math
import numpy as np

class DFT:

    def forward_transform(self, matrix):
        """Computes the forward Fourier transform of the input matrix
        takes as input:
        matrix: a 2d matrix
        returns a complex matrix representing fourier transform"""

        # Using direct function
        # matrix1 = np.fft.fft2(matrix)

        # Without using direct function
        rows, columns = np.shape(matrix)
        matrix1 = np.zeros((rows, columns), dtype=complex)
        for u in range(rows):
            for v in range(columns):
                a = []
                for i in range(rows):
                    for j in range(columns):
                        omega = np.exp(-2 * math.pi * 1J * (((u * i) / rows) + ((v * j) / columns)))
                        a.append(matrix[i, j] * omega)
                matrix1[u, v] = sum(a)

        return matrix1

    def inverse_transform(self, matrix):
        """Computes the inverse Fourier transform of the input matrix
        matrix: a 2d matrix (DFT) usually complex
        takes as input:
        returns a complex matrix representing the inverse fourier transform"""

        # Using direct function
        # matrix1 = np.fft.ifft2(matrix)

        # Without using direct function
        rows, columns = np.shape(matrix)
        matrix1 = np.zeros((rows, columns), dtype=complex)
        for u in range(rows):
            for v in range(columns):
                a = []
                for i in range(rows):
                    for j in range(columns):
                        omega = np.exp(2 * math.pi * 1J * (((u * i) / rows) + ((v * j) / columns)))
                        a.append(matrix[i, j] * omega)
                matrix1[u, v] = sum(a)
                # Normalized value of inverse dft
                # matrix1[u, v] = (1 / (rows * columns)) * sum(a)

        return matrix1

    def discrete_cosine_tranform(self, matrix):
        """Computes the discrete cosine transform of the input matrix
        takes as input:
        matrix: a 2d matrix
        returns a matrix representing discrete cosine transform"""

        rows, columns = np.shape(matrix)
        matrix1 = np.zeros((rows, columns), dtype=float)
        for u in range(rows):
            for v in range(columns):
                a = []
                for i in range(rows):
                    for j in range(columns):
                        temp = math.cos(2 * math.pi * (((u * i) / rows) + ((v * j) / columns)))
                        a.append(matrix[i, j] * temp)
                matrix1[u, v] = sum(a)

        return matrix1

    def discrete_sine_tranform(self, matrix):
        """Computes the discrete cosine transform of the input matrix
        takes as input:
        matrix: a 2d matrix
        returns a matrix representing discrete cosine transform"""

        rows, columns = np.shape(matrix)
        matrix1 = np.zeros((rows, columns), dtype=float)
        for u in range(rows):
            for v in range(columns):
                a = []
                for i in range(rows):
                    for j in range(columns):
                        temp = math.sin(2 * math.pi * (((u * i) / rows) + ((v * j) / columns)))
                        a.append(matrix[i, j] * temp)
                matrix1[u, v] = sum(a)

        return matrix1

    def magnitude(self, matrix):
        """Computes the magnitude of the DFT
        takes as input:
        matrix: a 2d matrix
        returns a matrix representing magnitude of the dft"""

        # method 1
        # matrix1 = np.abs(matrix)

        # method 2
        rows, columns = np.shape(matrix)
        matrix1 = np.zeros((rows, columns), dtype=float)
        for i in range(rows):
            for j in range(columns):
                # matrix1[i, j] = abs(matrix[i, j])
                matrix1[i, j] = math.sqrt(matrix[i, j].real ** 2 + matrix[i, j].imag ** 2)

        return matrix1


import numpy as np
from matplotlib import pyplot as plt
import math

def f1(a0,a1,x):
    return (a0+(a1*x))
def f2(a0,a1,a2,x):
    return (a0+(a1*x)+(a2*x*x))
def f3(a0,a1,a2,a3,x):
    return (a0+(a1*x)+(a2*x*x)+(a3*x*x*x))

inpo=open('data.txt','r').readlines()
n=len(inpo)
x=[0 for i in range(n)]
y=[0 for i in range(n)]
for i in range(n):
    spl=inpo[i].split(' ')
    x[i]=float(spl[0])
    y[i]=float(spl[1])
plt.scatter(x,y,s=3)
plt.xlabel('X Label')
plt.ylabel('Y label')
plt.legend()
plt.grid()
#deg 1
sumx=0
for i in range(n):
    sumx=sumx+x[i]

sumy=0
for i in range(n):
    sumy=sumy+y[i]
sumx2=0
for i in range(n):
    sumx2=sumx2+(x[i]*x[i])
sumy2=0
for i in range(n):
    sumy2=sumy2+(y[i]*y[i])
sumxy=0
for i in range(n):
    sumxy=sumxy+(x[i]*y[i])
a1=((n*sumxy)-(sumx*sumy))/((n*sumx2)-(sumx*sumx))
ybar=sumy/n
xbar=sumx/n
a0=ybar-(a1*xbar)
r=((n*sumxy)-(sumx*sumy))/(math.sqrt((n*sumx2)-(sumx*sumx))*math.sqrt((n*sumy2)-(sumy*sumy)))
print('for order 1')
print('a0=',a0)
print('a1=',a1)
print('r=',r)
#order 2
sumx3=0
for i in range(n):
    sumx3=sumx3+(x[i]*x[i]*x[i])
sumx4=0
for i in range(n):
    sumx4=sumx4+(x[i]*x[i]*x[i]*x[i])

meta= [[0 for i in range(3)] for j in range(3)]
metb=[0 for i in range(3)]
for i in range(3):
    for j in range(3):
        if i==0 and j==0:
            meta[i][j]=n
        else :
            k=i+j
            if k==1:
                meta[i][j]=sumx
            elif k==2:
                meta[i][j]=sumx2
            elif k==3:
                meta[i][j]=sumx3
            elif k==4:
                meta[i][j]=sumx4
sumx2y=0
for i in range(n):
    sumx2y=sumx2y+(x[i]*x[i]*y[i])
metb[0]=sumy
metb[1]=sumxy
metb[2]=sumx2y
#print(meta)
aord2=np.linalg.solve(meta,metb)
st=0
sr=0
for i in range (n):
    sr=sr+((y[i]-f2(aord2[0],aord2[1],aord2[2],x[i]))*(y[i]-f2(aord2[0],aord2[1],aord2[2],x[i])))
ybar=sumy/n
for i in range(n):
    st=st+((ybar-y[i])*(ybar-y[i]))
r=math.sqrt((st-sr)/st)
print('For order 2')
print('a0=',aord2[0])
print('a1=',aord2[1])
print('a2=',aord2[2])
print('r=',r)

#deg3
meta1= [[0 for i in range(4)] for j in range(4)]
metb1=[0 for i in range(4)]
sumx5=0
for i in range(n):
    sumx5=sumx5+(x[i]*x[i]*x[i]*x[i]*x[i])
sumx6=0
for i in range(n):
    sumx6=sumx6+(x[i]*x[i]*x[i]*x[i]*x[i]*x[i])

for i in range(4):
    for j in range(4):
        if i==0 and j==0:
            meta1[i][j]=n
        else :
            k=i+j
            if k==1:
                meta1[i][j]=sumx
            elif k==2:
                meta1[i][j]=sumx2
            elif k==3:
                meta1[i][j]=sumx3
            elif k==4:
                meta1[i][j]=sumx4
            elif k==5:
                meta1[i][j]=sumx5
            elif k==6:
                meta1[i][j]=sumx6
sumx3y=0
for i in range(n):
    sumx3y=sumx3y+(x[i]*x[i]*x[i]*y[i])
metb1[0]=sumy
metb1[1]=sumxy
metb1[2]=sumx2y
metb1[3]=sumx3y
aord3=np.linalg.solve(meta1,metb1)
#print(meta1)
st=0
sr=0
for i in range (n):
    sr=sr+((y[i]-f3(aord3[0],aord3[1],aord3[2],aord3[3],x[i]))*(y[i]-f3(aord2[0],aord2[1],aord2[2],aord3[3],x[i])))
ybar=sumy/n
for i in range(n):
    st=st+((ybar-y[i])*(ybar-y[i]))
r=math.sqrt((st-sr)/st)
print('For order 3')
print('a0=',aord3[0])
print('a1=',aord3[1])
print('a2=',aord3[2])
print('a3=',aord3[3])
print('r=',r)
z1=x
for i in range(n):
    for j in range(i+1,n):
        if z1[j]<z1[i]:
            t=z1[i]
            z1[i]=z1[j]
            z1[j]=t
z2=[0 for i in range(n)]
for i in range(n):
    z2[i]=f1(a0,a1,z1[i])
plt.plot(z1,z2,label='order 1 curve',color='red',linewidth=3)
plt.legend()
plt.grid()

z3=[0 for i in range(n)]
for i in range(n):
    z3[i]=f2(aord2[0],aord2[1],aord2[2],z1[i])
plt.plot(z1,z3,label='order 2 curve',color='black',linewidth=3)
plt.legend()
plt.grid()

z4=[0 for i in range(n)]
for i in range(n):
    z4[i]=f3(aord3[0],aord3[1],aord3[2],aord3[3],z1[i])
plt.plot(z1,z4,label='order 3 curve',color='purple',linewidth=3)
plt.legend()
plt.grid(True)
plt.title('this is graph of different order curve')


"""
Exercise4
Find the length of a curve between two points
"""

from sympy import Derivative, Integral, Symbol, sqrt, SympifyError, sympify

def find_length(fx, var, a, b):
    deriv = Derivative(fx, var).doit()
    return Integral(sqrt(1 + deriv ** 2), (var, a, b)).doit().evalf()

def validate_function(function):

    try:
        return sympify(function)
    except SympifyError:
        print("Invalid function entered")
        sys.exit(1)

    return None

if __name__ == "__main__":
    function = input("Provide a function of one variable: ")
    function = validate_function(function)

    variable = input("Enter the variable to differentiate with respect to: ")
    variable = Symbol(variable)

    lower_bound = float(input("Enter the lower limit of the variable: "))
    upper_bound = float(input("Enter the upper limit of the variable: "))

    print(
        "Length of {} between {} and {} is {} ".format(
            function,
            lower_bound,
            upper_bound,
            find_length(function, variable, lower_bound, upper_bound),
        )
    )


import numpy as np
import control
import matplotlib.pyplot as plt

def predict_lti(b, a, y, u, k=1, d=0):
    """
    Returns the k-step ahead prediction for the LTI with pulse-transfer function
       H(z) = B(z)/A(z)
    The coefficients of the numerator polynomial is contained in the array b, and those of
    the denominator polynomial in the array a.

    Input
       b  ->  numerator coefficients (1 x m) row vector
       a  ->  denominator coefficients (1 x n) row vector
       u  ->  input data (N x 1) column vector
       y  ->  output data (N x 1) column vector
       k  ->  prediction horizon (integer > 0)
       d  ->  input delay (integer >= 0)

    Output
       ypredk  <-  predicted output ((N-k) x 1) column vector
       tk      <-  for convenience, discrete time vector for predicted values
                   tk = [1+k, 2+k, ..., N]

    """

    # Kjartan Halvorsen
    # 2019-10-03

    # Adjust coefficient arrays in case of input delay
    aa = np.concatenate( (a, np.zeros(d)), axis=None)
    bb = np.concatenate( (np.zeros(len(aa)-len(b)), b), axis=None)

    n = len(aa) - 1 # System order
    N = len(u)      # Length of data

    print("Order n=%d" %n)
    print("Data points N=%d" %N)

    yy = np.concatenate( (np.zeros(n), y), axis=None ) # Prepad with initial values
    uu = np.concatenate( (np.zeros(n), u), axis=None )  # Prepad with initial values

    tk = np.arange(k, N)

    ypredk = np.zeros(len(tk))
    for i in range(N-k):
        ysim = yy[i:i+n+k+1] # First n are initial values, last k will be overwritten
        usim = uu[i:i+n+k+1]
        for l in range(k):
            ysim[n+l+1] = - np.dot(aa[1:], ysim[n+l:l:-1])
            ysim[n+l+1] += np.dot(bb, usim[n+l+1:l:-1])
        ypredk[i] = ysim[-1]

    return (ypredk, tk)

def predict_unit_test():
    # Generate some data and test method
    b = np.array([0, 0, 1,0.9])
    a = np.array([1, -np.sqrt(3), 1, 0])

    N = 50
    u = np.random.randn(N)
    t = np.arange(N)
    (t, y, x) = control.forced_response( control.tf(b,a,1.0), t, u)
    y = np.ravel(y)

    k = 10;
    (ypred, tpred) = predict_lti(b[2:], a[:-1], y,  u, k, 1);

    plt.figure
    plt.plot(tpred, ypred, 'rx')
    plt.plot(t, y, 'bo')

    return (ypred, tpred)


'''
This Code transforms a matrix into Row Reduced Echelon Form using the Row Echelon Form of a matrix
Stage : Developmental
Many test cases are checked, yet to check some corner cases. If you notice a corner case that gives
error, please create a branch and update accordingly
or mail me at: de.arkadipta05@gmail.com, ai20mtech14002@iith.ac.in , arkadipta.de@gcettb.ac.in

Author: Arkadipta De
'''
import numpy as np

def Row_Echelon(A):
  '''
  Function to transform a matrix into Row-Echelon Form
  '''
  row = A.shape[0]
  column = A.shape[1]

  if row==0 or column==0:
      return A
  for i in range(len(A)) :
      if A[i,0] !=0 :
          break
  else:
      A_Below = Row_Echelon(A[:,1:])
      return np.hstack([A[:,:1],A_Below])

  if i>0:
      ith_row = A[i].copy()
      A[i] = A[0]
      A[0] = ith_row

  pivot = A[0,0]

  A[0] = A[0]/pivot                 #Normalize Pivot's Row
  A[1:] = A[1:] - A[0]*A[1:,0:1]    #Make Pivot Column 0

  A_Below = Row_Echelon(A[1:,1:])
  A_Stacked = np.hstack([A[1:,:1],A_Below])
  return np.vstack([A[:1],A_Stacked])

def Reduced_Row_Echelon_Form(A):
  '''
  Function to transform a matrix into Reduced-Row-Echelon Form
  '''
  A = Row_Echelon(A)
  row = A.shape[0]-1
  col = A.shape[1]
  while (row > 0):
      i = row
      for j in range(col):
        if A[i,j]==1:
          while (i > 0):
            A[i-1] -= A[row]*A[i-1,j]
            i-=1
      row-=1
  return A

def Inverse(A):
  '''
  Function to Extract Inverse Matrix from Augmented RRE of a Matrix
  Input: Augmented RRE Form of a Matrix
  '''
  row = A.shape[0]
  col = A.shape[1]
  if (row!=col):
    print("\nInverse Does not exist")
  elif np.linalg.det(A)==0:
    print("\nInverse doesn't exist.")
  else:
    AI = np.hstack([A,np.identity(row)])
    AI_RREF = Reduced_Row_Echelon_Form(AI)
    AI_row = AI_RREF.shape[0]
    AI_col = AI_RREF.shape[1]
    return AI_RREF[:,int(AI_col/2):]

# Driver Code
A1 = np.array([[1,0,0],[0,1,0],[0,0,1]])
A2 = np.array([[0,0,1],[1,0,0],[0,1,0]])
A3 = np.array([[0,0,1,5],[0,1,2,1],[0,1,2,1]])
A4 = np.array([[1,3,1],[0,2,3],[34,-2,-1]])
A5 = np.array([[1,2,2],[-2,9,17],[1,0,0],[0,1,0]])
A6 = np.array([[1,1,1,1,1.0],[1,1,1,1,1],[1,1,1,1,1]])
A7 = np.array([[1,1,1,1,1.0],[1,1,2,2,2],[1,1,1,4,5]])
A8 = np.array([[1,1,1,1],[1,1,1,1],[1,2,3,4]])
A9 = np.array([[1,1,1],[1,1,2],[2,3,1]])

A = A5

RE = Row_Echelon(A)
print('Row Echelon Form = \n',RE)
RRE = Reduced_Row_Echelon_Form(A)
print('\nReduced Row Echelon Form = \n',RRE)
print("Inverse = \n",Inverse(A))


# Write a program called plottask.py that displays a plot of the functions f(x)=x, g(x)=x^2 and h(x)=x^3 in the range [0, 4] on the one set of axes.
# Author : Conor McCaffrey

import numpy as np, matplotlib.pyplot as plt   # importing libraries and assiging aliases

x = np.arange(0.0,4.0,1.0)   # setting the range as described in task and in RealPython reference

y1 = x   # plotting the points on y-axis
y2 = x**2 #plotting the points on y-axis
y3 = x**3 #plotting the points on y-axis

font = {'family': 'serif', 'color':'darkred','weight': 'normal', 'size':20} # adding in fonts
font1 = {'family':'serif','color':'navy','size':10}   # adding in fonts

plt.plot(y1, color='#575daa', ls='-.', lw = 2,
         marker='s', ms = '5', mec='black', mfc='hotpink', label='f(x)=x')  # plotting each point and making the plot pretty
plt.plot(y2, color='#57de5a', ls='-.', lw = 2,
         marker='o', ms ='5', mec='black', mfc = 'hotpink', label='g(x)=x^2')  # plotting each point and making the plot pretty
plt.plot(y3, color='#ff0202', ls='-.', lw = 2,
         marker='o', ms ='5', mec='black', mfc = 'hotpink', label='h(x)=x^3')  # plotting each point and making the plot pretty

plt.xlabel('x-axis', fontdict=font1)  # adding labels and setting the font
plt.ylabel('y-axis', fontdict=font1)   # adding labels and setting the font
plt.title('Plot Task', fontdict=font, loc = 'left')  # adding a title and properties
plt.legend()   # adding a legend for clarity
plt.grid(axis = 'y', color = 'green' , lw = 0.1) # adding a y=axis grid and setting properties
#plt.show()   # displaying the graph
plt.savefig('plottask.png')   # adding code here that would be used to save the file if we did not want to show it