import numpy as np
import csv

def sigmoid(soma):
    return 1/ (1+np.exp(-soma))

def sigmoidDerivada(sig):
    return sig * (1 - sig)

entradas = np.array([[0,0],
                     [0,1],
                     [1,0],
                     [1,1]])
saidas = np.array([[0],[1],[1],[0]])

pesos0 = 2 * np.random.random((2,3)) - 1
pesos1 = 2 * np.random.random((3,1)) - 1

epocas = 200_000
taxaAprendizagem = 0.6
momento = 1

#Roda o aprendizado a quantidade de epocas configuradas
for e in range(epocas):
    #Recupera os valores da camada de entrada para realizar as operações
    camadaEntrada = entradas

    #Multiplicação e soma da camada entrada com os pesos da camada de entrada para a camada oculta
    #pegando o dotproduct ou Produto escalar dos dois vetores
    somaSinapse0 = np.dot(camadaEntrada, pesos0)

    #Executa uma função para retornar o valor sigmoid da soma das sinapses da camada de entrada
    camadaOculta = sigmoid(somaSinapse0)

    #Recupera o produto escalar dos vetores da camada oculta e seus pesos
    somaSinapse1 = np.dot(camadaOculta, pesos1)

    #Recupera o valor da saida
    camadaSaida = sigmoid(somaSinapse1)

    #Recupera os valores de erro da rede neural
    erroSaida = saidas - camadaSaida

    #Obtem a média de erro da rede
    mediaAbsoluta = np.mean(np.abs(erroSaida))
    if (e % 1000 == 0):
        print("Epoca: " + str(e) + " | Erro: " + str(mediaAbsoluta))

    #Encontrar Delta da camada oculta para calculo do erro
    derivadaSaida = sigmoidDerivada(camadaSaida)
    deltaSaida = erroSaida * derivadaSaida

    pesos1Transposta = pesos1.T
    deltaSaidaXPeso = deltaSaida.dot(pesos1Transposta)
    deltaCamadaOculta = deltaSaidaXPeso * sigmoidDerivada(camadaOculta)

    #Utilizando o método Backpropagation
    #é feito o reajuste dos pesos da camada oculta para a camada de saida
    camadaOcultaTransposta = camadaOculta.T
    pesosNovo1 = camadaOcultaTransposta.dot(deltaSaida)
    pesos1 = (pesos1 * momento) + (pesosNovo1 * taxaAprendizagem)

    #Realiza o reajuste dos pesos da camada de entrada para a camada oculta
    camadaEntradaTransposta = camadaEntrada.T
    pesosNovos0 = camadaEntradaTransposta.dot(deltaCamadaOculta)
    pesos0 = (pesos0 * momento) + (pesosNovos0 * taxaAprendizagem)

print("Taxa de acerto é de aproximadamente " + str(int((1 - mediaAbsoluta)*100)) + "%")


import numpy as np

# removes redundant wrapper arrays generated with found contours
def unwrap_contour(cnt):
    return np.array([point[0] for point in cnt])

# Calculates the distance between two points
def distance(s1, s2):
    return np.linalg.norm(s1 - s2)

# Calculates the angle between 2 vectors
def angle(v1, v2):
    cos_theta = np.dot(v1, v2) / (np.linalg.norm(v1) * np.linalg.norm(v2))
    return np.degrees(np.arccos(cos_theta))

# Calculates the angle between 2 edges of a polygon
def edge_angle(dest1, origin, dest2):
    return angle(dest1 - origin, dest2 - origin)

# Calculates all angles of a polygon, given its vertic;e locations
def angles_of_shape(points):
    angles = []
    for offset in range(len(points)):
        adj1, origin, adj2 = np.roll(points, offset, axis=0)[:3]
        angles += [edge_angle(adj1, origin, adj2)]
    return np.array(angles)

# Calculates all edge lengths of a polygon, given its vertic;e locations
def edges_of_shape(points):
    lengths = []
    for offset in range(len(points)):
        p1, p2 = np.roll(points, offset, axis=0)[:2]
        lengths += [np.linalg.norm(p1 - p2)]
    return np.array(lengths)

# Create a contour that represents a perfect circle (used for circle detection)
def perfect_circle_contour(x, y, r, num_points):
    perfect_circle = []
    for theta in range(0, num_points):
        perfect_circle += [[[ r * np.sin(theta) + y, r * np.cos(theta) + x ]]]

    perfect_circle = np.array(perfect_circle, np.int32)
    return perfect_circle


"""
求矩阵的特征值和谱半径

"""
import numpy as np
from sympy.abc import lamda

def eig_and_specradi(A):
    """
    求矩阵M的特征值和谱半径
    :param: A: 待处理的矩阵
    :return: (特征值, 谱半径)
    """
    # 计算出 Lambda * I - A
    A_ = A.tolist()  # array -> list
    I = np.identity(len(A_))  # 单位阵
    print("lambda * I - A：")
    print(lamda * I - A_)
    # 计算出特征值和谱半径
    eigs, _ = np.linalg.eig(A)
    radius = np.max(eigs)
    print("特征值：", eigs)
    print("谱半径：", radius)
    return eigs, radius

if __name__ == '__main__':
    A = np.array([
        [0.1, 1],
        [0.8, -0.1]
    ])
    eigs, r = eig_and_specradi(A)


#!/usr/bin/python

import numpy as np
from matplotlib import pyplot as plt

def linear_lagrange0(xi):
  return 1-xi

def linear_lagrange1(xi):
  return xi

def quadratic_lagrange0(xi):
  return 2*xi*xi - 3*xi + 1

def quadratic_lagrange1(xi):
  return -4*xi*xi + 4*xi

def quadratic_lagrange2(xi):
  return 2*xi*xi - xi

def hermite0(xi):
  return 1 - 3*xi*xi + 2*xi*xi*xi

def hermite1(xi):
  return xi * (xi-1) * (xi-1)

def hermite2(xi):
  return xi*xi * (3 - 2*xi)

def hermite3(xi):
  return xi*xi * (xi-1)

xlist = np.linspace(0,1,100)

# as plot
plt.rcParams.update({'font.size': 20, 'lines.linewidth': 8, 'lines.markersize':8})
plt.figure()
plt.plot(xlist, [linear_lagrange0(x) for x in xlist], label=r"$\phi_1$")
plt.plot(xlist, [linear_lagrange1(x) for x in xlist], label=r"$\phi_2$")
plt.legend(loc="best")
plt.xlabel(r"$\xi$")
plt.savefig("lagrange.pdf")

# as geometry
# linear lagrange
plt.figure(figsize=(5,5))
plt.plot(xlist, [linear_lagrange0(x) for x in xlist], label=r"$\phi_1$")
plt.plot(xlist, [linear_lagrange1(x) for x in xlist], label=r"$\phi_2$")
plt.plot(xlist, [0 for x in xlist], color='k', lw=4)
#plt.legend(loc="best")
plt.xlabel(r"$\xi$")
plt.axis('off')
plt.savefig("lagrange1.pdf")

# quadratic lagrange
plt.figure(figsize=(5,5))
plt.plot(xlist, [quadratic_lagrange0(x) for x in xlist], label=r"$\phi_1$")
plt.plot(xlist, [quadratic_lagrange1(x) for x in xlist], label=r"$\phi_2$")
plt.plot(xlist, [quadratic_lagrange2(x) for x in xlist], label=r"$\phi_3$")
plt.plot(xlist, [0 for x in xlist], color='k', lw=4)
#plt.legend(loc="best")
plt.xlabel(r"$\xi$")
plt.axis('off')
plt.savefig("lagrange2.pdf")

# hermite
plt.figure(figsize=(5,5))
plt.plot(xlist, [hermite0(x) for x in xlist], label=r"$\phi_1$")
plt.plot(xlist, [hermite1(x) for x in xlist], label=r"$\phi_1$")
plt.plot(xlist, [hermite2(x) for x in xlist], label=r"$\phi_1$")
plt.plot(xlist, [hermite3(x) for x in xlist], label=r"$\phi_1$")
plt.plot(xlist, [0 for x in xlist], color='k', lw=4)
#plt.legend(loc="best")
plt.xlabel(r"$\xi$")
plt.axis('off')
plt.savefig("hermite.pdf")

plt.show()


import scipy.io as sio
import matplotlib.pyplot as plt
import os
import numpy as np
from scipy import optimize as opt

def h0x(X, theta):
    return np.dot(X, theta)

def cost_l2(theta, X, y, lamb=0):
    predictions = h0x(X, theta)
    squared_errors = np.sum(np.square(predictions - y))
    regularization = np.sum(lamb * np.square(theta[1:]))
    return (squared_errors + regularization) / (2 * len(y))

def gradient_l2(theta, X, y, lamb):
    predictions = h0x(X, theta)
    gradient = np.dot(X.transpose(), (predictions - y))
    regularization = lamb * theta
    regularization[0] = 0  # because formula for 0 member is different
    return (gradient + regularization) / len(y)

def learning_curves_chart(X_train, y_train, X_val, y_val, lambda_):
    m = len(y_train)
    train_err = np.zeros(m)
    val_err = np.zeros(m)
    for i in range(1, m):
        theta = opt.fmin_cg(cost_l2, np.zeros(X_train.shape[1]), gradient_l2,
                            (X_train[0:i + 1, :], y_train[0:i + 1], lambda_), disp=False)
        train_err[i] = cost_l2(theta, X_train[0:i + 1, :], y_train[0:i + 1])
        val_err[i] = cost_l2(theta, X_val, y_val)
    plt.plot(range(2, m + 1), train_err[1:], c="r", linewidth=2)
    plt.plot(range(2, m + 1), val_err[1:], c="b", linewidth=2)
    plt.xlabel("number of training examples", fontsize=14)
    plt.ylabel("error", fontsize=14)
    plt.legend(["training", "validation"], loc="best")
    plt.axis([2, m, 0, 100])
    plt.grid()
    plt.show()

def polynom(x, degree):
    X_poly = np.zeros(shape=(len(x), degree))
    for i in range(0, degree):
        X_poly[:, i] = x.squeeze() ** (i + 1);
    return X_poly

if __name__ == '__main__':
    # 1
    file_path = 'ex3data1.mat'
    dataset = sio.loadmat(file_path)
    x_train = dataset["X"]
    x_val = dataset["Xval"]
    x_test = dataset["Xtest"]

    # squeeze the target variables into one-dimensional arrays
    y_train = dataset["y"].squeeze()
    y_val = dataset["yval"].squeeze()
    y_test = dataset["ytest"].squeeze()

    # 2
    fig, ax = plt.subplots()
    ax.scatter(x_train, y_train)
    plt.xlabel("change in water level", fontsize=14)
    plt.ylabel("water flowing out of the dam", fontsize=14)
    plt.title("Training sample", fontsize=16)
    plt.show()

    # data preparation
    X_train = np.hstack((np.ones((len(x_train), 1)), x_train))
    theta = np.zeros(X_train.shape[1])

    # 3
    print(cost_l2(theta, X_train, y_train, 0))

    # 4
    print(gradient_l2(theta, X_train, y_train, 0))

    # 5
    theta = opt.fmin_cg(cost_l2, theta, gradient_l2, (X_train, y_train, 1000), disp=False)
    print(theta)

    h = np.dot(X_train, theta)
    fig, ax = plt.subplots()
    ax.scatter(x_train, y_train)
    ax.plot(X_train[:, 1], h, linewidth=2, color='red')
    plt.show()

    # 6
    X_val = np.hstack((np.ones((len(x_val), 1)), x_val))
    plt.title("Learning Curves for Linear Regression", fontsize=16)
    learning_curves_chart(X_train, y_train, X_val, y_val, 0)

    # 7
    x_train_poly = polynom(x_train, 8)
    x_val_poly = polynom(x_val, 8)
    x_test_poly = polynom(x_test, 8)

    # 8
    train_means = x_train_poly.mean(axis=0)
    train_std = np.std(x_train_poly, axis=0, ddof=1)

    x_train_poly = (x_train_poly - train_means) / train_std
    x_val_poly = (x_val_poly - train_means) / train_std
    x_test_poly = (x_test_poly - train_means) / train_std

    X_train_poly = np.hstack((np.ones((len(x_train_poly), 1)), x_train_poly))
    X_val_poly = np.hstack((np.ones((len(x_val_poly), 1)), x_val_poly))
    X_test_poly = np.hstack((np.ones((len(x_test_poly), 1)), x_test_poly))

    # 9
    theta = opt.fmin_cg(cost_l2, np.zeros(X_train_poly.shape[1]), gradient_l2, (X_train_poly, y_train, 0), disp=False)
    x = np.linspace(min(x_train) - 5, max(x_train) + 5, 1000)
    x_polynom = polynom(x, 8)
    print(x_polynom)
    x_polynom = (x_polynom - train_means) / train_std
    x_polynom = np.hstack((np.ones((len(x_polynom), 1)), x_polynom))

    # 10
    fig, ax = plt.subplots()
    plt.scatter(x_train, y_train, color='red')
    plt.plot(x, h0x(x_polynom, theta), linewidth=2)
    plt.xlabel("change in water level", fontsize=14)
    plt.ylabel("water flowing out ", fontsize=14)
    plt.title("Polynomial Fit", fontsize=16)
    plt.show()

    learning_curves_chart(X_train_poly, y_train, X_val_poly, y_val, 0)

    # 11
    # lambda = 1
    theta = opt.fmin_cg(cost_l2, np.zeros(X_train_poly.shape[1]), gradient_l2, (X_train_poly, y_train, 1), disp=False)
    x = np.linspace(min(x_train) - 5, max(x_train) + 5, 1000)
    x_polynom = polynom(x, 8)
    x_polynom = (x_polynom - train_means) / train_std
    x_polynom = np.hstack((np.ones((len(x_polynom), 1)), x_polynom))
    fig, ax = plt.subplots()
    plt.scatter(x_train, y_train, color='red')
    plt.plot(x, h0x(x_polynom, theta), linewidth=2)
    plt.xlabel("change in water level", fontsize=14)
    plt.ylabel("water flowing out ", fontsize=14)
    plt.title("Polynomial Fit (lambda=1)", fontsize=16)
    plt.show()

    learning_curves_chart(X_train_poly, y_train, X_val_poly, y_val, 1)

    # lambda = 100
    theta = opt.fmin_cg(cost_l2, np.zeros(X_train_poly.shape[1]), gradient_l2, (X_train_poly, y_train, 100), disp=False)
    x = np.linspace(min(x_train) - 5, max(x_train) + 5, 1000)
    x_polynom = polynom(x, 8)
    x_polynom = (x_polynom - train_means) / train_std
    x_polynom = np.hstack((np.ones((len(x_polynom), 1)), x_polynom))
    fig, ax = plt.subplots()
    plt.scatter(x_train, y_train, color='red')
    plt.plot(x, h0x(x_polynom, theta), linewidth=2)
    plt.xlabel("change in water level", fontsize=14)
    plt.ylabel("water flowing out ", fontsize=14)
    plt.title("Polynomial Fit (lambda=100)", fontsize=16)
    plt.show()

    learning_curves_chart(X_train_poly, y_train, X_val_poly, y_val, 100)

    # 12
    lambda_values = [0, 0.001, 0.003, 0.01, 0.03, 0.1, 0.3, 1, 3, 10]
    val_err = []
    for lamb in lambda_values:
        theta = opt.fmin_cg(cost_l2, np.zeros(X_train_poly.shape[1]), gradient_l2, (X_train_poly, y_train, lamb),
                            disp=False)
        val_err.append(cost_l2(theta, X_val_poly, y_val))
    plt.plot(lambda_values, val_err, c="b", linewidth=2)
    plt.axis([0, len(lambda_values), 0, val_err[-1] + 1])
    plt.grid()
    plt.xlabel("lambda", fontsize=14)
    plt.ylabel("error", fontsize=14)
    plt.title("Validation Curve", fontsize=16)
    plt.show()

    # 13
    theta = opt.fmin_cg(cost_l2, np.zeros(X_train_poly.shape[1]), gradient_l2, (X_train_poly, y_train, 3), disp=False)
    test_error = cost_l2(theta, X_test_poly, y_test)
    print("Test Error: ", test_error, "| Regularized Polynomial (lambda=3))")


import numpy as np
import scipy as sp
import scipy.stats

def gaussian_loglik(obs, mu, sigma):
    return sp.stats.multivariate_normal.logpdf(obs, mean=mu, cov=sigma) / mu.shape[0]

def gaussian_entropy(sigma):
    return 0.5 * (len(sigma) + np.log(np.linalg.det(sigma)) + np.log(2 * np.pi))

def r2_score(obs, pred):
    return 1 - np.sum((obs - pred) ** 2) / np.sum((obs - np.mean(obs)) ** 2)

def cal_error(obs, mu, sigma, bins=5):
    """
    Unweighted regression calibration error for GP predictions.

    We calculate the mean-squared error between predicted versus observed
    empirical CDFs, for the specified number of equally spaced bins on the
    interval [0,1].

    [Equation (9), Kuleshov et. al. 2018]

    Parameters
    ----------
    obs: m-length array of observations
    mu: m-length array of predicted means
    sigma: m x m array of predicted covariance
    bins: number of bins at which to evaluate

    Returns
    -------
    cal_error: float
    predicted: predicted CDFs corresponding to each bin
    empirical: observed CDFs corresponding to each bin
    """
    sigmas = np.diag(sigma)
    quantiles = sp.stats.norm.cdf(obs, mu, np.sqrt(sigmas))
    predicted = np.arange(1/bins, 1+1/bins, 1/bins)
    empirical = np.array([np.mean(quantiles < p) for p in predicted])
    return np.sum((predicted - empirical) ** 2) / bins, predicted, empirical


#  Created byMartin.cz

import numpy as np

def mean_squared_error(Y, Y_hat):
    """
    Calculates cost and its gradient using Mean Squared Error.

    (1/2m)*sum(Y - Y_hat)^.2

    Args:
        Y:
            Data labels.

        Y_hat:
            Predictions/activations from the output layer.

    Returns:
        cost:
            Mean Squared Error Cost result.

        dY_hat:
            Gradient.
    """

    m = Y.shape[0]

    # calc cost
    cost = (1 / (2 * m)) * np.sum(np.square(Y - Y_hat))
    cost = np.squeeze(cost)

    # calc gradient
    dY_hat = - (1 / m) * (Y - Y_hat)

    return cost, dY_hat

def binary_cross_entropy(Y, Y_hat, epsilon=1e-8):
    """
    Calculates cost and its gradient using Binary Cross Entropy.

    (1/m) * np.sum(-np.log(Y_hat) * Y - np.log(1-Y_hat) * (1-Y))

    Args:
        Y:
            Data labels.

        Y_hat:
            Predictions/activations from the output layer.

        epsilon: float
            Zero corrector.

    Returns:
        cost:
            Binary Cross-Entropy Cost result.

        dY_hat:
            Gradient.
    """

    m = Y.shape[0]

    # make data safe
    Y_hat = np.clip(Y_hat, a_min=epsilon, a_max=(1 - epsilon))

    # calc cost
    cost = (1 / m) * np.nansum(-np.log(Y_hat) * Y - np.log(1 - Y_hat) * (1 - Y))
    cost = np.squeeze(cost)

    # calc gradient
    dY_hat = -(Y / Y_hat) + (1 - Y) / (1 - Y_hat)

    return cost, dY_hat

def cross_entropy(Y, Y_hat, epsilon=1e-8):
    """
    Calculates cost and its gradient using Cross Entropy.

    (-1 / m) * np.sum(Y * np.log(Y_hat))

    Args:
        Y:
            Data labels.

        Y_hat:
            Predictions/activations from the output layer.

        epsilon: float
            Zero corrector.

    Returns:
        cost:
            Cross-Entropy Cost result.

        dY_hat:
            Gradient.
    """

    m = Y.shape[0]

    # make data safe
    Y_hat = np.clip(Y_hat, a_min=epsilon, a_max=(1 - epsilon))

    # calc cost
    cost = (-1 / m) * np.nansum(Y * np.log(Y_hat))
    cost = np.squeeze(cost)

    # calc gradient
    dY_hat = -Y / Y_hat

    return cost, dY_hat


import math
import numpy

from scipy.stats import pearsonr
from scipy.stats import spearmanr
from sklearn.metrics import accuracy_score, f1_score, precision_score
from sklearn.metrics import recall_score

def calc_pearson(y, y_hat):
	"""
		Compute Pearson correlation coefficient

	Args:
		y: (N,) list of ground_truth values
		y_hat: (N,) list of predicted values

	Return:
		a float, pearson correlation coefficient
	"""
	score, p_value = pearsonr(y, y_hat)
	if math.isnan(score):
		return 0
	else:
		return score

def calc_spearman(y, y_hat):
	"""
		Compute Spearman correlation coefficient

	Args:
		y: (N,) list of ground_truth values
		y_hat: (N,) list of predicted values

	Return:
		a float, spearman correlation coefficient
	"""
	score, p_value = spearmanr(y, y_hat)
	if math.isnan(score):
		return 0
	else:
		return score

def get_metrics(task):
	"""
		Wrapper to have at hand what metrics and how to monitor
		them at training time.

	Args:
		task: str, task goal [regression|binary|multilabel]

	Return:
		a tuple:
			- a metrics to eval on dict
			- a monitoring metric s name
			- a str, denoting whether to seek max or min value of
			the monitoring metric
	"""
	metrics_ = {
		"regression": {
			"pearson": calc_pearson,
			"spearman": calc_spearman
		},
		"binary": {
			"acc": lambda y, y_hat: accuracy_score(y, y_hat),
			"precision": lambda y, y_hat: precision_score(y, y_hat,
				average="weighted"),
			"recall": lambda y, y_hat: recall_score(y, y_hat,
				average="weighted"),
			"f1": lambda y, y_hat: f1_score(y, y_hat, average="weighted")
		},
		"multilabel": {
			"acc": lambda y, y_hat: accuracy_score(y, y_hat),
			"precision": lambda y, y_hat: precision_score(y, y_hat,
				average="weighted"),
			"recall": lambda y, y_hat: recall_score(y, y_hat,
				average="weighted"),
			"f1": lambda y, y_hat: f1_score(y, y_hat, average="weighted")
		}
	}

	monitor_ = {
		"regression": "spearman",
		"binary": "f1",
		"multilabel": "f1"
	}

	mode_ = {
		"regression": "max",
		"binary": "max",
		"multilabel": "max"
	}

	return metrics_[task], monitor_[task], mode_[task]


import matplotlib.pyplot as plt
import numpy as np
import math

x = np.arange(-5, 5, 0.001);
# y1 = np.sin(x)
# y2 = np.sin(x + math.pi / 3 * 2)
# y3 = np.sin(x - math.pi / 3 * 2)
#
# plt.plot(x, y1)
# plt.plot(x, y2)
# plt.plot(x, y3)
# y4 = y1 * 0.3 + y2 * 0.4 + y3 * 0.3
# plt.plot(x, y4)

arr = [ np.sin(n * x) for n in range(1,1000,2)]
t = np.zeros(len(x))
for tmp in arr:
    t += tmp

print(len(t))
print(len(x))
plt.plot(x, t)
print(t)
plt.show()


import numpy as np
import matplotlib.pyplot as plt
import integrators as intg

# Parameters mass-spring system
m = 1
b = 0.25
k = 1

#  xdot = f(t, x, u),
#   t: time
#   x: state vector
#   u: input
def f(t, x, u):
    # mass sprint system state space form:
    return np.array([x[1], -k*x[0]/m - b*x[1]/m])

# Initial conditions
t = 0
x0 = np.array([0, 1])
u = 0

# Numerical solution
tf = 30; dt = 0.001; n = int(np.floor(tf/dt));

#integrator = intg.Heun(dt, f)
integrator = intg.RungeKutta4(dt, f)

x = x0
t_history = [0]
x_history = [x]

for i in range(n):

    x = integrator.step(t, x, u)
    t = (i+1) * dt

    t_history.append(t)
    x_history.append(x)

# Convert numerical soln to numpy arrays
t_ = np.asarray(t_history)
x_ = np.asarray(x_history)

# Analytical solution
wn = np.sqrt(k/m)
z = b/(2*m*wn)
sigma = z*wn
wd = wn*np.sqrt(1 - z**2)
A = x0[0]
B = (x0[1] + sigma*x0[0])/wd
x_exact = np.exp(-sigma*t_)*(A*np.cos(wd*t_) + B*np.sin(wd*t_))

# Error
error = x_exact - x_[:,0]

# close all figures
plt.close('all')

plt.figure()
plt.plot(t_history, x_[:,0],'rx', label='RK4')
plt.plot(t_history, x_exact,'g--', label='exact')
plt.xlabel('t')
plt.ylabel('x')
plt.legend()
plt.show()

plt.figure()
plt.plot(t_history,error,'g--')
plt.xlabel('t')
plt.ylabel('error')
plt.show()


import numpy as np
import numpy.linalg as la

from orbit.utils.general import rotation_matrix

def phase_adv_matrix(mu1, mu2):
    """4x4 matrix to rotate x-x' by mu1 and y-y' by mu2, both clockwise."""
    R = np.zeros((4, 4))
    R[:2, :2] = rotation_matrix(mu1)
    R[2:, 2:] = rotation_matrix(mu2)
    return R

def V_matrix_2x2(alpha, beta):
    """2x2 normalization matrix for x-x' or y-y'."""
    return np.array([[beta, 0], [-alpha, 1]]) / np.sqrt(beta)

def V_matrix_4x4_uncoupled(alpha_x, alpha_y, beta_x, beta_y):
    """4x4 normalization matrix for x-x' and y-y'."""
    V = np.zeros((4, 4))
    V[:2, :2] = V_matrix_2x2(alpha_x, beta_x)
    V[2:, 2:] = V_matrix_2x2(alpha_y, beta_y)
    return V

def params_from_transfer_matrix(M):
    """Return dictionary of lattice parameters from a transfer matrix.

    Method is taken from `py/orbit/matrix_lattice/MATRIX_Lattice.py`.

    Parameters
    ----------
    M : ndarray, shape (4, 4)
        A transfer matrix.

    Returns
    -------
    lattice_params : dict
        Dictionary with the following keys: 'frac_tune_x', 'frac_tune_y',
        'alpha_x', 'alpha_y', 'beta_x', 'beta_y', 'gamma_x', 'gamma_y'.
    """
    keys = ['frac_tune_x', 'frac_tune_y',
            'alpha_x', 'alpha_y',
            'beta_x', 'beta_y',
            'gamma_x', 'gamma_y']
    lattice_params = {key: None for key in keys}

    cos_phi_x = (M[0, 0] + M[1, 1]) / 2
    cos_phi_y = (M[2, 2] + M[3, 3]) / 2
    if abs(cos_phi_x) >= 1 or abs(cos_phi_y) >= 1 :
        return lattice_params
    sign_x = sign_y = +1
    if abs(M[0, 1]) != 0:
        sign_x = M[0, 1] / abs(M[0, 1])
    if abs(M[2, 3]) != 0:
        sign_y = M[2, 3] / abs(M[2, 3])
    sin_phi_x = sign_x * np.sqrt(1 - cos_phi_x**2)
    sin_phi_y = sign_y * np.sqrt(1 - cos_phi_y**2)

    nux = sign_x * np.arccos(cos_phi_x) / (2 * np.pi)
    nuy = sign_y * np.arccos(cos_phi_y) / (2 * np.pi)
    beta_x = M[0, 1] / sin_phi_x
    beta_y = M[2, 3] / sin_phi_y
    alpha_x = (M[0, 0] - M[1, 1]) / (2 * sin_phi_x)
    alpha_y = (M[2, 2] - M[3, 3]) / (2 * sin_phi_y)
    gamma_x = -M[1, 0] / sin_phi_x
    gamma_y = -M[3, 2] / sin_phi_y

    lattice_params['frac_tune_x'] = nux
    lattice_params['frac_tune_y'] = nuy
    lattice_params['beta_x'] = beta_x
    lattice_params['beta_y'] = beta_y
    lattice_params['alpha_x'] = alpha_x
    lattice_params['alpha_y'] = alpha_y
    lattice_params['gamma_x'] = gamma_x
    lattice_params['gamma_y'] = gamma_y
    return lattice_params

def is_stable(M, tol=1e-5):
    """Return True if transfer matrix is stable.

    M : ndarray, shape (n, n)
        A transfer matrix.
    tol : float
        The matrix is stable if all eigenvalue norms are in the range
        (1 - tol, 1 + tol).
    """
    for eigval in la.eigvals(M):
        if abs(la.norm(eigval) - 1.0) > tol:
            return False
    return True

def get_eigtunes(M):
    """Compute transfer matrix eigentunes -- cos(Re[eigenvalue]).

    Parameters
    ----------
    M : ndarray, shape (4, 4)
        A transfer matrix.

    Returns
    -------
    ndarray, shape (2,)
        Eigentunes for the two modes.
    """
    return np.arccos(la.eigvals(M).real)[[0, 2]] / (2 * np.pi)

def unequal_eigtunes(M, tol=1e-5):
    """Return True if the eigentunes of the transfer matrix are the same.

    M : ndarray, shape (4, 4)
        A transfer matrix.
    tol : float
        Tunes are equal if abs(mu1 - mu2) > tol.
    """
    mu1, mu2 = get_eigtunes(M)
    return abs(mu1 - mu2) > tol


from __future__ import division
import numpy as np
from LinearSolve import LinearSolver
from utils import Grapher

class Interpolator():
    methods = ['lagrange', 'spline']

    def __init__(self, x, y, xstar):
        self.x = x
        self.y = y
        self.xstar = np.sort(xstar)
        self.ystar = None
        self.N = x.size
        self.s = np.array([0] * self.N)
        self.f = None

    ''' Returns i'th Lagrange polynomial evaluated at pt '''
    @staticmethod
    def L(i, x, pt):
        trans = map(lambda arg: (pt - arg) / (x[i] - arg), x[np.arange(len(x)) != i])
        return np.prod(trans)

    def lagrange(self):
        L = Interpolator.L

        def func(pt):
            return np.sum([L(i, self.x, pt) * self.y[i] for i in range(self.N)])
        self.evaluate(func)
        self.f = func

    def spline(self):
        n, x, y, s = self.N, self.x, self.y, self.s
        a, b = np.ndarray(shape=(n - 2, n)), np.ndarray(shape=(n - 2))

        ''' Divided Diference '''
        def dd(i, j):
            return (y[i] - y[j]) / (x[i] - x[j])

        for i in range(1, n - 1):
            a[i - 1, i - 1:i + 2] = [x[i] - x[i - 1], 2 *
                                     (x[i + 1] - x[i - 1]), x[i + 1] - x[i]]
            b[i - 1] = 6 * (dd(i + 1, i) - dd(i, i - 1))
        a = a[..., 1:n - 1]

        solver = LinearSolver(a, b)
        solver.GE_pivoted()
        s[1:-1] = solver.x

        def S(i, pt):
            return ((x[i + 1] - pt)**3 * s[i] + (pt - x[i])**3 * s[i + 1]) / 6 / (x[i + 1] - x[i]) \
                + (y[i] / (x[i + 1] - x[i]) - (x[i + 1] - x[i]) * s[i] / 6) * (x[i + 1] - pt) \
                + (y[i + 1] / (x[i + 1] - x[i]) - (x[i + 1] - x[i]) * s[i + 1] / 6) * (pt - x[i])

        def func(pt):
            for i, x in enumerate(self.x):
                if pt < x:
                    return S(i - 1, pt)
        self.evaluate(func)
        self.f = func

    def evaluate(self, f):
        self.ystar = np.array([f(i) for i in self.xstar])

def main():
    method_names = ['Lagrange polynomials', 'Natural Cubic Spline']

    for i, name in enumerate(method_names):
        print 'press ', i, 'for ', name
    choice = int(raw_input())

    with open("q1.txt", "r") as f:
        n = int(f.readline())
        a = np.array([map(float, f.readline().split()) for i in range(n)])
        x, y = a[..., 0], a[..., 1]
        m = int(f.readline())
        xstar = np.array([float(f.readline()) for i in range(m)])

    func = Interpolator(x, y, xstar)
    getattr(func, Interpolator.methods[choice])()
    Grapher.plot(func.f, func.x[0], func.x[-1], x=func.x, y=func.y)

    with open("out.txt", "w") as fout:
        if func.ystar is not None:
            formated_y = np.array([func.xstar, func.ystar]).transpose()
            np.savetxt(fout, formated_y, fmt='%f',
                       header=method_names[choice], footer='\n', comments='')

if __name__ == '__main__':
    main()


# Create by Etzion Harari
# https://github.com/EtzionR

# Load libraries:
import numpy as np
import matplotlib.pyplot as plt

# Define useful functions
create_matrix = lambda x, p: np.power(x, np.arange(p).reshape(p, 1)).T
create_polynm = lambda x, b: np.dot(create_matrix(x, len(b)), b)
b_coefficient = lambda x, y: np.dot(np.linalg.inv(np.dot(x.T, x)), np.dot(x.T, y))
rmse = lambda p, y: np.sqrt(np.mean(np.power(p - y, 2)))

# Define Polynom Object
class PolyR:
    """
    Polynomial Regression Object
    Optimize Model by fitting polynom degree
    """
    def __init__(self, max_p, cv=5):
        """
        initilize PolyR object
        :param max_p: max degree to check for fitting
        :param cv: number to K fold cross validation
        """
        self.p = None
        self.betas = None
        self.r_mse = None
        self.folds = None
        self.scores = {}
        self.max_p = max_p
        self.ary_p = np.arange(1, self.max_p + 1)
        self.cv = cv

    def fit(self, x, y):
        """
        fit betas vector for the input data
        :param x: x vector (independent variables)
        :param y: y vector (dependent variable)
        :return: The object
        """
        mat_x = create_matrix(x, self.max_p)
        self.folds = np.arange(mat_x.shape[0]) % self.cv

        for j in self.ary_p:
            self.scores[j] = self.cross_validation(mat_x[:, :j], y)

        self.p = min(self.scores, key=lambda k: self.scores[k])
        self.r_mse = self.scores[self.p]
        self.betas = b_coefficient(create_matrix(x, self.p), y)

        return self

    def predict(self, x):
        """
        predict y values using x values
        :param x: x vector (independent variables)
        :return:  y vector (dependent variable)
        """
        return create_polynm(x, self.betas)

    def cross_validation(self, x, y):
        """
        cross validation function
        using to prevent overfitting of the regression
        :param x: x matrix (independent variables)
        :param y: y vector (dependent variable)
        :return: RMSE for the model
        """
        cross_valid = []
        for k in np.arange(self.cv):
            trn = self.folds != k
            tst = self.folds == k

            x_trn = x[trn, :]
            y_trn = y[trn]
            x_tst = x[tst, :]
            y_tst = y[tst]

            betas = b_coefficient(x_trn, y_trn)
            y_prd = np.dot(x_tst, betas)

            cross_valid.append(rmse(y_prd, y_tst))
        return np.mean(cross_valid)

    def plot_rmse(self, size=10):
        """
        plot rmse result for each polynom degree
        :param size: plot size (default: 10)
        """
        vlu = [*self.scores.values()]
        axe = [*self.scores.keys()]
        plt.figure(figsize=(size, size * .6))
        plt.title(f'RMSE for each Polynomial degree:\n[Optimal degree = {self.p}, range from 1 to {self.max_p}]', fontsize=14)
        plt.plot(axe, vlu, color='r', label='RMSE')
        plt.plot([self.p, self.p], [min(vlu), max(vlu)], color='k', linestyle='dotted', label='Minimum RMSE')
        plt.xlabel('Polynomial degree', fontsize=12)
        plt.ylabel('RMSE', fontsize=12)
        plt.legend(fontsize=12)
        plt.show()

# License
# MIT © Etzion Harari


import numpy as np
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
import matplotlib.dates as mdate

plt.savefig("save.png", dpi=300, bbox_inches="tight")

pd.set_option('display.max_columns', 100)
pd.set_option('display.width', 500)

data = pd.read_csv("../data/covid_19_data.csv")

data_corre = data.drop("SNo", axis = 1)

corre1 = data_corre.corr(method ='pearson')
corre2 = data_corre.corr(method ='kendall')
corre3 = data_corre.corr(method ='spearman')

print("Standard Correlation Coefficients")
print(corre1)
print("-------------------------------------")
print("Kendall Tau Correlation Coefficients")
print(corre2)
print("-------------------------------------")
print("Spearman Rank Correlation Coefficients")
print(corre3)

f,ax = plt.subplots(figsize=(15, 13))
sns.heatmap(corre1, annot= True, fmt= '.2f',cmap="summer")
plt.tight_layout()
plt.show()

for a in data.columns:

    list1 = list(data[a])
    count = 0

    for i in list1:
        if (i == "Recovered"):
            count = count + 1
        else:
            count = count

    print(" The number of ""'Recovered'"" value inside {} column is {}".format(a, count))


# 2021.04.10
# Problem Statement:
# https://leetcode.com/problems/maximum-gap/
# Referred to the solution here:
# https://leetcode.com/problems/maximum-gap/discuss/50643/bucket-sort-JAVA-solution-with-explanation-O(N)-time-and-space
import numpy as np

class Solution:
    def maximumGap(self, nums: List[int]) -> int:
        # check corner cases
        if len(nums)<2: return 0
        if len(nums)==2: return abs(nums[0]-nums[1])

        # idea: bucket sort
        # the max gap must >= ceiling[(max - min ) / (len(nums) - 1)],
        # therefore, divide the list into (len(nums)-1) buckets,
        # the max gap will only show between buckets (the max of a smaller bucket and the min of a larger bucket)

        # set the division gap, to divide the list into buckets
        division_gap = (max(nums)-min(nums)) // (len(nums)-1)
        if division_gap != (max(nums)-min(nums)) / (len(nums)-1):
            division_gap = division_gap + 1

        # bucket[i][0] would store the min of bucket number i
        # bucket[i][1] would store the max of bucket number i
        bucket = np.zeros((len(nums)-1, 2))
        for i in range(0, len(nums)-1):
            bucket[i][0] = np.inf # initialization
            bucket[i][1] = -np.inf # initialization

        min_val, max_val = min(nums), max(nums)
        # check corner cases where all numbers are the same
        if min_val == max_val: return 0
        # do the division
        for i in range(0, len(nums)):
            if nums[i] == min_val:
                bucket[0][0] = nums[i]
                bucket[0][1] = max(bucket[0][1], nums[i])
            elif nums[i] == max_val:
                bucket[-1][1] = nums[i]
                bucket[-1][0] = min(bucket[-1][0], nums[i])
            else:
                # check which bucket it belongs to
                idx = (nums[i]-min_val) // division_gap
                if idx == (nums[i]-min_val) / division_gap:
                    idx = idx - 1
                bucket[idx][0] = min(bucket[idx][0], nums[i])
                bucket[idx][1] = max(bucket[idx][1], nums[i])

        # iterate over all buckets to see the max gap
        answer = 0
        prev = bucket[0][1]
        for i in range(0, len(nums)-2):
            if bucket[i+1][0] != np.inf:
                answer = max(answer, bucket[i+1][0]-prev)
                prev = bucket[i+1][1]

        return int(answer)


import numpy as np

def split(inte, h):
    """ Split the interval

    Args:
        inte: list/ndarray, the interval to be splited
        h: the step length of the split

    Returns:
        x: list/ndarray, the split of the interval
    """
    # the endpoints of the interval
    a, b = inte
    n = int((b - a)/h)
    x = np.linspace(a, b, n+1)

    return x

def classical_Runge_Kutta(x, func, ini):
    """ Solve ordinary differential equation by 4-th Runge Kutta Method

    Args:
        x: list/ndarray, the split of the interval
        func: the function which satisifies y'(x) = f(x, y)
        ini: double, the initial value for IVP

    Returns:
        y: list/ndarray, the values of the function y(x) at every point x
    """
    n = len(x)
    h = x[1] - x[0]
    A = np.diag([1/2, 1/2, 1], k=-1)
    B = np.array([1/6, 1/3, 1/3, 1/6])
    C = np.array([0, 1/2, 1/2, 1])

    # begin iteration
    y = np.zeros(n)
    y[0] = ini
    for i in range(1, n):
        k = np.zeros(4)
        for j in range(4):
            y_step = 0
            for m in range(4):
                y_step += A[j, m]*k[m]
            k[j] = func(x[i-1]+C[j]*h, y[i-1]+h*y_step)
        phi = 0
        for j in range(4):
            phi += B[j]*k[j]
        y[i] = y[i-1] + h*phi

    return y

def f1(x, y):
    z = y**2
    return z

def f2(x, y):
    z = x/y
    return z

if __name__ == '__main__':
    h = 0.1
    # solve the ordinary differential equation 1 by 4-th Runge-Kutta method
    interval1 = [0, 0.4]
    x1 = split(interval1, h)
    y1 = classical_Runge_Kutta(x1, f1, 1)
    y1_true = 1/(1 - x1)
    print("The numerical solution, analytic solution and the error of the question 1 are as follows:")
    for item in zip(x1, y1, y1_true):
        print(f"{item[0]:.1f}\t{item[1]:.7f}\t{item[2]:.7f}\t{item[2]-item[1]:.7f}")
    # solve the ordinary differential equation 2 by 4-th Runge-Kutta method
    interval2 = [2, 2.6]
    x2 = split(interval2, h)
    y2 = classical_Runge_Kutta(x2, f2, 1)
    y2_true = np.sqrt(x2**2 - 3)
    print("The numerical solution, analytic solution and the error of the question 2 are as follows:")
    for item in zip(x2, y2, y2_true):
        print(f"{item[0]:.1f}\t{item[1]:.7f}\t{item[2]:.7f}\t{item[2]-item[1]:.7f}")


# -*- coding: utf-8 -*-
import numpy as np

#def secante(x0, x1, f, error = 0.01):

def newtonraphson(f, x0, dx=0.01, error=0.0001, max_iter=100000):

    xn = x0

    for n in range(0, max_iter):

        fn = f(xn)

        if abs(fn) < error:
            print('Solución tras ', n, ' iteraciones.')
            return xn

        dfn = (f(xn + dx) - f(xn - dx)) / (2 * dx)

        if dfn == 0:
            print('Método se indefine (derivada se anula).')
            return None

        xn = xn - fn/dfn

    print('Método no converge (se supera el máximo de iteraciones).')
    return None

def cerof(x, y, error = 0.01):

    x0 = []
    y0 = []

    for i, X in enumerate(x):
        if np.abs(y[i]) <= error:
            x0.append(X)
            y0.append(y[i])

    x0 = np.asarray(x0)
    y0 = np.asarray(y0)

    return x0, y0

def ceros(a, b, f, error=0.01):
    """
    params x: arreglo en el que se quiere encontrar el cero de f
    params f: función a la que se le quiere encontrar el cero
    params a, b: extremos del intervalo que se desea explorar
    params error: nivel de precisión deseado
    returns: valor x en [a, b] en que f(x)=0
    """

    fa = f(a)
    fb = f(b)

    ceros = []
    fceros = []

    if fa * fb > 0:
        print('El método no aplica')
        return ceros, fceros
    elif fa * fb == 0:
        if fa == 0:
            ceros.append(a)
            fceros.append(fa)
        if fb == 0:
            ceros.append(b)
            fceros.append(fb)

        return ceros, fceros

    else:
        while (np.abs(fa).astype(float) > error) and (np.abs(fb).astype(float) > error):
            c = (a + b) / 2
            fc = f(c)

            if fa * fc > 0:
                a = c
                fa = fc

            else:
                b = c
                fb = fc

        if np.abs(fa).astype(float) <= error:
            ceros.append(a)
            fceros.append(fa)

        if np.abs(fb).astype(float) <= error:
            ceros.append(b)
            fceros.append(fb)

        return ceros, fceros

def derivadafun(a, b, f, dx=0.01):
    """
    Devuelve la derivada de una función, considerando que en extremo izquierdo sólo se puede
    aplicar derivada forward, en extremo derecho sólo se puede aplicar derivada backward y
    entremedio puede aplicad derivada central, que es más precisa.

    param x: vector en el que se evaluará la función, tiene largo N.
    param y: vector de la función evaluada en x.
    """

    N = int((b - a) / dx)
    x = np.linspace(a, b, N)

    derivada = []

    for i, X in enumerate(x):

        if i == 0:                   # Extremo izquierdo: usamos derivada forward
            deriv = (f(x[1]) - f(x[0])) / dx
            derivada.append(deriv)

        elif i == N - 1:             # Extremo derecho: usamos derivada backward (recordar que índice de x e y llega hasta N-1
            deriv = (f(x[N - 1]) - f(x[N - 2])) / dx
            derivada.append(deriv)

        else:                        # Extremo izquierdo: usamos derivada forward
            deriv = (f(x[i + 1]) - f(x[i - 1])) / (2 * dx)
            derivada.append(deriv)

    derivada = np.asarray(derivada)

    return derivada

def derivadavec(x, y):
    """
    Devuelve la derivada de una función, considerando que en extremo izquierdo sólo se puede
    aplicar derivada forward, en extremo derecho sólo se puede aplicar derivada backward y
    entremedio puede aplicad derivada central, que es más precisa.

    param x: vector en el que se evaluará la función, tiene largo N.
    param y: vector de la función evaluada en x.
    """

    N = len(x)

    # Calculamos dx, que tendrá N-1 valores
    dx = []

    for i in range(N-1):
            dx.append(x[i + 1] - x[i])

    # OJO: Si el vector x fuese definido de forma equiespaciada, por ejemplo, con linspace
    # , basta con dx = x[1] - x[0].
    # Además, en ese caso, dx[0] = dx[N - 1] = dx y dx[i + 1] + dx[i - 1] = 2*dx

    derivada = []

    for i in range(N):

        if i == 0:                   # Extremo izquierdo: usamos derivada forward
            deriv = (y[1] - y[0]) / dx[0]
            derivada.append(deriv)

        elif i == N - 1:             # Extremo derecho: usamos derivada backward (recordar que índice de x e y llega hasta N-1
            deriv = (y[N - 1] - y[N - 2]) / dx[N - 2]
            derivada.append(deriv)

        else:                        # Centro: usamos derivada central
            deriv = (y[i + 1] - y[i - 1]) / (dx[i] + dx[i - 1])
            derivada.append(deriv)

    derivada = np.asarray(derivada)

    return derivada


import numpy as np

# We create two rank 1 ndarrays
x = np.array([1,2,3,4])
y = np.array([5.5,6.5,7.5,8.5])

# We print x
print()
print('x = ', x)

# We print y
print()
print('y = ', y)
print()

# We perfrom basic element-wise operations
#  using arithmetic symbols and functions

print('x + y = ', x + y)
print('add(x,y) = ', np.add(x,y))
print()

print('x - y = ', x - y)
print('x - y = ', np.subtract(x,y))
print()

print('x * y = ', x * y)
print('x * y = ', np.multiply(x,y))
print()

print('x / y = ', x / y)
print('x / y = ', np.divide(x,y))
print()

# We can also perform the same element-wise arithmetic
# operations on rank 2 ndarrays. Again, remember that in
# order to do these operations the shapes of the ndarrays
#  being operated on, must have the same shape or be
# broadcastable.

# We create two rank 2 ndarrays
X = np.array([1,2,3,4]).reshape(2,2)
Y = np.array([5.5,6.5,7.5,8.5]).reshape(2,2)

# We print X
print()
print('X = \n', X)

# We print Y
print()
print('Y = \n', Y)
print()

# We perform basic element-wise operations using arithmetic symbols and functions
print('X + Y = \n', X + Y)
print()
print('add(X,Y) = \n', np.add(X,Y))
print()
print('X - Y = \n', X - Y)
print()
print('subtract(X,Y) = \n', np.subtract(X,Y))
print()
print('X * Y = \n', X * Y)
print()
print('multiply(X,Y) = \n', np.multiply(X,Y))
print()
print('X / Y = \n', X / Y)
print()
print('divide(X,Y) = \n', np.divide(X,Y))

# We can also apply mathematical functions,
# such as sqrt(x), to all elements of an ndarray at once.

# We create a rank 1 ndarray
x = np.array([1,2,3,4])

# We print x
print()
print('x = ', x)

# We apply different mathematical functions to all elements of x
print()
print('EXP(x) =', np.exp(x))
print()
print('SQRT(x) =',np.sqrt(x))
print()
print('POW(x,2) =',np.power(x,2)) # We raise all elements to the power of 2

# Another great feature of NumPy is that it has a wide variety of
#  statistical functions. Statistical functions provide us
#  with statistical information about the elements in an
#  ndarray.

# We create a 2 x 2 ndarray
X = np.array([[1,2], [3,4]])

# We print x
print()
print('X = \n', X)
print()

print('Average of all elements in X:', X.mean())
print('Average of all elements in the columns of X:', X.mean(axis=0))
print('Average of all elements in the rows of X:', X.mean(axis=1))
print()
print('Sum of all elements in X:', X.sum())
print('Sum of all elements in the columns of X:', X.sum(axis=0))
print('Sum of all elements in the rows of X:', X.sum(axis=1))
print()
print('Standard Deviation of all elements in X:', X.std())
print('Standard Deviation of all elements in the columns of X:', X.std(axis=0))
print('Standard Deviation of all elements in the rows of X:', X.std(axis=1))
print()
print('Median of all elements in X:', np.median(X))
print('Median of all elements in the columns of X:', np.median(X,axis=0))
print('Median of all elements in the rows of X:', np.median(X,axis=1))
print()
print('Maximum value of all elements in X:', X.max())
print('Maximum value of all elements in the columns of X:', X.max(axis=0))
print('Maximum value of all elements in the rows of X:', X.max(axis=1))
print()
print('Minimum value of all elements in X:', X.min())
print('Minimum value of all elements in the columns of X:', X.min(axis=0))
print('Minimum value of all elements in the rows of X:', X.min(axis=1))

# Finally, let's see how NumPy can add single numbers to all the elements of an ndarray without the use of complicated loops.

# We create a 2 x 2 ndarray
X = np.array([[1,2], [3,4]])

# We print x
print()
print('X = \n', X)
print()

print('3 * X = \n', 3 * X)
print()
print('3 + X = \n', 3 + X)
print()
print('X - 3 = \n', X - 3)
print()
print('X / 3 = \n', X / 3)

# n the examples above, NumPy is working behind the scenes
#  to broadcast 3 along the ndarray so that they have the
#  same shape. This allows us to add 3 to each element
#  of X with just one line of code.

# Subject to certain constraints, Numpy can do the same for
# two ndarrays of different shapes, as we can see below.

# We create a rank 1 ndarray
x = np.array([1,2,3])

# We create a 3 x 3 ndarray
Y = np.array([[1,2,3],[4,5,6],[7,8,9]])

# We create a 3 x 1 ndarray
Z = np.array([1,2,3]).reshape(3,1)

# We print x
print()
print('x = ', x)
print()

# We print Y
print()
print('Y = \n', Y)
print()

# We print Z
print()
print('Z = \n', Z)
print()

print('x + Y = \n', x + Y)
print()
print('Z + Y = \n',Z + Y)

## Quiz Questions


from typing import List, Optional
from dataclasses import dataclass

import numpy as np

from IPython import display
from d2l import torch as d2l

@dataclass
class PlotConfig:
    X: float
    Y: Optional[float]
    legend: Optional[List]

def function_(x):
    """
    example for this topic caculus
    f(x) = 3 * x^2 - 4 * x
    """
    return 3 * x ** 2 - 4 * x

def numerical_lim(f, x, h):
    """
    numerical result ( do Differentiation )
    f is what function used
    x is coordinate of x
    h is like dx
    """
    return (f(x + h) - f(x)) / h

def use_svg_display():
    """
    Use the svg format to display a plot in Jupyter.
    """
    display.set_matplotlib_formats("svg")

def set_figsize(figsize=(3.5, 2.5)):
    """
    Set the figure size for matplotlib.
    """
    use_svg_display()
    d2l.plt.rcParams["figure.figsize"] = figsize

def set_axes(axes, xlabel, ylabel, xlim, ylim, xscale, yscale, legend):
    """
    Set the axes for matplotlib.
    """
    axes.set_xlabel(xlabel)
    axes.set_ylabel(ylabel)
    axes.set_xscale(xscale)
    axes.set_yscale(yscale)
    axes.set_xlim(xlim)
    axes.set_ylim(ylim)
    if legend:
        axes.legend(legend)
    axes.grid()

def plot(
    config: PlotConfig,
    fmts=("-", "m--", "g-.", "r:"),
    figsize=(3.5, 2.5),
    axes=None,
):
    """
    Plot data points.
    return axes, xlabel, ylabel, xlim, ylim, xscale, yscale, legend for set_axes()
    """
    if config.legend is None:
        config.legend = []

    set_figsize(figsize)
    axes = axes if axes else d2l.plt.gca()

    # Return True if `X` (tensor or list) has 1 axis
    def has_one_axis(X):
        return (
            hasattr(X, "ndim")
            and X.ndim == 1
            or isinstance(X, list)
            and not hasattr(X[0], "__len__")
        )

    if has_one_axis(config.X):
        X = [config.X]
    if config.Y is None:
        X, Y = [[]] * len(X), X
    elif has_one_axis(config.Y):
        Y = [config.Y]
    if len(X) != len(Y):
        X = X * len(Y)

    axes.cla()

    for x, y, fmt in zip(X, Y, fmts):
        if len(x):
            axes.plot(x, y, fmt)
        else:
            axes.plot(y, fmt)

if __name__ == "__main__":
    x = np.arange(0, 3, 0.1)
    h = 0.1  # h is like dx

    plot_config = PlotConfig(
        X=x,
        Y=[function_(x), 2 * x - 3],
        legend=["function_(x)", "Tangent line (x=1)"],
    )

    plot(config=plot_config)

    for _ in range(5):
        print(
            f"h={h:.5f}, numerical limit={numerical_lim(function_, 1, h):.5f}"
        )
        h *= 0.1


# XGBoost & LightGBM

## XGBoost

mathematically, we can write our ensemble tree model in the form

$$\hat{y}_{i} = \sum_{k=1}^{K}f_{k}(x_{i}), f_{k}\in\mathcal{F}$$

where $K$ is the number of trees, $f$ is a function in the function space $\mathcal{F}$, and $\mathcal{F}$ is the set of all possible CARTs.

the objective function to be optimized is given by

$$\mbox{obj}(\theta) = \sum_{i=1}^{n}l(y_{i}, \hat{y}_{i}) + \sum_{k=1}^{K}\Omega(f_{k})$$

where $l$ is the loss term, $\Omega$ is the regularization term.

### Additive Learning

what are the parameters of trees? you can find that what we need to learn are those functions $f_{i}$, each containing the structure of the tree and the leaf scores.

learning tree strcture is much harder than traditional optimization problem where you can simply take the gradient, instead, we use an additive strategy: fix what we learned, and add one new tree at a time. we write the prediction value at step $t$ as $\hat{y}_{i}^{(t)}$, then

$$
\begin{equation}
\begin{split}
\mbox{obj}^{(t)} =& \sum_{i=1}^{n}l(y_{i}, \hat{y}_{i}^{(t)}) + \sum_{k=1}^{t}\Omega(f_{k}) \\
=& \sum_{i=1}^{n}l(y_{i}, \hat{y}_{i}^{(t - 1)} + f_{t}(x_{i})) + \Omega(f_{t}) + \mbox{constant}
\end{split}
\end{equation}
$$

$$\hat{y}_{i}^{(0)} = 0$$

in general case($l$ arbitrary), we take the taylor expansion of the loss function up to the second order

$$\mbox{obj}^{(t)} \approx \sum_{i=1}^{n}[l(y_{i}, \hat{y}_{i}^{(t - 1)}) + g_{i}f_{t}(x_{i}) + \frac{1}{2}h_{i}f_{t}(x_{i})^{2}] + \Omega(f_{t}) + \mbox{constant}$$

where $g_{i}$ and $h_{i}$ are defined as

$$g_{i} = \frac{\partial l(y_{i}, \hat{y}_{i}^{(t - 1)})}{\partial \hat{y}_{i}^{(t-1)}}, h_{i} = \frac{\partial^{2} l(y_{i}, \hat{y}_{i}^{(t - 1)})}{{\partial \hat{y}_{i}^{(t-1)}}^2}$$

after removing all the constants, the specific objective at step $t$ becomes

$$\sum_{i=1}^{n}[g_{i}f_{t}(x_{i}) + \frac{1}{2}h_{i}f_{t}(x_{i})^{2}] + \Omega(f_{t})$$

this becomes our optimization goal for the new tree.

### Model Complexity

we need to define the complexity of the tree $\Omega(f)$, in order to do so, let us first refine the definition of the tree $f(x)$ as

$$f(x) = w_{q(x)}, w \in \mathbb{R}^{T}, q: \mathbb{R}^{d} \to \{1,2,...,T\}$$

where $w$ is the vector of scores on leaves, $q$ is a function assigning each data point to the corresponding leaf, and $T$ is the number of leaves. in XGBoost, we define the complexity as

$$\Omega(f) = \gamma{T} + \frac{1}{2}\lambda\sum_{j=1}^{T}w_{j}^{2}$$

this works well in practice.

### The Structure Score

now we can write the objective value with the $t$-th tree as:

$$
\begin{equation}
\begin{split}
\mbox{obj}^{(t)} = & \sum_{i=1}^{n}[g_{i}f_{t}(x_{i}) + \frac{1}{2}h_{i}f_{t}(x_{i})^{2}] + \gamma{T} + \frac{1}{2}\lambda\sum_{j=1}^{T}w_{j}^{2} \\
=& \sum_{j=1}^{T}[(\sum_{i\in{I_{j}}}g_{i})w_{j} + \frac{1}{2}(\sum_{i\in{I_j}}h_{i} + \lambda)w_{j}^2] + \gamma{T}
\end{split}
\end{equation}
$$

where $I_{j} = \{i|q_{i}=j\}$ is the set of indices of data-points assign to the $j$-th leaf.

we could further compress the expression by defining $G_{j} = \sum_{i\in{I_{j}}}g_{i}, H_{j} = \sum_{i\in{I_j}}h_{i}$:

$$\mbox{obj}^{(t)} = \sum_{j=1}^{T}[G_{j}w_{j} + \frac{1}{2}(H_{j} + \lambda)w_{j}^2] + \gamma{T}$$

in this equation, $w_{j}$ are independent with respect to each other, the form $G_{j}w_{j} + \frac{1}{2}(H_{j} + \lambda)w_{j}^2$ is quadratic and the best $w_{j}$ for a given $q(x)$ and the best objective reduction we can get is:

$$w_{j}^{\ast} = -\frac{G_{j}}{H_{j} + \lambda}$$

$$\mbox{obj}^{\ast} = -\frac{1}{2}\sum_{i=1}^{T}\frac{G_{j}^2}{H_{j} + \lambda} + \gamma{T}$$

the last equation measures how good a tree structure $q(x)$ is.

### Learn the Tree Structure

now we have a way to measure how good a tree is, ideally we would enumerate all possible trees and pick the best one, but not practical.

instead we will try to optimize **one level** of the tree at a time, specifically we try to split a leaf into two leaves, and the score gain is:

$$Gain = \frac{1}{2}\left [\frac{G_L^2}{H_L + \lambda} + \frac{G_R^2}{H_R + \lambda} - \frac{(G_L + G_R)^2}{H_L + H_R + \lambda}\right ] - \gamma$$

if the first part of $Gain$ is smaller than $\gamma$, we would do better not add that branch, this is exactly pruning!

for real valued data, we places all instances in sorted order(by the split feature), then a left to right scan is sufficient to calculate the structure score of all possible split solutions, and we can find the best split efficiently.

in practice, since it is intractable to enumerate all possible tree structures, we add one split at a time, this approach works well at most of the time.

### XGBoost Practice

"""quadratic dataset"""
import numpy as np
from sklearn.model_selection import train_test_split

np.random.seed(42)
X = np.random.rand(100, 1) - 0.5
y = 3*X[:, 0]**2 + 0.05 * np.random.randn(100)
X_train, X_val, y_train, y_val = train_test_split(X, y, random_state=42)

"""basic xgboost"""
import xgboost
from sklearn.metrics import mean_squared_error

xgb_reg = xgboost.XGBRegressor()
xgb_reg.fit(X_train, y_train)
y_pred = xgb_reg.predict(X_val)
mean_squared_error(y_pred, y_val)

"""xgboost automatically taking care of early stopping"""
xgb_reg.fit(X_train, y_train, eval_set=[(X_val, y_val)], early_stopping_rounds=2)
y_pred = xgb_reg.predict(X_val)
mean_squared_error(y_pred, y_val)

## LightGBM

XGBoost uses level-wise tree growth:

![jupyter](images/level.png)

while LightGBM uses leaf-wise tree growth:

![jupyter](images/leaf.png)

we can formalize leaf-wise tree growth as:

$$(p_m, f_{m}, v_{m}) = \underset{(p,f,v)}{argmin}\ L(T_{m-1}(X).split(p, f, v), Y)$$

$$T_{m}(X) = T_{m-1}(X).split(p_m, f_{m}, v_{m})$$

finding the best split is costly, in XGBoost, we enumerate all features and all thresholds to find the best split.

LightGBM optimize that by using the histogram algorithm:


import numpy as np
import pandas as pd

def drop_non_numeric_values(data: pd.DataFrame) -> None:
    data.replace([np.inf, -np.inf], np.nan, inplace=True)
    data.dropna(inplace=True)

def downcast_to_int(data: pd.DataFrame) -> pd.DataFrame:
    for column in data.columns:
        data[column] = data[column].astype(int)
    return data

def encode_cyclic_data(df: pd.DataFrame, column_id: str, max_val: int, drop=True) -> None:
    df[column_id + '_sin'] = np.sin(2 * np.pi * df[column_id] / max_val)
    df[column_id + '_cos'] = np.cos(2 * np.pi * df[column_id] / max_val)
    if drop:
        df.drop(columns=[column_id], inplace=True)

def aggregate(data: pd.DataFrame, target_column: str, rule: str) -> pd.DataFrame:
    data.replace(np.nan, -np.inf, inplace=True)

    aggregator = {}
    for column in data.columns:
        if column is target_column:
            aggregator[column] = 'sum'
        else:
            aggregator[column] = 'first'

    data = data.resample(rule).agg(aggregator)

    data.replace(-np.inf, np.nan, inplace=True)
    return data

def interpolate(df: pd.DataFrame, method='linear') -> pd.DataFrame:
    # replace inf with NaN, so they are interpolated
    df = df.replace([np.inf, -np.inf], np.nan)
    df = df.interpolate(method=method, limit_direction='both')

    assert not df.isnull().values.any(), 'well that is awkward. maybe the entire dataframe contained NaNs only?'
    return df


"""
Author : Achintya Gupta
Date Created : 22-09-2020
"""

"""
Problem Statement
------------------------------------------------
Q) A permutation is an ordered arrangement of objects. For example, 3124 is one possible permutation of the digits 1, 2, 3 and 4.
   If all of the permutations are listed numerically or alphabetically, we call it lexicographic order.
   The lexicographic permutations of 0, 1 and 2 are:
    012   021   102   120   201   210
   What is the millionth lexicographic permutation of the digits 0, 1, 2, 3, 4, 5, 6, 7, 8 and 9?
"""
from utils import timing_decorator, get_factorial
import numpy as np

def recurr_loop(nums, nth, selected_nums = [], totald = 10):
    nums = sorted([k for k in nums if k not in selected_nums])
    total_digits = len(nums)
    blank_spaces = total_digits-1
    blank_combin = get_factorial(blank_spaces)
    if len(selected_nums) == totald-1:
        selected_nums = [str(k) for k in selected_nums+nums]
        return int("".join(selected_nums))
    numidx = int(np.floor(nth/blank_combin)) if nth%blank_combin != 0 else int(np.floor(nth/blank_combin))-1
    selected_digit = nums[numidx]
    selected_nums.append(selected_digit)
    return recurr_loop(nums, nth-blank_combin*numidx, selected_nums)

@timing_decorator
def nth_lexiperm(num_list=[0,1,2,4,3,5,6,7,8,9],N=1_000_000):
    perm_comb = recurr_loop(nums=num_list, nth=N)
    print(f'{N}`th Lexicographic permutation : ', perm_comb)

nth_lexiperm()


from math import radians
import unittest

from numpy import array, cos, sin, exp
from numpy.testing import assert_allclose

from ml_algorithms.utils import numerical_grad, g_grad

class TestLogisticRegression(unittest.TestCase):

    def test_numeric_grad_1(self):

        def J(x):
            return sum(3 * (x ** 2))

        theta = array([[0], [4], [10]])
        err = 0.0001

        assert_allclose(array([[0], [24], [60]]),
                        numerical_grad(J, theta, err),
                        rtol=0, atol=0.001, equal_nan=False)

    def test_numeric_grad_2(self):

        def J(x):
            return sum(1 / x)

        theta = array([[5], [8], [20]])
        err = 0.0001

        assert_allclose(array([[-0.04], [-0.015625], [-0.0025]]),
                        numerical_grad(J, theta, err),
                        rtol=0, atol=0.001, equal_nan=False)

    def test_numeric_grad_3(self):

        def J(x):
            return sum(cos(x))

        theta = array([[radians(30)],
                       [radians(45)],
                       [radians(60)],
                       [radians(90)]])
        err = 0.0001

        assert_allclose(array([[-sin(radians(30))],
                               [-sin(radians(45))],
                               [-sin(radians(60))],
                               [-sin(radians(90))]]),
                        numerical_grad(J, theta, err),
                        rtol=0, atol=0.001, equal_nan=False)

    def test_numeric_grad_4(self):

        def J(x):
            return sum(exp(x))

        theta = array([[-10], [-1], [0], [1], [10]])
        err = 0.0001

        assert_allclose(array([[exp(-10)],
                               [exp(-1)],
                               [exp(0)],
                               [exp(1)],
                               [exp(10)]]),
                        numerical_grad(J, theta, err),
                        rtol=0, atol=0.001, equal_nan=False)

    def test_numeric_grad_5(self):

        def J(x):
            return sum(7 * x)

        theta = array([[-10], [-1], [0], [1], [10]])
        err = 0.0001

        assert_allclose(array([[7],
                               [7],
                               [7],
                               [7],
                               [7]]),
                        numerical_grad(J, theta, err),
                        rtol=0, atol=0.001, equal_nan=False)

    def test_sigmoid_gradient(self):
        z = array([-1, -0.5, 0, 0.5, 1])
        assert_allclose(g_grad(z),
                        [0.196612, 0.235004, 0.25, 0.235004, 0.196612],
                        rtol=0, atol=0.001, equal_nan=False)


import numpy as np

def translate(point, vector, length=None):
    vector = np.array(vector)
    if length is not None:
        vector = length * vector / np.linalg.norm(vector)
    return (np.array(point) + vector).tolist()

def circle(center, perp_vect, radius, element_number=10):
    """
    Function computed the circle points. No drawing.
    perp_vect is vector perpendicular to plane of circle
    """
    # tl = [0, 0.2, 0.4, 0.6, 0.8]
    tl = np.linspace(0, 1, element_number)

    # vector form center to edge of circle
    # u is a unit vector from the centre of the circle to any point on the
    # circumference

    # normalized perpendicular vector
    n = perp_vect / np.linalg.norm(perp_vect)

    # normalized vector from the centre to point on the circumference
    u = perpendicular_vector(n)
    u = u / np.linalg.norm(u)

    pts = []

    for t in tl:
        # u = np.array([0, 1, 0])
        # n = np.array([1, 0, 0])
        pt = radius * np.cos(t * 2 * np.pi) * u +\
            radius * np.sin(t * 2 * np.pi) * np.cross(u, n) +\
            center

        pt = pt.tolist()
        pts.append(pt)

    return pts

def perpendicular_vector(v):
    r""" Finds an arbitrary perpendicular vector to *v*."""
    if v[1] == 0 and v[2] == 0:
        if v[0] == 0:
            raise ValueError('zero vector')
        else:
            return np.cross(v, [0, 1, 0])
    return np.cross(v, [1, 0, 0])

def cylinder_circles(nodeA, nodeB, radius, element_number=10):
    """
    Return list of two circles with defined parameters.
    """

    vector = (np.array(nodeA) - np.array(nodeB)).tolist()
    ptsA = circle(nodeA, vector, radius, element_number)
    ptsB = circle(nodeB, vector, radius, element_number)

    return ptsA, ptsB

def plane_fit(points):
    """
    p, n = plane_fit(points)

    Given an array, points, of shape (d,...)
    representing points in d-dimensional space,
    fit an d-dimensional plane to the points.
    Return a point, p, on the plane (the point-cloud centroid),
    and the normal, n.
    """
    import numpy as np
    from numpy.linalg import svd
    points = np.reshape(points, (np.shape(points)[0], -1)) # Collapse trialing dimensions
    assert points.shape[0] <= points.shape[1], "There are only {} points in {} dimensions.".format(points.shape[1], points.shape[0])
    ctr = points.mean(axis=1)
    x = points - ctr[:,np.newaxis]
    M = np.dot(x, x.T) # Could also use np.cov(x) here.
    return ctr, svd(M)[0][:,-1]


# %%
import numpy as np
import os
import matplotlib.pyplot as plt

plt.style.use("seaborn-whitegrid")

import tensorflow as tf

import tensorflow_probability as tfp

tfd = tfp.distributions
tfb = tfp.bijectors

# %%
# Coin flip example
# Use bernoulli samples
# Build beta distributions from conjugate priors

# Build Graph
rv_coin_flip_prior = tfp.distributions.Bernoulli(probs=0.5, dtype=tf.int32)

num_trials = tf.constant([0, 1, 2, 3, 4, 5, 8, 15, 50, 500, 1000, 2000])

coin_flip_data = rv_coin_flip_prior.sample(num_trials[-1])

# prepend a 0 onto tally of heads and tails, for zeroth flip
coin_flip_data = tf.pad(coin_flip_data, tf.constant([[1, 0]]), "CONSTANT")

# compute cumulative headcounts from 0 to 2000 flips, and then grab them at each of num_trials intervals
cumulative_headcounts = tf.gather(tf.cumsum(coin_flip_data), num_trials)

# Creates a series of beta distributions at each num_trials
rv_observed_heads = tfp.distributions.Beta(
    concentration1=tf.cast(1 + cumulative_headcounts, tf.float32),
    concentration0=tf.cast(1 + num_trials - cumulative_headcounts, tf.float32),
)

probs_of_heads = tf.linspace(start=0.0, stop=1.0, num=100, name="linspace")
observed_probs_heads = tf.transpose(
    rv_observed_heads.prob(probs_of_heads[:, tf.newaxis])
)

# %%
# For the already prepared, I'm using Binomial's conj. prior.
plt.figure(figsize=(16, 9))
for i in range(len(num_trials)):
    sx = plt.subplot(len(num_trials) / 2, 2, i + 1)
    plt.xlabel("$p$, probability of heads") if i in [
        0,
        len(num_trials) - 1,
    ] else None
    plt.setp(sx.get_yticklabels(), visible=False)
    plt.plot(
        probs_of_heads,
        observed_probs_heads[i],
        label="observe %d tosses,\n %d heads"
        % (num_trials[i], cumulative_headcounts[i]),
    )
    plt.fill_between(probs_of_heads, 0, observed_probs_heads[i], alpha=0.4)
    plt.vlines(0.5, 0, 4, color="k", linestyles="--", lw=1)
    leg = plt.legend()
    leg.get_frame().set_alpha(0.4)
    plt.autoscale(tight=True)

plt.suptitle(
    "Bayesian updating of posterior probabilities", y=1.02, fontsize=14
)
plt.tight_layout()

# %%

x = tf.range(start=0.0, limit=16.0, dtype=tf.float32)
lambdas = tf.constant([1.5, 4.25])

poi_pmf = tfd.Poisson(rate=lambdas[:, tf.newaxis]).prob(x)

# %%
temp = tfd.Poisson(rate=2.5)
tf.reduce_mean(temp.sample(1000))

temp = tfd.Exponential(rate=2.5)
print(tf.reduce_mean(temp.sample(1000)))
print(1 / temp.rate)

# %%
a = tf.range(start=0.0, limit=4.0, delta=0.04)
a = a[..., tf.newaxis]
lambdas = tf.constant([0.5, 1.0])

# Now we use TFP to compute probabilities in a vectorized manner.
expo_pdf = tfd.Exponential(rate=lambdas).prob(a)

# %%

# Defining our Data and assumptions
# fmt: off
count_data = tf.constant([
    13,  24,   8,  24,   7,  35,  14,  11,  15,  11,  22,  22,  11,  57,
    11,  19,  29,   6,  19,  12,  22,  12,  18,  72,  32,   9,   7,  13,
    19,  23,  27,  20,   6,  17,  13,  10,  14,   6,  16,  15,   7,   2,
    15,  15,  19,  70,  49,   7,  53,  22,  21,  31,  19,  11,  18,  20,
    12,  35,  17,  23,  17,   4,   2,  31,  30,  13,  27,   0,  39,  37,
    5,  14,  13,  22,
], dtype=tf.float32)
# fmt: on
n_count_data = tf.shape(count_data)
days = tf.range(n_count_data[0], dtype=tf.int32)

# %%
def joint_log_prob(count_data, lambda_1, lambda_2, tau):
    """
    Takes in data samples for count data and lambdas + tau
    Returns joint prob of all of those occurances, logged
    """
    # Estimate prior alpha as 1/average count
    alpha = 1.0 / tf.reduce_mean(count_data)
    rv_lambda_1 = tfd.Exponential(rate=alpha)
    rv_lambda_2 = tfd.Exponential(rate=alpha)

    rv_tau = tfd.Uniform()

    # Split days into different lambdas
    # Select lambda_1 for 0:tau-1, lambda_2 for tau:-1
    lambda_ = tf.gather(
        [lambda_1, lambda_2],
        indices=tf.cast(
            tau * tf.cast(tf.size(count_data), dtype=tf.float32)
            <= tf.cast(tf.range(tf.size(count_data)), dtype=tf.float32),
            dtype=tf.int32,
        ),
    )
    rv_observation = tfd.Poisson(rate=lambda_)

    return (
        rv_lambda_1.log_prob(lambda_1)
        + rv_lambda_2.log_prob(lambda_2)
        + rv_tau.log_prob(tau)
        + tf.reduce_sum(rv_observation.log_prob(count_data))
    )

# Define a closure over our joint_log_prob. - removes count_data as optimised parameter
def unnormalized_log_posterior(lambda1, lambda2, tau):
    return joint_log_prob(count_data, lambda1, lambda2, tau)

# %%
# wrap the mcmc sampling call in a @tf.function to speed it up
@tf.function(autograph=False)
def graph_sample_chain(*args, **kwargs):
    return tfp.mcmc.sample_chain(*args, **kwargs)

num_burnin_steps = 5000
num_results = 20000

# Set the chain's start state.
initial_chain_state = [
    tf.cast(tf.reduce_mean(count_data), tf.float32)
    * tf.ones([], dtype=tf.float32, name="init_lambda1"),
    tf.cast(tf.reduce_mean(count_data), tf.float32)
    * tf.ones([], dtype=tf.float32, name="init_lambda2"),
    0.5 * tf.ones([], dtype=tf.float32, name="init_tau"),
]

# Since HMC operates over unconstrained space, we need to transform the
# samples so they live in real-space.
unconstraining_bijectors = [
    tfp.bijectors.Exp(),  # Maps a positive real to R.
    tfp.bijectors.Exp(),  # Maps a positive real to R.
    tfp.bijectors.Sigmoid(),  # Maps [0,1] to R.
]

step_size = 0.2

kernel = tfp.mcmc.TransformedTransitionKernel(
    inner_kernel=tfp.mcmc.HamiltonianMonteCarlo(
        target_log_prob_fn=unnormalized_log_posterior,
        num_leapfrog_steps=2,
        step_size=step_size,
        state_gradients_are_stopped=True,
    ),
    bijector=unconstraining_bijectors,
)

kernel = tfp.mcmc.SimpleStepSizeAdaptation(
    inner_kernel=kernel, num_adaptation_steps=int(num_burnin_steps * 0.8)
)

# Sample from the chain.
(
    [lambda_1_samples, lambda_2_samples, posterior_tau],
    kernel_results,
) = graph_sample_chain(
    num_results=num_results,
    num_burnin_steps=num_burnin_steps,
    current_state=initial_chain_state,
    kernel=kernel,
)

tau_samples = tf.floor(
    posterior_tau * tf.cast(tf.size(count_data), dtype=tf.float32)
)

plt.hist(lambda_1_samples, bins=100)
plt.hist(posterior_tau)

# %%
w = 1.0 / tau_samples.shape[0] * np.ones_like(tau_samples)
plt.hist(
    tau_samples,
    bins=n_count_data[0],
    alpha=1,
    label=r"posterior of $\tau$",
    weights=w,
    rwidth=2.0,
)
plt.xticks(np.arange(n_count_data[0]))

plt.legend(loc="upper left")
plt.ylim([0, 0.75])
plt.xlim([35, len(count_data) - 20])
plt.xlabel(r"$\tau$ (in days)")
plt.ylabel(r"probability")

# %%
# Get samples of lambda1/2, tau for each day. Repeat 20k times. Use the posterior chains as samples
# Average lambda distribution per day and plot

# tau_samples, lambda_1_samples, lambda_2_samples contain
# N samples from the corresponding posterior distribution

N_ = tau_samples.shape[0]
expected_texts_per_day = tf.zeros(N_, n_count_data.shape[0])  # (10000,74)

plt.figure(figsize=(12.5, 9))

# Create matrix of days, copy over 20k samples
day_range = tf.range(0, n_count_data[0], delta=1, dtype=tf.int32)

# expand from shape of 74 to (10000,74)
day_range = tf.expand_dims(day_range, 0)
day_range = tf.tile(day_range, tf.constant([N_, 1]))

# Create multiple series of samples from tau, each day has 20k samples of tau
# expand from shape of 10000 to 10000,74
tau_samples_per_day = tf.expand_dims(tau_samples, 0)
tau_samples_per_day = tf.transpose(
    tf.tile(tau_samples_per_day, tf.constant([day_range.shape[1], 1]))
)

tau_samples_per_day = tf.cast(tau_samples_per_day, dtype=tf.int32)
# ix_day is (10000,74) tensor where axis=0 is number of samples, axis=1 is day. each value is true iff sampleXday value is < tau_sample value
ix_day = day_range < tau_samples_per_day

# Expand lambda samples over 20k samples, one sample pair per day
lambda_1_samples_per_day = tf.expand_dims(lambda_1_samples, 0)
lambda_1_samples_per_day = tf.transpose(
    tf.tile(lambda_1_samples_per_day, tf.constant([day_range.shape[1], 1]))
)
lambda_2_samples_per_day = tf.expand_dims(lambda_2_samples, 0)
lambda_2_samples_per_day = tf.transpose(
    tf.tile(lambda_2_samples_per_day, tf.constant([day_range.shape[1], 1]))
)

# Sum lambdas with binary on/off from tau
expected_texts_per_day = (
    tf.reduce_sum(
        lambda_1_samples_per_day * tf.cast(ix_day, dtype=tf.float32), axis=0
    )
    + tf.reduce_sum(
        lambda_2_samples_per_day * tf.cast(~ix_day, dtype=tf.float32), axis=0
    )
) / N_

# %% Add stats not just expectation
# My way
expected_texts_per_day_samples = tf.concat(
    [
        lambda_1_samples_per_day * tf.cast(ix_day, dtype=tf.float32),
        lambda_2_samples_per_day * tf.cast(~ix_day, dtype=tf.float32),
    ],
    axis=0,
)
# Removes 0s
expected_texts_per_day_samples
mask = tf.not_equal(expected_texts_per_day_samples, 0)
# different number of 0s per row
# need to transpose the mask and data and it iterates across rows
# reshape works along rows, so need long rows and then transpose back to full size
expected_texts_per_day_samples_mask = tf.transpose(
    tf.reshape(
        tf.boolean_mask(
            tf.transpose(expected_texts_per_day_samples), tf.transpose(mask)
        ),
        [-1, N_],
    )
)

# Find stats
expected_texts_per_day_mu = tf.reduce_mean(
    expected_texts_per_day_samples_mask, axis=0
)
expected_texts_per_day_std = tf.math.reduce_std(
    expected_texts_per_day_samples_mask, axis=0
)
expected_texts_per_day_std = tf.math.reduce
import numpy as np
expected_texts_per_day_quantiles = np.quantile(expected_texts_per_day_samples_mask.numpy(), [0.05, 0.95], axis=0)

assert (
    np.testing.assert_allclose(
        expected_texts_per_day.numpy(),
        expected_texts_per_day_mu.numpy(),
        rtol=1e-5,
    )
    is None
)

# %%
plt.figure(figsize=(12, 9))
plt.plot(
    range(n_count_data[0]),
    expected_texts_per_day_mu,
    lw=4,
    color="#E24A33",
    label="expected number of text-messages received",
)
if 0:
    plt.fill_between(
        range(n_count_data[0]),
        expected_texts_per_day_mu - expected_texts_per_day_std,
        expected_texts_per_day_mu + expected_texts_per_day_std,
        alpha=0.4,
    )
else:
    plt.fill_between(
        range(n_count_data[0]),
        expected_texts_per_day_quantiles[0],
        expected_texts_per_day_quantiles[1],
        alpha=0.4,
        color="#0000DA",
        label='ConfidenceInterval [0.05, 0.95]'
    )

plt.xlim(0, n_count_data.numpy()[0])
plt.xlabel("Day")
plt.ylabel("Expected # text-messages")
plt.title("Expected number of text-messages received")
plt.ylim(0, 60)
plt.bar(
    np.arange(len(count_data)),
    count_data,
    color="#5DA5DA",
    alpha=0.65,
    label="observed texts per day",
)

plt.legend(loc="upper left")

# %%


"""

Simple probability distribution model
==============================================

In the following we learn the mean and covariance of a univariate gaussian distribution from data.
"""

#%% md
#
# Initially we have to import the necessary modules.

#%%

import numpy as np
import matplotlib.pyplot as plt
from UQpy.inference import DistributionModel, MLE
from UQpy.distributions import Normal
from UQpy.inference import MinimizeOptimizer

#%% md
#
# First, for the sake of this example, we generate fake data from a gaussian distribution with mean 0 and
# standard deviation 1.

#%%

mu, sigma = 0, 0.1  # true mean and standard deviation
data_1 = np.random.normal(mu, sigma, 1000).reshape((-1, 1))
print('Shape of data vector: {}'.format(data_1.shape))

count, bins, ignored = plt.hist(data_1, 30, density=True)
plt.plot(bins, 1 / (sigma * np.sqrt(2 * np.pi)) * np.exp(- (bins - mu) ** 2 / (2 * sigma ** 2)),
         linewidth=2, color='r')
plt.title('Histogram of the data')
plt.show()

#%% md
#
# Create an instance of the class Model. The user must define the number of parameters to be estimated, in this case 2
# (mean and standard deviation), and set those parameters to be learnt as None when instantiating the Distribution
# object. For maximum likelihood estimation, no prior pdf is required.

#%%

# set parameters to be learnt as None
dist = Normal(loc=None, scale=None)
candidate_model = DistributionModel(n_parameters=2, distributions=dist)

ml_estimator = MLE(inference_model=candidate_model, data=data_1, n_optimizations=3)
print('ML estimates of the mean={0:.3f} (true=0.) and std. dev={1:.3f} (true=0.1)'.format(
    ml_estimator.mle[0], ml_estimator.mle[1]))

#%% md
#
# We can also fix one of the parameters and learn the remaining one

#%%

d = Normal(loc=0., scale=None)
candidate_model = DistributionModel(n_parameters=1, distributions=d)

optimizer = MinimizeOptimizer(bounds=[[0.0001, 2.]])
ml_estimator = MLE(inference_model=candidate_model, data=data_1,
                   n_optimizations=1)
print('ML estimates of the std. dev={0:.3f} (true=0.1)'.format(ml_estimator.mle[0]))


'''
File: simplex_method.py
Author: Yutong Dai (rothdyt@gmail.com)
File Created: Saturday, 2018-12-15 15:17
Last Modified: Saturday, 2018-12-15 15:35
--------------------------------------------
Desscription:
'''
import numpy as np
from simplex_step import simplex_step
from simplex_init import simplex_init

def simplex_method(A, b, c, irule):
    """
    Find a basic optimal solution for the linear program

     min:    c*x
     ST:     Ax=b
             x>=0,

    where A is an (m,n) matrix.

    Input Parameters:

    A - (n,m) constraint matrix
    b - (m,1) POSITIVE vector appearing in the constraint equation above
    c - (1,n) vector giving the coefficients of the objective function

    irule - integer parameter speciying which pivot rule to use:
        irule = 0 indicates that the smallest coefficient rule should be
            used
        irule = 1 indicates that Bland's rule should be used

    Output Parameters:

    istatus - integer parameter reporting the condition of the
        istatus = 0  indicates normal completeion (i.e., a solution
        has been found and reported)
        istatus = 4  indicates the program is infeasible
        istatus = 16 indicates the program is feasible but our initialization
        procedure has failed
        istatus = 32 indicates that the program is unbounded

    X  - vector of length n specifying the solution
    eta - the minimum value of the objective function
    iB - integer vector specifying the m indices of the basic variables
        after the simplex step
    iN - integer vector specifying the n-m indices of the nonbasic
        variables after the simplex step
    xB - vector of length m specifying the values of the basic
        variables after the simplex step

    """
    #init_status, iB, iN, xB, tableau = simplex_init_modified(A, b, c)
    init_status, iB, iN, xB = simplex_init(A, b, c)
    if init_status == 4:
        istatus, X, eta, iB, iN, xB = 16, None, None, None, None, None
        return istatus, X, eta, iB, iN, xB
    elif init_status == 16:
        istatus, X, eta, iB, iN, xB = 4, None, None, None, None, None
        return istatus, X, eta, iB, iN, xB
    else:
        istatus_step = 1000
        count = 0
        while istatus_step != -1:
            istatus_step, iB, iN, xB, _ = simplex_step(A, b, c, iB, iN, xB, irule=0, Binv="")
            count += 1
            if istatus_step == 16 or count > 100:
                istatus, X, eta, iB, iN, xB = 32, None, None, None, None, None
                return istatus, X, eta, iB, iN, xB

        istatus = 0
        iB = iB - 1
        eta = np.matmul(c[0, iB].reshape(1, -2), xB)
        X = np.zeros((A.shape[1], 1))
        X[iB] = xB
        return istatus, X, eta, iB+1, iN, xB

if __name__ == "__main__":
    import numpy as np
    import random
    from simplex_method import simplex_method
    from simplex_test import simplex_test
    print("===============================")
    print("Test9.py")
    # first form an invertible matrix
    R = np.matrix([[4, 1,  1],
                   [-1, 2,  1],
                   [1, 1, -1]], dtype=np.float64)

    # form a vector b which is in the span of R
    b = R*np.matrix([[1],
                     [2],
                     [1]], dtype=np.float64)

    B = np.matrix([[1, 1, 1],
                   [1, 1, 0],
                   [1, 0, 0]], dtype=np.float64)
    A = np.hstack((R, B))

    # form a random permutation
    p = list(range(0, 6))
    random.shuffle(p)
    A = A[:, p]

    c = np.matrix([[-2, 1, 1, -1, -1, -1]], dtype=np.float64)
    c = c[:, p]

    # test
    irule = 0
    [istatus, X, eta, iB, iN, xB] = simplex_method(A, b, c, irule)
    print(X, eta, iB, iN, xB)
    # return

    if (istatus != 0):
        print('istatus is wrong\n')

    [X, eta, isfeasible, isoptimal, zN] = simplex_test(A, b, c, iB, xB)

    if (isfeasible != 1):
        print('your solution is not feasible!!!\n')

    if (isoptimal != 1):
        print('your solution is not optimal!!!\n')

    print("===============================")
    print("Test10.py")
    R = np.matrix([[4, 1,  1],
                   [1, 2,  1],
                   [1, 1,  1]], dtype=np.float64)

    # form a vector b which is in the span of R
    b = R*np.matrix([[1],
                     [-4],
                     [-1]], dtype=np.float64)

    B = np.matrix([[1, 1, 1],
                   [1, 1, 0],
                   [1, 0, 0]], dtype=np.float64)

    A = np.hstack((R, B))

    c = np.matrix([[-2, 1, 1, -1, -1, -1]], dtype=np.float64)

    irule = 1
    [istatus, X, eta, iB, iN, xB] = simplex_method(A, b, c, irule)

    if (istatus != 4):
        print('istatus is wrong\n')
    print("===============================")
    print("Test11.py")
    A1 = np.matrix([[-1,    1,     2],
                    [-1,    1,     1],
                    [0,    1,     1]], dtype=np.float64)

    A = np.hstack((np.eye(3), A1))

    b = np.matrix([[1],
                   [2],
                   [3]], dtype=np.float64)

    iB = [1, 2, 3]
    iN = [4, 5, 6]
    xB = np.matrix(np.copy(b))
    c = np.matrix([[0, 0, 0, -1, 2, 1]], dtype=np.float64)

    # form an invertible matrix B and modify the problem
    B = np.matrix([[4, 1, 0],
                   [1, -2, -1],
                   [1, 2, 4]], dtype=np.float64)
    A = B*A
    b = B*b

    # modify c
    N = A[:, [index_N-1 for index_N in iN]]
    c1 = np.matrix([[1, 1, 0]], dtype=np.float64)
    c2 = c[:, (4-1):6]+c1*B.I*N
    c = np.hstack((c1, c2))

    irule = 0
    [istatus, X, eta, iB, iN, xB] = simplex_method(A, b, c, irule)

    if (istatus != 32):
        print('istatus is wrong\n')


import numpy as np
import scipy
import scipy.linalg
import time

# Set grid sizes
N1 = 30
N2 = 12
N3 = 10
N4 = 4
N = N1*N2*N3*N4

# Fix random seed
#np.random.seed(seed=0)

col1 = np.linspace(0.0,1.0,N1)
A = scipy.linalg.toeplitz(col1)

col2 = np.linspace(0.0,1.0,N2)
B = scipy.linalg.toeplitz(col2)

col3 = np.linspace(0.0,1.0,N3)
C = scipy.linalg.toeplitz(col3)

col4 = np.linspace(0.0,1.0,N4)
D = scipy.linalg.toeplitz(col4)

start_time = time.time()
T = np.kron(np.kron(np.kron(A,B),C),D)
end_time = time.time()
print('\nDirect Formation Time:  {:.5f} s'.format(end_time-start_time))

def kron_product(A,B,y):
    A = A.astype(np.float64)
    B = B.astype(np.float64)
    y = y.astype(np.float64)
    input_dim = y.size
    output_dim = A.shape[0]*B.shape[0]
    vals = np.zeros([output_dim])

    A_dim_out = A.shape[0]
    A_dim_in = A.shape[1]
    B_dim_out = B.shape[0]
    B_dim_in = B.shape[1]

    y = np.reshape(y, [B_dim_in, -1], order='F')
    B_times_y = np.matmul(B,y)

    for i in range(0, A_dim_out):
        vals[i*B_dim_out : (i+1)*B_dim_out] = np.sum(A[i,:] * B_times_y[:,:], axis=1)
        #for j in range(0,A_dim_in):
        #    vals[i*B_dim_out : (i+1)*B_dim_out] += A[i,j] * B_times_y[:,j]
    return vals

###
###  REFERENCE: https://arxiv.org/pdf/1601.01507.pdf
###
def kron_vec_trick_2(A,B,y):
    A_dim_out = A.shape[0]
    B_dim_in = B.shape[1]
    y = np.reshape(y, [B_dim_in, A_dim_out], order='F')
    vals = np.matmul(B, np.matmul(y, A.transpose()))
    vals = np.reshape(vals, [-1], order='F')
    return vals

### Three Matrix Implementation
def kron_vec_trick_3(A,B,C,y):

    y = np.reshape(y, [-1, A.shape[0]], order='F')

    intermediate_list = []
    for i in range(0, y.shape[1]):
        intermediate_list.append(kron_vec_trick_2(B,C,y[:,i]))
    intermediate_list = np.array(intermediate_list).transpose()

    vals = np.matmul(intermediate_list, A.transpose())
    vals = np.reshape(vals, [-1], order='F')
    return vals

### Four Matrix Implementation
def kron_vec_trick_4(A,B,C,D,y):

    y = np.reshape(y, [-1, A.shape[0]], order='F')

    intermediate_list = []
    for i in range(0, y.shape[1]):
        intermediate_list.append(kron_vec_trick_3(B,C,D,y[:,i]))
    intermediate_list = np.array(intermediate_list).transpose()

    vals = np.matmul(intermediate_list, A.transpose())
    vals = np.reshape(vals, [-1], order='F')
    return vals

### General Implementation
def kron_vec_trick(matrix_list,y):

    # Remove first matrix from copy of original list
    matrix_list = matrix_list.copy()
    A = matrix_list.pop(0)

    if len(matrix_list) == 1:
        B = matrix_list.pop(0)
        y = np.reshape(y, [B.shape[1], A.shape[0]], order='F')
        vals = np.matmul(B, np.matmul(y, A.transpose()))
        vals = np.reshape(vals, [-1], order='F')
        return vals
    else:
        y = np.reshape(y, [-1, A.shape[0]], order='F')
        intermediate_list = []
        for i in range(0, y.shape[1]):
            intermediate_list.append(kron_vec_trick(matrix_list,y[:,i]))
        intermediate_list = np.array(intermediate_list).transpose()
        vals = np.matmul(intermediate_list, A.transpose())
        vals = np.reshape(vals, [-1], order='F')
        return vals

y = np.random.uniform(0.0,1.0,N)

start_time = time.time()
Ty = np.matmul(T,y)
end_time = time.time()
print('\nDirect Time:  {:.5f} s'.format(end_time-start_time))

start_time = time.time()
#Ty_approx = kron_vec_trick_3(A,B,C,y)
#Ty_approx = kron_vec_trick_4(A,B,C,D,y)
Ty_approx = kron_vec_trick([A,B,C,D],y)
end_time = time.time()
print('\nKernel Trick Time:  {:.5f} s \n'.format(end_time-start_time))

print(np.linalg.norm(Ty_approx-Ty))


import numpy as np

def is_symmetric(a: np.ndarray, rtol: float = 1e-5, atol: float = 1e-8) -> bool:
    """
    Check whether matrix a is symmetric
    :param a: the matrix to be tested
    :type: numpy.ndarray
    :param rtol: relative tolerance
    :type: float
    :param atol: absolute tolerance
    :type: float
    :return: whether matrix a is symmetric
    :rtype: bool
    :raise ValueError: when the numpy array is not a matrix
    """
    if len(a.shape) == 2:
        return np.allclose(a, a.T, rtol=rtol, atol=atol)
    else:
        raise ValueError("The input numpy array must be a matrix")

def is_pos_def(x: np.ndarray) -> bool:
    """
    Check if a matrix is positive definite
    """
    return np.all(np.linalg.eigvals(x) > 0)

def is_spd(x: np.ndarray) -> bool:
    """
    Check if a matrix is symmetric positve definite
    """
    return is_symmetric(x) and is_pos_def(x)


import numpy as np
import numpy.linalg as npla

def pol2cart(pts: np.ndarray) -> np.ndarray:
  """Converts points from polar to cartesian.
  Args:
    pts (np.ndarray): points in polar coordinate [rho, theta, phi]
  Returns:
    np.ndarray: points in cartesian coordinate [x, y, z]
  """
  assert pts.shape[-2:] == (3, 1)
  x = np.cos(pts[..., 2:3, :]) * np.sin(pts[..., 1:2, :]) * pts[..., 0:1, :]
  y = np.sin(pts[..., 2:3, :]) * np.sin(pts[..., 1:2, :]) * pts[..., 0:1, :]
  z = np.cos(pts[..., 1:2, :]) * pts[..., 0:1, :]
  return np.concatenate((x, y, z), axis=-2)

def cart2pol(pts: np.ndarray) -> np.ndarray:
  """Converts points from cartesian to polar.
  Args:
    pts (np.ndarray): points in cartesian coordinate [x, y, z]
  Returns:
    np.ndarray: points in polar coordinate [rho, theta, phi]
  """
  assert pts.shape[-2:] == (3, 1)
  rho = npla.norm(pts, axis=-2, keepdims=True)
  phi = np.arctan2(pts[..., 1:2, :], pts[..., 0:1, :])
  theta = np.arctan2(np.sqrt(pts[..., 0:1, :]**2 + pts[..., 1:2, :]**2), pts[..., 2:3, :])
  return np.concatenate((rho, theta, phi), axis=-2)


from sympy import integer_nthroot
from math import log2, floor
from itertools import combinations

def subset_of(l):
    return sum(map(lambda r: list(combinations(l, r)), range(1, len(l)+1)), [])

class Pippenger:
    def __init__(self, group):
        self.G = group
        self.order = group.order
        self.lamb = group.order.bit_length()

    # Returns g^(2^j)
    def _pow2powof2(self, g, j):
        tmp = g
        for _ in range(j):
            tmp = self.G.square(tmp)
        return tmp

    # Returns Prod g_i ^ e_i
    def multiexp(self, gs, es):
        if len(gs) != len(es):
            raise Exception('Different number of group elements and exponents')

        es = [ei%self.G.order for ei in es]

        if len(gs) == 0:
            return self.G.unit

        lamb = self.lamb
        N = len(gs)
        s = integer_nthroot(lamb//N, 2)[0]+1
        t = integer_nthroot(lamb*N,2)[0]+1
        gs_bin = []
        for i in range(N):
            tmp = [gs[i]]
            for j in range(1,s):
                tmp.append(self.G.square(tmp[-1]))
            gs_bin.append(tmp)
        es_bin = []
        for i in range(N):
            tmp1 = []
            for j in range(s):
                tmp2 = []
                for k in range(t):
                    tmp2.append(int( bin(es[i])[2:].zfill(s*t)[-(j+s*k+1)]) )
                tmp1.append(tmp2)
            es_bin.append(tmp1)

        Gs = self._multiexp_bin(
                [gs_bin[i][j] for i in range(N) for j in range(s)],
                [es_bin[i][j] for i in range(N) for j in range(s)]
                )

        ans2 = Gs[-1]
        for k in range(len(Gs)-2,-1,-1):
            ans2 = self._pow2powof2(ans2, s)
            ans2 = self.G.mult(ans2, Gs[k])

        return ans2

    def _multiexp_bin(self, gs, es):
        assert len(gs) == len(es)
        M = len(gs)
        b = floor( log2(M) - log2(log2(M)) )
        b = b if b else 1
        subsets = [list(range(i,min(i+b,M))) for i in range(0,M,b)]
        Ts = [{sub: None for sub in subset_of(S)} for S in subsets]

        for T,S in zip(Ts, subsets):
            for i in S:
                T[(i,)] = gs[i]
            # Recursively set the subproducts in T
            def set_sub(sub):
                if T[sub] is None:
                    if T[sub[:-1]] is None:
                        set_sub(sub[:-1])
                    T[sub] = self.G.mult(T[sub[:-1]], gs[sub[-1]])
            for sub in T:
                set_sub(sub)

        Gs = []
        for k in range(len(es[0])):
            tmp = self.G.unit
            for T,S in zip(Ts, subsets):
                sub_es = [j for j in S if es[j][k]]
                sub_es = tuple(sub_es)
                if not sub_es:
                    continue
                tmp = self.G.mult(tmp, T[sub_es])
            Gs.append(tmp)

        return Gs


"""
Linear Regression
-----------------
Probabilistic Linear Regression with toy data in 1D.
"""
import numpy as np
import scipy.stats as st
import matplotlib.pyplot as plt

# Generate data
M = 20

X = np.linspace(0, 2, num=M)
# X = np.array([5, 14, 19], dtype=np.float)
t_real = np.sin(X)
t = t_real + np.random.randn(M) * 0.25

plt.scatter(X, t, label='Data points')

# Infer p(t|W,X,alpha) = N(t|XW+b,alpha); the predictive distribution
# MLE for W, b, and beta
W_ml = X.T @ t / (X.T @ X)  # Normal eq.
b_ml = np.mean(t) - W_ml * np.mean(X)

y = X * W_ml + b_ml

alpha_ml = np.mean((t - y)**2)

plt.plot(X, y, color='red', alpha=0.75, label='Regression line')

# Sample from predictive dist.
ys = np.random.normal(y, alpha_ml)

plt.scatter(X, ys, alpha=0.15, label='Posterior samples')
plt.legend(loc='best')
plt.show()


#!/usr/bin/env python

import numpy as np
import random

from q1_softmax import softmax
from q2_sigmoid import sigmoid, sigmoid_grad
from q2_gradcheck import gradcheck_naive

def forward_backward_prop(data, labels, params, dimensions):
    """
    Forward and backward propagation for a two-layer sigmoidal network

    Compute the forward propagation and for the cross entropy cost,
    and backward propagation for the gradients for all parameters.

    Arguments:
    data -- M x Dx matrix, where each row is a training example.
    labels -- M x Dy matrix, where each row is a one-hot vector.
    params -- Model parameters, these are unpacked for you.
    dimensions -- A tuple of input dimension, number of hidden units
                  and output dimension
    """

    ### Unpack network parameters (do not modify)
    ofs = 0
    Dx, H, Dy = (dimensions[0], dimensions[1], dimensions[2])

    W1 = np.reshape(params[ofs:ofs+ Dx * H], (Dx, H))
    ofs += Dx * H
    b1 = np.reshape(params[ofs:ofs + H], (1, H))
    ofs += H
    W2 = np.reshape(params[ofs:ofs + H * Dy], (H, Dy))
    ofs += H * Dy
    b2 = np.reshape(params[ofs:ofs + Dy], (1, Dy))

    ### YOUR CODE HERE: forward propagation
    l1 = sigmoid(np.dot(data, W1) + b1)
    l2 = softmax(np.dot(l1, W2) + b2)
    M, _ = data.shape
    cost = -np.sum(np.log(l2[labels == 1])) / M
    ### END YOUR CODE

    ### YOUR CODE HERE: backward propagation
    # gradient = y - y_head
    gradl2 = (l2 - labels) / M

    # input = gradl2, wight = l1.T
    gradW2 = np.dot(l1.T, gradl2)
    # bias is caused by input from the neuron by fixed activation 1
    gradb2 = np.sum(gradl2, axis=0, keepdims=True)

    # gradient = d(sigmoid(x)) / dx
    gradl1 = np.dot(gradl2, W2.T) * sigmoid_grad(l1)
    gradW1 = np.dot(data.T, gradl1)
    gradb1 = np.sum(gradl1, axis=0, keepdims=True)
    ### END YOUR CODE

    ### Stack gradients (do not modify)
    grad = np.concatenate((gradW1.flatten(), gradb1.flatten(),
        gradW2.flatten(), gradb2.flatten()))

    return cost, grad

def sanity_check():
    """
    Set up fake data and parameters for the neural network, and test using
    gradcheck.
    """
    print "Running sanity check..."

    N = 20
    dimensions = [10, 5, 10]
    data = np.random.randn(N, dimensions[0])   # each row will be a datum
    labels = np.zeros((N, dimensions[2]))
    for i in xrange(N):
        labels[i, random.randint(0,dimensions[2]-1)] = 1

    params = np.random.randn((dimensions[0] + 1) * dimensions[1] + (
        dimensions[1] + 1) * dimensions[2], )

    gradcheck_naive(lambda params:
        forward_backward_prop(data, labels, params, dimensions), params)

def your_sanity_checks():
    """
    Use this space add any additional sanity checks by running:
        python q2_neural.py
    This function will not be called by the autograder, nor will
    your additional tests be graded.
    """
    print "Running your sanity checks..."
    ### YOUR CODE HERE
    # raise NotImplementedError
    ### END YOUR CODE

if __name__ == "__main__":
    sanity_check()
    your_sanity_checks()