# -*- coding: utf-8 -*-
"""
Created on Mon Mar 11 10:36:13 2019

@author: Miguel Angel Velez
"""

import scipy.stats as ss
from mstats import generals, linregr

#------------------------------------------------------------------------------------------------#
#------------------------------------- Confidence Intervals -------------------------------------#
#------------------------------------------------------------------------------------------------#

def norm(n, x_, sd, alpha):
    """
    Function to get the confidence interval to estimate the average population
    when we have large samples (n >= 30).

    Parameters:
    --------------------------
    n : int
        Sample size.

    x_ : float, double
        Sample average.

    sd : float, double
        Sample or population standard deviation.

    alpha : float, double
        Confidence level to estimate our unknown parameter (the average population).
        Each value should be in the range [0, 1].

    Returns:
    --------------------------
    interval : tuple
        The confidence interval, with the minimum value at [0] and the maximum value
        at [1] for the unknown parameter (the average population).
    """
    if (n <= 30):
        print("The sample size must be greater than 30.")
    else:
        sd_error = sd / (n**0.5)
        z_value = ss.norm.interval(alpha)[1]

        minimum = x_ - (z_value * sd_error)
        maximum = x_ + (z_value * sd_error)

        return minimum, maximum

def norm_2p(n1, n2, x_1, x_2, sd1, sd2, alpha):
    """
    Function to get the confidence interval to estimate the difference between
    2 population averages in large samples (n >= 30).

    Parameters:
    --------------------------
    n1 : int
        Sample 1 size.

    n2 : int
        Sample 2 size.

    x_1 : float, double
        Sample 1 average.

    x_2 : float, double
        Sample 2 average.

    sd1 : float, double
        Sample or population 1 standard deviation.

    sd2 : float, double
        Sample or population 2 standard deviation.

    alpha : float, double
        Confidence level to estimate our unknown parameter (the difference between
        2 population averages).
        Each value should be in the range [0, 1].

    Returns:
    --------------------------
    interval : tuple
        The confidence interval, with the minimum value at [0] and the maximum value
        at [1] for the unknown parameter (the difference between
        2 population averages).
    """
    if (n1 <= 30 or n2 <= 30):
        print("The sample sizes must be greater than 30.")
    else:
        sd_error = ((sd1**2 / n1) + (sd2**2 / n2))**0.5
        z_value = ss.norm.interval(alpha)[1]

        minimum = (x_1 - x_2) - (z_value * sd_error)
        maximum = (x_1 - x_2) + (z_value * sd_error)

        return minimum, maximum

def t(n, x_, s, alpha):
    """
    Function to get the confidence interval to estimate the average population
    when we have small samples (n < 30).

    Parameters:
    --------------------------
    n : int
        Sample size.

    x_ : float, double
        Sample average.

    s : float, double
        Sample standard deviation.

    alpha : float, double
        Confidence level to estimate our unknown parameter (the average population).
        Each value should be in the range [0, 1].

    Returns:
    --------------------------
    interval : tuple
        The confidence interval, with the minimum value at [0] and the maximum value
        at [1] for the unknown parameter (the average population).
    """
    if (n > 30):
        print("The sample size must be less than 30.")
    else:
        df = n - 1
        s_error = s / (n**0.5)
        t_value = ss.t.interval(alpha, df)[1]

        minimum = x_ - (t_value * s_error)
        maximum = x_ + (t_value * s_error)

        return minimum, maximum

def t_2p(n1, n2, x_1, x_2, s1, s2, alpha, var):
    """
    Function to get the confidence interval to estimate the difference between
    2 population averages in small samples (n < 30).

    Parameters:
    --------------------------
    n1 : int
        Sample 1 size.

    n2 : int
        Sample 2 size.

    x_1 : float, double
        Sample 1 average.

    x_2 : float, double
        Sample 2 average.

    s1 : float, double
        Sample 1 standard deviation.

    s2 : float, double
        Sample 2 standard deviation.

    alpha : float, double
        Confidence level to estimate our unknown parameter (the difference between
        2 population averages).
        Each value should be in the range [0, 1].

    var : bool
        Are the population variances equal?
        True if population variances are equal.
        False if population variances are different.

    Returns:
    --------------------------
    interval : tuple
        The confidence interval, with the minimum value at [0] and the maximum value
        at [1] for the unknown parameter (the difference between
        2 population averages).
    """
    if (n1 > 30 or n2 > 30):
        print("The sample sizes must be less than 30.")
    else:
        if (var is True):
            df = n1 + n2 - 2
            s2p = (((s1**2) * (n1 - 1)) + (s2**2 * (n2 - 1))) / df
            sp_error = ((s2p / n1) + (s2p / n2))**0.5
            t_value = ss.t.interval(alpha, df)[1]

            minimum = (x_1 - x_2) - (t_value * sp_error)
            maximum = (x_1 - x_2) + (t_value * sp_error)

            return minimum, maximum
        else:
            df = generals.get_df_var(n1, n2, s1, s2)

            sp_error = (((s1**2) / n1) + ((s2**2) / n2))**0.5
            t_value = ss.t.interval(alpha, df)[1]

            minimum = (x_1 - x_2) - (t_value * sp_error)
            maximum = (x_1 - x_2) + (t_value * sp_error)

            return minimum, maximum

def prop(n, p, alpha):
    """
    Function to get the confidence interval to estimate the population proportion.

    Parameters:
    --------------------------
    n : int
        Sample size.

    p : float, double
        Sample proportion.
        Each value should be in the range [0, 1].

    alpha : float, double
        Confidence level to estimate our unknown parameter (the population proportion).
        Each value should be in the range [0, 1].

    Returns:
    --------------------------
    interval : tuple
        The confidence interval, with the minimum value at [0] and the maximum value
        at [1] for the unknown parameter (the population proportion).
    """
    sp_error = ((p * (1 - p)) / n)**0.5
    z_value = ss.norm.interval(alpha)[1]

    minimum = p - (z_value * sp_error)
    maximum = p + (z_value * sp_error)

    return minimum, maximum

def prop_2p(n1, n2, p1, p2, alpha):
    """
    Function to get the confidence interval to estimate the difference between
    2 population proportions.

    Parameters:
    --------------------------
    n1 : int
        Sample 1 size.

    n2 : int
        Sample 2 size.

    p1 : float, double
        Sample 1 proportion.
        Each value should be in the range [0, 1].

    p2 : float, double
        Sample 2 proportion.
        Each value should be in the range [0, 1].

    alpha : float, double
        Confidence level to estimate our unknown parameter (the population proportion).
        Each value should be in the range [0, 1].

    Returns:
    --------------------------
    interval : tuple
        The confidence interval, with the minimum value at [0] and the maximum value
        at [1] for the unknown parameter (the difference between
        2 population proportions).
    """
    sp_error = (((p1 * (1 - p1)) / n1) + ((p2 * (1 - p2)) / n2))**0.5
    z_value = ss.norm.interval(alpha)[1]

    minimum = (p1 - p2) - (z_value * sp_error)
    maximum = (p1 - p2) + (z_value * sp_error)

    return minimum, maximum

def pair_2p(n, cond1, cond2, alpha):
    """
    Function to get the confidence interval to estimate the difference between
    2 population averages in paired samples.

    Parameters:
    --------------------------
    n : int
        Sample sizes.

    cond1 : list
        Sample initial condition list.

    cond2 : list
        Sample final condition list.

    alpha : double, float
        Confidence level to estimate our unknown parameter (the difference between
        2 population averages).
        Each value should be in the range [0, 1].

    Returns:
    --------------------------
    interval : tuple
        The confidence interval, with the minimum value at [0] and the maximum value
        at [1] for the unknown parameter (the difference between
        2 population averages).

    """
    d1 = [cond1[x] - cond2[x] for x in range(n)]
    d21 = [x**2 for x in d1]
    sumd1 = sum(d1)
    sumd21 = sum(d21)
    d_ = sumd1 / n
    sd = ((sumd21 - (n * (d_**2))) / (n - 1))**0.5
    sd_error = sd / (n**0.5)

    if n <= 30:
        df = n - 1
        t_value = ss.t.interval(alpha, df)[1]

        minimum = d_ - (t_value * sd_error)
        maximum = d_ + (t_value * sd_error)

        return minimum, maximum

    else:
        z_value = ss.norm.interval(alpha)[1]

        minimum = d_ - (z_value * sd_error)
        maximum = d_ + (z_value * sd_error)

        return minimum, maximum

def n_avg(alpha, s, error):
    """
    Function to determine the appropiate sample size to estimate the average population.

    Parameters:
    --------------------------
    alpha : float, double
        Confidence level to determine our sample size.
        Each value should be in the range [0, 1].

    s: float, double
        Pilot sample standard deviation, or population standar deviation.

    error: float, double
        Tolerable size error.

    Returns:
    --------------------------
    n : int
        The appropiate sample size to estimate the average population.
        This sample size is rounded.
    """
    z_value = ss.norm.interval(alpha)[1]

    n = ((z_value**2)*(s**2)) / (error**2)

    return round(n)

def n_avg_2p(alpha, s1, s2, error):
    """
    Function to determine the appropiate sample size to estimate the difference
    between 2 average populations.

    Parameters:
    --------------------------
    alpha : float, double
        Confidence level to determine our sample size.
        Each value should be in the range [0, 1].

    s1 : float, double
        Pilot sample 1 standard deviation, or population standar deviation.

    s2 : float, double
        Pilot sample 2 standard deviation, or population standar deviation.

    error: float, double
        Tolerable size error.

    Returns:
    --------------------------
    n : int
        The appropiate sample size to estimate the he difference
        between 2 average populations.
        This sample size is rounded.
    """
    z_value = ss.norm.interval(alpha)[1]

    n = ((z_value**2)*(s1**2 + s2**2)) / (error**2)

    return round(n)

def n_prop(alpha, p, error):
    """
    Function to determine the appropiate sample size to estimate the population proportion.

    Parameters:
    --------------------------
    alpha : float, double
        Confidence level to determine our sample size.
        Each value should be in the range [0, 1].

    p: float, double
        Sample proportion.
        Each value should be in the range [0, 1].

    error: float, double
        Tolerable size error.

    Returns:
    --------------------------
    n : int
        The appropiate sample size to estimate the population proportion.
        This sample size is rounded.
    """
    z_value = ss.norm.interval(alpha)[1]

    n = ((z_value**2) * (p * (1 - p))) / (error**2)

    return round(n)

def n_prop_2p(alpha, p1, p2, error):
    """
    Function to determine the appropiate sample size to estimate the
    difference between 2 population proportions.

    Parameters:
    --------------------------
    alpha : float, double
        Confidence level to determine our sample size.
        Each value should be in the range [0, 1].

    p1: float, double
        Sample proportion 1.
        Each value should be in the range [0, 1].

    p2: float, double
        Sample proportion 2.
        Each value should be in the range [0, 1].

    error: float, double
        Tolerable size error.

    Returns:
    --------------------------
    n : int
        The appropiate sample size to estimate the difference between
        2 population proportions.
        This sample size is rounded.
    """
    z_value = ss.norm.interval(alpha)[1]

    n = ((z_value**2) * ((p1 * (1 - p1)) + ((p2 * (1 - p2))))) / (error**2)

    return round(n)

def f_2p(n1, n2, var1, var2, alpha):
    """
    """
    df1 = n1 - 1
    df2 = n2 - 1
    f_value = ss.f(df1, df2).interval(alpha)

    minimum = var1 / (f_value[1] * var2)
    maximum = var1 / (f_value[0] * var2)

    return minimum, maximum

def beta(x, y, alpha):
    """
    """
    df = len(x) - 1
    t_value = ss.t(df).interval(alpha)[1]

    minimum = linregr.slope(x, y) - (t_value * linregr.get_sb(x, y))
    maximum = linregr.slope(x, y) + (t_value * linregr.get_sb(x, y))

    return minimum, maximum

#------------------------------------------------------------------------------------------------#
#------------------------------------------------------------------------------------------------#
#------------------------------------------------------------------------------------------------#


import numpy
import sympy

print("System of nonlinear equations \n")

m = 0.3
a = 0.5

EPS = 10.0 ** -5

# Исходные уравнения в формате f(x,y) = 0
(x, y) = sympy.symbols("x y")
eq1 = sympy.tan(x * y + m) - x
eq2 = a * (x ** 2) + 2 * (y ** 2) - 1

print(eq1, "= 0")
print(eq2, "= 0")
print()

# Подстановка в исходные уравнения для вычисления значения f(x,y)
def val1(x, y):
    return numpy.tan(x * y + m) - x

def val2(x, y):
    return a * (x ** 2) + 2 * (y ** 2) - 1

# Выраженные из исходных уравнений x = функция(x,y), y = функция(x,y)
def eqx(x, y):
    return numpy.tan(x * y + m)

def eqy(x, y):
    return numpy.sqrt((1 - a * (x ** 2)) / 2)

# Вычисление матрицы Якоби
def W(x, y):
    return numpy.array([
        [(1 + numpy.tan(x * y + m) ** 2) * y - 1, (1 + numpy.tan(x * y + m) ** 2) * x],
        [2 * a * x, 4 * y]
    ])

# Графики исходных уравнений
plots = sympy.plot_implicit(sympy.Eq(eq1, 0), (x, -2, 2), (y, -2, 2), line_color="blue", show=False)
plots.extend(sympy.plot_implicit(sympy.Eq(eq2, 0), (x, -2, 2), (y, -2, 2), line_color="red", show=False))
# plots.show()

iters = 0

def SimpleSolve(x0, y0):
    global iters
    iters = 0
    (x, y) = (x0, y0)
    while True:
        iters += 1
        oldx = x
        oldy = y
        x = eqx(x, y)
        y = eqy(x, y)
        if not (numpy.isfinite(x) and numpy.isfinite(y)):
            raise RuntimeError("Sequence {x} is divergent")
        if max(abs(x - oldx), abs(y - oldy)) < EPS:
            return x, y

def NewtonSolve(x0, y0):
    global iters
    iters = 0
    (x, y) = (x0, y0)
    while True:
        iters += 1
        w = W(x, y)
        f = numpy.array([[val1(x, y)], [val2(x, y)]])
        deltas = numpy.linalg.solve(w, -f)
        x += deltas[0][0]
        y += deltas[1][0]
        if not (numpy.isfinite(x) and numpy.isfinite(y)):
            raise RuntimeError("Sequence {x} is divergent")
        if max(abs(deltas)) < EPS:
            return x, y

x0 = 1.0
y0 = 0.5
print("Initial guess =", (x0, y0))
print()

def run(method):
    global iters
    try:
        (x, y) = method(x0, y0)
        print("(x, y) = ({:.4f}, {:.4f})".format(x, y))
        print(" via {} method (with {} iterations)".format(method.__name__, iters))
    except Exception as ex:
        print("ERROR: {} - in {} method (with {} iterations)".format(ex, method.__name__, iters))
    print()

run(SimpleSolve)
run(NewtonSolve)

plots.show()


"""This module contains plane fitting procedures. This is only needed to *evaluate* RMSE of a plane

"""
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D
import numpy as np

np.set_printoptions(threshold=3600, linewidth=350, precision=6, suppress=True)

# Source: https://stackoverflow.com/a/38770513/9341063

def PCA(data, correlation=False, sort=True):
    """ Applies Principal Component Analysis to the data

    Parameters
    ----------
    data: array
        The array containing the data. The array must have NxM dimensions, where each
        of the N rows represents a different individual record and each of the M columns
        represents a different variable recorded for that individual record.
            array([
            [V11, ... , V1m],
            ...,
            [Vn1, ... , Vnm]])

    correlation(Optional) : bool
            Set the type of matrix to be computed (see Notes):
                If True compute the correlation matrix.
                If False(Default) compute the covariance matrix.

    sort(Optional) : bool
            Set the order that the eigenvalues/vectors will have
                If True(Default) they will be sorted (from higher value to less).
                If False they won't.
    Returns
    -------
    eigenvalues: (1,M) array
        The eigenvalues of the corresponding matrix.

    eigenvector: (M,M) array
        The eigenvectors of the corresponding matrix.

    Notes
    -----
    The correlation matrix is a better choice when there are different magnitudes
    representing the M variables. Use covariance matrix in other cases.

    """

    mean = np.mean(data, axis=0)

    data_adjust = data - mean

    #: the data is transposed due to np.cov/corrcoef syntax
    if correlation:
        matrix = np.corrcoef(data_adjust.T)
    else:
        matrix = np.cov(data_adjust.T)

    eigenvalues, eigenvectors = np.linalg.eig(matrix)

    if sort:
        #: sort eigenvalues and eigenvectors
        sort = eigenvalues.argsort()[::-1]
        eigenvalues = eigenvalues[sort]
        eigenvectors = eigenvectors[:, sort]

    return eigenvalues, eigenvectors

# Source: https://stackoverflow.com/a/38770513/9341063

def best_fitting_plane(points, equation=False):
    """ Computes the best fitting plane of the given points

    Parameters
    ----------
    points: array
        The x,y,z coordinates corresponding to the points from which we want
        to define the best fitting plane. Expected format:
            array([
            [x1,y1,z1],
            ...,
            [xn,yn,zn]])

    equation(Optional) : bool
            Set the oputput plane format:
                If True return the a,b,c,d coefficients of the plane.
                If False(Default) return 1 Point and 1 Normal vector.
    Returns
    -------
    a, b, c, d : float
        The coefficients solving the plane equation.

    or

    point, normal: array
        The plane defined by 1 Point and 1 Normal vector. With format:
        array([Px,Py,Pz]), array([Nx,Ny,Nz])

    """

    w, v = PCA(points)

    #: the normal of the plane is the last eigenvector
    normal = v[:, 2]
    normal = normal / np.linalg.norm(normal)

    #: get a point from the plane
    point = np.mean(points, axis=0)

    if equation:
        a, b, c = normal
        d = -(np.dot(normal, point))
        return a, b, c, d

    else:
        return point, normal

def point_to_plane_distance_simple(points, p_point, p_normal: np.array):
    distance = []
    for i in range(points.shape[0]):
        point = points[i, :]
        signed_distance = p_normal.dot(p_point - point)
        distance.append(signed_distance)
    distance = np.array(distance)
    rmse = np.sqrt(np.mean(distance ** 2))
    return distance, rmse

def calculate_rmse(points: np.array, p_point: np.array, p_normal: np.array):
    points_a = points - p_point
    distance = points_a @ p_normal
    distance = np.array(distance)
    rmse = np.sqrt(np.mean(distance ** 2))
    return distance, rmse

def fit_plane_and_get_rmse(points: np.array):
    """Fit Plane to 3D point and return RMSE

    Arguments:
        points {np.array} -- Point cloud, NX3

    Returns:
        tuple(point, normal, rmse) -- Return a for a point on the plane, its normal, and the RMSE
    """

    point, normal = best_fitting_plane(points)
    distance, rmse = calculate_rmse(points, point, normal)
    return point, normal, distance, rmse

def main():
    direction = 0.7071
    normalizing = np.sqrt(3 * direction * direction)
    unit = direction / normalizing
    a = unit
    b = unit
    c = unit
    d = 1

    noise = 1

    print(f"Ground Truth Normal: [{a:.2f},{b:.2f},{c:.2f}], d: {d}")

    # Create point cloud
    pc_gt = []
    pc_ns = []
    for x in range(-10, 10, 1):
        for y in range(-10, 10, 1):
            true_z = -(a * x + b * y) / c + d
            noise_z = true_z + np.random.rand() * noise
            pc_gt.append([x, y, true_z])
            pc_ns.append([x, y, noise_z])

    pc_gt = np.array(pc_gt)
    pc_ns = np.array(pc_ns)

    # fit plane and get rmse
    point, normal, distance, rmse = fit_plane_and_get_rmse(pc_ns)
    print(f"Predicted Normal: {normal}; point: {point}; RMSE: {rmse:.3f}")

    # plot data
    plt.figure()
    ax = plt.subplot(111, projection='3d')
    ax.scatter(pc_ns[:, 0], pc_ns[:, 1], pc_ns[:, 2], color='b')
    plt.show()

    # norms = np.apply_along_axis(np.linalg.norm, 1, pc_ns)
    # pc_ns_norm = pc_ns / np.expand_dims(norms, axis=1)
    # ax.scatter(pc_ns_norm[:, 0], pc_ns_norm[:, 1], pc_ns_norm[:, 2], color='g')
    # print(pc_ns)

if __name__ == "__main__":
    main()


import numpy as np

class Neural_Network(object):
    def __init__(self):
    #Define Hyper Parameters
        self.inputLayerSize = 2
        self.outputLayerSize = 1
        self.hiddenLayerSize = 3

    #Weights (Parameters)
        self.W1 = np.random.randn(self.inputLayerSize, self.hiddenLayerSize)
        self.W2 = np.random.randn(self.hiddenLayerSize, self.outputLayerSize)

#Forward Propagation
    def forward(self, X):
        #Propagate inputs through network
        #Synapses of ANN multiplies its weight with the input and output the result.
        #So to get the activity of second layer we multiply the input matrix with weight matrix
        #X(1)*W(2) = Z(2)
        #Z(2) is a 3x3 matrix
        self.z2 = np.dot(X, self.W1)
        #Apply activation function to the input in second layer
        #A(2) = 1/1-e^-z(2)
        self.a2 = self.sigmoid(self.z2)
        #A(2)*W(2) = Z(2)
        self.z3 = np.dot(self.a2, self.W2)
        #We apply the activation function on Z(3)
        yHat = self.sigmoid(self.z3)
        #We get a 3x1 matrix as output
        return yHat

    def sigmoid(self, z):
    #Apply sigmoid activation function to scalar, vector, or matrix
        return 1/(1+np.exp(-z))

if __name__ == '__main__':

    # X = (hours sleeping, hours studying), y = Score on test
    X = np.array(([3,5], [5,1], [10,2]), dtype=float)
    y = np.array(([75], [82], [93]), dtype=float)
    X = X/np.amax(X, axis=0)
    y = y/100 #Max test score is 100
    #Create an instance of neural network
    mANN = Neural_Network()

    #Run the forward propagation
    yn = mANN.forward(X)

    #Print the output of neural network
    print("Output of ANN:",yn)

    #Print the original network
    print("Original data:",y)


#!/usr/bin/env python

import sys
import argparse
import numpy as np
import scipy as sp
import scipy.stats

def mean_confidence_interval(data, confidence=0.95):
    a = 1.0*np.array(data)
    n = len(a)
    m, se = np.mean(a), scipy.stats.sem(a)
    h = se * sp.stats.t.ppf((1+confidence)/2., n-1)
    return m, m-h, m+h

parser = argparse.ArgumentParser(description="Compute average and confidence interval")
parser.add_argument('-c', '--col', default=0, type=int, help='Column number - default=0')
parser.add_argument('-i', '--int', default=95, type=float, help='Confidence interval - default=95')

parser.add_argument('file')
args = parser.parse_args()

#i = 0
data=[]
with open(args.file) as file:
    xsum = 0
    xmean = 0
    for  line in file:
        line = line.split()
        data.append(float(line[args.col]))
        #print(i,'  ',data[i])
        #i = i + 1

print(mean_confidence_interval(data,args.int*1.0/100.))


import numpy as np

class KNN:
    def __init__(self, K, metric_type):
        """
        Args:
            K (int): K-value
            metric_type (str): L1, L2, or L-inf
        """
        self.K = K
        self.metric_type = metric_type

    def distance_func(self, vec1, vec2):
        """
        Computes the distance between two d-dimension vectors.

        Args:
            vec1 ((d,) np.ndarray): d-dim vector
            vec2 ((d,)) np.ndarray): d-dim vector
        """
        diff = vec1 - vec2
        distance = 0
        if self.metric_type == "L1":
            # write your code here
            # todo
            distance = np.sum(np.abs(diff))

        elif self.metric_type == "L2":
            # write your code here
            distance = np.sqrt(np.sum(np.square(diff)))
        elif self.metric_type == "L-inf":
            # write your code here
            distance = np.max(np.abs(diff))

        return distance

    def fit(self, x_train, y_train):
        """
        Args:
            x_train ((n,d) np.ndarray): training data with n samples and d features
            y_train ((n,) np.ndarray): training labels
        """
        self.x_train = x_train
        self.y_train = y_train

    def compute_distances_neighbors(self, sample):
        """
        Computes the distance between every data point in the train set and the
        given sample and then finds the k-nearest neighbors.

        Returns a numpy array of the labels of the k-nearest neighbors.

        Args:
            sample ((d,) np.ndarray): the given sample to be computed

        Returns:
            neighbors (list): K-nearest neighbors' labels
        """

        # write your code here
        # todo

        dis = []
        for i in range(self.x_train.shape[0]):
            dis.append(self.distance_func(sample, self.x_train[i, :]))
        labels = []
        index = sorted(range(len(dis)), key=dis.__getitem__)
        for j in range(self.K):
            labels.append(self.y_train[index[j]])
        neighbors = labels

        return neighbors

    @staticmethod
    def majority(neighbors):
        """
        Performs majority voting and returns the predicted value for the test sample.
        Since we're performing binary classification, the possible values are [0,1].

        Args:
            neighbors (list): K-nearest neighbors' labels

        Returns:
            predicted_value (int): the predicted label for the given sample
        """

        # write your code here
        # todo
        bins = np.bincount(neighbors)
        predicted_value = np.argmax(bins)
        return predicted_value

    def predict(self, X_test):
        """
        Computes the predicted values for the entire test set.

        Args:
            x_train ((n,d) np.ndarray): training data with n samples and d features
            y_train ((n,) np.ndarray): training labels
            X_test ((n,d) np.ndarray): test data

        Returns:
            pred_test ((n,) np.ndarray): output for every entry in the test set
        """
        pred_test = np.array(
            [self.majority(self.compute_distances_neighbors(sample)) for sample in X_test])

        return pred_test

def accuracy_score(pred, y):
    """
    Computes the accuracy of the predicted data.

    Args:
        pred ((n,) np.ndarray): predicted values for n samples
        y ((n,) np.ndarray): labels for n samples

    Returns:
        acc (float): accuracy
    """

    acc = np.mean(pred == y)
    # print(acc)
    return acc

def main():
    n, m, d = input().split()
    x_train = []
    x_test = []
    y_train = []
    y_test = []
    for _ in range(int(n)):
        split_line = input().split()
        X = split_line[:int(d)]
        y = split_line[-1]
        x_train.append([float(x) for x in X])
        y_train.append(int(y))
    for _ in range(int(m)):
        split_line = input().split()
        X = split_line[:int(d)]
        y = split_line[-1]
        x_test.append([float(x) for x in X])
        y_test.append(int(y))
    x_train = np.array(x_train)
    x_test = np.array(x_test)
    y_train = np.array(y_train)
    y_test = np.array(y_test)

    metric_types = ["L1", "L2", "L-inf"]
    params = [(x, y) for x in range(1, 6, 1) for y in metric_types]
    value = -100
    list = []
    for param in params:
        model = KNN(param[0], param[1])
        model.fit(x_train, y_train)
        pred = model.predict(x_test)
        acc = accuracy_score(pred, y_test)
        # print(param)
        # print(acc)
        if acc > value:
            list.clear()
            list.append(param)
            value = acc
            continue
        elif acc == value:
            list.append(param)
            continue
        else:
            continue
    b = np.array(list)
    a = np.sort(b, kind='stable')
    for _ in a:
        print(_[0], end=" ")
        print(_[1])
    # print(1)

if __name__ == '__main__':
    main()


"""
Stick Breaking Construction
---------------------------
Generative story of Stick Breaking process
"""
import numpy as np

alpha = 1  # dirichlet prior
n_clusters = 10  # max num of clusters

G = lambda: np.random.normal()  # base distribution, theta ~ G

thetas = np.zeros(n_clusters)  # cluster params
phis = np.zeros(n_clusters)  # cluster mixing weights

prev_v = 1

for i in range(n_clusters):
    # v ~ Beta(1, alpha)
    v = np.random.beta(1, alpha)
    # theta ~ G
    theta = G()

    thetas[i] = theta
    phis[i] = v * prev_v  # v_1 * \prod_{i>1} v_i

    prev_v *= (1-v)

# Check: phis should sum to one as n_clusters -> inf
print(np.sum(phis))


import numpy as np
from numpy.testing import assert_almost_equal
from scipy.stats import t as stats_t
from scipy.stats import ttest_ind

def make_t_test(na: int, nb: int):
    """
    Returns a t-test that takes the sample mean and variance for a list of sample points from A,
    and those for a list of sample points for B.

    Python implementation of the univariate t-test from the work of Schneider & Moradi (2016):
        "Leakage assessment methodology: Extended version"
    """
    def welch_t_test(ma: np.array, va: np.array, mb: np.array, vb: np.array):
        m = ma - mb

        sa = va / na
        sb = vb / nb

        sab = sa + sb
        t = m / np.sqrt(sab)

        # Approximation from paper.
        dof = na + nb

        # Student's t CDF.
        p = 2 * stats_t(df=dof).cdf(-np.abs(t))

        return t, p

    return welch_t_test

if __name__ == '__main__':
    # Test case, comparing the Schneider variant of the t-test to the t-test implementation of scipy.

    def gen_example(mean, trace_num, trace_len=1400):
        return np.random.normal(mean, 2.2, size=(trace_num, trace_len)).astype(int)

    def get_mv(x: np.array):
        return np.array((x.mean(axis=0), x.var(axis=0)))

    num_traces = 10000

    ex_a = gen_example(2, num_traces)
    ex_b = gen_example(2, num_traces)

    ex_a_s = get_mv(ex_a).shape

    res_sp = ttest_ind(ex_a, ex_b, axis=0, equal_var=False)[0]

    test = make_t_test(num_traces, num_traces)
    res_custom = test(*get_mv(ex_a), *get_mv(ex_b))[0]

    assert_almost_equal(res_custom, res_sp, decimal=3)


"""
CS 156a: Homework #2
Anthony Bartolotta
Other
"""
import numpy as np

def pseudoInverse(X):
    tempM = np.linalg.inv(np.dot(np.transpose(X), X))
    xPseudo = np.dot(tempM, np.transpose(X))
    return xPseudo

def generateClassifyingLine():
    # Generate two random points in [-1,1]x[-1,1]
    [p1,p2] = np.random.uniform(-1,1,[2,2])
    # Find normal vector to line passing between points
    # This defines the positive direction.
    vec12 = p1 - p2
    rotation = np.array([[0,1],[-1,0]])
    normalVec = np.dot(rotation, vec12)
    nVec = normalVec / np.linalg.norm(normalVec)
    # Define 1+2 dimensional classification vector
    cVec = np.concatenate(([-np.vdot(normalVec, p1)], nVec))
    return cVec

def generateData(nSamples, cVec):
    # Generate points in [-1,1]x[-1,1]
    points = np.random.uniform(-1,1,[nSamples,2])
    # Zeroth component is always 1
    ones = np.ones([nSamples,1])
    vecs = np.concatenate((ones, points),1)
    # Classify each point
    classifications = np.sign(np.dot(vecs, cVec))
    data = [classifications, vecs]
    return data

def inSampleError(data, wVec):
    [trueClass, inSampleVecs] = data
    # Classify in sample points
    wClass = np.sign(np.dot(inSampleVecs, wVec))
    # Find fraction of mismatch
    fMismatch = float(sum(trueClass != wClass)) / len(trueClass)
    return fMismatch

def outSampleError(cVec, wVec, nOutSample):
    # Generate new out-of-sample points
    testPoints = np.random.uniform(-1,1,[nOutSample,2])
    ones = np.ones([nOutSample,1])
    testVecs = np.concatenate((ones, testPoints),1)
    # Classify these points using the target function
    trueClass = np.sign(np.dot(testVecs, cVec))
    # Classify these points using the learned function
    learnClass = np.sign(np.dot(testVecs, wVec))
    # Return fraction of mismatch
    fMismatch = float(sum(trueClass != learnClass)) / nOutSample
    return fMismatch

def perceptronLearning(data, wVec):
    [trueClassVec, sampleVecs] = data
    # Classify all data points
    percClassVec = np.sign(np.dot(sampleVecs, wVec))
    # See if any points are misclassified
    misclassVec = (percClassVec != trueClassVec)
    misclassified = any(misclassVec)
    # Randomly select a misclassified point if it exists
    if misclassified:
        misclassPoints = sampleVecs[misclassVec]
        misclassClasses = trueClassVec[misclassVec]
        rInt = np.random.randint(0, len(misclassPoints))
        rPoint = misclassPoints[rInt]
        rClass = misclassClasses[rInt]
        # Update perceptron vector
        wVec = wVec + rClass*rPoint
    return [misclassified, wVec]

def linearRegression(data):
    [trueClassVec, inSampleVecs] = data
    # Calculate pseudo-inverse
    xPseudo = pseudoInverse(inSampleVecs)
    # Perform linear regression
    wVec = np.dot(xPseudo, trueClassVec)
    return wVec

def perceptron(data):
    # Initialize weight vector
    wVec = np.array([0,0,0])
    # Begin Learning iterations
    misclassified = True
    steps = 0
    while misclassified:
        [misclassified, wVec] = perceptronLearning(data, wVec)
        steps +=1
    return [steps, wVec]

def perceptronWithLinearStart(data):
    # Use linear regression to initialize the weight vector
    wVec = linearRegression(data)
    # Begin Learning iterations
    misclassified = True
    steps = 0
    while misclassified:
        [misclassified, wVec] = perceptronLearning(data, wVec)
        steps +=1
    return [steps, wVec]

def individualTrial(nSamples, nOutSample):
    # Generate line for classification
    cVec = generateClassifyingLine()

    # Generate random data points and classify
    data = generateData(nSamples, cVec)
    [trueClassVec, inSampleVecs] = data

    # Use linear regression to approximate target function
    wVecLin = linearRegression(data)
    # Calculate linear regression in-sample error
    eInLin = inSampleError(data, wVecLin)
    # Calculate linear regression out-of-sample error
    eOutLin = outSampleError(cVec, wVecLin, nOutSample)

    # Use PLA to approximate target function
    [stepsPLA, wVecPLA] = perceptron(data)
    # Calculate PLA in-sample error
    eInPLA = inSampleError(data, wVecPLA)
    # Calculate PLA out-of-sample error
    eOutPLA = outSampleError(cVec, wVecPLA, nOutSample)

    # Use PLA w/ Lin. Reg. to approximate target function
    [stepsLinPLA, wVecLinPLA] = perceptron(data)
    # Calculate PLA w/ Lin. Reg. in-sample error
    eInLinPLA = inSampleError(data, wVecLinPLA)
    # Calculate PLA w/ Lin. Reg. out-of-sample error
    eOutLinPLA = outSampleError(cVec, wVecLinPLA, nOutSample)

    return [eInLin, eOutLin, stepsPLA, eInPLA, eOutPLA,
                                    stepsLinPLA, eInLinPLA, eOutLinPLA]

def main(nTrials, nSamples, nOutSample):
    eInLinArray = np.zeros([nTrials])
    eOutLinArray = np.zeros([nTrials])
    stepsPLAArray = np.zeros([nTrials])
    eInPLAArray = np.zeros([nTrials])
    eOutPLAArray = np.zeros([nTrials])
    stepsLinPLAArray = np.zeros([nTrials])
    eInLinPLAArray = np.zeros([nTrials])
    eOutLinPLAArray = np.zeros([nTrials])
    for j in range(nTrials):
        returnData = individualTrial(nSamples, nOutSample)
        eInLinArray[j] = returnData[0]
        eOutLinArray[j] = returnData[1]
        stepsPLAArray[j] = returnData[2]
        eInPLAArray[j] = returnData[3]
        eOutPLAArray[j] = returnData[4]
        stepsLinPLAArray[j] = returnData[5]
        eInLinPLAArray[j] = returnData[6]
        eOutLinPLAArray[j] = returnData[7]
    avg_eInLin = np.mean(eInLinArray)
    avg_eOutLin = np.mean(eOutLinArray)
    avg_stepsPLA = np.mean(stepsPLAArray)
    avg_eInPLA = np.mean(eInPLAArray)
    avg_eOutPLA = np.mean(eOutPLAArray)
    avg_stepsLinPLA = np.mean(stepsLinPLAArray)
    avg_eInLinPLA = np.mean(eInLinPLAArray)
    avg_eOutLinPLA = np.mean(eOutLinPLAArray)
    print("Number of Learning Samples = " + repr(nSamples))
    print("For Linear Regression, average E_in = " + repr(avg_eInLin))
    print("For Linear Regression, average E_out = " + repr(avg_eOutLin))
    print("For PLA, average steps = " + repr(avg_stepsPLA))
    print("For PLA, average E_in = " + repr(avg_eInPLA))
    print("For PLA, average E_out = " + repr(avg_eOutPLA))
    print("For Lin Reg w PLA, average steps = " + repr(avg_stepsLinPLA))
    print("For Lin Reg w PLA, average E_in = " + repr(avg_eInLinPLA))
    print("For Lin Reg w PLA, average E_out = " + repr(avg_eOutLinPLA))
    return

main(1000, 10, 10000)
main(1000, 100, 10000)


import logging

import numpy as np

import lin_prog as lp

logging.basicConfig(format='%(asctime)s %(levelname)s: %(message)s',
                    level=logging.DEBUG)

# 6.1
A = np.array([[  -1,  -6,   1,   3],
              [  -1,  -2,   7,   1],
              [   0,   3, -10,  -1],
              [  -6, -11,  -2,  12],
              [   1,   6,  -1,  -3],
             ])
b = np.array([[ -3],
              [  5],
              [ -8],
              [ -7],
              [  4],
             ])
c = np.array([[  47],
              [  93],
              [  17],
              [ -93],
             ])
x = np.array([[ 1],
              [ 1],
              [ 1],
              [ 1],
             ])

J = lp.active_constraints(A, b, x)
z = lp.get_dual_vars(A, c, J)
print "b - Ax =\n", b - np.dot(A, x)
print "Dual variables:\n", z

# 6.2
A = np.array([[ -1,  0,  0],
              [  0, -1,  0],
              [  0,  0, -1],
              [  1,  0,  0],
              [  0,  1,  0],
              [  0,  0,  1],
              [  1,  1,  1]
             ])
b = np.array([[ 0],
              [ 0],
              [ 0],
              [ 2],
              [ 2],
              [ 2],
              [ 4],
             ])
c = np.array([[  1],
              [  1],
              [ -1],
             ])
x_1 = np.array([[ 2],
                [ 2],
                [ 0],
               ])

x_s = lp.simplex(A, b, c, x_i=x_1, p_rule='bland', verbose=True)


import math
import numpy as np
from dezero import Function, Variable
from dezero.utils import plot_dot_graph

class Sin(Function):
    def forward(self, x):
        y = np.sin(x)
        return y

    def backward(self, gy):
        x = self.inputs[0].data
        gx = gy * np.cos(x)
        return gx

def sin(x):
    return Sin()(x)

def my_sin(x, threshold=0.0001):
    y = 0
    for i in range(100000):
        c = (-1) ** i / math.factorial(2 * i + 1)
        t = c * x ** (2 * i + 1)
        y = y + t
        if abs(t.data) < threshold:
            break
    return y

x = Variable(np.array(np.pi/4))
#y = sin(x)
y = my_sin(x)
y.backward()

x.name = 'x'
y.name = 'y'
plot_dot_graph(y, verbose=False, to_file='my_sin.png')
#print(y.data)
#print(x.grad)


# %% [markdown]
'''
#  Fitting geometric models with gradient descent
'''
# %% [markdown]
'''
The process of fitting known geometric models to scattered data is well known and resolved since a long time.
Indeed, Legendre was one of the first to use least-square to do such tasks legendre1805nouvelles,
when he wanted to fit an equation for the shape of the earth !

The idea is quite simple: knowing the model definition, it is possible to estimate its best parameters
that fits well to the input noisy data.
'''
# %% [markdown]
'''
## tl;dr
1. Mathematical model definition for your data
2. Cost function
3. Optimisation (gradient descent with cost function derivative on all the parameters)
4. Hyper-parameters optimisation with training set (learning rate)
5. Model bias with testing set
'''
# %% [markdown]
'''
## 1. Optimisation
'''
# %% [markdown]

# Given a mathematical model with parameters $\Theta$, we can define its minimization function $\xi(\Theta)$ (the inverse likelihood).
# This represents the fitness of the model given some data, so if $\xi(\Theta)$ is minimum, then the parameters are best suited to the data.

# To optimize the parameters, one could check every possible parameters $\Theta_i$ and compute directly $\xi(\Theta_i)$.
# Unfortunately, with our current compute power, this is time consuming.
# Especially if we have more than one parameter, then the manifold would be to large to explore.
# Imagine the difference between exploring a one dimension line vs exploring a 3D surface!

#

# This is why it is important to have a dynamic strategy to find the local minimum, and this can be done using gradients.
# We can calculate the gradient of the energy function among our parameters:
# \begin{equation}
# \frac{\delta \xi(\Theta)}{\delta \Theta} = \nabla\xi(\Theta)
# \end{equation}

# The gradient of the energy function can then be used to update the parameter every step, and converge to a local minima:
# \begin{equation}
# \Theta(t+1) = \Theta(t) - \mu\nabla\xi(\Theta(t))
# \end{equation}

# One could compute directly the optimal solution by equalizing the gradient of energy function with 0:
# \begin{equation}
# \frac{\delta \xi(\Theta)}{\delta \Theta} = 0
# \end{equation}

# When this method works well with little data and few parameters, it can be computationnaly impossible to compute the best solution [see this thread for more information](https://stats.stackexchange.com/questions/278755/why-use-gradient-descent-for-linear-regression-when-a-closed-form-math-solution).
# Morevover, it is not always easy to mathematically extract the optimal parameters.

# %% [markdown]
'''
## 2. Example with sphere fitting
'''
# %% [markdown]
'''
### 2.1 Cost function
'''
# %% [markdown]

# To extract the cost function, we first need to define what is a sphere.
# The equation of a 3D sphere with radius $r$ and center $(a, b, c)$ is known as:
# \begin{equation}
# r^2 = (x-a)^2 + (y-b)^2 + (z-c)^2
# \end{equation}

# Every point $(x,y,z)$ must satisfy this equation to be on the sphere, if not, the point is outside the sphere.
# Using this condition, we can deduce the energy function:
# \begin{equation}
# \xi(\Theta) = \sum_i^n(L_i - r)^2
# \end{equation}
# with,
# \begin{equation}
# L_i = \sqrt{(x_i-a)^2 + (y_i-b)^2 + (z_i-c)^2}
# \end{equation}
# This function will verify if every known point $i$ fits well with the parameters $a, b, c$ and $r$.
# Let's implement it in python !
# %%
## imports
import numpy as np
import math
import plotly.graph_objects as go

# fixing numpy random state for reproducibility
np.random.seed(0)
# %%
def sph_loss(T, x):
    L = np.sqrt((x[:,0] - T[0])**2 + (x[:,1] - T[1])**2 + (x[:,2] - T[2])**2)
    return L
# %%
def cost_function(T, x):
    L = sph_loss(T, x)
    return np.sum( (L - T[3])**2 )
# %% [markdown]
'''
### 2.2 Gradient of the cost function
'''
# %% [markdown]

# Now that we have the cost funtion, we can compute its gradient for every parameters (a, b, c, r).
# \begin{equation}
# \nabla\xi(\Theta) =
#   \begin{bmatrix}
#   \frac{\delta \xi(\Theta)}{\delta r} \\
#     \frac{\delta \xi(\Theta)}{\delta a} \\
#     \frac{\delta \xi(\Theta)}{\delta b} \\
#     \frac{\delta \xi(\Theta)}{\delta c}
#   \end{bmatrix}
# \end{equation}

# Then, with $m$ as the number of 3D points:

# \begin{equation}
# \frac{\delta \xi(\Theta)}{\delta r} = -2\sum_{i=1}^m (L_i -r)
# \end{equation}

# \begin{equation}
# \frac{\delta \xi(\Theta)}{\delta a} = 2\sum_{i=1}^{m}((x_i-a) + r\frac{\delta L_i}{\delta a}); \qquad  \frac{\delta L_i}{\delta a} = \frac{a-x_i}{L_i}
# \end{equation}

# \begin{equation}
# \frac{\delta \xi(\Theta)}{\delta b} = 2\sum_{i=1}^{m}((y_i-b) + r\frac{\delta L_i}{\delta b}); \qquad  \frac{\delta L_i}{\delta b} = \frac{b-y_i}{L_i}
# \end{equation}

# \begin{equation}
# \frac{\delta \xi(\Theta)}{\delta c} = 2\sum_{i=1}^{m}((z_i-c) + r\frac{\delta L_i}{\delta c}); \qquad  \frac{\delta L_i}{\delta c} = \frac{c-z_i}{L_i}
# \end{equation}

# %% [markdown]
# In python,

# %%
## cost function derivative
def derivative_cost_function(T, x):
    L = sph_loss(T, x)

    dr = (-1)*(-2)*np.sum( (L - T[3]) )

    dLa = (T[0] - x[:,0])/L
    da = 2*np.sum( (x[:,0] - T[0]) + T[3]*dLa )

    dLb = (T[1] - x[:,1])/L
    db = 2*np.sum( (x[:,1] - T[1]) + T[3]*dLb )

    dLc = (T[2] - x[:,2])/L
    dc = 2*np.sum( (x[:,2] - T[2]) + T[3]*dLc )

    return np.array([da, db, dc, dr])
# %% [markdown]
'''
### 2.3 Gradient descent
'''
# %% [markdown]
'''
Using the gradient of the cost function, it is now possible to optimize the best parameters with gradient descent.
'''
# %%
def grad_descent(data, param_init):
    T = param_init #initial guess
    lr = 5e-3
    it_max = 10000
    grad = [1e99, 1e99, 1e99]

    for it in range(it_max):
        if( abs(np.sum(grad))<1e-6 ):
            continue
        # gradient descent
        grad = derivative_cost_function(T, data)
        T = T + lr*grad
        it_max = it
    print("Done in %d epochs with grad: [%1.4e, %1.4e, %1.4e, %1.4e]" %(it_max, grad[0], grad[1], grad[2], grad[3]))

    return T, grad

# %% [markdown]
'''
### 2.4 Training phase
'''
# %% [markdown]
'''
We will first generated data for a standard 3D sphere, we can use spherical coordinates to sample random points given the sphere parameters. To reduce CPU time, we will not use a high number of points.
'''
# %%
def gen_points_from_sph(model, n):
    res = int(np.ceil(math.sqrt(n)))
    theta = np.linspace(0,2*np.pi,res)
    phi = np.linspace(0,np.pi,res)

    x = model[3]*np.outer(np.cos(theta),np.sin(phi))
    y = model[3]*np.outer(np.sin(theta),np.sin(phi))
    z = model[3]*np.outer(np.ones(res),np.cos(phi))

    X = x + model[0]
    Y = y + model[1]
    Z = z + model[2]

    points = np.zeros((n, 3))
    points[:,0] = X.ravel()[0:n]
    points[:,1] = Y.ravel()[0:n]
    points[:,2] = Z.ravel()[0:n]

    return points, X, Y, Z
# %% [markdown]
'''
Let's generate 250 points with gaussian noise, from an unknown sphere centered at (1,2,4) and 10 radius.
Then, we will use 2/3 of the data and try to find these parameters. Remember that the hyper-parameters for the gradient descent are supposed to be optimized during training set.
'''
# %%
#Training phase
sph_model = [1, 2, 4, 10]
n = 150
sph_points = gen_points_from_sph(sph_model, n)[0] + np.random.randn(n,3)
sph_points_train = sph_points[0:int(n/3),:]

param_init = np.array([0, 0, 0, 10]) #initial guess

param = grad_descent(sph_points_train, param_init)[0]

# We use the formula for the radius
print(param)
# %% [markdown]
'''
The optimization returned a sphere centered at (1.21, 2.09, 3.94) with 9.94 radius.
'''
# %% [markdown]
'''
### 2.5 Testing phase
'''
# %% [markdown]

# Now that we have an estimation on the model, we can test it and see how weel this model fits to the data. We use the 1/3 remaining points to estimate the error of the model.

# The error estimation can be done using the fitting function $\xi(\Theta)$.

# %%
#Testing phase
sph_points_test = sph_points[int(n/3)::,:]
model_error = cost_function(sph_model, sph_points_test)/sph_points_test.shape[0]

print("Model has an error of %.2f per point"%model_error)
# %% [markdown]
'''
The error per point is quite high. But considering the noise that was introduced, we see that the algorithm performs quite well.
'''
# %%
model_error = cost_function(param, sph_points_test)/sph_points_test.shape[0]

print("Noise of the data %.2f per point"%model_error)

# %% [markdown]
'''
This is using these error can we can optimized the hyper-parameters (optimization parameters), to find which one are best suited for the data.
'''
# %% [markdown]
'''
### 2.6 Qualitative result
'''
# %% [markdown]
'''
We will use [plotly](https://plot.ly/python/) to render the result.
'''
# %%
## Qualitative results with plotly
_,X, Y, Z = gen_points_from_sph(param, 5*n)

trace = go.Surface(
            x=X,
            y=Y,
            z=Z,
            showscale=False,
            opacity = 0.5,
            colorscale=[[0, 'rgb(50,50,125)'], [1, 'rgb(50,50,125)']])

trace2=go.Scatter3d(
           x=sph_points[:,0].ravel(),
           y=sph_points[:,1].ravel(),
           z=sph_points[:,2].ravel(),
           mode='markers',
           name='points',
           marker=go.scatter3d.Marker(symbol='circle',
                            size=3,
                            color='rgb(0,0,255)',
                            opacity=1),)

layout = go.Layout(
            title="Sphere fitting",
            scene=go.layout.Scene(
                aspectmode = "data",
                xaxis_title="x(mm)",
                yaxis_title="y(mm)",
                zaxis_title="z(mm)",
                camera=dict(center=dict(x=0.1, y=0.1, z=0))))

fig = go.Figure(data=[trace, trace2], layout=layout)
fig.show(renderer="iframe_connected", config={'showLink': False})
# %% [markdown]
'''
## To go further
'''
# %% [markdown]
'''
You can look at other examples for standard mathematical models [here](https://www.geometrictools.com/Documentation/LeastSquaresFitting.pdf).
'''
# %% [markdown]
'''
## Tags
'''
# %% [markdown]
'''
Data-Science; Geometry; Optimization
'''


import numpy as np

def JacobiSolve(A,b,tol=1.0e-6,max_iterations=100,LOUD=False):
    """Solve a linear system by Jacobi iteration.
    Note: system must be diagonally dominant
    Args:
        A: N by N array
        b: array of length N
        tol: Relative L2 norm tolerance for convergence
        max_iterations: maximum number of iterations
    Returns:
        The approximate solution to the linear system
    """
    [Nrow, Ncol] = A.shape
    assert Nrow == Ncol
    N = Nrow
    converged = False
    iteration = 1
    x = np.random.rand(N) #random initial guess
    x_new = np.zeros(N)
    while not(converged):
        x = x_new.copy() #replace old value
        x_new *= 0 #reset x_new
        for row in range(N):
            x_new[row] = b[row]
            for column in range(N):
                if column != row:
                    x_new[row] -= A[row,column]*x[column]
            x_new[row] /= A[row,row]
        relative_change = np.linalg.norm(x_new-x)/np.linalg.norm(x_new)
        if (LOUD):
            print("Iteration",iteration,": Relative Change =",relative_change)
        if (relative_change < tol) or (iteration >= max_iterations):
            converged = True
        iteration += 1
    return x_new

def JacobiSolve_Short(A,b,tol=1.0e-6,max_iterations=100,LOUD=False):
    """Solve a linear system by Jacobi iteration.
    This implementation removes the for loops to make it faster
    Note: system must be diagonally dominant
    Args:
        A: N by N array
        b: array of length N
        tol: Relative L2 norm tolerance for convergence
        max_iterations: maximum number of iterations
    Returns:
        The approximate solution to the linear system
    """
    [Nrow, Ncol] = A.shape
    assert Nrow == Ncol
    N = Nrow
    converged = False
    iteration = 1
    x = np.random.rand(N) #random initial guess
    x_new = np.zeros(N)
    while not(converged):
        x = x_new.copy() #replace old value
        x_new *= 0 #reset x_new
        #update is (b - whole row * x + diagonal part * x)/diagonal
        x_new = (b - np.dot(A,x)+ A.diagonal()*x)/A.diagonal()
        relative_change = np.linalg.norm(x_new-x)/np.linalg.norm(x_new)
        if (LOUD):
            print("Iteration",iteration,": Relative Change =",relative_change)
        if (relative_change < tol) or (iteration >= max_iterations):
            converged = True
        iteration += 1
    return x_new

def Gauss_Seidel_Solve(A,b,tol=1.0e-6,max_iterations=100,LOUD=False):
    """Solve a linear system by Gauss-Seidel iteration.
    Note: system must be diagonally dominant
    Args:
        A: N by N array
        b: array of length N
        tol: Relative L2 norm tolerance for convergence
        max_iterations: maximum number of iterations
    Returns:
        The approximate solution to the linear system
    """
    [Nrow, Ncol] = A.shape
    assert Nrow == Ncol
    N = Nrow
    converged = False
    iteration = 1
    x = np.random.rand(N) #random initial guess
    x_new = np.zeros(N)
    while not(converged):
        x = x_new.copy() #replace old value
        for row in range(N):
            x_new[row] = b[row]
            for column in range(N):
                if column != row:
                    #only change from before is that I use x_new in the update
                    x_new[row] -= A[row,column]*x_new[column]
            x_new[row] /= A[row,row]
        relative_change = np.linalg.norm(x_new-x)/np.linalg.norm(x_new)
        if (LOUD):
            print("Iteration",iteration,": Relative Change =",relative_change)
        if (relative_change < tol) or (iteration >= max_iterations):
            converged = True
        iteration += 1
    return x_new

def SOR_Solve(A,b,tol=1.0e-6,omega=1,max_iterations=100,LOUD=False):
    """Solve a linear system by Gauss-Seidel iteration with SOR.
    Note: system must be diagonally dominant
    Args:
        A: N by N array
        b: array of length N
        tol: Relative L2 norm tolerance for convergence
        omega: the over-relaxation parameter
        max_iterations: maximum number of iterations
    Returns:
        The approximate solution to the linear system
    """
    [Nrow, Ncol] = A.shape
    assert Nrow == Ncol
    N = Nrow
    converged = False
    iteration = 1
    x = np.random.rand(N) #random initial guess
    x_new = np.zeros(N)
    while not(converged):
        x = x_new.copy() #replace old value
        for row in range(N):
            x_new[row] = b[row]
            for column in range(N):
                if column != row:
                    x_new[row] -= A[row,column]*x_new[column]
            x_new[row] /= A[row,row]
            x_new[row] = (1.0-omega) * x[row] + omega*x_new[row]
        relative_change = np.linalg.norm(x_new-x)/np.linalg.norm(x_new)
        if (LOUD):
            print("Iteration",iteration,": Relative Change =",relative_change)
        if (relative_change < tol) or (iteration >= max_iterations):
            converged = True
        iteration += 1
    return x_new

def SOR_Solve_Opt(A,b,tol=1.0e-6,max_iterations=100,LOUD=False):
    """Solve a linear system by Gauss-Seidel iteration with SOR and automatic \omega
    Note: system must be diagonally dominant
    Args:
        A: N by N array
        b: array of length N
        tol: Relative L2 norm tolerance for convergence
        max_iterations: maximum number of iterations
    Returns:
        The approximate solution to the linear system
    """
    [Nrow, Ncol] = A.shape
    assert Nrow == Ncol
    N = Nrow
    converged = False
    iteration = 1
    omega = 1
    l = 5
    p = 2
    x = np.random.rand(N) #random initial guess
    x_new = np.zeros(N)
    while not(converged):
        x = x_new.copy() #replace old value
        for row in range(N):
            x_new[row] = b[row]
            for column in range(N):
                if column != row:
                    x_new[row] -= A[row,column]*x_new[column]
            x_new[row] /= A[row,row]
            x_new[row] = (1.0-omega) * x[row] + omega*x_new[row]
        relative_change = np.linalg.norm(x_new-x)/np.linalg.norm(x_new)
        #record change after iteration k
        if (l==iteration):
            dxl = np.linalg.norm(x_new-x)
        if (l + p == iteration):
            dxlp = np.linalg.norm(x_new-x)
            omega = 2.0/(1.0+np.sqrt(1-(dxlp/dxl)**(1.0/p)))
        if (LOUD):
            print("Iteration",iteration,": Relative Change =",relative_change)
        if (relative_change < tol) or (iteration >= max_iterations):
            converged = True
        iteration += 1
    return x_new

def CG(A,b,tol=1.0e-6,max_iterations=100,LOUD=False):
    """Solve a linear system by Conjugate Gradient
    Note: system must be SPD
    Args:
        A: N by N array
        b: array of length N
        tol: Relative L2 norm tolerance for convergence
        max_iterations: maximum number of iterations
    Returns:
        The approximate solution to the linear system
    """
    [Nrow, Ncol] = A.shape
    assert Nrow == Ncol
    N = Nrow
    converged = False
    iteration = 1
    x = np.random.rand(N) #random initial guess
    r = b - np.dot(A,x)
    s = r.copy()
    while not(converged):
        denom = np.dot(s, np.dot(A,s))
        alpha = np.dot(s,r)/denom
        x = x + alpha*s
        r = b - np.dot(A,x)
        beta = - np.dot(r,np.dot(A,s))/denom
        s = r + beta * s
        relative_change = np.linalg.norm(r)
        if (LOUD):
            print("Iteration",iteration,": Relative Change =",relative_change)
        if (relative_change < tol) or (iteration >= max_iterations):
            converged = True
        iteration += 1
    return x

def JacobiTri(A,b,tol=1.0e-6,max_iterations=100,LOUD=False):
    """Solve a linear system by Jacobi iteration.
    Note: system must be diagonally dominant
    Args:
        A: N by N array
        b: array of length N
        tol: Relative L2 norm tolerance for convergence
        max_iterations: maximum number of iterations
    Returns:
        The approximate solution to the linear system
    """
    [Nrow, Ncol] = A.shape
    assert 3 == Ncol
    N = Nrow
    converged = False
    iteration = 1
    x = np.random.rand(N) #random initial guess
    x_new = np.zeros(N)
    while not(converged):
        x = x_new.copy() #replace old value
        for i in range(1,N-1):
            x_new[i] = (b[i] - A[i,0]*x[i-1] - A[i,2]*x[i+1])/A[i,1]
        i = 0
        x_new[0] = (b[i]  - A[i,2]*x[i+1])/A[i,1]
        i = N-1
        x_new[i] = (b[i] - A[i,0]*x[i-1])/A[i,1]
        relative_change = np.linalg.norm(x_new-x)/np.linalg.norm(x_new)
        if (LOUD):
            print("Iteration",iteration,": Relative Change =",relative_change)
        if (relative_change < tol) or (iteration >= max_iterations):
            converged = True
        iteration += 1
    return x_new


# -------------------------------------------------------------------------------------- #
# -------------------------------------------------------------------------------------- #
# Vignette #XX
# APPM 4720/5720 Randomized Algorithms, Spring 2019
#
# Demo to show how to approximate a matrix-matrix product using randomization based on
# the approach of P. Drineas and M. Mahoney. See section four of "Lectures on Randomized
# Linear Algebra" by Drineas and Mahoney for additional details and analysis.
#
# Algorithm randomized matrix multiplication.
# given A \in \reals^{m \times n}, B \in \reals^{n \times p}, an integer c
#    (1 \le c \le n), and a probability distribution p of length n.
# repeat for k = 1,\dots, c:
#       1. pick i \in {1,\dots,n} with P(i = k) = p_k iid with replacement.
#       2. set C(:,k) = 1/sqrt(c * p_i) * A(:,i) and R(k,:) = 1/sqrt(c * p_i) * B(i,:).
# return C, R, and CR = sum_{k=1}^c 1/(c * p_i) * A(:,i) B(i,:).
# -------------------------------------------------------------------------------------- #
import numpy as np                      # import numpy package
import time as time                     # import time package
import matplotlib.pyplot as plt         # import matplotlib package
np.set_printoptions(precision = 2)      # display only four digits
# -------------------------------------------------------------------------------------- #
def vignette_rand_mat_mult(n_sims = 1000, m = 100, n = 20, p = 80):
    np.random.seed(seed = 2)                # set seed for reproducibility
    # ensure parameters are integers
    n_sims = np.int(n_sims);  m = np.int(m);  n = np.int(n);  p = np.int(p)
    c = min(max(np.int(np.round(0.5*n)), 1), n)
    # storage for simulations
    fro_norms = np.zeros(n_sims);   fro_norms_opt = np.zeros(n_sims)
    # generate "data" matrices
    A = np.random.normal(scale = 1 / np.sqrt(n), size = (m,n))  # isotropic rows
    B = np.random.normal(scale = 1 / np.sqrt(n), size = (n,p))  # isotropic columns
    AB = np.matmul(A, B);
    # ell_2 norms: columns of A and rows of B
    col_norm_A = np.linalg.norm(A, axis = 0); row_norm_B = np.linalg.norm(B, axis = 1)
    # probabilities for sampling
    probs = np.ones(n)/n                # naive probabilities---i.e., uniform distribution
    probs_opt = (col_norm_A * row_norm_B) / sum(col_norm_A * row_norm_B) # optimal probs
    probs_opt = probs_opt / sum(probs_opt)  # undo roundoff---ensure they sum to 1
    # compute theoretical upper bounds on expected squared Frobenius norms
    upper_bound = sum(col_norm_A**2 * row_norm_B**2 / (c*probs))   # naive probabilities
    upper_bound_opt = sum(col_norm_A * row_norm_B)**2 / c          # optimal probabilities
    # simulation
    for t in range(n_sims):
        # initialize / re-zero matrices
        C = np.zeros((m,c));     R = np.zeros((c,p))
        C_opt = np.zeros((m,c)); R_opt = np.zeros((c,p))
        for k in range(c):
            # step 1
            i = np.random.choice(a = np.arange(n), replace = True, p = probs)
            i_opt = np.random.choice(a = np.arange(n), replace = True, p = probs_opt)
            # calculate rescaling coefficients
            rescale = 1 / np.sqrt(c*probs[i]);
            rescale_opt = 1 / np.sqrt(c*probs_opt[i_opt])
            # step 2
            C[:,k] = rescale * A[:,i]; R[k,:] = rescale * B[i,:]
            C_opt[:,k] = rescale_opt * A[:,i_opt]; R_opt[k,:] = rescale_opt * B[i_opt,:]
        # compute random matrix product via outerproduct
        CR = np.matmul(C, R)
        CR_opt = np.matmul(C_opt, R_opt)
        # calculate Frobenius norms of actual vs. randomized
        fro_norms[t] = np.linalg.norm(AB - CR, ord = 'fro')**2
        fro_norms_opt[t] = np.linalg.norm(AB - CR_opt, ord = 'fro')**2
    # print comparisons averaged across the number of simulations
    print('||AB - CR||_F^2 simulation vs. upper bound:',
            np.round(np.mean(fro_norms),2), 'vs.',
            np.round(upper_bound,2), '(using naive sampling probabilities)')
    print('||AB - CR||_F^2 simulation vs. upper bound:',
            np.round(np.mean(fro_norms_opt),2), 'vs.',
            np.round(upper_bound_opt,2), '(using optimal sampling probabilities)')
    # return np.mean(fro_norms), upper_bound, np.mean(fro_norms_opt), upper_bound_opt

# small problem
vignette_rand_mat_mult(n_sims = 10000, m = 10, n = 4, p = 10)

# moderate problem
vignette_rand_mat_mult(n_sims = 1000, m = 1000, n = 20, p = 1000)

# big problem
vignette_rand_mat_mult(n_sims = 1000, m = 5000, n = 10, p = 5000)

# Question: If you see that the estimated squared Frobenius norm for the naive
#           probabilities is smaller than that for the optimal probabilities, what should
#           you do?


import numpy as np

array1 = np.ones((3,3))
""" array([[1., 1., 1.],
       [1., 1., 1.],
       [1., 1., 1.]]) """

# 2-Dim, with diagonal values 1 and rest of the values 0 i,e identity matrix
array2 = np.eye(3)
""" array([[1., 0., 0.],
       [0., 1., 0.],
       [0., 0., 1.]]) """

# create matrix using diag function
array3 = np.diag([10,20,30,40])
""" array([[10,  0,  0,  0],
       [ 0, 20,  0,  0],
       [ 0,  0, 30,  0],
       [ 0,  0,  0, 40]]) """

# retrieve diagonal values of the matrix
np.diag(array2)
# array([1., 1., 1.])

# create matrix using diag random
array4 = np.random.rand(4)
# array([0.79959924, 0.49119588, 0.60578439, 0.53968689])


import numpy as np
from matplotlib import pyplot as plt

np.random.seed(0)

def data_generate(n=3000):
    x = np.zeros(n)
    u = np.random.rand(n)
    index1 = np.where((0 <= u) & (u < 1 / 8))
    x[index1] = np.sqrt(8 * u[index1])
    index2 = np.where((1 / 8 <= u) & (u < 1 / 4))
    x[index2] = 2 - np.sqrt(2 - 8 * u[index2])
    index3 = np.where((1 / 4 <= u) & (u < 1 / 2))
    x[index3] = 1 + 4 * u[index3]
    index4 = np.where((1 / 2 <= u) & (u < 3 / 4))
    x[index4] = 3 + np.sqrt(4 * u[index4] - 2)
    index5 = np.where((3 / 4 <= u) & (u <= 1))
    x[index5] = 5 - np.sqrt(4 - 4 * u[index5])

    return x

def kernel_function(x):
    #compute kernel function
    return np.exp(- (x * x) / 2) / (np.sqrt(2 * np.pi))

def p_hat(x,train_data,h):
    tempsum = 0
    for xi in train_data:
        tempsum = tempsum + kernel_function((x - xi)/h)
    return tempsum / (len(train_data) * h)

def kernel_density_method(data,h):
    #exec kernel density method with band width h and draw the result

    #draw the function with the help of p_hat
    plt.figure()
    X = np.arange(0,5,h)
    Y = p_hat(X,data,h)
    plt.hist(data,bins = 30,density = True)
    plt.plot(X,Y)
    plt.savefig("h" + str(int(h*100)) + ".png")

    return

def cross_validation(data,n):
    #exec cross validation with n sunsets of data to decide band width
    hs = [0.01,0.1,0.5]
    hscore = np.array([])

    #first, shuffle the data
    ndata = np.array([])
    permutation = np.random.permutation(len(data))
    for i in permutation:
        ndata = np.append(ndata,data[i])

    subset_size = int(len(ndata) / n)
    for h in hs:
        score = 0
        for i in range(n):
            #select test data as ith subset
            #and create test and train data
            train_data = np.append(ndata[0:(i*subset_size)],ndata[((i+1)*subset_size):])
            test_data = ndata[(i*subset_size):((i+1)*subset_size)]

            # compute the score
            tempscore = 0
            for x in test_data:
                tempscore = tempscore + np.log(p_hat(x, train_data, h))
            tempscore = tempscore / len(test_data)
            score = score + tempscore

        score = score / n
        hscore = np.append(hscore, score)

    # finally plot the score for each h
    plt.figure()
    plt.plot(hs,hscore)
    plt.savefig("cross_validation.png")

    return

if __name__ == "__main__":
    n = 3000
    data = data_generate(n)

    kernel_density_method(data,0.01)
    kernel_density_method(data,0.1)
    kernel_density_method(data,0.5)

    cross_validation(data,5)


#!/usr/bin/env python3
# coding=utf-8

import matplotlib.pyplot as plt
import numpy as np
import tensorflow as tf
from tensorflow.python.framework import ops

ops.reset_default_graph()

# Create graph
sess = tf.Session()

# Regression Example:
# We will create sample data as follows:
# x-data: 100 random samples from a normal ~ N(1, 0.1)
# target: 100 values of the value 10.
# We will fit the model:
# x-data * A = target
# Theoretically, A = 10.

# Declare batch size
batch_size = 25

# Create data
x_vals = np.random.normal(1, 0.1, 100)
y_vals = np.repeat(10., 100)
x_data = tf.placeholder(shape=[None, 1], dtype=tf.float32)
y_target = tf.placeholder(shape=[None, 1], dtype=tf.float32)

# Split data into train/test = 80%/20%
lenXvals = len(x_vals)
lenXvals80 = round(lenXvals * 0.8)
train_indices = np.random.choice(lenXvals, lenXvals80, replace=False)
print('lenXvals: ' + str(lenXvals))
print('lenXvals80: ' + str(lenXvals80))
print('train_indices: ' + str(train_indices))

setTrainIndices = set(train_indices)
rangeLenXvals = range(lenXvals)
setRangeLenXvals = set(rangeLenXvals)
setMinus = setRangeLenXvals - setTrainIndices
listMinus = list(setMinus)
test_indices = np.array(listMinus)

print('setTrainIndices: ' + str(setTrainIndices))
print('rangeLenXvals: ' + str(rangeLenXvals))
print('setRangeLenXvals: ' + str(setRangeLenXvals))
print('setMinus: ' + str(setMinus))
print('listMinus: ' + str(listMinus))
print('test_indices: ' + str(test_indices) + ' type: ' + str(type(test_indices)))

x_vals_train = x_vals[train_indices]
x_vals_test = x_vals[test_indices]
y_vals_train = y_vals[train_indices]
y_vals_test = y_vals[test_indices]

# Create variable (one model parameter = A)
A = tf.Variable(tf.random_normal(shape=[1, 1]))

# Add operation to graph
my_output = tf.matmul(x_data, A)

# Add L2 loss operation to graph
loss = tf.reduce_mean(tf.square(my_output - y_target))

# Create Optimizer
my_opt = tf.train.GradientDescentOptimizer(0.02)
train_step = my_opt.minimize(loss)

# Initialize variables
init = tf.global_variables_initializer()
sess.run(init)

# Run Loop
for i in range(100):
    rand_index = np.random.choice(len(x_vals_train), size=batch_size)
    rand_x = np.transpose([x_vals_train[rand_index]])
    rand_y = np.transpose([y_vals_train[rand_index]])
    sess.run(train_step, feed_dict={x_data: rand_x, y_target: rand_y})
    if (i + 1) % 25 == 0:
        print('Step #' + str(i + 1) + ' A = ' + str(sess.run(A)))
        print('Loss = ' + str(sess.run(loss, feed_dict={x_data: rand_x, y_target: rand_y})))

# Evaluate accuracy (loss) on test set
mse_test = sess.run(loss, feed_dict={x_data: np.transpose([x_vals_test]), y_target: np.transpose([y_vals_test])})
mse_train = sess.run(loss, feed_dict={x_data: np.transpose([x_vals_train]), y_target: np.transpose([y_vals_train])})
print('MSE on test:' + str(np.round(mse_test, 2)))
print('MSE on train:' + str(np.round(mse_train, 2)))

# Classification Example
# We will create sample data as follows:
# x-data: sample 50 random values from a normal = N(-1, 1)
#         + sample 50 random values from a normal = N(1, 1)
# target: 50 values of 0 + 50 values of 1.
#         These are essentially 100 values of the corresponding output index
# We will fit the binary classification model:
# If sigmoid(x+A) < 0.5 -> 0 else 1
# Theoretically, A should be -(mean1 + mean2)/2

ops.reset_default_graph()

# Create graph
sess = tf.Session()

# Declare batch size
batch_size = 25

# Create data
x_vals = np.concatenate((np.random.normal(-1, 1, 50), np.random.normal(2, 1, 50)))
y_vals = np.concatenate((np.repeat(0., 50), np.repeat(1., 50)))
x_data = tf.placeholder(shape=[1, None], dtype=tf.float32)
y_target = tf.placeholder(shape=[1, None], dtype=tf.float32)

# Split data into train/test = 80%/20%
train_indices = np.random.choice(len(x_vals), round(len(x_vals) * 0.8), replace=False)
test_indices = np.array(list(set(range(len(x_vals))) - set(train_indices)))
x_vals_train = x_vals[train_indices]
x_vals_test = x_vals[test_indices]
y_vals_train = y_vals[train_indices]
y_vals_test = y_vals[test_indices]

# Create variable (one model parameter = A)
A = tf.Variable(tf.random_normal(mean=10, shape=[1]))

# Add operation to graph
# Want to create the operstion sigmoid(x + A)
# Note, the sigmoid() part is in the loss function
my_output = tf.add(x_data, A)

# Add classification loss (cross entropy)
xentropy = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits=my_output, labels=y_target))

# Create Optimizer
my_opt = tf.train.GradientDescentOptimizer(0.05)
train_step = my_opt.minimize(xentropy)

# Initialize variables
init = tf.global_variables_initializer()
sess.run(init)

# Run loop
for i in range(1800):
    rand_index = np.random.choice(len(x_vals_train), size=batch_size)
    rand_x = [x_vals_train[rand_index]]
    rand_y = [y_vals_train[rand_index]]
    sess.run(train_step, feed_dict={x_data: rand_x, y_target: rand_y})
    if (i + 1) % 200 == 0:
        print('Step #' + str(i + 1) + ' A = ' + str(sess.run(A)))
        print('Loss = ' + str(sess.run(xentropy, feed_dict={x_data: rand_x, y_target: rand_y})))

# Evaluate Predictions on test set
y_prediction = tf.squeeze(tf.round(tf.nn.sigmoid(tf.add(x_data, A))))
correct_prediction = tf.equal(y_prediction, y_target)
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
acc_value_test = sess.run(accuracy, feed_dict={x_data: [x_vals_test], y_target: [y_vals_test]})
acc_value_train = sess.run(accuracy, feed_dict={x_data: [x_vals_train], y_target: [y_vals_train]})
print('Accuracy on train set: ' + str(acc_value_train))
print('Accuracy on test set: ' + str(acc_value_test))

# Plot classification result
A_result = -sess.run(A)
bins = np.linspace(-5, 5, 50)
plt.hist(x_vals[0:50], bins, alpha=0.5, label='N(-1,1)', color='green')
plt.hist(x_vals[50:100], bins[0:50], alpha=0.5, label='N(2,1)', color='red')
plt.plot((A_result, A_result), (0, 8), 'k--', linewidth=3, label='A = ' + str(np.round(A_result, 2)))
plt.legend(loc='upper right')
plt.title('Binary Classifier, Accuracy=' + str(np.round(acc_value_test, 2)))
plt.show()


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Tue Sep 29 07:23:32 2020

Module for functions used in nonlinear equation solvers

@author: zettergm
"""

import numpy as np

def fun1(x):
    y=np.cos(x)
    return y

def fun1_derivative(x):
    yprime=-1.0*np.sin(x)
    return yprime

def fun3(x):
    y=x**2+6*x+10
    return y

def fun3_deriv(x):
    yprime=2*x+6
    return yprime

def fun2D1(x,y):
    f=x**3+y**3-3*x*y
    return f

def fun2D2(x,y):
    g=x**2+y**2-1
    return g

def fun2D1_deriv(x,y):
    fx=3*x**2-3*y
    fy=3*y**2-3*x
    return [fx,fy]

def fun2D2_deriv(x,y):
    gx=2*x
    gy=2*y
    return [gx,gy]


import numpy as np
import math

class Regression:
    """ Base model :
    n_iterations : float
        The number of training iterations the algorithm will tune the weights for.
    learning_rate : float
        The step length that will be used when updating the weights.
    """

    def __init__(self, n_iterations, learning_rate):
        self.n_interations = n_iterations
        self.learning_rate = learning_rate

    def initialize_weights(self, n_features):
        """ initialize weight randomly """
        limit = 1 / math.sqrt(n_features)
        self.w = np.random.uniform(-limit, limit, (n_features,))

    def fit(self, X, y):
        """ Insert constant one for bias weights """
        X = np.insert(X, 0, 1, axis=1)
        self.training_errors = []
        self.initialize_weights(n_features=X.shape[1])

        """ Gradient descent for n_iterations """
        for i in range(self.n_interations):
            y_pred = X.dot(self.w)
            mse = np.mean(0.5 * (y - y_pred) ** 2)
            self.training_errors.append(mse)
            grad_w = -(y - y_pred).dot(X)
            # update the weight
            self.w -= self.learning_rate * grad_w

    def predict(self, X):
        """insert constant ones for bias weights"""
        X = np.insert(X, 0, 1, axis=1)
        y_pred = X.dot(self.w)
        return y_pred

class LinearRegression(Regression):
    """ Linear Regression Model Parameter
    n_iterations : float
        The number of training iterations the algorithm will tune the weights for
    learning rate : float

    gradient_descent : boolean
        True : Gradient Descent
        False : batch optimization by least squares
    """

    def __init__(self, n_iterations=100, learning_rate=0.001, gradient_descent=True):
        self.gradient_descent = gradient_descent
        super(LinearRegression, self).__init__(
            n_iterations=n_iterations, learning_rate=learning_rate
        )

    def fit(self, X, y):
        """ If Gradient Descent is False """
        if not self.gradient_descent:
            pass
        else:
            super(LinearRegression, self).fit(X, y)


# QRFactors.py
#
# Daniel R. Reynolds
# SMU Mathematics
# Math 4315

# imports
import numpy

def QRFactors(A):
    """
    usage: Q, R = QRFactors(A)

    Function to compute the QR factorization of a (possibly rank-deficient)
    'thin' matrix A (m x n, with m >=n) using Householder reflection matrices.

    Input:    A - thin matrix
    Outputs:  Q - orthogonal matrix
              R - "upper triangular" matrix, i.e. R = [ Rhat ]
                                                      [  0   ]
                  with Rhat an (n x n) upper-triangular matrix
    """

    # get dimensions of A
    m, n = numpy.shape(A)

    # initialize results
    Q = numpy.identity(m)
    R = A.copy()

    # iterate over columns
    for k in range(n):

        # extract subvector from diagonal down and compute norm
        z = R[k:m,k]
        v = -z;
        v[0] = -numpy.sign(z[0])*numpy.linalg.norm(z) - z[0];
        vnorm = numpy.linalg.norm(v)

        # if subvector has norm zero, continue to next column
        if (vnorm < numpy.finfo(float).eps):
            continue

        # compute u = u = v/||v||;
        # the Householder matrix is then Qk = I-2*u*u'
        u = v/vnorm

        # update rows k through m of R
        for j in range(k,n):
            utR = 2 * u.T @ R[k:m, j]
            R[k:m, j] -= u*utR

        # update rows k through m of Q
        for j in range(m):
            utQ = 2 * u.T @ Q[k:m, j]
            Q[k:m, j] -= u*utQ

    # transpose Q before return
    Q = Q.T

    return [Q, R]

# end function


"""
Calculuates the probability that a given sum occurs using an ordinary generating function.
"""

import sympy
from sympy import poly, srepr
from sympy.abc import x, y

board = [
    4, 3, 2, 4, 1, 4, 2, 3, 4,
    3, 4, 5, 3, 6, 3, 5, 4, 3,
    4, 3, 2, 4, 1, 4, 2, 3, 4,
    3, 4, 5, 3, 6, 3, 5, 4, 3,
    4, 3, 2, 4, 1, 4, 2, 3, 4,
    3, 4, 5, 3, 6, 3, 5, 4, 3,
    4, 3, 2, 4, 1, 4, 2, 3, 4,
    3, 4, 5, 3, 6, 3, 5, 4, 3,
    4, 3, 2, 4, 1, 4, 2, 3, 4
]

# Compute the generating function
generating_function = 1
for k in board:
    generating_function *= poly(1 + x**k * y)

generating_function = generating_function.as_expr()

# Parse terms and map into data structure
# The coefficient of the x^k * y^n term as a tuple map.
coefficients = {}
for term in generating_function.as_ordered_terms():
    xpair = term.as_coeff_exponent(x)
    ypair = xpair[0].as_coeff_exponent(y)

    x_exponent = int(xpair[1])
    y_exponent = int(ypair[1])
    coefficient = int(ypair[0])

    coefficients[(x_exponent, y_exponent)] = coefficient

frequency = {}
# Calculate the frequency for rolling all sums (k in [4, 24]) with 4 (n) marbles
for k in range(4, 25):
    frequency[k] = coefficients[(k, 4)]

total = sum(frequency.values())
for key in sorted(frequency.keys()):
    print('{}: {} ({})'.format(key, frequency[key], frequency[key] / total))


#
# damped, driven nonlinear pendulum motion
#
# ODE system:   1) dtheta/dt = omega
#               2) domega/dt = -Omega0^2 * sin(theta) - q omega + fD sin(OmegaD t)
#
#     where Omega0^2 = g / L
#
# solved numerically via the 2nd-order Runge-Kutta method
#
#
# by Denes Molnar for PHYS 580, Fall 2020

from math import sin, cos, sqrt, floor
from math import pi as PI
import numpy as np

import matplotlib
matplotlib.use('TkAgg')
import matplotlib.pyplot as plt

# alternatively, you could use this function to limit any angle to [-pi,pi)
def chopAngle(th0):
   theta = th0 - floor(th0 / (2. * PI)) * 2. * PI    # chop to [0, 2pi)
   if theta >= PI: theta -= 2. * PI                  # shift to [-pi,pi)
   return theta

# compute evolution of theta and omega from given initconds
def computeTrajectory(initconds, params, dt, nsteps):
   (th0, om0) = initconds
   (Omega0, q, fD, OmegaD) = params

   Omega0sq = Omega0**2

   # allocate storage - store both theta and omega
   theta = np.zeros(nsteps + 1)
   omega = np.zeros(nsteps + 1)
   theta[0], omega[0] = th0, om0

   # solve using 2nd-order Runge-Kutta method
   t = 0
   for i in range(nsteps):
      # halfstep
      om_half = omega[i] + 0.5 * dt * ( -Omega0sq * sin(theta[i]) - q * omega[i] + fD * sin(OmegaD * t) )
      th_half = theta[i] + 0.5 * dt * omega[i]
      t_half = t + 0.5 * dt
      # full step
      omega[i + 1] = omega[i] + dt * ( -Omega0sq * sin(th_half) - q * om_half + fD * sin(OmegaD * t_half) )
      theta[i + 1] = theta[i] + dt * om_half
      t += dt
      # bound theta (or, you could use the chop function -> slower)
      if   theta[i + 1] >= PI:  theta[i + 1] -= 2. * PI
      elif theta[i + 1] < -PI:  theta[i + 1] += 2. * PI

   # return results
   return theta, omega

# get parameters
#

# physical parameters and initial conditions (units are in [] brackets)
L   = float( input('length of pendulum [m]: ') )
q   = float( input('dissipation coefficient [1/s]: ') )
fD  = float( input('driving coefficient [1/s^2]: ') )
OmegaD = float( input('driving angular frequency [rad/s]: ') )
th0  = float( input('initial angle wrt to vertical [rad]: ') )
om0  = float( input('initial angular velocity [rad/s]: ') )
#
g    = 9.8      # gravitational acceleration [m/s^2] (hardcoded)

Omega0 = sqrt(g / L)

# computational parameters: dt[s], nsteps
dt = float( input('dt [s]: ') )
nsteps = int( input('number of time steps to calculate: ') )

# compute evolution
#
params = (Omega0, q, fD, OmegaD)
initconds = (th0, om0)

theta, omega   = computeTrajectory(initconds, params, dt, nsteps)

# time values, for plotting
timevals = np.array( [ dt * i  for i in range(nsteps + 1) ] )
# calculate 'energy' at each timestep
energy  = 0.5 * omega**2  + Omega0**2 * (1. - np.cos(theta))   # use numpy vector ops

# plots
#

#fontsize = 30   # fontsize, need to be tuned to screen resolution
fontsize = 15    # fontsize, for 1920x1080
linewidth = 3
pointarea = 2.

# make a 2x2 array of plots
fig, axArray = plt.subplots(2, 2)

((ax0, ax1), (ax2, ax3)) = axArray

# theta-omega  (TOP LEFT)
#ax0.plot(theta, omega, linewidth = linewidth, label = "phase space")
ax0.scatter(theta, omega, s = pointarea, label = "phase space")

ax0.tick_params(labelsize = fontsize)
ax0.legend(fontsize = fontsize)
ax0.set_xlabel("theta [rad]", fontsize = fontsize)
ax0.set_ylabel("omega [rad/s]", fontsize = fontsize)

titleString  = "dt=" + str(dt) + ", L=" + str(L)
titleString += ", fD=" + str(fD) + ", OmegaD=" + str(round(OmegaD, 3))
titleString += ", omega0=" + str(om0) + ", theta0=" + str(th0)
ax0.set_title(titleString, fontsize = fontsize)

# omega (TOP RIGHT)
ax1.plot(timevals, omega, linewidth = linewidth, label = "omega")

ax1.tick_params(labelsize = fontsize)
ax1.legend(fontsize = fontsize)
ax1.set_xlabel("t[s]", fontsize = fontsize)
ax1.set_ylabel("omega[rad/s]", fontsize = fontsize)

# theta (BOTTOM LEFT)
ax2.plot(timevals, theta, linewidth = linewidth, label = "theta")

ax2.tick_params(labelsize = fontsize)
ax2.legend(fontsize = fontsize)
ax2.set_xlabel("t[s]", fontsize = fontsize)
ax2.set_ylabel("theta[rad]", fontsize = fontsize)

# energy (BOTTOM RIGHT)
ax3.plot(timevals, energy, linewidth = linewidth, label = "energy")

ax3.tick_params(labelsize = fontsize)
ax3.legend(fontsize = fontsize)
ax3.set_xlabel("t[s]", fontsize = fontsize)
ax3.set_ylabel("energy [a.u.]", fontsize = fontsize)

# show plot
plt.subplots_adjust(wspace = 0.4, hspace = 0.2)
#fig.tight_layout()
plt.show()

#EOF


#BT3051 Assignment 3b
#Roll number: BE17B037
#Collaborators: none
#Time: 1:30:00

import networkx as nx

def maze(MazeMatrix,start,finish):

    def MazeMatrix2Graph(MazeMatrix):
        MazeGraph = nx.Graph()
        for i in range(m):
            for j in range(n):
                if MazeMatrix[i][j] == 1:
                    MazeGraph.add_node((i, j))
                    if i-1>=0 and MazeMatrix[i-1][j]==1:
                        MazeGraph.add_edge((i, j), (i-1, j))
                    if j-1>=0 and MazeMatrix[i][j-1]==1:
                        MazeGraph.add_edge((i, j), (i, j-1))
        return MazeGraph

    def MazeAnswerBFS(MazeGraph,start,finish):
        queue = []
        distance = [[99999999 for i in range(n)] for j in range(m)]
        parent = [[(-1, -1) for i in range(n)] for j in range(m)]
        queue.append(start)
        (i, j) = start
        distance[i][j] = 0
        parent[i][j] = (i, j)
        for node_1 in queue:
            (i1, j1) = node_1
            for node_2 in list(MazeGraph.adj[node_1]):
                (i2, j2) = node_2
                if (distance[i1][j1] + 1) < distance[i2][j2]:
                    queue.append((i2, j2))
                    distance[i2][j2] = distance[i1][j1] + 1
                    parent[i2][j2] = node_1
        shortest_path = []
        (i, j) = finish
        while(not parent[i][j]==(i, j)):
            shortest_path.append((i, j))
            (i, j) = parent[i][j]
        shortest_path.append((i, j))
        shortest_path = shortest_path[::-1]
        return shortest_path

    def MazeComponentsDFS(MazeGraph):
        stack = []
        components = []
        count = 0
        Visited = [[0 for i in range(n)] for j in range(m)]
        for i in range(m):
            for j in range(n):
                if Visited[i][j] == 0 and (i, j) in list(MazeGraph.nodes):
                    count = count+1
                    stack.append((i, j))
                    Visited[i][j] = 1
                    individual_component = []
                    while(len(stack)!=0):
                        node_1 = stack.pop()
                        individual_component.append(node_1)
                        for node_2 in list(MazeGraph.adj[node_1]):
                            (i2, j2) = node_2
                            if (Visited[i2][j2] == 0):
                                stack.append(node_2)
                                Visited[i2][j2] = 1
                    components.append(individual_component)
        return components

    m = len(MazeMatrix)
    n = len(MazeMatrix[0])

    G = MazeMatrix2Graph(MazeMatrix)
    a = MazeAnswerBFS(G,start,finish)
    b = MazeComponentsDFS(G)

    print(a)
    print(b)

if __name__ == '__main__':
    #DO NOT MODIFY THE FOLLOWING
    hw3bmaze=    [[1,0,1,1,0,1],
                  [1,1,0,0,0,0],
                  [0,1,0,1,1,1],
                  [0,1,1,1,0,1],
                  [1,0,0,1,1,1],
                  [1,1,0,0,0,1],
                  [0,0,1,1,0,1]]

    hw3bstart=(0,0)
    hw3bfinish=(6,5)
    print(maze(hw3bmaze,hw3bstart,hw3bfinish))

#output for this example should be:
#[(0, 0), (1, 0), (1, 1), (2, 1), (3, 1), (3, 2), (3, 3), (4, 3), (4, 4), (4, 5), (5, 5), (6, 5)]
#[[(0, 0), (1, 0), (1, 1), (2, 1), (3, 1), (3, 2), (3, 3), (4, 3), (4, 4), (4, 5), (5, 5), (6, 5), (3, 5), (2, 5), (2, 4), (2, 3)], [(0, 2), (0, 3)], [(0, 5)], [(4, 0), (5, 0), (5, 1)], [(6, 2), (6, 3)]]


# -*- coding: utf-8 -*-

import numpy as np
import matplotlib.pyplot as plt

class Polynomial:

    def __init__(self, coefficients):
        '''
        This class implements a polynomial function.
        It takes input the coefficients of a n-order polynomial as a list
        and given m number of inputs X, it returns the corresponding Y.

        usage:
        p = Polynomial([a_0, a_1, ..., a_n])
        X = np.linspace(-3, 3, 50, endpoint=True)
        F = p(X)
        plt.plot(X,F)
        '''

        self.coefficients = coefficients

    def __repr__(self):
        '''
        method to return the canonical string representation
        of a polynomial.

        '''
        return "Polynomial" + str(self.coefficients)

    def __call__(self, x):
        '''
        This method calculates the ordinate (Y-axis) given the
        abscissa (X-axis) of a particular point on the polynomial.

        input: x can be a number or a numpy array
        '''

        # Initialize return value
        res = 0

        # Calculate the y_i given x_i
        # remember: y_i = f(x_i)
        # y_i = a_0 + a_1 * x_i ^ 1 + a_2 * x_i ^ 2 + ... + a_n * x_i ^ n
        #
        # use a for loop to calculate res
        # your loop might look like this
        for index, coeff in enumerate(self.coefficients):
            res += coeff*(x**index)

        return res

    def degree(self):
        return len(self.coefficients)

    def __str__(self):
        res = ""
        res += str(self.coefficients[0])
        for i in range(1, len(self.coefficients)):
            coeff = self.coefficients[i]
            if coeff < 0:
                res += " - " +  str(-coeff) + "x^" + str(i)
            else:
                res += " + " +  str(coeff) + "x^" + str(i)

        return res

def get_coefficients(x, y):
    '''
    Given a set of Xs and Ys, this function tries to form a
    Vandermonde matrix. It then uses np.linalg.pinv to find the
    inverse of the matrix to calculate the coefficients of the
    n-order polynomial to fit the given data points.

    usage:
    x = np.array([0,1,2])
    y = np.array([0,1,4])
    coefficients = get_coefficients(x, y) # array([-0.,  0.,  1.])
                                          # y = f(x) = 0 + 0 + x^2
    '''

    # Get length
    length = len(x)

    # Create (len X len) matrix
    coeff = np.zeros((length, length))

    # Populate the matrix

    # a_0 + a_1 * x_0 + a_2 * x_0^2 + .. a_n * x_0^n = y_0
    # a_0 + a_1 * x_1 + a_2 * x_1^2 + .. a_n * x_1^n = y_1
    # a_0 + a_1 * x_2 + a_2 * x_2^2 + .. a_n * x_2^n = y_2
    # .. .. ..
    # a_0 + a_1 * x_(n-1) + a_2 * x_(n-1)^2 + .. a_n * x_(n-1)^n = y_(n-1)

    # |1  x_0     x_0^2     ... x_0^n    | |a_0|   |  y_0  |
    # |1  x_1     x_1^2     ... x_1^n    | |a_1|   |  y_1  |
    # |1  x_2     x_2^2     ... x_2^n    | |a_2| = |  y_2  |
    # | .. ..                            | |...|   |  ...  |
    # |1  x_(n-1) x_(n-1_^2 ... x(n-1)^n | |a_n|   |y_(n-1)|

    for i in range(length):
        coeff[:, i] = x**i

    # Invert the matrix
    inverse = np.linalg.pinv(coeff)

    # calculate the coefficients
    coefficients = np.dot(inverse, y)
    return coefficients

x = np.array([-3.,-2.,-1.,0.,1.,3])
y = np.array([-80.,-13.,6.,1.,5., 16])
coefficients = get_coefficients(x, y)
p = Polynomial(list(coefficients))
X = np.linspace(-3, 3, 50, endpoint=True)
F = p(X)
plt.plot(X,F)
plt.plot(x,y, 'ro')
print(x,y)
print(coefficients)
print(p)


"""Pearson Correlation Coefficient for regression"""
import typing
import numpy as np
from h2oaicore.metrics import CustomScorer

class Pearson_Correlation(CustomScorer):
    _description = "Pearson Correlation coefficient"
    _regression = True
    _maximize = True
    _perfect_score = 1.
    _display_name = "PearsonR"

    def score(self,
              actual: np.array,
              predicted: np.array,
              sample_weight: typing.Optional[np.array] = None,
              labels: typing.Optional[np.array] = None) -> float:
        if sample_weight is None:
            sample_weight = np.ones(actual.shape[0])

        sx = np.sum(predicted * sample_weight)
        sy = np.sum(actual * sample_weight)
        sx_2 = np.sum(predicted * predicted * sample_weight)
        sy_2 = np.sum(actual * actual * sample_weight)
        sxy = np.sum(predicted * actual * sample_weight)
        n = np.sum(sample_weight)
        sq = np.sqrt(np.abs(n * sx_2 - (sx * sx)) * np.abs(n * sy_2 - (sy * sy)))
        cor = (n * sxy - sx * sy) / max(sq, 1E-30)

        return cor


import numpy as np
import math

# this function helps in calculation of sigmoid function value of a component of vector
def sigmoid_element_wise(vector_component):
    # if-else to prevent math overflow
    if vector_component >= 0:
        return 1 / (1 + math.exp(-vector_component))
    else:
        return math.exp(vector_component) / (math.exp(vector_component) + 1)

# this function calculated sigmoid of pre - activation layer
def sigmoid(pre_activation_vector):
    # create a vector of same shape as input
    activated_vector = np.empty_like(pre_activation_vector)
    # iterate over input
    for i, elem in np.ndenumerate(pre_activation_vector):
        # calculate component wise sigmoid
        activated_vector[i] = sigmoid_element_wise(elem)
    return activated_vector

# this function calculates softmax
def softmax(pre_activation_vector):
    post_act = np.copy(pre_activation_vector)
    # we are shifting the value of exponent because in case of large error, there can be nan problem,this is the fix
    max_exponent = np.max(post_act)
    post_act = np.exp(post_act - max_exponent)
    post_act = post_act / np.sum(post_act)
    return post_act

# this function calculates softmax
def relu(pre_activation_vector):
    post_act = np.copy(pre_activation_vector)
    # get the position of vector that is -ve and make them 0
    post_act[post_act < 0] = 0
    return post_act

# this function handles the input and redirects the request to proper function
def activation_function(pre_activation_vector, context):
    if context == 'softmax':
        # calling softmax
        return softmax(pre_activation_vector)
    elif context == 'sigmoid':
        # calling sigmoid
        return sigmoid(pre_activation_vector)
    elif context == 'tanh':
        # creating tanh
        return np.copy(np.tanh(pre_activation_vector))
    elif context == 'relu':
        # calling relu
        return relu(pre_activation_vector)
    else:
        # Error handling
        return None


import scipy as sp
import FiniteDiff as FD

def der(fc, x, h=.0001, degree=1, type='centered', accuracy=2):
    """ Computes the numerical of the callable function 'fc at all the
    points in array 'x'. 'degree' is the degree of the derivative to be
    computed. 'type' can be 'centered', 'forward', or 'backward'.
    'accuracy' is the desired order of accuracy. For forward and backward
    differences it can take a value of 1, 2, or 3. For centered differences
    it can take a value of 2, 4, or 6."""
    # Use these lists to manage the different coefficient options.
    A = np.array([[[0., 0., -.5, 0., .5, 0., 0.],
                   [0., 1/12., -2/3., 0., 2/3., -1/12., 0.],
                   [-1/60., 3/20., -3/4., 0., 3/4., -3/20., 1/60.]],
                  [[0., 0., 1., -2., 1., 0., 0.],
                   [0., -1/12., 4/3., -5/2., 4/3., -1/12., 0.],
                   [1/90., -3/20., 3/2., -49/18., 3/2., -3/20., 1/90.]]])
    B = np.array([[[-1., 1., 0., 0., 0.],
                   [-1.5, 2., -.5, 0., 0.],
                   [-11/6., 3., -1.5, 1/3., 0.]],
                  [[1., -2., 1., 0., 0.],
                   [2., -5., 4., -1., 0.],
                   [35/12., -26/3., 19/2., -14/3., 11/12.]]])
    if type == "centered":
        acc = int(accuracy/2) - 1
    else:
        acc = int(accuracy) - 1
    if int(degree) not in [1, 2]:
        raise ValueError ("Only first and second derivatives are supported")
    if acc not in [0, 1, 2]:
        raise ValueError ("Invalid accuracy")
    if type == 'centered':
        xdifs = np.array([fc(x+i*h) for i in xrange(-3, 4)])
        return np.inner(A[degree-1,acc], xdifs.T) / h**degree
    elif type == 'forward':
        xdifs = np.array([fc(x+i*h) for i in xrange(5)])
        return np.inner(B[degree-1,acc], xdifs.T) / h**degree
    elif type == 'backward':
        xdifs = np.array([fc(x-i*h) for i in xrange(5)])
        return np.inner(B[degree-1,acc], xdifs.T) / (-h)**degree
    else:
        raise ValueError ("invalid type")

def partial(fc, x, i, h=.0001, ty="centered", ac=2):
    """ Computes a partial derivative with respect to index 'i'.
    The rest of the options are the same as the numerical derivative function."""
    def fcpart(y):
        add = np.zeros(x.shape[0])
        add[i] = y
        return fc(x+add)
    return der(fcpart, 0., h=h, type=ty, accuracy=ac)

def jac(fc, x, ty="centered", ac=2, h=.0001):
    """Compute the Jacobian matrix of a function.
    'fc' is a callable function that operates on a 1D array.
    'x' is where to evaluate the Jacobian matrix.
    Dimensions of the domain and range are infered from 'x'
    and the output of 'fc'."""
    return np.array([partial(fc, x, [i], h=h, ty=ty, ac=ac) for i in xrange(x.size)]).T

def multipartial(fc, x, li, h=.0001, ty="centered", ac=2):
    """ Computes multiple partial derivatives via recursion.
    'fc' is a callable function. 'x' is where to take the derivatives.
    'li' is a list of indices corresponding to the partials to be taken."""
    if len(li) <= 1:
        return partial(fc, x, li[0], h=h, ty=ty, ac=ac)
    else:
        part = lambda x: partial(fc, x, li[-1], h=h, ty=ty, ac=ac)
        return multipartial(part, x, li[:-1], h=h, ty=ty, ac=ac)

def hessian(fc, x, h=.0001, ty="centered", ac=2):
    """ Hessian matrix of function 'fc' at point 'x'.
    Computed using difference 'h' and difference type 'ty' of acccuracy 'ac'.
    Exact options are the same as the numerical derivative function."""
    hes = np.empty((x.size, x.size))
    for i in xrange(x.size):
        for j in xrange(x.size):
            hes[i,j] = multipartial(fc, x, [i,j], h=h, ty=ty, ac=ac)
    return hes

# Here are alternate versions for the Jacobian and Hessian functions.
def Jacobian(f, m, n, pt, mode='centered', o=2, h=1e-5):
    '''
    Approximate the Jacobian of a function at a point.
    Inputs:
        f -- a callable function whose jacobian we will approximate
        m -- integer giving the dimension of the domain or f
        n -- integer giving the dimension of the range of f
        pt -- numpy array specifying the point at which to approximate the Jacobian
        mode -- specifies the type of difference scheme. Should take values in
                ['centered', 'backward', 'forward'].
        d -- the order of the derivative. Should take values in [1,2]
        o -- order of approximation. If mode = 'centered', should take values
             [2,4,6], otherwise should take values in [1,2,3]
        h -- the size of the difference step
    Returns:
        jac -- array the same shape as pts, giving the approximate Jacobian at
               each point in pts.
    '''
    jac = np.empty((m,n))
    for i in xrange(n):
        off = np.zeros(n)
        off[i] = h
        if mode == 'centered':
            if o == 2:
                jac[:, i] = (-.5*f(pt - off) + .5*f(pt + off))/h
            if o == 4:
                jac[:, i] = (f(pt-2*off)/12 - 2*f(pt-off)/3 + 2*f(pt+off)/3 - f(pt+2*off)/12)/h
            if o == 6:
                jac[:, i] = (-f(pt-3*off)/60 + 3*f(pt-2*off)/20 - 3*f(pt-off)/4 +
                          f(pt+3*off)/60 - 3*f(pt+2*off)/20 + 3*f(pt+off)/4)/h
        if mode == 'forward':
            if o == 1:
                jac[:, i] = (-f(pt)+f(pt+off))/h
            if o == 2:
                jac[:, i] = (-3*f(pt)/2 + 2*f(pt+off) - f(pt+2*off)/2)/h
            if o == 3:
                jac[:, i] = (-11*f(pt)/6 + 3*f(pt+off) - 3*f(pt+2*off)/2 + f(pt+3*off)/3)/h
        if mode == 'backward':
            if o == 1:
                jac[:, i] = -(-f(pt)+f(pt-off))/h
            if o == 2:
                jac[:, i] = -(-3*f(pt)/2 + 2*f(pt-off) - f(pt-2*off)/2)/h
            if o == 3:
                jac[:, i] = -(-11*f(pt)/6 + 3*f(pt-off) - 3*f(pt-2*off)/2 + f(pt-3*off)/3)/h
    return jac

def Hessian(f, n, pt, h=1e-5):
    '''
    Approximate the Hessian of a function at a given point.
    Inputs:
        f -- callable function object
        n -- integer giving the dimension of the domain of f
        pt -- flat numpy array giving the point at which to approximate
        h -- step size for difference scheme
    Returns:
        hess -- numpy array giving the approximated Hessian matrix
    '''
    hess = np.empty((n,n))
    off = np.eye(n)*h
    for i in xrange(n):
        for j in xrange(n):
            hess[i,j] = (f(pt+off[i]+off[j])-f(pt + off[i]-off[j]) -
                         f(x+off[j]-off[i]) + f(pt-off[i]-off[j]))/(4*h**2)
    return hess