from numpy import array
from numpy import mean
from numpy import cov
from numpy.linalg import eig
import pandas as pd

# define a matrix
A = array([[1, 2], [2, 3], [3, 2], [4, 4], [5, 4], [6, 7], [7, 6], [9, 7]])

print("We have the data:")
print(A)
# calculate the mean of each column
M = mean(A.T, axis=1)
print("Step1: Let's compute the mean vector:")
print(M)
# center columns by subtracting column means
C = A - M
print("Step2: Subtract the mean from data:")
print(C)
# calculate covariance matrix of centered matrix
V = cov(C.T)
print("Step3: Let's calculate the covariance matrix:")
print(V)
# eigendecomposition of covariance matrix
values, vectors = eig(V)
print("Step4: Let's calculate the eigen vectors and eigen values.")
print("Eigen vectors:")
print(vectors)
print("Eigen values:")
print(values)

# project data
P = vectors.T.dot(C.T)
print("Step5: Projected Data:")
print(P.T)


import numpy as np

def pascalindex2d(k):
    """
    The purpose of this function is to prepare a set of combinations of m and n for the creation of the Jacobi polynomials that satisfy (m+n)<=k.
    k: int, max order of the Jacobi polynomial desired

    Returns: array of (ncomb x 2), where each row corresponds to a tuple of (m, n) values for the Jacobi polynomials. Take, for example, the monomial basis: {1, x1, x2, x1^2, x1x2, x2^2, ...}. The order of the polynomials in the variables x1 and x2 are (0, 0), (1, 0), (0, 1), (2, 0), (1, 1), (0, 2), which is exactly what this function outputs.
    Remember the formula nplocal = (k+1)(k+2)/2 for 2D? This is just a formula for the number of elements up to that order (row) of the pascal triangle. Assuming each entry has equal area weighting, this formula basically calculates the "area" of the elements in pascal's triangule using A=(base1*base2)/2*(1/3)h - volume of a square pyramid is (1/3)bh.
    """

    if k==0:
        # Base case - top element in pascal triangle has index (0, 0)
        return np.array([0, 0])[None, :]

    # Builds the index pairs of the kth row of the triangle

    nth_row = np.zeros((k+1, 2))
    nth_row[:, 0] = np.arange(k+1)
    nth_row[:, 1] = k - nth_row[:, 0]

    nth_row = np.fliplr(nth_row)    # So it matches the 16.930 matlab code
    # And then stacks it with the index pairs of the elements in the rows of the pyramid above it
    prev_rows = pascalindex2d(k-1)
    pindx = np.concatenate((prev_rows, nth_row), axis=0).astype(int)

    return pindx

def pascalindex3d(k):
    """
    The purpose of this function is to prepare a set of combinations of m, n, and l for the creation of the Jacobi polynomials that satisfy (m+n+l)<=k.
    k: int, max order of the Jacobi polynomial desired

    Returns: array of (ncomb x 3), where each row corresponds to a tuple of (m, n, l) values for the Jacobi polynomials.
    Remember the formula nplocal = (k+1)(k+2)(k+3)/6 for 3D? This is just a formula for the number of elements up to that order (row) of the pascal pyramid. Assuming each entry has equal volume weighting, this formula basically calculates the "volume" of the elements in pascal's triangule using A=bh/2*(altitude)/3
    Recursively
    """

    if k==0:
        # Base case - top element in pascal pyramid has index (0, 0, 0)
        return np.array([0, 0, 0])[None,:]

    # Builds the index pairs of the kth level of the pyramid
    level2d_indices = pascalindex2d(k)  # 2D pascal indices on a level
    nth_level = np.zeros((level2d_indices.shape[0], 3))
    nth_level[:,1:] = level2d_indices
    nth_level[:, 0] = k-np.sum(nth_level[:,1:], axis=1)   # Adds the third column/dimension so that the rows add to k

    # nth_level = np.fliplr(nth_level)    # So it matches the 16.930 matlab code
    # And then stacks it with the index pairs of the elements in the levels of the pyramid above it
    prev_levels = pascalindex3d(k-1)
    pindx = np.concatenate((prev_levels, nth_level), axis=0).astype(int)

    return pindx

if __name__ == '__main__':
    print(pascalindex2d(3))
    # print(pascalindex3d(3))


import numpy as np

def sigmoid(z):
    #          |
    #          | .'''''   1
    #         .|          0.5
    #  .....'__|________  0
    #          |
    #          |
    #          |
    # sigmoid function
    # output ranges form 0.0 to 1.0
    # very useful for binary classification

    result = 1.0 / (1.0 + np.exp(-z))
    return result

def sigmoid_prime(z):
    # returns the derivative of sigmoid function as
    # s'(z) = s(z).(1 - s(z))

    result = sigmoid(z)*(1.0 - sigmoid(z))
    return result

def tanh(z):
    #        |
    #       1|  .....
    # _______|._______
    #       .|
    #  ..... |-1
    #        |
    # hyperbolic tangent function
    # output ranges from -1.0 to 1.0

    e = np.exp(z)
    e_ = np.exp(-z)
    result = (e - e_) / (e + e_)
    return result

def tanh_prime(z):
    # returns the derivative of hyperbolic tangent function as
    # tanh'(z) = 1 - tanh(z)^2

    result = 1 - np.power(tanh(z), 2)
    return result

def relu(z):
    #        |    .
    #        |  .
    # __.....|.______  y = [x for x > 0
    #        |             [0 otherwise
    #        |
    #        |
    # called rectified linear unit
    # returns the maximum of (0.0, z)

    result = np.maximum(0.0, z)
    return result

def relu_prime(z):
    # returns the derivative of relu function
    # when z < 0, relu(z) = 0, a constant. so, derivative is 0
    # when z > 0, relu(z) = z, so derivative is 1

    result = np.array(z, copy=True)
    result[z <= 0.0] = 0.0
    result[z > 0.0] = 1.0
    return result

def leaky_relu(z):
    #        |    .
    #        |  .
    # _______|.______  y = [x for x > 0
    #   .  ' |             [0.01 * x otherwise
    #        |
    #        |
    # called leaky rectified linear unit
    # returns the maximum of (0.01 * z, z)

    result = np.maximum(0.01 * z, z)
    return result

def leaky_relu_prime(z):
    # returns the derivative of leaky relu function
    # when z < 0, leaky_relu(z) = 0.01*z, so, derivative is 0.01
    # when z > 0, leaky_relu(z) = z, so derivative is 1

    result = np.array(z, copy=True)
    result[z <= 0.0] = 0.01
    result[z > 0.0] = 1.0
    return result

# dictionary containing the mapping for activation functions to their string names
activations_forward = {
    "sigmoid": sigmoid,
    "tanh": tanh,
    "relu": relu,
    "leaky_relu": leaky_relu
}

activations_backward = {
    "sigmoid": sigmoid_prime,
    "tanh": tanh_prime,
    "relu": relu_prime,
    "leaky_relu": leaky_relu_prime
}


import numpy as np

def identity_function(x):
    """
    (function) identity_function
    ----------------------------
    The identity function

    Parameter
    ---------
    - x : input value(s)

    Return
    ------
    - identity function output value(s)
    """
    return x

def step_function(x):
    """
    (function) step_function
    ------------------------
    The step function

    Parameter
    ---------
    - x : input value(s)

    Return
    ------
    - step function output value(s)
    """
    return np.array(x > 0, dtype=np.int)

def sigmoid(x):
    """
    (function) sigmoid
    ------------------
    The sigmoid function

    Parameter
    ---------
    - x : input value(s)

    Return
    ------
    - sigmoid function output value(s)
    """
    return 1 / (1 + np.exp(-x))

def relu(x):
    """
    (function) relu
    ---------------
    The ReLU function

    Parameter
    ---------
    - x : input value(s)

    Return
    ------
    - ReLU function output value(s)
    """
    return np.maximum(0, x)

# Softmax function
def softmax(x):
    """
    (function) softmax
    ------------------
    The softmax function

    Parameter
    ---------
    - x : input vector

    Return
    ------
    - softmax function output vector = probability vector
    """
    if x.ndim == 2:
        x = x.T
        x = x - np.max(x, axis=0)
        y = np.exp(x) / np.sum(np.exp(x), axis=0)
        return y.T

    x = x - np.max(x)
    return np.exp(x) / np.sum(np.exp(x))


import numpy as np

def sigmoid(x):
    """returns the value of x evaluated on the sigmoid function"""
    return 1 / (1 + np.exp(-x))

def sigmoid_d(x):
    """returns the value of x evaluated on the derivative
    of the sigmoid function"""
    return sigmoid(x) * (1 - sigmoid(x))

def tanh(x):
    """returns the value of x evaluated on the hyperbolic tangent function"""
    return np.tanh(x)

def tanh_d(x):
    """returns the value of x evaluated on the derivative
    of the hyperbolic tangent function"""
    return 1 - (tanh(x) ** 2)

def rrelu(x):
    """returns the value of x evaluated on the rectified linear unit function"""
    x = np.where(x <= 0, 0.01 * x, x)
    return x

def rrelu_d(x):
    """returns the value of x evaluated on the derivative
    of the rectified linear unit function"""
    return np.where(x <= 0, 0.01, 1)


import numpy as np
from math import sqrt
import matplotlib.pyplot as plt

'''
The following code solves the second order system:
        u' = x + v; u(0) = 0
        v' = u v^2; v(0) = 1

The second order Runge Kutta method is used to solve this system.
Here, x + v = f and u v^2 = g
'''
def f(x, v):
    return x + v #RHS of first order differential equation

def g(u, v):
    return u*v*v

def rk4(f, g, x0, u0, v0, x1, n): #Runge-Kutta fourth order
    #vx = [0] * (n + 1)
    vx=np.zeros((n+1,1)) #using numpy array to define vector x
    #print(vx) #for debugging only
    #vy = [0] * (n + 1)
    vu=np.zeros((n+1,1)) #using numpy array to define vector u
    vv=np.zeros((n+1,1)) #using numpy array to define vector v
    #print(vy) #for debugging only
    h = (x1 - x0) / float(n) #Step size
    vx[0] = x = x0 #x0 in (x0,x1)
    vu[0] = u = u0 #initial condition u(x=x0)=u0
    vv[0] = v = v0 #initial condition v(x=x0)=v0
    for i in range(1, n + 1): #RK4 loop
        #k1 = h * f(x, u, v) #this is a template for k1 for RK4
        k1 = h * f(x, v)
        #l1 = h * g(x, u, v) #this is a template for l1 for RK4
        l1 = h * g(u, v)

        #k2 = h * f(x + 0.5 * h, u + 0.5 * k1, v + 0.5 * l1) #this is a template for k2 for RK4
        k2 = h * f(x + 0.5 * h, v + 0.5 * l1)
        #l2 = h * g(x + 0.5 * h, u + 0.5 * k1, v + 0.5 * l1) #this is a template for l2 for RK4
        l2 = h * g(u + 0.5 * k1, v + 0.5 * l1)

        #k3 = h * f(x + 0.5 * h, u + 0.5 * k2, v + 0.5 * l2)
        k3 = h * f(x + 0.5 * h, v + 0.5 * l2)
        #l3 = h * g(x + 0.5 * h, u + 0.5 * k2, v + 0.5 * l2)
        l3 = h * g(u + 0.5 * k2, v + 0.5 * l2)

        #k4 = h * f(x + h, u + k3)
        k4 = h * f(x + h, v + l3)
        #l4 = h * g(x + h, u + k3)
        l4 = h * g(u + k3, v + l3)

        vx[i] = x = x0 + i * h #Proceed to next value of x
        vu[i] = u = u + (k1 + k2 + k2 + k3 + k3 + k4) / 6 #next value of u
        vv[i] = v = v + (l1 + l2 + l2 + l3 + l3 + l4) / 6 #next value of v

    return vx, vu, vv

x1=1
nodes=x1*20
vx, vu, vv = rk4(f, g, 0, 0, 1, x1, nodes) #f, g, u0, v0, x0,
m=nodes/float(10)
print vu
print vv

#plt.plot(vu, vv, 'rp')
#plt.axis([0, 5, 0, 100])
#plt.show()

##print(vx)
##print(vu)
##print(vv)

##for x in list(zip(vx, vu, vv))[::1]:
##    print x

##for x, y in list(zip(vx, vu, vv))[::m]: #in [::m], prints every m elements
##    print("%4.1f %10.5f %10.5f" % (x, u, v))
##print('\n')


import numpy as np
import math
from plot import plot as plot
import scipy.linalg

def read_data(file_name):
    file_object = open(file_name, "r")
    result = []
    for line in file_object.readlines():
        parts = map(lambda part: float(part.strip()), line.split(","))
        if len(parts) != 2:
            raise Exception("illegal line format")
        result.append((parts[0], parts[1]))
    file_object.close()
    return result

def get_b_from_data_list(data_list):
    return np.matrix(map(lambda pair: [pair[1]], data_list))

def get_a_from_data_list(data_list, d):
    e = math.e
    return np.matrix(map(lambda pair: [e**(d * pair[0]), e**(-d * pair[0]), 1], data_list))

def get_d_list(n):
    if n <= 2:
        raise Exception("n is not > 2")

    result = []
    for k in range(0, n):
        result.append((k, 0.1 + 0.4 * k / (n - 1)))
    return result

def r_rank(r):
    if r.shape[0] != r.shape[1]:
        raise Exception("r is not quadratic")

    counter = 0
    eps = 10**-12
    for i in range(0, r.shape[0]):
        if not (-eps < r[i, i] < eps):
            counter += 1
    return counter

def main(file_name="data.txt", n=7):
    data_list = read_data(file_name)
    b = get_b_from_data_list(data_list)
    parameter_list = []
    for k, d in get_d_list(n):
        print("k={0}; n={1}".format(k, n))
        a = get_a_from_data_list(data_list, d)
        q, r = np.linalg.qr(a)
        if r_rank(r) != 3:
            print("Rank of r or q is not 3!")
        else:
            z = np.dot(q.T, b)
            x = scipy.linalg.solve_triangular(r, z)

            r = np.dot(a, x) - b
            print("Residuum r = Ax - b = ")
            print(str(r))

            norm_r = np.linalg.norm(r)
            print("Norm of Residuum: |r| = " + str(norm_r))

            cond_a = np.linalg.cond(a)
            cond_ata = np.linalg.cond(np.dot(a.T, a))
            print("cond(A) = {0}; cond(A^T A) = {1}".format(cond_a, cond_ata))

            parameter_list.append((x.item(0), x.item(1), x.item(2), d, k, n, norm_r, cond_a, cond_ata))

        print("")

    plot(parameter_list, data_list)

if __name__ == "__main__":
    # available files:
    #  - data.txt          - from task sheet
    #  - data_subset.txt   - contains a subset from data.txt
    #  - data_sym.txt      - contains manipulated (symmetric) data from data.txt
    main(file_name="data.txt", n=5)


import sympy as sym
x, L, C, D, c_0, c_1, = sym.symbols('x L C D c_0 c_1')

def model1(f, L, D):
    """Solve -u'' = f(x), u(0)=0, u(L)=D."""
    # Integrate twice
    u_x = - sym.integrate(f, (x, 0, x)) + c_0
    u = sym.integrate(u_x, (x, 0, x)) + c_1
    # Set up 2 equations from the 2 boundary conditions and solve
    # with respect to the integration constants c_0, c_1
    r = sym.solve([u.subs(x, 0)-0,  # x=0 condition
                   u.subs(x,L)-D],  # x=L condition
                  [c_0, c_1])       # unknowns
    # Substitute the integration constants in the solution
    u = u.subs(c_0, r[c_0]).subs(c_1, r[c_1])
    u = sym.simplify(sym.expand(u))
    return u

def model2(f, L, C, D):
    """Solve -u'' = f(x), u'(0)=C, u(L)=D."""
    u_x = - sym.integrate(f, (x, 0, x)) + c_0
    u = sym.integrate(u_x, (x, 0, x)) + c_1
    r = sym.solve([sym.diff(u,x).subs(x, 0)-C,  # x=0 cond.
                   u.subs(x,L)-D],              # x=L cond.
                  [c_0, c_1])
    u = u.subs(c_0, r[c_0]).subs(c_1, r[c_1])
    u = sym.simplify(sym.expand(u))
    return u

def model3(f, a, L, C, D):
    """Solve -(a*u')' = f(x), u(0)=C, u(L)=D."""
    au_x = - sym.integrate(f, (x, 0, x)) + c_0
    u = sym.integrate(au_x/a, (x, 0, x)) + c_1
    r = sym.solve([u.subs(x, 0)-C,
                   u.subs(x,L)-D],
                  [c_0, c_1])
    u = u.subs(c_0, r[c_0]).subs(c_1, r[c_1])
    u = sym.simplify(sym.expand(u))
    return u

def demo():
    f = 2
    u = model1(f, L, D)
    print(('model1:', u, u.subs(x, 0), u.subs(x, L)))
    print((sym.latex(u, mode='plain')))
    u = model2(f, L, C, D)
    #f = x
    #u = model2(f, L, C, D)
    print(('model2:', u, sym.diff(u, x).subs(x, 0), u.subs(x, L)))
    print((sym.latex(u, mode='plain')))
    u = model3(0, 1+x**2, L, C, D)
    print(('model3:', u, u.subs(x, 0), u.subs(x, L)))
    print((sym.latex(u, mode='plain')))

if __name__ == '__main__':
    demo()


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Fri Feb  8 21:41:08 2019

@author: alankar
"""

#import numpy as np
#import matplotlib.pyplot as plt

def simpson(func, a, b, n,*args):
    h=(b-a)/n
    k=0.0
    x=a + h
    for i in range(1,int(n/2) + 1):
        k += 4*func(x,*args)
        x += 2*h

    x = a + 2*h
    for i in range(1,int(n/2)):
        k += 2*func(x,*args)
        x += 2*h
    return (h/3)*(func(a,*args)+func(b,*args)+k)

f = lambda x:x**4-2*x+1

print('\n\nSIMPSON RULE:')
n = 10
print('Slices = %d'%n)
I1 = simpson(f,0,2,n)

n = 20
I2 = simpson(f,0,2,n)

print('Fractional error estimate: %e'%abs((I2-I1)/I2))

"""
The results vary from the actual esult comparison of problem 2 for N=10
because of the truncation error associated with both I1 and I2.
I2 is closer to actual value but still off by the truncation error (which is smaller than I1)
"""


import matplotlib
import matplotlib.pyplot as plt
import matplotlib.animation as animation
matplotlib.use('Agg')

import numpy as np
import sys
from os.path import join, isfile

import warnings
warnings.filterwarnings("ignore")

def grad_desc(x, y, eta=1e-2, epsilon=1e-15):

    # eta = learning rate
    # epsilon = error
    # m = number of examples
    m = x.shape[0]

    # x[i][0] = 1 for intercept term
    x = np.hstack((np.ones(x.shape), x))

    # theta = parameters learnt by model
    theta = np.array([0., 0.])

    # number of iterations
    iterations = 0

    # all the iterations of theta
    theta_vals = []
    prev_cost_value = 0.0

    x = x.T
    y = y.T

    while True:

        # cost function and gradient calculation
        diff_errors = np.tile(np.matmul(theta[np.newaxis, :], x)[0] - y[0], (2, 1))
        grad = np.sum(diff_errors * x, axis=1) / m
        cost_value = np.sum(diff_errors[0] ** 2) / (2 * m)

        # append current value of theta to all theta values
        theta_vals.append(np.hstack((theta, np.array([cost_value]))))

        # change in the value of theta during gradient update
        diff = eta * grad
        theta = theta - diff
        iterations += 1

        # convergence criterion
        if abs(cost_value - prev_cost_value) < epsilon:
            break

        prev_cost_value = cost_value

    return np.array(theta_vals).T, theta, eta, ('change in cost function is less than ' + str(epsilon)), iterations

def main():

    # read command-line arguments
    data_dir = sys.argv[1]
    out_dir = sys.argv[2]
    part = sys.argv[3]

    # check for existence of input files
    for c in ['X', 'Y']:
        if not isfile(join(data_dir, 'linear' + c + '.csv')):
            raise Exception('linear' + c + '.csv not found')

    # read from csv file
    x = np.array([np.genfromtxt(join(data_dir, 'linearX.csv'))]).T
    y = np.array([np.genfromtxt(join(data_dir, 'linearY.csv'))]).T

    # normalization step
    x_mean = np.sum(x) / x.shape[0]
    x -= np.full_like(x, x_mean)
    x_stddev = np.sqrt(np.sum(x ** 2) / x.shape[0])
    x /= x_stddev

    # call gradient descent on the given data
    theta_vals, theta, learning_rate, stopping_criteria, total_iterations = grad_desc(x, y)

    if part == 'a':
        # write the output for 1a
        output_file = open(join(out_dir, '1aoutput.txt'), mode='w')
        output_file.write('learning_rate = ' + str(learning_rate) + '\n')
        output_file.write('stopping_criteria = ' + stopping_criteria + '\n')
        output_file.write('theta_0 = ' + str(theta[0]) + '\n')
        output_file.write('theta_1 = ' + str(theta[1]) + '\n')
        output_file.write('total_iterations = ' + str(total_iterations) + '\n')
        output_file.close()
        print('learning_rate = ' + str(learning_rate))
        print('stopping_criteria = ' + stopping_criteria)
        print('theta_0 = ' + str(theta[0]))
        print('theta_1 = ' + str(theta[1]))
        print('total_iterations = ' + str(total_iterations))
        return 0

    # PART B: plot the graphs for 1b

    fig1b, ax1b = plt.subplots()
    ax1b.scatter(x * x_stddev + x_mean, y)
    X0 = np.arange(-2, 5, 0.1)
    ax1b.plot(X0 * x_stddev + x_mean, theta[0] + theta[1] * X0)

    ax1b.set_xlabel('Acidity')
    ax1b.set_ylabel('Density')
    if part == 'b':
        fig1b.savefig(join(out_dir, 'regression_plot.png'))
        plt.show()
        return 0
    plt.close(fig1b)

    # PART C: plot the graph for 1c

    # X, Y, Z - theta_0, theta_1, cost function
    (X, Y), Z = np.meshgrid(np.linspace(-0.5, 2, 1000), np.linspace(-0.7, 0.7, 1000)), 0
    for i in range(x.shape[0]):
        Z += ((X + Y * x[i][0]) - y[i]) ** 2
    Z /= 2 * x.shape[0]

    # actually starting the plot
    fig1c = plt.figure()
    ax1c = fig1c.gca(projection='3d')
    ax1c.plot_surface(X, Y, Z)
    ax1c.set_xlabel('Theta_0')
    ax1c.set_ylabel('Theta_1')
    ax1c.set_zlabel('Cost function')
    plot1c = ax1c.plot([theta_vals[0][0]], [theta_vals[1][0]], [theta_vals[2][0]])

    # update function for animation
    def update1c(nums):
        plot1c[0].set_data(theta_vals[0:2, :nums])
        plot1c[0].set_3d_properties(theta_vals[2, :nums])
        return plot1c

    # performing the animation
    anim1c = animation.FuncAnimation(fig1c, update1c, theta_vals.shape[1], interval=200, blit=True)
    update1c(theta_vals.shape[1])
    if part == 'c':
        fig1c.savefig(join(out_dir, '1clast_frame.png'))
        plt.show()
        return 0
    plt.close(fig1c)

    # PART D: plot the graph for 1d

    fig1d, ax1d = plt.subplots()
    ax1d.contour(X, Y, Z, 100)
    ax1d.set_xlabel('Theta_0')
    ax1d.set_ylabel('Theta_1')
    plot1d = ax1d.plot([theta_vals[0, 0]], [theta_vals[1, 0]])
    # update function for animation
    def update1d(nums):
        plot1d[0].set_data(theta_vals[0:2, :nums])
        return plot1d
    # performing the animation
    anim1d = animation.FuncAnimation(fig1d, update1d, theta_vals.shape[1], interval=200, blit=True)
    update1d(theta_vals.shape[1])
    if part == 'd':
        fig1d.savefig(join(out_dir, '1dlast_frame.png'))
        plt.show()
        return 0
    plt.close(fig1d)

    # PART E: plot the graphs for 1e

    learning_parameters = [(1e-3, 1e-15), (25e-2, 1e-15), (1e-1, 1e-15)]

    eta, epsilon = learning_parameters[0]
    theta_vals1, _, _, _, _ = grad_desc(x, y, eta, epsilon)

    fig1e1, ax1e1 = plt.subplots()
    ax1e1.contour(X, Y, Z, 100)
    ax1e1.set_xlabel('Theta_0')
    ax1e1.set_ylabel('Theta_1')

    plot1e1 = ax1e1.plot([theta_vals1[0, 0]], [theta_vals1[1, 0]])

    # update function for animation
    def update1e1(nums):
        plot1e1[0].set_data(theta_vals1[0:2, :nums])
        return plot1e1

    # performing the animation
    anim1e1 = animation.FuncAnimation(fig1e1, update1e1, theta_vals1.shape[1], interval=200, blit=True)
    update1e1(theta_vals1.shape[1])
    if part == 'e':
        fig1e1.savefig(join(out_dir, '1e1last_frame.png'))

    eta, epsilon = learning_parameters[1]
    theta_vals2, _, _, _, _ = grad_desc(x, y, eta, epsilon)

    fig1e2, ax1e2 = plt.subplots()
    ax1e2.contour(X, Y, Z, 100)
    ax1e2.set_xlabel('Theta_0')
    ax1e2.set_ylabel('Theta_1')

    plot1e2 = ax1e2.plot([theta_vals2[0, 0]], [theta_vals2[1, 0]])

    # update function for animation
    def update1e2(nums):
        plot1e2[0].set_data(theta_vals2[0:2, :nums])
        return plot1e2

    # performing the animation
    anim1e2 = animation.FuncAnimation(fig1e2, update1e2, theta_vals2.shape[1], interval=200, blit=True)
    update1e2(theta_vals2.shape[1])
    if part == 'e':
        fig1e2.savefig(join(out_dir, '1e2last_frame.png'))

    eta, epsilon = learning_parameters[2]
    theta_vals3, _, _, _, _ = grad_desc(x, y, eta, epsilon)

    fig1e3, ax1e3 = plt.subplots()
    ax1e3.contour(X, Y, Z, 100)
    ax1e3.set_xlabel('Theta_0')
    ax1e3.set_ylabel('Theta_1')

    plot1e3 = ax1e3.plot([theta_vals3[0, 0]], [theta_vals3[1, 0]])

    # update function for animation
    def update1e3(nums):
        plot1e3[0].set_data(theta_vals3[0:2, :nums])
        return plot1e3

    # performing the animation
    anim1e3 = animation.FuncAnimation(fig1e3, update1e3, theta_vals3.shape[1], interval=200, blit=True)
    update1e3(theta_vals3.shape[1])
    if part == 'e':
        fig1e3.savefig(join(out_dir, '1e3last_frame.png'))

    if part == 'e':
        plt.show()

    return 0

if __name__ == "__main__":
    main()


#Ref: Sreenivas Sarwar Anik

"""
Spyder Editor

scipy.signal.convolve2d - https://docs.scipy.org/doc/scipy/reference/generated/scipy.signal.convolve2d.html
scipy.ndimage.filters.convolve
cv2.filter2D - https://docs.opencv.org/2.4/modules/imgproc/doc/filtering.html?highlight=filter2d#filter2d

"""
import cv2
import numpy as np
from scipy.signal import convolve2d
from scipy.ndimage.filters import convolve
from skimage import io, img_as_float

img_gaussian_noise = img_as_float(io.imread('images/BSE_25sigma_noisy.jpg', as_gray=True))
img_salt_pepper_noise = img_as_float(io.imread('images/BSE_salt_pepper.jpg', as_gray=True))

img = img_salt_pepper_noise

#convolving the image with a normalized box filter.
#It simply takes the average of all the pixels under kernel area
#and replaces the central element with this average.
#Can also use cv2.blur() to directly apply the convolution rather than defining
#kernel and applying it as a filter (or convolution) separately.
kernel = np.ones((5,5),np.float32)/25    #Averaging filter with 5x5 kernel
#Normalize by dividing with 25 so all numbers add to 1
gaussian_kernel = np.array([[1/16, 1/8, 1/16],   #3x3 kernel
                [1/8, 1/4, 1/8],
                [1/16, 1/8, 1/16]])

laplacian = np.array([[0.,1,0], [1,-4,1], [0,1,0]])
gabor = cv2.getGaborKernel((5, 5), 1.4, 45, 5, 1)

"""
Results from all approaches would be the same..
Remember that if the padding (border) is not done then the output image
and values would be different

"""

conv_using_cv2 = cv2.filter2D(img, -1, kernel, borderType=cv2.BORDER_CONSTANT)
# when ddepth=-1, the output image will have the same depth as the source
#example, if input is float64 then output will also be float64
# BORDER_CONSTANT - Pad the image with a constant value (i.e. black or 0)
#BORDER_REPLICATE: The row or column at the very edge of the original is replicated to the extra border.

conv_using_scipy = convolve2d(img, kernel, mode='same')
#mode ="same" - pads image so the output is same as input

conv_using_scipy2 = convolve(img, kernel, mode='constant', cval=0.0)
#mode=constant adds a constant value at the borders.

cv2.imshow("Original", img)
cv2.imshow("cv2 filter", conv_using_cv2)
cv2.imshow("Using scipy", conv_using_scipy)
cv2.imshow("Using scipy2", conv_using_scipy2)

cv2.waitKey(0)
cv2.destroyAllWindows()


import numpy as np
import matplotlib.pyplot as plt
from scipy.interpolate import CubicSpline
from scipy.optimize import curve_fit

'''
O codigo de geracao de splines cubicas e seus graficos foi modificado da seguinte fonte:
https://docs.scipy.org/doc/scipy/reference/generated/scipy.interpolate.CubicSpline.html
'''
#Funcao do enunciado
def f(x):
    return 1.0/(1.0 + 25.0*x*x)

x_k = np.linspace(2, 28, 26) #Vetor contendo o valor para k utilizados em cada iteracao
e_k = [] #Vetor para armazenar o erro maximo para determinado k de x_k[], correspondendo a k+1 pontos
xOriginal = np.linspace(-1.0, 1.0, 1000) #Recurso usado para impressao das funcoes
yOriginal = f(xOriginal)

#Calculo do erro maximo para i = k+1 pontos utilizados

#Splines Naturais
for i in range(3, 29, 1):

    #Gerando pontos igualmente espacados e seus respectivos valores em f(x)
    xPontos = np.linspace(-1.0, 1.0, i)
    yPontos = f(xPontos)

    #cs armaneza a spline cubica propriamente dita
    cs = CubicSpline(xPontos, yPontos, bc_type='natural')

    xcs = np.linspace(-1.0, 1.0, 1000) #Recurso usado para impressao da spline
    ycs = cs(xcs)

    #Calculo do erro
    e_aux = abs(f(xOriginal) - cs(xOriginal))
    e_k.append(float(np.amax(e_aux)))

'''
#Plot da funcao spline cubica
plt.plot(xOriginal,yOriginal,label = "Função Original")
plt.plot(xPontos, yPontos,'o',label = "Pontos")
plt.plot(xcs, ycs, label="Spline cubica")
plt.xlim(-1.5, 1.5)
plt.ylim(-0.4, 1.2)
plt.title('Funcao original e spline cubica')
plt.xlabel('x')
plt.ylabel('y')
plt.legend(loc='upper left')

plt.show()
'''

#Plot do erro para spline natural
plt.plot(x_k, e_k, label = "Função Erro")
plt.plot(x_k, e_k, 'o', label = "Pontos do Erro")
plt.title('Erro em funcao de k')
plt.xlabel('k')
plt.ylabel('Erro')
plt.legend(loc='upper right')
plt.xticks(range(0, 32, 2))
plt.show()

#Aproximacao Ch^q para Splines Naturais
'''
Com n pontos -> dividimos o intervalo [-1, 1], de tamanho 2, em (n-1) partes.
Assim os pontos distam: h = 2/(n-1)
'''
#Definindo o formato da funcao que relaciona o erro e a distancia entre os pontos
def errorFunc(h, C, q):
    return C*(h**q)

#Preenchendo vetor que representa a distancia entre os pontos para dado numero de pontos
h = []

for j in range(3, 29, 1):
    h.append(2 / (j-1))

#Usando minimos quadrados para encontrar os parametros C e q
parametros = curve_fit(errorFunc, h, e_k)
[C, q] = parametros[0]

print('\nNa aproximacao Ch^q para spline natural, temos:','\nC = ', C, '\nq = ', q, '\n')

#Definindo a funcao errorFunc com os parametros corretos
def error(h):
    return C*(h**q)

x_error = np.linspace(0.05, 1.1, 1000)
y_error = error(x_error)

#Plot dos pontos e da funcao error
plt.plot(h, e_k, 'o', label = 'Pontos do erro')
plt.plot(x_error, y_error, label = 'Curva ajustada por minimos quadrados (Ch^q)')
plt.title('Valor do erro em funcao da distancia entre os pontos (para spline natural)')
plt.xlabel('Distancia entre os pontos (h)')
plt.ylabel('Erro')
plt.legend(loc='upper left')
plt.show()

####### SPLINE COM DERIVADA CONHECIDA NOS EXTREMOS #######
'''
Como f'(-1) = 0.07396 e f'(1) = -0.07396, temos que as novas condicoes de contorno
sao cs''(-1) = 0.07396 e cs''(1) = -0.07396.
'''

#Calculando o erro
e_k_2 = []

for i in range(3, 29, 1):

    #Gerando pontos igualmente espacados e seus respectivos valores em f(x)
    xPontos_2 = np.linspace(-1.0, 1.0, i)
    yPontos_2 = f(xPontos_2)

    #cs armaneza a spline cubica propriamente dita
    cs2 = CubicSpline(xPontos_2, yPontos_2, bc_type = ((1, 0.07396),(1, -0.07396)))

    #Calculo do erro
    e_aux2 = abs(f(xOriginal) - cs2(xOriginal))
    e_k_2.append(float(np.amax(e_aux2)))

#Aproximacao Ch^q para Splines com Derivada Conhecida nos Extremos

#Usando minimos quadrados para encontrar os parametros C e q
parametros2 = curve_fit(errorFunc, h, e_k_2)
[C2, q2] = parametros2[0]

print('\nNa aproximacao Ch^q para derivada conhecida nos extremos, temos:','\nC = ', C2, '\nq = ', q2, '\n')

#Definindo a funcao errorFunc com os parametros corretos
def error_2(h):
    return C2*(h**q2)

x_error2 = np.linspace(0.05, 1.1, 1000)
y_error2 = error_2(x_error)

#Plot dos pontos e da funcao error
plt.plot(h, e_k_2, 'o', label = 'Pontos do erro')
plt.plot(x_error2, y_error2, label = 'Curva ajustada por minimos quadrados (Ch^q)')
plt.title('Valor do erro em funcao da distancia entre os pontos (derivada conhecida nos extremos)')
plt.xlabel('Distancia entre os pontos (h)')
plt.ylabel('Erro')
plt.legend(loc='upper left')
plt.show()


#!/usr/bin/env python3

import numpy as np
import math as m

A = [[4, 12, -16],[12, 37, -43], [-16, -43, 98]]
A = np.array(A)

# shape[0] --> lignes i
# shape[1] --> colonnes j

def cholesky(A):
    '''Returns cholesky decomposition of A = L * L.T'''

    n = A.shape[0]
    L = np.zeros((n, n))

    for i in range(n):
        L[i][i] = m.sqrt(A[i][i] - np.dot(L[i][:],L[i][:]))
        for j in range(i+1, n):
            L[j][i] = (A[j][i] - np.dot(L[i,:], L[j, :]))/L[i][i]
    return L
#print(cholesky(A))

########## THINKING OF THE ABY IMPLEMENTATION #########
#-----------------------------------------------------#

# Okay.. Let's rewrite this code as a lower level as possible!

# New input data as form of a list
A = [4, 12, -16, 12, 37, -43, -16, -43, 98]
n = 3
L = [0, 0, 0, 0, 0, 0, 0, 0, 0]
def cholesky_dec(A):

    # Initiating empty matrix L
    L = [0]*len(A)
    n = m.sqrt(len(A))

    for i in range(n):
        # Calculating the np.dot inside the square root..
        mul = 0
        for k in range(n): mul += L[i*n+k]**2
        # Getting diagonal elements
        L[i*n+i] = m.sqrt(A[i*n+i] - mul)
        for j in range(i+1, n):
            mul = 0
            for k in range(n):
                mul += L[i*n+k]*L[j*n+k]
            # Getting the [j][i] element
            L[j*n+i] = (A[j*n+i]-mul)/L[i*n+i]
    return L


import numpy as np

def gaussian_kernel(x1, x2, sigma):
    """
    Implementing a gaussian kernel function.
    Although scikit-learn has a gaussian kernel built in, for transparency we'll implement one from scratch:
    """
    return np.exp(-(np.sum((x1 - x2) ** 2) / (2 * (sigma ** 2))))

# x1 = np.array([1.0, 2.0, 1.0])
# x2 = np.array([0.0, 4.0, -1.0])
# sigma = 2
# gaussian_kernel(x1, x2, sigma)


"""

identity

The identity tool returns an identity array. An identity array is a square matrix with all the main diagonal elements as
and the rest as

. The default type of elements is float.

import numpy
print numpy.identity(3) #3 is for  dimension 3 X 3

#Output
[[ 1.  0.  0.]
 [ 0.  1.  0.]
 [ 0.  0.  1.]]

eye

The eye tool returns a 2-D array with
's as the diagonal and 's elsewhere. The diagonal can be main, upper or lower depending on the optional parameter . A positive is for the upper diagonal, a negative is for the lower, and a

(default) is for the main diagonal.

import numpy
print numpy.eye(8, 7, k = 1)    # 8 X 7 Dimensional array with first upper diagonal 1.

#Output
[[ 0.  1.  0.  0.  0.  0.  0.]
 [ 0.  0.  1.  0.  0.  0.  0.]
 [ 0.  0.  0.  1.  0.  0.  0.]
 [ 0.  0.  0.  0.  1.  0.  0.]
 [ 0.  0.  0.  0.  0.  1.  0.]
 [ 0.  0.  0.  0.  0.  0.  1.]
 [ 0.  0.  0.  0.  0.  0.  0.]
 [ 0.  0.  0.  0.  0.  0.  0.]]

print numpy.eye(8, 7, k = -2)   # 8 X 7 Dimensional array with second lower diagonal 1.

Task

Your task is to print an array of size
X with its main diagonal elements as 's and

's everywhere else.

Input Format

A single line containing the space separated values of
and .
denotes the rows.

denotes the columns.

Output Format

Print the desired
X

array.

Sample Input

3 3

Sample Output

[[ 1.  0.  0.]
 [ 0.  1.  0.]
 [ 0.  0.  1.]]
"""

import numpy
print(str(numpy.eye(*map(int,input().split())))
      .replace('1',' 1').replace('0',' 0'))


import numpy as np
from scipy import stats
import matplotlib.pyplot as plt
from math import sqrt

########################################################################################
# I.

# it is important to fix a seed in order to have the same random numbers every time we run the program
# therefore, we can compare every run with different parameters
# if we use different random numbers for each one, the comparison depends on the randomness
np.random.seed(10)
N = 10**4 #number of points of the data sample
mean = np.array([2,1])
sigma_1 = 1
sigma_2 = 2
rho = 0.999
cov = [[sigma_1**2, rho*sigma_1*sigma_2], [rho*sigma_1*sigma_2, sigma_2**2]]

L = np.linalg.cholesky(cov)
# print(L)

uncorrelated = np.random.normal(0,1,(2,N))

#now we apply the transformation

# we cannot do broadcasting if both arrays do not have the same dimensions (ie 2D)
# reshape of mean in order to do broadcasting
data = mean.reshape(2,1) + np.dot(L, uncorrelated)
# print(mean.reshape(2,1)[0,0], mean.reshape(2,1)[1,0])
# print(data.shape)
# (2, N)

plt.figure(1)
plt.scatter(data[0,:], data[1,:], color = "black", s = 0.1)
plt.xticks(np.arange(-12,12,1))
plt.yticks(np.arange(-10,10,1))
plt.xlim(-6,10)
plt.ylim(-7,9)
plt.xlabel(r"$x_1$")
plt.ylabel(r"$x_2$")

#################################################
# alternative: if instead we use
"""
plt.figure(2)
data2 = np.random.multivariate_normal(mean, cov, N)
plt.scatter(data2[:,0], data2[:,1], color = "black", s = 1)
"""

# plt.show()

########################################################################################
# III.

def posterior_pdf_2(mu):
    """
    this is the posterior pdf computed in II. We want to sample this for different values of mu.
    this function brings problems as explained at the end of the function
    """
    mean_1 = mu[0]
    mean_2 = mu[1]
    x_1 = data[0, :]  # data fixed from the first experiment
    x_2 = data[1, :]

    posterior = np.exp(-1 / (2 * (1 - rho ** 2)) * (
    (x_1 - mean_1) ** 2 / sigma_1 ** 2 - 2 * rho * (x_1 - mean_1) * (x_2 - mean_2) / (sigma_1 * sigma_2) +
    (x_2 - mean_2) ** 2 / sigma_2 ** 2))

    # for the moment we have an array 1D (N,)
    # print(posterior, np.shape(posterior))
    return posterior

    # if we perform the product of all the elements
    # posterior = np.prod(posterior)
    # return posterior
    # np.shape(posterior)
    # shape posterior (,) it is a number
    # PROBLEM: the posterior pdf computed alone with the prod operator, gives 0 because overflow
    # But if we compute the ratio, this gives =/ 0
    # Precisely, that's why we use MCMC and we don't plot directly from the pdf

def posterior_pdf(mu):
    """
    this is the posterior pdf computed in II. We want to sample this for different values of mu.
    It's an alternative method using log in order to evade numerical underflow
    """
    mean_1 = mu[0]
    mean_2 = mu[1]
    x_1 = data[0, :]  # data fixed from the first experiment
    x_2 = data[1, :]

    log_posterior = (x_1 - mean_1) ** 2 / sigma_1 ** 2 - 2. * rho * (x_1 - mean_1) * (x_2 - mean_2) / (
        sigma_1 * sigma_2) + (x_2 - mean_2) ** 2 / sigma_2 ** 2
    log_posterior = -(1. / (2. * (1 - rho ** 2))) * np.sum(log_posterior)

    # for the moment we a number, but still we need to perform the exponential of this
    return log_posterior

def proposal_distribution(mu):
    global c #we define c as global, because later (outside the function) we want to print this value
    # mu must be 1D array here
    cov_diag = np.array([[sigma_1 ** 2, 0], [0, sigma_2 ** 2]])
    #    c = 2.4/sqrt(2.) #in our case this optimal value brings problems
    c = 0.001
    proposal = np.random.multivariate_normal(mu, c ** 2 * cov_diag)
    return proposal
    # np.shape(proposal), np.shape(mu)
    # (2,) (2,) ie both are 1-D

def met_hast(f, proposal, old):
    """
    metropolis_hastings algorithm.
    allows proposal asymetric
    _f_ is the unnormalized density function to sample, i.e. the posterior pdf
    _proposal_ is the proposal distirbution J
    _old_ is the value mu^{t-1}, i.e. the last iteration
    """

    # we sample mu* from the proposal distribution
    new = proposal(old)

    # we must be careful with the overflow problems in the ratio.
    # the posterior pdf computed alone with the prod operator, gives 0

    ratio_posterior_pdf_log = f(new) - f(old)
    # print(ratio_posterior_pdf_log)  # this is a number

    ratio_posterior_pdf = np.exp(ratio_posterior_pdf_log)
    # print(ratio_posterior_pdf)

    ratio = ratio_posterior_pdf * proposal(new)/new
    # note we call new = proposal(old) instead of proposal(old) because if we use proposal(old)
    # we would be calling proposal(old) again, so it will be different than new because is random.
    # print(ratio, np.shape(ratio))
    # returns [ratio_1, ratio_2]
    # shape (2,) 1-D array

    # alpha is the acceptance probability, can be 1 or less. That's why we have np.min
    # if it is 1, the acceptance ratio will be 100%
    # we have 2 ratios, then we have to compare 2 objects
    alpha_1 = np.min([ratio[0], 1])
    alpha_2 = np.min([ratio[1], 1])

    # now we start the acceptance or rejection
    # we generate a random uniform number u and we compare it with alpha_1 and alpha_2
    u = np.random.uniform()
    cnt = 0
    if (u <= alpha_1 and u <= alpha_2):
        old = new #this is a 1D array (2,)
        cnt = 1

    print(old)
    return old, cnt
    # if accepted we return old = new and cnt=1, if rejected we return old and cnt=0

def run_chain(chainer, f, proposal, start, n, take=1):
    """
    _chainer_ is the method used: Metropolis, MH, Gibbs ...
    _f_ is the unnormalized density function to sample, i.e. the posterior pdf
    _proposal_ is the proposal distirbution J
    _start_ is the initial start of the Markov Chain, i.e. the initial value of mu_1,mu_2
    _start_ must be an array 1D (2,), ie a list.
    _n_ length of the chain. We can modify it to improve the convergance
    _take_ thinning
    """
    # we initialise the counter of the number of accepted values
    count = 0
    # samples recolect all the mu_1,mu_2 values along the chain
    # but we want to have an array (2, n)
    samples = np.array(start.reshape(2,1))
    for i in range(n): # we iterate
        print(i)
        start, c = chainer(f, proposal, start) # start will be the value of mu_1,mu_2, it's no longer the initial value
        count = count + c # this count only adds +1 when is mu* accepted
        if i%take is 0: # we recolect the values for each iteration, even if not accepted
            samples = np.append(samples, start.reshape(2,1), axis=1)
            # now samples will be an array (2, n+1)
            # ie two rows and n+1 columns. The two rows correspond to mu_1 and mu_2
    return samples, count

start_point = np.array([0,0])
n_iterations = 30000 #number of iterations of the chain
samples, count = run_chain(met_hast, posterior_pdf, proposal_distribution, start=start_point, n=n_iterations)

#print(samples)
#print(np.shape(samples))

# we apply the burn-in period
# the conventional choice is to discard the first half (gelman pag 297)
# however, analyzing the values at each iteration, we see that from 1000 iteration there is already convergence

burn_in = 15000
samples_burn_in = samples[:, burn_in : n_iterations+1]
# remember slices must be integers that's why we use //
print(np.shape(samples_burn_in))

# now we compute the sample mean and sample standard deviation
# we select the axis 1 to obtain 2 means: mean_1 mean_2
# the standard deviation is biased with ddof=0 and ddof=1
# the var is the unbiased estimator for ddof=1
sample_mean = np.mean(samples_burn_in, axis=1)
standard_deviation_mean = np.std(samples_burn_in, axis=1, ddof=1)
print("number iterations =", n_iterations)
print("c =", c)
print("burn-in =", burn_in)
print('Acceptance fraction:', count / float(n_iterations))
print("sample mean:", sample_mean)
print("sample standard deviation:", standard_deviation_mean)

# we try different values of c in order to find an appropiate acceptance fraction
# and we want the sample mean and standard deviation to fit with the true values

plt.figure(3)
plt.scatter(samples_burn_in[0,:], samples_burn_in[1,:], color = "black", s = 0.1)
plt.xticks(np.arange(1.90, 2.10, 0.02))
plt.yticks(np.arange(0.90, 1.10, 0.02))
plt.xlim(1.92,2.08)
plt.ylim(0.92,1.08)
plt.xlabel(r"$\mu_1$")
plt.ylabel(r"$\mu_2$")

np.savetxt("samples_burn_in", samples_burn_in)

plt.show()

###########################################################################################
# IV.
# Gibbs Sampler
# we have two parameters mu_1 and mu_2. Then two steps per iteration t.

# we define a function for Gibbs which fits with the run_chain function.
def gibbs(mu):
    """
    Gibbs sampling algorithm
    _mu_ is the value of mu^{t-1}, ie the last iteration. It must be an array 1D (2,) (mu_1, mu_2)
    output: mu of the iteration t
    """

    mu_1 = mu[0] #note that we don't use this value
    mu_2 = mu[1]
    x_1 = data[0, :]  # data fixed from the first experiment
    x_2 = data[1, :]

    x_1_mean = np.mean(x_1) # number
    x_2_mean = np.mean(x_2)

    # we choose the order (mu_1, mu_2)
    # we start sampling mu_new_1

    # be careful the arguments of the normal are the mean and the std

    normal_mean_1 = x_1_mean + rho * sigma_1 / sigma_2 * (mu_2 - x_2_mean)
    normal_std_1 = sqrt(sigma_1 ** 2 * (1 - rho ** 2) / N)
    # they are numbers

    mu_new_1 = np.random.normal(normal_mean_1, normal_std_1)
    # number

    # now we sample mu_new_2  using mu_new_1
    normal_mean_2 = x_2_mean + rho * sigma_2 / sigma_1 * (mu_new_1 - x_1_mean)
    normal_std_2 = sqrt(sigma_2 ** 2 * (1 - rho ** 2) / N)

    mu_new_2 = np.random.normal(normal_mean_2, normal_std_2)

    mu_new = np.array([mu_new_1, mu_new_2])
    print(mu_new)

    cnt = 1
    return mu_new, cnt

""" #######################################################################
    Here there is a wrong attempt.
    I tried to compute mu_new_1 as     mu_new_1 = np.prod(np.random.normal(normal_mean_1, normal_std_1))
    where i computed a one-d gaussian for every data, ie 10^4 gaussian and then the product.
    the result was clearly bigger than the desired and it had overflow problems.

    # be careful the arguments are the mean and the std
    normal_mean_1 = x_1 - rho*sigma_1*(x_2-mu_2)/sigma_2
    normal_std_1 = sqrt(sigma_1**2*(1-rho**2))
    # print(np.shape(normal_mean_1))
    # (N, ) 1D array
    # print(np.shape(normal_std_1))
    # () scalar

    mu_new_1 = np.random.normal(normal_mean_1, normal_std_1)
    # print(mu_new_1)
    # print(np.shape(mu_new_1))
    # (N,) 1D array

    # for the moment, for any value of the data, we have obtained one mu_1
    # now we have to make the product of all of them so we obtain only one mu_1
    # but we have overflow problems. It is obvious because we are multiplying sth 10**4 times
    # way out: apply log

    # however we can have log(negative number) which gives nan.
    # in order to solve this, we perform a mask which filters out the negative values
    # at the end, i think we can make this approximation because most of the values are positive

    check the values of mu_new_1
    for i in range(0,N):
        print(mu_new_1[i])

    # we perform the mask
    mask = mu_new_1 > 0
    mu_new_1 = mu_new_1[mask]

    # check the values of mu_new_1 after the mask
    for i in range(0,len(mu_new_1)):
        print(mu_new_1[i])

    # we perform the natural logarithm of the array element-wise
    mu_new_1 = np.log(mu_new_1)

    #then we sum each element
    mu_new_1 = np.sum(mu_new_1)

    # now we sample mu_new_2  using mu_new_1
    normal_mean_2 = x_2 - rho*sigma_2*(x_1-mu_new_1)/sigma_1
    normal_std_2 = sqrt(sigma_2**2*(1-rho**2))

    mu_new_2 = np.random.normal(normal_mean_2, normal_std_2)

    # we perform the natural logarithm of the array element-wise
    mu_new_2 = np.log(mu_new_2)

    #then we sum each element
    mu_new_2 = np.sum(mu_new_2)

    mu_new = np.array([mu_new_1, mu_new_2])
    print(mu_new)

    cnt = 1
    return mu_new, cnt

    """

def run_chain_gibbs(chainer, start, n, take=1):
    """
    _chainer_ is the method used: Metropolis, MH, Gibbs ...
    _start_ is the initial start of the Markov Chain, i.e. the initial value of mu_1,mu_2
    _start_ must be an array 1D (2,), ie a list.
    _n_ length of the chain. We can modify it to improve the convergance
    _take_ thinning
    """
    # we initialise the counter of the number of accepted values
    count = 0
    # samples recolect all the mu_1,mu_2 values along the chain
    # but we want to have an array (2, n)
    samples = np.array(start.reshape(2,1))
    for i in range(n): # we iterate
        start, c = chainer(start) # start will be the value of mu_1,mu_2, it's no longer the initial value
        print(i)
        count = count + c # this count only adds +1 when is mu* accepted
        if i%take is 0: # we recolect the values for each iteration, even if not accepted
            samples = np.append(samples, start.reshape(2,1), axis=1)
            # now samples will be an array (2, n+1)
            # ie two rows and n+1 columns. The two rows correspond to mu_1 and mu_2
    return samples, count

"""
start_point = np.array([0,0])
n_iterations = 30000 #number of iterations of the chain
samples, count = run_chain_gibbs(gibbs, start=start_point, n=n_iterations)

# print(samples)
print(np.shape(samples))

# we apply the burn-in period
# the conventional choice is to discard the first half (gelman pag 297)

burn_in = 15000
samples_burn_in = samples[:, burn_in : n_iterations+1]
# remember slices must be integers that's why we use //
#print(samples_burn_in)
print(np.shape(samples_burn_in))

# now we compute the sample mean and sample standard deviation
# we select the axis 1 to obtain 2 means: mean_1 mean_2
# the standard deviation is biased with ddof=0 and ddof=1
# the var is the unbiased estimator for ddof=1
sample_mean = np.mean(samples_burn_in, axis=1)
standard_deviation_mean = np.std(samples_burn_in, axis=1, ddof=1)
print("number iterations =", n_iterations)
print("burn-in =", burn_in)
print('Acceptance fraction:', count / float(n_iterations))
print("sample mean:", sample_mean)
print("sample standard deviation:", standard_deviation_mean)

# we try different values of c in order to find an appropiate acceptance fraction
# and we want the sample mean and standard deviation to fit with the true values

plt.figure(4)
plt.scatter(samples_burn_in[0,:], samples_burn_in[1,:], color = "black", s = 0.1)
plt.xticks(np.arange(1.90, 2.10, 0.02))
plt.yticks(np.arange(0.90, 1.10, 0.02))
plt.xlim(1.92,2.08)
plt.ylim(0.92,1.08)
plt.xlabel(r"$\mu_1$")
plt.ylabel(r"$\mu_2$")

np.savetxt("samples_burn_in", samples_burn_in)

plt.show()
"""

Projects/Parameter Estimation and Bayesian Statistics/_3_Exercise_3.py


import numpy as np
import warnings

warnings.filterwarnings('ignore')

x = np.arange(12)

print(x)
# Dimension of Numpy Array
print(x.shape)

# No. of elements in Array
print(x.size)

# Change the shape of a Tensor without altering either the number of elements or their values
X = x.reshape(3, 4)
print(X)

# Automatically Inferring One Dimension by providing rest and putting -1 for Unknown Dim
X1 = x.reshape(-1, 4)
X2 = x.reshape(3, -1)

print(X1)
print(X2)

# Setting Arrays with Initial Value Zero
print('Setting Arrays with Initial Value Zero')
A1 = np.zeros((2, 3, 4))
print(A1)

# Setting Arrays with Initial Value One
print('Setting Arrays with Initial Value One')
A2 = np.ones((2, 3, 4))
print(A2)

# Random Initialization of Tensors and Type of Distribution Used
# Gaussian or Normal Distribution
X3 = np.random.normal(0, 1, size=(3, 4))  # Ist Para = Mean, IInd Para = Standard Deviation

# Converting Python List / Array to Numpy Array
X4 = np.array([[1, 2, 3], [4, 5, 6]])

## Operations

# Elementwise Operations
x = np.array([1, 2, 4, 8])
y = np.array([2, 2, 2, 2])
sum_op = x + y
sub_op = x - y
mul_op = x * y
div_op = x / y
exp_op = x ** y

# Unary Exponentiation - Element-wise Exponentiation (e^x)
uExp = np.exp(x)

# Concatenation - (Axis along which to concatenate is to be mentioned)
X = np.arange(12).reshape(3, 4)
Y = np.array([[2, 1, 4, 3], [1, 2, 3, 4], [4, 3, 2, 1]])
# Axis == 1: Means along column and Axis == 0: Means along row Here
np.concatenate([X, Y], axis=0), np.concatenate([X, Y], axis=1)

# Binary Tensor (Obtained by equating 2 Tensor: Each element = True if same value in both Tensors else: False)
bin_tensor = X == Y
print('Binary Tensor Obtained from Tensors X and Y is:')
print(bin_tensor)

# Summing all Elements in a Tensor
sum_all = X.sum()
print(sum_all)

## BroadCasting Mechanism: (Element-wise Operation on 2 Tensors with Different Dimensions)
a = np.arange(3).reshape(3, 1)
b = np.arange(2).reshape(1, 2)
print('a:', a)
print('b:', b)
print('BroadCast Result:', a + b)

## Indexing and Slicing

# Indexing
a = np.arange(12)
print('Last Element:', a[-1])
print('Access Matrix', b[0, 1])

a = np.arange(12).reshape(3, 4)
print(a)

# Assigning Multiple Indices Same Value
a[0:2, :] = 12
print('Setted 1st 2 row to 12:', a)

## Saving Memory

# For example, if we
# write Y = X + Y, we will dereference the tensor that Y used to point to and instead point Y at
# the newly allocated memory.
save_id = id(Y)
Y = Y + X
new_id = id(Y)
print('Is both Y same:', save_id == new_id)

# Handling Memory Issue
Z = np.zeros_like(Y)
print('Initial Id:', id(Z))
Z[:] = X + Y
print('Id After Op:', id(Z))
# Other Method

save_id = id(X)
X += Y
print('Is Id Same?', id(X) == save_id)

## Type Conversions

print(type(Y))
a = np.array([3.2])
print(a, a.item(), int(a), float(a))


from scipy.special import softmax
import numpy as np

"""
Self Attn
https://colab.research.google.com/github/mrm8488/shared_colab_notebooks/blob/master/basic_self_attention_.ipynb

Attentions
https://www.youtube.com/watch?v=S27pHKBEp30

Position Embeddings
https://www.youtube.com/watch?v=dichIcUZfOw

@ symbol
https://www.python.org/dev/peps/pep-0465/#semantics

"""
x = np.array([
  [1, 0, 1, 0],   # Input 1
  [0, 2, 0, 2],   # Input 2
  [1, 1, 1, 1],   # Input 3
  [1, 2, 1, 2],   # Input 4
  [2, 2, 2, 2],   # Input 5
 ])

w_key = np.array([
  [0, 0, 1],
  [1, 1, 0],
  [0, 1, 0],
  [1, 1, 0]
])
w_query = np.array([
  [1, 0, 1],
  [1, 0, 0],
  [0, 0, 1],
  [0, 1, 1]
])
w_value = np.array([
  [0, 2, 0],
  [0, 3, 0],
  [1, 0, 3],
  [1, 1, 0]
])
key = []
query = []
value = []

# Generate Query, Key, and Value
for i in range(len(x)):
    # The out dim: 1X4 @ 4X3 = 1X3 = array(3)
    query_i = x[i] @ w_query
    key_i = x[i] @ w_key
    value_i = x[i] @ w_value
    query.append(query_i)
    key.append(key_i)
    value.append(value_i)

# Convert list into numpy array
query = np.stack(query)
key = np.stack(key)
value = np.stack(value)
# print(query)
# print(key)
# print(value)
# exit()
this_query_contextual = []
for i in range(len(x)):
    this_query = query[i]
    relevance = []
    # Compute this_query relevance to all the Keys (keys-row)
    for j in range(len(key)):
        # Calculate inner product in between this_query and each row of the key matrix
        rel_key_j = this_query @ key[j]
        relevance.append(rel_key_j)

    relevance = np.array(relevance)
    # Apply softmax to get probability scores of relevance
    relevance_scores = softmax(relevance, axis=-1)
    # relevance_scores = relevance_scores.round(decimals=1)
    out = 0
    # Each values-row is multiplied with relevance score and added point-wise
    for k in range(len(relevance)):
        # Here value[k] :is vector (of head_dim), and relevance_scores[k]: is a scalar score
        out += value[k] * relevance_scores[k]
    this_query_contextual.append(out.round(decimals=1))

print(np.stack(this_query_contextual))

# For Multi-Head, repeat the above process for n-separate w_query, w_key, and w_value,
# that will be n multi-head attn (In an optimized implementation, all the heads are packed
# in a single matrix for query, key, and value)


'''
Author:    Michael Sherif Naguib
Date:      March 13, 2020
@:         University of Tulsa
Description: see README.md

After my initial code written in montecarlopi.py that performs a Monte Carlo simulation using a circle inscribed in a
square ... I realized a similar simulation could be achieved in 3D ... I wondered if I could generalize the simulation
to an N Dimensional space ... I spent time to derive the necessary formulae ...in cases where n is an positive integer.

Note however .... I should also note that as the higher dimensional formulas involve a kth root the simulation will be
limited by the precision of the nth root function. (in addition to long division ... as in the 2D case)

'''

#imports
import numpy as np
import tqdm
import math
import matplotlib.pyplot as plt

def magSquared(vector):
    '''
    :description: Calculate the squared magnitude of a vector
    :param vector: a numpy vector
    :return: the distance squared
    '''
    return sum([math.pow(vector[i],2) for i in range(len(vector))])

def nDFuncGenerator(dimension):
    '''
    :Description: Since a good portion of the function is itself seperable (i.e constant) with respect to the
    hypervolumes for the Hypersphere and Hypercube, much of the calculation can be precomputed... this function
    returns a function which has those values precomputed...
    :param dimension: the positive integer value that specifies the dimension (note the formula changes based on this dim)
    :return: a function that provides an estimate of PI in an N Dimensional Monte Carlo Simulation,
    (based on the hypervolumes for the Hypersphere and Hypercube)

    the returned function accepts two parameters
    param1: countForHyperSphere (or 2d circle 3d etc... )
    param2: countForHyperCube   (or 2d square 3d etc...)
    '''
    assert type(dimension) == type(1)# Assert it is an integer
    assert dimension >=2             # Assert we are at least in 2 Dimensions

    # 2 Dimensional case
    if dimension == 2:
        def d2Case(numInCircle,numInSquare):
            assert numInSquare != 0  # Catch Div by Zero
            return 4 * numInCircle / numInSquare
        return d2Case
    # 3 Dimensional Case
    elif dimension == 3:
        def d3Case(numInSphere,numInCube):
            assert numInCube != 0  # Catch Div by Zero
            return 6 * numInSphere / numInCube
        return d3Case
    # N Dimensional Case
    else:
        # Init a variable for the constant
        constant = None
        k = None
        # For the N dimensional case there are two scenarios: dimension is even or n is odd
        if dimension%2 ==0: #Even Case
            k = dimension //2
            assert k!=0
            # Derived formula
            constant = math.pow(math.factorial(k)*math.pow(2,2*k),1/k)
        else:# Odd Case
            k = (dimension -1)//2
            assert k!=0
            # Derived formula
            constant = math.pow(math.factorial(2*k +1)*math.pow(2,2*k+1)/(2*math.factorial(k)*math.pow(4,k)),1/k)
        # Return the function bound with the constant ... it turns out the seperable part of both the even and
        # the odd case are the same ... i.e    c*f(x)
        def dnCase(hypersphereCount,hypercubeCount):
            assert hypercubeCount!=0 # Catch Divide by zero
            if k%2==1:
                assert hypercubeCount>=0 and hypersphereCount>=0   # Catch - in nth roots odd ensure neither are negative (they never should be)
            return constant*math.pow(hypersphereCount/hypercubeCount,1/k)
        return dnCase

def monteCarloPiItrGenerator(dimension):
    '''
    :description: since points need to be generated differently i.e a point in 2D vs 3D vs nD ...
    a function is needed that can do this ... this function returns a function which is a generator ( in the sense of yield)
    for successive estimates of pi gi
    :param dimension: specifies the dimension for the simulation
    :return: a function which accepts two parameters
    a radius and a maximum iteration count... (this is based off the code in montecarlopi.py)
    '''

    def monteCarloPiItr(radius=10, iterations=100_000_000):
        '''
        :description: A generator that generates successive estimates of pi
        :param radius: the Radius of the Circle inscribed in the Square (both centered at 0,0)
        :param iterations: the number points to compute and use to update the estimate
        :yeild: the next estimate of pi
        '''

        # Count how many are in the hypersphere ( circle etc...)
        inHypersphereCount = 0
        # Store an estimate for Pi (will be updated as the simulation progresses)

        # Calculate the radius squared (once)
        radiusSquared = math.pow(radius, 2)
        # A numpy array used to shift the random nums into the desired range
        s = np.ones(dimension) / 2
        # Generate the pi esitmate calculator function (precomputing the constants)
        calcPiEstimate = nDFuncGenerator(dimension)

        # Begin iteration: i is the total up to that iteration .. (all points are in the square)
        for i in range(0, iterations):
            # Pick random coords: the circle is centered @ 0,0 ... 0  so shift over the range of the square by subtracting 0.5
            # before scaling by the sidelength of the square ( side length = 2*radius)
            x = (np.random.rand(dimension)-s)*2*radius

            # Check if the point is within the circle: increment if it is (use squared distance to be efficient)
            inHypersphereCount = inHypersphereCount + 1 if  magSquared(x)<= radiusSquared else inHypersphereCount + 0

            # Update the pi estimate
            piEst = calcPiEstimate(inHypersphereCount,i+1)

            # Yield the pi esitmate
            yield piEst

    # Return the N- Dimensional configured Monte Carlo Pi calculator
    return monteCarloPiItr

def reject_outliers(data, m=2):
    '''
    NOT MY CODE: i take no credit for this code...
    thanks to: https://stackoverflow.com/questions/11686720/is-there-a-numpy-builtin-to-reject-outliers-from-a-list
    :Description: this code rejects outliers ....
    :param data: array
    :param m:
    :return: data without outliers
    '''
    return data[abs(data - np.mean(data)) < m * np.std(data)]

def updateRunningAvg(curAvg,val,valCnt):
    '''
    :description: compute a running average
    :param curAvg: the current value of the average
    :param val: the new value to factor into the average
    :param valCnt: the number of values taken into account for the curAvg
    :return: the updated average

    (NOTE! this function does not change any of the parameters --> no side effect...)
    it is the prgmr's responsibility to update those vals
    '''
    assert valCnt != 0
    return (curAvg*valCnt + val)/(valCnt+1)

def monteCarloSim(dimension,radius=1,iterations=10_000,redos=100,log=True):
    '''
    :description: performs a monte carlo simulation configured by the parameters and then takes each redundant
    simulation and averages it at the coresponding timestep.
    :param dimension: the dimension to make the calculation in
    :param radius: the radius for the hypersphere
    :param iterations: the number of successive estimates of pi
    :param redos: the number of times to repete the simulation
    :param log: if true prints progress of the redundant sims
    :return: list of avg values of pi at each time step
    '''
    assert redos>=1
    #init the first
    avgData =  [newEst for newEst in monteCarloPiItrGenerator(dimension)(radius=radius, iterations=iterations)]
    logger = tqdm.tqdm if log else lambda x: x
    # do the remainder
    for r in logger(range(0,redos-1)):
        latestSim = [newEst for newEst in monteCarloPiItrGenerator(dimension)(radius=radius, iterations=iterations)]
        for i in range(len(avgData)):
            avgData[i] = updateRunningAvg(avgData[i],latestSim[i],r+1)
    return avgData

if __name__ == "__main__":
    # Settings
    dimension = 4          # Run the simulation in nth Dimensional space
    radius = 1              # Radius of the hypersphere
    iterations = 10_000 # maximum iterations
    '''
    logEvery = 10_000       # Log every so many iterations
    # Run the simulation
    cnt=0
    for newEst in monteCarloPiItrGenerator(dimension)(radius=radius,iterations=iterations):
        cnt+=1
        if cnt%logEvery==0:
            print(newEst)
    '''
    dimensions= list(range(2,10))
    bins = 100
    allSeries=[]
    rejectionStrength = 2
    for d in dimensions:
        #series=[newEst for newEst in tqdm.tqdm(monteCarloPiItrGenerator(d)(radius=radius,iterations=iterations))]
        series = monteCarloSim(d)
        series = reject_outliers( np.array(series),m=rejectionStrength)
        allSeries.append(series)
    assert(len(allSeries)==len(dimensions))
    meanOfAllData = 0
    for series in allSeries:
        meanOfAllData+= series.mean()/len(allSeries)
    fig, ax = plt.subplots()
    for i in range(len(allSeries)):
        ax.hist(allSeries[i],bins,label="{0}D".format(dimensions[i]),alpha=0.5)
    # Dashed line indicate the estimate of pi
    ax.axvline(meanOfAllData, color='k', linestyle='dashed', linewidth=1)
    #Solid line indicate what is taken as the true value of pi
    ax.axvline(3.14159265358979, color='k', linestyle='solid', linewidth=1)
    #Plot info
    plt.legend(loc="best")
    plt.title("Estimates for Pi (redos=100,radius=1,sample=10k)".format())
    plt.xlabel("Estimate Values (Outliers)")
    plt.show()


import numpy as np
import matplotlib.pyplot as plt

xlim = 10
x = np.linspace(-xlim, xlim, 500)

plt.figure(figsize=(3,3)) # 400x400 Pixel

plt.title('Heaviside Funktion', fontsize=20)
plt.plot(x, np.heaviside(x, 1.0), linewidth=10, color='red')
plt.xlim([-10, 10])
plt.grid(True)
plt.show()

input("Drück eine Taste...")

plt.title('Sigmoid (Logistische Funktion)', fontsize=20)
plt.plot(x, 1./(1.+np.exp(-x)), linewidth=10, color='green')
plt.text(-9, 0.85, r'$\sigma(x)=\frac{1}{1+e^{-x}}$', fontsize=30)
plt.xlim([-10, 10])
plt.grid(True)
plt.show()

input("Drück eine Taste...")

plt.title('Aktivierungsfunktionen')
plt.plot(x, np.heaviside(x, 10.0), label='Heaviside', linewidth=10, color='red')
plt.plot(x, 1./(1.+np.exp(-x)), label='Logistische Funktion', linewidth=10, color='green')
plt.xlim([-10, 10])
plt.grid(True)
plt.legend()
plt.show()

input("Drück eine Taste...")

fig, axs = plt.subplots(2, 1)

axs[0].plot(x, 1./(1.+np.exp(-x)), color='green')
axs[0].set_title('Logistische Funktion')
axs[0].text(-10, 0.85, r'$\sigma(x)=\frac{1}{1+e^{-x}}$', fontsize=25, color='green')
axs[0].grid(True)

axs[1].plot(x, 1./(1.+np.exp(-x))*(1-1./(1.+np.exp(-x))), color='red')
axs[1].set_title('Änderung (1. Ableitung)')
axs[1].text(-10, 0.15, r'$\sigma\'(x)=\sigma(x).(1-\sigma(x))$', fontsize=15, color='red')
axs[1].grid(True)

plt.show()

input("Drück eine Taste...")

plt.title('Tangens Hyperbolicus', fontsize=20)
plt.plot(x, np.tanh(x), linewidth=10, color='orange')
plt.text(-9, 0.75, r'$\sigma(x)=tanh(x)'
                   r'$', fontsize=25)
plt.xlim([-10, 10])
plt.grid(True)
plt.legend()
plt.show()

input("Drück eine Taste...")

plt.title('ReLU-Funktion', fontsize=20)
plt.plot(x, np.maximum(0,x), linewidth=10, color='blue')
plt.xlim([-10, 10])
plt.grid(True)

plt.legend()
plt.show()

input("Drück eine Taste...")


from src.functions.objectivefn import ObjectiveFn
import math
import numpy as np
import matplotlib.pyplot as plt
from matplotlib import cm

class RastriginFn(ObjectiveFn):

    def __init__(self, dims):
        super(RastriginFn, self).__init__(dims)
        self.domain = (-5.12, 5.12)

    def get_minima(self):
        minima_coords = [0 for i in range(self.dims)]  # Function value is 0 here.
        minima = tuple(minima_coords)
        return self.evaluate(minima)

    def evaluate(self, params):
        A = 10
        f = 0
        try:
            if len(params) != self.dims:
                raise Exception('number of paramters passd is not the same as the number of expected dimensions')

            for param in params:
                f = f + A + (param ** 2 - (A * math.cos(2 * math.pi * param)))

            return f
        except Exception as error:
            print('Exception raised in rastrigin_fn.eval_fn: ', repr(error))

    def eval_vectors(self, *X, **kwargs):
        A = kwargs.get('A', 10)
        return A + sum([(x ** 2 - A * np.cos(2 * math.pi * x)) for x in X])

    def graph_fn(self):
        A = 10
        X = np.linspace(-5.12, 5.12, 200)
        Y = np.linspace(-5.12, 5.12, 200)

        X, Y = np.meshgrid(X, Y)

        Z = self.eval_vectors(X, Y, A=10)

        fig = plt.figure()
        ax = fig.gca(projection='3d')
        ax.plot_surface(X, Y, Z, rstride=1, cstride=1, cmap=cm.plasma, linewidth=0, antialiased=False)
        plt.savefig('rastrigin.png')

    def contour_plot(self, save_file_name, points):
        A = 10
        # X = np.linspace(-5.12, 5.12, 200)
        # Y = np.linspace(-5.12, 5.12, 200)
        X = np.linspace(self.domain[0], self.domain[1], 200)
        Y = np.linspace(self.domain[0], self.domain[1], 200)

        X, Y = np.meshgrid(X, Y)

        Z = self.eval_vectors(X, Y, A=10)
        plt.contour(X, Y, Z)
        for point in points:
            # print(point)
            plt.scatter(point[0], point[1], marker='X', color='r')
        plt.savefig(save_file_name, dpi=300, bbox_inches='tight')
        plt.close()

    def is_defined_only_for_2d(self):
        return False

    def name(self):
        return "rastrigin"


from __future__ import division
import numpy as np
from utils import Grapher, memoize, SafeEval

class Romberg():
    def __init__(self, func, a, b, err):
        self.f = SafeEval.functionize(func)
        self.a = a
        self.b = b
        self.err = err
        self.intervals = None
        self.integral = None
        self.final_err = None

    @staticmethod
    def trapezoidal(f, l, r, n):
        h = (r - l) / n
        return h * ((f(l) + f(r)) / 2 + sum(map(f, np.arange(l + h, r, h))))

    def integrate(self):
        @memoize
        def T(n):
            return Romberg.trapezoidal(self.f, self.a, self.b, n)

        @memoize
        def R(o, n):
            if o == 2:
                return T(n)
            else:
                return (4 * R(o - 2, 2 * n) - R(o - 2, n)) / 3

        def err(o):
            e = R(o, 1) - R(o - 2, 2)
            if R(o, 1) is not 0:
                e /= R(o, 1)
            return e
        o = 4
        while abs(err(o)) > self.err:
            o *= 2
        self.integral = R(o, 1)
        self.intervals = o / 2
        self.final_err = err(o)

    def plot(self):
        x = np.linspace(self.a, self.b, self.intervals + 1)
        y = [self.f(_) for _ in x]
        Grapher.plot(x=x, y=y)

class Quadrature():
    def __init__(self, func, a, b, err):
        self.f = SafeEval.functionize(func)
        self.a = a
        self.b = b
        self.err = err
        self.points = None
        self.integral = None
        self.final_err = None

    @staticmethod
    def gaussxw(N):
        # Initial approximation to roots of the Legendre polynomial
        a = np.linspace(3, 4 * N - 1, N) / (4 * N + 2)
        x = np.cos(np.pi * a + 1 / (8 * N * N * np.tan(a)))

        # Find roots using Newton's method
        epsilon, delta = 1e-15, 1.0
        while delta > epsilon:
            p0 = np.ones(N, float)
            p1 = np.copy(x)
            for k in range(1, N):
                p0, p1 = p1, ((2 * k + 1) * x * p1 - k * p0) / (k + 1)
            dp = (N + 1) * (p0 - x * p1) / (1 - x * x)
            dx = p1 / dp
            x -= dx
            delta = np.max(abs(dx))

        # Calculate the weights
        w = 2 * (N + 1) * (N + 1) / (N * N * (1 - x * x) * dp * dp)

        return x, w

    @staticmethod
    def gaussxwab(N, a, b):
        x, w = Quadrature.gaussxw(N)
        return 0.5 * (b - a) * x + 0.5 * (b + a), 0.5 * (b - a) * w

    def intergrate(self):
        @memoize
        def G(n):
            x, w = Quadrature.gaussxwab(n, self.a, self.b)
            fx = np.array([self.f(i) for i in x])
            w = np.array(w)
            return np.sum(fx * w)

        def err(n):
            e = G(n) - G(n - 1)
            if G(n) is not 0:
                e /= G(n)
            return e
        n = 2
        while abs(err(n)) > self.err:
            n += 1
        self.points = n
        self.integral = G(n)
        self.final_err = err(n)

    def plot(self):
        x, w = Quadrature.gaussxwab(self.points, self.a, self.b)
        y = [self.f(_) for _ in x]
        Grapher.plot(x=x, y=y)

def main():

    with open("q1.txt", "r") as f:
        func = f.readline()
        a, b = map(float, f.readline().split(', '))
        err = float(f.readline())
        op = int(f.readline())

    if op == 1:
        romberg = Romberg(func, a, b, err / 100)
        romberg.integrate()
        with open("out.txt", "w") as fout:
            if romberg.integral is not None:
                fout.write('I= %f\n' % romberg.integral)
                fout.write('Number of intervals= %d\n' % romberg.intervals)
                fout.write('Approximate relative error (%%)= %f\n' %
                           (romberg.final_err * 100))
        if raw_input('Plot f(x) vs x?[Press 1 for yes]: ') is '1':
            romberg.plot()
    else:
        gauss = Quadrature(func, a, b, err / 100)
        gauss.intergrate()
        with open("out.txt", "w") as fout:
            if gauss.integral is not None:
                fout.write('I= %f\n' % gauss.integral)
                fout.write('Number of points= %d\n' % gauss.points)
                fout.write('Approximate relative error (%%)= %f\n' %
                           (gauss.final_err * 100))
        if raw_input('Plot f(x) vs x?[Press 1 for yes]: ') is '1':
            gauss.plot()

if __name__ == '__main__':
    main()


def mc_pi(n):  # Sampling n points to calculate pi.
    #import time
    import numpy as np
    #a = time.time()
    m = 0
    pi = [None] * n
    x = np.random.uniform(-1, 1, n)
    y = np.random.uniform(-1, 1, n)
    for i in range(0, n):
        if (x[i]**2 + y[i]**2) <= 1:
            m = m + 1
        pi[i] = 4.0 * m / (i + 1)
    #b = time.time() - a
    #print("Toal time: %.1fsec\n" % (b))
    return (pi[n - 1])

# use every core of CPU to parallel calculate pi, loop t times, every time sampling n points.
def pl_mc_pi(n, t):
    import time
    import sys
    import multiprocessing
    import numpy as np
    a = time.time()
    cores = multiprocessing.cpu_count()
    pool = multiprocessing.Pool(processes=cores)
    cnt = 0
    pi = [None] * t
    for y in pool.imap_unordered(mc_pi, [n] * t):
        pi[cnt] = y
        m = np.mean(pi[0:cnt + 1])
        cnt += 1
        sys.stdout.write('done %d/%d, current pi is %f\r' % (cnt, t, m))
    b = time.time() - a
    print("\nToal time: %.1fsec\n" % (b))
    return np.mean(pi)

def mc_pi_plot(n, tt):  # want to plot an animation to show the progress?
    import time
    import numpy as np
    import matplotlib.pyplot as plt
    %matplotlib osx
    plt.close()
    a = time.time()
    m = 0
    pi = [None] * n
    co = [None] * n
    x = np.random.uniform(-1, 1, n)
    y = np.random.uniform(-1, 1, n)
    plt.axis('scaled')
    plt.axis([-1, 1, -1, 1])
    for i in range(0, n):
        if (x[i]**2 + y[i]**2) <= 1:
            m = m + 1
            co[i] = 'r'
        else:
            co[i] = 'k'
        pi[i] = 4.0 * m / (i + 1)
        plt.scatter(x[i], y[i], s=0.75, marker='.', c=co[i], alpha=.5)
        if tt:
            plt.pause(tt)
    b = time.time() - a
    print("Toal time: %.1fsec\n" % (b))
    if tt:
        plt.show()
    return (pi[n - 1])


'''
Computing the Pearson correlation coefficient
100xp
As mentioned in the video, the Pearson correlation coefficient, also called the
Pearson r, is often easier to interpret than the covariance. It is computed using
the np.corrcoef() function. Like np.cov(), it takes two arrays as arguments and
returns a 2D array. Entries [0,0] and [1,1] are necessarily equal to 1 (can you
think about why?), and the value we are after is entry [0,1].

In this exercise, you will write a function, pearson_r(x, y) that takes in two
arrays and returns the Pearson correlation coefficient. You will then use this
function to compute it for the petal lengths and widths of I. versicolor.

Again, we include the scatter plot you generated in a previous exercise to remind
you how the petal width and length are related.

Instructions
-Define a function with signature pearson_r(x, y).
    -Use np.corrcoef() to compute the correlation matrix of x and y (pass them to
    np.corrcoef() in that order).
    -The function returns entry [0,1] of the correlation matrix.
-Compute the Pearson correlation between the data in the arrays versicolor_petal_length
and versicolor_petal_width. Assign the result to r.
-Print the result.
'''
import numpy as np

versicolor_petal_length = np.array([4.7,  4.5,  4.9,  4.,  4.6,  4.5,  4.7,  3.3,  4.6,  3.9,  3.5,
                                    4.2,  4.,  4.7,  3.6,  4.4,  4.5,  4.1,  4.5,  3.9,  4.8,  4.,
                                    4.9,  4.7,  4.3,  4.4,  4.8,  5.,  4.5,  3.5,  3.8,  3.7,  3.9,
                                    5.1,  4.5,  4.5,  4.7,  4.4,  4.1,  4.,  4.4,  4.6,  4.,  3.3,
                                    4.2,  4.2,  4.2,  4.3,  3.,  4.1])

versicolor_petal_width = np.array([1.4,  1.5,  1.5,  1.3,  1.5,  1.3,  1.6,  1.,  1.3,  1.4,  1.,
                                   1.5,  1.,  1.4,  1.3,  1.4,  1.5,  1.,  1.5,  1.1,  1.8,  1.3,
                                   1.5,  1.2,  1.3,  1.4,  1.4,  1.7,  1.5,  1.,  1.1,  1.,  1.2,
                                   1.6,  1.5,  1.6,  1.5,  1.3,  1.3,  1.3,  1.2,  1.4,  1.2,  1.,
                                   1.3,  1.2,  1.3,  1.3,  1.1,  1.3])

def pearson_r(x, y):
    """Compute Pearson correlation coefficient between two arrays."""
    # Compute correlation matrix: corr_mat
    corr_mat = np.corrcoef(x, y)

    # Return entry [0,1]
    return corr_mat[0, 1]

# Compute Pearson correlation coefficient for I. versicolor: r
r = pearson_r(versicolor_petal_length, versicolor_petal_width)

# Print the result
print(r)


import pandas as pd
import matplotlib.pyplot as plt
import numpy as np
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
import seaborn as sns

sns.set()
sns.set_context("talk")

#1) Load (all features, using Pandas), standardize this d-dimension dataset (d is number of features) and Split
# the Iris dataset to training and test sets with ratio 70% and 30%, respectively.

iris_df = pd.read_csv('https://archive.ics.uci.edu/ml/machine-learning-databases/iris/iris.data', header=None)
print("iris_df.head(): ",iris_df.head())
print("iris_df.tail(): ",iris_df.tail())

X = iris_df.iloc[:,0:4].values
y = iris_df.iloc[:,4].values

X_train, X_test, y_train, y_test =  train_test_split(X, y, test_size=0.3, random_state=0)

sc = StandardScaler()
X_train_std = sc.fit_transform(X_train)
X_test_std = sc.transform(X_test)

# 2) Write your own function to calculate the covariance matrix. Then compute the eigenvalues and
# eigenvectors of this matrix.
mean_vec = np.mean(X_train_std, axis=0)
cov_mat = (X_train_std - mean_vec).T.dot((X_train_std - mean_vec)) / (X_train_std.shape[0]-1)
print('Covariance matrix \n%s' %cov_mat)
eigen_vals, eigen_vecs = np.linalg.eig(cov_mat)
print('\nEigenvalues \n%s' % eigen_vals)

# 3) Plot the cumulative variance ratio (c.f. block [11] of the nbviewer of Chapter 5).
tot = sum(eigen_vals)
var_exp = [(i / tot) for i in sorted(eigen_vals, reverse=True)]
cum_var_exp = np.cumsum(var_exp)
plt.bar(range(0, 4), var_exp, alpha=0.5, align='center',label='individual explained variance')
plt.step(range(0, 4), cum_var_exp, where='mid', label='cumulative explained variance')
plt.ylabel('Explained variance ratio')
plt.xlabel('Principal components')
plt.legend(loc='center right')
plt.tight_layout()
plt.show()

# 4) Choose the k=3 eigenvectors that correspond to the k largest eigenvalues to construct a d×k-dimensional
# transformation matrix W ; the eigenvectors are the columns of this matrix
eigen_pairs = [(np.abs(eigen_vals[i]), eigen_vecs[:, i])
               for i in range(len(eigen_vals))]
print(eigen_pairs)
W = np.hstack((eigen_pairs[0][1][:, np.newaxis],
               eigen_pairs[1][1][:, np.newaxis], eigen_pairs[2][1][:, np.newaxis]))
print('Matrix W:\n', W)
print("X_train_std[0].dot(W) ",X_train_std[0].dot(W))

# 5) Project the samples onto the new feature subspace, and plot the projected data using the
# transformation matrix W (c.f. block [14] of the nbviewer of Chapter 5)
X_train_pca = X_train_std.dot(W)
print("X_train_pca ",X_train_pca)

colors = ['r', 'b', 'g']
markers = ['s', 'x', 'o']
#PCA 1 and PCA2
for l, c, m in zip(np.unique(y_train), colors, markers):
    plt.scatter(X_train_pca[y_train == l, 0],
                X_train_pca[y_train == l, 1],
                c=c, label=l, marker=m)
plt.title('PC 1 and PC 2')
plt.xlabel('PC 1')
plt.ylabel('PC 2')
plt.legend(loc='upper right')
plt.tight_layout()
plt.show()

#PCA 1 and PCA 3
for l, c, m in zip(np.unique(y_train), colors, markers):
    plt.scatter(X_train_pca[y_train == l, 0],
                X_train_pca[y_train == l, 2],
                c=c, label=l, marker=m)
plt.title('PC 1 and PC 3')
plt.xlabel('PC 1')
plt.ylabel('PC 3')
plt.legend(loc='upper right')
plt.tight_layout()
plt.show()

#PCA 2 and PCA 3
for l, c, m in zip(np.unique(y_train), colors, markers):
    plt.scatter(X_train_pca[y_train == l, 1],
                X_train_pca[y_train == l, 2],
                c=c, label=l, marker=m)
plt.title('PC 2 and PC 3')
plt.xlabel('PC 2')
plt.ylabel('PC 3')
plt.legend(loc='upper right')
plt.tight_layout()
plt.show()

from mpl_toolkits.mplot3d import Axes3D
fig = plt.figure(1, figsize=(8, 6))
ax = Axes3D(fig, elev=-150, azim=110)
for l, c, m in zip(np.unique(y_train), colors, markers):
    ax.scatter(X_train_pca[y_train == l, 0], X_train_pca[y_train == l, 1], X_train_pca[y_train == l, 2],c=c,   label=l,marker=m, edgecolor='k', s=40)

ax.set_title("First three PC directions")
ax.set_xlabel("PC 1")
ax.w_xaxis.set_ticklabels([])
ax.set_ylabel("PC 2")
ax.w_yaxis.set_ticklabels([])
ax.set_zlabel("PC 3")
ax.w_zaxis.set_ticklabels([])
plt.legend(loc='upper right')

plt.show()


import numpy as np

def minmax_scaling(x, m=None, r=None):
    """
    m: midrange
    r: range
    """

    xmin = np.min(x, axis=0)
    xmax = np.max(x, axis=0)

    if m is None:
        m = (xmin + xmax) / 2
    if r is None:
        r = xmax - xmin

    #with np.errstate(divide='ignore'):
    np.seterr(divide='ignore')
    ret = (x-m)/(r/2)

    ret[np.where(np.isnan(ret))] = 0.0
    return ret, m, r

def standardize(x, m=None, r=None):
    """
    m: midrange
    r: range
    """

    if m is None:
        m = np.mean(x, axis=0)
    if r is None:
        r = np.std(x, axis=0, ddof=1)

    #with np.errstate(divide='ignore'):
    np.seterr(divide='ignore', invalid='ignore')
    ret = (x-m)/r

    ret[np.where(np.isnan(ret))] = 0.0
    return ret, m, r

def norm_l2(x, m = None, r = None):
    """
    m: midrange
    r: range
    """

    if m is None:
        m = np.mean(x) # not axis = 0 because result = (x-E(x))/normL2(X)

    # centering
    x = x-m

    if r is None:
        r = np.sqrt(np.sum((x)**2, axis = 0))

    #with np.errstate(divide='ignore'):
    np.seterr(divide='ignore')
    ret = x/r

    ret[np.where(np.isnan(ret))] = 0.0

    return ret, m, r


# Implement the Binomial Tree method for American call/put options

import numpy as np

def american_call_binomial_tree(s_0, sigma, r, T, K, N):
    """Calculate the value of American call option by the method of
    Binomial Tree.

    Args:
        s_0: The value of asset S at time 0
        sigma: The volatility
        r: The risk-free rate
        T: The time to maturity(in years)
        K: Strike
        N: The number of steps in Binomial Tree

    Returns:
        The American call option value
    """
    def _calculate_node_value(v_0, remain_layer):
        """Calculate the option value at this node

        Args:
            v_0: The stock price at this node
            remain_layer: The remaining layer number from leaf node

        Returns:
            The call option value at this node
        """
        if remain_layer <= 0:
            return max(v_0-K, 0)
        else:
            return max(v_0-K, _DF*(_p*_calculate_node_value(v_0*_u, remain_layer-1)
                              + (1-_p)*_calculate_node_value(v_0*_d, remain_layer-1)))

    _delta_T = T/N
    _u = np.exp(sigma*np.sqrt(_delta_T))
    _d = 1/_u
    _p = (np.exp(r*_delta_T)-_d)/(_u-_d)
    _DF = np.exp(-r*_delta_T) # discount factor DF = e^(-r*deltaT)

    return _calculate_node_value(s_0, N)

def american_put_binomial_tree(s_0, sigma, r, T, K, N):
    """Calculate the value of American put option by the method of
    Binomial Tree.

    Args:
        s_0: The value of asset S at time 0
        sigma: The volatility
        r: The risk-free rate
        T: The time to maturity(in years)
        K: Strike
        N: The number of steps in Binomial Tree

    Returns:
        The American put option value
    """
    def _calculate_node_value(v_0, remain_layer):
        """Calculate the option value at this node

        Args:
            v_0: The stock price at this node
            remain_layer: The remaining layer number from leaf node

        Returns:
            The put option value at this node
        """
        if remain_layer <= 0:
            return max(K-v_0, 0)
        else:
            return max(K-v_0, _DF*(_p*_calculate_node_value(v_0*_u, remain_layer-1)
                              + (1-_p)*_calculate_node_value(v_0*_d, remain_layer-1)))

    _delta_T = T/N
    _u = np.exp(sigma*np.sqrt(_delta_T))
    _d = 1/_u
    _p = (np.exp(r*_delta_T)-_d)/(_u-_d)
    _DF = np.exp(-r*_delta_T) # discount factor DF = e^(-r*deltaT)

    return _calculate_node_value(s_0, N)

def main():
    K = 52
    T = 2
    r = 0.05
    sigma = 0.223144
    s_0 = 50
    N = 2
    print("American put value: " + str(american_put_binomial_tree(s_0, sigma, r, T, K, N)))

    K = 50
    T = 0.25
    r = 0.05
    sigma = 0.3
    s_0 = 50
    N = 1
    print("American call value: " + str(american_call_binomial_tree(s_0, sigma, r, T, K, N)))

    N = 2
    print("American call value: " + str(american_call_binomial_tree(s_0, sigma, r, T, K, N)))

    N = 3
    print("American call value: " + str(american_call_binomial_tree(s_0, sigma, r, T, K, N)))

if __name__ == '__main__':
    main()


#Creating Arrays
"""Zero-dimensional Arrays in Numpy

It's possible to create multidimensional arrays in numpy. Scalars are zero dimensional. In the following example, we will create the scalar 42. Applying the ndim method to our scalar, we get the dimension of the array. We can also see that the type is a "numpy.ndarray" type.
"""
import numpy as np
x = np.array(42)
print("x: ", x)
print("The type of x: ", type(x))
print("The dimension of x:", np.ndim(x))

"""One-dimensional Arrays

We have already encountered a 1-dimenional array - better known to some as vectors - in our initial example. What we have not mentioned so far, but what you may have assumed, is the fact that numpy arrays are containers of items of the same type, e.g. only integers. The homogenous type of the array can be determined with the attribute "dtype", as we can learn from the following example:
"""
F = np.array([1, 1, 2, 3, 5, 8, 13, 21])
V = np.array([3.4, 6.9, 99.8, 12.8])
print("F: ", F)
print("V: ", V)
print("Type of F: ", F.dtype)
print("Type of V: ", V.dtype)
print("Dimension of F: ", np.ndim(F))
print("Dimension of V: ", np.ndim(V))
"""
Two- and Multidimensional Arrays

Of course, arrays of NumPy are not limited to one dimension. They are of arbitrary dimension. We create them by passing nested lists (or tuples) to the array method of numpy.
"""
A = np.array([ [3.4, 8.7, 9.9],
               [1.1, -7.8, -0.7],
               [4.1, 12.3, 4.8]])
print(A)
print(A.ndim)


#prompt:
#Good morning! Here's your coding interview problem for today.
#This problem was asked by Uber.
#Given an array of integers, return a new array such that each element at index i of the new array is the product of all the numbers in the original array except the one at i.
#For example, if our input was [1, 2, 3, 4, 5], the expected output would be [120, 60, 40, 30, 24]. If our input was [3, 2, 1], the expected output would be [2, 3, 6].
#Follow-up: what if you can't use division?

import random
import numpy as np

#for each element in a list, get the product of all other elements the list.
def DCP_2(L):
    #create container for list of products
    P = []
    #iterate through the list elements
    for i,p in enumerate(L):
        #DEBUG: test that the right indexes will be excluded from the product
        #print([j for j,q in enumerate(L) if j != i])
        #calculate the product. remember to use indexes for item exclusion so that duplicate values are retained in other index positions
        P.append(np.product([q for j,q in enumerate(L) if j != i]))
    return P

#save products to a list :

#test example lists:
#1:
L = [3, 2, 1]
D = DCP_2(L)
print(D)

#2:
L = [1, 2, 3, 4, 5]
D = DCP_2(L)
print(D)

#test a random list of ints of user-specified len n:
n = 10
L = [ random.randint(1,10) for x in range(n) ]
D = DCP_2(L)
print(D)

#NOTE: follow-up irrelevant, division not used in sol'n (numpy was allowed, right? :) )


import numpy as np
from scipy import optimize
from matplotlib import pyplot as plt, cm, colors
from math import sqrt, pi

def calc_R(x,y, xc, yc):
    """
    calculate the distance of each 2D points from the center (xc, yc)
    """
    return np.sqrt((x-xc)**2 + (y-yc)**2)

def f(c, x, y):
    """
    calculate the algebraic distance between the data points
    and the mean circle centered at c=(xc, yc)
    """
    Ri = calc_R(x, y, *c)
    return Ri - Ri.mean()

def sigma(coords, x, y, r):
    """Computes Sigma for circle fit."""
    dx, dy, sum_ = 0., 0., 0.

    for i in range(len(coords)):
        dx = coords[i][1] - x
        dy = coords[i][0] - y
        sum_ += (sqrt(dx*dx+dy*dy) - r)**2
    return sqrt(sum_/len(coords))

def hyper_fit(coords, IterMax=99, verbose=False):
    """
    Fits coords to circle using hyperfit algorithm.

    Inputs:
        - coords, list or numpy array with len>2 of the form:
        [
    [x_coord, y_coord],
    ...,
    [x_coord, y_coord]
    ]
        or numpy array of shape (n, 2)

    Outputs:

        - xc : x-coordinate of solution center (float)
        - yc : y-coordinate of solution center (float)
        - R : Radius of solution (float)
        - residu : s, sigma - variance of data wrt solution (float)

    """
    X, X = None, None
    if isinstance(coords, np.ndarray):
        X = coords[:, 0]
        Y = coords[:, 1]
    elif isinstance(coords, list):
        X = np.array([x[0] for x in coords])
        Y = np.array([x[1] for x in coords])
    else:
        raise Exception("Parameter 'coords' is an unsupported type: " + str(type(coords)))

    n = X.shape[0]

    Xi = X - X.mean()
    Yi = Y - Y.mean()
    Zi = Xi*Xi + Yi*Yi

    #compute moments
    Mxy = (Xi*Yi).sum()/n
    Mxx = (Xi*Xi).sum()/n
    Myy = (Yi*Yi).sum()/n
    Mxz = (Xi*Zi).sum()/n
    Myz = (Yi*Zi).sum()/n
    Mzz = (Zi*Zi).sum()/n

    #computing the coefficients of characteristic polynomial
    Mz = Mxx + Myy
    Cov_xy = Mxx*Myy - Mxy*Mxy
    Var_z = Mzz - Mz*Mz

    A2 = 4*Cov_xy - 3*Mz*Mz - Mzz
    A1 = Var_z*Mz + 4.*Cov_xy*Mz - Mxz*Mxz - Myz*Myz
    A0 = Mxz*(Mxz*Myy - Myz*Mxy) + Myz*(Myz*Mxx - Mxz*Mxy) - Var_z*Cov_xy
    A22 = A2 + A2

    #finding the root of the characteristic polynomial
    y = A0
    x = 0.
    for i in range(IterMax):
        Dy = A1 + x*(A22 + 16.*x*x)
        xnew = x - y/Dy
        if xnew == x or not np.isfinite(xnew):
            break
        ynew = A0 + xnew*(A1 + xnew*(A2 + 4.*xnew*xnew))
        if abs(ynew)>=abs(y):
            break
        x, y = xnew, ynew

    det = x*x - x*Mz + Cov_xy
    Xcenter = (Mxz*(Myy - x) - Myz*Mxy)/det/2.
    Ycenter = (Myz*(Mxx - x) - Mxz*Mxy)/det/2.

    x = Xcenter + X.mean()
    y = Ycenter + Y.mean()
    r = sqrt(abs(Xcenter**2 + Ycenter**2 + Mz))
    s = sigma(coords,x,y,r)
    iter_ = i
    if verbose:
        print('Regression complete in {} iterations.'.format(iter_))
        print('Sigma computed: ', s)
    return x, y, r, s

def least_squares_circle(coords, radius):
    """
    Circle fit using least-squares solver.
    Inputs:

        - coords, list or numpy array with len>2 of the form:
        [
    [x_coord, y_coord],
    ...,
    [x_coord, y_coord]
    ]
        or numpy array of shape (n, 2)

    Outputs:

        - xc : x-coordinate of solution center (float)
        - yc : y-coordinate of solution center (float)
        - R : Radius of solution (float)
        - residu : MSE of solution against training data (float)
    """

    x, y = None, None
    if isinstance(coords, np.ndarray):
        x = coords[:, 0]
        y = coords[:, 1]
    elif isinstance(coords, list):
        x = np.array([point[0] for point in coords])
        y = np.array([point[1] for point in coords])
    else:
        raise Exception("Parameter 'coords' is an unsupported type: " + str(type(coords)))

    # coordinates of the barycenter
    #x_m = np.mean(x)
    #y_m = np.mean(y)
    center_estimate = np.mean(x), np.mean(y)
    center, _ = optimize.leastsq(f, center_estimate, args=(x,y))
    xc, yc = center
    #Ri       = calc_R(x, y, *center)
    #R        = Ri.mean()
    R = radius
    #residu   = np.sum((Ri - R)**2)
    #return xc, yc, R#, residu
    return xc, yc, R, center_estimate

def plot_data_circle(x, y, xc, yc, R):
    """
    Plot data and a fitted circle.
    Inputs:

        x : data, x values (array)
        y : data, y values (array)
        xc : fit circle center (x-value) (float)
        yc : fit circle center (y-value) (float)
        R : fir circle radius (float)

    Output:
        None (generates matplotlib plot).
    """
    f = plt.figure(facecolor='white')
    plt.axis('equal')

    theta_fit = np.linspace(-pi, pi, 180)

    x_fit = xc + R*np.cos(theta_fit)
    y_fit = yc + R*np.sin(theta_fit)
    plt.plot(x_fit, y_fit, 'b-' , label="fitted circle", lw=2)
    plt.plot([xc], [yc], 'bD', mec='y', mew=1)
    plt.xlabel('x')
    plt.ylabel('y')
    # plot data
    plt.scatter(x, y, c='red', label='data')

    plt.legend(loc='best',labelspacing=0.1 )
    plt.grid()
    plt.title('Fit Circle')


# coding: utf-8

# #Numpy Tutorial

# Numpy is a computational library for Python that is optimized for operations on multi-dimensional arrays. In this notebook we will use numpy to work with 1-d arrays (often called vectors) and 2-d arrays (often called matrices).
#
# For a the full user guide and reference for numpy see: http://docs.scipy.org/doc/numpy/

# In[1]:

import numpy as np # importing this way allows us to refer to numpy as np

# # Creating Numpy Arrays

# New arrays can be made in several ways. We can take an existing list and convert it to a numpy array:

# In[7]:

mylist = [1., 2., 3., 4.]
mynparray = np.array(mylist)
mynparray

# You can initialize an array (of any dimension) of all ones or all zeroes with the ones() and zeros() functions:

# In[8]:

one_vector = np.ones(4)
print one_vector # using print removes the array() portion

# In[12]:

one2Darray = np.ones((2, 4)) # an 2D array with 2 "rows" and 4 "columns"
print one2Darray

# In[13]:

zero_vector = np.zeros(4)
print zero_vector

# You can also initialize an empty array which will be filled with values. This is the fastest way to initialize a fixed-size numpy array however you must ensure that you replace all of the values.

# In[14]:

empty_vector = np.empty(5)
print empty_vector

# #Accessing array elements

# Accessing an array is straight forward. For vectors you access the index by referring to it inside square brackets. Recall that indices in Python start with 0.

# In[15]:

mynparray[2]

# 2D arrays are accessed similarly by referring to the row and column index separated by a comma:

# In[20]:

my_matrix = np.array([[1, 2, 3], [4, 5, 6]])
print my_matrix

# In[21]:

print my_matrix[1, 2]

# Sequences of indices can be accessed using ':' for example

# In[22]:

print my_matrix[0:2, 2] # recall 0:2 = [0, 1]

# In[23]:

print my_matrix[0, 0:3]

# You can also pass a list of indices.

# In[24]:

fib_indices = np.array([1, 1, 2, 3])
random_vector = np.random.random(10) # 10 random numbers between 0 and 1
print random_vector

# In[25]:

print random_vector[fib_indices]

# You can also use true/false values to select values

# In[28]:

my_vector = np.array([1, 2, 3, 4])
select_index = np.array([True, False, True, False])
print my_vector[select_index]

# For 2D arrays you can select specific columns and specific rows. Passing ':' selects all rows/columns

# In[29]:

select_cols = np.array([True, False, True]) # 1st and 3rd column
select_rows = np.array([False, True]) # 2nd row

# In[30]:

print my_matrix[select_rows, :] # just 2nd row but all columns

# In[31]:

print my_matrix[:, select_cols] # all rows and just the 1st and 3rd column

# #Operations on Arrays

# You can use the operations '\*', '\*\*', '\\', '+' and '-' on numpy arrays and they operate elementwise.

# In[33]:

my_array = np.array([1., 2., 3., 4.])
print my_array*my_array

# In[34]:

print my_array**2

# In[35]:

print my_array - np.ones(4)

# In[36]:

print my_array + np.ones(4)

# In[37]:

print my_array / 3

# In[38]:

print my_array / np.array([2., 3., 4., 5.]) # = [1.0/2.0, 2.0/3.0, 3.0/4.0, 4.0/5.0]

# You can compute the sum with np.sum() and the average with np.average()

# In[39]:

print np.sum(my_array)

# In[40]:

print np.average(my_array)

# In[41]:

print np.sum(my_array)/len(my_array)

# #The dot product

# An important mathematical operation in linear algebra is the dot product.
#
# When we compute the dot product between two vectors we are simply multiplying them elementwise and adding them up. In numpy you can do this with np.dot()

# In[42]:

array1 = np.array([1., 2., 3., 4.])
array2 = np.array([2., 3., 4., 5.])
print np.dot(array1, array2)

# In[43]:

print np.sum(array1*array2)

# Recall that the Euclidean length (or magnitude) of a vector is the squareroot of the sum of the squares of the components. This is just the squareroot of the dot product of the vector with itself:

# In[44]:

array1_mag = np.sqrt(np.dot(array1, array1))
print array1_mag

# In[46]:

print np.sqrt(np.sum(array1*array1))

# We can also use the dot product when we have a 2D array (or matrix). When you have an vector with the same number of elements as the matrix (2D array) has columns you can right-multiply the matrix by the vector to get another vector with the same number of elements as the matrix has rows. For example this is how you compute the predicted values given a matrix of features and an array of weights.

# In[47]:

my_features = np.array([[1., 2.], [3., 4.], [5., 6.], [7., 8.]])
print my_features

# In[48]:

my_weights = np.array([0.4, 0.5])
print my_weights

# In[49]:

my_predictions = np.dot(my_features, my_weights) # note that the weights are on the right
print my_predictions # which has 4 elements since my_features has 4 rows

# Similarly if you have a vector with the same number of elements as the matrix has *rows* you can left multiply them.

# In[50]:

my_matrix = my_features
my_array = np.array([0.3, 0.4, 0.5, 0.6])

# In[51]:

print np.dot(my_array, my_matrix) # which has 2 elements because my_matrix has 2 columns

# #Multiplying Matrices

# If we have two 2D arrays (matrices) matrix_1 and matrix_2 where the number of columns of matrix_1 is the same as the number of rows of matrix_2 then we can use np.dot() to perform matrix multiplication.

# In[52]:

matrix_1 = np.array([[1., 2., 3.],[4., 5., 6.]])
print matrix_1

# In[53]:

matrix_2 = np.array([[1., 2.], [3., 4.], [5., 6.]])
print matrix_2

# In[54]:

print np.dot(matrix_1, matrix_2)

# In[ ]: