"""
==================
Fill Betweenx Demo
==================

Using ``fill_betweenx`` to color between two horizontal curves.
"""
import matplotlib.pyplot as plt
import numpy as np

y = np.arange(0.0, 2, 0.01)
x1 = np.sin(2 * np.pi * y)
x2 = 1.2 * np.sin(4 * np.pi * y)

fig, [ax1, ax2, ax3] = plt.subplots(3, 1, sharex=True)

ax1.fill_betweenx(y, 0, x1)
ax1.set_ylabel('(x1, 0)')

ax2.fill_betweenx(y, x1, 1)
ax2.set_ylabel('(x1, 1)')

ax3.fill_betweenx(y, x1, x2)
ax3.set_ylabel('(x1, x2)')
ax3.set_xlabel('x')

# now fill between x1 and x2 where a logical condition is met.  Note
# this is different than calling
#   fill_between(y[where], x1[where], x2[where])
# because of edge effects over multiple contiguous regions.

fig, [ax, ax1] = plt.subplots(2, 1, sharex=True)
ax.plot(x1, y, x2, y, color='black')
ax.fill_betweenx(y, x1, x2, where=x2 >= x1, facecolor='green')
ax.fill_betweenx(y, x1, x2, where=x2 <= x1, facecolor='red')
ax.set_title('fill between where')

# Test support for masked arrays.
x2 = np.ma.masked_greater(x2, 1.0)
ax1.plot(x1, y, x2, y, color='black')
ax1.fill_betweenx(y, x1, x2, where=x2 >= x1, facecolor='green')
ax1.fill_betweenx(y, x1, x2, where=x2 <= x1, facecolor='red')
ax1.set_title('Now regions with x2 > 1 are masked')

# This example illustrates a problem; because of the data
# gridding, there are undesired unfilled triangles at the crossover
# points.  A brute-force solution would be to interpolate all
# arrays to a very fine grid before plotting.

plt.show()


"""
Baby steps in Theano library

"""
import numpy
from matplotlib import pyplot as plt
import theano.tensor as T
from theano import function
from theano import pp

'''
this is a brief insight of how the theano library works

everything we do, every operation, instead of a usual for example "x + y" is declared differently

variables are first declared as Theano based ones and then we assemble an expression as theano function

the reason for this is that theano creates a C code in the backend to make an operation

then the operation's code is optimized by compiler

following code shows a baseline on how to create simple equations, though it may seem intuitive, it
does things in quite unusual way
'''

x = T.dscalar('x')
y = T.dscalar('y')
z = x + y
f = function([x, y], z)
print(f(2, 3))
print(numpy.allclose(f(16.3, 12.1), 28.4))

'''
here we defined simple addition function in more steps than usual

it's because theano acts on symbols to perform creation of C code that runs in background

we declare 2 "scalars" (numbers), assigned them to x and y, respectively, then we
hinted the operation (addition) that gets stored in z variable.

a function declaration takes inputs and outputs, in our case it takes an array [x, y]
and returns z. Z is evaluated as addition
'''

print(pp(z))
'''
here we see how theano assigned the operation
'''

print(z.eval({x : 16.3, y : 12.1}))
'''
we could also invoke the "eval()" from a z variable that had addition operation
"eval()" takes a dictionary with names of variables and values to be assigned to them

eval() utimately imports a "function()", so we end up in the same situation, so it is
slower the first time we invoke this, subsequent invocations are faster since it saves
"function()" imported already

that way we don't need to import "function()", but importing and using it is more
flexible than relyin on "eval()" itself
'''

'''
Addition of two matrices is also very simple, the only difference is using
T.dmatrix instead of T.dscalar
'''

x = T.dmatrix('x')
y = T.dmatrix('y')
z = x + y
f2 = function([x, y], z)

'''
now we do not assign the matrices to x and y directly, but instead we pass matrices
to function as variables, invoking it like: "f2([matrix_nr_1], [matrix_nr_2])

here we add, elementwise, 2 2-dimensional matrices
'''

a1 = [[1, 2],
      [3, 4]]
a2 = [[10, 20],
      [30, 40]]

result = f2(a1, a2)
print(result)

'''
we could also utilize numpy.array
'''
a3 = numpy.array([[1, 2],
                  [3, 4]])
a4 = numpy.array([[10, 20],
                  [30, 40]])

print(f2(a3, a4))

'''
ex1. modify the code at the bottom,

import theano
a = theano.tensor.vector() # declare variable
out = a + a ** 10
# build symbolic expression
f = theano.function([a], out)
# compile function
print(f([0, 1, 2]))

to reflect the expression: a ** 2 + b ** 2 + 2 * a * b
'''

c = 16
d = 13

sample_result = [c**2 + d**2 + 2*c*d]

a = T.vector('a') # declare variable
b = T.vector('b')
add = a + b
out = T.power(add, 2)
# build symbolic expression
f3 = function([a, b], out)
# compile function
print(f3([0, 1, 2], [0, 1, 2]))

assert sample_result == f3([16], [13])
print(f3([16], [13]))

'''
calculation more elaborate functions is also possible
here we calculate a result of a "logistic" function, sometimes referred to as "sigmoid"

'''

matrix = T.dmatrix('matrix')
s = 1/ (1+ T.exp(-matrix))
sigmoid = function([matrix], s)
X = numpy.linspace(-6, 6, 100)
values = sigmoid([X])[0]

print(X)
print(values)

plt.plot(X, values)
plt.show()

'''
We could also produce gradients.
Theano provides efficient symbolic differentiaition, using T.grad as a macro

lets compute a gradient of logistic function

note that we compute it for SCALAR values here!!!
'''

x2 = T.scalar('x2')
sigm = 1/ (1+ T.exp(-x2))
sigm_grad = T.grad(sigm, x2)
# uncomment this if you copy code to another program, i already declared this above
# X = numpy.linspace(-6, 6, 100)
'''
ok we declared derivative (gradient) and a regular function, at the coordinate passed
to the function

theano "function()" can also accept multiple outputs, not only multiple inputs

here we calculate the regular function output, as well as the derivative at the same point

KNOWN PROBLEM: the result is a list of 0-dimensional arrays, which can't be accessed in regular way
use "list[index].sum()" to extract a desired value from it
'''

multiple_sigmoid_outputs = function([x2], [sigm, sigm_grad])

val_out = multiple_sigmoid_outputs(4)
print(val_out)
print(val_out[0].sum(), val_out[1].sum())

'''
ok so we defined a derivative of a function at a single point, but what about computing
whole lists of points?

you have to wrap the result of a regular with the "T.sum" to make that happen
'''

x3 = T.matrix('x3')
sigm2 = T.sum(1/ (1+ T.exp(-x3)))
sigm2_grad = T.grad(sigm2, x3)
logistics = function([x3], [sigm2, sigm2_grad])
values2 = logistics([X])

print(values2)
Y = values2[1][0]
plt.plot(X, Y, c='red')
plt.plot(X, values, c='blue')
plt.show()

print(logistics.maker.fgraph.outputs)
pp(logistics.maker.fgraph.outputs[0])

'''
Back to the basics

How to set a default value for a variable in theano function?

it isn't that hard as it looks, just use the "In" class

"In" from theano accepts a variable and a "value=" which initializes default variable
if not present

You can think of a "In(variable_name, value=default_value)" class as an "Input" to a function

We will also use a "T.dscalars" macro, which creates multiple variables in one line, unlike the
"T.dscalar"
'''

from theano import In

x5, y5 = T.dscalars('x5', 'y5')
z5 = x5 + y5
func5 = function([x5, In(y5, value=4)], z5)
print(func5(32))
print(func5(20, 56.7))

assert func5(25) == 29

'''
and how to share a value between theano functions?

you can allocate memory space that will be accessible from any theano function, even after
it finishes work

the additional "set_value()" and "get_value()" help show and modify shared varaible's contents without a need of creting
a function

functions use shared variables in slightly different way than regular variables; we have to pass the in "updates" list
declaring what we want to do with the variable
'''

from theano import shared

state = shared(0)
inc = T.iscalar('inc')
accumulator = function([inc], state, updates=[(state, state+inc)])

print(state.get_value())
accumulator(1)
print(state.get_value())
accumulator(300)
print(state.get_value())
state.set_value(-1)
accumulator(3)
print(state.get_value())

decrementor = function([inc], state, updates=[(state, state-inc)])
decrementor(2)
print(state.get_value())


"""
Problem 1

If we list all the natural numbers below 10 that are multiples of 3 or 5,
we get 3, 5, 6 and 9. The sum of these multiples is 23.

Find the sum of all the multiples of 3 or 5 below 1000.

"""

import math
import numpy as np

def multiples_sum(lim,*numeros):
    div = np.lcm.reduce(numeros)
    return sum([i for i in range(0,lim,div)])

def main():
    # Defining the range limit (given by the problem statement)
    value = 1000

    # Defining the multiples of 3 and 5
    mult_3 = multiples_sum(value,3)
    mult_5 = multiples_sum(value,5)

    # Defining the repeated multiples of 3 and 5 at the same time
    mult_3_5 = multiples_sum(value,3,5)

    # Calculating the sum of all the multiples of 3 or 5 below 1000
    answer = mult_3 + mult_5 - mult_3_5

    # Printing the result
    print(answer)

main()


import tensorflow as tf
import numpy as np
tf.set_random_seed(777)

# data (CSV):
# 0,0.08379999999998593,0.5622259218589494,0
# 171,1.6777333333333564,0.7347358779862057,1
xy = np.loadtxt('cluster.csv', delimiter=',', dtype=np.float32)
x_data = xy[:, 1:-1]
y_data = xy[:, [-1]]
print(x_data.shape, y_data.shape)

# input output
X = tf.placeholder(tf.float32, shape=[None, 2])
Y = tf.placeholder(tf.float32, shape=[None, 1])

W = tf.Variable(tf.random_normal([2, 1]), name='weight')
b = tf.Variable(tf.random_normal([1]), name='bias')

# Hypothesis using sigmoid:
hypothesis = tf.sigmoid(tf.matmul(X, W) + b)

# cost fuction
cost = -tf.reduce_mean(Y * tf.log(hypothesis) + (1 - Y) * tf.log(1 - hypothesis))

train = tf.train.GradientDescentOptimizer(learning_rate=0.01).minimize(cost)

# abnormal if hypothesis>0.5 else normal
predicted = tf.cast(hypothesis > 0.5, dtype=tf.float32)
accuracy = tf.reduce_mean(tf.cast(tf.equal(predicted, Y), dtype=tf.float32))

#Launch graph
with tf.Session() as sess:
    sess.run(tf.global_variables_initializer())

    for step in range(10001):
    #for step in range(10001):
        cost_val, _ = sess.run([cost, train], feed_dict={X: x_data, Y: y_data})
        if step % 200 == 0:
            print(step, cost_val)

    h, c, a = sess.run([hypothesis, predicted, accuracy], feed_dict={X: x_data, Y: y_data})
    print("\nHypothesis: ", h, "\nCorrect (Y): ", c, "\nAccuracy: ", a)
    print("test input ::: should be 1", sess.run(hypothesis, feed_dict={X: [[1.9980000000000002, 0.2774374445897312]]}))
    print("test input ::: should be 0", sess.run(hypothesis, feed_dict={X: [[0.5418000000000014, 0.15445027502694608]]}))
    print("test input ::: should be 1", sess.run(hypothesis, feed_dict={X: [[1.9040539729735135,0.788305155932941]]}))
    print("test input ::: should be 0 cjb", sess.run(hypothesis, feed_dict={X: [[0.1, 0.1]]}))
    print("test input ::: should be 1 cjb", sess.run(hypothesis, feed_dict={X: [[1.9, 0.8]]}))
    print("test input ::: should be 1 cjb", sess.run(hypothesis, feed_dict={X: [[0.1, 0.9]]}))

#print("TEST abnormal (CPU Memory)", sess.run(hypothesis, feed_dict={X: [[0.1013340088932656, 0.5580860958318363]]}))
#print(sess.run(hypothesis, feed_dict={X: [[0.1, 0.5]]}))


import numpy as np
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
from matplotlib import cm
from mpl_toolkits.mplot3d import Axes3D
import math
import sys
import os

data_dir = sys.argv[1]
out_dir = sys.argv[2]
test_set = os.path.join(sys.argv[1], 'q2test.csv')
out = os.path.join(sys.argv[2], 'Q2c.txt')
outfile = open(out, "w")

# 2. Sampling and Stochastic Gradient
print("\n################ 2. Sampling and Stochastic Gradient ################", file=outfile)

# (a) Sampling

w = np.array([[3],[1],[2]])
X1 = np.random.normal(3, 2, (int)(1e6))
X2 = np.random.normal(-1, 2, (int)(1e6))
noise = np.random.normal(0, np.sqrt(2), int(1e6)).reshape(-1,1)

X = np.column_stack((X1, X2))
X = np.column_stack((np.ones(X.shape[0]), X))
Y = np.dot(X, w) + noise
m = len(Y)

# (b) Stochastic Gradient

def hw(theta, i, r):
	x = X[i*r:(i+1)*r,]
	return np.dot(x, theta)

def Jw(h, i, r):
	y = Y[i*r:(i+1)*r,]
	return (0.5/r)*np.sum((y - h)**2)

def dJw(h, i, r):
	x = X[i*r:(i+1)*r,]
	y = Y[i*r:(i+1)*r,]
	return (1.0/r)*np.dot(x.T, (y - h))

def stochasticGradientDescent(x, y, r):
	b = (int)(m/r)
	eta = 0.001
	theta = np.zeros((x.shape[1], 1))
	thetaL = [theta]
	prevCost = 0
	converged = False
	itr = 0
	while not converged:
		for i in range(b):
			h = hw(theta, i, r)
			cost = Jw(h, i, r)
			theta = theta + eta*dJw(h, i, r)
			thetaL.append(theta)
		itr += 1
		error = abs(cost - prevCost)
		prevCost = cost
		if error < 1e-7 or itr > 25000:
			converged = True

	return thetaL, itr, cost

thetaL1, maxit1, cost1 = stochasticGradientDescent(X, Y, 1)
thetaL2, maxit2, cost2 = stochasticGradientDescent(X, Y, 100)
thetaL3, maxit3, cost3 = stochasticGradientDescent(X, Y, 10000)
thetaL4, maxit4, cost4 = stochasticGradientDescent(X, Y, 1000000)

# (c) Test Data

data = np.loadtxt(test_set, delimiter=',', skiprows=1)
q2X1 = data[:,0]
q2X2 = data[:,1]
q2X = np.column_stack((q2X1, q2X2))
q2X = np.column_stack((np.ones(q2X.shape[0]), q2X))
q2Y = data[:,-1].reshape(-1,1)
q2m = len(q2Y)

def testError(theta):
	h = np.dot(q2X, theta)
	return (0.5/q2m)*np.sum((q2Y - h)**2)

error1 = testError(thetaL1[-1])
error2 = testError(thetaL2[-1])
error3 = testError(thetaL3[-1])
error4 = testError(thetaL4[-1])

print('\nFor r = 1 : Training Error = {}, Test Error = {}'.format(cost1, error1), file=outfile)
print('For r = 100 : Training Error = {}, Test Error = {}'.format(cost2, error2), file = outfile)
print('For r = 10000 : Training Error = {}, Test Error = {}'.format(cost3, error3), file=outfile)
print('For r = 1000000 : Training Error = {}, Test Error = {}'.format(cost4, error4), file=outfile)


## 2. Systems of equations as matrices ##

import numpy as np

# Set the dtype to float to do float math with the numbers.
matrix = np.asarray([
    [2, 1, 25],
    [3, 2, 40]
], dtype=np.float32)
matrix[0] = matrix[0] * 2
matrix[0] = matrix[0] - matrix[1]
matrix[1] = matrix[1] - (matrix[0] * 3)
matrix[1] /= 2
print(matrix)

## 4. Solving more complex equations ##

import numpy as np

matrix = np.asarray([
    [1, 2, 0, 7],
    [0, 3, 3, 11],
    [1, 2, 2, 11]
], dtype=np.float32)
matrix[2] = matrix[2] - matrix[0]
matrix[2] = matrix[2] / 2
matrix[1] = matrix[1] / 3
matrix[1] = matrix[1] - matrix[2]
matrix[0] = matrix[0] - 2 * matrix[1]

## 5. Echelon form ##

matrix = np.asarray([
    [0, 0, 0, 7],
    [0, 0, 1, 11],
    [1, 2, 2, 11],
    [0, 5, 5, 1]
], dtype=np.float32)

# Swap the first and the third rows - first swap
matrix[[0,2]] = matrix[[2,0]]
matrix[[1,3]] = matrix[[3,1]]
matrix[[2,3]] = matrix[[3,2]]

## 6. Reduced row echelon form ##

A = np.asarray([
        [0, 2, 1, 5],
        [1, 2, 1, 8],
        [3, 0, 1, 10],
        ], dtype=np.float32)

# First, we'll swap the second row with the first to get a non-zero coefficient in the first column
A[[0,1]] = A[[1,0]]

# The leading coefficient is already 1, so there's no need to divide
# Now, we need to make sure that our 1 coefficient is the only coefficient in its column
# We have to subtract three times the first row from the third row
A[2] -= 3 * A[0]

# Now, we move to row 2
# We divide by 2 to get a one as the leading coefficient
A[1] /= 2

# We subtract 2 times the second row from the first to get rid of
# the second column coefficient in the first row
A[0] -= 2 * A[1]

# And we'll add 6 times the second row to the third to eliminate the leading coefficient there
A[2] += 6 * A[1]

# Now, we can move to the third row where the leading coefficient is already 1
# We just need to subtract half of the third from the second
A[1] -= 0.5 * A[2]

# We're finished, and our system is solved!
print(A)

## 7. Inconsistency ##

A = np.asarray([
    [10, 5, 20, 60],
    [3, 1, 0, 11],
    [8, 2, 2, 30],
    [0, 4, 5, 13]
], dtype=np.float32)

B = np.asarray([
    [5, -1, 3, 14],
    [0, 1, 2, 8],
    [0, -2, 5, 1],
    [0, 0, 6, 6]
], dtype=np.float32)

A_consistent,B_consistent = True,False

## 8. Infinite solutions ##

A = np.asarray([
        [2, 4, 8, 20],
        [4, 8, 16, 40],
        [20, 5, 5, 10]
], dtype=np.float32)
A_infinite = True

B = np.asarray([
        [1, 1, 1, 4],
        [3, -2, 5, 8],
        [8, -4, 5, 10]
        ], dtype=np.float32)

B_infinite = False

Linear Algebra/Solving systems of equations with matrices-55.py


#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
@author: Ashutosh Vyas

A function to compute cost for multivariate linear regression.

This function is used for the Optional Part of the Exercise 1: Linear Regression.
cost_J = compute_cost(x_data, y_data, theta) computes the cost of using theta
as the parameter for linear regression to fit the data points in x_data, y_data.
"""

import numpy

def compute_cost_multi(x_data, y_data, theta, num_examples):
    """
    Compute cost for single iteration of multivariate linear regression.

    Given the multivariate hypothesis,
    h(x) = theta0 + theta1*x1 + theta2*x2 = theta^transpose.x,
    this function calculates the cost incurred for the given value of theta.
    Thus computes J(theta)

    Parameters
    ----------
    x_data : ndarray
        X 2D array dataset with appended theta0 initalized as 1.
    y_data : ndarray
        A vector of actual values of y from the dataset.
    theta : ndarray
        theta0 (y-intercept) and theta1 (slope) of the linear model.
    num_examples : scalar
        Number of training examples.

    Returns
    -------
    cost_J : scalar
        Cost incurred as difference between hypothesis and actual value.
        Represented as sum -of squared errors

    """

    x_dot_theta = numpy.reshape(numpy.dot(x_data, theta), y_data.shape)  # calculating hypothesis
    difference = x_dot_theta - y_data  # error = hypothesis - actual
    squared = numpy.power(difference, 2.0)  # squared error
    factor = (1.0/(2.0*num_examples))
    cost_J = factor*numpy.sum(squared)  # sum of squared error
    return cost_J


####################################################################################
## PROBLEM1: Gradient Descent
## Gradient descent is a popular optimization technique to solve many
## machine learning problems. In this case, we will explore the gradient
## descent algorithm to fit a line for the given set of 2-D points.
## ref: https://tinyurl.com/yc4jbjzs
## ref: https://spin.atomicobject.com/2014/06/24/gradient-descent-linear-regression/
##
##
## input: directory of faces in ./data/1_points.csv/
## function for reading points is provided
##
##
## your task: fill the following functions:
## evaluate_cost
## evaluate_gradient
## udpate_params
## NOTE: do NOT change values of 'init_params' and 'max_iterations' in optimizer
##
##
## output: cost after convergence (rmse, lower the better)
##
##
## NOTE: all required modules are imported. DO NOT import new modules.
## NOTE: references are given intline
## tested on Ubuntu14.04, 22Oct2017, Abhilash Srikantha
####################################################################################

import numpy as np
import matplotlib.pyplot as plt
import time

def load_data(fname):
    points = np.loadtxt(fname, delimiter=',')
    y_ = points[:,1]
    # append '1' to account for the intercept
    x_ = np.ones([len(y_),2])
    x_[:,0] = points[:,0]
    # display plot
    #plt.plot(x_[:,0], y_, 'ro')
    #plt.xlabel('x-axis')
    #plt.ylabel('y-axis')
    #plt.show()
    print('data loaded. x:{} y:{}'.format(x_.shape, y_.shape))
    return x_, y_

def evaluate_cost(x_,y_,params):
    tempcost = 0
    for i in range(len(y_)):
        tempcost += (y_[i] - ((params[0] * x_[i,0]) + params[1])) ** 2
    return tempcost / float(10000)

def evaluate_gradient(x_,y_,params):
    m_gradient = 0
    b_gradient = 0
    N = float(len(y_))
    for i in range(len(y_)):
        m_gradient += -(2/N) * (x_[i,0] * (y_[i] - ((params[0] * x_[i,0]) + params[1])))
        b_gradient += -(2/N) * (y_[i] - ((params[0] * x_[i,0]) + params[1]))
    return [m_gradient,b_gradient]

def update_params(old_params, grad, alpha):
    new_m = old_params[0] - (alpha * grad[0])
    new_b = old_params[1] - (alpha * grad[1])
    return [new_m,new_b]

# initialize the optimizer
optimizer = {'init_params':np.array([4.5,2.0]) ,
             'max_iterations':10000,
             'alpha':0.69908,
             'eps':0.0000001,
             'inf':1e10}

# load data
x_, y_ = load_data("./data/1_points.csv")

# time stamp
start = time.time()

try:
    # gradient descent
    params = optimizer['init_params']
    old_cost = 1e10
    for iter_ in range(optimizer['max_iterations']):
        # evaluate cost and gradient
        cost = evaluate_cost(x_,y_,params)
        grad = evaluate_gradient(x_,y_,params)
        # display
        if(iter_ % 10 == 0):
            print('iter: {} cost: {} params: {}'.format(iter_, cost, params))
        # check convergence
        if(abs(old_cost - cost) < optimizer['eps']):
            break
        # udpate parameters
        params = update_params(params,grad,optimizer['alpha'])
        old_cost = cost
except:
    cost = optimizer['inf']

# final output
print('time elapsed: {}'.format(time.time() - start))
print('cost at convergence: {} (lower the better)'.format(cost))


from numpy import sqrt
import matplotlib as pl
'''
@author: Arleigh Dickerson
@see: p17 and 18 of Sauer's Numerical Analysis, example 0.6
'''

# square root of the result of b squared minus the quantity of 4 times a times c
def sqrtTerm(a, b, c): return sqrt((b ** 2) - (4 * a * c))

# call to get x1 for a, b, c
def naiveX1(a, b, c): return (-b + sqrt((b ** 2) - (4 * a * c))) / (2 * a)

# call to get x2 for a, b, c
# copy and paste with - instead of +...
def naiveX2(a, b, c): return (-b - sqrt((b ** 2) - (4 * a * c))) / (2 * a)

# second root (- part of eq) can be problematic if b is large compared to a or c
def naiveQuadratic(a, b, c):
    # a tuple with both root values. Values will be identical if double root.
    return (naiveX1(), naiveX2())

# use if b and sqrt(bsquared minus 4ac) are nearly equal in magniture
# and b is positive
def impl0dot13(a, b, c):
    def x1(): return -(b + sqrtTerm(a, b, c)) / (2 * a)
    def x2(): return -((2 * c) / (b + sqrtTerm(a, b, c)))
    return (x1(), x2())

# use if b and sqrt(bsquared minus 4ac) are nearly equal in magniture
# and b is negative
def impl0dot14(a, b, c):
    def x1(): return naiveX1(a, b, c)
    def x2(): return ((2 * c) / (-b + sqrtTerm(a, b, c)))
    return (x1(), x2())

def nearlyEqual(term0, term1) :
    threshold = 0.001  # arbitrary (for now)
    difference = abs(term0 - term1)
    return difference < threshold


from __future__ import print_function
import numpy as np
import matplotlib.pyplot as plt
from scipy import stats

np.random.seed(3)
a = np.random.random(10)
b = np.random.normal(size=10)
two_samp = stats.ks_2samp(a,b)
# since the p value is less than 0.05 I reject that the
# two samples came from the same distribution
# with a 95% confidence level.

np.random.seed(121)
binom_mars = stats.binom(100,0.0925)
mars_rv = binom_mars.rvs(10)
print(np.mean(mars_rv))
print(binom_mars.cdf(10))

x = np.arange(0,50,1)
plt.figure()
plt.plot(x, binom_mars.sf(x))
plt.show()

np.random.seed(32332)
# generate 1000 samples from the
rv = stats.laplace.rvs(loc=75, scale=5, size=1000)

# save the random variables as numpy array
np.save('Aluminum_youngs_moduli',rv)

# fit a normal distribion
norm_param = stats.norm.fit(rv)
laplace_param = stats.laplace.fit(rv)
maxwell_param = stats.maxwell.fit(rv)
# vonmises_param = stats.vonmises.fit(rv)
logistic_param = stats.logistic.fit(rv)

x = np.linspace(10,150,200)

plt.figure()
plt.hist(rv, bins=21, normed=True)
plt.plot(x, stats.norm.pdf(x, *norm_param), label='norm')
plt.plot(x, stats.maxwell.pdf(x, *maxwell_param), label='maxwell_param')
# plt.plot(x, stats.vonmises.pdf(x, *vonmises_param), label='vonmises_param')
plt.plot(x, stats.logistic.pdf(x, *logistic_param), label='logistic_param')
plt.plot(x, stats.laplace.pdf(x, *laplace_param), label='laplace')
plt.legend()
plt.show()

plt.figure()
plt.hist(rv, cumulative=True, bins=21, normed=True)
plt.plot(x, stats.norm.cdf(x, *norm_param), label='norm')
plt.plot(x, stats.maxwell.cdf(x, *maxwell_param), label='maxwell_param')
# plt.plot(x, stats.vonmises.pdf(x, *vonmises_param), label='vonmises_param')
plt.plot(x, stats.logistic.cdf(x, *logistic_param), label='logistic_param')
plt.plot(x, stats.laplace.cdf(x, *laplace_param), label='laplace')
plt.legend()
plt.show()

# ks_statistic
ks_norm = stats.kstest(rv, 'norm', norm_param)
ks_laplace = stats.kstest(rv, 'laplace', laplace_param)
ks_maxwell = stats.kstest(rv, 'maxwell', maxwell_param)
ks_logistic = stats.kstest(rv, 'logistic', logistic_param)

# accept either the logistic or laplace!


import solver.algorithms as alg
import numpy as np

def problem5(t0, tf, NA0, NB0, tau, n, returnlist=False):
    """Uses Euler's method to model the solution to a radioactive decay problem where dNA/dt = NB/tau - NA/tau and dNB/dt = NA/tau - NB/tau.

    Args:
        t0 (float): Start time
        tf (float): End time
        NA0 (int): Initial number of NA nuclei
        NB0 (int): Initial number of NB nuclei
        tau (float): Decay time constant
        n (float): Number of points to sample at
        returnlist (bool) = Controls whether the function returns the list of points or not. Defaults to false

    Returns:
        solution (list): Points on the graph of the approximate solution. Each element in the list has the form (t, array([NA, NB]))

    In the graph, NA is green and NB is blue
    """
    print("Problem 5: ~Radioactive Decay~ dNA/dt = NB/tau - NA/tau & dNB/dt = NA/tau - NB/tau.")
    N0 = np.array([NA0, NB0])
    A = np.array([[-1/tau, 1/tau],[1/tau, -1/tau]])
    def dN_dt(t, N):
        return A @ N
    h = (tf-t0)/(n-1)
    print("Time step of %f seconds." % h)
    solution = alg.euler(t0, tf, n, N0, dN_dt)
    if returnlist:
        return solution


"""
Created on Sat Apr 18 21:11:50 2020
@author: Mohammad Asif Zaman
Differentiaion matrix operators in 1D and 2D
-   First derivatives d/dx, d/dy
-   Second derivatives d2/dx2, d2/dy2
Notes:
        - kron() is different in python compared to MATLAB. The matrix order is reveresed here.
          MATLAB version:
          Dx_2d = sp.kron(Dx_1d,Iy)
          Dy_2d = sp.kron(Ix,Dy_1d)
"""

import scipy.sparse as sp

def Diff_mat_1D(Nx):

    # First derivative
    D_1d = sp.diags([-1, 1], [-1, 1], shape = (Nx,Nx)) # A division by (2*dx) is required later.
    D_1d = sp.lil_matrix(D_1d)
    D_1d[0,[0,1,2]] = [-3, 4, -1]               # this is 2nd order forward difference (2*dx division is required)
    D_1d[Nx-1,[Nx-3, Nx-2, Nx-1]] = [1, -4, 3]  # this is 2nd order backward difference (2*dx division is required)

    # Second derivative
    D2_1d =  sp.diags([1, -2, 1], [-1,0,1], shape = (Nx, Nx)) # division by dx^2 required
    D2_1d = sp.lil_matrix(D2_1d)
    D2_1d[0,[0,1,2,3]] = [2, -5, 4, -1]                    # this is 2nd order forward difference. division by dx^2 required.
    D2_1d[Nx-1,[Nx-4, Nx-3, Nx-2, Nx-1]] = [-1, 4, -5, 2]  # this is 2nd order backward difference. division by dx^2 required.

    return D_1d, D2_1d

def Diff_mat_2D(Nx,Ny):
    # 1D differentiation matrices
    Dx_1d, D2x_1d = Diff_mat_1D(Nx)
    Dy_1d, D2y_1d = Diff_mat_1D(Ny)

    # Sparse identity matrices
    Ix = sp.eye(Nx)
    Iy = sp.eye(Ny)

    # 2D matrix operators from 1D operators using kronecker product
    # First partial derivatives
    Dx_2d = sp.kron(Iy,Dx_1d)
    Dy_2d = sp.kron(Dy_1d,Ix)

    # Second partial derivatives
    D2x_2d = sp.kron(Iy,D2x_1d)
    D2y_2d = sp.kron(D2y_1d,Ix)

    # Return compressed Sparse Row format of the sparse matrices
    return Dx_2d.tocsr(), Dy_2d.tocsr(), D2x_2d.tocsr(), D2y_2d.tocsr()


# -*- coding: utf-8 -*-
""" spfs/utils/normalization """

import numpy as np

class Normalization:
    """ Holds several normalization methods """

    @staticmethod
    def lp(vector, p):
        """
        Returns the lp normalized vector

        Args:
            vector (np.ndarray): vector to normalize
            p             (int): type of normalization

        Returns:
            np.ndarray
        """
        assert isinstance(vector, np.ndarray)
        assert isinstance(p, int)
        assert 0 <= p <= 2

        lp_norm = np.linalg.norm(vector, p)

        if lp_norm in (0, 1, np.nan):
            return vector

        return vector/lp_norm

    @classmethod
    def l0(cls, vector):
        """
        Returns the l0 normalized vector

        Args:
            vector (np.ndarray): vector to normalize

        Returns:
            np.ndarray
        """
        return cls.lp(vector, 0)

    @classmethod
    def l1(cls, vector):
        """
        Returns the l1 normalized vector

        Args:
            vector (np.ndarray): vector to normalize

        Returns:
            np.ndarray
        """
        return cls.lp(vector, 1)

    @classmethod
    def l2(cls, vector):
        """
        Returns the l2 normalized vector

        Args:
            vector (np.ndarray): vector to normalize

        Returns:
            np.ndarray
        """
        return cls.lp(vector, 2)

    @staticmethod
    def standardization(vector):
        """
        Returns normalized vector following the formula: (vector - mean) / std

        Args:
            vector (np.ndarray): vector to normalize

        Returns:
            np.ndarray
        """
        dev = np.std(vector)
        return (vector-np.mean(vector))/dev


import numpy as np
import pylab as pl
import random as rd
import time

# Part a
# First we use the 'random' package
# Single seed used for generating a sequence of random numbers
seed = 1
rd.seed(seed)

num_sample = 10**5
# Count the number of numbers generated
count = 0

# Contains the numbers generated, starting the seed
rannumrd = np.array([seed])

while count < num_sample:
    rannumrd = np.append(rannumrd, rd.random())
    count = count + 1

pl.figure()
pl.hist(rannumrd, bins = np.linspace(0., 1., 100)) # 100 bins between 0 and 1
pl.xlabel("x")
pl.ylabel("Number of counts in bin")
pl.title("Distribution of 1e5 random numbers using 'random' package")

# Compare with 'numpy.random.uniform' for uniform deviation
# numpy.seed does not allow seed dtype = float, must be integers
# Also all numerical inputs below must be integers
np.random.seed(seed)

rannumnp = np.array([seed])
rannumnp = np.append(rannumnp, np.random.uniform(low = 0, high = 1, size = num_sample))

pl.figure()
pl.hist(rannumnp, bins = np.linspace(0., 1., 100)) # 100 bins
pl.xlabel("x")
pl.ylabel("Number of counts in bin")
pl.title("Distribution of 1e5 random numbers using 'numpy.random.uniform' package")

# Conclusion: random.random is a better method to use

# Part b - transformation method
# Transforming from uniform P(x) into the PDF given, integrating the PDF between
# 0 and y and equating to x,
# the relation between x and y was found to be y = cos^-1 (1 - 2*x)

# Using x values computed from 'random' package in part a, compute corresponding y
start_time_trans = time.clock() # Records the start time
y = np.arccos(1-2*rannumrd)
time_trans = time.clock() - start_time_trans # Records the end time and subtract from
# start to check running time for this method

# Plotting sine function to check
check = np.linspace(0., np.pi, 500) # Used for plotting comparison function and predicted PDF
pl.figure()
(count, bins, patches) = pl.hist(y, bins = np.linspace(0., np.pi, 100),
normed=True, label = "Histogram of sinusoidal distribution")
# 100 bins between 0 and pi
pl.plot(check, 0.5*np.sin(check), 'r-', label = "Predicted PDF")
pl.xlabel("y")
pl.ylabel("Normalised PDF")
pl.legend()
pl.title("Distribution of 1e5 random numbers using a sinusoidal distribution")

# Part c - rejection method
# PDF (y) takes the values of the sin function in part b and substitute into
# the new PDF given.
PDF = 2./np.pi*(np.sin(y))**2.
comp = np.linspace(0., np.pi, 10**5.+1.)
PDFcomp = 2./np.pi*(np.sin(y))

# Accept = array containing all accepted values
accept = np.array([])
# Counters to keep track of when to stop iterating
num_accept = 0.
num_trials = 0.

start_time_rej = time.clock() # Records the start time
while num_accept < num_sample:
    indexP = rd.randrange(PDF.size)
    compval = rd.uniform(0, PDFcomp[indexP])
    if PDF[indexP] >= compval:
        accept = np.append(accept, y[indexP])
        num_accept += 1.
    num_trials += 1.
time_rej = time.clock() - start_time_rej # Records the end time and subtract from
# start to check running time for this method

pl.figure()
(count2, bins2, patches2) = pl.hist(accept, bins = np.linspace(0., np.pi, 100),
normed=True, label = "Histogram of sine squared distribution") # 100 bins between 0 and pi
pl.plot(check, 2./np.pi*(np.sin(check))**2, 'r-', label = "Predicted PDF")
pl.plot(check, 2./np.pi*(np.sin(check)), 'k-', label = "Comparison function")
pl.legend()
pl.xlabel("y")
pl.ylabel("Normalised PDF")
pl.title("Distribution of 1e5 random numbers using a sine squared distribution")

# Check measuring times
print(time_trans, time_rej, time_trans/time_rej)


import numpy as np
from matplotlib import pyplot as plt
from scipy.misc import comb
from numpy.random import rand

# Decasteljau's algorithm problem
def decasteljau(p,t):
    """ Evaluates a Bezier curve with control points 'p' at time 't'.
    The points in 'p' are assumed to be stored in its rows.
    'p' is assumed to be 2-dimensional."""
    pass

# Bernstein polynomial problem
def bernstein(i, n):
    """ Returns the 'i'th Bernstein polynomial of degree 'n'."""
    pass

# Coordinate function problem.
def bernstein_pt_aprox(X):
    """ Returns the 'x' and 'y' coordinate functions for a 2-dimensional
    Bezier curve with control points 'X'."""
    pass

# plot demonstrating numerical instability in Bernstein polys.
def compare_plot(n, res=501):
    """ Produces a plot showing a Bezier curve evaluated via
    the Decasteljau algorithm and a Bezier curve evaluated using
    Bernstein polynomials. Control points are chosen randomly.
    Instability should be evident for moderately large values of 'n'."""


import numpy as np
import scipy

__all__ = ['slepian_filter','slepian_psd']

def slepian_filter(length, filter_band_length, order=None):
    '''
    Slepian filters bank.

    Parameters
    --------------------
    * length: int,
        the length pf signal.
    * filter_band_length: [int, int],
        filter bandwidth with respect to length
        (in points).
    * order: int,
        order of the filter.

    Returns
    ------------------
    * Matrix of filter windows: 2d ndarray,
        base_band_ratio x order with Slepian
        sequence in the colunms.

    Refernce
    ---------------
    [1a] P. Stoica, R.L. Moses,
        Spectral analysis of signals
        - New-York: Present-Hall, 2005.
    [1b] http://www2.ece.ohio-state.edu/~randy/SAtext/
            - Dr.Moses Spectral Analysis of Signals: Resource Page.

    Examples
    ---------------
        h = slepian_filter(256,4,4)
        plt.plot(h[:,1])

    '''
    N = length
    K = filter_band_length

    if(order is None): order = filter_band_length

    vect = K/N*np.sinc(K/N*np.arange(N))
    gamma = scipy.linalg.toeplitz(vect)
    D,V=np.linalg.eig(gamma)

    h=V[:,:order]

    if np.sum(h[:,1])<0:
        h[:,1]=-h[:,1]

    return h

#---------------------------------------
def slepian_psd(x, order, n_psd=None):
    '''
    Estimation of the pseudo-spectrum based on the
        Slepian - Refil algorithm.

    Parameters
    -------------------
    * x: 1d ndarray.
    * order: int,
        order of the model.
    * n_psd: int or None,
        number of samples in psd.

    Returns
    ------------------------
    * pseudo-spectrum: 1d ndarray.

    Refernce
   ------------------------
    [1a] P. Stoica, R.L. Moses,
        Spectral analysis of signals
        - New-York: Present-Hall, 2005.
    [1b] http://www2.ece.ohio-state.edu/~randy/SAtext/
         Dr.Moses Spectral Analysis of Signals: Resource Page.

    Examples
    ------------------------

    See also
    ------------------------
    capone

    '''
    x=np.array(x)
    N=x.shape[0]

    if(n_psd is None):
        n_psd=N

    h=slepian_filter(N,order,order)

    psd = np.zeros(n_psd)

    for i in np.arange(order):
        sp   = np.fft.fft(x*h[:,i],n_psd)
        psd += np.real(sp*np.conj(sp))# 0 imagenary part remain

    return psd


import numpy as np
from scipy.stats import logistic
import matplotlib.pyplot as plt

class LogisticRegressioner(object):
    def __init__(self, tol=1e-6):
        self.tol = tol

    def first_derivative(self, X, Y, w):
        """
        Calculate the 1st derivative of log-loss function
        """
        d, T = X.shape
        sigma = logistic.cdf(np.multiply(-Y, np.dot(w.T, X)))
        ret = np.multiply(sigma, Y)
        ret = np.multiply(np.repeat(ret[np.newaxis, :], d, axis=0), X)
        ret = np.sum(ret, axis=1)
        return ret

    def log_loss(self, X, Y, w):
        L = np.log(logistic.cdf(np.multiply(Y, np.dot(w.T, X))))
        L = np.sum(L)
        L = -L
        return L

    def classifier_w(self, X, Y,eta=0.05):
        """
        and returns a classification vector w \in Rp obtained by gradient
        descent on the logistic regression loss function
        """
        X = X.T
        d, T = X.shape
        X = np.vstack([np.ones((T, )), X])  # add bias 1
        d += 1
        w = np.zeros(d)
        t = 0
        while True:
            t += 1
            w_ = w + eta*self.first_derivative(X, Y, w)
            L_old, L_new = self.log_loss(X, Y, w_), self.log_loss(X, Y, w)
            if np.abs(L_old - L_new) < self.tol: break
            w = w_
            if t % 300 == 1: yield w  # yield for trace the w

        yield w

    def test_sample(self):
        X = np.array([
            [2, 1],
            [1, 20],
            [1, 5],
            [4, 1],
            [1, 40],
            [3, 30],
        ])

        Y = np.array([
            -1,
            -1,
            -1,
            1,
            1,
            1,
        ])
        ws = list(self.classifier_w(X, Y))
        print ws[-1]


import numpy as np
import numpy.linalg as npla

def qr_iteration(A, tol):
    # Your implementation goes here

    for i in range(A.shape[1] - 1, 0, -1):
        while npla.norm(A[i - 1, :i - 1], ord=2) > tol:
            sigma = A[i, i]
            Q, R = npla.qr(A - sigma * np.diag(np.diag(np.ones(A.shape))))
            A = np.dot(R, Q) + sigma * np.diag(np.diag(np.ones(A.shape)))

    return np.diag(A)

def main():

	tol = 10 ** (-16)
	A_1 = np.array([[2, 3, 2], [10, 3, 4], [3, 6, 1]])
	eigenvalues_1 = qr_iteration(A_1.copy(), tol)
	print("Matrix:\n", A_1)
	print("\n")
	print ("Computed eigenvalues: ", eigenvalues_1)
	print ("Actual eigenvalues: ", np.linalg.eigvals(A_1))

	print("\n\n")

	tol = 10 ** (-16)
	A_2 = np.array([[6, 2, 1], [2, 3, 1], [1, 1, 1]])
	eigenvalues_2 = qr_iteration(A_2.copy(), tol)
	print("Matrix:\n", A_2)
	print("\n")
	print ("Computed eigenvalues: ", eigenvalues_2)
	print ("Actual eigenvalues: ", np.linalg.eigvals(A_2))

if __name__ == '__main__':
	main()


"""
Ankit Khandelwal
15863
Exercise 10
"""

from math import exp

import numpy as np

i = 3 * 10 ** -9
Vt = 0.05
V = 5.
R1 = 1000.
R2 = 4000.
R3 = 3000.
R4 = 2000.

def f1(V1, V2):
    return R2 * (V1 - V) + R1 * V1 + R1 * R2 * i * (exp((V1 - V2) / Vt) - 1)

def f2(V1, V2):
    return R4 * (V2 - V) + R3 * V2 - R3 * R4 * i * (exp((V1 - V2) / Vt) - 1)

def df1_x(V1, V2):
    return R2 + R1 + R1 * R2 * i * exp((V1 - V2) / Vt) / Vt

def df1_y(V1, V2):
    return -R1 * R2 * i * exp((V1 - V2) / Vt) / Vt

def df2_x(V1, V2):
    return -R3 * R4 * i * exp((V1 - V2) / Vt) / Vt

def df2_y(V1, V2):
    return R4 + R3 + R3 * R4 * i * exp((V1 - V2) / Vt) / Vt

x1 = 4.
y1 = 2.
f = np.zeros([2, 1])
J = np.zeros([2, 2])
r1 = np.zeros([2, 1])
r1[0, 0] = x1
r1[1, 0] = y1
d = np.ones([2, 1])
while abs(d[0]) > 10 ** -7 and abs(d[1]) > 10 ** -7:
    V1 = r1[0, 0]
    V2 = r1[1, 0]
    f[0, 0] = f1(V1, V2)
    f[1, 0] = f2(V1, V2)
    f1x = df1_x(V1, V2)
    f1y = df1_y(V1, V2)
    f2x = df2_x(V1, V2)
    f2y = df2_y(V1, V2)
    J[0, 0] = f2y
    J[0, 1] = -f1y
    J[1, 0] = -f2x
    J[1, 1] = f1x
    J = (1 / (f1x * f2y - f1y * f2x)) * J
    d = np.dot(J, f)
    r1 = r1 - d

print("V1 = ", r1[0, 0], "V\nV2 = ", r1[1, 0], "V")
print("V = V1 - V2 = ", r1[0, 0] - r1[1, 0], "V")

Ankit Khandelwal_HW3/Exercise 10/exercise 10.py


import math
import numpy
from numpy import linalg

class HMM(object):

    def __init__(self,
        pi: numpy.ndarray,
        T: numpy.matrix,
        O: numpy.matrix,
        states=None,
        emissions=None):

        """
        pi: initial probability distribution
        T : transition probability matrix
        O : emission probability matrix
        states: list of state names
        emissions: list of emission names
        """

        self.pi = pi
        assert(len(pi) == T.shape[0])
        assert(sum(pi) == 1)

        self.T = T
        assert(T.shape[0] == T.shape[1])
        assert(int(sum(T.A1)) == T.shape[0])

        self.O = O
        assert(O.shape[0] == T.shape[0])
        assert(int(sum(O.A1)) == O.shape[0])

        self.states = states
        if self.states:
            assert(len(self.states) == T.shape[0])

        self.emissions = emissions
        if self.emissions:
            assert(len(emissions) == O.shape[1])

    def allclose(self, A: numpy.matrix, B: numpy.matrix, **kwargs):
        """
        Since `numpy.allclose` method doesn't work as expected
        """
        return numpy.all(list(map(
            lambda a,b: math.isclose(a, b, **kwargs), A.A1, B.A1
        )))

    def stationary_dist(self, type: str, **kwargs):
        """
        Given a transition probabilities matrix T, this function computes
        the stationary distribution pi. One can choose two different methods:
        - naive: power method; loop until T^k == T^(k+1), within a certain tolerance
        (look at the method `numpy.allclose`)
        - linear: solve a linear system pi' = (pi * T)', where pi is a row vector of
        indipendent variables
        It returns the array of stationary distribution and the power k+1 such that
        T^k = T^(k+1), None if type is linear

        **kwargs is the keyword argument accepted by `math.isclose` function
        """
        # Check that T is a square matrix
        # assert(reduce(operator.and_, [len(row) == len(T) for row in T]) == True)
        assert(self.T.shape[0] == self.T.shape[1])
        if type == 'naive':
            T_k = self.T
            T_k_1 = self.T ** 2
            power = 2
            while (not self.allclose(T_k, T_k_1, **kwargs)):
                T_k = T_k_1
                T_k_1 = T_k * self.T
                power += 1
            return numpy.array(T_k_1[0].flat), power
        elif type == 'linear':
            """
            The linear system that one might resolve is,
            with pi a row vector of indipendent variables:
            (pi * T)' = pi'
            T' * pi' - pi' = 0
            Since pi is the vector of indipendent variables, the operation (- pi')
            is the same as subtract 1 from every element in the diagonal of T' * pi'.
            In order to obtain a unique solution we must replace one of the equation
            with sum(pi_i) = 1 for all i from 0 to n-1. In this case I choose the first one
            """
            self.T = self.T.T
            self.T[numpy.diag_indices_from(self.T)] -= 1
            self.T[0,:] = numpy.ones(self.T.shape[0])
            b = numpy.zeros(self.T.shape[0])
            b[0] = 1
            return linalg.solve(self.T, b), None
        else:
            raise Exception('Type unspecified')

    def forward(self, observations):
        f = self.pi
        message_f = [f]
        for obs in observations:
            f = (f @ self.T @ numpy.diag(self.O[:, obs].T.A1)).A1
            message_f.append(f / sum(f))
        return message_f

    def backward(self, observations):
        b = [1, 1]
        message_b = [b]
        for i in range(len(observations)-1, -1, -1):
            b = (self.T @ numpy.diag(self.O[:, observations[i]].T.A1) @ b).A1
            message_b.append(b / sum(b))
        return message_b

    def smoothing(self, observations):
        def normalized_hadamard(v1, v2):
            res = numpy.multiply(v1, v2)
            return res / sum(res)

        return list(
            map(normalized_hadamard,
                self.forward(observations),
                list(reversed(self.backward(observations)))
            )
        )

    def viterbi(self, observations: list):
        """
        Viterbi algorithm in matrix form
        """
        # Compute initial probability given the first observation
        S = numpy.matrix(numpy.diag(self.pi) @ numpy.diag(self.O[:, observations[0]].T.A1))
        saved_max = []
        for obs in observations[1:]:
            # Get, for every cols, the row of the max value
            amax = numpy.argmax(S, axis=0).A1
            max_per_col = []
            # Get max values per columns
            for col in range(len(self.T)):
                max_per_col.append(S[amax[col], col])
            # 'Forward' pass
            S = numpy.diag(max_per_col) @ self.T @ numpy.diag(self.O[:, obs].T.A1)
            # Save where the maximum comes from (needed fro the 'backward' pass)
            saved_max.append(amax)
        # Get indices of the global max
        row, col = numpy.unravel_index(numpy.argmax(S, axis=None), S.shape)
        # Cols will be the state, row becomes col while running backward
        sequence = [self.states[col] if self.states else col]
        for i in range(len(saved_max)-1, -1, -1):
            sequence.append(self.states[row] if self.states else row)
            row = saved_max[i][row]
        return list(reversed(sequence))

if __name__ == "__main__":
    pi = numpy.array([0.52, 0.48])
    T = numpy.matrix([[0.6, 0.4], [0.17, 0.83]])
    O = numpy.matrix([[1/10, 1/10, 1/10, 1/10, 1/10, 1/2], [1/6]*6])

    # pi = numpy.array([0.98, 0.02])
    # T = numpy.matrix([[0.4, 0.6], [0.1, 0.9]])
    # O = numpy.matrix([[0.8, 0.2], [0.1, 0.9]])

    # pi = numpy.array([.5, .5])
    # T = numpy.matrix([[.7, .3], [.3, .7]])
    # O = numpy.matrix([[.9, .1], [.2, .8]])

    hmm = HMM(pi, T, O, ['Loaded', 'Fair'])

    print(hmm.viterbi([2,0,5,5,5,3]))
    # print(hmm.viterbi([1, 0, 1]))
    # print(hmm.forward([0, 0, 1, 0]))
    # print(list(reversed(hmm.backward([0, 0, 1, 0]))))
    print(hmm.smoothing([0, 0, 1, 0]))


import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import datetime

df = pd.read_json('amsterdam_sold_geo_4pp.json')
df.reset_index(drop = True)

df['price per unit area'] = df['price'] / df['area']

df['postdate']=pd.to_datetime(df['posting_date'], dayfirst = True)
df['saledate']=pd.to_datetime(df['sale_date'], dayfirst = True)

df['time_to_sell'] = df['saledate'] - df['postdate']

dfp=df[df['price'] > 50000]
dfp['days_to_sell'] = dfp['time_to_sell'].apply(lambda x: x.days)
dfp=dfp[dfp['days_to_sell']>0]

dfp_week = dfp.groupby(pd.TimeGrouper(key='saledate', freq='W')).mean()

fig, (ax1,ax2) = plt.subplots(2, sharex = True)
ax1.scatter(dfp['saledate'].values, dfp['price']/1000.0, s=10)
ax1.plot_date(dfp_week.index, dfp_week['price']/1000.0, 'r-', linewidth = 2)
ax1.set_xlim([datetime.datetime(2015,4,1), datetime.datetime(2016,8,1)])
ax1.set_xlabel('Date of sale')
ax1.set_ylabel(u'Asking price (1,000 EUR)')
ax1.set_title('Property sales in Amsterdam')
ax1.set_ylim([0,800])
ax1.grid('on')

ax2.scatter(dfp['saledate'].values,dfp['days_to_sell'].values, s=10)
ax2.plot_date(dfp_week.index, dfp_week['days_to_sell'].values,'r-', linewidth = 2)
ax2.set_xlabel('Date of sale')
ax2.set_ylabel('Days on the market')
ax2.set_ylim([0,300])
ax2.grid('on')

fig.autofmt_xdate()

plt.show(block = False)

# plt.show()

# df.to_json('amsterdam_sold_geo_4pp_dates.json')

# plt.scatter(df['longitude_4pp'], df['latitude_4pp'], c=df['price per unit area'])
# plt.axis('equal')
# plt.show()

# df_by_4pp = df.groupby('postal_code_4pp').mean()
#
# new_df = pd.DataFrame()
# new_df['lat'] = df_by_4pp['latitude_4pp'].values
# new_df['lon'] = df_by_4pp['longitude_4pp'].values
# new_df['price_per_unit_area'] = df_by_4pp['price per unit area'].values
#
# new_df=new_df[new_df['lat'].notnull()]

# plt.figure()
# plt.scatter(new_df['lon'],new_df['lat'],c=new_df['price_per_unit_area'])
# plt.axis('equal')

# plt.figure()
# plt.scatter(df['price'],df['time_to_sell'])

# dfp = df[df['price']>50000]

# fig, ax = plt.subplots()
# ax.scatter(dfp['area'],dfp['price']/1000.0,c=dfp['latitude_4pp'])
# ax.set_title('Price per unit area colored by latitude')

# fig.colorbar()

# new_df.to_csv('coords_4pp_and_price_per_unit_area4.csv', index=False)

# plt.show()


"""
Module: libfmp.c2.c2_fourier
Author: Frank Zalkow, Meinard Müller
License: The MIT license, https://opensource.org/licenses/MIT

This file is part of the FMP Notebooks (https://www.audiolabs-erlangen.de/FMP)
"""
import numpy as np
from numba import jit
import librosa

@jit(nopython=True)
def generate_matrix_dft(N, K):
    """Generates a DFT (discrete Fourier transfrom) matrix

    Notebook: C2/C2_DFT-FFT.ipynb

    Args:
        N (int): Number of samples
        K (int): Number of frequency bins

    Returns:
        dft (np.ndarray): The DFT matrix
    """
    dft = np.zeros((K, N), dtype=np.complex128)
    for n in range(N):
        for k in range(K):
            dft[k, n] = np.exp(-2j * np.pi * k * n / N)
    return dft

@jit(nopython=True)
def generate_matrix_dft_inv(N, K):
    """Generates an IDFT (inverse discrete Fourier transfrom) matrix

    Notebook: C2/C2_STFT-Inverse.ipynb

    Args:
        N (int): Number of samples
        K (int): Number of frequency bins

    Returns:
        dft (np.ndarray): The IDFT matrix
    """
    dft = np.zeros((K, N), dtype=np.complex128)
    for n in range(N):
        for k in range(K):
            dft[k, n] = np.exp(2j * np.pi * k * n / N) / N
    return dft

@jit(nopython=True)
def dft(x):
    """Compute the disrcete Fourier transfrom (DFT)

    Notebook: C2/C2_DFT-FFT.ipynb

    Args:
        x (np.ndarray): Signal to be transformed

    Returns:
        X (np.ndarray): Fourier transform of x
    """
    x = x.astype(np.complex128)
    N = len(x)
    dft_mat = generate_matrix_dft(N, N)/N
    return np.dot(dft_mat, x)

@jit(nopython=True)
def idft(X):
    """Compute the inverse discrete Fourier transfrom (IDFT)

    Args:
        X (np.ndarray): Signal to be transformed

    Returns:
        x (np.ndarray): Inverse Fourier transform of X
    """
    X = X.astype(np.complex128)
    N = len(X)
    dft_mat = generate_matrix_dft_inv(N, N)
    return np.dot(dft_mat, X)

@jit(nopython=True)
def twiddle(N):
    """Generate the twiddle factors used in the computation of the fast Fourier transform (FFT)

    Notebook: C2/C2_DFT-FFT.ipynb

    Args:
        N (int): Number of samples

    Returns:
        sigma (np.ndarray): The twiddle factors
    """
    k = np.arange(N // 2)
    sigma = np.exp(-2j * np.pi * k / N)
    return sigma

@jit(nopython=True)
def twiddle_inv(N):
    """Generate the twiddle factors used in the computation of the Inverse fast Fourier transform (IFFT)

    Args:
        N (int): Number of samples

    Returns:
        sigma (np.ndarray): The twiddle factors
    """
    n = np.arange(N // 2)
    sigma = np.exp(2j * np.pi * n / N)
    return sigma

@jit(nopython=True)
def fft(x):
    """Compute the fast Fourier transform (FFT)

    Notebook: C2/C2_DFT-FFT.ipynb

    Args:
        x (np.ndarray): Signal to be transformed

    Returns:
        X (np.ndarray): Fourier transform of x
    """
    x = x.astype(np.complex128)
    N = len(x)
    log2N = np.log2(N)
    assert log2N == int(log2N), 'N must be a power of two!'
    X = np.zeros(N, dtype=np.complex128)

    if N == 1:
        return x
    else:
        this_range = np.arange(N)
        A = fft(x[this_range % 2 == 0])
        B = fft(x[this_range % 2 == 1])
        C = twiddle(N) * B
        X[:N//2] = A + C
        X[N//2:] = A - C
        return X

@jit(nopython=True)
def ifft_noscale(X):
    """Compute the inverse fast Fourier transform (IFFT) without the final scaling factor of 1/N

    Args:
        X (np.ndarray): Fourier transform of x

    Returns:
        x (np.ndarray): Inverse Fourier transform of X
    """
    X = X.astype(np.complex128)
    N = len(X)
    log2N = np.log2(N)
    assert log2N == int(log2N), 'N must be a power of two!'
    x = np.zeros(N, dtype=np.complex128)

    if N == 1:
        return X
    else:
        this_range = np.arange(N)
        A = ifft_noscale(X[this_range % 2 == 0])
        B = ifft_noscale(X[this_range % 2 == 1])
        C = twiddle_inv(N) * B
        x[:N//2] = A + C
        x[N//2:] = A - C
        return x

@jit(nopython=True)
def ifft(X):
    """Compute the inverse fast Fourier transform (IFFT)

    Args:
        X (np.ndarray): Fourier transform of x

    Returns:
        x (np.ndarray): Inverse Fourier transform of X
    """
    return ifft_noscale(X) / len(X)

def stft_basic(x, w, H=8, only_positive_frequencies=False):
    """Compute a basic version of the discrete short-time Fourier transform (STFT)

    Notebook: C2/C2_STFT-Basic.ipynb

    Args:
        x (np.ndarray): Signal to be transformed
        w (np.ndarray): Window function
        H (int): Hopsize (Default value = 8)
        only_positive_frequencies (bool): Return only positive frequency part of spectrum (non-invertible)
            (Default value = False)

    Returns:
        X (np.ndarray): The discrete short-time Fourier transform
    """
    N = len(w)
    L = len(x)
    M = np.floor((L - N) / H).astype(int) + 1
    X = np.zeros((N, M), dtype='complex')
    for m in range(M):
        x_win = x[m * H:m * H + N] * w
        X_win = np.fft.fft(x_win)
        X[:, m] = X_win

    if only_positive_frequencies:
        K = 1 + N // 2
        X = X[0:K, :]
    return X

def istft_basic(X, w, H, L):
    """Compute the inverse of the basic discrete short-time Fourier transform (ISTFT)

    Notebook: C2/C2_STFT-Inverse.ipynb

    Args:
        X (np.ndarray): The discrete short-time Fourier transform
        w (np.ndarray): Window function
        H (int): Hopsize
        L (int): Length of time signal

    Returns:
        x (np.ndarray): Time signal
    """
    N = len(w)
    M = X.shape[1]
    x_win_sum = np.zeros(L)
    w_sum = np.zeros(L)
    for m in range(M):
        x_win = np.fft.ifft(X[:, m])
        # Avoid imaginary values (due to floating point arithmetic)
        x_win = np.real(x_win)
        x_win_sum[m * H:m * H + N] = x_win_sum[m * H:m * H + N] + x_win
        w_shifted = np.zeros(L)
        w_shifted[m * H:m * H + N] = w
        w_sum = w_sum + w_shifted
    # Avoid division by zero
    w_sum[w_sum == 0] = np.finfo(np.float32).eps
    x_rec = x_win_sum / w_sum
    return x_rec, x_win_sum, w_sum

@jit(nopython=True)
def stft(x, w, H=512, zero_padding=0, only_positive_frequencies=False):
    """Compute the discrete short-time Fourier transform (STFT)

    Args:
        x (np.ndarray): Signal to be transformed
        w (np.ndarray): Window function
        H (int): Hopsize (Default value = 512)
        zero_padding (bool): Number of zeros to be padded after windowing and before the Fourier transform of a frame
            (Note: The purpose of this step is to increase the frequency sampling.) (Default value = 0)
        only_positive_frequencies (bool): Return only positive frequency part of spectrum (non-invertible)
            (Default value = False)

    Returns:
        X (np.ndarray): The discrete short-time Fourier transform
    """

    N = len(w)
    x = np.concatenate((np.zeros(N // 2), x, np.zeros(N // 2)))

    L = len(x)
    M = int(np.floor((L - N) / H)) + 1

    X = np.zeros((N + zero_padding, M), dtype=np.complex128)
    zero_padding_vector = np.zeros((zero_padding, ), dtype=x.dtype)

    for m in range(M):
        x_win = x[m * H:m * H + N] * w
        if zero_padding > 0:
            x_win = np.concatenate((x_win, zero_padding_vector))
        X_win = fft(x_win)
        # Note: X_win = np.fft.fft(x_win) does not work in combination with @jit
        X[:, m] = X_win

    if only_positive_frequencies:
        K = 1 + (N + zero_padding) // 2
        X = X[0:K, :]
    return X

@jit(nopython=True)
def istft(X, w, H, L, zero_padding=0):
    """Compute the inverse discrete short-time Fourier transform (ISTFT)

    Args:
        X (np.ndarray): The discrete short-time Fourier transform
        w (np.ndarray): Window function
        H (int): Hopsize
        L (int): Length of time signal
        zero_padding (bool): Number of zeros to be padded after windowing and before the Fourier transform of a frame
            (Default value = 0)

    Returns:
        x (np.ndarray): Reconstructed time signal
    """
    N = len(w)
    L = L + N
    M = X.shape[1]
    w_sum = np.zeros(L)
    x_win_sum = np.zeros(L)
    w_sum = np.zeros(L)
    for m in range(M):
        start_idx, end_idx = m * H, m * H + N + zero_padding
        if start_idx > L:
            break

        x_win = ifft(X[:, m])
        # Note: x_win = np.fft.ifft(X[:, m]) does not work in combination with @jit
        if end_idx > L:
            end_idx = L
            x_win = x_win[:end_idx-start_idx]
            cur_w = w[:end_idx-start_idx]
        else:
            cur_w = w

        # Avoid imaginary values (due to floating point arithmetic)
        x_win_real = np.real(x_win)
        x_win_sum[start_idx:end_idx] = x_win_sum[start_idx:end_idx] + x_win_real
        w_shifted = np.zeros(L)
        w_shifted[start_idx:start_idx + len(cur_w)] = cur_w
        w_sum = w_sum + w_shifted
    # Avoid division by zero
    w_sum[w_sum == 0] = np.finfo(np.float32).eps
    x_rec = x_win_sum / w_sum
    x_rec = x_rec[N // 2:-N // 2]
    return x_rec

def stft_convention_fmp(x, Fs, N, H, pad_mode='constant', center=True, mag=False, gamma=0):
    """Compute the discrete short-time Fourier transform (STFT)

    Notebook: C2/C2_STFT-FreqGridInterpol.ipynb

    Args:
        x (np.ndarray): Signal to be transformed
        Fs (scalar): Sampling rate
        N (int): Window size
        H (int): Hopsize
        pad_mode (str): Padding strategy is used in librosa (Default value = 'constant')
        center (bool): Centric view as used in librosa (Default value = True)
        mag (bool): Computes magnitude STFT if mag==True (Default value = False)
        gamma (float): Constant for logarithmic compression (only applied when mag==True) (Default value = 0)

    Returns:
        X (np.ndarray): Discrete (magnitude) short-time Fourier transform
    """
    X = librosa.stft(x, n_fft=N, hop_length=H, win_length=N,
                     window='hann', pad_mode=pad_mode, center=center)
    if mag:
        X = np.abs(X)**2
        if gamma > 0:
            X = np.log(1 + gamma * X)
    F_coef = librosa.fft_frequencies(sr=Fs, n_fft=N)
    T_coef = librosa.frames_to_time(np.arange(X.shape[1]), sr=Fs, hop_length=H)
    # T_coef = np.arange(X.shape[1]) * H/Fs
    # F_coef = np.arange(N//2+1) * Fs/N
    return X, T_coef, F_coef


from sklearn.datasets import make_regression
from matplotlib import pyplot as plt
import numpy as np
from sklearn.linear_model import LinearRegression

# Functions
def gram_schmidt(a):
    q = []
    for i in range(len(a)):
        #orthogonalization
        q_tilde = a[i]
        for j in range(len(q)):
            q_tilde = q_tilde - (q[j] @ a[i])*q[j]
        #Test for dependennce
        if np.sqrt(sum(q_tilde**2)) <= 1e-10:
            print('Vectors are linearly dependent.')
            print('GS algorithm terminates at iteration ', i+1)
            return q
        #Normalization
        else:
            q_tilde = q_tilde / np.sqrt(sum(q_tilde**2))
            q.append(q_tilde)
    print('Vectors are linearly independent.')
    return q

def QR_factorization(A):
    Q_transpose = np.array(gram_schmidt(A.T))
    R = Q_transpose @ A
    Q = Q_transpose.T
    return Q, R

def back_subst(R,b_tilde):
    n = R.shape[0]
    x = np.zeros(n)
    for i in reversed(range(n)):
        x[i] = b_tilde[i]
        for j in range(i+1,n):
            x[i] = x[i] - R[i,j]*x[j]
        x[i] = x[i]/R[i,i]
    return x

def solve_via_backsub(A,b):
    Q,R = QR_factorization(A)
    b_tilde = Q.T @ b
    x = back_subst(R,b_tilde)
    return x

#########

# # Least squares problem
# A = np.array([[2,0],[-1,1],[0,2]])
# b = np.array([1,0,-1])
# x_hat = np.array([1/3, -1/3])
# r_hat = A @ x_hat - b
# print(np.linalg.norm(r_hat))
# x = np.array([1/2, -1/2]) #other value of x
# r = A @ x - b
# print(np.linalg.norm(r))
# print(np.linalg.inv(A.T @ A) @ A.T @ b)
# print(np.linalg.pinv(A) @ b)
# print((A.T @ A) @ x_hat - A.T @ b) #Check that normal equations hold
# # Principio da ortogonalidade
# z = np.array([-1.1,2.3])
# print(A @ z).T @ r_hat)
# z = np.array([5.3, -1.2])
# print((A @ z).T @ r_hat)

# # Resolvendo problemas de quadrados mínimos
# A = np.random.normal(size = (100,20))
# b = np.random.normal(size = 100)
# x1 = solve_via_backsub(A,b)
# x2 = np.linalg.inv(A.T @ A) @ (A.T @ b)
# x3 = np.linalg.pinv(A) @ b
# print(np.linalg.norm(x1-x2))
# print(np.linalg.norm(x2-x3))
# print(np.linalg.norm(x3-x1))

# Exemplo página 234
n = 10 # número de lâmpadas
# posições (x,y) das lâmpadas e altura acima do chão
lamps = np.array([[4.1 ,20.4, 4],
    [14.1, 21.3, 3.5],
    [22.6, 17.1,6],
    [5.5 ,12.3, 4.0],
    [12.2, 9.7, 4.0],
    [15.3, 13.8, 6],
    [21.3, 10.5, 5.5],
    [3.9 ,3.3, 5.0],
    [13.1, 4.3, 5.0],
    [20.3,4.2, 4.5]])

N = 25 # grid size
m = N*N # Número de pixels
# construct m x 2 matrix with coordinates of pixel centers
pixels = np.hstack([np.outer(np.arange(0.5,N,1),np.ones(N)).reshape(m,1), np.outer(np.ones(N),np.arange(0.5,N,1)).reshape(m,1)])
# The m x n matrix A maps lamp powers to pixel intensities.
# A[i,j] is inversely proportional to the squared distance of
# lamp j to pixel i.
A = np.zeros((m,n))
for i in range(m):
    for j in range(n):
        A[i,j] = 1.0 / (np.linalg.norm(np.hstack([pixels[i,:], 0]) - lamps[j,:])**2)

A = (m / np.sum(A)) * A # scale elements of A
# Least squares solution
x = solve_via_backsub(A, np.ones(m))
rms_ls = (sum((A @ x - 1)**2)/m)**0.5
print(rms_ls)
import matplotlib.pyplot as plt
plt.ion()
plt.hist(A @ x, bins = 25)
plt.show()
plt.pause(10)
# Intensity if all lamp powers are one
rms_uniform = (sum((A @ np.ones(n) - 1)**2)/m)**0.5
print(rms_uniform)
plt.hist(A @ np.ones(n), bins = 25)
plt.show()
plt.pause(10)


#! bin/usr/env python

import numpy as np

def make_diff_matrix(h,N):
    """This function takes in a value h which is the distance between 2
    sample points and the amount of points it would sample
    N. It will then use these parameters to create an NxN matrix that is
    used to creat N central difference
    equations to get the derivative of a function."""
    D = np.zeros((N+1,N+1))
    D[0][0] = -1.0/h
    D[0][1] = 1.0/h
    D[N][N-1] = -1.0/h
    D[N][N] = 1.0/h
    for i in range(1,N):
        for j in range(1,N+1):
            if (j==i-1):
                D[i][j] = -1.0/(2.0*h)
            elif (j==i+1):
                D[i][j] = 1/(2.0*h)
    return D

def vec_central_first_diff(f,a,b,N):
    """Takes a function and an array of equidistant inputs (all discrete)
    and returns the derivative. Achieves this using a derivative matrix.
    Takes args: function f, N sample intervals, [a,b]"""
    x_i = np.linspace(a,b,N+1)
    f_x = f(x_i)
    l = vec_central_first_diff1(x_i, f_x)
    return l

def vec_central_first_diff1(x_i, f_x):
    """This function takes in a list of points x_i and the function that is
     produced by a function acting on x_i.
    Then it takes the difference matrix defined above and applies it to the
    function to get the derivative points of the
    function."""
    N = len(x_i) - 1
    h = x_i[1]-x_i[0]
    D = make_diff_matrix(h,N)
    d1 = np.dot(D,f_x)
    return x_i, f_x, d1

def vec_central_second_diff1(x_i,f_x):
    """This function does almost the same thing as vec_central_first_diff1,
    except this is to get the second derivative.
    So the only difference here is that we multiply the difference matrix
    to itself and then apply it to the function."""
    N = len(x_i) - 1
    h = x_i[1] - x_i[0]
    D = np.dot(make_diff_matrix(h,N), make_diff_matrix(h,N))
    d1 = np.dot(D,f_x)
    return x_i, f_x, d1

def vec_central_second_diff(f, a, b, N):
    """This is the application function for the second derivative function.
    Takes in a mathematical function an interval
    [a,b] and number of sample points N, and returns the second derivative."""
    x_i = np.linspace(a,b,N+1)
    f_x = f(x_i)
    l = vec_central_second_diff1(x_i, f_x)
    return l

def vec_central_diff(f,a,b,N,order=1):
    """Application function for vcd. Takes in function f, interval [a,b]
    and sample points N and the order of the derivatives
    (default to 1)."""
    x_i = np.linspace(a,b,N+1)
    f_x = f(x_i)
    l = vcd(x_i,f_x,order)
    return l

def vcd(x_i,f_x,order):
    """Takes in list of points x_i, function points f_x and a derivative
    order. Will iterate 'order' amount of times, multiplying the difference
    matrix to itself every time and then applying D^(order) to the function
    points f_x."""
    N = len(x_i) - 1
    h = x_i[1] - x_i[0]
    D = make_diff_matrix(h,N)
    if order >= 2:
        for i in range(1,order):
            D = np.dot(make_diff_matrix(h,N),D)
    d1 = np.dot(D, f_x)
    return x_i, f_x, d1

def make_trap_matrix(h,N):
    """Takes in step size h and the number of partition points used for integration N.
    Creates an NxN integration matrix which is a lower triangular matrix. With main
    diagonal and first column equal to h/2 and the rest of the lower triangle equal to h.
    """
    I = np.zeros((N+1,N+1))
    for i in range(1,N+1):
        for j in range(0,N+1):
            if j==i:
            	I[i][j] = float(h)/2.0
            elif j==i-1:
                I[i][j] = float(h)/2.0
            I[i][j] += I[i-1][j]
    return I

def vec_trapz(f,a,b,N):
    """Parameters: Function, interval [a,b] and sample points N. Returns
    integrated function"""
    x_i = np.linspace(a,b,N+1)
    f_x = f(x_i)
    l = vt(x_i,f_x)
    return l

def vt(x_i,f_x):
    """Takes in an array of points and a vectorized function. Creates the trapezoidal matrix and acts it onto the
    vectorized function. Returns the input parameters and the integral"""
    N = len(x_i) - 1
    h = x_i[1] - x_i[0]
    I = make_trap_matrix(h,N)
    integral = np.dot(I,f_x)
    return x_i, f_x, integral

def test_vectrapz():
    """ testing the vect_trapz function with a moderately
    high error tolerance"""
    #On the interval from 0 to 10 the integral of x^3 = 2500
    xCubedInt = 2500
    a = vec_trapz(lambda x: x**3, 0,10,100)
    l = a[2]
    if(abs(l[100] - 2500) < 1):
        return True
    else:
        return False


"""Example implementation of the steepest descent algorithm using a hard coded obbjective function

    F(X1, X2) = 5X1**2 + X2**2 + 4X1X2 - 14X1 - 6X2 + 20

- At each step the value of X will be updated using
    X_k+1 = X_k + alpha * del_F(Xk)

- Initial guess of the minimiser is a point X_0 = [0, 0]
- The value of alpha will be calculated using an exact line search

    alpha = || gk || ** 2 / gk_T * A * gk

    where gk is the gradient vector at point Xk and A is the hessian matrix
"""
import math
import logging
import numpy as np

logging.basicConfig(level=logging.DEBUG)
log = logging.getLogger(__name__)

class GradientDescent:
    def __init__(self, max_iterations=100, hessian_matrix=None, linear_terms=None):
        self.Hessian: np.ndarray = hessian_matrix or np.array([[10, 4], [4, 2]])
        self.LinearCoefficients = linear_terms or np.array([-14, -6])
        self.Minimiser = np.array([0, 0])
        self.CurrentIteration = 0
        self.MaxIterations = max_iterations
        self.Epsilon = math.pow(10, -6)  # Stop if magnitude of gradient is less than this value

    @property
    def __current_gradient_value(self):
        gradient = self.del_f(self.Minimiser)
        return np.sqrt(gradient.dot(gradient))

    @staticmethod
    def f(xk, decimal_places=3):
        x1 = xk[0]
        x2 = xk[1]
        fn_value = 5*math.pow(x1, 2) + math.pow(x2, 2) + 4*x1*x2 - 14*x1 - 6*x2 + 20
        return round(fn_value, decimal_places)

    def del_f(self, xk: np.ndarray) -> np.ndarray:
        gradient_vector = np.matmul(self.Hessian, xk)
        if isinstance(self.LinearCoefficients, np.ndarray):
            gradient_vector = np.add(gradient_vector, self.LinearCoefficients)
        return gradient_vector

    def exact_step_length(self, xk: np.ndarray, decimal_places=3) -> np.float64:
        """Get the step lengh by minimising f(x + ad_k) wrt to a

        alpha(or lambda) = || gk || ** 2 / gk_T * A * gk
        """
        gk = self.del_f(xk)
        numerator = gk.dot(gk)
        denominator = gk.dot(self.Hessian).dot(gk)
        step_len = numerator/denominator
        return round(step_len, decimal_places)

    def _find_next_candidate(self):
        """Get next potential minimum using X_k+1 = X_k + alpha * del_F(Xk)"""
        xk = self.Minimiser
        dk = -1 * self.del_f(xk)
        step_length = self.exact_step_length(xk)
        x_k_plus1 = xk + (step_length * dk)
        log.debug(f"X**{self.CurrentIteration} = {xk} - {step_length} * {dk}")
        return x_k_plus1

    def execute(self):
        log.info('Gradient descent iteration started')
        # self.__preview_config()
        for k in range(self.MaxIterations):
            log.debug(f"-----------Iteration {k}--------------")
            self.CurrentIteration = k
            self.Minimiser = self._find_next_candidate()
            log.debug(f"------Minimiser = {self.Minimiser}. Gradient = {self.__current_gradient_value}--------\n")
            if self.__current_gradient_value <= self.Epsilon:
                log.warning(f"Iteration stopped at k={k}. Stopping condition reached!")
                break
        log.info(f"------Minimiser = {self.Minimiser}. Gradient = {self.__current_gradient_value}--------")
        log.info('Gradient descent completed successfully')
        return self.Minimiser, self.f(self.Minimiser), round(self.__current_gradient_value, 3)


import numpy as np
import matplotlib.pyplot as plt
import cvxpy as cp

S = np.loadtxt("S.txt")
pMean = np.loadtxt("pbar.txt")
n = pMean.shape[0]

#%%
# What is the risk of the uniform portfolio?

xU = np.ones(n)/n
riskU = xU.T @ S @ xU

print(f"uniform portfolio risk: {riskU**0.5}")

#%%
# What is the risk of an optimal portfolio with no (additional) constraints?
print("min risk with expected return equals uniform portfolio")

x = cp.Variable(n)
constraints = \
[
    cp.sum(x) == 1,
    pMean.T @ x == pMean.T @ xU
]

prob = cp.Problem(cp.Minimize(cp.quad_form(x, S)), constraints)
prob.solve()
assert prob.status == cp.OPTIMAL
print(f"optimal risk {prob.value**0.5}")

#%%
# What is the risk of a long-only portfolio 𝑥⪰0?
print("x >= 0")
prob = cp.Problem(cp.Minimize(cp.quad_form(x, S)), constraints + [x >= 0])
prob.solve()
assert prob.status == cp.OPTIMAL
print(f"optimal risk {prob.value**0.5}")

#%%
# What is the risk of a portfolio with a limit on total short position:  1𝑇(𝑥−)≤0.5 , where  (𝑥−)𝑖=max{−𝑥𝑖,0} ?
print("limit on short")
prob = cp.Problem(cp.Minimize(cp.quad_form(x, S)), constraints + [cp.sum(cp.neg(x)) <= 0.5])
prob.solve()
assert prob.status == cp.OPTIMAL
print(f"optimal risk {prob.value**0.5}")

Convex Optimization Problems/Simple Portfolio Optimization/solve.py


# -*- coding: utf-8 -*-
"""
Created on Fri Aug 23 19:07:15 2019

@author: marcus.bonifacio
"""

import numpy as np

print('1) Faça um script para estimar o valor de π utilizando o método de Monte Carlo.')

N = 10000000 #define quantidade de pontos
x = np.random.rand(N) #gera números aleatórios de 0 a 1
y = np.random.rand(N)
deltax = (0.5-x)**2 #calcula a diferença das coordenadas do ponto para o centro do círculo
deltay = (0.5-y)**2
d = np.sqrt(deltax + deltay) #calcula a distancia do ponto para o centro do círculo

circ = d[d <= 0.5] #seleciona os pontos dentro do círculo

razao = len(circ)/N #razão dos pontos dentro do círculo e todos os pontos
pi = 4*razao #aproxima pi
print('Valor aproximado de PI pelo método Monte Carlo =',pi)


import cvxpy as cp
import numpy as np
from numpy.linalg import pinv, inv, norm
from scipy.linalg import eig

def alternative_minimization(X, alpha, beta, num_iters, dual=False):

    n = X.shape[0]

    """
    helper functions
    """

    def dual_helper(Y):
        m = n * (n + 1) // 2

        # position matrix
        Q = np.zeros((n, n))
        count = 0
        for d in np.arange(0, n):
            for i in np.arange(d, n):
                j = i - d
                Q[i, j] = count
                count += 1
        for d in np.arange(1, n):
            for i in np.arange(0, n - d):
                j = i + d
                Q[i, j] = count
                count += 1

        # C = vec(Y @ Y.T)
        YYt = Y @ Y.T
        C = np.zeros(((n**2), 1))
        for i in np.arange(0, n):
            for j in np.arange(0, n):
                loc = int(Q[i, j])
                C[loc] = YYt[i, j]

        # Duplication matrix M
        nsqr = n ** 2
        M1 = np.eye(m)
        M2 = np.hstack((np.zeros((nsqr - m, n)), np.eye(m - n)))
        M = np.vstack((M1, M2))

        # Equality constraints matrix A
        A1 = np.hstack((np.ones((1, n)), np.zeros((1, m - n))))
        A2 = np.hstack((np.eye(n), np.zeros((n, m - n))))
        for i in np.arange(0, n):
            row = i
            for j in np.arange(0, i):
                col = int(Q[i, j])
                A2[row, col] = 1
            for j in np.arange(i + 1, n):
                col = int(Q[j, i])
                A2[row, col] = 1
        A = np.vstack((A1, A2))

        # vector b
        b = np.vstack((n * np.ones((1, 1)), np.zeros((n, 1))))

        # Inequality constraints matrix G
        G = np.hstack((np.zeros((m - n, n)), np.eye(m - n)))

        # vector h
        h = np.zeros((m - n, 1))

        P = 2 * beta * M.T @ M
        q = (alpha * M.T @ C).squeeze()

        return Q, P, q, G, h, A, b

    def make_L(x, Q):
        L = np.zeros((n, n))
        for i in np.arange(0, n):
            for j in np.arange(0, i + 1):
                loc = int(Q[i, j])
                L[i, j] = x[loc]

        L = L + L.T - np.diag(np.diag(L))
        return L

    """
    QP optimizations for L and Y
    """

    def optimize_L_primal(Y):
        L = cp.Variable((n,n), symmetric=True)

        obj = cp.Minimize(alpha * cp.trace(Y.T @ L @ Y) + beta * (cp.norm(L, p='fro')**2))

        constraints = [cp.trace(L) == n]
        constraints += [L >> 0]
        constraints += [L @ np.ones((n)) == np.zeros((n))]
        for i in range(n):
            for j in range(i+1,n):
                constraints += [L[i,j] <= 0]

        prob = cp.Problem(obj, constraints)

        p_star = prob.solve()
        print(f"primal value is {p_star}")
        L_opt = L.value
        return L_opt

    def optimize_L_dual(Y):
        Q, P, q, G, h, A, b = dual_helper(Y)

        lam = cp.Variable((G.shape[0]))
        nu = cp.Variable((A.shape[0]))

        Pinv = pinv(P)
        r = q + G.T @ lam + A.T @ nu

        obj_dual = cp.Maximize((-0.5) * cp.quad_form(r, Pinv) - h.T @ lam - b.T @ nu)

        constraints_dual = [lam >= 0]

        prob_dual = cp.Problem(obj_dual, constraints_dual)

        p_dual_star = prob_dual.solve()
        print(f"dual value is {p_dual_star}")
        lam_opt, nu_opt = lam.value, nu.value

        x = - Pinv @ (r.value)
        L_opt = make_L(x, Q)
        return L_opt

    def optimize_Y(L):
        Y_opt = inv(np.eye(n) + alpha * L) @ X

        return Y_opt

    """
    alternate minimization algorithm
    """

    Y_iter = X
    obj_old_val = np.inf
    for idx_iter in np.arange(num_iters):

        print(f"iteration #{idx_iter}")

        if dual is True:
            L_iter = optimize_L_dual(Y_iter)
        else:
            L_iter = optimize_L_primal(Y_iter)

        Y_iter = optimize_Y(L_iter)

        obj_val = norm(X - Y_iter, 'fro')**2 + alpha * np.trace(Y_iter.T @ L_iter @ Y_iter) + beta * (norm(L_iter, 'fro')**2);

        print(f"iteration-{idx_iter}: obj value = {obj_val}")

        if np.abs(obj_old_val - obj_val) < 1e-4:
            break
        else:
            obj_old_val = obj_val

    return L_iter, Y_iter

import numpy,from numpy,from scipy,import cvxpy


import numpy as np
from numpy import linalg as LA
from numpy import diag
from numpy import zeros
A=np.array([[-5,-5,-1,0,0,0,500,500,100],[0,0,0,-5,-5,-1,500,500,100],
           [-150,-5,-1,0,0,0,30000,1000,200],[0,0,0,-150,-5,-1,12000,400,80],
           [-150,-150,-1,0,0,0,33000,33000,220],[0,0,0,-150,-150,-1,12000,12000,80],
           [-5,-150,-1,0,0,0,500,15000,100],[0,0,0,-5,-150,-1,1000,30000,200]])
print("The A matrix:\n{}".format(A))
A_T=A.transpose()
print("The transpose of the A matrix is: \n{}".format(A_T))
A1=A.dot(A_T)
A2=A_T.dot(A)
#To find the eigen vectors and eigenvalues of the A^T*A and A*A^T.
w1, v1 = LA.eig(A1)
w2, v2 = LA.eig(A2)
#Taking the absolute values of the eigen values to keep them postive
w= abs(w2)
print("The adjusted eigen values : \n{}".format(w))
U=v1
print("The Eigen Vector U is: \n{}".format(U))
#transpose of the V eigen vector matrix
V_T=v2.transpose()
print("Eigen vectors 2\n{}".format(v2))
print("The transpose of Eigen Vector V is: \n{}".format(V_T))
#Sigma Matrix matrix can be formed using the square root of eigenvalues of A^T*A
#Sigma matrix is a diagonal matrix with the squareroot of the eigen values
sig=np.sqrt(w)
print("The eigenvectors considered for the Sigma matrix are: \n{}".format(sig))
#To form the Sigma diagonal matrix
Sigma = zeros((A.shape[0], A.shape[1]))
print(Sigma)
for i in range(len(Sigma)):
    Sigma[i][i]= sig[i]
print("The Sigma Matrix(Diagonal Matrix) is : \n{}".format(Sigma))
#SVD of A= U*Sigma*V'
SVD=U.dot(Sigma.dot(V_T))
print("The SVD of Matrix A is:")
print(SVD)
#To find the Homography matrix
H=v2[:,-1]
H1=np.reshape(H,(3,3))
print("The Homography matrix H is:")
print(H1)


# using higher order derivatives to find extrema values

from sympy import Symbol, solve, Derivative

x = Symbol( 'x' )
f = x**5 - 30*x**3 + 50*x

d1 = Derivative( f, x ).doit()

critical_points = solve( d1 )

critical_points

A = critical_points[ 2 ]
B = critical_points[ 0 ]
C = critical_points[ 1 ]
D = critical_points[ 3 ]

d2 = Derivative( f, x, 2 ).doit()

d2.subs( { x: B } ).evalf()
d2.subs( { x: C } ).evalf()
d2.subs( { x: A } ).evalf()
d2.subs( { x: D } ).evalf()

x_min = -5
x_max = 5

f.subs( { x: C } ).evalf()
f.subs( { x: A } ).evalf()

f.subs( { x: x_min } ).evalf()
f.subs( { x: x_max } ).evalf()


# define: Sigmoid function version 1
# date: 2021.12.23.
# Resource: 밑바닥부터 시작하는 인공지능(사이토고키, 2017)

def sigmoid(x):
	return 1 / (1+ np.exp(-x)) # exp is function to caculate the exponential of all elements in the input array.

if __name__ == '__main__':
	import numpy as np
	import matplotlib.pylab as plt

	x = np.arange(-5.0, 5.0, 0.1)
	y = sigmoid(x)

	plt.plot(x, y)
	plt.ylim(-0.1, 1.1)
	plt.show()


import matplotlib.pyplot as plt
import numpy as np

def numeric_better(x, h):
    return (np.cos(x + h) - np.cos(x - h)) / (2 * h)

def numeric_worse(x, h):
    return (np.cos(x + h) - np.cos(x)) / h

# line 1 points
x = 1
distances = [10 ** (-i) for i in range(1, 16)]
real_y = -np.sin(x)
numeric_better_y = [abs(numeric_better(x, distance) - real_y) for distance in distances]
numeric_worse_y = [abs(numeric_worse(x, distance) - real_y) for distance in distances]

plt.plot(distances, numeric_worse_y, label="real derivative")
plt.plot(distances, numeric_better_y, label="numeric derivative")

# plt.loglog(x, real_y, label="real derivative")

# # plotting the line 1 points
# line 2 points
# plotting the line 2 points
plt.xlabel("h")
# Set the y axis label of the current axis.
plt.ylabel("f`(x)")
# Set a title of the current axes.
plt.title("Two or more lines on same plot with suitable legends ")
# show a legend on the plot
plt.legend()
# Display a figure.
plt.show()


# Summary Statistics and More:
# mean, median, stdevs, min, max, quantiles, sum, product, etc.

# Summing Values in NumPy Arrays
import numpy as np
L = np.random.random(100)
big_array = np.random.rand(1000000)

sum(L)                  # Python   sum. 1D  arrs only
np.sum(L)               # NP ufunc sum. 2D+ arrs
sum(L) == np.sum(L)

np.sum(big_array)
big_array.sum()
# Generally, we prefer calling from object instance syntax.

# Minimum and Maximum
big_array.min(), big_array.max()

# Multi-Dimensional Aggregates
# For 2D arrays, we often want aggregates organized along rows or columns.
# We can toggle 'axis' argument of methods to perform this.
A = np.random.random((3, 4))
A
A.sum()

# NOTE: Axis controls dimension to collapse, and returns the complement.
A.min(axis=0)   # collapse row --> return min of cols
A.max(axis=1)   # collapse col --> return max of rows

# Most NumPy agg funcs have a NaN-safe equivalents (ignore NaN values).
A.sum; np.nansum(A)

# Partial List of Aggregation Functions:
np.sum
np.prod
np.mean
np.std
np.var          # compute variance (max-min)
np.min
np.max
np.argmin       # return index of min value
np.argmax       # return index of max value
np.median
np.percentile
np.any          # evaluate whether any elements are true
np.all          # evaluate whether all elements are true

# Example: Get Average Height of US Presidents
# !head -4 data/president_heights.csv
import pandas as pd
data = pd.read_csv('data/president_heights.csv')
heights = np.array(data['height(cm)'])

heights.mean()
heights.std()
heights.min()
heights.max()

np.percentile(heights, 25)
np.median(heights)
np.percentile(heights, 75)

## we can also visualize with Matplotlib
# %matplotlib inline
import matplotlib.pyplot as plt
import seaborn; seaborn.set()
plt.hist(heights)
plt.title('Height Distribution of US Presidents')
plt.xlabel('height (cm)')
plt.ylabel('number')


import numpy as np

### Functions for you to fill in ###

def polynomial_kernel(X, Y, c, p):
    """
        Compute the polynomial kernel between two matrices X and Y::
            K(x, y) = (<x, y> + c)^p
        for each pair of rows x in X and y in Y.

        Args:
            X - (n, d) NumPy array (n datapoints each with d features)
            Y - (m, d) NumPy array (m datapoints each with d features)
            c - a coefficient to trade off high-order and low-order terms (scalar)
            p - the degree of the polynomial kernel

        Returns:
            kernel_matrix - (n, m) Numpy array containing the kernel matrix
    """
    return np.power((X @ Y.T + c), p)

def rbf_kernel(X, Y, gamma):
    """
        Compute the Gaussian RBF kernel between two matrices X and Y::
            K(x, y) = exp(-gamma ||x-y||^2)
        for each pair of rows x in X and y in Y.

        Args:
            X - (n, d) NumPy array (n datapoints each with d features)
            Y - (m, d) NumPy array (m datapoints each with d features)
            gamma - the gamma parameter of gaussian function (scalar)

        Returns:
            kernel_matrix - (n, m) Numpy array containing the kernel matrix
    """
    n = X.shape[0]
    m = Y.shape[0]
    K = np.zeros((n,m))
    for i in range(n):
        for j in range(m):
            K[i,j] = np.exp(-gamma * (np.linalg.norm(X[i] - Y[j]) ** 2))
    return K
    # np.vectorize
    # n = X.shape[0]
    # m = Y.shape[0]
    # a = np.mat((X@X.T).diagonal())
    # a = np.tile(a.T, (1,m))

    # b = np.mat((Y@Y.T).diagonal())
    # b = np.tile(b, (n,1))

    # k = a + b
    # k -= 2*(X @ np.transpose(Y))

    # return np.exp(-gamma * k)

courses/MITx/MITx 6.86x Machine Learning with Python-From Linear Models to Deep Learning/project2/mnist/part1/kernel.py


import scipy.io
import os
import numpy as np
import numpy.matlib
import matplotlib.pyplot as plt

from toolbox.pcaPmtk import *

data_path = os.path.join("..", "bigData", "olivettiFaces", "olivettiFaces.mat")
mat = scipy.io.loadmat(data_path)

X = mat['faces'].T
y = np.matlib.repmat(np.arange(1, 41), 10, 1).flatten(order='F')

n, d = X.shape
h, w = 64, 64
name = 'faces'

perm = np.random.permutation(n)

f, ax_arr = plt.subplots(2, 2)
for i in range(2):
    for j in range(2):
        ax_arr[i, j].imshow(X[perm[i*2+j], :].reshape(h, w).T, cmap="gray")
        ax_arr[i, j].axis('off')
f.suptitle("pcaImages-%s-images" % name)
plt.show()

XC = X - np.mean(X, axis=0)
print('Performing PCA.... stay tuned')

V, Z, evals, _, mu = pcaPmtk(X)

f, ax_arr = plt.subplots(2, 2)
for i in range(2):
    for j in range(2):
        if i == 0 and j == 0:
            ax_arr[i, j].imshow(mu.reshape(h, w).T, cmap="gray")
            ax_arr[i, j].axis('off')
            ax_arr[i, j].set_title('mean')
        else:
            idx = i*2+j-1
            ax_arr[i, j].imshow(V[:, idx].reshape(h, w).T, cmap="gray")
            ax_arr[i, j].axis('off')
            ax_arr[i, j].set_title('principal basis %d' % idx)
f.suptitle("pcaImages-%s-basis" % name)
plt.show()

# Plot reconstructed image
ndx = 125
Ks = [5, 10, 20, np.linalg.matrix_rank(XC)]

f, ax_arr = plt.subplots(2, 2)
for ki in range(len(Ks)):
    k = Ks[ki]
    Xrecon = np.dot(Z[ndx, 0:k], V[:, 0:k].T) + mu
    i = ki//2
    j = ki%2
    ax_arr[i, j].imshow(Xrecon.reshape(h, w).T, cmap="gray")
    ax_arr[i, j].axis('off')
    ax_arr[i, j].set_title('Using %d bases' % k)
f.suptitle("pcaImages-%s-reconImages" % name)
plt.show()

Ks = []
Ks += range(1, 10)
Ks += range(10, 50, 5)
Ks += range(50, np.linalg.matrix_rank(XC), 25)
mse = []

for ki in range(len(Ks)):
    k = Ks[ki]
    Xrecon = np.dot(Z[:, 0:k], V[:, 0:k].T) + mu
    err = Xrecon - X
    mse.append(np.sqrt(np.mean(np.power(err, 2))))
plt.plot(Ks, mse, '-o')
plt.ylabel("mse")
plt.xlabel("K")
plt.title('reconstruction error')
plt.show()

plt.plot(np.cumsum(evals)/np.sum(evals), 'ko-')
plt.show()


"""Estimate pi using a parallel Monte Carlo simulation.

   Implements an MPI version of a parallel Monte Carlo simulation for estimating pi.
   MPI's default channel is used.

   Usage:
        python monte_carlo_pi.py [total_num_trials]

        Options:
               -total_num_trials    the total number of trials ( (x,y) points) for the simulation.
                                    By default it is set to 1 million since a (probabilistic) precision of 3 decimal digits
                                    requires 10^(2*3) trials (1 million trials).
                                    see https://stackoverflow.com/questions/18139934/can-a-monte-carlo-pi-calculation-be-used-for-a-world-record

"""

from mpi4py import MPI
import numpy as np
from sys import argv

def random_points_generator(trials_per_node=0):
    """Evaluate if a pair (x,y) drawn from an uniform distribution falls inside the unit circle.

    Given a random distribution between 0 and 1, it evaluates if a pair (x,y) of random points
    falls inside the unit circle. If that's the case, a counter is incremented.

    Args:
        trials_per_node (int): the number of trials/drawns for this node/process.

    Returns:
        int: a counter who keeps track of the overall number of points fallen inside the circle.

    """
    points_fallen_inside_circle = 0
    # draw a pair (x,y) from an uniform distribution
    # between 0 and 1 for each trial
    # NOTE: each child process has its own seed
    # since np.random.seed() hasn't been invoked
    # This is good for randomness but terrible for
    # reproducibility (which we don't consider in this project)
    for _ in range(trials_per_node):
        x = np.random.uniform(0,1)
        y = np.random.uniform(0,1)
        # if (x,y) falls inside the unit circle
        # increment the counter by 1
        if (x*x+y*y) <= 1.0:
            points_fallen_inside_circle += 1

    return points_fallen_inside_circle

if __name__ == "__main__":
    # Check if the number of trials has been supplied via
    # the CLI. If not, use 1 million as default value.
    total_num_trials = int(argv[1]) if len(argv) > 1 else 1e6
    # MPI_Init() is automatically invoked
    # when MPI module is imported
    # Rename default channel for convenience
    comm = MPI.COMM_WORLD
    # Get the number of nodes on comm channel
    num_nodes = comm.Get_size()
    # Get their rank
    rank = comm.Get_rank()
    # Print a start checkpoint if I am the master node
    if rank == 0:
        print("Start", flush=True)
    # Compute number of trials per node
    trials_per_node = np.ceil(total_num_trials / num_nodes).astype(int)
    # Sync all nodes here to start a 'more accurate' time measurement
    comm.Barrier()
    # Init send (i.e. elapsed_buff) and receive (i.e. longest_elapsed_buff) buffer for MAX reduction
    # as a one dimensional zero-filled arrays
    elapsed_buff         = np.zeros(1,dtype=np.float64)
    longest_elapsed_buff = np.zeros(1,dtype=np.float64)
    # Get start time
    start = MPI.Wtime()
    # Compute (x,y) random points which fall inside the circle
    points_in_circle_per_node = random_points_generator(trials_per_node)
    # Record the time when the function is done
    finish = MPI.Wtime()
    # Compute the time delta
    elapsed_buff[0] = (finish - start)

    #######################################################################################
    # If needed, print the wall-clock time for each processor
    # print("Processor {0} finished in {1:.6f}s.".format(rank,elapsed_buff[0]), flush=True)
    #######################################################################################

    # Do a MAX reduction to record the slowest processor
    comm.Reduce([elapsed_buff, MPI.DOUBLE],[longest_elapsed_buff, MPI.DOUBLE],op=MPI.MAX,root=0)
    # Get rid of the array, just need a number
    longest_elapsed = longest_elapsed_buff[0]
    # Init send (i.e. points_in_buff) and receive (i.e. tot_points_in_buff) buffer
    # for SUM reduction
    points_in_buff     = np.zeros(1,dtype=np.int32)
    tot_points_in_buff = np.zeros(1,dtype=np.int32)
    # Store points_in_circle_per_node in the buffer
    points_in_buff[0] = points_in_circle_per_node
    # Do a SUM reduction to retrieve the total number of points fallen
    # inside the circle
    comm.Reduce([points_in_buff, MPI.INT],[tot_points_in_buff, MPI.INT],op=MPI.SUM,root=0)
    # Get rid of the array, just need a number
    tot_points_in = tot_points_in_buff[0]
    # Master node estimates pi
    if rank == 0:
        # print the slowest processor
        print("Slowest processor wall-clock time: {:.6f}s".format(longest_elapsed), flush=True)
        print("End", flush=True)

        ############################################################################
        # If needed, print the estimated pi and the estimation error
        # estimate pi according to 4*(total points in the circle) / overall trials)
        # estimated_pi = 4*(tot_points_in / total_num_trials)
        # compute the error
        # error_pi = np.abs(np.pi - estimated_pi)
        # print("DEBUG: Pi is approximately: {:.6f}".format(estimated_pi))
        # print("DEBUG: Error is: {:.6f}".format(error_pi))
        ############################################################################

    # No need to MPI_Finalize() since mpi4py
    # implements an exit hook who does the job for us


import numpy as np                         # Linear algebra library
import matplotlib.pyplot as plt            # library for visualization
from sklearn.decomposition import PCA      # PCA library
import pandas as pd                        # Data frame library
import math                                # Library for math functions
import random                              # Library for pseudo random numbers

n = 1  # The amount of the correlation
x = np.random.uniform(1,2,1000) # Generate 1000 samples from a uniform random variable
y = x.copy() * n # Make y = n * x

# PCA works better if the data is centered
x = x - np.mean(x) # Center x. Remove its mean
y = y - np.mean(y) # Center y. Remove its mean

data = pd.DataFrame({'x': x, 'y': y}) # Create a data frame with x and y
plt.scatter(data.x, data.y) # Plot the original correlated data in blue

pca = PCA(n_components=2) # Instantiate a PCA. Choose to get 2 output variables

# Create the transformation model for this data. Internally, it gets the rotation
# matrix and the explained variance
pcaTr = pca.fit(data)

rotatedData = pcaTr.transform(data) # Transform the data base on the rotation matrix of pcaTr
# # Create a data frame with the new variables. We call these new variables PC1 and PC2
dataPCA = pd.DataFrame(data = rotatedData, columns = ['PC1', 'PC2'])

# Plot the transformed data in orange
plt.scatter(dataPCA.PC1, dataPCA.PC2)
plt.show()

print('Eigenvectors or principal component: First row must be in the direction of [1, n]')
print(pcaTr.components_)

print()
print('Eigenvalues or explained variance')
print(pcaTr.explained_variance_)

import matplotlib.lines as mlines
import matplotlib.transforms as mtransforms

random.seed(100)

std1 = 1     # The desired standard deviation of our first random variable
std2 = 0.333 # The desired standard deviation of our second random variable

x = np.random.normal(0, std1, 1000) # Get 1000 samples from x ~ N(0, std1)
y = np.random.normal(0, std2, 1000)  # Get 1000 samples from y ~ N(0, std2)
#y = y + np.random.normal(0,1,1000)*noiseLevel * np.sin(0.78)

# PCA works better if the data is centered
x = x - np.mean(x) # Center x
y = y - np.mean(y) # Center y

#Define a pair of dependent variables with a desired amount of covariance
n = 1 # Magnitude of covariance.
angle = np.arctan(1 / n) # Convert the covariance to and angle
print('angle: ',  angle * 180 / math.pi)

# Create a rotation matrix using the given angle
rotationMatrix = np.array([[np.cos(angle), np.sin(angle)],
                 [-np.sin(angle), np.cos(angle)]])

print('rotationMatrix')
print(rotationMatrix)

xy = np.concatenate(([x] , [y]), axis=0).T # Create a matrix with columns x and y

# Transform the data using the rotation matrix. It correlates the two variables
data = np.dot(xy, rotationMatrix) # Return a nD array

# Print the rotated data
plt.scatter(data[:,0], data[:,1])
plt.show()

plt.scatter(data[:,0], data[:,1]) # Print the original data in blue

# Apply PCA. In theory, the Eigenvector matrix must be the
# inverse of the original rotationMatrix.
pca = PCA(n_components=2)  # Instantiate a PCA. Choose to get 2 output variables

# Create the transformation model for this data. Internally it gets the rotation
# matrix and the explained variance
pcaTr = pca.fit(data)

# Create an array with the transformed data
dataPCA = pcaTr.transform(data)

print('Eigenvectors or principal component: First row must be in the direction of [1, n]')
print(pcaTr.components_)

print()
print('Eigenvalues or explained variance')
print(pcaTr.explained_variance_)

# Print the rotated data
plt.scatter(dataPCA[:,0], dataPCA[:,1])

# Plot the first component axe. Use the explained variance to scale the vector
plt.plot([0, rotationMatrix[0][0] * std1 * 3], [0, rotationMatrix[0][1] * std1 * 3], 'k-', color='red')
# Plot the second component axe. Use the explained variance to scale the vector
plt.plot([0, rotationMatrix[1][0] * std2 * 3], [0, rotationMatrix[1][1] * std2 * 3], 'k-', color='green')

plt.show()

nPoints = len(data)

# Plot the original data in blue
plt.scatter(data[:,0], data[:,1])

#Plot the projection along the first component in orange
plt.scatter(data[:,0], np.zeros(nPoints))

#Plot the projection along the second component in green
plt.scatter(np.zeros(nPoints), data[:,1])

plt.show()


import numpy as np

def longest_subseq_recur(x,y,i,j):
    if i==-1 or j==-1:
        return 0
    elif x[i]==y[j]:
        return longest_subseq_recur(x,y,i-1,j-1)+1
    else:
        return max(longest_subseq_recur(x,y,i-1,j),\
                longest_subseq_recur(x,y,i,j-1))

def lcs_len_topdwn_strt(x,y):
    m=len(x); n=len(y)
    c = np.ones((m+1,n+1))*np.inf
    return lcs_len_topdwn(x,y,m-1,n-1,c)

def lcs_len_topdwn(x,y,i,j,c):
    if c[i,j] < np.inf:
        return c[i,j]
    elif i==-1 or j==-1:
        c[i,j]=0
    elif x[i]==y[j]:
        c[i,j]=lcs_len_topdwn(x,y,i-1,j-1,c)+1
    else:
        c[i,j] = max(lcs_len_topdwn(x,y,i,j-1,c),\
            lcs_len_topdwn(x,y,i-1,j,c))
    return c[i,j]

def lcs_len_bottom_up(x,y):
    m=len(x)
    n=len(y)
    c=np.zeros((m+1,n+1))
    for i in range(1,m+1):
        for j in range(1,n+1):
            if x[i-1]==y[j-1]:
                c[i,j]=c[i-1,j-1]+1
            elif c[i-1,j]>=c[i,j-1]:
                c[i,j]=c[i-1,j]
            else:
                c[i,j]=c[i,j-1]
    return c[m,n]

def lcs_bottom_up(x,y):
    m=len(x)
    n=len(y)
    c=np.zeros((m+1,n+1)); b=np.copy(c)
    for i in range(1,m+1):
        for j in range(1,n+1):
            if x[i-1]==y[j-1]:
                c[i,j]=c[i-1,j-1]+1
                b[i,j]=-1#top-left
            elif c[i-1,j]>=c[i,j-1]:
                c[i,j]=c[i-1,j]
                b[i,j]=1#up
            else:
                c[i,j]=c[i,j-1]
                b[i,j]=0#left
    return c, b

def print_lcs(b,x,i,j):
    if i==0 or j==0:
        return
    if b[i,j]==-1:
        print_lcs(b,x,i-1,j-1)
        print(x[i-1],end='')
    elif b[i,j]==1:
        print_lcs(b,x,i-1,j)
    else:
        print_lcs(b,x,i,j-1)

if __name__=="__main__":
    x="abcbdab"
    y="bdcaba"
    res = longest_subseq_recur(x,y,len(x)-1,len(y)-1)
    print(res)
    c,b=lcs_bottom_up(x,y)
    print(b)
    print_lcs(b,x,len(x),len(y)); print()
    res = lcs_len_bottom_up(x,y)
    print(res)
    res = lcs_len_topdwn_strt(x,y)
    print(res)
    x="ACCGGTCGAGTGCGCGGAAGCCGGCCGAA"
    y="GTCGTTCGGAATGCCGTTGCTCTGTAAA"
    res = lcs_len_bottom_up(x,y)
    print(res)
    res = lcs_len_topdwn_strt(x,y)
    print(res)
    c,b=lcs_bottom_up(x,y)
    print_lcs(b,x,len(x),len(y)); print()
    #print(b)


"""
looking at wallis product for pi/2
"""

# import libraries
import numpy as np
import matplotlib.pyplot as plt

# close all plots
plt.close('all')

# generate numbers and sequences via an upper bound
ub = 100
num = np.append(np.repeat(np.arange(2,ub,2),2),ub)
den = np.insert(np.repeat(np.arange(3,ub+1,2),2),0,1)

# empty lists for two methods
wallis1, wallis2, wallis3 = [], [], []

### method 1 - calculate numerator, then denominator, then divide ###

# iterate through arrays and calculate
for i in range(len(num)):
    wallis1.append(np.prod(num[:i+1])/np.prod(den[:i+1]))

### method 2 - calculate num/denom of each term, then multiply ###

# iterate through arrays and calculate
for i in range(len(num)):
    wallis2.append(np.prod(num[:i+1]/den[:i+1]))

### method 3 - using production summation definition ###

# start with the first term of the series
wallis3_val = 4.0/3.0

# now start iterations from n=2
for n in range(2, ub):
    wallis3_val *= (4*n**2)/(4*n**2-1)
    wallis3.append(wallis3_val)

### create figures

fig,ax = plt.subplots()
plt.rcParams.update({'font.size': 18})
ax.plot(wallis1,label='Method 1')
ax.plot(wallis2,label='Method 2',color='g')
ax.plot(wallis3,label='Method 3',color='r')
ax.set_xlabel('# of Iterations $(n)$')
ax.set_ylabel('Value')
ax.axhline(np.pi/2.0,color='k',linestyle='--',label=r'$\pi/2$')
ax.axhline(0,color='k',linewidth=0.8)
ax.axvline(0,color='k',linewidth=0.8)
plt.legend()
ax.set_title('Wallis Product')
plt.tight_layout()