from manimlib.imports import *
import numpy as np
import bats
# slide scene
from manim_reveal import SlideScene
# manim interface with bats
from manimtda import *

import matplotlib

from scipy.spatial import ConvexHull

def gen_circle2(n, r=1.0):
    theta = np.linspace(0, 2*np.pi*(1 - 1./n), n)
    pts = np.array([r*np.cos(theta), r*np.sin(theta)],  dtype=np.float)
    return pts.T

def gen_point_cloud(n, scale=1.0):
	"""
	sample points on the unit square
	"""
	return np.random.uniform(low=-scale, high=scale, size=(n,2))

def get_dimension_filt(C):
	"""
	get dimension filtration data from bats complex
	"""
	spxs = []
	ts = []
	for d in range(C.maxdim() + 1):
		ts.extend([d for _ in range(C.ncells(d))])
		spxs.extend(C.get_simplices(d))

	return spxs, ts

def dimension_filtration_from_bats(C, pts, **kwargs):
	"""
	get manimtda filtration from bats filtration
	"""
	spxs, ts = get_dimension_filt(C)
	return SimplicialFiltration(pts, spxs, ts, **kwargs)

def get_convex_hull(s, pts, color=BLUE, **kwargs):
	"""
	get convex hull of pts[s] as a Manim polygon
	"""
	spts = np.array([pts[i] for i in s])
	hull_2d = ConvexHull(spts[:,:-1])
	P = Polygon(*[spts[i] for i in hull_2d.vertices],
				color=color,
				**kwargs
				)
	P.set_fill(color, opacity=0.2)
	#P.round_corners(0.05)
	#P.scale(1.3)
	return P

def get_zzbar(k, color=BLUE):
	ZZbar = VGroup(
		Dot(3*LEFT, color=color),
		Dot(ORIGIN, color=color),
		Dot(3*RIGHT, color=color),
		Line(3*LEFT, ORIGIN, color=color),
		Line(ORIGIN, 3*RIGHT, color=color)
	)
	label = TexMobject("H_{}".format(k), color=color)
	label.next_to(ZZbar, LEFT)
	return VGroup(ZZbar, label)

def get_convex_hull_barycenter(s, pts):
	"""
	get barycenter of convex hull
	"""
	spts = np.array([pts[i] for i in s])
	hull_2d = ConvexHull(spts[:,:-1])
	hull_pts = spts[hull_2d.vertices]
	return np.mean(hull_pts, axis=0)

class ZigzagNerve(SlideScene):
	CONFIG={
        "camera_config":{"background_color":"#F0F1EB"},
        "video_slides_dir":"../video_slides"
    }
	def construct(self):
		pts = gen_circle2(60, r=2.5)
		pts = pts + np.random.normal(scale=0.15, size=pts.shape)
		# pts = gen_point_cloud(100, scale=2.5)

		# put points into bats
		pbats = bats.DataSet(bats.Matrix(pts))
		# get landmarks
		lbats = bats.greedy_landmarks(pbats, 3, bats.Euclidean(), 0)
		lbats2 = bats.greedy_landmarks(pbats, 3, bats.Euclidean(), 5)
		lms = np.array(lbats.data()) # numpy array
		# get cover based on landmarks - assign each point to nearest landmarks
		coverU = bats.landmark_eps_cover(pbats, lbats, bats.LInfDist(), 1.0)
		coverV = bats.landmark_eps_cover(pbats, lbats2, bats.LInfDist(), 1.0)
		coverUV, fU, fV = bats.bivariate_cover(coverU, coverV)

		# Fbats = WeakAlphaFiltration(pts)
		Fbats = bats.RipsFiltration(pbats, bats.Euclidean(), 0.0, 2)
		RFC = bats.ReducedFilteredChainComplex(Fbats, bats.F2())

		pts = np.hstack((pts, np.zeros((pts.shape[0], 1))))
		lms = np.hstack((lms, np.zeros((lms.shape[0], 1))))
		# compute barycenters of hulls
		bcU = np.array([get_convex_hull_barycenter(s, pts) for s in coverU])
		bcV = np.array([get_convex_hull_barycenter(s, pts) for s in coverV])
		bcUV = np.array([get_convex_hull_barycenter(s, pts) for s in coverUV])

		# this is just for point cloud
		F = filtration_from_bats(Fbats, pts, color=BLACK)
		F.shift(3*RIGHT)

		#title = TextMobject(r"Comparing Covers", color=BLACK).shift(3.5*UP)
		anim = []
		# this just generates the point cloud
		Pcloud = F.step_to(0)
		anim.append(FadeIn(Pcloud))
		self.play(*anim)
		self.slide_break()

		# show the open sets in U
		line0 = TexMobject(r"\mathcal{U}", r"=\{U_0, U_1, U_2\}", color=BLACK).shift(3*LEFT + 1*UP)
		anim = [FadeIn(line0)]
		hulls = [get_convex_hull(coverU[i], pts).shift(3*RIGHT) for i in range(len(coverU))]
		hullsU = VGroup(*hulls)
		anim.extend([FadeInFrom(h, DOWN) for h in hulls])
		self.play(*anim)
		self.slide_break()

		# show the other cover V
		line1 = TexMobject(r"\mathcal{V}", r"=\{V_0, V_1, V_2\}", color=BLACK).shift(3*LEFT + 1*UP)
		line1.next_to(line0, DOWN)
		anim = [FadeIn(line1)]
		hulls = [get_convex_hull(coverV[i], pts, color=RED).shift(3*RIGHT) for i in range(len(coverV))]
		hullsV = VGroup(*hulls)
		anim.extend([FadeInFrom(h, DOWN) for h in hulls])
		self.play(*anim)
		self.slide_break()

		line2 = TexMobject(r"\mathcal{U} \times_X \mathcal{V}", color=BLACK).shift(3*LEFT + 1*UP)
		line2.next_to(line1, DOWN)
		anim = [FadeIn(line2)]
		hulls = [get_convex_hull(coverUV[i], pts, color=PURPLE).shift(3*RIGHT) for i in range(len(coverUV))]
		hullsUV = VGroup(*hulls)
		anim.extend([FadeOut(hullsU), FadeOut(hullsV), FadeInFrom(hullsUV, UP)])
		self.play(*anim)
		self.slide_break()

		anim = [FadeOut(Pcloud), FadeOut(hullsUV)]
		line0a = TexMobject(r"\mathcal{U}", color=BLACK).move_to((-3,2.5,0))
		line1a = TexMobject(r"\mathcal{V}", color=BLACK).move_to((3,2.5,0))
		# anim.append(ApplyMethod(line0.move_to, (-3,2.5,0)))
		# anim.append(ApplyMethod(line1.move_to, (3,2.5,0)))
		anim.append(Transform(line0, line0a))
		anim.append(Transform(line1, line1a))
		anim.append(ApplyMethod(line2.move_to, (0,2.5,0)))
		self.play(*anim)
		self.slide_break()

		# shift and scale hulls
		hullsU.shift(6*LEFT)
		hullsUV.shift(3*LEFT)

		hullsU.scale(0.3)
		hullsV.scale(0.3)
		hullsUV.scale(0.3)
		self.play(FadeIn(hullsU), FadeIn(hullsV), FadeIn(hullsUV))
		self.slide_break()

		# Nerve Functor

		# construct nerves
		NU = bats.Nerve(coverU, 2)
		NV = bats.Nerve(coverV, 2)
		NUV = bats.Nerve(coverUV, 2)

		# construct filtration on Nerve
		FU = dimension_filtration_from_bats(NU, bcU, color=BLUE, tri_opacity=0.4)
		FU.shift(3*LEFT)
		FU.scale(0.3)
		CU = FU.step_to(2)
		labelU = TexMobject(r"\mathcal{N}(\mathcal{U})", color=BLACK).move_to((-3,2.5,0))

		# construct filtration on Nerve
		FV = dimension_filtration_from_bats(NV, bcV, color=RED, tri_opacity=0.4)
		FV.shift(3*RIGHT)
		FV.scale(0.3)
		CV = FV.step_to(2)
		labelV = TexMobject(r"\mathcal{N}(\mathcal{V})", color=BLACK).move_to((3,2.5,0))

		# construct filtration on Nerve
		FUV = dimension_filtration_from_bats(NUV, bcUV, color=PURPLE, tri_opacity=0.4)
		FUV.scale(0.3)
		CUV = FUV.step_to(2)
		labelUV = TexMobject(r"\mathcal{N}(\mathcal{U}, \mathcal{V})", color=BLACK).move_to((0,2.5,0))

		anim = [Transform(hullsU, CU), Transform(hullsV, CV), Transform(hullsUV, CUV)]
		anim.extend([Transform(line0, labelU), Transform(line1, labelV), Transform(line2, labelUV)])
		self.play(*anim)
		self.slide_break()

		# arrows for maps
		arU = TexMobject(r"\xleftarrow{p_U}",color=BLACK).move_to((-1.5,2.5,0))
		arV = TexMobject(r"\xrightarrow{p_V}",color=BLACK).move_to((1.5,2.5,0))
		arUc = TexMobject(r"\xleftarrow{}",color=BLACK).move_to((-1.5,0,0))
		arVc = TexMobject(r"\xrightarrow{}",color=BLACK).move_to((1.5,0,0))
		self.play(*[FadeIn(ar) for ar in [arU, arV, arUc, arVc]])
		self.slide_break()

		# apply homology functor

		homtxt = TexMobject(
			r"H_k(\mathcal{N}(\mathcal{U}))",
			r"\xleftarrow{H_k(p_U)}",
			r"H_k(\mathcal{N}(\mathcal{U}, \mathcal{V}))",
			r"\xrightarrow{H_k(p_V)}",
			r"H_k(\mathcal{N}(\mathcal{V}))",
			color=BLACK
		).shift(2.5*UP)
		homtxt1 = TexMobject(r"H_k(\mathcal{N}(\mathcal{U}))", color=BLACK)
		homtxt2 = TexMobject(r"\xleftarrow{H_k(p_U)}", color=BLACK)
		homtxt3 = TexMobject(r"H_k(\mathcal{N}(\mathcal{U}, \mathcal{V}))", color=BLACK).shift(2.5*UP)
		homtxt4 = TexMobject(r"\xrightarrow{H_k(p_V)}", color=BLACK)
		homtxt5 = TexMobject(r"H_k(\mathcal{N}(\mathcal{V}))", color=BLACK)
		homtxt2.next_to(homtxt3, LEFT)
		homtxt1.next_to(homtxt2, LEFT)
		homtxt4.next_to(homtxt3, RIGHT)
		homtxt5.next_to(homtxt4, RIGHT)
		# arUh = TexMobject(r"\xleftarrow{H_k(p_U)}",color=BLACK).move_to((-1.5,2.5,0))
		# arVh = TexMobject(r"\xrightarrow{H_k(p_V)}",color=BLACK).move_to((1.5,2.5,0))
		# labelUh = TexMobject(r"H_k(\mathcal{N}(\mathcal{U}))", color=BLACK).move_to((-3,2.5,0))
		# labelVh = TexMobject(r"H_k(\mathcal{N}(\mathcal{V}))", color=BLACK).move_to((3,2.5,0))
		# labelUVh = TexMobject(r"H_k(\mathcal{N}(\mathcal{U}, \mathcal{V}))", color=BLACK).move_to((0,2.5,0))

		# produce zigzag barcode
		ZZ0 = get_zzbar(0, color=BLUE)
		ZZ0.shift(2*DOWN)
		ZZ1 = get_zzbar(1, color=RED)
		ZZ1.shift(3*DOWN)

		dgrp =VGroup(line0, arU, line2, arV, line1)
		anim = [ Transform( dgrp, homtxt), ShowCreation(ZZ0), ShowCreation(ZZ1)]
		# anim = [ FadeOut( dgrp), FadeIn(homtxt), ShowCreation(ZZ0), ShowCreation(ZZ1)]
		#anim.extend([FadeOut(l) for l in [arU, arV, label0, label1, label2]])

		self.play(*anim)
		self.wait()


'''
   based on: http://nipunbatra.github.io/2013/05/simulating-a-discrete-hidden-markov-model/
   Unfair cassino problem: there may be two die, one fair and other biased.
   The biased die is much more likely to produce a 6 than others (p=0.5).
   The observer is only able to observe the values of die being thrown without
   having a knowledge whether a fair or biased die were used:

   observed states: 1 to 6 on the die faces
   hidden states: fair or biased die
   prior: probability that the first thworn is made from a fair or biased die
   transition matrix A: matrix enconding the prob of the 4 possible transition
       between fair and biased die
   emission matrix B: matrix enconding the prob of an obsevation given the hidden
       state
'''

import numpy as np
import matplotlib.pyplot as plt
import matplotlib
import random

# plot setup
matplotlib.rcParams.update({'font.size': 11})

'''
    setting the components of HMM
    prior: a fair die is twice as likely as biased die
    A :
    	1. Fair -> Fair: .95
    	2. Fair -> Biased: 1-.95 =.05
    	3. Biased -> Biased: .90
    	4. Biased -> Biased: 1-.90 =.10

    B:
    	Pr(6)= 0.5
        Pr(1)= Pr(2)= Pr(3)= Pr(4)= Pr(5)= 0.1
'''

prior = np.array([2.0/3,1.0/3])
A = np.array([[.95,.05],[.1,.9]])
B = np.array([[1.0/6 for i in range(6)],[.1,.1,.1,.1,.1,.5]])

# return next state to the weighted probability array
def next_state(weights):
    choice = random.random() * sum(weights)
    for i, w in enumerate(weights):
        choice -= w
        if choice < 0:
            return i

def create_hidden_sequence(prior, A, length):
    out=[None]*length
    out[0]=next_state(prior)
    for i in range(1,length):
        out[i]=next_state(A[out[i-1]])
    return out

def create_observation_sequence(hidden_sequence, B):
    length=len(hidden_sequence)
    out=[None]*length
    for i in range(length):
        out[i]=next_state(B[hidden_sequence[i]])
    return out

# group all contiguous values in tuple
def group(L):
    first = last = L[0]
    for n in L[1:]:
        if n - 1 == last:
            last = n
        else:
            yield first, last
            first = last = n
    yield first, last

# create tuples of the form (start, number_of_continuous values)
def create_tuple(x):
    return [(a,b-a+1) for (a,b) in x]

if __name__ == '__main__':
	count = 0
	num_calls = 500

	for i in range(num_calls):
	    count += next_state(prior)

	print("Expected number of Fair states:", num_calls-count)
	print("Expected number of Biased states:", count)

    # Create the sequences
	hidden = np.array(create_hidden_sequence(prior, A, num_calls))
	observed = np.array(create_observation_sequence(hidden, B))
	print('Observed: ', observed)
	print('Hidden: ', hidden)

	# Tuples of form index value, number of continuous values corresponding to Fair State
	indices_hidden_fair = np.where(hidden==0)[0]
	tuples_contiguous_values_fair = list(group(indices_hidden_fair))
	tuples_start_break_fair = create_tuple(tuples_contiguous_values_fair)

	# Tuples of form index value, number of continuous values corresponding to Biased State
	indices_hidden_biased = np.where(hidden==1)[0]
	tuples_contiguous_values_biased = list(group(indices_hidden_biased))
	tuples_start_break_biased = create_tuple(tuples_contiguous_values_biased)

	# Tuples for observations
	observation_tuples=[]
	for i in range(6):
	    observation_tuples.append(create_tuple(group(list(np.where(observed==i)[0]))))

    # Make plots
	plt.subplot(2,1,1)
	plt.xlim((0, num_calls));
	plt.title('Observations');
	for i in range(6):
	    plt.broken_barh(observation_tuples[i],(i+0.5,1),facecolor='k');
	plt.subplot(2,1,2);
	plt.xlim((0, num_calls));
	plt.title('Hidden States Blue:Fair, Red: Biased');
	plt.broken_barh(tuples_start_break_fair,(0,1),facecolor='b');
	plt.broken_barh(tuples_start_break_biased,(0,1),facecolor='r');
	plt.savefig('hmm.png')


#!/usr/bin/python
# -*- coding: utf-8 -*-
#
# Author: Vadym Tkachuk
# Date: 02.03.2019
# License: MIT
# Purpose: This is a module for testing if a number is a prime.
# 		   The secondary objective is to test speed of python
#		   and of algos of finding primes.

""" This module contains several definitions of
	function desiding if a number is a prime
	i.e it has no divisors except itself.
    Pre-condition: n is a nonnegative integer
    post-condition: return True if n is prime and False otherwise."""

import math

def is_prime1(x):
    # If number(x) is evenly divided by following dividers then number(x) is not prime
    divider = [2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43, 47, 53, 59, 61, 67, 71]
    # An empty list to be able to check whether number(x) is evenly divided:
    remainder = []
    # exceptions for numbers 1,2,3,5,7:
    if x < 2:
        return False
    if x in divider:
        return True
    else:
        for nums in divider:
            remainder.append(x % nums)
        if 0 in remainder:
            return False
        else:
            return True

def is_prime2(n):
    if n == 2: return True
    for i in range(3, int(n**0.5)+1, 2): # n**0.5 is a square root
        if n%i==0:
            return False
    return True

def is_prime3(n):
    if n == 2 or n == 3: return True
    if n < 2 or n%2 == 0: return False
    if n < 9: return True
    if n%3 == 0: return False
    r = int(n**0.5)
    f = 5
    while f <= r:
        if n%f == 0: return False
        if n%(f+2) == 0: return False
        f += 6
    return True

# This is probably among the fastest as it both replaces square root
# with square, and leaves out not only the multiplies of 2
# from the loop but the multiplies of 3 as well.
# %timeit isPrime(800)

def is_prime4(n):
    # Corner Cases
    if (n<= 1): return False
    elif (n<= 3): return True
    elif (n%2 == 0 or n%3 == 0): return False

    i = 5
    while i*i<=n:
        if (n%i==0 or n%(i+2)==0): return False
        i += 6

    return True;

def is_prime5(n):
    if n < 2:
        return False;
    if n % 2 == 0:
        return n == 2  # return False
    i = 3
    while i*i <= n:
        if n % i == 0:
            return False
        i += 2
    return True

# Using lambda definition:
import numpy as np

def is_prime6(n):
    is_prime=lambda x: all(x % i != 0 for i in range(int(x**0.5)+1)[2:])
    # To find all primes you might use:
    # filter(isPrime, range(4000)[2:])[:5]
    return is_prime(n)

def is_prime7(Number):
    return 2 in [Number,2**Number%Number]

def is_prime8(x):
    if x < 2:
        return False
    for n in range(2, (x) - 1):
        if x % n == 0:
            return False
    return True

from math import factorial
def is_prime9(x):
    """
    This method will be slower than the the recursive and enumerative methods here,
    but uses Wilson's theorem, and is just a single line.
    """
    return factorial(x - 1)  % x == x - 1

def is_prime10(x):
    if x < 4:
        return True
    if all([(x > 2), (x % 2 == 0)]):
        return False
    else:
        return np.array([*map(lambda y: ((x % y) == 0).sum(), np.arange(1, x + 1))]).sum() == 2

# Method with sieve of Atkin
def atkin(limit):
    if limit > 2:
        yield 2
    if limit > 3:
        yield 3

    import math
    is_prime = [False] * (limit + 1)

    for x in range(1,int(math.sqrt(limit))+1):
        for y in range(1,int(math.sqrt(limit))+1):
            n = 4*x**2 + y**2

            if n<=limit and (n%12==1 or n%12==5):
                # print "1st if"
                is_prime[n] = not is_prime[n]
            n = 3*x**2+y**2
            if n<= limit and n%12==7:
                # print "Second if"
                is_prime[n] = not is_prime[n]
            n = 3*x**2 - y**2
            if x>y and n<=limit and n%12==11:
                # print "third if"
                is_prime[n] = not is_prime[n]

    for n in range(5,int(math.sqrt(limit))):
        if is_prime[n]:
            for k in range(n**2,limit+1,n**2):
                is_prime[k] = False

    for n in range(5,limit):
        if is_prime[n]: yield n

def is_prime11(n):
    r = list(atkin(n+1))
    if not r: return False
    return True if n == r[-1] else False

import re
# Explanation is here: http://montreal.pm.org/tech/neil_kandalgaonkar.shtml
#
#   ^1?$   # matches beginning, optional 1, ending.
#          # thus matches the empty string and "1".
#          # this matches the cases where N was 0 and 1
#          # and since it matches, will not flag those as prime.
# |   # or...
#   ^                # match beginning of string
#     (              # begin first stored group
#      1             # match a one
#       1+?          # then match one or more ones, minimally.
#     )              # end storing first group
#     \1+            # match the first group, repeated one or more times.
#   $                # match end of string.
def is_prime12(n):
    return not re.match(r'^1?$|^(11+?)\1+$',n*'1')

import unittest_primes
import benchmark_primes
import runpy

if __name__ == '__main__':
    file = "./unittest_primes.py"
    exec(compile(open(file).read(), file, 'exec'))
    # The following lines do not run. Do not know why.
    file = "./benchmark_primes.py"
    exec(compile(open(file).read(), file, 'exec'))


"""
https://en.wikipedia.org/wiki/Rayleigh_quotient
"""
import numpy as np

def is_hermitian(matrix:np.matrix) -> bool:
    """
    Checks if a matrix is Hermitian.

    >>> import numpy as np
    >>> A = np.matrix([
    ... [2,    2+1j, 4],
    ... [2-1j,  3,  1j],
    ... [4,    -1j,  1]])
    >>> is_hermitian(A)
    True
    >>> A = np.matrix([
    ... [2,    2+1j, 4+1j],
    ... [2-1j,  3,  1j],
    ... [4,    -1j,  1]])
    >>> is_hermitian(A)
    False
    """
    return np.array_equal(matrix, matrix.H)

def rayleigh_quotient(A:np.matrix, v:np.matrix) -> float:
    """
    Returns the Rayleigh quotient of a Hermitian matrix A and
    vector v.
    >>> import numpy as np
    >>> A = np.matrix([
    ... [1,  2, 4],
    ... [2,  3,  -1],
    ... [4, -1,  1]
    ... ])
    >>> v = np.matrix([
    ... [1],
    ... [2],
    ... [3]
    ... ])
    >>> rayleigh_quotient(A, v)
    matrix([[3.]])
    """
    v_star = v.H
    return (v_star * A * v) / (v_star * v)

def tests() -> None:
    A = np.matrix([[2, 2 + 1j, 4], [2 - 1j, 3, 1j], [4, -1j, 1]])
    v = np.matrix([[1], [2], [3]])
    assert is_hermitian(A), f"{A} is not hermitian."
    print(rayleigh_quotient(A, v))

    A = np.matrix([[1, 2, 4], [2, 3, -1], [4, -1, 1]])
    assert is_hermitian(A), f"{A} is not hermitian."
    assert rayleigh_quotient(A, v) == float(3)

if __name__ == "__main__":
    import doctest

    doctest.testmod()
    tests()


import numpy as np

def l2_regularization(W, reg_strength):
    '''
    Computes L2 regularization loss on weights and its gradient

    Arguments:
      W, np array - weights
      reg_strength - float value

    Returns:
      loss, single value - l2 regularization loss
      gradient, np.array same shape as W - gradient of weight by l2 loss
    '''
    loss = reg_strength * np.sum(np.power(W, 2))
    grad = 2 * reg_strength * W

    return loss, grad

def softmax(predictions):
    '''
    Computes probabilities from scores

    Arguments:
      predictions, np array, shape is either (N) or (batch_size, N) -
        classifier output

    Returns:
      probs, np array of the same shape as predictions -
        probability for every class, 0..1
    '''
    # Your final implementation shouldn't have any loops
    # From every batch substact its maximum
    predictions = (predictions.T - np.max(predictions, axis=-1)).T

    exp_preds = np.exp(predictions)
    return (exp_preds.T / np.sum(exp_preds, axis=-1)).T

import math

def cross_entropy_loss(probs, target_index):
    '''
    Computes cross-entropy loss

    Arguments:
      probs, np array, shape is either (N) or (batch_size, N) -
        probabilities for every class
      target_index: np array of int, shape is (1) or (batch_size) -
        index of the true class for given sample(s)

    Returns:
      loss: single value
    '''
    # TODO implement cross-entropy
    # Your final implementation shouldn't have any loops
    if type(target_index) != np.ndarray:
        target_index = np.array([target_index])
    if len(probs.shape) == 1:
        probs = probs[np.newaxis, :]

    result = np.mean(-np.log(
        probs[np.arange(target_index.shape[0]), target_index]
    ))
    if math.isinf(result):
        raise Exception('loss is inf')
    return result

def softmax_with_cross_entropy(predictions, target_index):
    '''
    Computes softmax and cross-entropy loss for model predictions,
    including the gradient

    Arguments:
      predictions, np array, shape is either (N) or (batch_size, N) -
        classifier output
      target_index: np array of int, shape is (1) or (batch_size) -
        index of the true class for given sample(s)

    Returns:
      loss, single value - cross-entropy loss
      dprediction, np array same shape as predictions
        - gradient of predictions by loss value
    '''
    # Your final implementation shouldn't have any loops
    softmaxf = softmax(predictions)
    loss = cross_entropy_loss(softmaxf, target_index)

    if type(target_index) != np.ndarray:
        target_index = np.array([target_index])

    do_flatten = False

    if len(softmaxf.shape) == 1:
        do_flatten = True
        softmaxf = softmaxf[np.newaxis, :]

    num_samples = softmaxf.shape[0]

    a = tuple([np.arange(target_index.shape[0]), target_index])
    dprediction = softmaxf.copy()

    np.subtract.at(dprediction, a, 1)
    dprediction /= num_samples
    if do_flatten:
        dprediction = dprediction.flatten()

    return loss, dprediction

class Param:
    """
    Trainable parameter of the model
    Captures both parameter value and the gradient
    """

    def __init__(self, value):
        self.value = value
        self.grad = np.zeros_like(value)

    def __str__(self):
        return f'value={np.mean(self.value)}; grad={np.mean(self.grad)}'

    def __repr__(self):
        return f'value={np.mean(self.value)}; grad={np.mean(self.grad)}'

class ReLULayer:
    def __init__(self):
        self._X = None

    def forward(self, X):
        self._X = X
        return np.maximum(X, 0)

    def backward(self, d_out):
        """
        Backward pass

        Arguments:
        d_out, np array (batch_size, num_features) - gradient
           of loss function with respect to output

        Returns:
        d_result: np array (batch_size, num_features) - gradient
          with respect to input
        """
        return d_out * (self._X > 0).astype(int)

    def params(self):
        # ReLU Doesn't have any parameters
        return {}

class FullyConnectedLayer:
    def __init__(self, n_input, n_output):
        self.W = Param(0.001 * np.random.randn(n_input, n_output))
        self.B = Param(0.001 * np.random.randn(1, n_output))
        self.X = None

    def forward(self, X):
        # TODO: Implement forward pass
        # Your final implementation shouldn't have any loops
        self.X = X
        return X.dot(self.W.value) + self.B.value

    def backward(self, d_out):
        """
        Backward pass
        Computes gradient with respect to input and
        accumulates gradients within self.W and self.B

        Arguments:
        d_out, np array (batch_size, n_output) - gradient
           of loss function with respect to output

        Returns:
        d_result: np array (batch_size, n_input) - gradient
          with respect to input
        """
        # TODO: Implement backward pass
        # Compute both gradient with respect to input
        # and gradients with respect to W and B
        # Add gradients of W and B to their `grad` attribute

        # It should be pretty similar to linear classifier from
        # the previous assignment

        # WHY WHY WHY!?!?!?!?!?!?!
        grad = d_out.dot(self.W.value.T)
        self.W.grad = self.X.T.dot(d_out)
        self.B.grad = np.ones((1, self.X.shape[0])).dot(d_out)

        return grad

    def params(self):
        return {'W': self.W, 'B': self.B}


import numpy as np
import logging
from tqdm import tqdm

class Perceptron:
  def __init__(self , eta , epochs):
    self.weights = np.random.randn(3) * 1e-4 # SMALL WEIGHT INITILIZATION
    logging.info(f"initial weights before trainings: \n{self.weights}")
    self.eta = eta  # Learning Rate
    self.epochs = epochs

  def activationFunction(self , inputs , weights):
    z = np.dot(inputs,weights) #z = W *X
    return np.where(z>0 , 1,0) # Condition, if True , Else

  def fit(self , X , y):
    self.X = X
    self.y = y

    X_with_bias = np.c_[self.X, -np.ones((len(self.X),1))] # Concatination
    logging.info(f"X with bias: \n{X_with_bias}")

    for epoch in tqdm (range(self.epochs), total=self.epochs,desc = "training the model"):
     logging.info("--"*10)
     logging.info(f"for epoch: {epoch}")
     logging.info("--"*10)

     y_hat = self.activationFunction(X_with_bias,self.weights) # forward propagation
     logging.info(f"predicted value after forward pass: \n{y_hat}")
     self.error = self.y - y_hat
     logging.info(f"error:\n{self.error}")
     self.weights = self.weights + self.eta * np.dot(X_with_bias.T,self.error) # backward propagation
     logging.info(f"updated weights after epoch:\n{epoch}/{self.epochs} : {self.weights}")
     logging.info("#####"*10)

  def predict(self , X):
    X_with_bias = np.c_[X, -np.ones((len(X), 1))]
    return self.activationFunction(X_with_bias , self.weights)

  def total_loss(self):
    total_loss = np.sum(self.error)
    logging.info(f"total_loss:{total_loss}")
    return total_loss


import numpy as np
import matplotlib.pyplot as probabilities

#some data
X=np.array([[1, 1],[0, 0],[1, 0], [0, 1]])

#Output
y=np.array([[1],[0],[0], [0]])

# initialize parameters randomly
h = 5 # size of hidden layer
W = 0.01 * np.random.randn(X.shape[1], h)
b = np.zeros((1,h))
W2 = 0.01 * np.random.randn(h,2)
b2 = np.zeros((1,2))

# some hyperparameters
step_size = 1e-0
reg = 1e-3 # regularization strength

# gradient descent loop
num_examples = X.shape[0]
for i in range(500):

  # evaluate class scores, [N x K]
  hidden_layer = np.maximum(0, np.dot(X, W) + b) # note, ReLU activation
  scores = np.dot(hidden_layer, W2) + b2

  # compute the class probabilities
  exp_scores = np.exp(scores)
  probs = exp_scores / np.sum(exp_scores, axis=1, keepdims=True) # [N x K]

  # compute the loss: average cross-entropy loss and regularization
  corect_logprobs = -np.log(probs[range(num_examples),y])
  data_loss = np.sum(corect_logprobs)/num_examples

  reg_loss = 0.5*reg*np.sum(W*W) + 0.5*reg*np.sum(W2*W2)

  loss = data_loss + reg_loss

  if i % 1000 == 0:
    print("iteration %d: loss %f" % (i, loss) )

  # compute the gradient on scores
  dscores = probs
  dscores[range(num_examples),y] -= 1
  dscores /= num_examples

  # backpropate the gradient to the parameters
  # first backprop into parameters W2 and b2
  dW2 = np.dot(hidden_layer.T, dscores)
  db2 = np.sum(dscores, axis=0, keepdims=True)
  # next backprop into hidden layer
  dhidden = np.dot(dscores, W2.T)
  # backprop the ReLU non-linearity
  dhidden[hidden_layer <= 0] = 0
  # finally into W,b
  dW = np.dot(X.T, dhidden)
  db = np.sum(dhidden, axis=0, keepdims=True)

  # add regularization gradient contribution
  dW2 += reg * W2
  dW += reg * W

  # perform a parameter update
  W += -step_size * dW
  b += -step_size * db
  W2 += -step_size * dW2
  b2 += -step_size * db2

# evaluate training set accuracy
hidden_layer = np.maximum(0, np.dot(X, W) + b)
scores = np.dot(hidden_layer, W2) + b2
predicted_class = np.argmax(scores, axis=1)
print('training accuracy: %.2f' % (np.mean(predicted_class == y)) )


#Linear regression for G_pml + explicit formula for polynomial approximation + truncation

import numpy as np
from scipy.misc import comb
from scipy.special import hermitenorm
from tqdm import tqdm
from joblib import Parallel, delayed
from itertools import product
from sklearn.preprocessing import PolynomialFeatures
from matplotlib import pyplot as plt
import math

def H(k, x):
    if k==0:
        return 1.0
    if k ==1:
        return x
    if k==2:
        return (x**2 - 1)/np.sqrt(2)
    h = hermitenorm(k)(x) /  np.sqrt(math.factorial(k))
    return h

def Hermite_val(k_vec,x_vec):
    P = 1.0
    d = x_vec.shape[0]
    for i in range(d):
        P = P * H(k_vec[i],x_vec[i])
    return P

def generate_X_poly(train_traj, r, max_deg):
    N_train = train_traj.shape[0]
    N = train_traj.shape[1]
    d = train_traj.shape[2]
    X = np.empty((N_train * (N-r) ,d+1 + int(d*(d+1)/2)))
    all_points = train_traj[:, :N-r].reshape(-1,d)
    poly = PolynomialFeatures(max_deg)
    X = poly.fit_transform(all_points)
    return X, poly.powers_

def generate_y_mean(train_traj, r, f_target = "sum"):
    N_train = train_traj.shape[0]
    N = train_traj.shape[1]
    y = np.zeros(N_train * (N-r))
    if f_target == "sum":
        y = train_traj[:, r:].sum(axis = 2).reshape(-1)
    elif f_target == "sum_squared":
        y = np.square(train_traj[:, r:]).sum(axis = 2).reshape(-1)
    elif f_target == "sum_4th":
        y = (train_traj[:, r:]**4).sum(axis = 2).reshape(-1)
    elif f_target == "exp_sum":
        y = np.exp(train_traj[:, r:].sum(axis =2)).reshape(-1)
    else:
        raise Exception('unrecognized target function')
    return y

def G_pml_fit_mean(train_traj, f_target="sum", max_deg = 1):
    N_train = train_traj.shape[0]
    N = train_traj.shape[1]
    d = train_traj.shape[2]
    Betas = np.zeros((N, d+ 1 + (max_deg-1) * int(d*(d+1)/2)))
    for r in tqdm(range(N)):
        # Linear Regression
        if 0 < max_deg < 3:
            X, degrees = generate_X_poly(train_traj, r, max_deg)
        else:
            raise Exception('max_deg should be 1 or 2')
        y = generate_y_mean(train_traj, r, f_target)

        beta = np.linalg.inv(X.T @ X) @ X.T @ y
        Betas[r] = beta
    return Betas, degrees

def G_pml_predict(x, pml, Betas, max_deg = 1):
    poly = PolynomialFeatures(max_deg)
    x_pol = poly.fit_transform(x.reshape(1,-1))
    beta = Betas[pml]
    return (x_pol @ beta)

def a_plk(traj, traj_grad, p, l, k_vec, step, degrees, Betas):
    '''

    Explicit formula for polynomial regression

    '''
    dim = traj.shape[1]
    S = 0
    x_hat = traj[l-1] - step*traj_grad[l-1]/2
    Small_s = np.zeros(dim)
    for ind,deg in enumerate(degrees):
        Small_s[:] = 0
        for d, i in enumerate(deg):
            for t in range (i+1):
                for s in range (int(t/2 +1)):
                    if (k_vec[d] == t - 2*s):
                        Small_s[d] = Small_s[d] + comb(N=i, k = t, exact = True) * x_hat[0]**(i-t) * math.factorial(t)*1/math.factorial(s)*1 / np.sqrt(math.factorial(t-2*s)) *np.sqrt(step)**t /2**s
                    else:
                        pass
        S = S + Betas[p-l,ind] * Small_s.prod()
    return S

def M_bias(k_vec, traj, traj_grad, traj_noise, step, degrees, Betas, n_tilde):
    N = traj.shape[0]
    S = 0
    for p in range(N):
        for l in range (p+1):
            if (p-l<n_tilde):   #TRUNCATED
                s = a_plk(traj, traj_grad, p, l, k_vec, step, degrees, Betas)* Hermite_val(k_vec,traj_noise[l])
                S = S + s
    return S/N

def estimator_bias(k_vec, test_traj, test_traj_grad, test_traj_noise, step, degrees, Betas, n_tilde, n_jobs = -1):
    N_test = test_traj.shape[0]
    M_results = Parallel(n_jobs = n_jobs)(delayed(M_bias)(k_vec, test_traj[i], test_traj_grad[i], test_traj_noise[i], step, degrees, Betas,n_tilde)for i in range(N_test))
    return np.array(M_results).reshape(-1)


#!/usr/bin/env python
# coding: utf-8

# In[1]:

import numpy as np
np.set_printoptions(edgeitems=30, linewidth=100000,
    formatter=dict(float=lambda x: "%.10g" % x))

# In[2]:

#Romberg from book p.219
def Romberg(f, a, b, n):
    R = np.array([[0.0]*n]*n, float)
    h = b - a
    R[0][0] = (h/2) * (f(a) + f(b))
    for i in range(1,n):
        h = h/2
        sum1 = 0
        for k in range(1,2**(i)+1, 2):
            sum1 = sum1+f(a+(k*h))
        R[i][0] = (1/2)*R[i-1][0] + sum1*h
        for j in range(1,i+1):
            R[i][j] = R[i][j-1] + (R[i][j-1] - R[i-1][j-1])/(4**(j) - 1)
    return R

# In[3]:

#test integral
def f(x):
    return 4/(1+x**2)
a = 0
b = 1
n = 5
print(Romberg(f,a,b,n))

# In[ ]:


import numpy as np

def monotonicity(x, rounding_precision = 3):
    """Calculates monotonicity metric of a value of[0-1] for a given array.\nFor an array of length n, monotonicity is calculated as follows:\nmonotonicity=abs[(num. positive gradients)/(n-1)-(num. negative gradients)/(n-1)]."""

    n = x.shape[0]
    grad = np.gradient(x)
    pos_grad = np.sum(grad>0)
    neg_grad = np.sum(grad<0)
    monotonicity = np.abs( pos_grad/(n-1) - neg_grad/(n-1) )
    return np.round(monotonicity, rounding_precision)

def trendability(x):
    """Calculate trendability metric of a value of [0-1] for a given array of features."""

    t_array = np.array([])

    for feature in x:
        n = feature.shape[0]
        grad_first = np.gradient(feature)
        grad_second = np.gradient(grad_first)
        pos_grad_first = np.sum(grad_first>0)
        pos_grad_second = np.sum(grad_second>0)
        t = (pos_grad_first)/(n-1) + (pos_grad_second)/(n-2)
        t_array = np.append(t_array, t)

    trendability = 1-np.std(t_array)
    return trendability


import numpy as np
import matplotlib.pyplot as plt
from pandas import datetime
from statsmodels.graphics.tsaplots import plot_predict
from statsmodels.tsa.arima_process import arma_generate_sample
from statsmodels.tsa.arima.model import ARIMA
from sklearn.metrics import mean_squared_error
from math import sqrt
####################################################################
# AR(2)
N=1000
a1=0.4
a2=0.35
x=np.arange(2)
e_t=np.random.normal(0,1)
y=np.append(x,a1*x[1]+a2*x[0]+e_t)

for i in range(N):
   e_t=np.random.normal(0,1)
   y=np.append(y,a1*y[-1]+a2*y[-2]+e_t)

y.mean()
y.std()
#plt.plot(y);plt.show()

# Predict
l=20
ar2 = ARIMA(y, order=(2, 0, 0))
ar2_res = ar2.fit()
print(ar2_res.summary())

fig, ax = plt.subplots(figsize=(10, 8))
fig = plot_predict(ar2_res, start=0,end=N+l, ax=ax)
plt.plot(y)
legend = ax.legend(loc="upper left")
plt.show()

####################################################################

# AR(1)
N=1000
a1=0.5
sigma=0.1
x=np.arange(1)
y=np.append(x,a1*x+np.random.normal(0,sigma))

for i in range(N):
   y=np.append(y,a1*y[-1]+np.random.normal(0,sigma))

y.mean()
y.std()
plt.plot(y);plt.show()

l=10
ar1 = ARIMA(y, order=(1, 0, 0))
ar1_res = ar1.fit()
print(ar1_res.summary())

fig, ax = plt.subplots(figsize=(10, 8))
fig = plot_predict(ar1_res, start=0,end=N+l, ax=ax)
plt.plot(y)
legend = ax.legend(loc="upper left")
plt.show()

####################################################################
# MA(1)
N =1000
b1 =0.25
e_t0 = np.random.normal(0,1)
e_t1 = np.random.normal(0,1)
y = np.append(e_t0,b1*e_t0+e_t1)  # y1 = b1*e_t0+e_t1

for i in range(N):
   e_t0 = e_t1
   e_t1 = np.random.normal(0,1)
   y = np.append(y,b1*e_t0+e_t1)

y.mean()
y.std()
plt.plot(y);plt.show()

# Predict
ma1 = ARIMA(y, order=(0, 0, 1))
ma1_res = ma1.fit()
print(ma1_res.summary())

fig, ax = plt.subplots(figsize=(10, 8))
fig = plot_predict(ma1_res, start=1,end=1010, ax=ax)
plt.plot(y)
legend = ax.legend(loc="upper left")
plt.show()

y_ma1=y
####################################################################
#ARMA

N=10000
a1=0.4
a2=0.3
b1 =-0.3

x=np.arange(2)
e_t=np.random.normal(0,1)
y=np.append(x,a1*x[1]+a2*x[0]+e_t)

for i in range(N):
   e_0 = e_t
   e_t=np.random.normal(0,1)
   y=np.append(y,a1*y[-1]+a2*y[-2]+ b1*e_0 + e_t)

y.mean()
y.std()
#plt.plot(y);plt.show()

# Predict
y_arma = ARIMA(y, order=(2, 0, 1))
y_arma_res = y_arma.fit()
print(y_arma_res.summary())

fig, ax = plt.subplots(figsize=(10, 8))
fig = plot_predict(y_arma_res, start=1,end=N+10, ax=ax)
plt.plot(y)
legend = ax.legend(loc="upper left")
plt.show()

####################################################################
#ARIMA

N=10000
a1=0.35
a2=0.25
b1 =-0.35

x=np.arange(2)
e_t=np.random.normal(0,1)
y=np.append(x,a1*x[1]+a2*x[0]+e_t)

for i in range(N):
   e_0 = e_t
   e_t=np.random.normal(0,1)
   y=np.append(y,a1*y[-1]+a2*y[-2]+ b1*e_0 + e_t)

t = np.arange(int(N*0.01), step=0.01)
y=y[2:N+2]

y = y+t
y.mean()
y.std()
#plt.plot(y);plt.show()

# Predict
y_arma = ARIMA(y, order=(2, 1, 1), trend='t')
y_arma_res = y_arma.fit()
print(y_arma_res.summary())

fig, ax = plt.subplots(figsize=(10, 8))
fig = plot_predict(y_arma_res, start=1,end=N+1000, ax=ax)
plt.plot(y)
legend = ax.legend(loc="upper left")
plt.show()


#Program: Chi-Square fitting to a straight line
#NOTICE: fit y=a+bx with error bar of y. Give out
#a and b with error bars, no output goodness-of-fit.
#REFERENCE: NUMERICAL RECIPES IN FORTRAIN II P656

import numpy as np
import pylab

def fit(x, y, dy):
    data_len = len(x)

    inv_y2 = 1.0/(dy*dy)
    S      = inv_y2.sum()
    Sx     = (x*inv_y2).sum()
    Sy     = (y*inv_y2).sum()
    Sxx    = (x*x*inv_y2).sum()
    Sxy    = (x*y*inv_y2).sum()
    delta  = S*Sxx-Sx*Sx

    a=(Sxx*Sy-Sx*Sxy)/delta
    b=(S*Sxy-Sx*Sy)/delta
    da=np.sqrt(Sxx/delta)
    db=np.sqrt(S/delta)

    print "parameters:", a   , b
    print "param err :", da  , db
    print "relative  :", da/a, db/b

    fit_y=a+x*b
    chi=np.sum( (fit_y-y)**2/dy**2 )
    print "Chi-sqaure with data lenth:", chi, data_len

    return np.array([a,b]),np.array([da,db])

def fit_plot(x, y, dy, popt, xmin="none", xmax="none", save=0):
    pylab.figure()
    pylab.xlabel('$x$')
    pylab.ylabel('$y$')
    pylab.errorbar(x, y, yerr=dy,fmt='bo',label='data')

    if xmin=="none":
       xmin=np.amin(x)
    if xmax=="none":
       xmax=np.amax(x)
    fitx =np.linspace(xmin,xmax,512,endpoint=True)
    fity =popt[0]+fitx*popt[1]
    pylab.plot(fitx, fity,'r-',label='fit curve')

    pylab.legend(loc='upper right',frameon=False)

    if save==0:
       pylab.show()
    else:
       pylab.savefig('fit.pdf',bbox_inches='tight')


#! /usr/bin/env python

import numpy as np

def numpy_solution(Xdata, ydata):

    # half examples for training
    Xtrain, ytrain = Xdata[::2], ydata[::2]
    # half examples for validation
    Xvalid, yvalid = Xdata[1::2], ydata[1::2]

    # closed-form solution to minimize cost (loss) function
    # min(cost) = inverse(XT*X) * (XT*y)
    XT_X = np.dot(Xtrain.T, Xtrain)
    XT_y = np.dot(Xtrain.T, ytrain)
    weights = np.dot(np.linalg.inv(XT_X), XT_y)

    print("First few weights = ", weights[:5].T)

    train_predictions = np.dot(Xtrain, weights)
    train_cost = np.mean(np.square(train_predictions-ytrain))
    print("Cost on training data = ", train_cost)

    valid_predictions = np.dot(Xvalid, weights)
    valid_cost = np.mean(np.square(valid_predictions-yvalid))
    print("Cost on validation data = ", valid_cost)

    return weights

def tensorflow_solution_closed_form(Xdata, ydata):

    import tensorflow as tf

    # half examples for training
    Xt, yt = Xdata[::2], ydata[::2]
    # half examples for validation
    Xv, yv = Xdata[1::2], ydata[1::2]

    X = tf.placeholder(dtype=tf.float64, name="X")
    y = tf.placeholder(dtype=tf.float64, name="y")
    weights = tf.Variable(tf.zeros(shape=(Xdata.shape[1],1), dtype=tf.float64), name="weights")

    XT = tf.transpose(X, name="XT")
    XT_X = tf.matmul(XT, X, name="XT_X")
    XT_y = tf.matmul(XT, y, name="XT_y")

    compute_weights = weights.assign(
        tf.matmul(tf.matrix_inverse(XT_X), XT_y, name="weights")
    )

    predictions = tf.matmul(X, weights, name="predictions")
    cost = tf.reduce_mean(tf.square(predictions-y))

    init = tf.global_variables_initializer()

    with tf.Session() as sesh:
        weights_value = sesh.run(compute_weights, feed_dict={X: Xt, y: yt})
        print("First few weights = ", weights_value[:5].T)

        train_cost = sesh.run(cost, feed_dict={X: Xt, y: yt})
        print("Cost on training data = ", train_cost)

        valid_cost = sesh.run(cost, feed_dict={X: Xv, y: yv})
        print("Cost on validation data = ", valid_cost)

    return weights

def tensorflow_solution_gradient_descent(Xdata, ydata, diff="automatic"):

    import tensorflow as tf

    # half examples for training
    Xt, yt = Xdata[::2], ydata[::2]
    # half examples for validation
    Xv, yv = Xdata[1::2], ydata[1::2]

    # setup for gradient descent
    niters = 10000000

    X = tf.placeholder(dtype=tf.float64, name="X")
    y = tf.placeholder(dtype=tf.float64, name="y")
    weights = tf.Variable(tf.zeros(shape=(Xdata.shape[1],1), dtype=tf.float64), name="weights")

    # penalty = tf.constant(0.001, dtype=tf.float64, name="penalty")
    learn_rate = tf.constant(0.0055, dtype=tf.float64, name="learn_rate")
    examples = tf.constant(Xdata.shape[0], dtype=tf.float64, name="examples")

    predictions = tf.matmul(X, weights, name="predictions")
    residuals = predictions-y
    cost = tf.reduce_mean(tf.square(residuals))
    # Tried some regularized linear regression here (need more info)
    # cost = tf.add(
    #     tf.reduce_mean(tf.square(residuals)),
    #     tf.multiply(tf.reduce_sum(tf.square(weights)), penalty)
    # )

    if diff == "automatic":
        print("Using automatic differentiation for gradient descent")
        cost_gradient = tf.gradients(cost, [weights])[0] # automatic differentiation
    else:
        print("Using closed-form gradient for gradient descent")
        XT = tf.transpose(X, name="XT")
        cost_gradient = 2./Xt.shape[0] * tf.matmul(XT, residuals)

    update_weights = weights.assign(weights - learn_rate * cost_gradient)
    # update_weights = weights.assign(weights*(1-penalty*learn_rate/examples) - (learn_rate/examples) * cost_gradient)

    init = tf.global_variables_initializer()
    graph = tf.get_default_graph()
    graph.finalize()

    with tf.Session() as sesh:
        sesh.run(init)

        for i in range(niters):
            weights_value = sesh.run(update_weights, feed_dict={X: Xt, y: yt})
            if i % 1000 == 0:
                train_cost = sesh.run(cost, feed_dict={X: Xt, y: yt})
                valid_cost = sesh.run(cost, feed_dict={X: Xv, y: yv})
                print(
                    "Iteration %s :: Train Cost %s :: Valid Cost %s" % (
                        i, train_cost, valid_cost
                    )
                )

        print("First few weights = ", weights_value[:5].T)
        print("Cost on training data = ", train_cost)
        print("Cost on validation data = ", valid_cost)

    return weights

def main():
    import os, sys
    data_dir = os.path.dirname(os.path.abspath(__file__)) + "/data/"

    features_path = data_dir + "Xtrain.txt"
    labels_path = data_dir + "Ytrain.txt"
    Xdata = np.loadtxt(features_path)
    ydata = np.loadtxt(labels_path)
    ydata = ydata.reshape(-1,1)

    # results from numpy (weight vector)
    np_linreg_wvector = numpy_solution(Xdata, ydata)

    tf_linreg_wvector = tensorflow_solution_closed_form(
        Xdata, ydata
    )

    tf_gradient_wvector = tensorflow_solution_gradient_descent(
        Xdata, ydata, diff="automatic"
    )

if __name__ == '__main__':
    main()


import pandas as pd
import numpy as np
from math import floor

def get_pp(data_1d: (pd.Series, np.array),
            USL: (int, float), LSL: (int, float)):

    std=data_1d.std()
    if USL>LSL and std>0 :
        pp = (USL - LSL) /( 6 * std)
    else:
        pp=0

    return round(pp,3)

def get_ppk(data_1d: (pd.Series, np.array),
            USL: (int, float), LSL: (int, float)):

    mean=data_1d.mean()
    std=data_1d.std()
    if USL>LSL and std>0 :
        ppk = min((USL-mean),(mean - LSL)) /( 3 *std)
    else:
        ppk=0

    return round(ppk,3)

def get_cp(data_1d: (pd.Series, np.array),
            USL: (int, float), LSL: (int, float),subgroupsize=5):
    rbar=get_rbar(data_1d,subgroupsize)
    d2=get_d2(subgroupsize)
    if USL>LSL and round(rbar,3)>0:
        cp=(USL-LSL)/(6*rbar/d2)
    else:
        cp=0

    return round(cp,3)

def get_cpk(data_1d: (pd.Series, np.array),
            USL: (int, float), LSL: (int, float),subgroupsize=5):

    mean=data_1d.mean()
    rbar=get_rbar(data_1d,subgroupsize)
    d2=get_d2(subgroupsize)

    if USL>LSL and round(rbar,3)>0:
        cpk=min((USL-mean),(mean - LSL)) /(3 * rbar/d2)
    else:
        cpk=0

    return round (cpk,3)

def get_xbar(data_1d: (pd.Series, np.array),
            USL: (int, float), LSL: (int, float)):
    xbar=data_1d.mean()
    return xbar

def get_rbar(data_1d: (pd.Series, np.array),subgroupsize=5):
    subgroupnumber=floor(data_1d.size/subgroupsize)
    r = data_1d.groupby(data_1d.index//subgroupsize).max()-data_1d.groupby(data_1d.index//subgroupsize).min()
    if subgroupnumber>0:
        rbar=r[:subgroupnumber-1].mean()
    else:
        rbar=r.mean()

    return rbar

def get_d2(subgroupsize):
    d2List=[1.128,1.693,2.059,2.326,2.534,2.704,2.847,2.970,3.078,3.173,3.259,3.336,3.407,3.472]
    if subgroupsize>15:
        subgroupsize=15
    d2=d2List[subgroupsize-2]
    return d2


from sage.all import xgcd

def attack(n, e1, c1, e2, c2):
    """
    Recovers the plaintext from two ciphertexts, encrypted using the same modulus and different public exponents.
    :param n: the common modulus
    :param e1: the first public exponent
    :param c1: the ciphertext of the first encryption
    :param e2: the second public exponent
    :param c2: the ciphertext of the second encryption
    :return: the plaintext
    """
    _, u, v = xgcd(e1, e2)
    p1 = pow(c1, u, n) if u > 0 else pow(pow(c1, -1, n), -u, n)
    p2 = pow(c2, v, n) if v > 0 else pow(pow(c2, -1, n), -v, n)
    return p1 * p2 % n


import csv
import matplotlib.pyplot as plt
import numpy as np

if __name__ == '__main__':
    x, y = np.array([]), np.array([])
    with open('datasets/aerogerador.dat') as f:
        reader = csv.reader(f, delimiter='\t')
        for line in reader:
            x = np.append(x, float(line[0]))
            y = np.append(y, float(line[1]))

    n_x = x.size
    matrix_ones = np.ones((n_x, 1))
    x_big = np.column_stack([matrix_ones, x])

    grau = int(input('Escolha o grau: '))
    if grau >= 2 and grau <= 5:
        if grau == 2:
            x_big = np.column_stack([x_big, x**2])
            print(x_big)
            beta = np.linalg.inv(np.matrix.transpose(x_big) @ x_big) @ np.matrix.transpose(x_big) @ y
            y_chapeu = beta[0] + (beta[1] * x) + (beta[2] * (x**2))

        elif grau == 3:
            x_big = np.column_stack([x_big, x**2])
            x_big = np.column_stack([x_big, x**3])
            beta = np.linalg.inv(np.matrix.transpose(x_big) @ x_big) @ np.matrix.transpose(x_big) @ y
            y_chapeu = beta[0] + (beta[1] * x) + (beta[2] * (x**2) + (beta[3] * (x**3)))

        elif grau == 4:
            x_big = np.column_stack([x_big, x**2])
            x_big = np.column_stack([x_big, x**3])
            x_big = np.column_stack([x_big, x**4])
            beta = np.linalg.inv(np.matrix.transpose(x_big) @ x_big) @ np.matrix.transpose(x_big) @ y
            y_chapeu = beta[0] + (beta[1] * x) + (beta[2] * (x**2) + (beta[3] * (x**3)) + (beta[4] * (x**4)))
        elif grau == 5:
            x_big = np.column_stack([x_big, x**2])
            x_big = np.column_stack([x_big, x**3])
            x_big = np.column_stack([x_big, x**4])
            x_big = np.column_stack([x_big, x**5])
            beta = np.linalg.inv(np.matrix.transpose(x_big) @ x_big) @ np.matrix.transpose(x_big) @ y
            y_chapeu = beta[0] + (beta[1] * x) + (beta[2] * (x**2) + (beta[3] * (x**3)) + (beta[4] * (x**4) + (beta[5] * (x**5))))

    y_media = np.mean(y)
    sq_e, s_yy = 0, 0
    for i in range(0, len(y)):
        sq_e = sq_e + (y[i] - y_chapeu[i])**2
        s_yy = s_yy + (y[i] - y_media)**2
    r2 = 1 - sq_e/s_yy
    r2_aj = 1 - (sq_e/(len(y) - (grau + 1)))/(s_yy/(len(y) - 1))
    print(r2, r2_aj)

    plt.plot(x, y_chapeu)
    plt.scatter(x=x, y=y, color='red', s=20, alpha=0.5)
    plt.title('               R2: ' + str(round(r2, 5)) + '\nR2 Ajustado: ' + str(round(r2_aj, 5)))
    plt.show()


"""
page 93
"""
import numpy as np
import matplotlib.pyplot as plt

x = [1, 2, 3, 4, 5, 6, 7, 8, 9]
y = [0.199, 0.389, 0.580, 0.783, 0.980, 1.177, 1.380, 1.575, 1.771]

A = np.vstack([x, np.ones(len(x))]).T
print(A)

a, b = np.linalg.lstsq(A, y)[0]
print("a=" + str(a) + ", b=" + str(b))

x = np.array(x)
y = np.array(y)

plt.plot(x, y, 'o', label='Original data', markersize=10)
plt.plot(x, a * x + b, 'r', label='Fitted line')
plt.show()


import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import random
import math

#functions
def get_data(n): #function that will read the dataset from the csv file

    data = pd.read_csv('task2_dataset.csv').values #load the dataset

    np.random.shuffle(data) #shuffles the rows of the data

    dataset = [data[:,0],data[:,n]]  #chooses the columns that will be used for the dataset based on the paramater n

    return dataset #returns the dataset

def compute_euclidean_distance(vec_1, vec_2): #Function that will calculate the shortest distance between two points

    dist = pow(vec_1[0] - vec_2[0], 2) + pow(vec_1[1] - vec_2[1], 2) #Euclidian formula

    return math.sqrt(dist) #returns the distance between two points

def initialise_centroids(dataset, k): #function to initialise the first centroids
    size = np.size(dataset, 1) #gets the size of the dataset
    centroids = [] #create an array for the centroids to be stored in

    for j in range(k): #for loop that will loop k times to create a random point to be the centroid
        n = random.randint(0, size) #random number assigned to n from 0 to size of dataset
        coord_x = dataset[0][n] #once random number has been assigned the corresponding x value is chosen
        coord_y = dataset[1][n] #once the random number has been assigned the corresponding y value is chosen
        centroids.append([coord_x, coord_y]) #adds the centroids to the centroid array

    return centroids #returns the centroid

def kmeans(dataset, k): #function that will perform the k-means algorithm
    centroids = initialise_centroids(dataset, k) #runs the function to initalise the centroids
    end = False
    while(end == False):

        cluster_assigned = [] #create an array cluster_assigned that will assign data points to closest centroids
        for i in range(k): #loop that adds a list to the list depending on k
            cluster_assigned.append([])

        for i in range(np.size(dataset, 1)): #loop that will use the datapoint to find its closest centroid
            vec_1 = [dataset[0][i],dataset[1][i]] #data point
            distance = 100 #base distance that is used to store the lowest distance from the data point to the centroid

            cluster_assigned.append([]) #allows to add data to the cluster_assigned list
            for j in range(k): #for loop that will loop until the lowest distance for a data point has been found
                current_distance = compute_euclidean_distance(vec_1, centroids[j]) #Euclidian function to work out the distance

                if current_distance < distance: #if statement that compares the distance calculated to its distance from the centroid
                    distance = current_distance
                    cluster = j

            cluster_assigned[cluster].append(vec_1) #appends the data point to its closest centroid
        #print(cluster_assigned[0][0])
        #print(cluster_assigned[0][0][0])
        #print(cluster_assigned[0][0][1])
        centroid = []

        for i in range(k):
            x_sum = 0
            y_sum = 0

            for j in range(len(cluster_assigned[i])):
                #print(cluster_assigned[i][j][0])
                x_sum += cluster_assigned[i][j][0]
                y_sum += cluster_assigned[i][j][1]

            x_mean = x_sum / len(cluster_assigned[i])
            y_mean = y_sum / len(cluster_assigned[i])

            #print(x_mean)
            #print(y_mean)

            centroid.append([x_mean, y_mean])

        for i in range(k):
            dist = compute_euclidean_distance(centroid[i], centroids[i])

        if centroids == centroid:
            end = True

        else:
            centroids = centroid

    return centroid, cluster_assigned #returns values centroid and cluster_assigned list

#main
dataset1 = get_data(1) #gets the dataset for height and tail length
dataset2 = get_data(2) #gets the dataset for height and leg length

#clustering for height to tail length on k = 2
centroid, cluster_assigned = kmeans(dataset1, 2) #runs the k-means algorithm to set the centroids and cluster points
plt.scatter(dataset1[:][0], dataset1[:][1]) #creates a scatter plot to display the datapoints in the dataset
for i in range(len(cluster_assigned[1])): # for loop to display datapoints to centroid 1
    plt.scatter(cluster_assigned[1][i][0], cluster_assigned[1][i][1],color = 'g') #the datapoints closest to centroid 1 are green
for i in range(len(cluster_assigned[0])): # for loop to display datapoints to centroid 2
    plt.scatter(cluster_assigned[0][i][0], cluster_assigned[0][i][1], color = 'y') #the datapoints closest to centroid 2 are yellow
for i in range(2): #for loop to show the centroids with k=2
    plt.scatter(centroid[i][0], centroid[i][1], color = 'r', marker = '+') #the centroids are displayed as red crosses
plt.show() # plots the scatter graph.

#clustering for height to tail length on k = 3
centroid, cluster_assigned = kmeans(dataset1, 3) #runs the k-means algorithm to set the centroids and cluster points
plt.scatter(dataset1[:][0], dataset1[:][1]) # creates a scatter plot to display the datapoints in the dataset
for i in range(len(cluster_assigned[1])): #for loop to display datapoints to centroid 1
    plt.scatter(cluster_assigned[1][i][0], cluster_assigned[1][i][1],color = 'g')#the datapoints closes to centroid 1 are green
for i in range(len(cluster_assigned[0])): # for loop to display datapoints to centroid 2
    plt.scatter(cluster_assigned[0][i][0], cluster_assigned[0][i][1], color = 'y') # the datapoints closest to centroid 2 are yellow
for i in range(3): #for loop to show the centroids with k=3
    plt.scatter(centroid[i][0], centroid[i][1], color = 'r', marker = '+') #the centroids are displayed as red crosses
plt.show() # plots the scatter graph

#clustering for height to leg length on k = 2
centroid, cluster_assigned = kmeans(dataset2, 2) # runs the k-means algorithm to set the centroids and cluster points
plt.scatter(dataset2[:][0], dataset2[:][1]) # creates a scatter plot to display the datapoints in the dataset
for i in range(len(cluster_assigned[1])): # for loop to display datapoints to centroid 1
    plt.scatter(cluster_assigned[1][i][0], cluster_assigned[1][i][1],color = 'g')#the datapoints closest to centroid 1 are green
for i in range(len(cluster_assigned[0])): # for loop to display datapoints to centroid 2
    plt.scatter(cluster_assigned[0][i][0], cluster_assigned[0][i][1], color = 'y')#the datapoints closest to centroid 2 are yellow
for i in range(2): # for loop to show the centroids with k=2
    plt.scatter(centroid[i][0], centroid[i][1], color = 'r', marker = '+') # the centroids are displayed as red corsses
plt.show() # plots the scatter graph

#clustering for height to leg length on k = 3
centroid, cluster_assigned = kmeans(dataset2, 3) # runs the k-means algorithm to set the centroids and cluster points
plt.scatter(dataset2[:][0], dataset2[:][1]) #creates a scatter plot to display the datapoints in the dataset
for i in range(len(cluster_assigned[1])): # for loop to display datapoints to centroids 1
    plt.scatter(cluster_assigned[1][i][0], cluster_assigned[1][i][1],color = 'g')#the datapoints closest to centroid 1 are green
for i in range(len(cluster_assigned[0])):# for loop to display datapoints to centroid 2
    plt.scatter(cluster_assigned[0][i][0], cluster_assigned[0][i][1], color = 'y') # the datapoints closest to centroid 2 are yellow
for i in range(3): # for loop to show the centroids with k=3
    plt.scatter(centroid[i][0], centroid[i][1], color = 'r', marker = '+') # the centroids are displayed as red crosses
plt.show() # plots the scatter graph


from typing import Callable, Tuple

import numpy as np

def student_id():
    """
    Returns a tuple of student's ID and TAU email address.
    """

    return '123456789', r'template@mail.tau.ac.il'

class QuadraticFunction:
    """
    Implementation of a quadratice function f(x) = 1/2 * x^T*Q*x + b^T*x
    with its gradient and Hessian matrix.
    """

    def __init__(
        self,
        Q: np.ndarray,
        b: np.ndarray
    ) -> None:
        """
        Initializes a quadratic function object with Q, b

        Arguments:
            Q : (n,n) matrix

            b : (n,) vector
        """

        self.Q = Q
        self.b = b

    def __call__(
        self,
        x: np.ndarray
    ) -> np.ndarray:
        """
        Evaluates f(x)

        Arguments:
            x : (n,) vector

        Return:
            fx : scalar
        """

        qx = np.dot(self.Q, x)
        fx = 0.5 * np.dot(x, qx) + np.dot(self.b,x)

        return fx

    def grad(
        self,
        x: np.ndarray
    ) -> np.ndarray:
        """
        Evaluates g(x), the gradient of f, at x

        Arguments:
            x : (n,) vector

        Return:
            gx : (n,) vector
        """

        gx = 0.5 * np.dot(self.Q.T + self.Q, x) + self.b

        return gx

    def hessian(
        self,
        x: np.ndarray
    ) -> np.ndarray:
        """
        Evaluates h(x), the Hessian matrix of f, at x

        Arguments:
            x : (n,) vector

        Return:
            hx : (n,n) matrix
        """

        hx = 0.5 * (self.Q.T + self.Q)

        return hx

class NewtonOptimizer:
    """
    Implements Newton's method optimizer with constant alpha.
    """

    def __init__(
        self,
        objective: Callable,
        x_0: np.ndarray,
        alpha: float,
        threshold: float = 1e-10,
        max_iters: int = 100
    ) -> None:
        """
        Arguments:

            objective : callable to an objective function

            x_0 : initial point, (n,) numpy array

            alpha : a scalar step size

            threshold : (float) stopping criteria |x_k-▁x_(k-1)|<threshold

            max_iters : (int) maximal number of iterations (stopping criteria)

        """

        self.objective = objective
        self.curr_x = x_0
        self.alpha = alpha
        self.threshold = threshold
        self.max_iters = max_iters

    def step(self):
        """
        Executes a single step of Newton's method.

        Return:

            next_x : an N-dimensional numpy vector, next x

            gx : vector, the gradient of f(x) evaluated at the current x
                 (the one used for the step calculation)

            hx : matrix, the Hessian of f(x) evaluated current x
                 (the one used for the step  calculation)
        """

        gx = self.objective.grad(self.curr_x)
        hx = self.objective.hessian(self.curr_x)
        next_x = self.curr_x - self.alpha * np.dot(np.linalg.inv(hx), gx)

        return next_x, gx, hx

    def optimize(self):
        """
        Execution of optimization flow

        Return:
            fmin : objective function evaluated at x_opt

            minimizer : x_opt

            num_iters : number of iterations until convergence
        """

        for iter_idx in range(self.max_iters):
            prev_x = self.curr_x
            self.curr_x, _, _ = self.step()  # update self.curr_x internally
            if np.linalg.norm(self.curr_x - prev_x) < self.threshold:
                break

        return self.objective(self.curr_x), self.curr_x, iter_idx+1

class BFGSOptimizer:
    """
    Implements the Quasi-Newton algorithm, using BFGS algorithm
    for approximating the inverse Hessian.
    """

    def __init__(
        self,
        objective: Callable,
        x_0: np.ndarray,
        B_0: np.ndarray,
        alpha_0: float,
        beta: float = 0.2,
        sigma: float = 0.3,
        threshold: float = 1e-10,
        max_iters: int = 100
    ) -> None:
        """
        Arguments:
            objective : callable to an objective function

            x_0 : initial point, (n,) numpy array

            B_0 : initial guess of the inverse Hessian

            alpha_0 : initial step size of Armijo line search (scalar)

            beta : (float) beta parameter of Armijo line search, a float in range (0,1)

            sigma : (float) sigma parameter of Armijo line search, a float in range (0,1)

            threshold : (float) stopping criteria |x_k-▁x_(k-1)|<threshold

            max_iters : (int) maximal number of iterations (stopping criteria)

        """

        self.objective = objective
        self.init_step_size = alpha_0
        self.beta = beta
        self.sigma = sigma
        self.threshold = threshold
        self.max_iters = max_iters

        self.curr_x = x_0
        self.curr_inv_hessian = B_0
        self.curr_step = 0
        self.curr_dir = np.zeros_like(x_0)

    def update_dir(self) -> np.ndarray:
        """
        Compute step direction

        Return:
            next_d : updated direction, (n,) numpy array
        """

        grad = self.objective.grad(self.curr_x)

        return -np.dot(self.curr_inv_hessian, grad)

    def update_step_size(self) -> np.ndarray:
        """
        Compute the new step size using Backtracking Line Search algorithm (Armijo rule).

        Return:
            step_size : scalar
        """

        # Initialize
        curr_step = self.init_step_size
        objective_1d = lambda alpha: self.objective(self.curr_x + alpha*self.curr_dir)  # restrict f to a line
        h = lambda alpha: objective_1d(alpha) - objective_1d(0)
        g = np.dot(self.objective.grad(self.curr_x), self.curr_dir)  # Note: d_phi(alpha)/dalpha = g(x+alpha*d)^T*d, alpha=0

        if h(curr_step) > self.sigma * g * curr_step:
            while h(curr_step) > self.sigma * g * curr_step:
                prev_step = curr_step
                curr_step = self.beta * curr_step
            step_size = curr_step
        else:
            while h(curr_step) <= self.sigma * g * curr_step:
                prev_step = curr_step
                curr_step = curr_step / self.beta
            step_size = prev_step

        return step_size

    def update_x(self) -> np.ndarray:
        """
        Take a step of size curr_step in direction dk.

        Return:
            next_x : updated point, (n,) numpy array
        """

        return self.curr_x + self.curr_step * self.curr_dir

    def update_inv_hessian(
        self,
        prev_x: np.ndarray
    ) -> np.ndarray:
        """
        Take a rank 2 BFGS update step to update the inverse Hessian.

        Arguments:
            prev_x : previous point, (n,) numpy array

        Return:
            next_inv_hessian : updated inverse Hessian, (n,n) numpy array
        """

        p = self.curr_x - prev_x
        q = self.objective.grad(self.curr_x) - self.objective.grad(prev_x)
        s = np.dot(self.curr_inv_hessian, q)
        tau = np.dot(s, q)
        mu = np.dot(p, q)
        v = p / mu - s / tau

        next_inv_hessian = self.curr_inv_hessian + np.outer(p, p) / mu - np.outer(s, s) / tau + tau * np.outer(v, v)

        return next_inv_hessian

    def step(self):
        """
        Executes a single step of Quasi-Newton algorithm.

        Return:
            next_x : previous point, (n,) numpy array

            next_d : vector, updated direction

            next_step_size : scalar

            next_inv_hessian : (n,n) numpy array, approximator of the inverse Hessian matrix
        """

        self.curr_dir = self.update_dir()
        self.curr_step = self.update_step_size()
        prev_x = self.curr_x
        self.curr_x = self.update_x()
        self.curr_inv_hessian = self.update_inv_hessian(prev_x)

        return self.curr_x, self.curr_dir, self.curr_step, self.curr_inv_hessian

    def optimize(self) -> Tuple:
        """
        Execution of optimization flow

        Return:
            fmin : objective function evaluated at x_opt

            minimizer : x_opt

            num_iters : number of iterations until convergence
        """

        for iter_idx in range(self.max_iters):
            prev_x = self.curr_x
            _ = self.step()  # update self.curr_x internally
            if np.linalg.norm(self.curr_x - prev_x) < self.threshold:
                break

        return self.objective(self.curr_x), self.curr_x, iter_idx+1

class TotalVariationObjective:
    """
    Implementation of a Total Variation objective function (MSE + TV) for image denoising.
    """

    def __init__(
        self,
        src_img: np.ndarray,
        mu: float = 1e-3,
        eps: float = 1e-8
    ) -> None:
        """
        Initializes a FastRoute problem

        Arguments:
            src_img : (n,m) matrix, input noisy image

            mu : regularization parameter, determines the weight of total variation term

            eps : small number for numerical stability
        """

        self.src_img = src_img
        self.mu = mu
        self.eps = eps

    def __call__(
        self,
        img: np.ndarray
    ) -> np.ndarray:
        """
        Compute TV objective.

        Arguments:
            img : (nxm,) vector, denoised image

        Return:
            total_variation : scalar, objective's value
        """

        # Convert vector to matrix
        img = self._vec_to_mat(img)

        mse = np.square(self.src_img - img).mean()

        row_diff, col_diff = self._diff(img)
        row_diff_pad = np.pad(row_diff, ((0,1), (0,0)), 'constant', constant_values=0)
        col_diff_pad = np.pad(col_diff, ((0,0), (0,1)), 'constant', constant_values=0)
        total_variation = np.sqrt(row_diff_pad**2  + col_diff_pad**2 + self.eps).sum()

        return mse + self.mu * total_variation

    def grad(
        self,
        img: np.ndarray
    ) -> np.ndarray:
        """
        Evaluate the gradient of the objective.

        Arguments:
            img : (nxm,) row-major vector, denoised image

        Return:
            grad : (nxm,) row-major vector, the objective's gradient
        """

        # Convert vector to matrix
        img = self._vec_to_mat(img)

        # MSE term
        mse_grad = -2*(self.src_img - img) / img.size

        # TV terms
        eps = self.eps
        row_diff, col_diff = self._diff(img, pad=True)
        term_1 = row_diff[:-1,1:-1] / np.sqrt(row_diff[:-1,1:-1]**2 + col_diff[:-2,1:]**2 + eps)
        term_2 = (row_diff[1:,1:-1] + col_diff[1:-1,1:]) / np.sqrt(row_diff[1:,1:-1]**2 + col_diff[1:-1,1:]**2 + eps)
        term_3 = col_diff[1:-1,:-1] / np.sqrt(row_diff[1:,:-2]**2 + col_diff[1:-1,:-1]**2 + eps)

        grad = mse_grad + self.mu * (term_1 - term_2 + term_3)

        return self._mat_to_vec(grad)

    def _mat_to_vec(
        self,
        img: np.ndarray
    ) -> np.ndarray:
        """
        Converts a vector to an (n,m) matrix.
        """

        return img.flatten()

    def _vec_to_mat(
        self,
        img: np.ndarray
    ) -> np.ndarray:
        """
        Converts an image to a row major (nxm,) vector
        """

        return img.reshape(self.src_img.shape)

    def _diff(
        self,
        x: np.ndarray,
        pad: bool = False
    ) -> Tuple:
        """
        Calculate the row and collumn difference arrays

        Arguments:
            x : (n,m) matrix

            pad : False, zero padding on all boundaries

        Return:
            row_diff : row_diff[i,:] = x[i+1,:] - x[i,:],
                        output dimensions:
                            pad is False: (n-1,m)
                            pad is True: (n+1,m+2)

            col_diff : col_diff[:,i] = x[:,i+1] - x[:,i]
                        output dimensions:
                            pad is False: (n,m-1)
                            pad is True: (n+2,m+1)
        """

        row_diff = np.diff(x, axis=0)
        col_diff = np.diff(x, axis=1)

        if pad:
            row_diff = np.pad(row_diff, ((1,1),(1,1)), 'constant', constant_values=0)
            col_diff = np.pad(col_diff, ((1,1),(1,1)), 'constant', constant_values=0)

        return row_diff, col_diff

def denoise_img(
    noisy_img: np.ndarray,
    B_0: np.ndarray,
    alpha_0: float,
    beta: float = 0.2,
    sigma: float = 0.3,
    threshold: float = 1e-10,
    max_iters: int = 1000,
    mu: float = 1e-4,
    eps: float = 1e-8
) -> Tuple:
    """
    Optimizes a Total Variantion objective using BFGS optimizer to
    denoise a noisy image.

    Arguments:
    	noisy_img : (n,m) matrix, input noisy image

        For the rest: see BFGSOptimizer and TotalVariationObjective docstrings.

    Return:
        total_variation : loss at minimum

        img : matrix, denoised image

        num_iters : number of iterations until convergence

    """

    # Initialize objective and optimizer
    denoising_objective = TotalVariationObjective(src_img=noisy_img, mu=mu, eps=eps)

    opt = BFGSOptimizer(
        objective=denoising_objective,
        x_0=noisy_img.flatten(),
        B_0=B_0,
        alpha_0=alpha_0,
        beta=beta,
        sigma=sigma,
        threshold=threshold,
        max_iters=max_iters
    )

    # Optimize
    total_variation, denoised_img, num_iters = opt.optimize()

    # Post process
    denoised_img = np.clip(denoised_img, a_min=0, a_max=1)
    denoised_img = denoised_img.reshape(32,32)

    return total_variation, denoised_img, num_iters


"""
solve the diffusion equation:

 phi_t = k phi_{xx}

with a Crank-Nicolson implicit discretization

M. Zingale (2013-04-03)
"""

import numpy as np
from scipy import linalg
import matplotlib.pyplot as plt
from diffusion_explicit import Grid1d

import matplotlib as mpl

# Use LaTeX for rendering
mpl.rcParams['mathtext.fontset'] = 'cm'
mpl.rcParams['mathtext.rm'] = 'serif'
mpl.rcParams['font.size'] = 12
mpl.rcParams['legend.fontsize'] = 'large'
mpl.rcParams['figure.titlesize'] = 'small'

class Simulation(object):

    def __init__(self, grid, k=1.0):
        self.grid = grid
        self.t = 0.0
        self.k = k  # diffusion coefficient

    def init_cond(self, name, *args):
        # initialize the data

        if name == "gaussian":
            t0, phi1, phi2 = args
            self.grid.phi[:] = self.grid.phi_a(0.0, self.k, t0, phi1, phi2)

    def diffuse_CN(self, dt):
        """
        diffuse phi implicitly through timestep dt, with a C-N
        temporal discretization
        """

        gr = self.grid
        phi = gr.phi

        phinew = gr.scratch_array()

        alpha = self.k*dt/gr.dx**2

        # create the RHS of the matrix
        gr.fill_BCs()
        R = 0.5*self.k*dt*self.lap()
        R = R[gr.ilo:gr.ihi+1]
        R += phi[gr.ilo:gr.ihi+1]

        # create the diagonal, d+1 and d-1 parts of the matrix
        d = (1.0 + alpha)*np.ones(gr.nx)
        u = -0.5*alpha*np.ones(gr.nx)
        u[0] = 0.0

        l = -0.5*alpha*np.ones(gr.nx)
        l[gr.nx-1] = 0.0

        # set the boundary conditions by changing the matrix elements

        # homogeneous neumann
        d[0] = 1.0 + 0.5*alpha
        d[gr.nx-1] = 1.0 + 0.5*alpha

        # Dirichlet
        #d[0] = 1.0 + 1.5*alpha
        #d[gr.nx-1] = 1.0 + 1.5*alpha

        #R[0] += alpha*phi1
        #R[gr.nx-1] += alpha*phi1

        # solve
        A = np.matrix([u,d,l])
        phinew[gr.ilo:gr.ihi+1] = linalg.solve_banded((1,1), A, R)

        return phinew

    def lap(self):
        """ compute the Laplacian of phi """

        gr = self.grid
        phi = gr.phi

        lapphi = gr.scratch_array()

        ib = gr.ilo
        ie = gr.ihi

        lapphi[ib:ie+1] = \
            (phi[ib-1:ie] - 2.0*phi[ib:ie+1] + phi[ib+1:ie+2])/gr.dx**2

        return lapphi

    def evolve(self, C, tmax):
        """
        the main evolution loop.  Evolve

        phi_t = k phi_{xx}

        from t = 0 to tmax
        """

        gr = self.grid

        # time info
        dt = C*0.5*gr.dx**2/self.k

        while self.t < tmax:

            gr.fill_BCs()

            # make sure we end right at tmax
            if self.t + dt > tmax:
                dt = tmax - self.t

            # diffuse for dt
            phinew = self.diffuse_CN(dt)

            gr.phi[:] = phinew[:]
            self.t += dt

if __name__ == "__main__":

    #--------------------------------------------------------------------------
    # Convergence of a Gaussian

    # a characteristic timescale for diffusion is L^2/k
    tmax = 0.005

    t0 = 1.e-4
    phi1 = 1.0
    phi2 = 2.0

    k = 1.0

    N = np.array([32, 64, 128, 256, 512])

    # CFL number
    CFL = [0.8, 8.0]

    for C in CFL:
        err = []

        for nx in N:

            # the present C-N discretization
            print(C, nx)
            g = Grid1d(nx, ng=1)
            s = Simulation(g, k=k)
            s.init_cond("gaussian", t0, phi1, phi2)
            s.evolve(C, tmax)

            xc = 0.5*(g.xmin + g.xmax)
            phi_analytic = g.phi_a(tmax, k, t0, phi1, phi2)

            err.append(g.norm(g.phi - phi_analytic))

        plt.clf()

        err = np.array(err)

        plt.scatter(N, err, color="C0", label="C-N implicit diffusion")
        plt.loglog(N, err[len(N)-1]*(N[len(N)-1]/N)**2,
                   color="C1", label="$\mathcal{O}(\Delta x^2)$")

        plt.xlabel(r"$N$", fontsize="large")
        plt.ylabel(r"L2 norm of absolute error")
        plt.title("C-N Implicit Diffusion, C = %3.2f, t = %5.2g" % (C, tmax))

        plt.ylim(1.e-6, 1.e-2)
        plt.legend(frameon=False, fontsize="small")

        plt.tight_layout()

        plt.savefig("diffimplicit-converge-{}.pdf".format(C))

    #-------------------------------------------------------------------------
    # solution at multiple times

    # diffusion coefficient
    k = 1.0

    # reference time
    t0 = 1.e-4

    # state coeffs
    phi1 = 1.0
    phi2 = 2.0

    nx = 128

    # a characteristic timescale for diffusion is 0.5*dx**2/k
    dt = 0.5/(k*nx**2)
    tmax = 100*dt

    # analytic on a fine grid
    nx_analytic = 512

    CFL = [0.8, 8.0]

    for C in CFL:

        plt.clf()

        ntimes = 5
        tend = tmax/2.0**(ntimes-1)

        c = ["C0", "C1", "C2", "C3", "C4"]

        while tend <= tmax:

            g = Grid1d(nx, ng=2)
            s = Simulation(g, k=k)
            s.init_cond("gaussian", t0, phi1, phi2)
            s.evolve(C, tend)

            ga = Grid1d(nx_analytic, ng=2)
            xc = 0.5*(ga.xmin + ga.xmax)
            phi_analytic = ga.phi_a(tend, k, t0, phi1, phi2)

            color = c.pop()
            plt.plot(g.x[g.ilo:g.ihi+1], g.phi[g.ilo:g.ihi+1],
                     "x", color=color, label="$t = %g$ s" % (tend))
            plt.plot(ga.x[ga.ilo:ga.ihi+1], phi_analytic[ga.ilo:ga.ihi+1],
                     color=color, ls="-")

            tend = 2.0*tend

        plt.xlim(0.35,0.65)
        plt.ylim(0.95,1.7)

        plt.legend(frameon=False, fontsize="small")

        plt.xlabel("$x$", fontsize="large")
        plt.ylabel(r"$\phi$", fontsize="large")
        plt.title(r"implicit diffusion, N = %d, $C$ = %3.2f" % (nx, C))

        f = plt.gcf()
        f.set_size_inches(8.0, 6.0)

        plt.tight_layout()
        plt.savefig("diff-implicit-{}-CFL_{}.pdf".format(nx, C))

    # under-resolved example
    plt.clf()
    nx = 64
    C = 10.0
    tmax = 0.001

    g = Grid1d(nx, ng=2)
    s = Simulation(g, k=k)
    s.init_cond("gaussian", t0, phi1, phi2)
    s.evolve(C, tend)

    ga = Grid1d(nx_analytic, ng=2)
    xc = 0.5*(ga.xmin + ga.xmax)
    phi_analytic = ga.phi_a(tend, k, t0, phi1, phi2)

    plt.plot(g.x[g.ilo:g.ihi+1], g.phi[g.ilo:g.ihi+1],
             color="r", marker="x", ls="-", label="$t = %g$ s" % (tend))
    plt.plot(ga.x[ga.ilo:ga.ihi+1], phi_analytic[ga.ilo:ga.ihi+1],
             color=color, ls="-")

    plt.xlim(0.2,0.8)

    plt.xlabel("$x$", fontsize="large")
    plt.ylabel(r"$\phi$", fontsize="large")
    plt.title(r"implicit diffusion, N = {}, $C$ = {:3.2f}, $t$ = {}".format(nx, C, tmax))

    f = plt.gcf()
    f.set_size_inches(8.0, 6.0)

    plt.tight_layout()

    plt.savefig("diff-implicit-{}-CFL_{}.pdf".format(nx, C))


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Sun Feb 24 14:28:26 2019

@author: Emma
"""
import numpy as np
import os,sys
sys.path.append(os.pardir)
from dataset.mnist import load_mnist
import matplotlib.pylab as plt

def cross_entropy_error(y,t):
    if y.ndim==1:
        t=t.reshape(1,t.size())
        y=y.reshape(1,y.size())
    batch_size=y.shape[0]
    delta=1e-7
    return -np.sum(t*np.log(y+delta))/batch_size

def numerical_diff(f,x,i):
    delta=1e-4
    x2=x.copy()
    x1=x.copy()
    x2[i]=x2[i]+delta
    x1[i]=x1[i]-delta

    return (f(x2)-f(x1))/(2*delta)

def numerical_gradient(f,x):
    grad=np.zeros_like(x)
    for i in range(x.size):
        grad[i]=numerical_diff(f,x,i)
    return grad

def gradient_descent(f,init_x,lr=0.1,step_num=100):
    x=init_x

    for i in range(step_num):
        grad=numerical_gradient(f,x)
        x-=lr*grad
    return x

def function_1(x):
    return 0.01*x**2+0.1*x

def function_2(x):
    return np.sum(x**2)

if __name__=="__main__":
#    (x_train,t_train),(x_test,t_test)=load_mnist(normalize=True,one_hot_label=True)
#    train_size=x_train.shape[0]
#    batch_size=10
#    mask_batch=np.random.choice(train_size,batch_size)
#    x_batch=x_train[mask_batch]
#    t_batch=t_train[mask_batch]

#    x=np.arange(-20,10,0.1)
#    y=function_1(x)
#    plt.xlabel("x")
#    plt.ylabel("f(x)")
#    plt.plot(x,y)
#    plt.show()
#
    print(numerical_diff(function_2,np.array([4.5,3.6]),0))
    print(numerical_diff(function_2,np.array([4.5,3.6]),1))
    print(numerical_gradient(function_2,np.array([4.5,3.6])))
    print(gradient_descent(function_2,np.array([-3.0,4.0])))


# coding: utf-8

# In[1]:

# %matplotlib inline
get_ipython().run_line_magic('matplotlib', 'ipympl')
# for interactive

# In[2]:

get_ipython().run_line_magic('config', "InlineBackend.figure_format = 'retina'")

# In[3]:

get_ipython().run_line_magic('cat', '0Source_Citation.txt')

# For high dpi displays.

# # 0. General note

# - This notebook shows how to propagate uncertainties to obtain reasonable error bars for pressure.
# - We use MgO as an example.

# # 1. General setup

# In[20]:

import numpy as np
import matplotlib.pyplot as plt
import uncertainties as uct
from uncertainties import unumpy as unp
import pandas as pd
import pytheos as eos

# # 2. Assign uncertainties to the EOS parameters

# Uncertainties of parameters can be defined using the `uncertainties` package.  The parameter values used in this example are just for demonstration purpose.  For more accurate values, please refer to the recent literature.

# In[21]:

v0 = uct.ufloat(74.698, 0.004)
k0 = uct.ufloat(160., 3.)
k0p = uct.ufloat(4.0, 0.3)

# We make a `numpy` array for volume at high pressure.

# In[22]:

n_pts = 20
vv0 = np.linspace(1.,0.8, n_pts)
v = vv0 * v0

# Calculate pressure from `pytheos`.

# In[23]:

p = eos.bm3_p(v, v0, k0, k0p)

# ### How to get help

# You may get help by using `help(function_name)`.

# In[24]:

help(eos.bm3_p)

# Now you can see that error bars for the EOS parameters are used in error propagation calculation for pressure value.  Note that the uncertainties in the EOS parameters are correctly applied for propagating uncertainties to both molar volume and pressure.

# We use `pandas` to organize the data more effectively.  It also presents nice looking tables.

# In[26]:

df = pd.DataFrame()
df['unit-cell volume'] = v
df['pressure'] = p
df
#print(df.to_string(index=False)) # for fancier print

# Unfortunately to plot with `matplotlib`, you need to separate nominal values from standard deviation.

# In[27]:

f = plt.figure()
plt.errorbar(unp.nominal_values(p), unp.nominal_values(v), fmt='ko',              xerr=unp.std_devs(p), yerr=unp.std_devs(v))
plt.xlabel('Pressure (GPa)'); plt.ylabel('Unit-cell volume ($\mathrm{\AA}^3$)');

# # 3. Calculate volume from pressure using pytheos

# `Pytheos` provides functions to calculate volumes at given pressures with error propagation.

# In[28]:

v_cal = eos.bm3_v(p, v0, k0, k0p)

# In[29]:

df = pd.DataFrame()
df['pressure'] = p
df['unit-cell volume'] = v_cal
df
# print(df.to_string(index=False))

# Compare this table with the one we showed above for accuracy check.

# # 4. Bulk modulus at high pressure

# You can also propagate uncertainties in bulk modulus calculation.

# In[30]:

k = eos.bm3_k(p, v0, k0, k0p)

# In[31]:

df = pd.DataFrame()
df['pressure'] = p
df['bulk modulus'] = k
df
#print(df.to_string(index=False))

# In[32]:

f = plt.figure()
plt.errorbar( unp.nominal_values(p), unp.nominal_values(k),              xerr=unp.std_devs(p), yerr=unp.std_devs(k), fmt='o')
plt.xlabel('Pressure (GPa)'); plt.ylabel('Bulk modulus (GPa)');

# # 5. High temperature equation of state

# The constant $q$ assumption has been used widely for the thermal part of the mantle phases.  Below, we assign uncertainties to the thermal parameters of MgO.

# In[33]:

gamma0 = uct.ufloat(1.45, 0.02)
q = uct.ufloat(0.8, 0.3)
theta0 = uct.ufloat(800., 0.)

# We will use `constq_pth` for calculating the thermal pressure part of the EOS.  Below, I demonstrate how to get help for the function.

# In[34]:

help(eos.constq_pth)

# We calculate total pressure at 2000 K below.  `eos.constq_pth` requires input of volume and temperature with the same number of elements.  For 2000-K isotherm, we generate a temperature array with 2000 for all elements.

# In[35]:

p_hT = eos.bm3_p(v, v0, k0, k0p) +     eos.constq_pth(v, np.ones_like(v)*2000., v0, gamma0, q, theta0, 2, 4)

# In[36]:

df = pd.DataFrame()
df['unit-cell volume'] = v
df['pressure@300K'] = p
df['pressure@2000K'] = p_hT
df
# print(df.to_string(index=False))


# In 2 part for 1D Linear Regression, we will train the modeil with PyTorch

import numpy as np
import matplotlib.pyplot as plt
import torch

# define class plot_diagram - to visualize data space and parameter space during training

class plot_diagram():

    def __init__(self,X,Y,w,stop,go=False): #Constructor
        start = w.data

        self.error=[]
        self.parameter=[]
        self.X=X.numpy()
        self.Y=Y.numpy()

        self.parameter_values=torch.arange(start,stop)
        self.Loss_function=[criterion(forward(X),Y) for w.data in self.parameter_values]
        w.data=start

    def __call__(self, Yhat,w,error,n): #Executor
        self.error.append(error)
        self.parameter.append(w.data)

        plt.subplot(212)
        plt.plot(self.X,Yhat.detach().numpy())
        plt.plot(self.X,self.Y,'ro')

        plt.xlabel('A')
        plt.ylim(-20,20)

        plt.subplot(211)
        plt.title('Data Space (Top) Estimated Line (Bottom) Iteration' + str(n))
        plt.plot(self.parameter_values.numpy(),self.Loss_function)
        plt.plot(self.parameter,self.error,'ro')
        plt.xlabel('B')
        plt.figure()

    def __del__(self): #Destructor
        plt.close('all')

## Make some Data - generate values -3 to 3 that create a line with a slope of 3

X=torch.arange(-3,3,0.1).view(-1,1)
f=-3*X
# Add some noise
Y=f+0.1*torch.randn(X.size())

plt.figure() #plotting line and added noise
plt.plot(X.numpy(),Y.numpy(),'rx',label='Y')
plt.plot(X.numpy(),f.numpy(),label='f')
plt.xlabel('x')
plt.ylabel('y')
plt.legend()

# Model and cost function built

def forward(x): #for prediction
    return w*x
def forward2(x): #for prediction
    return w2*x

def criterion(yhat,y): # for MSE to evaluate the result
    return torch.mean((yhat - y) ** 2)

lr=0.1
LOSS=[] #loss matrix 1
LOSS2=[] #loss matrix 2

w=torch.tensor(-10.0,requires_grad=True) #Learnable paramter 1
w2=torch.tensor(-15.0,requires_grad=True) #Learnable paramter 2

gradient_plot=plot_diagram(X,Y,w,stop=5)
gradient_plot2=plot_diagram(X,Y,w2,stop=15)

# Train model

#define a function to train a model:

def train_model(iter): # for w and LOSS
    for epoch in range(iter):

        Yhat=forward(X) #make prediction
        loss=criterion(Yhat,Y) # calculate iteration
        gradient_plot(Yhat,w,loss.item(),epoch) #plot diagram

        LOSS.append(loss.item()) #store loss into list

        loss.backward() # compute gradient of loss with wrt to all learnable parameters
        w.data=w.data-lr*w.grad.data #update parameters
        w.grad.data.zero_() # xero the gradient before backward pass

def my_train_model(iter): # for w2 and LOSS2
    for epoch in range(iter):

        Yhat2=forward2(X) #make prediction
        loss2=criterion(Yhat2,Y) # calculate iteration
        gradient_plot2(Yhat2,w2,loss2.item(),epoch) #plot diagram

        LOSS2.append(loss2.item()) #store loss into list

        loss2.backward() # compute gradient of loss with wrt to all learnable parameters
        w2.data=w2.data-lr*w2.grad.data #update parameters
        w2.grad.data.zero_() # xero the gradient before backward pass

train_model(4) # model train with plot for 4 iterations
my_train_model(4)

# Plot loss for each iteration

plt.figure()
plt.plot(LOSS)
plt.plot(LOSS2)
plt.tight_layout()
plt.xlabel('Epoch/Iterations')
plt.ylabel('Cost')

#Disply plots
plt.show()

Deep Neural Networks with PyTorch/Module2/Lab2_LR_1D_Prediction_2.py


import numpy as np

def yche_matrix_pow(a, b):
    if b == 1:
        return a
    else:
        return np.dot(yche_matrix_pow(a, b / 2), yche_matrix_pow(a, (b + 1) / 2))

def fib(n):
    primitive_matrix = np.matrix([[1, 1], [1, 0]])
    if n < 2:
        return n
    else:
        return yche_matrix_pow(primitive_matrix, n - 1).item(0, 0)

answer = ','.join([str(fib(i)) for i in range(43)])
print len(answer)
print answer

tmp=[0,1,1,2,3,5,8,13,21,34,55,89,144,233,377,610,987,1597,2584,4181,6765,10946,17711,28657,46368,75025,121393,196418,317811,514229,832040,1346269,2178309,3524578,5702887,9227465,14930352,24157817,39088169,63245986,102334155,165580141]
print len(tmp)


import numpy as np
from scipy.optimize import minimize
# problem conditions
demand_1 = 110; demand_2 = 100; demand_3 = 80
prob_1 = 0.2; prob_2 = 0.6; prob_3 = 0.2
price_1 = 50; price_2 = 46; price_3 = 44
price_c = 45; contract_max = 90.0
# define objective function
def objective(p):
    p_c1, p_c2, p_c3, p1, p2, p3 = p
    sce_1 = np.dot([price_c,price_1],[p_c1,p1])
    sce_2 = np.dot([price_c,price_2],[p_c2,p2])
    sce_3 = np.dot([price_c,price_3],[p_c3,p3])
    obj = 24*7*np.dot([prob_1, prob_2, prob_3],[sce_1,sce_2,sce_3])
    return obj
# define constraints
def constraint1(p):
    return p[0]+p[3]-demand_1
def constraint2(p):
    return p[1]+p[4]-demand_2
def constraint3(p):
    return p[2]+p[5]-demand_3

# p_c1=p_c2=p_c3
def constraint_c12(p):
    return p[0]-p[1]
def constraint_c23(p):
    return p[1]-p[2]
'''
def constraint2(x):
    sum_eq = 40.0
    for i in range(4):
        sum_eq = sum_eq - x[i]**2
    return sum_eq
'''
# initial guesses
n = 6
p0 = np.zeros(n)
for i in range(len(p0)):
    p0[i] = 80.0
# show initial objective
print('Initial Objective: ' + str(round(objective(p0),0)))

# set constraints/bounds
b_c = (0.0, contract_max)
b_pos = (0.0, None)
con1 = {'type': 'ineq', 'fun': constraint1}
con2 = {'type': 'ineq', 'fun': constraint2}
con3 = {'type': 'ineq', 'fun': constraint3}
con_c12 = {'type': 'eq', 'fun': constraint_c12}
con_c23 = {'type': 'eq', 'fun': constraint_c23}
cons = [con1,con2,con3,con_c12,con_c23]
bnds = (b_c, b_c, b_c, b_pos, b_pos, b_pos)
# optimize
solution = minimize(objective,p0,method='SLSQP',bounds=bnds, constraints=cons)
p = solution.x
for i in range(len(p)):
    p[i]=round(p[i],0)
# show final objective
print('Final Objective: ' + str(round(objective(p),0)))

# print solution
print('Solution')
print('p_c1 = ' + str(p[0]))
print('p1 = ' + str(p[3]))
print('p_c2 = ' + str(p[1]))
print('p2 = ' + str(p[4]))
print('p_c3 = ' + str(p[2]))
print('p3 = ' + str(p[5]))

# calculate EVPI

# stochastic solution
z_s = objective(p)
# EVPI problem (with perfect infomation) -- relax the non-anticipativity constrainsts
cons = [con1,con2,con3]
# optimize
solution = minimize(objective,p0,method='SLSQP',bounds=bnds, constraints=cons)
p = solution.x
for i in range(len(p)):
    p[i]=round(p[i],0)
print('Final Objective: ' + str(round(objective(p),0)))
print('Solution given perfect information')
print('p_c1 = ' + str(p[0]))
print('p1 = ' + str(p[3]))
print('p_c2 = ' + str(p[1]))
print('p2 = ' + str(p[4]))
print('p_c3 = ' + str(p[2]))
print('p3 = ' + str(p[5]))

# perfect info. solution
z_p = objective(p)

# calculate EVPI value
EVPI_min = z_s - z_p
print('EVPI value: ' + str(EVPI_min))


from numpy import sqrt, pi, exp
import numpy as np

class muedge(object):
    def _init_(self,x,y,e=1):
        '''
        generic __init__ for all mufunction classes
        x, y, e are numpy arrays
        e is always defined,
        either provided by the caller
        or default to np.ones(x.shape[0])
        '''
        if x.shape[0]!=y.shape[0]:
            raise ValueError('x, y have different lengths')
        else:
            self.x = x
            self.y = y
        try:
            if e==1:
                self.e = np.ones(x.shape[0])
        except:
            if e.shape[0]!=x.shape[0]:
                raise ValueError('x, e have different lengths')
            else:
                self.e = e
    # ---- end generic __init__

    def f(self,t, t00, N, D):
        '''
        fit function for ISIS t0 fit on edge
        see mulab function f=mathedge(t,N,Delta,TauPi,TauMu,t0)
        obtained from MuonBeamSimple.nb in Mathematica
        here time in musec (vs mulab in ns)
        '''
        tp = 0.026003 # mus
        t0 = 0.82*tp-t00
        tm = 2.197 # mus
        f = (6*N/(D**3*(tm-tp)))*\
             exp(-(((t+t0+D/2)*tm+(t+t0+D/2)*tp)/(tm*\
                                               tp)))*\
             ((-2*exp(((t+t0+D/2)*tm+D*tp)/(tm*tp))*\
             tm**2*(-D/2+tm)+\
             2*exp((D*tm+(t+t0+D/2)*tp)/(tm*tp))*\
             tp**2*(-(D/2)+tp)+\
             exp(((t+t0+D/2)*(tm+tp))/(tm*tp))*(tm-tp)*\
             (t**2+t0**2+2*t0*(t-tm-tp)-2*t*(tm+tp)+\
             2*(-(D**2/8)+tm**2+tm*tp+tp**2)))*\
             ((t+t0-D/2)>0)+\
             (2*exp((t+t0+D/2)/tp)*tm**2*(D/2+tm)-\
              2*exp((t+t0+D/2)/tm)*tp**2*(D/2+tp)-\
              exp(((t+t0+D/2)*(tm+tp))/(tm*tp))*(tm-tp)*\
              (t**2+t0**2+2*t0*(t-tm-tp)-2*t*(tm+tp)+\
              2*(-(D**2/8)+tm**2+tm*tp+tp**2)))*((t+t0+D/2)>0))
        return f

    def __call__(self,t00, N, D):
        '''
        chisquare, normalized if self.e was provided, unnormalized otherwise
        '''
        return sum(((self.f(self.x, t00, N, D)-self.y)/self.e)**2)


import numpy as np
from scipy.integrate import solve_ivp
import matplotlib.pyplot as plt
from numpy.random import randint, rand
from sir import *

def SIR_continuous_reinfected(b,k,time,ii,r):
    """
    Simulates continuous SIR model
    ii = initial percentage of infected
    time = Days of simulation
    b = probability that people getting infectious
    k = probability that people getting recovered
    r = reinfected probability

    returns sol from solve_ivp
    """
    def SIR(t, X):

        #The main set of equations
        Y = np.zeros((3))
        Y[0] = -b * X[0] * X[2]
        Y[1] = k * X[2] - r * X[1]
        Y[2] = b * X[0] * X[2] - (k * X[2]) + r * X[1]
        return Y
    t_eval = np.linspace(0, time, time)
    sol1 = solve_ivp(SIR, [0, time], [1-ii, 0, ii], method='RK45', t_eval=t_eval)    # solve the equation
    return sol1

## Discrete

class Person_reinfection(Person):
    """
    An agent representing a person.

    By default, a person is susceptible but not infectious. They can become infectious by exposing with disease method.

    Status: 0 = susceptible        1 = infected          2 = removed
    """

    def __init__(self,startpos=None):
        self.status = 0
        if startpos==None:
            self.pos = np.random.rand(2)
        else:
            self.pos = np.array(startpos)
        self.reinfection=1

    def reinfectionrate(self):
        return self.reinfection
    def immunization(self,p):
        q=self.reinfection-p
        if q<0:
            q=0
        self.reinfection=q

def count_susceptible(pop):
    """
    counts number of susceptible
    """
    return sum(p.is_susceptible() for p in pop)

def count_infected(pop):
    """
    counts number of infected
    """
    return sum(p.is_infected() for p in pop)

def count_removed(pop):
    """
    counts number of removed
    """
    return sum(p.is_removed() for p in pop)

def SIR_discrete_reinfection(N,ii,b,T,k):
    """
    Simulates discrete SIR model
    N = Total number of people
    ii = initial percentage of infected
    b = number of contacts per day
    T = Days of simulation
    k = probability that people getting recovered

    returns list of s,i,r
    """
    pop = [Person_reinfection() for i in range(N)]
    initial_infection = randint(N,size=np.int(N*ii))
    for i in initial_infection:
        pop[i].infection()

    counts_susceptible = [count_susceptible(pop)]
    counts_infected = [count_infected(pop)]
    counts_removed = [count_removed(pop)]

    for t in range(T):
        # update the population
        for i in range(N):
            if pop[i].is_infected():
                # person i infected all their contacts
                contacts = randint(N, size=b)
                for j in contacts:
                    if not pop[j].is_removed():
                        pop[j].infection()
                        #if rand() < p:
                        #    pop[j].infection()
                    if pop[j].is_removed():
                        if rand()<pop[j].reinfectionrate():
                            pop[j].infection()
                if rand()< k:
                    pop[i].remove()
                    pop[i].immunization(rand())

        # add to our counts
        counts_susceptible.append(count_susceptible(pop))
        counts_infected.append(count_infected(pop))
        counts_removed.append(count_removed(pop))
    return np.array([counts_susceptible,counts_infected,counts_removed])


import cv2 as cv
import numpy as np
from matplotlib import pyplot as plt

'''
QUESTION 2
'''
'''
Read the data.
'''
data = np.loadtxt('data/hands_aligned_train.txt',skiprows=1)
'''
Caclulate mu
'''
mu=np.mean(data,axis=1)
'''
Calculate W = w-mu
'''
W = data - mu.reshape(-1,1)
'''
Caclulate covariance matrix W.Wt
'''
cov = np.cov(W)
'''
Find the eigen vectors and eigen value of covariance matrix
'''
eigen_value,eigen_vectors=np.linalg.eig(cov)
'''
Take the first 5 eigen vectors and find phi and sigma square.
'''
K=5
sigma_square = 1/(112-K)*(np.sum(eigen_value[K+1:]))
L2 = np.diag(eigen_value)
second = L2[:K,:K] - sigma_square*np.eye(K,K)
phi = np.dot(eigen_vectors[:,:K],np.sqrt(second))
'''
Calculate wi with the weights given in the sheet.
'''
wi=(mu + np.dot(phi,np.array([-0.4,-0.2,0,0.2,0.4])).real)
'''
Plot the wi.
'''
plt.suptitle("Statistical Shape Model , K = 5 and W = [-0.4,-0.2,0,0.2,0.4]")
plt.plot(wi[:56],wi[56:])
plt.show()
plt.pause(2)
plt.close()

'''
QUESTION 3
'''

'''
Read the test shape
'''
test_shape = np.loadtxt('data/hands_aligned_test.txt',skiprows=1)
h = np.array([0,0,0,0,0]) #Assume random h
h_org = np.copy(h)
count_iter=0
while(True):
    w = mu + np.dot(phi,h) # w
    w=w.real
    '''
    Shape changes to calculate psuedo inverse equation components
    '''
    w_xy = np.array([np.split(w,2)[0],np.split(w,2)[1]]).T
    A= []
    for point in w_xy:
        A = A + [[point[0], point[1], 0, 0, 1, 0]]
        A = A + [[0, 0, point[0], point[1], 0, 1]]
    A = np.array(A)
    test_split = np.split(test_shape, 2)
    y = np.array([test_split[0], test_split[1]]).T.flatten()
    '''
    Calculate transformation vector psi
    '''
    psi = np.dot(np.dot(np.linalg.inv(np.dot(A.T, A)), A.T), y)
    psi_1 = psi[:4].reshape(2,2)
    psi_2 = psi[4:]
    '''
    Transform points according to psi
    '''
    w = np.dot(A,psi).real
    '''
    Caclulate parts of the h update equation
    '''
    second_term = y - (np.dot(A,psi))
    second_term=np.array(np.split(second_term, 56)).T.flatten()
    second_sum = np.zeros((5,1))
    for n in range(56):
        phi_n=np.array([phi[n, :], phi[n+ 56, :]])
        first_n = np.dot(psi_1,phi_n).T.real
        second_n = np.array([second_term[n],second_term[n+56]]).reshape(-1,1)
        second_sum = second_sum + np.dot(first_n,second_n)

    sum_first = np.zeros((5,5))
    for n in range(56):
        phi_n = np.array([phi[n, :], phi[n + 56, :]])
        sum_first += phi_n.T.dot(psi_1.T).dot(psi_1).dot(phi_n).real
    sum_first = sum_first+ sigma_square*np.eye(5,5)
    sum_first_inv = np.linalg.inv(sum_first)
    '''
    Evalulate h
    '''
    h = np.dot(sum_first_inv,second_sum).real.flatten()

    print(h)
    '''
    Check if h has converged
    '''
    if count_iter>15:
        plt.plot(np.array(np.split(w, 56))[:, 0], np.array(np.split(w, 56))[:, 1], c='r')
        plt.show()
        break
    else:
        h_org=h
    count_iter+=1
    '''
    Calculate RMS error
    '''
    RMS = np.sqrt(np.sum(np.square(w-y)))
    plt.suptitle("Inference Step - RMS" + str(RMS))
    plt.plot(np.array(np.split(w,56))[:,0],np.array(np.split(w,56))[:,1],c='r')
    plt.plot(test_shape[:56],test_shape[56:])
    plt.pause(0.2)
    plt.cla()