# -*- coding: utf-8 -*-
"""
Created on Wed Jan 22 15:32:30 2020

Script to demonstrate small angles approximation sin(theta) = theta

@author: https://www.linkedin.com/in/gabrieltribeiro/
"""

import math
import numpy as np
import matplotlib.pyplot as plt

theta = np.linspace(0, 45, 46)
y_sin = np.sin(theta*math.pi/180)

err_sin = abs(100*(y_sin[1:] - theta[1:]*math.pi/180) / y_sin[1:])

fig, axes = plt.subplots(nrows=1, ncols=2, figsize=(10,5))

axes[0].plot(theta, theta*math.pi/180)
axes[0].plot(theta, y_sin)
axes[0].legend(['theta', 'sin(theta)'])
axes[0].set_xlabel('Angle (deg)', style='italic', fontsize='large')
axes[0].set_ylabel('Function value', style='italic', fontsize='large')
axes[0].set_xticks(ticks=np.linspace(0, 45, 10))
axes[0].grid()

axes[1].bar(theta[1:16], err_sin[0:15])
axes[1].hlines(y=1, xmin=theta[1], xmax=theta[16], linestyles='dashed')
axes[1].set_xlabel('Angle (deg)', style='italic', fontsize='large')
axes[1].set_ylabel('% of error', style='italic', fontsize='large')
axes[1].set_xticks(ticks=np.linspace(0, 16, 15))


"""
randmat.py:  Random matrix methods
"""
from __future__ import division

import numpy as np

def rand_rot_invariant_mat(nz1,nz0,cond_num=10,is_complex=False):
    """
    Creates a rotationally invariant random matrix.

    A rotationally invariant matrix is of the form :math:`A=USV^*` where
    :math:`U` and :math:`V` are uniformly distributed on the unitaries
    (for complex matrices) and orthogonal matrices (for real matrices).
    The singular values :math:`S=diag(s_1,\ldots,s_r)` are logarithmically
    spaced from :math:`1/cond_num` to 1.  The singular values are then
    scaled to have an average magnitude squared of one.
    """

    # Generate a random Gaussian matrix
    if is_complex:
        A = np.random.randn(nz1,nz0) + 1j*np.random.randn(nz1,nz0)
    else:
        A = np.random.randn(nz1,nz0)

    # Take the SVD
    U,s,V = np.linalg.svd(A,full_matrices=0)

    # Reset the singular values
    r = len(s)
    s = np.logspace(-np.log10(cond_num),0,r)
    s = s / np.sqrt(np.mean(s**2))

    # Rebuild the matrix
    A = (U*s[None,:]).dot(V)

    return A


import numpy as np
import numpy.random

rng = np.random.default_rng()

def monte_carlo_definite_integral(f, low, high, n):
    xs = rng.uniform(low, high, n)
    f_v = np.vectorize(f)
    ys = f_v(xs)
    mean = np.mean(ys)
    std = np.std(ys, ddof=1)
    range_size = high - low
    return range_size*mean, range_size*std/np.sqrt(n-1)


import numpy as np

'''
This method is used to get the KL-Transform of the image
The function requires an image as argument
It returns the transform, the transformation matrix, and the eigen values
in descending order.
'''
def kl_transform(img):
  '''Finding the eigen vector and value of covariance matrix of the image'''
  eigenVal, eigenVec = np.linalg.eig(np.cov(img))

  '''Sorting the eigen vector in descending order of thier values'''
  idx = eigenVal.argsort()[::-1]
  eigenVal = eigenVal[idx]
  eigenVec = eigenVec[:,idx]

  '''Finding the transform'''
  # klt = np.dot(eigenVec[:100,:512], img)
  klt = np.dot(eigenVec, img)

  return klt, eigenVal, eigenVec


from sympy import symbols, RR, poly

class InferenceEngine():
    def __init__(self):
        return

    def intersection(self, poly1, poly2):
        '''
        input:
            poly1 = poly
            poly2 = poly
        return:
            float
        '''
        inters = poly1 - poly2
        y = inters.root(0)
        return y

    def findMax(self, poly_subject, poly_target, interval_f):
        '''
        input:
            poly_subject = poly
            poly_target = poly
            interval_f = (start, end)

        output:
            poly = list[poly]
            interval_f = list[(start, end)]
        '''
        polyD = poly_subject.degree()
        targetD = poly_target.degree()
        if poly_target(interval_f[0]) == 0 and poly_target(interval_f[1]) == 0:
            # poly_target = 0
            return [poly_subject], [interval_f]
        if polyD == 0 and targetD == 0:
            # poly_subject = horizontal line & poly_target = horizontal line
            subject = poly_subject(interval_f[1])
            target = poly_target(interval_f[1])
            if target > subject:
                # poly_target above
                return [poly_target], [interval_f]
            else:
                # poly_subject above
                return [poly_subject], [interval_f]
        elif polyD == 1:
            # y = mx + c. m != 0
            # gradient poly_subject != 0
            if targetD == 1:
                # y = mx + c. m != 0
                # gradient poly_target != 0
                subject_start = poly_subject(interval_f[0])
                target_start = poly_target(interval_f[0])
                subject_end = poly_subject(interval_f[1])
                target_end = poly_target(interval_f[1])
                if subject_start <= target_start:
                    # poly_subject bottom in start
                    if subject_end <= target_end:
                        # poly_subject bottom in end
                        # not intersect, poly_target above
                        return [poly_target], [interval_f]
                    else:
                        # poly_subject above in end
                        # intersect with poly_target first
                        inter = self.intersection(poly_subject, poly_target)
                        poly_res = [poly_target, poly_subject]
                        intervals_res = [(interval_f[0], inter), (inter, interval_f[1])]
                        return poly_res, intervals_res
                elif subject_start > target_start:
                    # poly_subject above in start
                    if subject_end < target_end:
                        # poly_subject bottom in end
                        # intersect with poly_subject first
                        inter = self.intersection(poly_subject, poly_target)
                        poly_res = [poly_subject, poly_target]
                        intervals_res = [(interval_f[0], inter), (inter, interval_f[1])]
                        return poly_res, intervals_res
                    else:
                        # poly_subject above in end
                        # no intersect
                        return [poly_subject], [interval_f]
            elif targetD == 0:
                # gradient poly_target == 0
                # horizontal line
                if poly_subject(interval_f[0]) > poly_target(interval_f[0]):
                    # poly_subject above in start
                    if poly_subject(interval_f[1]) < poly_target(interval_f[1]):
                        # poly_subject bottom in end'
                        # intersect with poly_subject first
                        inter = self.intersection(poly_subject, poly_target)
                        poly_res = [poly_subject, poly_target]
                        interval_res = [(interval_f[0], inter), (inter, interval_f[1])]
                        return poly_res, interval_res
                    else:
                        # poly_subject above in end
                        # no intersect
                        return [poly_subject], [interval_f]
                elif poly_subject(interval_f[0]) <= poly_target(interval_f[0]):
                    # poly_subject bottom in start
                    if poly_subject(interval_f[1]) > poly_target(interval_f[1]):
                        # poly_subject above in end
                        # intersect with poly_target first
                        inter = self.intersection(poly_subject, poly_target)
                        poly_res = [poly_target, poly_subject]
                        interval_res = [(interval_f[0], inter), (inter, interval_f[1])]
                        return poly_res, interval_res
                    else:
                        # poly_subject bottom in end
                        return [poly_target], [interval_f]
        elif polyD == 0 and targetD == 1:
            # poly_subject gradient = 0 / horizontal line
            # poly_ target gredient != 0
            if poly_target(interval_f[0]) < poly_subject(interval_f[0]):
                # poly_target bottom in start
                if poly_target(interval_f[1]) > poly_subject(interval_f[1]):
                    # poly_target above in end
                    # intersect with poly_subject first
                    inter = self.intersection(poly_subject, poly_target)
                    poly_res = [poly_subject, poly_target]
                    interval_res = [(interval_f[0], inter), (inter, interval_f[1])]
                    return poly_res, interval_res
                else:
                    # poly_target bottom in end
                    return [poly_subject], [interval_f]
            else:
                # poly_target above in start
                if poly_target(interval_f[1]) < poly_subject(interval_f[1]):
                    # poly_target bottom in end
                    # intersect with poly_target first
                    inter = self.intersection(poly_subject, poly_target)
                    poly_res = [poly_target, poly_subject]
                    interval_res = [(interval_f[0], inter), (inter, interval_f[1])]
                    return poly_res, interval_res
                else:
                    # poly_target above in end
                    return [poly_target], [interval_f]

    def supremum(self, members, intervals, sistem_interval):
        '''
        input:
            members = list[[poly]]
            intervals = list[[(start, end)]]
            sistem_inter
        return
            members as list[poly -> variable z]
            intervals as list[(start, end)]
        '''
        z = symbols("z")
        # initial supremum
        members_supremum = [poly(0.0000000000000000000000000001, z, domain=RR)]
        intervals_supremum = [sistem_interval]
        for member_role, interval_role in zip(members, intervals):
            for member, interval_m in zip(member_role, interval_role):
                for index_supremum in range(len(members_supremum)):
                    member_supremum = members_supremum[index_supremum]
                    interval_supremum = intervals_supremum[index_supremum]
                    if interval_supremum[0] == interval_m[0]:
                        # interval supremum start == interval member start
                        '''
                        analogi
                        ---------------------- supremum
                        -------     member => K1
                        --------    supremum
                        --------    member => K2
                        ----- supremum
                        -------------- member => K3
                        K2 identik K3
                        '''
                        # K1
                        if interval_supremum[1] > interval_m[1]:
                            # interval_supremum more than interval member
                            supremum_temp = member_supremum
                            interval_temp = (interval_supremum[0], interval_m[1])
                            poly_list, interval_list = self.findMax(supremum_temp, member, interval_temp)
                            if len(poly_list) == 1:
                                # poly_list max no intersect
                                if poly_list[0](interval_supremum[0]) == supremum_temp(interval_supremum[0]):
                                    # no update poly_supremum
                                    continue
                                else:
                                    # update poly_supremum
                                    index_hasil = len(poly_list)-1
                                    while index_hasil >= 0:
                                        poly_hasil = poly_list[index_hasil]
                                        interval_hasil = interval_list[index_hasil]

                                        # insert update supremum
                                        members_supremum.insert(index_supremum, poly_hasil)
                                        intervals_supremum.insert(index_supremum, interval_hasil)

                                        # update interval supremum
                                        list_tuple = list(intervals_supremum[index_supremum+1])
                                        list_tuple[0]  = interval_hasil[1]
                                        intervals_supremum[index_supremum+1] = tuple(list_tuple)
                                        index_hasil -= 1
                            else:
                                # poly_list max with intersect
                                index_hasil = len(poly_list)-1
                                while index_hasil >= 0:
                                    poly_hasil = poly_list[index_hasil]
                                    interval_hasil = interval_list[index_hasil]

                                    # insert update supremum
                                    members_supremum.insert(index_supremum, poly_hasil)
                                    intervals_supremum.insert(index_supremum, interval_hasil)

                                    # update interval supremum
                                    list_tuple = list(intervals_supremum[index_supremum+1])
                                    list_tuple[0]  = interval_hasil[1]
                                    intervals_supremum[index_supremum+1] = tuple(list_tuple)
                                    index_hasil -= 1
                        # K2 K3
                        elif interval_supremum[1] <= interval_m[1]:
                            # poly_supremum less than member
                            supremum_temp = member_supremum
                            interval_temp = (interval_supremum[0], interval_supremum[1])
                            poly_list, interval_list = self.findMax(supremum_temp, member, interval_temp)
                            if len(poly_list) == 1:
                                # poly_list max no intersect
                                if poly_list[0](interval_supremum[0]) == supremum_temp(interval_supremum[0]):
                                    # no update poly_supremum
                                    continue
                                else:
                                    # update poly_supremum
                                    index_hasil = len(poly_list)-1
                                    while index_hasil >= 0:
                                        poly_hasil = poly_list[index_hasil]
                                        interval_hasil = interval_list[index_hasil]

                                        # insert update supremum
                                        members_supremum.insert(index_supremum, poly_hasil)
                                        intervals_supremum.insert(index_supremum, interval_hasil)

                                        # update interval supremum
                                        del members_supremum[index_supremum+1]
                                        del intervals_supremum[index_supremum+1]
                                        index_hasil -= 1
                            else:
                                # poly_list max with intersect
                                index_hasil = len(poly_list)-1
                                while index_hasil >= 0:
                                    poly_hasil = poly_list[index_hasil]
                                    interval_hasil = interval_list[index_hasil]

                                    # insert update supremum
                                    members_supremum.insert(index_supremum, poly_hasil)
                                    intervals_supremum.insert(index_supremum, interval_hasil)
                                    index_hasil -= 1

                                # update supremum
                                del members_supremum[index_supremum+2]
                                del intervals_supremum[index_supremum+2]
                    elif interval_supremum[0] < interval_m[0]:
                        # supremum less than member in start
                        '''
                        analogi kemungkinan
                        -------------       supremum
                            -----           member => K1
                        -------------       supremum
                            -------------   member
                        -------------       supremum
                                        --- member => K3
                        '''
                        # K3
                        if interval_supremum[1] <= interval_m[0]:
                            # no update supremum
                            continue
                        # K1
                        elif interval_supremum[1] > interval_m[1]:
                            # supremum more than member in end
                            supremum_temp = member_supremum
                            interval_temp = (interval_m[0], interval_m[1])
                            poly_list, interval_list = self.findMax(supremum_temp, member, interval_temp)
                            if len(poly_list) == 1:
                                # poly_list max no intersect
                                if poly_list[0](interval_supremum[0]) == supremum_temp(interval_supremum[0]):
                                    # no update poly_supremum
                                    continue
                                else:
                                    # update poly_supremum
                                    index_hasil = len(poly_list)-1
                                    while index_hasil >= 0:
                                        poly_hasil = poly_list[index_hasil]
                                        interval_hasil = interval_list[index_hasil]

                                        # insert update supremum
                                        members_supremum.insert(index_supremum, poly_hasil)
                                        intervals_supremum.insert(index_supremum, interval_hasil)

                                        # update interval supremum
                                        list_tuple = list(intervals_supremum[index_supremum+1])
                                        awal = list_tuple[0]
                                        list_tuple[0]  = interval_hasil[1]
                                        intervals_supremum[index_supremum+1] = tuple(list_tuple)
                                        members_supremum.insert(index_supremum, members_supremum[index_supremum+1])
                                        intervals_supremum.insert(index_supremum, (awal, interval_hasil[0]))
                                        index_hasil -= 1
                            else:
                                # poly_list max with intersect
                                index_hasil = len(poly_list)-1
                                while index_hasil >= 0:
                                    poly_hasil = poly_list[index_hasil]
                                    interval_hasil = interval_list[index_hasil]

                                    # insert update supremum
                                    members_supremum.insert(index_supremum, poly_hasil)
                                    intervals_supremum.insert(index_supremum, interval_hasil)

                                    # update interval supremum
                                    list_tuple = list(intervals_supremum[index_supremum+1])
                                    awal = list_tuple[0]
                                    list_tuple[0]  = interval_hasil[1]
                                    intervals_supremum[index_supremum+1] = tuple(list_tuple)
                                    if index_hasil == 0:
                                        # update supremum first in
                                        members_supremum.insert(index_supremum, members_supremum[index_supremum+1])
                                        intervals_supremum.insert(index_supremum, (awal, interval_hasil[0]))
                                    index_hasil -= 1
                        # K2
                        elif interval_supremum[1] <= interval_m[1]:
                            # interval_supremum less than interval member
                            supremum_temp = member_supremum
                            interval_temp = (interval_m[0], interval_supremum[1])
                            poly_list, interval_list = self.findMax(supremum_temp, member, interval_temp)
                            if len(poly_list) == 1:
                                # poly_list max no intersect
                                if poly_list[0](interval_supremum[0]) == supremum_temp(interval_supremum[0]):
                                    # no update poly_supremum
                                    continue
                                else:
                                    # update poly_supremum
                                    index_hasil = len(poly_list)-1
                                    while index_hasil >= 0:
                                        poly_hasil = poly_list[index_hasil]
                                        interval_hasil = interval_list[index_hasil]

                                        # insert update supremum
                                        members_supremum.insert(index_supremum+1, poly_hasil)
                                        intervals_supremum.insert(index_supremum+1, interval_hasil)

                                        # update interval supremum
                                        list_tuple = list(intervals_supremum[index_supremum])
                                        list_tuple[1]  = interval_hasil[0]
                                        intervals_supremum[index_supremum] = tuple(list_tuple)
                                        index_hasil -= 1
                            else:
                                # poly_list max with intersect
                                index_hasil = len(poly_list)-1
                                while index_hasil >= 0:
                                    poly_hasil = poly_list[index_hasil]
                                    interval_hasil = interval_list[index_hasil]

                                    # insert update supremum
                                    members_supremum.insert(index_supremum+1, poly_hasil)
                                    intervals_supremum.insert(index_supremum+1, interval_hasil)

                                    if index_hasil == 0:
                                        # update supremum first in
                                        list_tuple = list(intervals_supremum[index_supremum])
                                        list_tuple[1]  = interval_hasil[0]
                                        intervals_supremum[index_supremum] = tuple(list_tuple)
                                    index_hasil -= 1

                    elif interval_supremum[0] > interval_m[0]:
                        # interval_supremum more than interval member in start
                        '''
                        analogi
                            --------------  supremum
                        ------------        member => K1
                            --------------  supremum
                        --                  member => K2
                            --------------  supremum
                        ------------------- member => K3
                        '''
                        # K2
                        if interval_m[1] <= interval_supremum[0]:
                            # no update supremum
                            continue
                        # K1
                        elif interval_supremum[1] > interval_m[1]:
                            # supremum more than member in end
                            supremum_temp = member_supremum
                            interval_temp = (interval_supremum[0], interval_m[1])
                            poly_list, interval_list = self.findMax(supremum_temp, member, interval_temp)
                            if len(poly_list) == 1:
                                # poly_list no intersect
                                if poly_list[0](interval_supremum[0]) == supremum_temp(interval_supremum[0]):
                                    # no update poly_supremum
                                    continue
                                else:
                                    index_hasil = len(poly_list)-1
                                    while index_hasil >= 0:
                                        poly_hasil = poly_list[index_hasil]
                                        interval_hasil = interval_list[index_hasil]

                                        # insert update supremum
                                        members_supremum.insert(index_supremum, poly_hasil)
                                        intervals_supremum.insert(index_supremum, interval_hasil)

                                        # update interval supremum
                                        list_tuple = list(intervals_supremum[index_supremum+1])
                                        list_tuple[0]  = interval_hasil[1]
                                        intervals_supremum[index_supremum+1] = tuple(list_tuple)
                                        index_hasil -= 1
                            else:
                                # poly_list max with intersect
                                index_hasil = len(poly_list)-1
                                while index_hasil >= 0:
                                    poly_hasil = poly_list[index_hasil]
                                    interval_hasil = interval_list[index_hasil]

                                    # insert update supremum
                                    members_supremum.insert(index_supremum, poly_hasil)
                                    intervals_supremum.insert(index_supremum, interval_hasil)

                                    if index_hasil == 1:
                                        # update supremum last in
                                        list_tuple = list(intervals_supremum[index_supremum+1])
                                        list_tuple[0]  = interval_hasil[1]
                                        intervals_supremum[index_supremum+1] = tuple(list_tuple)
                                    index_hasil -= 1
                        # K3
                        elif interval_supremum[1] <= interval_m[1]:
                            # interval supremum less than member
                            supremum_temp = member_supremum
                            interval_temp = (interval_supremum[0], interval_supremum[1])
                            poly_list, interval_list = self.findMax(supremum_temp, member, interval_temp)
                            if len(poly_list) == 1:
                                # poly_list no intersect
                                if poly_list[0](interval_supremum[0]) == supremum_temp(interval_supremum[0]):
                                    # no update supremum
                                    continue
                                else:
                                    # update poly_supremum
                                    index_hasil = len(poly_list)-1
                                    while index_hasil >= 0:
                                        poly_hasil = poly_list[index_hasil]
                                        interval_hasil = interval_list[index_hasil]

                                        # insert update supremum
                                        members_supremum.insert(index_supremum, poly_hasil)
                                        intervals_supremum.insert(index_supremum, interval_hasil)
                                        index_hasil -= 1

                                    # update supremum
                                    del members_supremum[index_supremum+1]
                                    del intervals_supremum[index_supremum+1]
                            else:
                                # poly_list max with intersect
                                index_hasil = len(poly_list)-1
                                while index_hasil >= 0:
                                    poly_hasil = poly_list[index_hasil]
                                    interval_hasil = interval_list[index_hasil]

                                    # insert update supremum
                                    members_supremum.insert(index_supremum, poly_hasil)
                                    intervals_supremum.insert(index_supremum, interval_hasil)
                                    index_hasil -= 1

                                # update supremum
                                del members_supremum[index_supremum+2]
                                del intervals_supremum[index_supremum+2]

        # delete with interval start == interval end
        length = len(members_supremum)
        index_del = 0
        while index_del < length:
            member = members_supremum[index_del]
            interval_y = intervals_supremum[index_del]
            index_del2 = 0
            while index_del2 < length:
                interval2 = intervals_supremum[index_del2]
                if index_del == index_del2:
                    index_del2 += 1
                elif interval2[0] == interval_y[0] and interval2[1] == interval_y[1]:
                    del members_supremum[index_del2]
                    del intervals_supremum[index_del2]
                    length -= 1
                else:
                    index_del2 += 1
            if interval_y[0] >= interval_y[1]:
                del members_supremum[index_del]
                del intervals_supremum[index_del]
                length -= 1
            elif member(interval_y[0]) < 0.000000000000000000000000001:
                del members_supremum[index_del]
                del intervals_supremum[index_del]
                length -= 1
            else:
                index_del += 1

        return members_supremum, intervals_supremum


import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import pymc3 as pm

data = pd.read_csv('Howell1.csv', delimiter=';')

#d2 = data[data['age'] >= 18]
d2 = data

height = d2['height']
weight = d2['weight']

#optionally downsample data
#ind = np.random.permutation(height.shape[0])
#height = np.array(height)[ind]
#weight = np.array(weight)[ind]

weightBar = np.mean(weight)
'''
#----------------------------------#
#	linear fit		   #
#----------------------------------#
with pm.Model() as normal_approximation:
	alpha = pm.Normal('alpha', mu=178, sigma=20)
	beta = pm.Lognormal('beta', mu=0, sigma=1)

	s = pm.Uniform('sigma', 0, 50)

	h = pm.Normal('height', mu=alpha + beta*(weight-weightBar), sigma=s, observed=height)

	mean_q = pm.find_MAP()

	hessian = pm.find_hessian(mean_q, vars=[alpha, beta, s])

	covariance_q = np.linalg.inv(hessian)

	std_q = np.sqrt(np.diag(covariance_q))

#draw samples from posterior
samples = np.random.multivariate_normal([mean_q['alpha'], mean_q['beta'], mean_q['sigma']], covariance_q, size=10000)

fig = plt.figure()
ax = fig.add_subplot(111)
ax.scatter(weight, height, color='w', edgecolor='C0', alpha=0.5)

a_map, b_map, s_map = np.mean(samples,0)

ww = np.arange(0, 65)
ax.plot(ww, a_map + b_map * (ww-weightBar), color='k')

mus = samples[:, 0 ] + samples[:, 1] * np.tile(ww.reshape(1,ww.shape[0]) - weightBar, (samples.shape[0],1)).T

ci89 = np.percentile(mus, [5.5, 94.5], axis=1).T

ax.plot(ww, ci89[:,0], color='k', linestyle='--', alpha=0.7)
ax.plot(ww, ci89[:,1], color='k', linestyle='--', alpha=0.7)

ax.fill_between(ww, ci89[:,0], ci89[:,1], color='k', alpha=0.2)

posteriorDraws = np.random.normal(mus, samples[:,2])
posteriorCi89 = np.percentile(posteriorDraws, [5.5, 94.5], axis=1).T

ax.fill_between(ww, posteriorCi89[:,0], posteriorCi89[:,1], color='k', alpha=0.1)
'''

#----------------------------------#
#	polynomial fit		   #
#----------------------------------#
from sklearn.preprocessing import StandardScaler
weight = np.array(weight)
height = np.array(height)

mm = np.mean(weight)
ss = np.std(weight, ddof=1)

weightS = (weight - mm) / ss
#preproc = StandardScaler()
#weightS = preproc.fit_transform(weight.reshape(-1,1))
#this uses biased variance estimation

weightS2 = weightS ** 2

with pm.Model() as normal_approximation:
	alpha = pm.Normal('alpha', mu=178, sigma=20)
	beta1 = pm.Lognormal('beta1', mu=0, sigma=1)
	beta2 = pm.Normal('beta2', mu=0, sigma=1)

	s = pm.Uniform('sigma', 0, 50)

	h = pm.Normal('height', mu=alpha + beta1* weightS + beta2 * weightS2, sigma=s, observed=height)

	mean_q = pm.find_MAP()

	hessian = pm.find_hessian(mean_q, vars=[alpha, beta1, beta2, s])

	covariance_q = np.linalg.inv(hessian)

	std_q = np.sqrt(np.diag(covariance_q))

plt.show()


import numpy as np

def simulate_topography(numCraters = 10, # number of craters
                        widthMax   = 1.0,     # maximal width of Gaussian crater
                        heightMin  = -1.0,    # maximal depth of craters / valleys
                        heightMax  = 2.0):     # maximal height of hills / mountains

    # 1-D Gaussian
    np.random.seed()   # for reproducability
    def gaussian(x, A, mu, sigma):
        return A * np.exp(-(x - mu)**2 / 2.0 / sigma**2)

    # 1-D Gaussian (same thing using lambda)
    #gaussian = lambda x, A, mu, sigma: A * np.exp(-(x - mu)**2 / 2. / sigma**2)

    # Create an array of linearly spaced x values
    xArr = np.linspace(0, 10, 500)  # km

    # Create an array of initially flat landscape (aka filled with 0's)
    yArr = np.zeros_like(xArr)

    # Add craters / mountains to landscape
    for _ in range(numCraters):

        # Amplitude between heightMin and heightMax
        A = np.random.rand() * (heightMax - heightMin) + heightMin

        # Center location of the crater
        center = np.random.rand() * xArr.max()

        # Width of the crater
        sigma = np.random.rand() * widthMax

        # Add crater to landscape!
        yArr += gaussian(xArr, A=A, mu=center, sigma=sigma)

    return xArr, yArr


"""Create plots of signals generated by chirp() and sweep_poly()."""

import numpy as np
from scipy.signal.waveforms import chirp, sweep_poly
from numpy import poly1d

from pylab import figure, plot, show, xlabel, ylabel, subplot, grid, title, \
                    yscale, savefig, clf

FIG_SIZE = (7.5, 3.75)

def make_linear(f0, t1, f1, filename=None, fig_size=FIG_SIZE):
    t = np.linspace(0, t1, 5001)
    w = chirp(t, f0=f0, f1=f1, t1=t1, method='linear')

    figure(1, figsize=fig_size)
    clf()

    subplot(2,1,1)
    plot(t, w)
    tstr = "Linear Chirp, f(0)=%g, f(%g)=%g" % (f0, t1, f1)
    title(tstr)

    subplot(2,1,2)
    plot(t, f0 + (f1-f0)*t/t1, 'r')
    grid(True)
    ylabel('Frequency (Hz)')
    xlabel('time (sec)')
    if filename is None:
        show()
    else:
        savefig(filename)

def make_quadratic(f0, t1, f1, filename=None, fig_size=FIG_SIZE):
    t = np.linspace(0, t1, 5001)
    w = chirp(t, f0=f0, f1=f1, t1=t1, method='quadratic')

    figure(1, figsize=fig_size)
    clf()

    subplot(2,1,1)
    plot(t, w)
    tstr = "Quadratic Chirp, f(0)=%g, f(%g)=%g" % (f0, t1, f1)
    title(tstr)

    subplot(2,1,2)
    plot(t, f0 + (f1-f0)*t**2/t1**2, 'r')
    grid(True)
    ylabel('Frequency (Hz)')
    xlabel('time (sec)')
    if filename is None:
        show()
    else:
        savefig(filename)

def make_quadratic_v0false(f0, t1, f1, filename=None, fig_size=FIG_SIZE):
    t = np.linspace(0, t1, 5001)
    w = chirp(t, f0=f0, f1=f1, t1=t1, method='quadratic', vertex_zero=False)

    figure(1, figsize=fig_size)
    clf()

    subplot(2,1,1)
    plot(t, w)
    tstr = "Quadratic Chirp, f(0)=%g, f(%g)=%g (vertex_zero=False)" % (f0, t1, f1)
    title(tstr)

    subplot(2,1,2)
    plot(t, f1 - (f1-f0)*(t1-t)**2/t1**2, 'r')
    grid(True)
    ylabel('Frequency (Hz)')
    xlabel('time (sec)')
    if filename is None:
        show()
    else:
        savefig(filename)

def make_logarithmic(f0, t1, f1, filename=None, fig_size=FIG_SIZE):
    t = np.linspace(0, t1, 5001)
    w = chirp(t, f0=f0, f1=f1, t1=t1, method='logarithmic')

    figure(1, figsize=fig_size)
    clf()

    subplot(2,1,1)
    plot(t, w)
    tstr = "Logarithmic Chirp, f(0)=%g, f(%g)=%g" % (f0, t1, f1)
    title(tstr)

    subplot(2,1,2)
    plot(t, f0 * (f1/f0)**(t/t1), 'r')
    # yscale('log')
    grid(True)
    ylabel('Frequency (Hz)')
    xlabel('time (sec)')
    if filename is None:
        show()
    else:
        savefig(filename)

def make_hyperbolic(f0, t1, f1, filename=None, fig_size=FIG_SIZE):
    t = np.linspace(0, t1, 5001)
    w = chirp(t, f0=f0, f1=f1, t1=t1, method='hyperbolic')

    figure(1, figsize=fig_size)
    clf()

    subplot(2,1,1)
    plot(t, w)
    tstr = "Hyperbolic Chirp, f(0)=%g, f(%g)=%g" % (f0, t1, f1)
    title(tstr)

    subplot(2,1,2)
    plot(t, f0 * f1 * t1 / ((f0 - f1)*t + f1*t1), 'r')
    grid(True)
    ylabel('Frequency (Hz)')
    xlabel('time (sec)')
    if filename is None:
        show()
    else:
        savefig(filename)

def make_sweep_poly(filename=None, fig_size=FIG_SIZE):
    p = poly1d([0.05, -0.75, 2.5, 5.0])

    t = np.linspace(0, t1, 5001)
    w = sweep_poly(t, p)

    figure(1, figsize=fig_size)
    clf()

    subplot(2,1,1)
    plot(t, w)
    tstr = "Sweep Poly, $f(t) = 0.05t^3 - 0.75t^2 + 2.5t + 5$"
    title(tstr)

    subplot(2,1,2)
    plot(t, p(t), 'r')
    grid(True)
    ylabel('Frequency (Hz)')
    xlabel('time (sec)')
    if filename is None:
        show()
    else:
        savefig(filename)

if __name__ == "__main__":
    f0 = 12.5
    t1 = 10.0
    f1 = 2.5
    make_linear(f0, t1, f1, 'chirp_linear.png')
    make_quadratic(f0, t1, f1, 'chirp_quadratic.png')
    make_quadratic_v0false(f0, t1, f1, 'chirp_quadratic_v0false.png')
    make_hyperbolic(f0, t1, f1, 'chirp_hyperbolic.png')
    make_logarithmic(f0, t1, f1, 'chirp_logarithmic.png')

    make_sweep_poly(filename='sweep_poly.png')


"""Weighted Least Squares
"""

import numpy as np
from scipy import stats
import statsmodels.api as sm
import matplotlib.pyplot as plt
from statsmodels.sandbox.regression.predstd import wls_prediction_std
from statsmodels.iolib.table import (SimpleTable, default_txt_fmt)
np.random.seed(1024)

#WLS Estimation
#--------------

#Artificial data: Heteroscedasticity 2 groups
#^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
#Model assumptions:
#
# * Misspecificaion: true model is quadratic, estimate only linear
# * Independent noise/error term
# * Two groups for error variance, low and high variance groups

nsample = 50
x = np.linspace(0, 20, nsample)
X = np.c_[x, (x - 5)**2, np.ones(nsample)]
beta = [0.5, -0.01, 5.]
sig = 0.5
w = np.ones(nsample)
w[nsample * 6/10:] = 3
y_true = np.dot(X, beta)
e = np.random.normal(size=nsample)
y = y_true + sig * w * e
X = X[:,[0,2]]

#WLS knowing the true variance ratio of heteroscedasticity
#^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
mod_wls = sm.WLS(y, X, weights=1./w)
res_wls = mod_wls.fit()
print res_wls.summary()

#OLS vs. WLS
#-----------

# Estimate an OLS model for comparison
res_ols = sm.OLS(y, X).fit()

# Compare the estimated parameters in WLS and OLS
print res_ols.params
print res_wls.params

# Compare the WLS standard errors to  heteroscedasticity corrected OLS standard
# errors:
se = np.vstack([[res_wls.bse], [res_ols.bse], [res_ols.HC0_se], [res_ols.HC1_se], [res_ols.HC2_se], [res_ols.HC3_se]])
se = np.round(se,4)
colnames = ['x1', 'const']
rownames = ['WLS', 'OLS', 'OLS_HC0', 'OLS_HC1', 'OLS_HC3', 'OLS_HC3']
tabl = SimpleTable(se, colnames, rownames, txt_fmt=default_txt_fmt)
print tabl

# Calculate OLS prediction interval
covb = res_ols.cov_params()
prediction_var = res_ols.mse_resid + (X * np.dot(covb,X.T).T).sum(1)
prediction_std = np.sqrt(prediction_var)
tppf = stats.t.ppf(0.975, res_ols.df_resid)

# Draw a plot to compare predicted values in WLS and OLS:
prstd, iv_l, iv_u = wls_prediction_std(res_wls)
plt.figure()
plt.plot(x, y, 'o', x, y_true, 'b-')
plt.plot(x, res_ols.fittedvalues, 'r--')
plt.plot(x, res_ols.fittedvalues + tppf * prediction_std, 'r--')
plt.plot(x, res_ols.fittedvalues - tppf * prediction_std, 'r--')
plt.plot(x, res_wls.fittedvalues, 'g--.')
plt.plot(x, iv_u, 'g--')
plt.plot(x, iv_l, 'g--')
#@savefig wls_ols_0.png
plt.title('blue: true, red: OLS, green: WLS')

# Feasible Weighted Least Squares (2-stage FWLS)
# ----------------------------------------------

resid1 = res_ols.resid[w==1.]
var1 = resid1.var(ddof=int(res_ols.df_model)+1)
resid2 = res_ols.resid[w!=1.]
var2 = resid2.var(ddof=int(res_ols.df_model)+1)
w_est = w.copy()
w_est[w!=1.] = np.sqrt(var2) / np.sqrt(var1)
res_fwls = sm.WLS(y, X, 1./w_est).fit()
print res_fwls.summary()

#..plt.show()

import numpy,from scipy,import statsmodels,from statsmodels


import scipy as sp
import matplotlib.pyplot as plt

# Read file into tables:
table = sp.genfromtxt("flightlog.dat",delimiter="=",comments="C")
xtab=table[:,0]
ytab=table[:,1]
plt.title('Noisy data')
plt.plot(xtab,ytab,"r+")
plt.show()

# Polyfitting 10th order polynomial

coefficients = sp.polyfit(xtab, ytab, 10)
polynomial = sp.poly1d(coefficients)
ysmooth = sp.polyval(polynomial,xtab)

plt.plot(xtab,ysmooth)
plt.plot(xtab,ytab,"r+")
plt.title('Polynomial 10th order')
plt.show()

# Polyfitting 20th order

coefficients = sp.polyfit(xtab, ytab, 20)
polynomial = sp.poly1d(coefficients)
ysmooth = sp.polyval(polynomial,xtab)

plt.plot(xtab,ysmooth)
plt.plot(xtab,ytab,"r+")
plt.title('Polynomial 20th order')
plt.show()

g = open("filtered.log","w")
g.write("C\n")
g.write("C Data smoothed by fitting a 20th order polynomial\n")
g.write("C\n")
for i in range(len(xtab)):
    line = str(xtab[i])+" = "+str(ysmooth[i])+"\n"
    g.write(line)
#    print line
g.close()


import collections
from functools import partial

import jax
import jax.numpy as np

RotParams = collections.namedtuple("Params", ["projection"])

@jax.jit
def compute_projection(X: np.ndarray) -> np.ndarray:
    """Compute PCA projection matrix
    Using SVD, this computes the PCA components for
    a dataset X and computes the projection matrix
    needed to do the PCA decomposition.

    Parameters
    ----------
    X : np.ndarray, (n_samples, n_features)
        the data to calculate to PCA projection matrix

    Returns
    -------
    VT : np.ndarray, (n_features, n_features)
        the projection matrix (V.T) for the PCA decomposition

    Notes
    -----
    Can find the original implementation here:
    https://bit.ly/2EBDV9o
    """

    # center the data
    X = X - np.mean(X, axis=0)

    # Compute SVD
    _, _, VT = np.linalg.svd(X, full_matrices=False, compute_uv=True)

    return VT.T

def init_pca_params(X):

    # compute projection matrix
    R = compute_projection(X)

    return (
        np.dot(X, R),
        RotParams(R),
        partial(forward_transform, R=R),
        partial(inverse_transform, R=R),
    )

@jax.jit
def forward_transform(X, R):
    return np.dot(X, R), np.zeros(X.shape)

@jax.jit
def inverse_transform(X, R):
    return np.dot(X, R.T)


import numpy as np
import matplotlib.pyplot as plt

def f(x):
    return np.sin(np.log(x)/x)

def pf(x):
    return(1/x**2-np.log(x)/x**2)*np.cos(np.log(x)/x)

def krok_newton(x):
    return x - f(x) / pf(x)

xw0 = 0.1
xwk = 2
nxw = 100
xw = np.linspace(xw0,xwk,nxw)

yw = f(xw)
pyw = pf(xw)

fig = plt.figure()
ax = fig.add_subplot(111)
ax.plot([xw[0],xw[-1]],[0,0], '-k')
ax.plot(xw, yw, "-b")
# ax.plot(xw, pyw, 'or')

x0 = 1.6

x = x0
dok = 1e-10
jeszcze = True
licz = 0
while jeszcze:
    licz += 1
    x = krok_newton(x)
    if abs(f(x)) < dok:
        jeszcze = False

ax.plot(x, 0, 'xr', x, f(x), 'xr')
plt.show()


from autograd.blocks.block import SimpleBlock
import numpy as np

class sin(SimpleBlock):
    """
    vectorized sinus function on vectors
    """

    def data_fn(self, args):
        new_data = np.sin(args.data)
        return(new_data)

    def gradient_fn(self, args):
        grad = np.cos(args.data)
        return(grad)

class cos(SimpleBlock):
    """
    vectorized cosine function on vectors
    """
    def data_fn(self, args):
        new_data = np.cos(args.data)
        return(new_data)

    def gradient_fn(self, args):
        grad = -np.sin(args.data)
        return(grad)

class tan(SimpleBlock):
    """
    vectorized cosine function on vectors
	"""
    def data_fn(self, args):
        new_data = np.tan(args.data)
        return(new_data)

    def gradient_fn(self, args):
        grad = 1/np.cos(args.data)**2
        return(grad)

class arcsin(SimpleBlock):
    """
    vectorized arcsin function on vectors
    """

    def data_fn(self, args):
        new_data = np.arcsin(args.data)
        return(new_data)

    def gradient_fn(self, args):
        grad = 1/(np.sqrt(1 - args.data**2))
        return(grad)

class arccos(SimpleBlock):
    """
    vectorized arcsin function on vectors
    """

    def data_fn(self, args):
        new_data = np.arccos(args.data)
        return(new_data)

    def gradient_fn(self, args):
        grad = -1/(np.sqrt(1 - args.data**2))
        return(grad)

class arctan(SimpleBlock):
    """
    vectorized arcsin function on vectors
    """

    def data_fn(self, args):
        new_data = np.arctan(args.data)
        return(new_data)

    def gradient_fn(self, args):
        grad = 1/(1 + args.data**2)
        return(grad)


# Fix paths for imports to work in unit tests ----------------

if __name__ == "__main__":

    from _fix_paths import fix_paths
    fix_paths()

# ------------------------------------------------------------

# Load libraries ---------------------------------------------

import numpy as np

# ------------------------------------------------------------

def find_optimal_bid(theta_0, theta_1, rpc, average_cpc_diff):
    """
    Find the optimal bid assuming the logistic log function as the bid --> click
    probability relation. The following objective function is maximized:

    .. math::
        \\frac{1}{1 + e^{-theta_0 - theta_1 \\ln(bid)}} (rpc - bid + average\_cpc\_diff)

    :param theta_0: Theta_0 parameter of the logistic curve.
    :param theta_1: Theta_1 parameter of the logistic curve.
    :param rpc: Revenue per click.
    :param average_cpc_diff: Estimated difference between the bid and the actual
        cost per click.
    :return: The optimal bid.
    :rtype: float
    """

    u = max(min(theta_0, -0.001), -1000000)
    v = max(min(theta_1, 1000000), 0.0001)
    a = min(max(rpc, 0.001), 1000000000)
    d = min(max(average_cpc_diff, 0.0), 1000000)

    # Define the derivative of the profit function

    df = lambda b: -(1.0 / (1 + np.exp(-u - v * np.log(b)))) \
        + ((a - b + d) * np.exp(-u - v * np.log(b)) * v) / (b * (1 + np.exp(-u - v * np.log(b)))**2)

    # Find first positive value starting from very big bids

    b = 1000000000.0

    while df(b) < 0 and b > 0.01:
        b /= 2

    if b <= 0.01:

        return 0.01

    else:

        # Using bisection to find zero

        b_low = b
        b_high = b * 2

        eps = 0.01

        while b_high - b_low > eps:
            b_avg = (b_low + b_high) / 2

            if df(b_avg) < 0:
                b_high = b_avg
            else:
                b_low = b_avg

        return b_low

if __name__ == "__main__":

    #print(find_optimal_bid(0, 0, 0, 0))

    print(find_optimal_bid(-0.6392939760285917, -1.4251676868313812, 12.961876832844576, 1.213751269647543))


import numpy as np

from .base import Base
from .activation import Softmax

class BCELoss(Base):
    """
    Class that implements the Binary Cross Entropy loss function.

    Given a target math:`y` and an estimate math:`\hat{y}` the
    Binary Cross Entropy loss can be written as:

    .. math::
        \begin{aligned}
            l_n & = \left[ y_n \cdot \log \hat{y_n} + (1 - y_n) \cdot \log (1 - \hat{y_n}) \right], \\
            L(\hat{y}, y) = \frac{\sum_{i=1}^{N}l_i}{N},
        \end{aligned}

    References
    ----------
    .. [1] Wikipedia - Cross entropy:
       https://en.wikipedia.org/wiki/Cross_entropy
    """
    def __init__(self):
        self.type = 'BCELoss'
        self.eps = 1e-15

    def forward(self, Y_hat, Y):
        """
        Computes the forward propagation.

        Parameters
        ----------
        Y_hat : numpy.array
            Array containing the predictions.
        Y : numpy.array
            Array with the real labels.

        Returns
        -------
        Numpy.arry containing the cost.
        """
        self.Y = Y
        self.Y_hat = Y_hat

        m = len(self.Y)

        sub1 = np.dot(self.Y, np.log(self.Y_hat + self.eps).T)
        sub2 = np.dot(1 - self.Y, np.log(1 + self.eps - self.Y_hat).T)

        loss = - (1./m) * (sub1 + sub2)

        return np.squeeze(loss)

    def backward(self):
        """
        Computes the backward propagation.

        Returns
        -------
        grad : numpy.array
            Array containg the gradients of the weights.
        """
        neg = np.divide(self.Y, self.Y_hat + self.eps)
        pos = np.divide(1 - self.Y, 1 + self.eps - self.Y_hat)
        grad = - (neg - pos)

        return grad

class MSELoss(Base):
    """
    Class that implements the Mean Squared Error loss function.

    Given a target math:`y` and an estimate math:`\hat{y}` the
    mean squared error can be written as:

    .. math::
        \begin{aligned}
            l_n & = (y_n - \hat{y_n})^2, \\
            L(\hat{y}, y) = \frac{\sum_{i=1}^{N}l_i}{N},
        \end{aligned}

    References
    ----------
    .. [1] Wikipedia - Mean squared error:
       https://en.wikipedia.org/wiki/Mean_squared_error
    """
    def __init__(self):
        self.type = 'MSELoss'

    def forward(self, Y_hat, Y):
        """
        Computes the forward propagation.

        Parameters
        ----------
        Y_hat : numpy.array
            Array containing the predictions.
        Y : numpy.array
            Array with the real labels.

        Returns
        -------
        Numpy.arry containing the cost.
        """
        self.Y = Y
        self.Y_hat = Y_hat

        se = np.power(self.Y - self.Y_hat, 2)
        mse = np.mean(se)

        return np.squeeze(mse)

    def backward(self):
        """
        Computes the backward propagation.

        Returns
        -------
        grad : numpy.array
            Array containg the gradients of the weights.
        """
        grad = - 2 * (self.Y - self.Y_hat)

        return grad

class MAELoss(Base):
    """
    Class that implements the Mean Absolute Error loss function.

    Given a target math:`y` and an estimate math:`\hat{y}` the
    mean absolute error can be written as:

    .. math::
        \begin{aligned}
            l_n & = |y_n - \hat{y_n}|, \\
            L(\hat{y}, y) = \frac{\sum_{i=1}^{N}l_i}{N},
        \end{aligned}

    References
    ----------
    .. [1] Wikipedia - Mean absolute error:
       https://en.wikipedia.org/wiki/Mean_absolute_error
    """
    def __init__(self):
        self.type = 'MAELoss'

    def forward(self, Y_hat, Y):
        """
        Computes the forward propagation.

        Parameters
        ----------
        Y_hat : numpy.array
            Array containing the predictions.
        Y : numpy.array
            Array with the real labels.

        Returns
        -------
        Numpy.arry containing the cost.
        """
        self.Y = Y
        self.Y_hat = Y_hat

        ae = np.abs(self.Y - self.Y_hat)
        mae = np.mean(ae)

        return np.squeeze(mae)

    def backward(self):
        """
        Computes the backward propagation.

        Returns
        -------
        grad : numpy.array
            Array containg the gradients of the weights.
        """
        grad = - np.sign(self.Y - self.Y_hat)

        return grad

class CrossEntropyLoss(Base):
    """
    Class that implements the Cross entropy loss function.

    Given a target math:`y` and an estimate math:`\hat{y}` the
    Cross Entropy loss can be written as:

    .. math::
        \begin{aligned}
            l_{\hat{y}, class} = -\log\left(\frac{\exp(\hat{y_n}[class])}{\sum_j \exp(\hat{y_n}[j])}\right), \\
            L(\hat{y}, y) = \frac{\sum^{N}_{i=1} l_{i, class[i]}}{\sum^{N}_{i=1} weight_{class[i]}},
        \end{aligned}

    References
    ----------
    .. [1] Wikipedia - Cross entropy:
       https://en.wikipedia.org/wiki/Cross_entropy
    """
    def __init__(self):
        self.type = 'CELoss'
        self.eps = 1e-15
        self.softmax = Softmax()

    def forward(self, Y_hat, Y):
        """
        Computes the forward propagation.

        Parameters
        ----------
        Y_hat : numpy.array
            Array containing the predictions.
        Y : numpy.array
            Array with the real labels.

        Returns
        -------
        Numpy.arry containing the cost.
        """
        self.Y = Y
        self.Y_hat = self.softmax.forward(Y_hat)

        _loss = - Y * np.log(self.Y_hat)
        loss = np.sum(_loss, axis=0).mean()

        return np.squeeze(loss)

    def backward(self):
        """
        Computes the backward propagation.

        Returns
        -------
        grad : numpy.array
            Array containg the gradients of the weights.
        """
        grad = self.Y_hat - self.Y

        return grad

class NLLLoss(Base):
    """
    Class that implements the Negative log likelihood loss function.

    Given a target math:`y` and an estimate math:`\hat{y}` the
    Negative log likelihood loss can be written as:

    .. math::
        \begin{aligned}
            l_{\hat{y}, class} = -\log\left(\frac{\exp(\hat{y_n}[class])}{\sum_j \exp(\hat{y_n}[j])}\right), \\
            L(\hat{y}, y) = \frac{\sum^{N}_{i=1} l_{i, class[i]}}{\sum^{N}_{i=1} weight_{class[i]}},
        \end{aligned}

    References
    ----------
    .. [1] Pytorch - NLLLOSS:
       https://pytorch.org/docs/stable/generated/torch.nn.NLLLoss.html
    """
    def __init__(self):
        self.type = 'NLLLoss'
        self.eps = 1e-15

    def forward(self, Y_hat, Y):
        """
        Computes the forward propagation.

        Parameters
        ----------
        Y_hat : numpy.array
            Array containing the predictions.
        Y : numpy.array
            Array with the real labels.

        Returns
        -------
        Numpy.arry containing the cost.
        """
        self.Y = Y
        self.Y_hat = Y_hat

        _loss = - Y * np.log(self.Y_hat)
        loss = np.sum(_loss, axis=0).mean()

        return np.squeeze(loss)

    def backward(self):
        """
        Computes the backward propagation.

        Returns
        -------
        grad : numpy.array
            Array containg the gradients of the weights.
        """
        grad = self.Y_hat - self.Y

        return grad


#!/usr/bin/python
##example2_4
import numpy as np
from LUdecomp import *

def vandermode(v):
    n=len(v)
    a=np.zeros((n,n))
    for j in range(n):
        a[:,j]=v**(n-j-1)

    return a

v= np.array([1.0, 1.2, 1.4, 1.6, 1.8, 2.0])
b= np.array([0.0, 1.0, 0.0, 1.0, 0.0,1.0])

a= vandermode(v)
aOrig=a.copy()
bOrig=b.copy()

a = LUdecomp(a)

x = LUsolve(a,b)

det = np.prod(np.diagonal(a))

print('x=\n',x)


from .BaseModel import BaseModel
from .Function import Function
from .utils import *
import numpy as np

from numpy.polynomial.polynomial import polyfit
from sklearn.metrics import mean_squared_error

class PolynomialModel(BaseModel):

    def __init__(self):
        pass

    def get_model_name(self):
        return "Polynomial"

    def get_equation_string(self, coef):
        degrees = []

        for i in range(len(coef)):
            if i == 0:
                degrees.append('{:g}'.format(round_sig(coef[i], 2)) )
            elif i == 1:
                degrees.append('{:+g}x'.format(round_sig(coef[i], 2)) )
            else:
                degrees.append('{:+g}x^{}'.format(round_sig(coef[i], 2), i) )

        return " ".join(degrees)

    def get_best_fit(self, complexity_level, x0, y0):

        coef = polyfit(x0, y0, complexity_level)
        y1 = np.polyval(coef[::-1], x0).tolist()

        mse = mean_squared_error(y0, y1)

        equation_string = self.get_equation_string(coef)

        return Function(self.get_model_name(), complexity_level, f"y = {equation_string}", (x0, y1), round_sig(mse))


import numpy as np
import scipy as sp
import scipy.stats
from matplotlib import pyplot as plt
from conjugate_bayes.models import NormalInverseGamma

if __name__ == "__main__":

    x = np.random.normal(loc=1, scale=1, size=200)
    model = NormalInverseGamma(m=0, tau2=0.5, a=0.5, b=0.5)
    model.fit(x)

    mu = model.get_marginal_mu()
    sigma2 = model.get_marginal_sigma2()
    pred = model.get_posterior_prediction()
    mle = sp.stats.norm(loc=x.mean(), scale=x.std(ddof=1))
    true = sp.stats.norm(loc=1, scale=1)

    plt.figure(figsize=(12, 3))
    plt.subplot(1, 3, 1)
    axis = np.linspace(-2, 2, 500)
    plt.plot(axis, mu.pdf(axis), color="black")
    plt.axvline(mu.mean(), color="black")
    plt.title("Marginal distribution over $\mu$")
    plt.subplot(1, 3, 2)
    axis = np.linspace(0, 2, 500)
    plt.plot(axis, sigma2.pdf(axis), color="black")
    plt.axvline(sigma2.mean(), color="black")
    plt.title("Marginal distribution over $\sigma^2$")
    plt.subplot(1, 3, 3)
    axis = np.linspace(-2, 2, 500)
    plt.plot(axis, pred.pdf(axis), color="black", label="Posterior")
    plt.title("Posterior predictive distribution")
    plt.plot(axis, mle.pdf(axis), color="blue", label="MLE")
    plt.plot(axis, true.pdf(axis), color="green", label="True")
    plt.legend()
    plt.tight_layout()
    plt.show()


# -*- coding: utf-8 -*-
"""
Finding Maxima and Minma with Higher-Order Derivatives

Created on Wed Feb 24 09:12:25 2016

@author: Liu Lixiang
"""
from sympy import Symbol, solve, Derivative

x = Symbol('x')
f = x**5 - 30*x**3+50*x
d1 = Derivative(f, x).doit()
critical_points = solve(d1)
print(critical_points)
A, B, C, D = critical_points
# Second derivate
d2 = Derivative(f, x, 2).doit()
print('Second-Order Derivative:')
print(d2.subs({x:A}).evalf())
print(d2.subs({x:B}).evalf())
print(d2.subs({x:C}).evalf())
print(d2.subs({x:D}).evalf())
x_min = -5
x_max = 5
# Global Maxima
print('Global Maxima:')
print(f.subs({x:A}).evalf())
print(f.subs({x:C}).evalf())
print(f.subs({x:x_min}).evalf())
print(f.subs({x:x_max}).evalf())
# Global Minima
print('Global Minima:')
print(f.subs({x:B}).evalf())
print(f.subs({x:D}).evalf())
print(f.subs({x:x_min}).evalf())
print(f.subs({x:x_max}).evalf())


import numpy as np
import skfuzzy as fuzz
from skfuzzy import control as ctrl

error = ctrl.Antecedent(np.arange(-2*np.pi,2*np.pi), 'error')
phi_ = ctrl.Consequent(np.arange(-2*np.pi,2*np.pi),'phi_')

phi_['poor'] = fuzz.trimf(phi_.universe, [-8, -6,-3 ])
phi_['mediocre'] = fuzz.trimf(phi_.universe, [-4, -2, 0])
phi_['average'] = fuzz.trimf(phi_.universe, [-2, 0, 2])

phi_['decent'] = fuzz.trimf(phi_.universe, [0, 2,4 ])
phi_['good'] = fuzz.trimf(phi_.universe, [3, 6, 8])

error['poor'] = fuzz.trimf(error.universe, [-8, -6,-3 ])
error['mediocre'] = fuzz.trimf(error.universe, [-4, -2, 0])
error['average'] = fuzz.trimf(error.universe, [-2, 0, 2])

error['decent'] = fuzz.trimf(error.universe, [0, 2,4 ])
error['good'] = fuzz.trimf(error.universe, [3, 6, 8])

#rule1 = ctrl.Rule(error['dismal'], phi_['excellent'])
rule2 = ctrl.Rule(error['poor'], phi_['good'])
rule3 = ctrl.Rule(error['mediocre'], phi_['decent'])
rule4 = ctrl.Rule(error['average'], phi_['average'])
rule5 = ctrl.Rule(error['decent'], phi_['mediocre'])
rule6 = ctrl.Rule(error['poor'], phi_['good'])
#rule7 = ctrl.Rule(error['excellent'], phi_['dismal'])
phi_ctrl = ctrl.ControlSystem( [rule2, rule3,rule4,rule5,rule6])
phi =  ctrl.ControlSystemSimulation(phi_ctrl)
phi.input['error'] = -6.5

def get_value(err):
	Gain = 1
	phi.input['error']  = err
	phi.compute()
	return phi.output['phi_'] * Gain


import numpy as np

def sigmoid(z):
    return 1 / (1 + np.exp(-z))

def sigmoid_prime(z):
    return sigmoid(z) * (1 - sigmoid(z))

def softmax(z):
    z = z - np.max(z)
    ex = np.exp(z)
    return ex / np.sum(ex)

def softmax_prime(z):
    return np.ones(z.shape)

def relu(z):
    return np.maximum(0, z)

def relu_prime(z):
    return (z > 0).astype(float)

def tanh_prime(z):
    return 1.0 - np.tanh(z) ** 2

'''
Activation

compute(X): return the activation function applied element wise
compute_prime(X): return the derivate of the activation function applied element wise

'''

class SigmoidActivation:

    def __init__(self):
        self.name = 'sigmoid'

    def compute(self, x):
        return  sigmoid(x)

    def compute_prime(self, x):
        return  sigmoid_prime(x)

class SoftmaxActivation:

    def __init__(self):
        self.name = 'softmax'

    def compute(self, x):
        return softmax(x)

    def compute_prime(self, x):
        return softmax_prime(x)

class ReluActivation:

    def __init__(self):
        self.name = 'relu'

    def compute(self, x):
        return  relu(x)

    def compute_prime(self, x):
        return  relu_prime(x)

class TanhActivation:

    def __init__(self):
        self.name = 'tanh'

    def compute(self, x):
        return  np.tanh(x)

    def compute_prime(self, x):
        return  tanh_prime(x)

class LinearActivation:

    def __init__(self):
        self.name = 'linear'

    def compute(self, x):
        return x

    def compute_prime(self, x):
        return np.ones(x.shape)


import numpy as np

# Defining the matrices
A = np.matrix([[3, 6, -5],
               [1, -3, 2],
               [5, -1, 4]])

B = np.matrix([[12],
               [-2],
               [10]])

# Solving for the variables, where we invert A
X = A ** (-1) * B
print(X)

# matrix([[ 1.75],
#   [ 1.75],
#   [ 0.75]])

a = np.array([[3, 6, -5],
              [1, -3, 2],
              [5, -1, 4]])

# Defining the array
b = np.array([12, -2, 10])

# Solving for the variables, where we invert A
x = np.linalg.inv(a).dot(b)
print x
# array([ 1.75,  1.75,  0.75])


import numpy as np
from math import pow

class NetPresentValue:
    def __init__(self, cash_flow, discount_rate):
        self.cash_flow = cash_flow
        self.discount_rate = discount_rate
        self.npv = 0

    def calculate(self):
        temp = 0
        to_calc = []

        while temp < len(self.cash_flow):
            cash = self.cash_flow[temp]
            denom = pow((1 + (self.discount_rate / 100)), temp + 1)
            to_calc.append(cash / denom)
            temp += 1

        self.npv = np.sum(to_calc)
        return self.npv


from tqdm import tqdm

import numpy as np

def PCA(points):
    mean_points = np.mean(points, axis=0)
    centered_points = points - mean_points
    conv_matrix = centered_points.T @ centered_points / len(centered_points)

    eigenvalues, eigenvectors = np.linalg.eigh(conv_matrix)

    return eigenvalues, eigenvectors

def compute_local_PCA(query_points, cloud_points, radius_or_k_neighbors, use_radius):
    from classifier_3D.utils.neighbors import compute_index_neighbors

    idx_neighbors_queries = compute_index_neighbors(
        query_points, cloud_points, radius_or_k_neighbors, use_radius
    )

    all_eigenvalues = np.zeros((query_points.shape[0], 3))
    all_eigenvectors = np.zeros((query_points.shape[0], 3, 3))

    for idx_query, idx_neighbors in tqdm(enumerate(idx_neighbors_queries)):
        all_eigenvalues[idx_query], all_eigenvectors[idx_query] = PCA(
            cloud_points[idx_neighbors]
        )

    return all_eigenvalues, all_eigenvectors


# Bit manipulation utilities
#
# Long Le
# University of Illinois
#

import numpy as np

def reverseBits(x):
    mask = 1
    rx = 0
    for k in range(16):
        #print('k = '+str(k))
        #print('mask = '+format(mask,'#018b'))
        #print('mask&x = '+format(mask&x,'#018b'))
        if mask&x != 0:
            rx <<= 1
            rx += 1
        else:
            rx <<= 1
        #print('rx = '+format(rx,'#018b'))

        mask <<= 1

    return rx

def addWithLogic(x,y):
    s = x^y # sum
    c = x&y # carry out
    while c != 0:
        #print('----------')
        #print('s = '+format(s,'#018b'))
        #print('c = '+format(c,'#018b'))
        sTmp = s^(c<<1)
        cTmp = s&(c<<1)
        s = sTmp
        c = cTmp

    return s

def numOf1s(x):
    cnt = 0
    while x != 0:
        cnt += 1
        x = x&(x-1)
    return cnt

def ffs(x):
    # find first set/one
    if x == 0:
        return 0
    mask = 1
    cnt = 0
    while x & mask == 0:
        mask <<= 1
        cnt += 1
    return cnt

def ffs_table(x):
    # time-space tradeoff with assistant tables
    if x == 0:
        return 0

    n = 8
    table = [ffs(i) for i in range(2**8)]
    cnt = 0
    while True:
        if x & (2**n-1) != 0:
            return cnt + table[x & (2**n-1)]
        x >>= n
        cnt += n

def ctz(x):
    # count trailing zero == find first set
    # count the # of zeros following the LS 1
    # using de Bruijn sequences
    table = {(( (1<<i) * 0x077CB531 ) >> 27):i for i in range(32)}
    # x & (-x) isolates the LS 1
    return table[((x & (-x)) * 0x077CB531) >> 27]

def ctz_bs(x):
    # binary search
    nB = 32
    if x == 0:
        return nB

    cnt = 0
    mask = 0x0000FFFF
    maskN = int(nB//2)
    for k in range(int(np.log2(nB))):
        if x & mask == 0:
            cnt += maskN
            x >>= maskN
        maskN //= 2
        #print('maskN = %s' % maskN)
        mask >>= maskN

    return cnt

def clz(x):
    # count leading zero
    # count the # of zeros preceding the MS 1
    if x == 0:
        return 32

    # index map for indices 0,1,...,15
    table = [4, 3, 2, 2, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0]
    cnt = 0
    while True:
        #print('cnt = %d' % cnt)
        if x & 0xF0000000 != 0:
            #print('cnt + table = %d' % (cnt+table[x>>28]))
            return cnt + table[x >> (32-4)]
        x <<= 4
        cnt += 4


import numpy as np
from scipy import optimize
from scipy.stats import weibull_min
from math import gamma

# Problem 3
T1 = 44 * 10 ** 3
T2 = 26 * 10 ** 3
T3 = 100 * 10 ** 3
T4 = 19 * 10 ** 3
T5 = 45 * 10 ** 3

# def L(gamma, alpha, t1, t2, t3, t4, t5) :
#     # uncensored contribution
#     A = ((alpha/gamma)**2) * ((1/gamma)**(alpha-1)) * (t1*t2) * np.exp((1/gamma)*((-t1)**alpha + (-t2)**alpha))
#     # censored contribution
#     B = np.exp((1/gamma)**alpha) * ((t3**alpha) + (t4**alpha) + (t5**alpha))
#     return A * B

def neg_log_L(gamma, alpha) -> float:
    A = 2 * np.log(alpha / gamma) + (alpha - 1) * np.log(T1 * T2 / (gamma ** 2)) - \
        ((1 / gamma) ** alpha) * ((T1 ** alpha) + (T2 ** alpha))
    B = -((1 / gamma) ** alpha) * (T3 ** alpha + T4 ** alpha + T5 ** alpha)
    return -A - B

## Numerically solve MLE equations.
epsilon = 1 * 10 ** -12
x0 = [10, 10]
bnds = ((epsilon, np.inf), (epsilon, np.inf))
fun = lambda x: neg_log_L(x[0], x[1])
solver = optimize.minimize(fun, x0=x0, bounds=bnds)
print(" {} iterations \n".format(solver.nit),
      "lambda = {} and alpha = {}".format(1 / solver.x[0], solver.x[1]))

## Parametric Boostrap (1-delta) confidence interval
lmda = 1 / solver.x[0]
alpha = solver.x[1]
# reference parameter
mu_star = (1 / lmda) * gamma(1 + 1 / alpha)
print("In average, {} kms before a reliability problem occurs.".format(mu_star))

m = 100  # number of iterations to aggregate
bootstrap_estimates = []
for i in range(m):
    T_bootstrap = weibull_min.rvs(c=alpha, scale=1 / lmda, size=100)
    bootstrap_estimates.append(np.mean(T_bootstrap))

delta = 0.1
# Upper bound : we estimate prob(estimate - mu_star < x) = 1-delta/2
x = 2000
count = 0
while (count / m != 1 - delta / 2):
    count = 0
    for i in range(m):
        if bootstrap_estimates[i] - mu_star < x:
            count += 1
    print(count / m)
    x += 10
print(" Prob(mu_n - mu_estimate < {}) = {}".format(x, count / m))

# Lowe bound : we estimate prob(estimate - mu_star < y ) = delta/2
y = -10000
count = 0
while (count / m != delta / 2):
    count = 0
    for i in range(m):
        if bootstrap_estimates[i] - mu_star < y:
            count += 1
    print(count / m)
    y += 10
print(" Prob(mu_n - mu_estimate < {}) = {}".format(y, count / m))

print(" Parametric Boostrap {}% confidence interval : [{},{}]".format(1 - delta, -x + np.mean(bootstrap_estimates),
                                                                      - y + np.mean(bootstrap_estimates)))

# Problem 4
## Posterior distribution
X = np.array([6.00, 4.82, 3.35, 2.38, 3.59, 4.12, 4.98, 2.69, 6.24, 6.77,
              6.22, 5.42, 5.42, 3.10, 4.65, 4.24, 4.53, 4.62, 5.36, 2.57])

Sigma = (1 + (0.5) * np.sum(X[0:19]**2))**-1
X_i_1 = X[0:19]
X_i = X[1:20]
Mu = Sigma * ( (0.5) * np.sum(X_i_1*(X_i-2)) + 0.5)
print("rho*|X follows a normal distribution with mean = {} and variance = {}".format(Mu, Sigma))

## Prob(X_23 > 4)
n = 100000
X23_estimates = []
count = 0
for i in range(n) :
    rho = np.random.normal(Mu, Sigma, 1)
    epsilon = np.random.normal(0,2,3)
    X23 = (rho**3) * X[19] + 2 * (1 + rho + rho**2) + epsilon[0] + epsilon[1] * rho + epsilon[2] * rho**2
    X23_estimates.append(X23)
    if X23 >= 4 :
        count += 1

print("X23 estimate = {}".format(np.mean(X23_estimates)))
print("Prob(X_23 > 4) = {}".format(count/n))


# EM

## Mixtures of Gaussians

given a training set $\left\{x^{(1)},...,x^{(n)}\right\}$ with no labels. each point $x^{(i)}$ has a latent variable $z^{(i)}$, the data is specified by a joint probability $p(x^{(i)}, z^{(i)}) = p(x^{(i)}| z^{(i)})p(z^{(i)})$

mixtures of gaussians assumes $z^{(i)}\sim Multinomial(\phi)$ (where $\phi_{j} \ge 0, \sum_{j=1}^{k}\phi_{j}=1$, $\phi_{j}$ gives $p(z^{(i)}=j)$), and $x^{(i)}|z^{(i)}=j \sim \mathcal{N}(\mu_{j}, \Sigma_{j})$. we let $k$ denote the number of values $z^{(i)}$'s can take on.

thus our model posits that each $x^{(i)}$ was generated by randomly choosing $z^{(i)}$ from $\{1,...,k\}$, and $x^{(i)}$ was drawn from one of $k$ gaussians depending on $z^{(i)}$.

the parameters of our model are $\phi, \mu, \Sigma$, to estimate them, we can write down the likelihood of our data:

$$
\begin{equation}
\begin{split}
l(\phi, \mu, \Sigma) =& \sum_{i=1}^{n}log\,p(x^{(i)};\phi,\mu,\Sigma)\\
=& \sum_{i=1}^{n}log\sum_{z^{(i)}=1}^{k}p(x^{(i)}|z^{(i)}; \mu,\Sigma)p(z^{(i)};\phi)
\end{split}
\end{equation}
$$

it is not possible to find the maximum likelihood estimate of the parameters in closed form.

on the other hand, if we knew what $z^{(i)}$'s were, the maximum likelihood problem would have been easy.

specifically, we could then write down the likelihood as:

$$l(\phi, \mu, \Sigma) = \sum_{i=1}^{n}log\,p(x^{(i)}|z^{(i)};\mu,\Sigma) + log\,p(z^{(i)}; \phi)$$

maximizing this with respect to $\phi, \mu, \Sigma$ gives the parameters:

$$\phi_{j} = \frac{1}{n}\sum_{i=1}^{n}1\{z^{(i)} = j\}$$

$$\mu_{j} = \frac{\sum_{i=1}^{n}1\{z^{(i)} = j\}x^{(i)}}{\sum_{i=1}^{n}1\{z^{(i)} = j\}}$$

$$\Sigma_{j} = \frac{\sum_{i=1}^{n}1\{z^{(i)} = j\}(x^{(i)} - \mu_{j})(x^{(i)} - \mu_{j})^{T}}{\sum_{i=1}^{n}1\{z^{(i)} = j\}}$$

it is identical to the gaussian discriminant analysis(GDA).

however, in our density estimation problems, $z^{(i)}$'s are not known, what can we do?

the EM algorithm is an iterative algorithm that has two main steps:

**1.the E-step, it tries to "guess" the values of the $z^{(i)}$'s.**

**2.the M-step, it updates the parameters of our model based on our guesses.**

since in the M-step we are pretending that the guesses in the first part were correct, the maximization becomes easy, here's the algorithm:

(E-step) for each $i, j$ set

$$w_{j}^{(i)} := p(z^{i}=j|x^{(i)}; \phi,\mu,\Sigma)$$

(M-step) update the parameters:

$$\phi_{j} := \frac{1}{n}\sum_{i=1}^{n}w_{j}^{(i)}$$

$$\mu_{j} := \frac{\sum_{i=1}^{n}w_{j}^{(i)}x^{(i)}}{\sum_{i=1}^{n}w_{j}^{(i)}}$$

$$\Sigma_{j} := \frac{\sum_{i=1}^{n}w_{j}^{(i)}(x^{(i)} - \mu_{j})(x^{(i)} - \mu_{j})^{T}}{\sum_{i=1}^{n}w_{j}^{(i)}}$$

the EM-algorithm likes the K-means-algorithm.

except that instead of "hard" cluster assignments $c(i)$, we have the "soft" assignments $w_{j}^{(i)}$.

from sklearn.mixture import GaussianMixture
from sklearn.datasets import make_blobs
import numpy as np

X1, y1 = make_blobs(n_samples=1000, centers=((4, -4), (0, 0)), random_state=42)
X1 = X1.dot(np.array([[0.374, 0.95], [0.732, 0.598]]))
X2, y2 = make_blobs(n_samples=250, centers=1, random_state=42)
X2 = X2 + [6, -8]
X = np.r_[X1, X2]
y = np.r_[y1, y2]

import matplotlib.pyplot as plt

def plot_clusters(X, y=None):
    plt.scatter(X[:, 0], X[:, 1], c=y, s=1)
    plt.xlabel("$x_1$", fontsize=14)
    plt.ylabel("$x_2$", fontsize=14, rotation=0)

plot_clusters(X)

gm = GaussianMixture(n_components=3, n_init=10, random_state=42)
gm.fit(X)

gm.weights_

gm.means_

gm.covariances_

gm.converged_, gm.n_iter_

gm.predict(X), gm.predict_proba(X)

X_new, y_new = gm.sample(7)

gm.score_samples(X_new)

gm.bic(X), gm.aic(X)

## The EM algorithm

here comes the general view of EM.

like the assumption before, we have the training set $\left\{x^{(1)},...,x^{(n)}\right\}$, each point $x^{(i)}$ has a latent variable $z^{(i)}$.

denote $\theta$ as the distribution parameters($\phi,\mu,\Sigma$ in mixtures of gaussian), then density of $x$ can be obtained by:

$$p(x;\theta) = \sum_{z}p(x,z;\theta)$$

the log-likelihood of the data:

$$
\begin{equation}
\begin{split}
l(\theta) =& \sum_{i=1}^{n}log\,p(x^{(i)}; \theta) \\
=& \sum_{i=1}^{n}log\sum_{z^{(i)}}p(x^{(i)}, z^{(i)}; \theta)
\end{split}
\end{equation}
$$

instead of solving the maximum-likelihood directly(usually very hard), the EM algorithm repeatedly construct a lower-bound on $l$(E-step), and then optimize that lower-bound(M-step).

we first consider optimizing the likelihood $log\,p(x)$ for a single example $x$, optimize $log\,p(x;\theta)$:

$$log\,p(x;\theta) = log\sum_{z}p(x,z;\theta)$$

let $Q$ be a distribution over the possible values of $z$, then:

$$
\begin{equation}
\begin{split}
log\,p(x;\theta) =& log\sum_{z}p(x,z;\theta)\\
=& log\sum_{z}Q(z)\frac{p(x,z;\theta)}{Q(z)}\\
\ge& \sum_{z}Q(z)log\frac{p(x,z;\theta)}{Q(z)}
\end{split}
\end{equation}
$$

the last step uses Jensen's inequality, for $(log\,x)'' = -1/x^{2} < 0$ so that strictly concave.

for any distribution $Q$, the above formula gives a lower-bound on $log\,p(x;\theta)$.

to make the bound tight for a particular value of $\theta$, we want the Jensen's inequality to hold with equality. it is sufficient the expecation take on "constant"-valued random variables, i.e:

$$\frac{p(x,z;\theta)}{Q(z)} = c$$

this leads to:

$$
\begin{equation}
\begin{split}
Q(z) =& \frac{p(x,z;\theta)}{\sum_{z}p(x,z;\theta)}\\
=& \frac{p(x,z;\theta)}{p(x;\theta)}\\
=& p(z|x;\theta)
\end{split}
\end{equation}
$$

thus, we simply set $Q$ to be the posterior distribution of $z$'s given $x$ and $\theta$.

for convenience, we call the bound as **evidence lower bound(ELBO)**:

$$ELBO(x;Q,\theta) = \sum_{z}Q(z)log\frac{p(x,z;\theta)}{Q(z)}$$

the estimation now can re-write as:

$$\forall Q,\theta,x,\quad log\,p(x;\theta) \ge ELBO(x;Q,\theta)$$

Intuitively, the EM algorithm alternatively update $Q$ and $\theta$ by:

a.setting $Q(z) = p(z|x;\theta)$ so that $ELBO(x;Q,\theta) = log\,p(x;\theta)$

b.maximizing $ELBO(x;Q,\theta)$ with respect to $\theta$.

now consider multiple examples $\left\{x^{(1)},...,x^{(n)}\right\}$, note the optimal choice of $Q$ is $p(z|x;\theta)$, and it depends on the particular example $x$.

therefore we introduce $n$ distributions $Q_{1},...,Q_{n}$, one for each example $x^{(i)}$, for each example:

$$log\,p(x^{(i)};\theta) \ge ELBO(x^{(i)};Q_{i},\theta) = \sum_{z^{(i)}}Q_{i}(z^{(i)})log\frac{p(x^{(i)},z^{(i)};\theta)}{Q_{i}(z^{(i)})}$$

taking sum over all examples, we obtain a lower bound for the log-likelihood:

$$
\begin{equation}
\begin{split}
l(\theta) \ge& \sum_{i}ELBO(x^{(i)};Q_{i},\theta)\\
=& \sum_{i}\sum_{z^{(i)}}Q_{i}(z^{(i)})log\frac{p(x^{(i)},z^{(i)};\theta)}{Q_{i}(z^{(i)})}
\end{split}
\end{equation}
$$

the above inequality holds for **any** distributions $Q_{1},...,Q_{n}$. equality holds when:

$$Q_{i}(z^{(i)}) =  p(z^{(i)}|x^{(i)};\theta)$$

we now come to the definition of EM algorithm:

(E-step) for each $i$, set:

$$Q_{i}(z^{(i)}) :=  p(z^{(i)}|x^{(i)};\theta)$$

(M-step) for fixed $Q_{i}$'s, set:

$$
\begin{equation}
\begin{split}
\theta :=& \underset{\theta}{argmax}\sum_{i=1}^{n}ELBO(x^{(i)};Q_{i},\theta)\\
=& \underset{\theta}{argmax}\sum_{i=1}^{n}\sum_{z^{(i)}}Q_{i}(z^{(i)})log\frac{p(x^{(i)},z^{(i)};\theta)}{Q_{i}(z^{(i)})}
\end{split}
\end{equation}
$$

## Convergence of EM algorithm

suppose $\theta^{(t)}$ and $\theta^{(t+1)}$ are the parameters from two successive iterations of EM, we will now prove that $l(\theta^{(t)}) \le l(\theta^{(t+1)})$.

$$
\begin{equation}
\begin{split}
l(\theta^{(t+1)}) \ge& \sum_{i=1}^{n}ELBO(x^{(i)};Q_{i}^{(t)};\theta^{(t+1)})\quad (\mbox{by Jensen's inequality})\\
\ge& \sum_{i=1}^{n}ELBO(x^{(i)};Q_{i}^{(t)};\theta^{(t)})\quad (\mbox{by M-step}) \\
=& l(\theta^{(t)})\quad (\mbox{by E-step})
\end{split}
\end{equation}
$$

if we define:

$$ELBO(Q, \theta) = \sum_{i=1}^{n}ELBO(x^{(i)}; Q_{i}, \theta)$$

then we know $l(\theta) \ge ELBO(Q, \theta)$.

the EM algorithm can be viewed an alternating maximization algorithm on $ELBO(Q, \theta)$:

the E-step maximizes it with respect to $Q$.

the M-steo maximizes it with respect to $\theta$.

## Bayesian Gaussian Mixture Models

from sklearn.mixture import BayesianGaussianMixture

bgm = BayesianGaussianMixture(n_components=10, n_init=10)
bgm.fit(X)
np.round(bgm.weights_, 2)

## Excercises

### 1.clustering

from sklearn.datasets import fetch_olivetti_faces

olivetti = fetch_olivetti_faces()

print(olivetti.DESCR)

olivetti.target

from sklearn.model_selection import StratifiedShuffleSplit

strat_split = StratifiedShuffleSplit(n_splits=1, test_size=40, random_state=42)
train_valid_idx, test_idx = next(strat_split.split(olivetti.data, olivetti.target))
X_train_valid = olivetti.data[train_valid_idx]
y_train_valid = olivetti.target[train_valid_idx]
X_test = olivetti.data[test_idx]
y_test = olivetti.target[test_idx]

strat_split = StratifiedShuffleSplit(n_splits=1, test_size=80, random_state=43)
train_idx, valid_idx = next(strat_split.split(X_train_valid, y_train_valid))
X_train = X_train_valid[train_idx]
y_train = y_train_valid[train_idx]
X_valid = X_train_valid[valid_idx]
y_valid = y_train_valid[valid_idx]

print(X_train.shape, y_train.shape)
print(X_valid.shape, y_valid.shape)
print(X_test.shape, y_test.shape)

from sklearn.decomposition import PCA

pca = PCA(0.99)
X_train_pca = pca.fit_transform(X_train)
X_valid_pca = pca.transform(X_valid)
X_test_pca = pca.transform(X_test)

pca.n_components_

from sklearn.cluster import KMeans

k_range = range(5, 150, 5)
kmeans_per_k = []
for k in k_range:
    print("k={}".format(k))
    kmeans = KMeans(n_clusters=k, random_state=42).fit(X_train_pca)
    kmeans_per_k.append(kmeans)

from sklearn.metrics import silhouette_score

silhouette_scores = [silhouette_score(X_train_pca, model.labels_)
                     for model in kmeans_per_k]
best_index = np.argmax(silhouette_scores)
best_k = k_range[best_index]
best_score = silhouette_scores[best_index]

plt.figure(figsize=(8, 3))
plt.plot(k_range, silhouette_scores, "bo-")
plt.xlabel("$k$", fontsize=14)
plt.ylabel("Silhouette score", fontsize=14)
plt.plot(best_k, best_score, "rs")
plt.show()

best_k

inertias = [model.inertia_ for model in kmeans_per_k]
best_inertia = inertias[best_index]

plt.figure(figsize=(8, 3.5))
plt.plot(k_range, inertias, "bo-")
plt.xlabel("$k$", fontsize=14)
plt.ylabel("Inertia", fontsize=14)
plt.plot(best_k, best_inertia, "rs")
plt.show()

best_model = kmeans_per_k[best_index]

def plot_faces(faces, labels, n_cols=5):
    faces = faces.reshape(-1, 64, 64)
    n_rows = (len(faces) - 1) // n_cols + 1
    plt.figure(figsize=(n_cols, n_rows * 1.1))
    for index, (face, label) in enumerate(zip(faces, labels)):
        plt.subplot(n_rows, n_cols, index + 1)
        plt.imshow(face, cmap="gray")
        plt.axis("off")
        plt.title(label)
    plt.show()

for cluster_id in np.unique(best_model.labels_):
    print("Cluster", cluster_id)
    in_cluster = best_model.labels_==cluster_id
    faces = X_train[in_cluster]
    labels = y_train[in_cluster]
    plot_faces(faces, labels)

### 2.clustering + classification

from sklearn.ensemble import RandomForestClassifier

clf = RandomForestClassifier(n_estimators=150, random_state=42)
clf.fit(X_train_pca, y_train)
clf.score(X_valid_pca, y_valid)

X_train_reduced = best_model.transform(X_train_pca)
X_valid_reduced = best_model.transform(X_valid_pca)
X_test_reduced = best_model.transform(X_test_pca)

clf = RandomForestClassifier(n_estimators=150, random_state=42)
clf.fit(X_train_reduced, y_train)

clf.score(X_valid_reduced, y_valid)

from sklearn.pipeline import Pipeline

for n_clusters in k_range:
    pipeline = Pipeline([
        ("kmeans", KMeans(n_clusters=n_clusters, random_state=42)),
        ("forest_clf", RandomForestClassifier(n_estimators=150, random_state=42))
    ])
    pipeline.fit(X_train_pca, y_train)
    print(n_clusters, pipeline.score(X_valid_pca, y_valid))

X_train_extended = np.c_[X_train_pca, X_train_reduced]
X_valid_extended = np.c_[X_valid_pca, X_valid_reduced]
X_test_extended = np.c_[X_test_pca, X_test_reduced]

clf = RandomForestClassifier(n_estimators=150, random_state=42)
clf.fit(X_train_extended, y_train)
clf.score(X_valid_extended, y_valid)

### 3.Gaussian Mixture Model

from sklearn.mixture import GaussianMixture

gm = GaussianMixture(n_components=40, random_state=42)
y_pred = gm.fit_predict(X_train_pca)

n_gen_faces = 20
gen_faces_reduced, y_gen_faces = gm.sample(n_samples=n_gen_faces)
gen_faces = pca.inverse_transform(gen_faces_reduced)

plot_faces(gen_faces, y_gen_faces)

n_rotated = 4
rotated = np.transpose(X_train[:n_rotated].reshape(-1, 64, 64), axes=[0, 2, 1])
rotated = rotated.reshape(-1, 64*64)
y_rotated = y_train[:n_rotated]

n_flipped = 3
flipped = X_train[:n_flipped].reshape(-1, 64, 64)[:, ::-1]
flipped = flipped.reshape(-1, 64*64)
y_flipped = y_train[:n_flipped]

n_darkened = 3
darkened = X_train[:n_darkened].copy()
darkened[:, 1:-1] *= 0.3
y_darkened = y_train[:n_darkened]

X_bad_faces = np.r_[rotated, flipped, darkened]
y_bad = np.concatenate([y_rotated, y_flipped, y_darkened])

plot_faces(X_bad_faces, y_bad)

X_bad_faces_pca = pca.transform(X_bad_faces)

gm.score_samples(X_bad_faces_pca)

gm.score_samples(X_train_pca[:10])

### 4.Using Dimensionality Reduction Techniques for Anomaly Detection

X_train_pca

def reconstruction_errors(pca, X):
    X_pca = pca.transform(X)
    X_reconstructed = pca.inverse_transform(X_pca)
    mse = np.square(X_reconstructed - X).mean(axis=-1)
    return mse

reconstruction_errors(pca, X_train).mean()

reconstruction_errors(pca, X_bad_faces).mean()

plot_faces(X_bad_faces, y_bad)

X_bad_faces_reconstructed = pca.inverse_transform(X_bad_faces_pca)
plot_faces(X_bad_faces_reconstructed, y_bad)


import numpy as np
from numpy import linalg as LA
from utils import SingleValueDecompositon
import cv2

def calcHomography(P1, P2):
    """Calculates the homography matrix between set of src and dst points P1, and P2

    Args:
        P1 (list or numpy array): Set of source Points
        P2 (list or numpy array): Set of destination Points

    Returns:
        _type_: Returns the homograph matrix of numpy array type
    """
    P1 = np.array(P1)
    P2 = np.array(P2)
    x = P1[:,0]
    y = P1[:,1]
    xp = P2[:,0]
    yp = P2[:,1]

    A=[]
    # Generate A matrix for Ax=0 formulation
    for i in range(4):
        A.append(np.array([-x[i], -y[i], -1, 0, 0, 0, x[i]*xp[i], y[i]*xp[i], xp[i]]))
        A.append(np.array([0, 0, 0, -x[i], -y[i], -1, x[i]*yp[i], y[i]*yp[i], yp[i]]))
    A = np.array(A)
    # Compute the singular value decompostion of A
    svd = SingleValueDecompositon()
    U, S, V = svd.SVD(A)
    # Least square approximate of x corresponds to the last column of V
    # Reshape and divide each element by H33 to obtain the final matrix
    H = V[:,V.shape[1]-1].reshape(3,3)
    H = H/H[2,2]
    return H
if __name__ == "__main__":

    P1 = np.array([[5, 5], [150, 5], [150, 150], [5, 150]])
    P2 = np.array([[100, 100], [200, 80], [220, 80], [100, 200]])
    H = calcHomography(P1,P2)
    h, _ = cv2.findHomography(P1, P2)
    print("Homograph Matrix: ", H)
    if np.allclose(H, h):
        print("Verified by OpenCV findHomography function")
    else:
        print("Estimated Homography Matrix doesn't match exactly with OpenCV estiamtion")


import torch
import numpy as np
import matplotlib.pyplot as plt
from scipy.spatial import KDTree

def generate_points(sim_range, nsamples, ndim, points_type, seed=1, plot=0, min_dist=0, nodes1d=None):
    '''
    Generate points using different methods
    '''

    out = []
    np.random.seed(seed)
    if(type(sim_range) is float or type(sim_range) is int):

        sim_ranges = np.zeros((2, ndim))
        sim_ranges[0, :] = -sim_range  # low bound
        sim_ranges[1, :] = sim_range  # up bound

        sim_range = sim_ranges

    if(points_type == "random"):

        x_input = np.zeros((nsamples, ndim))
        soboleng = torch.quasirandom.SobolEngine(
            dimension=ndim, scramble=True, seed=seed)
        x_input = soboleng.draw(nsamples).cpu().detach().numpy()
        x_input = (sim_range[1]-sim_range[0])*x_input+sim_range[0]

        out.append(x_input)

    elif points_type == "regular":

        npts = round(pow(nsamples, 1./ndim))
        xs1d = np.zeros((ndim, npts))
        for d in range(ndim):

            xs1d[d] = np.linspace(sim_range[0][d], sim_range[1][d], npts)
            dx = xs1d[d][1]-xs1d[d][0]
            print("dx[", d, "]= ", dx)

        if ndim == 1:

            x1 = xs1d[0]
            x_input = x1.reshape(-1, 1)
            out.append(x_input)
            out.append(x1)

        elif ndim == 2:

            x1, x2 = np.meshgrid(xs1d[0], xs1d[1])
            x_input = np.stack((x1.reshape(-1), x2.reshape(-1)), axis=-1)

            out.append(x_input)
            out.append(x1)
            out.append(x2)

        elif ndim == 3:

            x1, x2, x3 = np.meshgrid(xs1d[0], xs1d[1], xs1d[2])
            x_input = np.stack(
                (x1.reshape(-1), x2.reshape(-1), x3.reshape(-1)), axis=-1)

            out.append(x_input)
            out.append(x1)
            out.append(x2)
            out.append(x3)

        else:
            raise ValueError("regular grid in not implemented for ndim>3")

    elif(points_type == "randomgrid"):

        if nodes1d is None:
            raise ValueError(
                f'nodes1d is required for points_type = {points_type}')

        x_input = np.zeros((nsamples, ndim))
        np.random.seed(seed)
        x_input = np.random.choice(nodes1d, size=(nsamples, ndim))
        out.append(x_input)

    else:
        raise ValueError("unknown points type")

    if ndim == 2 and plot:

        plt.figure()
        plt.plot(out[0][:, 0], out[0][:, 1], '.')
        plt.show()

    return out

def average_distance(points):
    '''
    calculates average distance between given points
    '''
    xs = points
    tree = KDTree(xs)
    dd, _ = tree.query(xs, k=2)
    avd = dd[:, 1].mean()
    return avd


# coding: utf-8

#
 Circuit Analysis Using Sympy

#
 Assignment 7

#
 M V A Suhas kumar,EE17B109

#
March 16,2019

# # Introduction
# In this assignment, we use Sympy to analytically solve a matrix equation governing an analog circuit. We look at two circuits, an active low pass filter and an active high pass filter. We create matrices using node equations for the circuits in sympy, and then solve the equations analytically. We then convert the resulting sympy solution into a numpy function which can be called. We then use the signals toolbox we studied in the last assignment to understand the responses of the two circuits to various inputs.

# Importing required packages

# In[1]:

from sympy import *
import numpy as np
import matplotlib.pyplot as plt
import scipy.signal as sp
from pylab import *
from IPython.display import *

# # Low pass Filter
#

# ![Circuit1](circuit1.png)
#

# where G  =1.586 and R1 = R2 = 10kΩ and C1=C2=10pF. This gives a 3dB Butter-worth filter with cutoff frequency of 1/2πMHz.
#
# Circuit Equations are as follows:
# $$V_{m}=\frac{V_{o}}{G}$$
# $$ V_{p} =V_{1} \frac{1}{1+s R_{2}C_{2}}$$
# $$ V_{o} = G(V_{p} - V_{m})$$
# $$\frac{V_{i}-V_{1}}{R_{1}} + \frac{V_{p}-V_{1}}{R_{2}} + s C_{1}(V_{0}-V_{1}) = 0$$
# Solving the above equations with approxmtion gives
#
# $$ V_{o} \approx \frac{V_{i}}{s R_{1} C_{1}}$$
#
# We would like to solve this in Python and also get (and plot) the exact result.  For this we need the sympy module.

# To solve the equtions exactly we use matrix method of solving:
#

# In[2]:

init_printing()
R1,R2,C1,C2,G = symbols("R1 R2 C1 C2 G")
V1,Vp,Vm,Vo,Vi = symbols("V1 Vp Vm Vo Vi")
s = symbols("s")
A = Matrix([[0,0,1,-1/G],
            [-1/(1+s*R2*C2),1,0,0],
            [0,-G,G,1],
          [-1/R1-1/R2-s*C1,1/R2,0,s*C1]])
M = Matrix([V1,Vp,Vm,Vo])
b = Matrix([0,0,0,Vi/R1])
display(Eq(MatMul(A,M),b))

# Solving the above matrix yield exact result

# Function defining low pass filter:

# In[3]:

def lowpass(R1=10**4,R2=10**4,C1=10**-11,C2=10**-11,G=1.586,Vi=1):
    s=symbols("s")
    A=Matrix([[0,0,1,-1/G],
              [-1/(1+s*R2*C2),1,0,0],
              [0,-G,G,1],
              [-1/R1-1/R2-s*C1,1/R2,0,s*C1]])
    b=Matrix([0,0,0,Vi/R1])
    V = A.inv()*b
    return(A,b,V)

# Function which can take input in laplace domain or time domain and give the output of low pass filter:

# In[4]:

def low_pass_output(laplace_fn = None,time_fn=None,t=np.linspace(0,1e-5,1e5),C=10**-11):
    A,b,V = lowpass(C1=C,C2=C)
    v_low_pass = V[-1]
    temp = expand(simplify(v_low_pass))
    n,d = fraction(temp)
    n,d = Poly(n,s),Poly(d,s)
    num,den = n.all_coeffs(),d.all_coeffs()
    H_v_low_pass = sp.lti([-float(f) for f in num],[float(f) for f in den])
    if laplace_fn !=None:
        temp = expand(simplify(laplace_fn))
        n,d = fraction(temp)
        n,d = Poly(n,s),Poly(d,s)
        num,den = n.all_coeffs(),d.all_coeffs()
        lap = sp.lti([float(f) for f in num],[float(f) for f in den])
        t,u = sp.impulse(lap,None,t)
    else:
        u = time_fn
    t,V_out,svec = sp.lsim(H_v_low_pass,u,t)
    return (t,V_out)

# # High pass filter

# ![High pass filter](high.png)

# values you can use are R1=R3=10kΩ, C1=C2=1nF, and G=1.586
#
# Circuit Equations are as follows:
# $$V_{n}=\frac{V_{o}}{G}$$
# $$ V_{p} =V_{1} \frac{s R_{3}C_{2}}{1+s R_{3}C_{2}}$$
# $$ V_{o} = G(V_{p} - V_{n})$$
# $$(V_{1}-V_{i})sC_{1} + \frac{(V_{1}-V_{o})}{R_{1}} + (V_{i}-V_{p})sC_{2} = 0 $$

# In[5]:

R1, R3, C1, C2, G, Vi = symbols('R_1 R_3 C_1 C_2 G V_i')
V1,Vn,Vp,Vo = symbols('V_1 V_n V_p V_o')
x=Matrix([V1,Vn,Vp,Vo])

A=Matrix([[0,-1,0,1/G],
        [s*C2*R3/(s*C2*R3+1),0,-1,0],
        [0,G,-G,1],
        [-s*C2-1/R1-s*C1,0,s*C2,1/R1]])

b=Matrix([0,0,0,-Vi*s*C1])
init_printing
display(Eq(MatMul(A,x),b))

# Function defining high pass filter:

# In[6]:

def highpass(R1=10**4,R3=10**4,C1=10**-9,C2=10**-9,G=1.586,Vi=1):
    s= symbols("s")
    A=Matrix([[0,-1,0,1/G],
        [s*C2*R3/(s*C2*R3+1),0,-1,0],
        [0,G,-G,1],
        [-s*C2-1/R1-s*C1,0,s*C2,1/R1]])

    b=Matrix([0,0,0,-Vi*s*C1])
    V =A.inv() * b
    return (A,b,V)

# Function which can take input in laplace domain or time domain and give the output of high pass filter:

# In[7]:

def high_pass_output(laplace_fn = None,time_fn=None,t=np.linspace(0,1e-4,1e5),C=10**-11):
    A,b,V = highpass(C1=C,C2=C)
    v_high_pass = V[-1]
    temp = expand(simplify(v_high_pass))
    n,d = fraction(temp)
    n,d = Poly(n,s),Poly(d,s)
    num,den = n.all_coeffs(),d.all_coeffs()
    H_v_high_pass = sp.lti([float(f) for f in num],[float(f) for f in den])
    if laplace_fn !=None:
        temp = expand(simplify(laplace_fn))
        n,d = fraction(temp)
        n,d = Poly(n,s),Poly(d,s)
        num,den = n.all_coeffs(),d.all_coeffs()
        lap = sp.lti([float(f) for f in num],[float(f) for f in den])
        t,u = sp.impulse(lap,None,t)
    else:
        u = time_fn
    t,V_out,svec = sp.lsim(H_v_high_pass,u,t)
    return (t,V_out)

# # Question1

# Step Response for low pass filter

# In[8]:

t,V_low_step = low_pass_output(laplace_fn=1/s)

# In[9]:

plt.plot(t,V_low_step)
plt.grid(True)
plt.xlabel("t ------>",size=14)
plt.ylabel(r"$Step\ Response\ V_{o}(t)$",size=14)
plt.title("Step Response When Capacitance = 10pF in low pass filter")
plt.show()

# Step response is starting from zero and reaching 0.793 at steady state.This is because DC gain oftransfer function  is 0.793.Initial value is 0 because AC gain of low pass filter is zero(impulse can be assumed as High frequency signal and we know low pass filter dosen't pass high frequency signal).

# # Question2

# Finding Output when input signal is $$(sin(2000πt)+cos(2×106πt))u_{o}(t)$$

# In[10]:

t = np.linspace(0,1e-3,1e5)
plt.plot(t,np.sin(2000*np.pi*t)+np.cos(2e6*np.pi*t))
plt.grid(True)
plt.xlabel("t ------>",size=14)
plt.ylabel(r"$V_{i}(t)$",size=14)
plt.title("Mixed frequency input")
plt.show()

# Band is high frequency wave and envolope is the low frequency wave

# In[11]:

t = linspace(0,1e-5,1e5)
t,vout = low_pass_output(time_fn=np.sin(2000*np.pi*t)+np.cos(2e6*np.pi*t),t=t,C=10**-9)

# In[12]:

plt.plot(t,vout)
plt.grid(True)
plt.xlabel("t ------>",size=14)
plt.ylabel(r"$V_{o}(t)$",size=14)
plt.title("Output for mixed frequency Sinusoid in lowpass filter in transient time")
plt.show()

# From above we can clearly see that Output is superposition of High Amplitude low frequency wave and Low amplitude High frquency wave(Since Low pass filter attenuates the High frequencies)

# In[13]:

t = linspace(0,1e-5,1e5)
t,vout = high_pass_output(time_fn=np.sin(2000*np.pi*t)+np.cos(2e6*np.pi*t),t=t,C=10**-9)

# In[14]:

plt.plot(t,vout)
plt.grid(True)
plt.xlabel("t ------>",size=14)
plt.ylabel(r"$V_{o}(t)$",size=14)
plt.title("Output for mixed frequency Sinusoid in High pass filter in transient time")
plt.show()

# The plot which is appearing to be band(closely placed lines) is superposition of High Amplitude High frequency wave and Low amplitude Low frquency wave(Since High pass filter attenuates the Low frequencies) which inturn appears to be non distorted sine wave.

# In[15]:

t = linspace(0,1e-3,1e5)
t,vout = low_pass_output(time_fn=np.sin(2000*np.pi*t)+np.cos(2e6*np.pi*t),t=t,C=10**-9)

# In[16]:

plt.plot(t,vout)
plt.grid(True)
plt.xlabel("t ------>",size=14)
plt.ylabel(r"$V_{o}(t)$",size=14)
plt.title("Output for mixed frequency Sinusoid in lowpass filter in steady time")
plt.show()

# From graph we can see frequency is close to 1000Hz(which is low frquency input)

# In[17]:

t = linspace(0,1e-4,1e5)
t,vout = high_pass_output(time_fn=np.sin(2000*np.pi*t)+np.cos(2e6*np.pi*t),t=t,C=10**-9)

# In[18]:

plt.plot(t,vout)
plt.grid(True)
plt.xlabel("t ------>",size=14)
plt.ylabel(r"$V_{o}(t)$",size=14)
plt.title("Output for mixed frequency Sinusoid in High pass filter in steay time")
plt.show()

# From graph we can see frequency is close to 1000KHz(which is high frquency input)

# # Question 3,4

# Damped Sinusoid -----> $exp(-300t)sin(10^{6}t)$

# In[19]:

t = linspace(0,1e-3,1e6)
f = np.exp(-3000*t) * np.sin(10**6 *t)

# In[20]:

plt.title("High frequency damped sinusoid")
plt.xlabel("$t$")
plt.ylabel("$v_i(t)$",size=20)
plt.plot(t,f)
plt.grid()
plt.show()

# In[21]:

t = linspace(0,1e-3,1e6)
t,vout = high_pass_output(time_fn=f,t=t,C=10**-9)

# In[22]:

plt.plot(t,vout)
plt.grid(True)
plt.xlabel("t ------>",size=14)
plt.ylabel(r"$V_{o}(t)$",size=14)
plt.title("Output for High frequency damped input in High pass filter")
plt.show()

# From above graph we can clearly see that High pass filter passed high frequency sinusoid with out attenuating much.(Since property of high pass filter)

# In[27]:

t = linspace(0,1e-3,1e6)
t,vout = low_pass_output(time_fn=f,t=t,C=10**-9)

# In[28]:

plt.plot(t,vout)
plt.grid(True)
plt.xlabel("t ------>",size=14)
plt.ylabel(r"$V_{o}(t)$",size=14)
plt.title("Output for High frequency damped input in low pass filter")
plt.show()

# From above graph Low pass filter quickly attenuated the High frequency Sinusoid and gives distorted Output

# # Question 5

# In[29]:

t,V_high_step = high_pass_output(laplace_fn=1/s,C=10**-9)

# In[30]:

plt.plot(t,V_high_step)
plt.grid(True)
plt.xlabel("t ------>",size=14)
plt.ylabel(r"$Step\ Response\ V_{o}(t)$",size=14)
plt.title("Step Response When Capacitance = 1nF in high pass filter")
plt.show()

# Step response here saturates at zero and this is because DC gain of High pass filter is 0. We can clearly see from graph that it starts from 0.793 and this because AC gain of transfer function  at high frequencies is 0.793(Step can assumed as infinite frequency signal and we know high pass filter only allows high frequency signals)
#
# step response overshoots the steady state value of 0, reaches an
# extremum, then settles back to 0, unlike the response of the low pass filter which steadily
# approaches the steady state value with no extrema. This occurs because of the presence of
# zeros at the origin in the transfer function of the high pass filter(which imply that the DC
# gain is 0). Since the steady state value of the step response is 0, the total signed area under
# the curve of the impulse response must also be 0. This means that the impulse response must
# equal zero at one or more time instants. Since the impulse response is the derivative of the
# step response, this therefore means that the step response must have at least one extremum.
# This explains the behaviour of the step response of the high pass filter.

# # Conclusions:

# The low pass filter responds by letting the low frequency sinusoid pass through without
# much additional attenuation. The output decays as the input also decays.
#
# The high pass filter responds by quickly attenuating the input. Notice that the time scales
# show that the high pass filter response is orders of magnitudes faster than the low pass
# response. This is because the input frequency is below the cutoff frequency, so the output
# goes to 0 very fast.
#
# In conclusion, the sympy module has allowed us to analyse quite complicated circuits by
# analytically solving their node equations. We then interpreted the solutions by plotting time
# domain responses using the signals toolbox. Thus, sympy combined with the scipy.signal
# module is a very useful toolbox for analyzing complicated systems like the active filters in
# this assignment.