# -*- coding: utf-8 -*-
"""
Created on Wed Apr 24 08:51:47 2019

@author: mhossa12
"""
## https://www.kaggle.com/andyxie/k-means-clustering-implementation-in-python
# Import necessary libraries
from copy import deepcopy
import numpy as np # linear algebra
from matplotlib import pyplot as plt
# Set three centers, the model should predict similar results
center_1 = np.array([1,1])
center_2 = np.array([5,5])
center_3 = np.array([8,1])

# Generate random data and center it to the three centers
data_1 = np.random.randn(200, 2) + center_1
data_2 = np.random.randn(200,2) + center_2
data_3 = np.random.randn(200,2) + center_3

data = np.concatenate((data_1, data_2, data_3), axis = 0)

plt.scatter(data[:,0], data[:,1], s=7)
plt.grid()
plt.show()

# Number of clusters
k = 3
# Number of training data
n = data.shape[0]
# Number of features in the data
c = data.shape[1]

# Generate random centers, here we use sigma and mean to ensure it represent the whole data
mean = np.mean(data, axis = 0)
std = np.std(data, axis = 0)
centers = np.random.randn(k,c)*std + mean

# Plot the data and the centers generated as random
plt.scatter(data[:,0], data[:,1], s=7)
plt.scatter(centers[:,0], centers[:,1], marker='*', c='g', s=150)

plt.grid()
plt.show()

centers_old = np.zeros(centers.shape) # to store old centers
centers_new = deepcopy(centers) # Store new centers

data.shape
clusters = np.zeros(n)
distances = np.zeros((n,k))

error = np.linalg.norm(centers_new - centers_old)

# When, after an update, the estimate of that center stays the same, exit loop
while error != 0:
    # Measure the distance to every center
    for i in range(k):
        distances[:,i] = np.linalg.norm(data - centers[i], axis=1)
    # Assign all training data to closest center
    clusters = np.argmin(distances, axis = 1)

    centers_old = deepcopy(centers_new)
    # Calculate mean for every cluster and update the center
    for i in range(k):
        centers_new[i] = np.mean(data[clusters == i], axis=0)
    error = np.linalg.norm(centers_new - centers_old)
centers_new

# Plot the data and the centers generated as random
plt.scatter(data[:,0], data[:,1], s=7)
plt.scatter(centers_new[:,0], centers_new[:,1], marker='*', c='g', s=150)

plt.grid()
plt.show()

Clustering K-Means/00Develope and Verify a K-Means++ Clustering Model with a Randomly Generated Dataset in Python.py


import sympy as sp

if __name__ == '__main__':
    j = complex(0, 1)
    A = sp.Matrix([
        [1, 0, -1],
        [0, 1, j],
        [-1, -j, 0]
    ])

    A_eigenvectors = A.eigenvects()

    print(f"A := {A}", '-'*25, sep='\n')
    for eigenvalue, multiplicity, eigenvector in A_eigenvectors:
        print(f"lambda = {eigenvalue}, m = {multiplicity}", eigenvector, '-'*175, sep='\n')

    E, a, theta = sp.symbols("E a theta")
    H = sp.Matrix([
        [E, a, 0],
        [a, E, a],
        [0, a, E]
    ])
    H_eigenvectors = H.eigenvects()

    print(f"H := {H}", '-' * 25, sep='\n')
    for eigenvalue, multiplicity, eigenvector in H_eigenvectors:
        print(f"lambda = {eigenvalue}, m = {multiplicity}", eigenvector, '-'*175, sep='\n')

    R = sp.Matrix([
        [1, 0, 0],
        [0, sp.cos(theta), -sp.sin(theta)],
        [0, sp.sin(theta), sp.cos(theta)]
    ])
    RHRinv = R*H*R.inv()
    RHRinv_eigenvectors = RHRinv.eigenvects()

    print(f"RHRinv := {RHRinv}", '-' * 25, sep='\n')
    for eigenvalue, multiplicity, eigenvector in RHRinv_eigenvectors:
        print(f"lambda = {eigenvalue}, m = {multiplicity}", eigenvector, '-' * 175, sep='\n')

    print(H.eigenvals(), RHRinv.eigenvals(), sep='\n')
    check = all([e in RHRinv.eigenvals() for e in H.eigenvals()])
    print(f"H.eigenvals() == RHRinv.eigenvals(): {check}")


"""Probability function"""
from scipy.stats import norm

# Finding the probability of a discrete value
def prob_norm_single_value(discrete_value=None, mean=None,stdev=None):
    """Probability function for a discrete value"""
    x_value = norm(loc = mean, scale= stdev).cdf(discrete_value)
    return x_value

# Finding the probability between two ranges
def prob_norm_between_values(lower_val=None, upper_val=None, mean=None,stdev=None):
    """Probability between two values"""
    l_value = norm(loc = mean, scale= stdev).cdf(lower_val)
    u_value = norm(loc = mean, scale= stdev).cdf(upper_val)
    delta = u_value - l_value
    return delta

# Finding the probability above a discrete value
def prob_norm_above_value(discrete_value=None, mean=None,stdev=None):
    """Probability above a discrete value"""
    x_value = norm(loc = mean, scale= stdev).cdf(discrete_value)
    delta = 1 - x_value
    return delta

def main():
    """Main logic flow"""
    func_type = input ('Enter the function type (single/range/above): ')
    lower_val = float ( input("Enter the single or lower range (value): ") )
    if func_type == 'range':
        upper_val = float ( input("Enter the upper range (value): ") )
    mean = float ( input("Enter the mean (value): ") )
    stdev = float ( input("Enter the standard deviation (value): ") )
    if func_type == 'range':
        return_value = prob_norm_between_values(
                lower_val=lower_val,
                upper_val=upper_val,
                mean=mean,
                stdev=stdev
                )
        print(f'Probability between {lower_val} and {upper_val} is: {return_value}')
    elif func_type == 'above':
        return_value = prob_norm_above_value(
                discrete_value=lower_val,
                mean=mean,
                stdev=stdev
                )
        print(f'Probability above {lower_val} is: {return_value}')
    else:
        return_value = prob_norm_single_value(
                discrete_value=lower_val,
                mean=mean,
                stdev=stdev
                )
        print(f'Probability of {lower_val} is: {return_value}')

if __name__ == "__main__":
    main()


from pyomo.environ import *
infinity = float('inf')

# Creation of a Concrete Model
model = AbstractModel()

#DEFINE SETS
# Products
model.F = Set()
# Nutrients
model.N = Set()

# DEFINE PARAMETERS
# Cost
model.c = Param(model.F, within = PositiveReals, doc = 'price in $')

# Amount of nutrient
model.a    = Param(model.F, model.N, within = NonNegativeReals)

# Max and Min for each Nutrient
model.Nmin = Param(model.N, within = NonNegativeReals, default = 0.0)
model.Nmax = Param(model.N, within = NonNegativeReals, default = infinity)

# Number of servings
model.x = Var(model.F, within = NonNegativeIntegers)

# Minimize z(cost)
def cost(model):
    return sum(model.c[i]*model.x[i] for i in model.F)
model.cost = Objective(rule=cost)

# LIMITS

#Max
def nutrients_max(model, j):
    value = sum(model.a[i,j]*model.x[i] for i in model.F)
    return value <= model.Nmax[j]
model.nutrient_limit_max = Constraint(model.N, rule=nutrients_max)
#Min
def nutrient_min(model, j):
    value = sum(model.a[i,j]*model.x[i] for i in model.F)
    return model.Nmin[j] <= value
model.nutrient_limit_min = Constraint(model.N, rule=nutrient_min)

#def pyomo_postprocess(options=None, instance=None, results=None):
#    model.x.display()


# Question 4 Lab Assignment 2
# AB Satyaprakash - 180123062

# imports ----------------------------------------------------------------------
from math import log, tan, factorial
from sympy import *
import numpy as np

# functions --------------------------------------------------------------------

def f(x):
    return log(tan(x))/log(10)

def thirdLagrange(x0, x1, x2, x3, val):
    # Using the values given in the quesiton
    f0, f1, f2, f3 = 0.1924, 0.2414, 0.2933, 0.3492

    # compute l0(x) with x = val
    l0 = ((val-x1)*(val-x2)*(val-x3))/((x0-x1)*(x0-x2)*(x0-x3))
    l1 = ((val-x0)*(val-x2)*(val-x3))/((x1-x0)*(x1-x2)*(x1-x3))
    l2 = ((val-x0)*(val-x1)*(val-x3))/((x2-x0)*(x2-x1)*(x2-x3))
    l3 = ((val-x0)*(val-x1)*(val-x2))/((x3-x0)*(x3-x1)*(x3-x2))
    px = f0*l0 + f1*l1 + f2*l2 + f3*l3
    return px

def derivative(x0, n):
    x = symbols('x')
    f = log(tan(x), 10)
    fn = f.diff(x, n)
    fn = lambdify(x, fn)
    return fn(x0)

def maxError(nodes, x):
    err = 1
    for n in nodes:
        err *= (x-n)
    err /= factorial(len(nodes))
    a = min(nodes)
    b = max(nodes)
    l = np.linspace(a, b, 250)
    ret = 0
    for z in l:
        der = derivative(z, len(nodes))
        ret = max(ret, abs(err*der))
    return ret

# ------------------------------------------------------------------------------
# Use the following values and four-digit rounding arithmetic to construct a third Lagrange polynomial approximation to f(1.09).
# The function being approximated is f(x) = log10(tan x). Use this knowledge to find a bound for the error in the approximation.
# f(1.00) = 0.1924, f(1.05) = 0.2414, f(1.10) = 0.2933, f(1.15) = 0.3492.
# Using f(1.00) = 0.1924, f(1.05) = 0.2414, f(1.10) = 0.2933, f(1.15) = 0.3492
# and four-digit rounding arithmetic to construct a third Lagrange polynomial approximation for f(1.09)
x0, x1, x2, x3, val = 1, 1.05, 1.1, 1.15, 1.09
res = thirdLagrange(x0, x1, x2, x3, val)
print('The third Lagrange polynomial approximation for f(1.09) is {}'.format(res))
print('The third Lagrange polynomial approximation for f(1.09) rounded to 4 decimal places is {}'.format(round(res, 4)))

# The bound for error in approximation:
print('The bound for the error in this approximation is {}'.format(maxError([x0, x1, x2, x3], val)))

Semester 6/MA 322 (Scientific Computing Theory)/Labs/Lab 2/Code/q4.py


import argparse
from math import inf

import matplotlib.pyplot as plt
import numpy as np

# Create k random initial center points
def randCent(dataSet, k):
    n = np.shape(dataSet)[1]
    cpoints = np.zeros((k, n))
    # create centroid mat
    for j in range(n):
        # create random cluster centers, within bounds of each dimension
        minJ = np.min(dataSet[:, j])
        rangeJ = float(np.max(dataSet[:, j]) - minJ)
        cpoints[:, j] = minJ + rangeJ * np.random.rand(k)
    return cpoints

# Euclidean distance
def eucDist(pA, pB):
    return np.linalg.norm(pA - pB)

# K-means main method.
def kmeans(dataSet, k, distMeasure=eucDist, centerGen=randCent):
    m = np.shape(dataSet)[0]  # number of data
    cluResult = np.zeros((m, 2))  # center and distance of each point
    centers = centerGen(dataSet, k)  # the k centers
    cluChanged = True
    while cluChanged:
        cluChanged = False
        for i in range(m):
            minDist = inf
            minIndex = -1
            for j in range(k):
                # Step 1: find the nearest center for each point
                newDist = distMeasure(centers[j, :], dataSet[i, :])
                if newDist < minDist:
                    minDist = newDist
                    minIndex = j
            if cluResult[i, 0] != minIndex:
                cluChanged = True
                cluResult[i, :] = minIndex, minDist
        # Step 2: re-evaluate the center points
        for cent in range(k):
            ptsInClust = dataSet[np.where(cluResult[:, 0] == cent)]
            centers[cent, :] = np.mean(ptsInClust, axis=0)
    return centers, cluResult

def main():
    data = np.loadtxt('rawdata.csv', delimiter=',')
    parser = argparse.ArgumentParser(
        description='K-means clustering program.')
    parser.add_argument('-k', type=int, default=3,
                        help='The number of clusters. Default is 3.')
    args = parser.parse_args()
    dataSet = data[:, 0:-1]
    centers, cluResult = kmeans(dataSet, args.k)
    print('Centers:', centers)
    print('Clustering result:', cluResult)
    # Drawing
    colors=['black', 'blue', 'green', 'lime', 'maroon', 'olive', 'orange', 'purple', 'red', 'teal', 'yellow']
    np.random.shuffle(colors)
    for i in range(len(centers)):
        center=centers[i]
        cluPoint=dataSet[np.where(cluResult[:,0]==i)]
        plt.scatter(center[0],center[1],c=colors[i],marker='*')
        plt.scatter(cluPoint[:,0],cluPoint[:,1], c=colors[i])
    plt.show()

if __name__ == '__main__':
    main()


# coding: utf-8

# # Solving nonlinear equations (finding roots of functions)
# Find $x$ such that $f(x) = 0$

import numpy as np
import matplotlib.pyplot as plt

tol = 1e-10
max_steps = 100
f = lambda x: x**2 - 2.0
df = lambda x: 2*x
print("root of x^2 - 2:", np.sqrt(2))

# ## Newtons method
# From:
# $$ f(x^*) = 0 = f(x_0) + f'(x_0)(x^* - x_0) + \mathcal{O}(|x^* - x_0|^2) $$
# we get:
# $$ x_{n+1} \leftarrow x_n - \frac{f(x_n)}{f'(x_n)}$$

def newton(f, df, x0, tol=tol, max_steps=max_steps):
    x = x0
    step = 0
    while True:
        x_new = x - f(x)/df(x)
        if abs(x - x_new) <= tol or step == max_steps:
            return x_new, step
        x = x_new
        step += 1

print("newton:", newton(f, df, 10.0))

# ## Bisection
# We have two points $a$ and $b$. Now we compute the point between thous $c = \frac{a + b}{2}$. If $f(c)$ has the same sign as $f(a)$ then we can use $c$ as our new $a$, if $f(c)$ as the same sign as $f(b)$ we can use $c$ as our new $b$.

def shrink_interval(f, a, b, c):
    if np.sign(f(c)) == np.sign(f(a)):
        return c, b
    else:
        return a, c

def bisect(f, a, b, tol=tol, max_steps=max_steps):
    step = 0
    assert np.sign(f(a)) != np.sign(f(b))
    while True:
        c = (a + b)/2
        if abs(a - b) <= tol or step == max_steps or f(c) == 0.0:
            return c, step
        a, b = shrink_interval(f, a, b, c)
        step += 1

print("bisection:", bisect(f, 0.0, 10.0, tol=1e-15, max_steps=200))

# ## Secant method
# We have two points $a$ and $b$. Then we can approx. the root between these points by the root of the secant between the points.
# The secant hits $(a, f(a))$ and $(b, f(b))$.
# $$m = \frac{f(a) - f(b)}{a - b}$$
# $$b = f(a) - m a$$
# $$ 0 = mx + b \Rightarrow x = -\frac{b}{m}$$

def secant_root(f, a, b):
    m = (f(a) - f(b))/(a - b)
    d = f(a) - m*a
    return -d/m

def secant(f, x0, x1, tol=tol, max_steps=max_steps):
    step = 0
    while True:
        x2 = secant_root(f, x0, x1)
        if abs(x1 - x2) <= tol or step == max_steps:
            return x2, step
        x0, x1 = x1, x2
        step += 1

print("secant:", secant(f, 0.0, 10.0))

# ## Regula falsi (false position)
# Lets combine secant and bisection!
#
# Start with an interval $(a_0,b_0)$ where $sgn(f(a_0)) \neq sgn(f(b_0))$
# - Find root of the secant $c$ between $a_0$ and $b_0$
# - We are done if $f(c) = 0$ or we reached some tolerance.
# - Choose $a_1 = c$ if $sgn(f(c)) = sgn(f(a))$ otherwise choose $b_1 = c$ if $sgn(f(c)) = sgn(f(b))$

def regula_falsi(f, a, b, tol=tol, max_steps=max_steps):
    assert np.sign(a) != np.sign(b)
    step = 0
    while True:
        c = secant_root(f, a, b)
        if min(abs(a - c), abs(b - c)) <= tol or c == 0.0 or step == max_steps:
            return c, step
        a, b = shrink_interval(f, a, b, c)
        step += 1

print("regula_falsi:", regula_falsi(f, 0.0, 10.0))

# ## Fixpoint iteration
# Instead of solving $f(x) = 0$ for $x$, we turn this equation into $g(x) = x$ and then compute $x_{n + 1} = g(x_n)$ for some sufficently large $n$.

def fixpoint(g, x, tol=tol, max_steps=max_steps, debug=False):
    step = 0
    while True:
        new_x = g(x)
        if debug:
            print(x, new_x)
        if abs(x - new_x) <= tol or step == max_steps:
            return new_x, step
        x = new_x
        step += 1

# $$f(x) = x^2 - 2$$
# $$x^2 - 2 = 0$$
# $$g(x) = x^2 + x - 2$$
# $$x = x^2 + x - 2$$

fixpoint(lambda x: x**2 + x - 2., 1.4, max_steps=10, debug=True)

# $$f(x) = sin(x)$$
# $$sin(x) = 0$$
# $$g(x) = sin(x) + x$$
# $$x = sin(x) + x$$

# In[15]:

fixpoint(lambda x: np.sin(x) + x, 1.0, debug=True)


import numpy as np
import scipy.sparse as spar
import scipy.linalg as la
from scipy.sparse import linalg as sla

def to_matrix(filename,n):
    '''
    Return the nxn adjacency matrix described by datafile.
    INPUTS:
    datafile (.txt file): A .txt file describing a directed graph. Lines
        describing edges should have the form '<from node>\t<to node>\n'.
        The file may also include comments.
    n (int): The number of nodes in the graph described by datafile
    RETURN:
        Return a SciPy sparse `dok_matrix'.
    '''
    pass

def calculateK(A,N):
    '''
    Compute the matrix K as described in the lab.
    Input:
        A (array): adjacency matrix of an array
        N (int): the datasize of the array
    Return:
        K (array)
    '''
    pass

def iter_solve(adj, N=None, d=.85, tol=1E-5):
    '''
    Return the page ranks of the network described by `adj`.
    Iterate through the PageRank algorithm until the error is less than `tol'.
    Inputs:
    adj - A NumPy array representing the adjacency matrix of a directed graph
    N (int) - Restrict the computation to the first `N` nodes of the graph.
            Defaults to N=None; in this case, the entire matrix is used.
    d     - The damping factor, a float between 0 and 1.
            Defaults to .85.
    tol  - Stop iterating when the change in approximations to the solution is
        less than `tol'. Defaults to 1E-5.
    Returns:
    The approximation to the steady state.
    '''
    pass

def eig_solve( adj, N=None, d=.85):
    '''
    Return the page ranks of the network described by `adj`. Use the
    eigenvalue solver in \li{scipy.linalg} to calculate the steady state
    of the PageRank algorithm
    Inputs:
    adj - A NumPy array representing the adjacency matrix of a directed graph
    N - Restrict the computation to the first `N` nodes of the graph.
            Defaults to N=None; in this case, the entire matrix is used.
    d     - The damping factor, a float between 0 and 1.
            Defaults to .85.
    Returns:
    The approximation to the steady state.
    '''
    pass

def team_rank(filename='ncaa2013.csv'):
    '''
    Use your iterative PageRank solver to predict the rankings of the teams in
    the given dataset of games.
    The dataset should have two columns, representing winning and losing teams.
    Each row represents a game, with the winner on the left, loser on the right.
    Parse this data to create the adjacency matrix, and feed this into the
    solver to predict the team ranks.
    Inputs:
    filename (optional) - The name of the dataset.
    Returns:
    ranks - A list of the ranks of the teams in order "best" to "worst"
    teams - A list of the names of the teams, also in order "best" to "worst"
    '''
    pass


from __future__ import print_function

import numpy as np
from scipy import stats
import matplotlib.pyplot as plt

# ========== HW07 SOLUTION [Python2/3] ========== #

# ========== 1 ========== #

a = np.array([0.5507979, 0.70814782, 0.29090474, 0.51082761, 0.89294695, 0.89629309, 0.12558531, 0.20724288, 0.0514672, 0.44080984])
b = np.array([-0.04381817, -0.47721803, -1.31386475, 0.88462238, 0.88131804, 1.70957306, 0.05003364, -0.40467741, -0.54535995, -1.54647732])

# K-S 2 sample test
# null hypothesis: a and b are sampled from the same distribution
kstest = stats.ks_2samp(a, b)
kstest.pvalue < 0.05  # True; reject null hypothesis.
# conclude a and b are NOT from the same distribution.

# a histogram can show visually that they are not from the same distribution.
plt.figure()
plt.hist(a)
plt.hist(b)
plt.show()

# ========== 2 ========== #

np.random.seed(121)
binom_mars = stats.binom(100, 0.0925)
mars_rv = binom_mars.rvs(10)
print(np.mean(mars_rv))
print(binom_mars.cdf(10.0))

# ========== 3 ========== #

rv = np.load('Aluminum_youngs_moduli.npy')
est_loc = np.mean(rv)
est_scale = np.std(rv)

# fitting different distributions
norm_param = stats.norm.fit(rv)
laplace_param = stats.laplace.fit(rv)
maxwell_param = stats.maxwell.fit(rv)
logistic_param = stats.logistic.fit(rv)

# plot PDFs
x = np.linspace(10, 150, 200)
plt.figure()
plt.hist(rv, bins=21, normed=True)
plt.plot(x, stats.norm.pdf(x, *norm_param), label='norm')
plt.plot(x, stats.maxwell.pdf(x, *maxwell_param), label='maxwell_param')
plt.plot(x, stats.logistic.pdf(x, *logistic_param), label='logistic_param')
plt.plot(x, stats.laplace.pdf(x, *laplace_param), label='laplace')
plt.legend()
plt.show()

# plot CDFs
plt.figure()
plt.hist(rv, cumulative=True, bins=21, normed=True)
plt.plot(x, stats.norm.cdf(x, *norm_param), label='norm')
plt.plot(x, stats.maxwell.cdf(x, *maxwell_param), label='maxwell_param')
plt.plot(x, stats.logistic.cdf(x, *logistic_param), label='logistic_param')
plt.plot(x, stats.laplace.cdf(x, *laplace_param), label='laplace')
plt.legend()
plt.show()

# visually, we can eliminate the Maxwell distribution

# K-S tests
ks_norm = stats.kstest(rv, 'norm', norm_param)
ks_laplace = stats.kstest(rv, 'laplace', laplace_param)
ks_maxwell = stats.kstest(rv, 'maxwell', maxwell_param)
ks_logistic = stats.kstest(rv, 'logistic', logistic_param)

# ks_norm.pvalue -> 0.00016185237120902585
# ks_laplace.pvalue -> 0.75696987136141258
# ks_maxwell,pvalue -> 0.0
# ks_logistic.pvalue -> 0.14502969211826544

# with alpha=0.05, accept either the laplace or logistic.
# however, it is clear to be laplace due to the high p-value.

# note that we are looking for a LARGE p-value
# to accept the null hypothesis of coming from that distribution.


import numpy as np

# Core functions for logistics regression

def sigmoid(z):
    return 1 / (1 + np.exp(-z))

def hypothesis(X, theta):
    return X @ theta

def cost(theta,X,y):
    m,n = X.shape
    hx = sigmoid(hypothesis(X,theta.reshape(len(theta),1)))
    return np.sum((-y * np.log(hx)) - ((1-y) * np.log(1-hx))) * 1/m

def gradient(theta, X,y):
    m,n = X.shape
    hx = sigmoid(hypothesis(X,theta.reshape(len(theta),1)))
    g = ((hx - y).T @ X) * 1/m
    return g.flatten() if theta.ndim==1 else g

def map_features(x1,x2,degree):
    features = np.ones((len(x1),1))
    for i in range(1,degree+1):
        for j in range(0,i+1):
            z = np.power(x1 , (i-j)) * np.power(x2 , j)
            features = np.hstack([features,z])
    return features

def cost_regularized(theta, X, y, lmbda):
    m,n = X.shape
    n_cost = cost(theta,X,y)
    n_cost += np.sum((theta[2:len(theta):] ** 2)) * (lmbda/(2*m))
    return n_cost

def gradient_regularized(theta,X,y,lmbda):
    grad = gradient(theta,X,y)
    m,n = X.shape
    return grad + np.hstack([[[0]], ((lmbda/m) *  theta[1:,:]).T])


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# @Date    : Aug-13-20 17:12
# @Author  : Your Name (you@example.org)
# @RefLink : https://numpy.org/doc/stable/reference/generated/numpy.linalg.eig.html#numpy.linalg.eig
# @RefLink : https://numpy.org/doc/stable/reference/generated/numpy.linalg.solve.html

import numpy as np

def solve_test():
    """solve equation a * x = b
    """
    a = np.array([[3, 1], [1, 2]])
    b = np.array([9, 8])
    x = np.linalg.solve(a, b)
    print(f"x:{x}")

def eigen_test():
    """
    v: The normalized (unit “length”) eigenvectors, such that the column v[:,i] is the eigenvector corresponding to the eigenvalue w[i].
    """
    # A = np.matrix([[1, 2, 3], [4, 5, 6], [7, 8, 9]])
    A = np.asarray([[1, 2], [3, 4]])
    print(f"A:{A}")

    w, v = np.linalg.eig(A)

    print(f"w:{w}")
    print(f"v:{v}")
    A_restore = np.dot(np.dot(v, np.diag(w)), np.linalg.inv(v))
    print(f"A_restore:{A_restore}")

    error = np.sum(A_restore-A)
    print(f"error:{error}")

def main():
    eigen_test()

if __name__ == "__main__":
    main()


import numpy as np

def unit_step(t):
    """ Takes time as argument and returns a unit-step function """
    return 1*(t>=0)

def dirac_delta(t,Delta=1e-3):

	x_t = (1/Delta)*(unit_step(t+Delta/2) - unit_step(t-Delta/2) )
	return x_t

def Fourier_Transfrom(x_t,t):
	""" Computes CTFT using FFT algorithm.
	Frequency and amplitude are scaled appropriately.
	It is assumed that the time varible is equispaced.
	Numpy FFT algorithm is used to compute the Fourier Transfrom,
	therefore it is also assumed that the signal is periodic in time
	with a period T_range.
	Note that you should define the time from -T_range/2 to +T_range/2"""

	N_pts = np.size(x_t)
	T_range = t.max() - t.min()

	delta_t = T_range/N_pts

	X_fft = np.fft.fft(x_t)
	X_fft = np.fft.fftshift(X_fft)
	X_omega = X_fft*delta_t

	omega = (2*np.pi/delta_t)*np.fft.fftshift(np.fft.fftfreq(N_pts))
	#omega = (2*np.pi/delta_t)*np.fft.fftfreq(N_pts)

	return X_omega, omega

def Inverse_Fourier_Transform(X_omega,omega):
	"""Computes and returns continuous inverse Fourier Transfrom using numpy
	ifft function. Time and frequency are scaled appropriately."""

	omega_range = omega.max() - omega.min()
	delta_omega = omega_range/np.size(omega)

	x_t_ifft = np.fft.ifft(np.fft.fftshift(X_omega))*omega_range/(2*np.pi)
	t_ifft = (2*np.pi/delta_omega)*np.fft.fftshift(np.fft.fftfreq(np.size(omega)))
	return x_t_ifft, t_ifft


# -*- coding: utf-8 -*-
"""Quadrature implementations"""

from __future__ import division

import numpy as np

from typing import Tuple
from .typing import ArrayLike

def clenshaw_curtis(n: int, a: float = -1.0, b: float = 1.0) -> Tuple[ArrayLike,ArrayLike]:
    """
    Computes the points and weights for a Clenshaw-Curtis integration
    from a to b. In other words, for the approximation to the integral

    \int_a^b f(x) dx \approx \sum_{i=0}^{n} w_i f(x_i)

    with the Clenshaw-Curtis quadrature, this function returns the
    positions x_i and the weights w_i.
    """
    assert b > a and n > 1

    npoints = n
    nsegments = n - 1
    theta = np.pi * np.flip(np.arange(npoints)) / nsegments
    xx = np.cos(theta) * 0.5 * (b - a) + 0.5 * (a + b)

    wcc0 = 1.0/(nsegments*nsegments - 1 + (nsegments%2))

    # build v vector
    v = np.zeros(nsegments)
    v[:nsegments//2] = 2.0/(1.0 - 4.0 * np.arange(nsegments//2)**2)
    v[nsegments//2] = (nsegments - 3) / (2 * (nsegments//2) - 1) - 1

    kk = np.arange(1, npoints//2)
    v[nsegments-kk] = np.conj(v[kk])

    # build g vector
    g = np.zeros(nsegments)
    g[:nsegments//2] = -wcc0
    g[nsegments//2] = wcc0 * ( (2 - (nsegments%2)) * nsegments - 1 )
    g[nsegments-kk] = np.conj(g[kk])

    h = v + g
    wcc = np.fft.ifft(h)

    # sanity check
    imag_norm = np.linalg.norm(np.imag(wcc))
    assert imag_norm < 1e-14

    out = np.zeros(npoints)
    out[:nsegments] = np.real(wcc)
    out[nsegments] = out[0]
    out = np.flip(out) # might be redundant, but for good measure
    out *= 0.5 * (b - a)

    return xx, out

def midpoint(n: int, a: float = -1.0, b: float = 1.0) -> Tuple[ArrayLike,ArrayLike]:
    """
    Returns the points and weights for a midpoint integration
    from a to b. In other words, for the approximation to the integral

    \int_a^b f(x) dx \approx \frac{b-a}{n} \sum_{i=0}^n f((x_0 + x_1)/2)
    """
    assert b > a and n > 1

    weights = np.ones(n) * (b - a) / n
    points = a + ((b - a) / n * (np.arange(n) + 0.5))

    return points, weights

def trapezoid(n: int, a: float = -1.0, b: float = 1.0) -> Tuple[ArrayLike,ArrayLike]:
    """
    Returns the points and weights for a trapezoid integration
    from a to b. In other words, for the approximation to the integral

    \int_a^b f(x) dx \approx \frac{b-a}{n} \sum_{i=0}^n f((x_0 + x_1)/2)
    """
    assert b > a and n > 1

    ninterval = n - 1

    weights = np.ones(n) * (b - a) / ninterval
    weights[0] *= 0.5
    weights[-1] *= 0.5
    points = a + ((b - a) / ninterval) * np.arange(n)

    return points, weights

def simpson(n: int, a: float = -1.0, b: float = 1.0) -> Tuple[ArrayLike,ArrayLike]:
    """
    Returns the points and weights for a simpson rule integration
    from a to b. In other words, for the approximation to the integral

    \int_a^b f(x) dx \approx \frac{b-a}{n} \sum_{i=0}^n f((x_0 + x_1)/2)
    """
    assert b > a and n > 1

    if n%2 != 1:
        raise Exception("Simpson's rule must be defined with an odd number of points (even number of intervals)")

    ninterval = n - 1

    weights = np.ones(n)
    for i in range(1, ninterval-1, 2):
        weights[i] = 4.0
        weights[i+1] = 2.0
    weights[ninterval-1] = 4.0
    weights *= (b-a) / ninterval / 3.0

    points = a + ((b - a) / ninterval) * np.arange(n)

    return points, weights

def quadrature(n: int, a: float = -1.0, b: float = 1.0, method: str = "gl") -> Tuple[ArrayLike,ArrayLike]:
    """
    Returns a quadrature rule for the specified method and bounds
    """
    if method.lower() == "cc" or method.lower() == "clenshaw-curtis":
        return clenshaw_curtis(n, a, b)
    elif method.lower() == "gl" or method.lower() == "gauss-legendre":
        points, weights = np.polynomial.legendre.leggauss(n)
        points = points * 0.5 * (b - a) + 0.5 * (a + b)
        weights *= 0.5
        return points, weights
    elif method.lower() == "midpoint" or method.lower() == "mp":
        return midpoint(n, a, b)
    elif method.lower() == "trapezoid":
        return trapezoid(n, a, b)
    elif method.lower() == "simpson":
        return simpson(n, a, b)
    else:
        raise Exception("Unrecognized quadrature choice")


# %%
import numpy as np
import sympy as sym

# %%

def lagrange_fn(x_array, y_array, out_type):
    # Inputs-
    # x_array: evaluation points
    # y_array: function values at evaluation points
    #          (associated with the evaluation points of the same index)
    # out_type: 3 options - "fun" returns a function, "sym" returns symbolic
    # math object, "lat" returns latex code for the function

    # for consistency ensure arrays are numpy arrays
    x_array = np.asarray(x_array)
    x_array = np.asarray(x_array)

    # catch easy error
    if len(x_array) != len(y_array):
        raise ValueError('arrays not same length')

    # find length of array
    n = len(x_array)
    # create x
    x = sym.symbols('x')
    # create var
    expr = 0

    for i in range(n):
        # reset term
        term = 1
        for j in range(n):
            if i != j:
                # Add factor
                term = term * ((x - x_array[j]) / (x_array[i] - x_array[j]))

        # addend new term with forrect function value
        expr = expr + y_array[i] * term

    # simplify expression (effect will probably depend on terms)
    # Presumably will convert to the monomial basis(if you aren't a fan you
    # can comment this out)
    expr = sym.simplify(expr)

    # Function case
    if out_type == "fun":
        fun_out = sym.lambdify(x, expr)

    # symbolic math object case
    if out_type == "sym":
        fun_out = expr

    # Latex code case
    if out_type == "lat":
        fun_out = sym.latex(expr)

    return fun_out

    raise ValueError('bad type (need to set out_type as fun, sym, or lat)')


import numpy as np
import numpy.polynomial as pol
from typing import Callable, List, Union
Vector = List[float]
Function = Callable[[float], float]
Polynomial = pol.Polynomial

def lagrange2(X: Vector, Y: Vector) -> Function:
    """Returns the lagrange interpolation
    Args:
        X (Vector): X-data
        Y (Vector): Y-data
    Returns:
        Callable[[Vector], Vector]: returns the function that evaluates the lagrange interpolation
    """
    X = np.asarray(X)

    def lagran(x):
        # Checks the input's type
        if type(x) is np.ndarray:
            x = x.reshape(-1, 1)

        if isinstance(x, list):
            x = np.array(x).reshape(-1, 1)

        else:
            x = np.array([x]).reshape(-1, 1)

        out = 0
        for i in range(len(X)):
            #pi_x = (x - np.array([(X[X != xi]).reshape(-1)] * len(x))) because X[X != xi] autoreshape (-1)
            pi_x = x - np.array([(X[X != X[i]])] * len(x))

            out += Y[i] * (pi_x / (X[i] - X[X != X[i]])).prod(axis = 1)

        return out

    return lagran

def lagrange(X: Vector, Y: Vector) -> Polynomial:
    p = 0
    for i in range(len(X)):
        qn = pol.Polynomial.fromroots(X[X != X[i]])
        p += Y[i] * qn / qn(X[i])
    return p


# coding: utf-8

#
Table of Contents

#

1  Linear Regression

1.1  Baseline your model

1.1.1  Plot the initial dataset

1.1.2  Split datasets, fit a linear regression and measure $R^2$ over the test set

1.1.3  Interpretation

1.1.4  Visualization

1.1.5  Cross Validation

1.1.6  Main pipeline call

1.1.7  Visualize Validation Scores

1.1.8  Increase the number of folds in train and test

1.2  Using SKLEARN Pipelines

1.3  Features preprocessing

1.3.1  Simple Cross Validation score

1.4  Multiple Linear Regression

1.4.1  What's important in Multiple linear regression

1.5  Bias and variance over training

1.5.1  Plot the polynomial expression.

1.5.2  Training size effect

1.5.3  What about the degree of the polynomial?

# # Linear Regression
# Let's practice with basic linear regression concepts in Python. We will read the Advertising dataset used in the ISLR book, and we will perform linear with single and multiple coefficients.

# In[1]:

import random
import sys
import pandas as pd
import matplotlib.pyplot as plt
import numpy as np
import seaborn as sns
import statsmodels.api as sm

from sklearn.pipeline import Pipeline, make_pipeline
from sklearn.preprocessing import PolynomialFeatures, StandardScaler
from sklearn.model_selection import cross_validate, cross_val_score,         train_test_split, learning_curve, validation_curve
from sklearn import datasets, linear_model
from sklearn.metrics import mean_squared_error, r2_score
from sklearn.model_selection import cross_val_score, ShuffleSplit
from sklearn_pandas import DataFrameMapper
from statsmodels.api import add_constant

import warnings
warnings.filterwarnings('ignore')

# ## Baseline your model
#
# Start by reading the data into a pandas DataFrame

# In[2]:

url="http://www-bcf.usc.edu/~gareth/ISL/Advertising.csv"
adv=pd.read_csv(url)
adv.head(5)

# Remove the first column containing the index of each row as it's not necessary when using Pandas.

# In[3]:

adv = adv.drop(adv.columns[0], axis=1)

# There're only have fours predictors or variables or dimensions to play with. Our **target** variable is **Sales**: We want to predict the amount of sales that we will get, basen on the money that we invest in TV, radio or newspaper advertising channels.

# ### Plot the initial dataset
#
# Plot the two variables to see how they look like.

# In[4]:

plt.scatter(adv.TV, adv.sales,  color='grey')
plt.xlabel('TV'); plt.ylabel('Sales')
plt.show();

# ### Split datasets, fit a linear regression and measure $R^2$ over the test set
#
# As usual, we must start by splitting the dataset into training & test. This time I will do once again to get a validation split. To do so, I will appply the `train_test_split` twice over the training set to get the validation from there.
#
# The idea is to finally evaluate the model using the validation set.

# In[5]:

X = pd.DataFrame(adv.TV)
y = pd.DataFrame(adv, columns=['sales'])

X_train, X_test, y_train, y_test = train_test_split(
    X, y, test_size=0.2, random_state=25)
X_train, X_val, y_train, y_val = train_test_split(
    X_train, y_train, test_size=0.2, random_state=12)

lm = linear_model.LinearRegression()
lm.fit(X_train, y_train)
y_pred = lm.predict(X_test)

print('Intercept:', lm.intercept_)
print('Coefficients:', lm.coef_)
print('Mean squared error (MSE): {:.2f}'.format(mean_squared_error(y_test, y_pred)))
print('Variance score (R2): {:.2f}'.format(r2_score(y_test, y_pred)))

# ### Interpretation
#
# - The intercept is the expected mean value of the target variable when all predictors are set to 0.
# - The coeffient is the slope of the line that explains the regression.
# - The R2 is the percentage of explained variance of the predictions.
#
# But, to really see how good is our model, we must evaluate the score ($R^2$) over a validation split --a different split of the data set. We will understand later, why this is important. I'm getting $R^2=0.51$ over this specific test set.
#
# However, if you check the results below, you'll see that the $R^2$ got when predicting over the validation dataset are slightlit better: from $R^2=0.51$ to $R^2=0.64$ !!
#
# **Why is that?** In this case, for this specific splits, it turns out that the validations split is probably more similar to the training set, than the test set. And that is why our model is better predicting values from that split, than from the test split.

# In[6]:

y_generalize = lm.predict(X_val)

print('Intercept:', lm.intercept_)
print('Coefficients:', lm.coef_)
print('Mean squared error: {:.2f}'.format(mean_squared_error(y_val, y_generalize)))
print('Variance score (R2): {:.2f}'.format(r2_score(y_val, y_generalize)))

# ### Visualization
#
# Now, plot the original points used to train the linear regression (grey) and the line representing the function that better approximates this relationship between TV and sales that we got from applying the linear regression.
#
# If you look at the code, you'll see that the way to obtain the values of Y later plotted using `plt.plot` come from directly applying the function describing the linear regression:
#
# $$\hat{y} = \beta_0 + \beta_1 X_{val}$$
#
# Correspondence:
# - $\beta_0$ is `lm.intercept_`
# - $\beta_1$ is `lm.coef_[0]`

# In[7]:

plt.scatter(X_train, y_train, color='gray')
x_vals = X_val.TV.values
y_vals = lm.intercept_ + lm.coef_[0] * x_vals
plt.plot(x_vals, y_vals, '--');

# ### Cross Validation
#
# As you might possible have figured out, we haven't used the validation dataset, and our model has learnt only from the first (and only) split that we made between training and test. In order to improve our results, a good approach is to repeat the splitting process again and again to obtain the best possible subset of individuals that lead to the best possible model.
#
# To do so, we should repeat the split and validation in a loop to select the best one. Here you can find a possible way of accomplishing that task.

# In[8]:

def split_datasets(X, y, seed,
                   split_test_size=0.2,
                   validation_split=True):
    """
    Split X and y dataframes into training, test and validation datasets,
    using the provided random seed.
    Returns a dictionary with the datasets
    """
    X_train, X_test, y_train, y_test = train_test_split(
        X, y, test_size=split_test_size, random_state=seed)

    split = dict()
    if validation_split is True:
        X_train, X_val, y_train, y_val = train_test_split(
            X_train, y_train, test_size=split_test_size, random_state=seed)
        split['X_val'] = X_val
        split['y_val'] = y_val

    split['X_train'] = X_train
    split['X_test'] = X_test
    split['y_train'] = y_train
    split['y_test'] = y_test

    return split

# In[9]:

def prepare_datasets(data, features, target,
                     seed=1024,
                     test_size=0.2,
                     validation_split=False):
    """
    From an input dataframe, separate features from target, and
    produce splits (with or without validation).
    """
    X = pd.DataFrame(adv, columns=features)
    y = pd.DataFrame(adv.loc[:, target])
    split = split_datasets(X, y, seed=seed,
                           split_test_size=test_size,
                           validation_split=validation_split)
    return split

# In[10]:

def train_and_predict(X, y, num_folds=1):
    """
    Call the splitting of the dataset, train a linear regression
    and predict with the test set.
    Returns the model, the splits used and the R2 score.
    """
    lm = linear_model.LinearRegression()
    test_r2 = 0.0
    for i in range(num_folds):
        seed = random.randrange(2**32 - 1)
        split = prepare_datasets(adv, ['TV'], 'sales',
                                 seed=seed, validation_split=True)

        lm.fit(split['X_train'], split['y_train'])
        y_pred = lm.predict(split['X_test'])
        test_r2 += r2_score(split['y_test'], y_pred)

    test_r2 = test_r2/num_folds
    return lm, split, test_r2

# In[11]:

def validate(model, X_val, y_val):
    """
    Scores a model over the validation datasets, by measuring the R2 score.
    Returns the r2.
    """
    y_generalized = lm.predict(X_val)
    r2 = r2_score(y_val, y_generalized)
    return r2

# In[12]:

def my_pipeline(X, y, max_iterations, num_folds=1):
    """
    Repeats during `max_iterations` the process of
      1) Splitting a dataset, training a linear regression
         and measuring its score over test set
      2) Store its R2 and compute its R2 over the validation dataset
    """
    history = dict()
    history['test_r2'] = np.array([])
    history['val_r2'] = np.array([])
    for num_iterations in range(max_iterations):
        model, split, test_r2 = train_and_predict(X, y, num_folds=num_folds)
        val_r2 = validate(model, split['X_val'], split['y_val'])
        history['test_r2'] = np.append(history['test_r2'], test_r2)
        history['val_r2'] = np.append(history['val_r2'], val_r2)
    return history

# ### Main pipeline call
#
# We are ready now to call the pipeline. The required arguments are the X and y dataframes, and the numner of iterations we want it to run. We can also specify how many runs we want each model to be trained over the same split of training and test, which by default is 1. We will keep it like that for the moment.
#
# It's quite impressive to check how our simple linear regression jumps from a bad 0.64 validation $R^2$ to an impressive $0.84$ ! And only by repeating the shuffling of the training and test sets.

# In[13]:

scores = my_pipeline(X, y, max_iterations=100)
max_val_r2 = np.amax(scores['val_r2'])
print('Best Validation R2: {:.2f}'.format(max_val_r2))

# ### Visualize Validation Scores
#
# To get an idea of what we got, let's draw a density plot with the validation score values saved along the different iterations. The density plot illustrates how important is to shuffle the training, test and validation sets to find out what is the one which is producing the best generalization.

# In[14]:

sns.distplot(scores['val_r2'], hist = False, kde = True,
             kde_kws = {'shade': True, 'linewidth': 3});

#
# ### Increase the number of folds in train and test
#
# What if I increase the number of folds from 1 to 10? The results are plotted below and there's n significant improvement in the results of the validation scores.
#
# Why? Well, It's namely because of the way we implemented our train & predict method: we're training `num_folds` linear regression without keeping it for test scoring. If we do that, we will explore which model will better generalize over different test sets, but keeping a single training split. And that will for sure, be a better method to assess over the the validation split.

# **Important**
#
# By partitioning the available data into three sets, we drastically reduce the number of samples which can be used for learning the model, and the results can depend on a particular random choice for the pair of (train, validation) sets.
#
# A test set should still be held out for final evaluation, but the validation set is no longer needed when doing CV. In the basic approach, called k-fold CV, the training set is split into k smaller sets.

# In[15]:

score_2 = my_pipeline(X, y, max_iterations=100, num_folds=10)
max_val_r2 = np.amax(score_2['val_r2'])
print('Best Validation R2: {:.2f}'.format(max_val_r2))

fig, ax = plt.subplots()
sns.distplot(scores['val_r2'], hist = False, kde = True,
             label='1-fold',
             kde_kws = {'shade': True, 'linewidth': 3}, ax=ax);
sns.distplot(score_2['val_r2'], hist = False, kde = True,
             label='10-fold',
             kde_kws = {'shade': True, 'linewidth': 3}, ax=ax);
plt.legend(loc='best')
plt.show();

# ## Using SKLEARN Pipelines
#
# Instead of having you to program everything from scratch, you can also access `sklearn` API for building preprocessing and cross validation pipelines. In the example below, we put in practice the very same approach used before, but using only three lines of code (!).
#
# I strongly recommend you to explore deeper the capabilities of the cross validation functions in SKLearn.

# ## Features preprocessing
#
# Before we go deeper, we must ensure that our features fulfill the set of requirements that are imposed by the linear regression method: centered and with variance = 1, following a normal ditribution and no correlation among them, namely. To do so, we will build a transformation pipeline. For more details, please, refere to the page: https://www.kaggle.com/baghern/a-deep-dive-into-sklearn-pipelines
#
# This case we're using the sklearn-pandas integration library `sklearn_pandas` which makes transformation pipelines much easier to read and understand. In the example below we're simply applying a standard scaler.

# In[16]:

mapper = DataFrameMapper([(list(adv), StandardScaler())], df_out=True)
res = np.round(mapper.fit_transform(adv), 2)
res.columns = list(adv)  # Set the columns names as they were.

# ### Simple Cross Validation score
#
# As we said, instead of having to program every single function in charge of splitting the datasets, setting the random seed, feeding a model with the training part and evaluating with test part, we will explore the `cross_val_score` method. It will be able to produce a single scoring (if you want more than one, you must use `cross_validate`) and will allow you to decide what CV strategy to use, or build a custom one.
#
# In the example below, we feed the `cross_val_score` with
# - the model (or pipeline) to be fit (`model`),
# - the features (`X`),
# - the target datasets (`y`),
# - the cross validation strategy (`cv`), and
# - the validation metric we want as a result (`r2`)
#
# This will run the cross validation with the model over the features and target using a ShuffleSplit (similar to _k-fold_ but with random splits), _n_splits_ (1000) times, evaluating with a 20% split as a test set, and returning the R2 achieved.

# In[17]:

model = linear_model.LinearRegression()
cv = ShuffleSplit(n_splits=100, test_size=0.2, random_state=666)
scores = cross_val_score(model, X, y, cv=cv, scoring='r2')
print('Best Validation R2: {:.2f}'.format(max(scores)))

sns.distplot(scores, hist = False, kde = True,
             kde_kws = {'shade': True, 'linewidth': 3});

# ## Multiple Linear Regression
#
# We want to consider all possible predictors available when it comes to train model. So, it's time to use the other two columns in the original dataset: radio and newspapers.

# In[18]:

X = pd.DataFrame(adv.loc[:, ['TV','radio','newspaper']])
y = pd.DataFrame(adv, columns=['sales'])
X.head(2)

# To train a multiple linear regression, the only thing that we must do is to feed the model with all the variables that you want to consider for the final equation. Obviously, building a linear model with 3 features, implies that our model is better described in 3+1 dimensions, which makes this task slightly more difficult to plot.

# In[19]:

mlm, split, test_r2 = train_and_predict(X, y)
print('Multiple linear regression R2: {:.2f}'.format(test_r2))
print('Multiple linear regression coeffcients: {}'.format(mlm.coef_))

# Repeating our CV experiment with SKLearn, this time replacing the single-feature $X$ by a 3-features dataframe $X$, leads to a similar result ($0.96$).

# In[20]:

clf = linear_model.LinearRegression()
cv = ShuffleSplit(n_splits=100, test_size=0.2, random_state=0)
scores = cross_val_score(clf, X, y, cv=cv, scoring='r2')
print('Best Validation R2: {:.2f}'.format(max(scores)))

sns.distplot(scores, hist = False, kde = True,
             kde_kws = {'shade': True, 'linewidth': 3});

# ### What's important in Multiple linear regression
#
# There's two concepts that are important when running a multiple linear regression: _p_values_ and the R-squared. Take a look to the values below in order to understand if this multiple linear regression is a better approximation to the original problem than the single-predictor linear regression.

# In[21]:

X = add_constant(X)
model = sm.OLS(y, X).fit()
print(model.summary())

# ## Bias and variance over training
#
# One of the advantages of linear regression is that we can make it as much complicated or fitted to data as we want. We simply add polynomial interactions between variables the following way:
#
# - Split datasets to hold out a test set, and use the training set for a cross-validation model selection
# - Build an sklearn _pipeline_ to randomly produce polynomial expressions before fitting a linear regression
# - Perform a CV validation score over the test set to check how do the models perform.

# We will extract some parts of the code to separate functions for the sake of code clarity, and better readibility. The first function (`xy_values`) will simply produce a dataframe with columns `x` and `y` after applying the prediction stage to an input dataframe. The second function (`polynomial_pipeline`) will construct a pipeline in which at first, a polynomial is found over input dataset, and second, a linear regression is fit (OLS) with that polynomial.

# In[22]:

def xy_values(pipeline, X):
    """ Returns the x and y values sorted by X in a dataframe """
    y_pred = pipeline.predict(X)
    return pd.DataFrame({'x': list(X.values), 'y': list(y_pred)}).                        sort_values(by=['x'])

# In[23]:

def polynomial_pipeline(X, y, degree):
    """
    Build a pipeline with polinomial expressions and linear regression over it.
    """
    cv = ShuffleSplit(n_splits=100, test_size=0.2, random_state=555)
    poly = PolynomialFeatures(degree=degree, interaction_only=False,
                              include_bias=False)
    lm = linear_model.LinearRegression()
    pipeline = make_pipeline(poly, lm)
    pipeline.fit(X, y)
    return pipeline

# In[24]:

split = prepare_datasets(adv, ['TV'], 'sales')
pipeline = polynomial_pipeline(split['X_train'], split['y_train'], degree=2)
scores = cross_val_score(pipeline, split['X_test'], split['y_test'],
                         scoring="r2", cv=cv)
print('The polynomial features used are:\n',
      pipeline.get_params('polynomialfeatures')['polynomialfeatures'].\
      get_feature_names())
print('Best Validation R2: {:.2f}'.format(max(scores)))

# ### Plot the polynomial expression.
#
# As you can easily check for this simulation, the higher degree polynomial means a better fit to the training but poorer generalization. It seems that here we might be overfitting a bit.

# In[25]:

f, (ax1, ax2) = plt.subplots(nrows=1, ncols=2)
f.set_figwidth(11)
ax1.scatter(split['X_train'].values, split['y_train'].values, color='grey')

pred = xy_values(pipeline, split['X_test'])
ax1.plot(pred.x.values, pred.y.values, label='prediction',
         linewidth=3, color='green')
ax1.grid()
ax1.set_title('2 degree polynomial prediction')

ax2.hist(scores[scores>0], 50, normed=1, alpha=0.5);
ax2.set_title('histogram of scores')
plt.show();

# ### Training size effect
#
# A learning curve shows the validation and training score of an estimator for varying numbers of training samples. It is a tool to find out how much we benefit from adding more training data and whether the estimator suffers more from a variance error or a bias error.
#
# A cross-validation generator splits the whole dataset $k$ times in training and test data. Subsets of the training set with varying sizes will be used to train the estimator and a score for each training subset size and the test set will be computed. Afterwards, the **scores will be averaged** over all $k$ runs for each training subset size.
#
# Can you interpret what is the number of samples you need to feed your model in training, in order to get the best possible result? When adding more samples does not add significant advantage?

# In[26]:

def plot_training_curve():
    plt.plot(train_sizes, np.mean(train_scores, axis=1), '-o',
             label='train_scores')
    plt.plot(train_sizes, np.mean(valid_scores, axis=1), '-o',
             label='validation_scores')
    plt.title('Training curve')
    plt.legend(loc='best')
    plt.xlabel('Training set size')
    plt.ylabel('Explained variance ($R^2$)')
    plt.grid()
    plt.show()

# In[27]:

split = prepare_datasets(data=adv, features=['TV'], target='sales', test_size=0.0)
cv = ShuffleSplit(n_splits=100, test_size=0.2, random_state=666)
train_sizes, train_scores, valid_scores = learning_curve(
    lm, split['X_train'], split['y_train'],
    train_sizes=[50, 75, 100, 125, 150, 175],
    cv=10)
plot_training_curve()

# ### What about the degree of the polynomial?
#
# Let's work on the `sklearn.model_selection.validation_curve` to see how the degree of the polynomial affects the R2 score. We pass an array of possible degrees for the polynomials to the `validation_curve` method.
#
# Compute scores for an estimator with different values of a specified parameter (degree of polynomial). This is similar to grid search with one parameter. However, this will also compute training scores and is merely a utility for plotting the results.

# In[28]:

def plot_validation_curve():
    # Plot the mean train error and validation error across folds
    plt.figure(figsize=(6, 4))
    plt.plot(degrees, train_scores.mean(axis=1), '-o',
             lw=2, label='training')
    plt.plot(degrees, validation_scores.mean(axis=1), '-o',
             lw=2, label='cross-validation')
    plt.title('Validation curve')
    plt.legend(loc='best')
    plt.xlabel('degree of polynomial')
    plt.ylabel('explained variance ($R^2$)')
    plt.tight_layout()
    plt.grid()
    plt.show()

# In[29]:

split = prepare_datasets(data=adv, features=['TV'], target='sales', test_size=0.0)
lm = linear_model.LinearRegression()
degrees = np.arange(1, 10)
model = make_pipeline(PolynomialFeatures(), lm)
cv = ShuffleSplit(n_splits=100, test_size=0.2, random_state=666)

train_scores, validation_scores = validation_curve(
                 model, split['X_train'], split['y_train'],
                 param_name='polynomialfeatures__degree',
                 cv=cv,
                 param_range=degrees)
plot_validation_curve()

# It seems that increasing the degree of the polynomial beyond '2' produces a decrease in validation accuracy.
#
# Now try to iterate over different maximum degrees for the polinomials to be generated to see how they affect the overall score of the model.

# In[30]:

plt.figure(figsize=(6, 4))
plt.scatter(split['X_train'].values, split['y_train'].values, color='grey')

split = prepare_datasets(adv, ['TV'], 'sales', seed=1024)
cv = ShuffleSplit(n_splits=100, test_size=0.2, random_state=666)

for degree in range(1, 6):
    pipeline = polynomial_pipeline(split['X_train'], split['y_train'], degree)
    scores = cross_val_score(pipeline, split['X_test'], split['y_test'],
                             scoring="r2", cv=10)
    pred = xy_values(pipeline, split['X_test'])
    plt.plot(pred.x.values, pred.y.values,
             label='deg.{}'.format(degree))

plt.title('Linear regression with different polynomial degrees')
plt.legend(loc='best')
plt.grid()
plt.show()

import numpy,import statsmodels,from statsmodels


import pathlib  # needed to create folder
import matplotlib.pyplot as plt  # needed for graphs
from tqdm import tqdm
import numpy as np

print('Program started running!')

pi = np.pi
dpisiz = 1000  # for images resolution

#######################################
# Lambda anonymous functions          #
#######################################

f = lambda x, e, k: x - k - e * np.sin(x)
df = lambda x, e: 1 - e * np.cos(x)

########################################
# Creating Our Newton-Raphson function #
########################################
def newton_raphson(f, df, x, e, TOL, k):
    error = 1
    iterations = 0
    while error > TOL:
        new_x = x - f(x, e, k) / df(x, e)
        error = abs(new_x - x)
        x = new_x
        iterations += 1
        # print(f'x{iterations}: {x:.15f} \t | \t error: {error} \t | \t f(x):{f(x, e,k)} \t | \t df(x):{df(x, e)} \t | \t line:{df(x, e)} * x+{f(x, e,k)}')
    # print(f"Newton's Estimate = {x:.15f}\nIterations: {iterations}")
    return x

#####################################
# Function to save images to folder #
#####################################
def savim(dir, name):
    path = pathlib.Path(f"./{dir}")
    path.mkdir(exist_ok=True,  # Without exist_ok=True,FileExistsError show up if folder already exists
               parents=True)  # Missing parents of the path are created.
    plt.savefig(f'./{dir}/{name}.png', dpi=dpisiz)
    print(f'Saved image at location: /{dir}/{name}.png')

numberoftries = 10000
i = np.empty((numberoftries, 6))
i[:, 0] = np.linspace(0, 2 * pi, numberoftries)
e = [0.1, 0.3, 0.5, 0.7, 0.9]

for j in tqdm(range(1, 6)):
    for k in range(numberoftries):
        i[k, j] = newton_raphson(f, df, 1, e[j - 1], 1e-15, i[k, 0])  # Adding NP-estimates to array
e = [0.1, 0.3, 0.5, 0.7, 0.9]

# Plotting our function
fig = plt.figure()
ax = plt.axes()
lines = ax.plot(i[:, 0], i[:, 0:5])
ax.set_ylabel('E')
ax.set_xlabel('M')
ax.set_title('Project 1 - Task 1 - Newton-Raphson method')
labels = [0] * 5
for o in range(0, 5):
    labels[o] = f'E(M) for e={e[o]}'
ax.legend(lines, labels)
savim('./pr1_task1/images', 'plot')

print('Program finished running!')

##############################################################################
#    A faster convergence could be achieved with a better initial value.     #
#          Current code uses a standard initial value of one.                #
##############################################################################


# -*- coding: utf-8 -*-
"""
Created on Wed Oct  7 08:25:49 2020
@author: Ashlee
ODE Example 1
$\frac{dx}{dt} = b1-b2*x
"""

import numpy as np
from scipy.optimize import curve_fit
from scipy.integrate import odeint
import matplotlib.pyplot as plt

# Data for example 1
xaxisData = np.array( [0.0, 0.5, 1.0, 5.0, 30.0] ) # time, independent variable
yaxisData = np.array( [0.0, 0.5, 1.2, 2.5, 2.7] ) # x, dependent variable

# guesses for parameters
b1guess = 1.0
b2guess = 1.0
parameterguesses = np.array([b1guess, b2guess])

# Need two functions for our model
# 1. to define the system of ODE(s)
# 2. to solve the ODE(s) and return ypredicted values in same shape as yaxisData

# 1. define ODE
def system_of_ODEs(x,t,parameters): # yvar, xvar, args
    # unpack the parameters
    b1 = parameters[0]
    b2 = parameters[1]
    dxdt = b1-b2*x
    return dxdt
# end of function

# 2. Solve ODEs at xaxisData points
    # and return calculated yaxisCalculated
    # using current values of the parameters
def model(xaxisData,*params):
    # initial condition(s) for the ODE(s)
    yaxis0 = 0.0 # should include a decimal
    yaxisCalc = np.zeros(xaxisData.size)
    for i in np.arange(0,len(xaxisData)):
        if xaxisData[i] == 0.0: # should include a decimal
            yaxisCalc[i] = yaxis0
        else:
            xaxisSpan = np.linspace(0,xaxisData[i],101)
            ySoln = odeint(system_of_ODEs,yaxis0,xaxisSpan,args = (params,)) # soln for entire xaxisSpan
            yaxisCalc[i] = ySoln[-1] # calculated y at the end of the xaxisSpan
    return yaxisCalc
    # end of for loop
# end of model function

# Estimate the parameters
parametersoln, pcov = curve_fit(model,xaxisData,yaxisData,p0=parameterguesses)
print(parametersoln)
plt.plot(xaxisData,yaxisData,'o',label='data')
yaxis0 = 0.0
xaxisForPlotting = np.linspace(0,xaxisData[-1],101)
yaxisCalcFromGuesses = odeint(system_of_ODEs,yaxis0,xaxisForPlotting,args = (parameterguesses,))
yaxisCalc = odeint(system_of_ODEs,yaxis0,xaxisForPlotting,args = (parametersoln,))
plt.plot(xaxisForPlotting,yaxisCalcFromGuesses,'r-',label='output with parameter guesses') # before fitting
plt.plot(xaxisForPlotting,yaxisCalc, 'g--', label='output with estimated parameters') # at soln parameters
plt.xlabel('t')
plt.ylabel('x')
plt.legend()
plt.show()

CHEclassFa20/In Class Problem Solutions/Python/ODEParamEstimExample1.py


import matplotlib.pyplot as plt
import numpy as np
import imageio
from _utils import *

def DCT2D(x):
    '''
    Discrete space cosine transform
    x: Input matrix
    '''
    N1, N2 = x.shape
    X = np.zeros((N1, N2))
    n1, n2 = np.mgrid[0:N1, 0:N2]
    for w1 in range(N1):
        for w2 in range(N2):
            l1 = (2/N1)**0.5 if w1 else (1/N1)**0.5
            l2 = (2/N2)**0.5 if w2 else (1/N2)**0.5
            cos1 = np.cos(np.pi*w1*(2*n1 + 1)/(2*N1))
            cos2 = np.cos(np.pi*w2*(2*n2 + 1)/(2*N2))
            X[w1, w2] = l1*l2*np.sum(x*cos1*cos2)
    return X

def iDCT2D(X, shift=True):
    '''
    Inverse discrete space cosine transform
    X: Input spectrum matrix
    '''
    N1, N2 = X.shape
    x = np.zeros((N1, N2))
    k1, k2 = np.mgrid[0:N1, 0:N2]
    l1 = np.ones((N1, N2))*(2/N1)**0.5
    l2 = np.ones((N1, N2))*(2/N2)**0.5
    l1[0] = (1/N1)**0.5; l2[:,0] = (1/N2)**0.5
    for n1 in range(N1):
        for n2 in range(N2):
            cos1 = np.cos(np.pi*k1*(2*n1 + 1)/(2*N1))
            cos2 = np.cos(np.pi*k2*(2*n2 + 1)/(2*N2))
            x[n1, n2] = np.sum(l1*l2*X*cos1*cos2)
    return x

if __name__ == "__main__":
    image = imageio.imread('./sample/cameraman.png')
    s = 4
    image = image[::s, ::s] / 255
    N1, N2 = image.shape
    histogram(image, interval=[0, 1])

    IMAGE = DCT2D(image)
    xX = np.array([image, np.log10(1 + abs(IMAGE))])
    panel(xX, [2, 1], text_color='green',
          texts=['Input image', 'DCT Spectrum'])

    image_ = iDCT2D(IMAGE)
    Xx_ = np.array([np.log10(1 + abs(IMAGE)), image_])
    panel(Xx_, [2, 1], text_color='green',
          texts=['DCT Spectrum', 'Reconstructed image'])

    u, v = np.mgrid[0:N1, 0:N2] / max(N1, N2)
    r = (u ** 2 + v ** 2) ** 0.5
    theta = np.arctan2(v, u)
    H = np.exp(-3 * r ** 2) * (np.cos(4 * 2 * theta) / 2 + 1 / 2)
    image__ = iDCT2D(H * IMAGE)
    Hx__ = np.array([H, abs(image__ * 0.5 + 0.5)])
    panel(Hx__, (2, 1), text_color='green',
          texts=['Filter', 'Filtered image'])


import numpy as np
import matplotlib.pyplot as plt

# Return fitted model parameters to the dataset at datapath for each choice in degrees.
# Input: datapath as a string specifying a .txt file, degrees as a list of positive integers.
# Output: paramFits, a list with the same length as degrees, where paramFits[i] is the list of
# coefficients when fitting a polynomial of d = degrees[i].
def main(datapath, degrees):
    paramFits = []
    # fill in
    # read the input file, assuming it has two columns, where each row is of the form [x y] as
    # in poly.txt.
    # iterate through each n in degrees, calling the feature_matrix and least_squares functions to solve
    # for the model parameters in each case. Append the result to paramFits each time.

    file_path = open( datapath,  "r")
    X = []
    Y = []
    #X = []
    #Y = []
    y_1 = []
    y_2 = []
    y_3 = []
    y_4 = []
    #count = 0
    y_5 = []
    file_data_path = file_path.readlines()

    for i in file_data_path:
        i = i.split(" ")
        l1 = float(i[0])
        l2 = float(i[1])
        X.append(l1)
        Y.append(l2)

    for j in degrees:
        mitrx = feature_matrix(X, j)
        ##matrix
        paramFits.append(least_squares(mitrx, Y))

    xsort = sorted(X)

    for l in sorted(X):
        res1 = paramFits[0][0] * l + paramFits[0][1]
        res2 = paramFits[1][0] * (l ** 2) + paramFits[1][1] * l + paramFits[1][2]
        res3 = paramFits[2][0] * (l ** 3) + paramFits[2][1] * (l ** 2) + paramFits[2][2] * l + paramFits[2][3]
        res4 = paramFits[3][0] * (l ** 4) + paramFits[3][1] * (l ** 3) + paramFits[3][2] * (l ** 2) + paramFits[3][3] * l + paramFits[3][4]
        res5 = paramFits[4][0] * (l ** 5) + paramFits[4][1] * (l ** 4) + paramFits[4][2] * (l ** 3) + paramFits[4][3] * (l ** 2) + paramFits[4][4] * l + paramFits[4][5]
        y_1.append(res1)
        y_2.append(res2)
        y_3.append(res3)
        y_4.append(res4)
        y_5.append(res5)

    #close
    file_path.close()

    plt.scatter(X, Y, color='b', marker='*')

    plt.plot(sorted(X), y_1, color='g', linestyle='-.')
    plt.plot(sorted(X), y_2, color='b', linestyle='-.')
    plt.plot(sorted(X), y_3, color='m', linestyle='-.')
    plt.plot(sorted(X), y_4, color='y', linestyle='-.')
    plt.plot(sorted(X), y_5, color='r', linestyle='-.')

    plt.legend(["data = 1", "data = 2", "data = 3", "data = 4", "data = 5", "Path Data"], loc='upper right')
    plt.ylabel("Y Data")
    plt.xlabel("X Data")

    plt.show()
    ##
    return paramFits

# Return the feature matrix for fitting a polynomial of degree d based on the explanatory variable
# samples in x.
# Input: x as a list of the independent variable samples, and d as an integer.
# Output: X, a list of features for each sample, where X[i][j] corresponds to the jth coefficient
# for the ith sample. Viewed as a matrix, X should have dimension #samples by d+1.
def feature_matrix(x, d):
    # fill in
    # There are several ways to write this function. The most efficient would be a nested list comprehension
    # which for each sample in x calculates x^d, x^(d-1), ..., x^0.
    X_list = []
    ind = 0
    for i in x:
        d1 = d

        X_list.append([])

        while d1 >= 0:
            X_list[ind].append(i**d1)
            d1 -= 1
        # d = 1
        ind += 1
    return X_list

# Return the least squares solution based on the feature matrix X and corresponding target variable samples in y.
# Input: X as a list of features for each sample, and y as a list of target variable samples.
# Output: B, a list of the fitted model parameters based on the least squares solution.
def least_squares(X, y):
    X_array = np.array(X)
    # X_array = list(X)
    Y_array = np.array(y)

    # fill in
    # Use the matrix algebra functions in numpy to solve the least squares equations. This can be done in just one line.

    return (np.linalg.inv(X_array.T @ X_array)) @ (X_array.T @ Y_array)

if __name__ == '__main__':
    datapath = 'poly.txt'
    degrees = [1, 2, 3, 4, 5]
    paramFits = main(datapath, degrees)
    print(paramFits)


import numpy     as np

from typing     import Type, List, NoReturn, Union, Dict

RealNumber    = Union[np.float16, np.float32, np.float64, float];
ComplexNumber = Union[np.complex64, np.complex128];

def cayley_to_upper(z:ComplexNumber) -> ComplexNumber:
    #
    # Compute the image of z via the Cayley map into the uper half plane.
    #
    return 1j*(1.0+z)/(1.0-z);

def cayley_from_upper(z:ComplexNumber) -> ComplexNumber:
    #
    # Compute the image of z via the Cayley map from the uper half plane into the unit disk.
    #
    return (z-1j)/(z+1j);

def moebius_transformation(z:ComplexNumber,a:RealNumber,b:RealNumber,c:RealNumber,d:RealNumber) -> Union[ComplexNumber, NoReturn]:
    #
    # Compute the image of z via the Moebius transformation defined by the real paramters a,b,c,d.
    #
    if a*b-c*d==0 :
        raise ValueError("In order to be a proper Moebius Transformaion, it must be a*b-c*d!=0");
    else:
        return (a*z-b)/(c*z+d);

def moebius_unit_disk(z:ComplexNumber, theta:RealNumber=np.pi, a:ComplexNumber=np.complex(0,0)) -> ComplexNumber:
    #
    # Compute the image of z via the Moebius transformation which is an automorphism of the unit disk.
    #
    # Input:
    #   z       : complex number (most likely, in the unit disk)
    #   theta   : angle of rotation
    #   a       : point of the unit disk such that 0-->a
    #
    # Output:
    #   w       : complex number (in the unit disk)
    #
    return np.exp(1j*theta*np.pi)*(z-a)/(1-np.conjugate(a)*z);

def moebius_upper_half_plane(z:ComplexNumber,a:RealNumber,b:RealNumber,c:RealNumber,d:RealNumber) -> Union[ComplexNumber, NoReturn]:
    #
    # Compute the image of z via the Moebius transformation which is an automorphism of the upper-half plane.
    #
    # Input:
    #   z       : complex number (most likely, in the upper half plane)
    #   a,b,c,d : real parameters such that a*b-c*d>0
    #
    # Output:
    #   w       : complex number (in the upper half plane)
    #
    if not a*b-c*d>0 :
        raise ValueError("In order to be a proper Moebius Transformaion of the upper-half plane, it must be a*b-c*d>0");
    else:
        return (a*z-b)/(c*z+d);

def principal_sqrt(x:RealNumber) -> RealNumber:
    #
    # Compute the principal squareroot of the real number x.
    #
    if np.any(np.angle(x) > 0):
        return np.exp(1j * np.angle(x) / 2) * np.sqrt(np.sqrt(np.real(x)**2 + np.imag(x)**2));
    else:
        return np.exp(1j * (np.angle(x)+2*np.pi) / 2) * np.sqrt(np.sqrt(np.real(x)**2 + np.imag(x)**2));


from __future__ import division
from mpl_toolkits.mplot3d import Axes3D
import matplotlib.pyplot as plt
import pandas as pd
import numpy as np
import time

print("BSUIR: Machine Learning, L1")

def h0x(x_value, theta):
    return theta[0] + theta[1] * x_value

def compute_cost(X, Y, theta):
    m = len(X)
    diff = []
    for i in range(0, m):
        val = pow(h0x(X[i], theta) - Y[i], 2)
        diff.append(val)
    cost = (1 / (2 * m)) * sum(diff)
    return cost

def gradient_descent(X, Y, theta, iterations, alpha):
    """
    From Andrew Ng implementation: without ones vector in X
    """
    m = len(X)
    J = []
    for i in range(iterations):
        val = np.zeros(len(theta))
        for j in range(0, m):
            val[0] += h0x(X[j], theta) - Y[j]
            for k in range(1, len(theta)):
                val[k] += (h0x(X[j], theta) - Y[j]) * X[j]
        for z in range(0, len(theta)):
            theta[z] = theta[z] - (alpha / m) * val[z]
        J.append(compute_cost(X, Y, theta))
    return [theta, J]

def gradient_descent_not_vectorized(X, Y, theta, iterations, alpha):
    """
    For reducing impact of compute_cost_vectorized vs compute_cost
    """
    m = len(X)
    J = []
    for i in range(iterations):
        val = np.zeros(len(theta))
        for j in range(0, m):
            for k in range(0, len(theta)):
                val[k] += (h0x_vectorized(X[j], theta) - Y[j]) * X[j][k]
        for z in range(0, len(theta)):
            theta[z] = theta[z] - (alpha / m) * val[z]
        J.append(compute_cost_vectorized(X, Y, theta))
    return [theta, J]

def h0x_vectorized(X, theta):
    return X.dot(theta)

def compute_cost_vectorized(X, Y, theta):
    # J = (1 / (2 * m)) * (X * theta - y)' * (X * theta - y); % equally (sum(power(X, 2)))
    m = len(X)
    temp = (h0x_vectorized(X, theta) - Y)
    return (1 / (2 * m)) * np.dot(temp.T, temp)[0][0]

def gradient_descent_vectorized(X, Y, theta, iterations, alpha):
    m = len(Y)
    J_history = []
    for i in range(iterations):
        # theta = theta - alpha * (1/m) * (((X*theta) - y)' * X)'; % Vectorized
        h0x = (h0x_vectorized(X, theta) - Y).T
        dt = np.dot(h0x, X).T
        a = alpha * (1 / m) * dt
        theta = theta - a
        J_history.append(compute_cost_vectorized(X, Y, theta))
    return [theta, J_history]

def feature_normalization(X):
    X = X.T
    for i in range(1, len(X)):
        mu = np.mean(X[i])
        s = np.std(X[i], ddof=1)
        X[i] = (X[i] - mu) / s
    return X.T

def normal_eqn(X, Y):
    # theta = pinv(X' * X) * (X' * y); % Vectorized
    return np.dot(np.linalg.inv(np.dot(X.T, X)), np.dot(X.T, Y))

if __name__ == '__main__':
    # 1
    file_path = 'ex1data1.csv'
    data_frames = pd.read_csv(file_path)

    x = data_frames['population']
    y = data_frames['profit']

    x = list(x)  # np.array(x)
    y = list(y)

    # 2
    fig, ax = plt.subplots()
    ax.scatter(x, y)

    plt.show()

    # 3
    theta = [0, 0]
    print('With theta = [0 ; 0]\nCost computed: ', compute_cost(x, y, theta))
    print('Expected cost value (approx) 32.07\n')

    theta = [-1, 2]
    print('\nWith theta = [-1 ; 2]\nCost computed: ', compute_cost(x, y, theta))
    print('Expected cost value (approx) 54.24\n')

    # 4
    iterations = 1500
    alpha = 0.01

    print('Running Gradient Descent ...\n')
    # run gradient descent
    theta = [0, 0]
    [theta, J1] = gradient_descent(x, y, theta, iterations, alpha)

    print('Theta found by gradient descent:', theta)
    print("Cost: ", compute_cost(x, y, theta))
    print('Expected theta values (approx):  -3.6303  1.1664\n\n')

    ax.plot([4, 23], [h0x(0, theta), h0x(23, theta)], 'red')
    plt.show()

    # 5
    u = np.arange(-5, 5, 0.1)
    v = np.arange(-5, 5, 0.1)
    z = np.zeros((len(u), len(v)))
    for i in range(len(u)):
        for j in range(len(u)):
            z[i][j] = compute_cost(x, y, [u[i], v[j]])

    u, v = np.meshgrid(u, v)

    fig = plt.figure()
    ax = fig.gca(projection='3d')
    surf = ax.plot_surface(u, v, z, linewidth=0, antialiased=False)
    plt.show()
    fig, ax = plt.subplots()
    plt.contour(u, v, z, np.logspace(-2, 3, 20))
    plt.show()

    # 6
    file_path = 'ex1data2.csv'
    data = pd.read_csv(file_path)

    # 7-8
    X = data.iloc[:, 0:2]  # read first two columns into X
    Y = data.iloc[:, 2]  # read the third column into y
    m = len(Y)
    ones = np.ones((m, 1))
    X = np.hstack((ones, X))  # [x1, x2] => [1, x1, x2]
    theta = np.zeros((3, 1))
    Y = Y[:, np.newaxis]  # convert to a matrix

    print('With theta = [0; 0; 0]\nCost computed: ', compute_cost_vectorized(X, Y, theta))
    print('Expected cost value (approx) 65591548106.45744\n')

    print('Without normalization: \n')
    # Can not be calculated since numbers are too large
    # print(gradient_descent_vectorized(X, Y, theta, iterations, alpha)[0])
    print('With normalization: \n')
    X = feature_normalization(X)

    start1 = time.time()
    [theta, J1] = gradient_descent_not_vectorized(X, Y, theta, iterations, alpha)
    end1 = time.time()
    print('Time gradient not vectorized: ', end1 - start1, theta)

    start2 = time.time()
    [gdv, J] = gradient_descent_vectorized(X, Y, np.zeros((3, 1)), iterations, alpha)
    end2 = time.time()

    print('Vectorized time: ', end2 - start2)
    print('Solution: ')
    print(gdv)

    # 9

    objects = ('Vectorized', 'Not-Vectorized')
    y_pos = np.arange(len(objects))
    performance = [end2 - start2, end1 - start1]

    plt.bar(y_pos, performance, align='center', alpha=0.5)
    plt.xticks(y_pos, objects)
    plt.ylabel('Time (s)')
    plt.title('Performance')

    plt.show()

    # 10

    year = range(0, iterations)

    plt.plot(year, J, color='orange')
    plt.xlabel('Iterations')
    plt.ylabel('Cost J')
    plt.title('Cost by iterations')
    plt.show()

    # 11

    theta = normal_eqn(X, Y)
    print('Normal equation: \n')
    print(theta)


import numpy as np

def dist(X1,X2):
    """
    Compute the Euclidean distance between each row of X1 and X2
    """
    X1sq = np.sum(np.square(X1),1)
    X2sq = np.sum(np.square(X2),1)
    sqdist = -2.*np.dot(X1, X2.T) + (X1sq[:,None] + X2sq[None,:])
    sqdist = np.clip(sqdist, 0, np.inf)
    return sqdist

def d(x1,x2):
    ''' Compute absolute distance between all elments of two vectors and output outersubtract matrix '''
    return np.abs(np.subtract.outer(x1,x2))

def SE_kernel(X1,X2,theta):
    l = theta[1]
    sigma_f = theta[2]
    if l <=0 or sigma_f <= 0:
        print("Check hyperparameter values!")
    sqdist = dist(X1,X2)
    return sigma_f**2 * np.exp(-0.5*sqdist/(l**2))

def RQ_kernel(X1,X2,theta):
    alpha = 2
    l = theta[1]
    sigma_f = theta[2]
    if l <=0 or sigma_f <= 0:
        print("Check hyperparameter values!")
    sqdist = dist(X1,X2)
    return sigma_f**2 * (1+sqdist/(2*alpha*l**2))**(-alpha)

def camphor_copper_kernel(X1,X2,theta):
    #This hyperparameter vector will work well:  theta=[0.001,0.26,0.1] (given that 0.05 added to lengthscale of RBF for z-variable)
    #OLD hyperparameters that stood out in testing: theta=[0.09,0.2,0.35]
    l = theta[1]
    sigma_f = theta[2]
    if l <=0 or sigma_f <= 0:
        print("Check hyperparameter values!")
    ''' Same lenghtscale l across dimensions '''
    ''' Periodic kernels has same period p=1 since data normalized to [0,1] '''
    p = 1
    kernelX = np.exp((-2*np.square(np.sin(np.pi*d(X1[:,0],X2[:,0])/p)))/l**2)
    kernelY = np.exp((-2*np.square(np.sin(np.pi*d(X1[:,1],X2[:,1])/p)))/l**2)
    kernelZ = np.exp(-0.5*np.square(d(X1[:,2],X2[:,2]))/((l+0.05)**2))
    ''' NOTE: 0.05 added to lengthscale of RBF for z-variable'''
    kernelalpha = np.exp((-2*np.square(np.sin(np.pi*d(X1[:,3],X2[:,3])/p)))/l**2)
    kernelbeta = np.exp((-2*np.square(np.sin(np.pi*d(X1[:,4],X2[:,4])/p)))/l**2)
    kernelgamma = np.exp((-2*np.square(np.sin(np.pi*d(X1[:,5],X2[:,5])/p)))/l**2)
    return sigma_f**2 * kernelX * kernelY * kernelZ * kernelalpha * kernelbeta * kernelgamma


#!/usr/bin/env python3
# -*- coding: utf-8 -*-

import numpy as np
import matplotlib.pyplot as plt

def derivative(a,b,n):
    """derivative(a,b,n)
    Function to create the derivative matrix.  Takes 3 inputs:
        - a = float, starting point of the domain
        - b = float, end point of domain
        - n = number of points created in the domain
    Output: an 'n x n' 2-dimensional numpy array that, when matrix multiplied by a 1-D array,
    returns the approximate derivative at each point."""
    t = np.linspace(a,b,n)
    dx = t[1]-t[0]
    empty = np.zeros(n, dtype = 'float64')
    a,b = np.meshgrid(empty, empty)
    A = a + b
    #Forward Difference
    A[0,0] = -1/dx
    A[0,1] = 1/dx
    #Symmetric Difference
    index = np.arange(1,n-1)
    A[index, index-1] = -1/(2*dx)
    A[index, index+1] = 1/(2*dx)
    #Backward Difference
    A[n-1,n-2] = -1/dx
    A[n-1,n-1] = 1/dx
    return A

def second_derivative(a,b,n):
    """second_derivative(a,b,n)
    Function to create the second derivative matrix.  Takes 3 inputs:
        - a = float, starting point of the domain
        - b = float, end point of domain
        - n = number of points created in the domain
    Output: an 'n x n' 2-dimensional numpy array that, when matrix multiplied by a 1-D array,
    returns the approximate second derivative at each point."""
    t = np.linspace(a,b,n)
    dx = t[1]-t[0]
    empty = np.zeros(n, dtype = 'float64')
    a,b = np.meshgrid(empty, empty)
    A = a + b
    #Forward Difference
    A[0,0] = 1
    A[0,1] = -2
    A[0,2] = 1
    #Symmetric/Forward Difference
    A[1,0] = 2
    A[1,1] = -3
    A[1,3] = 1
    #Symmetric Difference
    index = np.arange(2,n-2)
    A[index, index-2] = 1/2
    A[index, index] = -1
    A[index, index+2] = 1/2
    #Symmetric/Backward Difference
    A[n-2,n-4] = 1
    A[n-2,n-2] = -3
    A[n-2,n-1] = 2
    #Backward Difference
    A[n-1,n-3] = 1
    A[n-1,n-2] = -2
    A[n-1,n-1] = 1
    return A*(1/(2*dx**2))

def f(a,b,n):
    """Returns an array of the square of the input (a float)"""
    t = np.linspace(a,b,n)
    return t**2

def s(a,b,n):
    """Returns an array of the sine of the input (float)"""
    t = np.linspace(a,b,n)
    sin = np.vectorize(np.sin)
    sin = sin(t)
    return sin

def g(a,b,n):
    """Returns an array of the gaussian function"""
    def gauss(x):
        """Returns the Gaussian function dependent on the input float"""
        c = 1/(np.sqrt(2*np.pi))
        gauss = c*np.exp(-x**2/2)
        return gauss
    t = np.linspace(a,b,n)
    gs = np.vectorize(gauss)
    gs = gs(t)
    return gs

def plot_function(a,b,n,f,string):
    """plot_function(a,b,n,f,string)"""
    t = np.linspace(a,b,n)
    deriv = np.dot(derivative(a,b,n),f)
    sec_deriv = np.dot(second_derivative(a,b,n),f)
    plt.plot(t,f,'b', label=string)
    plt.plot(t,deriv,'r', label='Derivative')
    plt.plot(t,sec_deriv,'g', label='Second Derivative')
    plt.title(string)
    plt.legend()
    plt.show()


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Sat Nov 25 16:13:40 2017

@author: ratnadeepb
@License: MIT
"""

'''
Square matrices (n x n) are also vector space where an inner product can
be defined.

In M2,2 space let:
    A = [[a11, a21], [a12, a22]]
    B = [[b11, b21], [b12, b22]]

Inner Product in M2,2:
    <A, B> = a11*b11 + a21*b21 + a12*b12 + a22*b22
'''

import numpy as np
import sys

# Dot product of A and B
def mat_dot(A, B):
    try:
        A = np.array(A)
        B = np.array(B)
    except:
        sys.exit("Not compatible")

    bl = A.ndim == B.ndim
    if not bl:
        sys.exit("Not compatible")
    s = 0
    for i in range(A.ndim):
        for a, b in zip(A[i], B[i]):
            s += a * b
    return s

# Norm of a Matrix
def mat_norm(A):
    return np.sqrt(mat_dot(A, A))

# Angle between two matrices
def mat_angle(A, B, op="radians"):
    if op not in ["radians", "degrees"]:
        sys.exit("At this time we only handle radians and degrees")
    if op == "degrees":
        return (mat_dot(A, B) / (mat_norm(A) * mat_norm(B))) * (180 / np.pi)
    else:
        return (mat_dot(A, B) / (mat_norm(A) * mat_norm(B)))

if __name__ == "__main__":
    A = [[11, -10], [4, 3]]
    B = [[8, 7], [9, 16]]

    print("Dot product of these matrices is:", mat_dot(A, B))
    print("Angle betwee the matrices is:", mat_angle(A, B, "degrees"))


import numpy as np

# First method.... #np.array(p_object)
mylist = [1,2,3]
x = np.array(mylist)
print x

# Direct Method
y = np.array([2,4,5])
print y

#2 dimensional array
q = np.array([[136,53,64],[133,564,23]])
print q
print q.shape  #printing the dimension of the ARRAY

#arange(start,stop,step,dtype) funtion
z = np.arange(0,30,2) # arange(START, END, STEP size)
print z

#reshape(a,newshape,order) function
z = z.reshape(3,5)     #reshape(ROW,COLUMN)
print z

#linspace(start,stop, HOW MANY Number we want in return) function
w = np.linspace(0,4,8)
print w

#resize(a, newshape) fuction OR give it matrix dimention you desire like below...
w.resize((3,4))
print w

#resize() another example        #Changing the dimensions of the array.
a = np.array([[0,1],[2,3]])
a = np.resize(a,(2,6))
print a

b = np.ones((3,2))   #creates 1's matrix of 3by2
print b

c = np.zeros((5,2))  #creates 0's matrix of 5by2
print c

d = np.eye((5))  #creates identity matrix of 5by5
print d

e = np.diag(x)  #creates a diagonal matrix of the given NUMPY ARRAY
print e

f = np.array([1,3,2]*4)  #passing a repeated list..
print f

#repeat(a, repeats, axis) function
g = np.repeat([1,2,3],3)  #repeating each element of the list to the given times..
print g

#We can also combine arrays to create new ones. Let's create a two by three array of ones and stack it vertically
# with itself, multiplied by 2. And here's the same thing, but stacking horizontally.

h = np.ones([2,3], int)
#print h                      #adding a matrix of 2 vertically below to the exsisting array
h = np.vstack([h, 2*h])
print h

i = np.ones([2,3], int)
i = np.hstack([i, 2*i])      #adding a matrix of 2 horizontally side to the exsisting array
print i

#Mathematical OPERATIONS, element wise...

j = np.array([1,2,3],float)
k = np.array([4,5,6], float)

print j+k   #Addition

print j*k   #Multiplication

print j**2  #j to the power of 2..

print k/j  #Divison

print k.dot(j) # here  j = [1,2,3] and k = [4,5,6] => [(4.1)+(5.2)+(6.3)] = 14  DOT product

#Let's create a new array using a previous array y and its squared values. The shape of this array is two by three.
l = np.array([j, j**2])
print l
print l.shape

#We can also take the transpose of an array using the T method, which swaps the rows and columns.
# The shape of the transposed array is three by two.
m = l.T
print m
print m.shape

print m.dtype #Type of the Array

m = m.astype('i')   #typecasting of the Array...
print m.dtype

#Some more inbuilt fuctions of NumPy
o = np.array([46,45,21,53,75])   #New Array...

print o.sum()       #SUM of the Array
print o.max()       #Maximum number/value in the Array
print o.min()       #Minimum number/value in the Array
print o.mean()      #Mean value of the Array
print o.std()       #Standard Diviation of the Array
print o.argmax()    #To find the index value of the Maximum Value in the Array... Indexing starts with 0th address..
print o.argmin()    #To find the index value of the Minimum Value in the Array...  Indexing starts with 0th address..

#Indexing/Slicing

p = np.arange(0,13) #an array of squared value of 0 to 12..
print p**2

q = np.arange(13)**2 #Another method
print q

#We can use bracket notation to get the value at a particular index, and the colon notation to get a range.
print q[0], q[2], q[0:7]

print q[1:5]
print q[-4:]

#And here, we're starting fifth from the end, to the beginning of the array,
# and counting backwards by the difference of two.
print q[-5::-2]

#Let's see how this extends to a two-dimensional array. First, let's make a two dimensional array, 0 to 35.
# We can get a specific value by using the comma notation.
# Here's the value at the second row and second column
r = np.arange(36)
r.resize(6,6)
print r
print r[2,2]  #element at 2nd Row- 2nd Column

print r[3, 3:6] #Slice of the 3rd Row.. from column 3 to 6...

print r[:2 , :-2] #Slicing(getting) first 2 rows except the last 2 columns..

print r[-1 , ::2] #Getting every alternate 2nd element from the last Row

# [] operator for Conditional indexing and assignment
r[r>20] #finding all the elements greater than 20 and repalcing them with 30
r[r>20] = 30  #
print r

#Copying element in NumPy
r2 = r[:3, :3] #printing first 3 rows and columns from the orignal array 'r'....
print r2

r2[:] = 0 #Assigning 0's to the sliced array...
print r2

print r   #In the original array that sliced part also becomes 0...

# USE ...r.copy() function is used if we want to create copy of the original array
# and doesn't change the original array..

r_copy = r.copy() #creating the copy of the original array...
print r_copy

r_copy[:] =10
print r_copy
print r         #Notice that the element of the original array has not changed...

#Iterating over Arrays...
#Creating a 4by3 matrix of random numbers from 0to9...
test = np.random.randint(0, 10, (4,3))
print test

for row in test: #iterating by ROW through array...
    print (row)

for i in range(len(test)): #iterating by ROW INDEX...
    print (test [i])

for i, row in enumerate(test): #Enumerate Gives the ROW and the Index of the ROW...
    print('row', i, 'is', row)

test2 = test**2
print test2

#ZIP() fucntion to iterate between 2 Matrix simultaneously...
for i, j in zip(test, test2):
    print(i, '+', j, '=', i+j)

#Using resize and reshape functions to find different patterns of matrix...
s = np.arange(0,36)
#print s
s.resize(6,6)
print s

print s.reshape(36)[::7]
print s[2:4,2:4]


import numpy as np
import random

#import the data. Note that the txt file has to be in the same folder
data = np.loadtxt('cdata.txt')

#function for calculating standard deviation by using eq (5) from lab 2
def std5(val):
    mean = (1./len(val))*np.sum(val) # 1st pass to find mean
    sumOfDifference = 0
    for i in val: # 2nd pass to find difference
        sumOfDifference += (i - mean) ** 2
    ans5 = np.sqrt((1./(len(val)-1)) * sumOfDifference)
    return ans5

#function for calculating standard deviation by using eq (6) from lab 2
def std6(val):
    sumForMean = 0 #this is for calculating the mean
    sumOfSquare = 0 #this is for calculating the sum of sqaured element
    for i in val: #only pass through data set
        sumForMean += i
        sumOfSquare += (i ** 2)
        if sumOfSquare < sumForMean: #check to prevent sqrt negative number
            print('you are wrong, but the code will still run')
            sumOfSquare += sumForMean #will make it larger than sum for mean but std will be wrong
    ans6 = np.sqrt((1./(len(val)-1)) * (sumOfSquare - len(val)*((sumForMean/len(val)) ** 2)))
    return ans6

#using numpy.std method as our reference answer
correctAnswer = np.std(data, ddof=1)

#using (x-y)/y to calculate relative error from lab 2
print('The relative error of method 5: ', np.abs(std5(data) - correctAnswer)/correctAnswer)
print('The relative error of method 6: ', np.abs(std6(data) - correctAnswer)/correctAnswer)

#constants for generating set 1
mean1, sigma1, n1 = (0., 1., 2000)
#constants for generating set 2
mean2, sigma2, n2 = (1.e7, 1., 2000)

#set 1 data
data1= np.random.normal(mean1, sigma1, n1)
#set 2 data
data2= np.random.normal(mean2, sigma2, n2)

#using numpy.std method to calculate respective answers
correctAnswer1 = np.std(data1, ddof=1)
correctAnswer2 = np.std(data2, ddof=1)

#using same method as above to calculate relative error for set 1
print('The relative error of method 5 of set 1: ', np.abs(std5(data1) - correctAnswer1)/correctAnswer1)
print('The relative error of method 6 of set 1: ', np.abs(std6(data1) - correctAnswer1)/correctAnswer1)

#using same method as above to calculate relative error for set 2
print('The relative error of method 5 of set 2: ', np.abs(std5(data2) - correctAnswer2)/correctAnswer2)
print('The relative error of method 6 of set 2: ', np.abs(std6(data2) - correctAnswer2)/correctAnswer2)

def std6Improved(val):
    sumForMean = 0 #same setup as above
    sumOfSquare = 0
    adjust = random.choice(val) #randomly pick an element to shift data set close to 0 and 1
    for i in val:
        sumForMean += i - adjust #shifting each data point
        sumOfSquare += ((i - adjust) ** 2) #shifting each data point
        if sumOfSquare < 0:
            print('you are wrong, but the code will still run')
            np.abs(sumOfSquare)
    #no need to account the adjustment into std calculation because shifting data set does not effect value of std
    ans6 = np.sqrt((1./(len(val)-1))*(sumOfSquare - (sumForMean ** 2)/len(val)))
    return ans6

#using same method as above to calculate relative error for improved eq 6
print('The relative error of method 6 with improved method: ', np.abs(std6Improved(data) - correctAnswer)/correctAnswer)


import numpy as np
import matplotlib.pyplot as plt
from Generaldistribution import Distribution

class Gaussian (Distribution):

    """
    Gaussian distribution class for calculating and
    visualizing a Gaussian distribution.

    Attributes:
        mean (float) represent the mean values of the distribution
        stdev (float) represent standard deviation of the distribution
        dat_list (list of float) extracted from the text file
    """
    def __init__(self, mu = 0 , sigma = 1):

        Distribution.__init__(self, mu , sigma)

    def calcualte_mean(self):
        """
        Function calculated the mean of the dataset

        Args:
            None
        Returns:
            float: mean of the dataset
        """

        avg = float(sum(self.data / len(self.data)))
        self.mean = avg

        return self.mean

    def calculate_stdev(self, sample=True):
        """
        Function calculates the standard deviation of the dataset

        Args:
            sample (bool) whether the data is population or a sample
        Returns:
            float: standard deviation of the data set
        """

        if sample:
            n = len(self.data) - 1
        else:
            n = len(self.data)
        mean = self.calcualte_mean()
        sigma = 0

        for data in self.data:
            sigma += (data - mean)**2
        sigma = np.sqrt(sigma/n)

        self.stdev = sigma

        return self.stdev

     def plot_histogram(self):
        """
        Function to output a histogram of the instance variable data
        using matplotlib.pyplot library

        Args:
            None
        Returns:
            None
        """

        plt.hist(self.data)
        plt.title("Histogram of data")
        plt.xlabel('data')
        pltylable('count')

    def pdf(self , x):
        """
        Probability Density Function calculates the Gaussian distribution

        Args:
            x (float): point for calculating the probability density function

        Returns:
            float: probability density function as output
        """

        prob = (1. /(self.stdev * np.sqrt(2*math.pi))) * np.exp(-.5*((x - self.mean) / self.stdev)**2)
        return prob

    def plot_histogram_pdf(self, n_spaces = 50):

        """Function to plot the normalized histogram of the data and a plot of the
        probability density function along the same range

        Args:
            n_spaces (int): number of data points

        Returns:
            list: x values for the pdf plot
            list: y values for the pdf plot

        """

        mu = self.mean
        sigma = self.stdev

        min_range = min(self.data)
        max_range = max(self.data)

         # calculates the interval between x values
        interval = 1.0 * (max_range - min_range) / n_spaces

        x = []
        y = []

        # calculate the x values to visualize
        for i in range(n_spaces):
            tmp = min_range + interval*i
            x.append(tmp)
            y.append(self.pdf(tmp))

        # make the plots
        fig, axes = plt.subplots(2,sharex=True)
        fig.subplots_adjust(hspace=.5)
        axes[0].hist(self.data, density=True)
        axes[0].set_title('Normed Histogram of Data')
        axes[0].set_ylabel('Density')

        axes[1].plot(x, y)
        axes[1].set_title('Normal Distribution for \n Sample Mean and Sample Standard Deviation')
        axes[0].set_ylabel('Density')
        plt.show()

        return x, y

    def __add__(self, other):

        """Function to add together two Gaussian distributions

        Args:
            other (Gaussian): Gaussian instance

        Returns:
            Gaussian: Gaussian distribution

        """

        result = Gaussian()
        result.mean = self.mean + other.mean
        result.stdev = math.sqrt(self.stdev ** 2 + other.stdev ** 2)

        return result

    def __repr__(self):

        """Function to output the characteristics of the Gaussian instance

        Args:
            None

        Returns:
            string: characteristics of the Gaussian

        """

        return "mean {}, standard deviation {}".format(self.mean, self.stdev)

if __name__ == "__main__":
    # initialize two gaussian distributions
    gaussian_one = Gaussian(25, 3)
    gaussian_two = Gaussian(30, 2)

    # initialize a third gaussian distribution reading in a data efile
    gaussian_three = Gaussian()
    gaussian_three.read_data_file('numbers.txt')
    gaussian_three.calculate_mean()
    gaussian_three.calculate_stdev()

    # print out the mean and standard deviations
    print(gaussian_one.mean)
    print(gaussian_two.mean)

    print(gaussian_one.stdev)
    print(gaussian_two.stdev)

    print(gaussian_three.mean)
    print(gaussian_three.stdev)

    # plot histogram of gaussian three
    gaussian_three.plot_histogram_pdf()

    # add gaussian_one and gaussian_two together
    gaussian_one + gaussian_two


import sympy as sp
x, y, z = sp.symbols('x y z')
sp.init_printing()

# Here we simply give the input needed as The Following:
# ( Xi ) is the initial point
# ( Xminus1 ) is the point before the Initial toint
# ( Fx ) is the equation of the function
# ( n ) is the number of Iterations needed

Xi = 0

Xminus1 = -0.5

Fx = sp.cos(x)+2*sp.sin(x)+x**2

n = 5

print("\n____________________________________")
print(" Iteration |    Xi     ||   Error "
      "\n------------------------------------")
Error = 0 ; XiPlus1 = 0
for i in range (n):
    FxVal = float(Fx.subs(x, Xi).evalf())
    FminusVal = float(Fx.subs(x, Xminus1).evalf())
    XiPlus1 = Xi - FxVal*(Xi - Xminus1)/(FxVal - FminusVal)
    Error = abs(((XiPlus1 - Xi)/XiPlus1) *100)
    print("      {0:1d}    | {1:.6f} || {2:.6f}%\n"
          .format(i+1, Xi, Error))
    Xminus1 = Xi ; Xi = XiPlus1

# In The End The Program Prints The Output as well as The Percentage of error between Iterations


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Sun Jan 27 13:31:03 2019

@author: alankar
"""
import numpy as np
import matplotlib.pyplot as plt

def f(x,c):
    return 1-np.exp(-c*x)

def fpi(func,x0, tol, verbose, *args): #x0 is initial guess
    x_old = x0
    x_new = x0+10  #some arbitrary initialization
    counter = 0
    while(np.abs(x_new-x_old)>=tol):
        counter += 1
        x_old = x_new
        x_new = func(x_old,*args)
    if (verbose==True): print ("Converged in %d steps"%counter)
    return x_new

c = 2
print('Fixed Point Iteration:')
print('Solution for x=1-exp(-%.2f x) is: x=%f\n'%(c,fpi(f,0.5,1e-6,True,c)))

c = np.arange(0,3,0.01)
x = np.zeros(len(c))
for i in range(len(c)):
    x[i] = fpi(f,0.5,1e-6,False,c[i])

plt.plot(c,x)
plt.grid()
plt.xlabel('c',size=18)
plt.ylabel('x',size=18)
#plt.savefig('tansition.jpg')

def steffensen_fpi(func,x0, tol, verbose, *args):
    x_old = x0
    x_new = x0+10  #some arbitrary initialization
    counter = 0
    while(np.abs(x_new-x_old)>=tol):
        counter +=1
        x_old = x_new
        g = (func(func(x_old,*args),*args)-func(x_old,*args))/(func(x_old,*args)-x_old)-1
        x_new = x_old-(func(x_old,*args)-x_old)/g
    if (verbose==True): print ("Converged in %d steps"%counter)
    return x_new

c = 2
print('Accelerated Fixed Point Iteration')
print('Solution for x=1-exp(-%.2f x) is: x=%f'%(c,steffensen_fpi(f,0.5,1e-6,True,c)))

plt.show()

"""
Output

Fixed Point Iteration:
Converged in 15 steps
Solution for x=1-exp(-2.00 x) is: x=0.796813

Accelerated Fixed Point Iteration
Converged in 4 steps
Solution for x=1-exp(-2.00 x) is: x=0.796812
"""


from pyccel.decorators import types

#==============================================================================

@types( int )
def sum_natural_numbers( n ):
    x = 0
    for i in range( 1, n+1 ):
        x += i
    return x

# ...
@types( int )
def factorial( n ):
    x = 1
    for i in range( 2, n+1 ):
        x *= i
    return x

# ...
@types( int )
def fibonacci( n ):
    x = 0
    y = 1
    for i in range( n ):
        z = x+y
        x = y
        y = z
    return x

# ...
@types( int )
def double_loop( n ):
    x = 0
    for i in range( 3, 10 ):
        x += 1
        y  = n*x
        for j in range( 4, 15 ):
            z = x-y
    return z

# ...
@types( 'int[:,:](order=C)' )
def double_loop_on_2d_array_C( z ):

    from numpy import shape

    s = shape( z )
    m = s[0]
    n = s[1]

    for i in range( m ):
        for j in range( n ):
            z[i,j] = i-j

# ...
@types( 'int[:,:](order=F)' )
def double_loop_on_2d_array_F( z ):

    from numpy import shape

    s = shape( z )
    m = s[0]
    n = s[1]

    for i in range( m ):
        for j in range( n ):
            z[i,j] = i-j

# ...
@types( 'int[:,:](order=C)' )
def product_loop_on_2d_array_C( z ):

    from numpy     import shape
    from itertools import product

    s = shape( z )
    m = s[0]
    n = s[1]

    x = [i for i in range(m)]
    y = [j for j in range(n)]

    for i,j in product( x, y ):
        z[i,j] = i-j

# ...
@types( 'int[:,:](order=F)' )
def product_loop_on_2d_array_F( z ):

    from numpy     import shape
    from itertools import product

    s = shape( z )
    m = s[0]
    n = s[1]

    x = [i for i in range(m)]
    y = [j for j in range(n)]

    for i,j in product( x, y ):
        z[i,j] = i-j

# ...
@types( 'int[:]' )
def map_on_1d_array( z ):

    @types( int )
    def f( x ):
        return x+5

    res = 0
    for v in map( f, z ):
        res *= v

    return res

# ...
@types( 'int[:]' )
def enumerate_on_1d_array( z ):

    res = 0
    for i,v in enumerate( z ):
        res += v*i

    return res

# ...
@types( int )
def zip_prod( m ):

    x = [  i for i in range(m)]
    y = [2*j for j in range(m)]

    res = 0
    for i1,i2 in zip( x, y ):
        res += i1*i2

    return res