"""

A simple optimal savings problem.  The Bellman equation is

    v(x, z) = max_x' { u(R x + w z - x') + β E v(x', z')}

where 0 <= x' <= R x + w z and

    E v(x', z') = Σ_{z'} v(x', z') Q(z, z')

We take

    u(c) = c^{1 - γ} / (1 - γ)

and obtain the transition kernel p by discretizing

    log z' = ρ log z + d + σ η

using Rouwenhorst's method.

"""

import numpy as np
import quantecon as qe
from numba import jit, prange

@jit(nopython=True)
def u(c, γ):
    return (c + 1e-10)**(1 - γ) / (1 - γ)

class SavingsProblem:

    def __init__(self,
                 β=0.96,
                 γ=2.5,
                 ρ=0.9,
                 d=0.0,
                 σ=0.1,
                 r=0.04,
                 w=1.0,
                 z_grid_size=25,
                 x_grid_size=200,
                 x_grid_max=15):

        self.β, self.γ = β, γ
        self.R = 1 + r
        self.w = w
        self.z_grid_size, self.x_grid_size = z_grid_size, x_grid_size

        mc = qe.rouwenhorst(z_grid_size, d, σ, ρ)
        self.Q = mc.P
        self.z_grid = np.exp(mc.state_values)

        self.x_grid = np.linspace(0.0, x_grid_max, x_grid_size)

    def pack_parameters(self):
        return self.β, self.γ, self.R, self.w, self.Q, self.x_grid, self.z_grid

    def value_function_iteration(self,
                                tol=1e-4,
                                max_iter=1000,
                                verbose=True,
                                print_skip=25):

        # Set initial condition, set up storage
        v_in = np.ones((self.x_grid_size, self.z_grid_size))
        v_out = np.empty_like(v_in)
        π = np.empty_like(v_in, dtype=np.int)

        # Set up loop
        params = self.pack_parameters()
        i = 0
        error = tol + 1

        while i < max_iter and error > tol:
            T(v_in, v_out, π, params)
            error = np.max(np.abs(v_in - v_out))
            i += 1
            if i % print_skip == 0:
                print(f"Error at iteration {i} is {error}.")
            v_in[:] = v_out

        if i == max_iter:
            print("Failed to converge!")

        if verbose and i < max_iter:
            print(f"\nConverged in {i} iterations.")

        return v_out, π

@jit(nopython=True, parallel=True)
def T(v, v_out, π, params):
    """
    Given v, compute Tv and write it to v_out.

    At the same time, compute the v-greedy policy and write it to π

    """
    n, m = v.shape
    β, γ, R, w, Q, x_grid, z_grid = params

    for j in prange(m):
        z = z_grid[j]

        for i in range(n):
            x = x_grid[i]

            # Cash in hand at start of period
            y = R * x + w * z
            # A variable to store largest recorded value
            max_so_far = - np.inf

            # Find largest x_grid index s.t. x' <= y
            idx = np.searchsorted(x_grid, y)

            # Step through x' with 0 <= x' <= y, find max
            for k in range(idx):
                x_next = x_grid[k]
                val = u(y - x_next, γ) + β * np.sum(v[k, :] * Q[j, :])
                if val > max_so_far:
                    max_so_far = val
                    k_star = k

            π[i, j] = k_star
            v_out[i, j] = max_so_far


import numpy as np
import gd
import matplotlib.pyplot as plt

m = 15
# X: m x 2 matrix, X[i,:] is the 2D feature vector of the i-th sample
X = np.array([[1, 1.5],
              [1.2, 2.5],
              [1, 3.5],
              [2, 2.25],
              [1.8, 3],
              [2.5, 4],
              [3, 1.9],
              [1.5, .5],
              [2.5, .8],
              [2.8, .3],
              [3.2, .3],
              [3, .8],
              [3.8, 1],
              [4, 2],
              [1.8, 1.8]])
# y: m-D vector, y[i] is the label of the i-th sample
y = np.append(np.ones((7,)), -np.ones((8,)))

# append a constant 1 to each feature vector, so X is now a m x 3 matrix
X = np.append(X, np.ones((15, 1)), axis=1)

# Xy[i,:] = X[i,:] * y[i]
Xy = X * y.reshape((-1, 1))

def sigmoid(z):
    return 1 / (1 + np.exp(-z))

def fp(w):
    s = np.array([0.0, 0.0, 0.0])
    for i in range(m):
        s += (sigmoid(y[i] * (X[i, :].T @ w)) - 1) * y[i] * X[i, :]
    return s.reshape(-1, 1)

stepsize = 0.1
maxiter = 1000000
w0 = np.array([[0.0], [0.0], [0.0]])
w_traces = gd.gd_const_ss(fp, w0, stepsize=stepsize, maxiter=maxiter)

print(
    f'stepsize={stepsize}, number of iterations={len(w_traces)-1}')
print(f'optimal solution:\n{w_traces[-1]}')

# compute the accuracy on the training set
w = w_traces[-1]
y_hat = 2*(X@w > 0)-1

accuracy = sum(y_hat.T[0] == y) / float(len(y))
print(f"accuracy = {accuracy}")

# visualization
plt.figure(figsize=(4, 4))

plt.scatter(*zip(*X[y > 0, 0:2]), c='r', marker='x')
plt.scatter(*zip(*X[y < 0, 0:2]), c='g', marker='o')

# plot the decision boundary w[0] * x1 + w[1] * x2 + w[0] = 0
x1 = np.array([min(X[:, 0]), max(X[:, 0])])
x2 = -(w[0] * x1 + w[2]) / w[1]
plt.plot(x1, x2, 'b-')

plt.xlabel('x1')
plt.ylabel('x2')

plt.tight_layout()
plt.savefig('../figures/p3.pdf')


import pickle
import numpy as np

def eucl_dist(a, b, axis=1):
    return np.linalg.norm(a - b, axis=axis)

def get_center(k, X):
    temp = []
    temp.append(X[np.random.randint(0, len(X))])
    while len(temp)<k:
        d2 = np.array([min([np.square(eucl_dist(i,c, None)) for c in temp]) for i in X])
        prob = d2/d2.sum()
        cum_prob = prob.cumsum()
        r = np.random.random()
        ind = np.where(cum_prob >= r)[0][0]
        temp.append(X[ind])
    return np.array(temp)

def k_mean_pp(x, k):
    # error = {key:0.0 for key in range(2,11)}

    # initalizing cluster variable
    center = get_center(k, x)

    # for k in range(2,11):

    # assigining zeros to old centroids value
    center_old = np.zeros(center.shape)

    # initial error
    err = eucl_dist(center, center_old, None)

    while err != 0:

        # calculatin distance of data points from centroids and assiging min distance cluster centroid as data point cluster
        for i in range(len(x)):
            distances = eucl_dist(x[i], center)
            clust = np.argmin(distances)
            cluster[i] = clust

        # changing old centroids value
        center_old = np.copy(center)

        # Finding the new centroids by taking the average value
        for i in range(k):
            points = [x[j] for j in range(len(x)) if cluster[j] == i]
            if points:
                center[i] = np.mean(points, axis=0)

        # calculation difference between new centroid and old centroid values
        err = eucl_dist(center, center_old, None)

    # calculation total difference between cluster centroids and cluster data points
    error = 0
    for i in range(k):
        d = [eucl_dist(x[j], center[i], None) for j in range(len(x)) if cluster[j] == i]
        error += np.sum(d)

    # counting data points in all clusters
    count = {key: 0.0 for key in range(k)}
    for i in range(len(x)):
        count[cluster[i]] += 1

    # displaying cluster number, average distance between centroids and data points and cluster count
    print k, error / len(x), count

    return cluster

if __name__ == '__main__':

    # loading dataset of form [[data1],[data2], ....]
    inp = pickle.load(open('test.pickle', 'rb'))
    x = np.array([i[0] for i in inp])

    # return cluster number for every data
    cluster = k_mean_pp(x)


import numpy as np
from sympy import *
import matplotlib.pyplot as plt
import time
from math import *
from scipy.misc import derivative
import scipy.integrate as integrate

global iteration

def newtonTrapezoidal(f,a,b,n):
    array = []
    h=(b-a)/n
    trapezoidalSum=0
    part1=(0.5)*h*(f(a)+f(b))
    for i in range(1,n):
        xi=a+i*h
        trapezoidalSum=trapezoidalSum+f(xi)
        array.append(part1+h*trapezoidalSum)

    plt.plot(array, 'k--', label="Trapezoidal")
    plt.legend(loc='best')

    print('Trapezoidal result: ' + str(part1+h*trapezoidalSum))

    h = (b - a) / n
    c = derivative(f, float(a + b) / 2, n=2)
    error = (((b - a) ** 2) * (h ** 2) * c) / 25

    print('Trapezoidal error: ' + str(error))

def newtonSimpson(f, a, b, n):
    if a > b:
        print('Incorrect bounds')
        return None
    else:
        if n%2 != 0:
            n = n+1
        h = (b - a)/float(n)
        sum1 = 0
        for i in range(1, int(n/2 + 1)):
            sum1 += f(a + (2*i - 1)*h)
        sum1 *= 4
        sum2 = 0
        for i in range(1, int(n/2)):
            sum2 += f(a + 2*i*h)

        sum2 *= 2
        approx = (b - a)/(3.0*n)*(f(a) + f(b) + sum1 + sum2)
        print ('Simpson result: ' + str(approx))

        actual = integrate.quad(f, 0, 1)[0]
        error = round(abs(actual - approx), 8)
        print ('Simpson error: ' + str(error))

def newtTrapSimp(func, a, b, n):
    #### SIMPSON
    newtonSimpson(func, a, b, n)

    ### TRAPEZOIDAL
    newtonTrapezoidal(func, a, b, n)

if __name__ == '__main__':
    func = lambda x: x**2
    a = 0.0
    b = 1.0
    n = 4

    newtTrapSimp(func, a, b, n)

import numpy,import scipy,from scipy,from sympy


"""
Input: An example problem using a deck of cards will be used to demonstrate definition of the terms
What is the probability of receiving 2 Kings from a deck of 52 playing cards (containing 4 Kings)
when 6 cards are drawn from the deck
type_threshold is the number of a specific item from which the probability is calculated (e.g. 2 Kings)
no_of_type_in_set is the number of the specific item found amongst the total number of items (e.g.4 Kings within the deck of 52 playing cards)
sample_size is the set of items that are drawn from the total (e.g. 6 cards drawn from the deck )
set_size is the total number of the items in the set (e.g. the 52 cards in a deck of playing cards)

@author: kevoulee
"""

from IPython import get_ipython
get_ipython().magic('clear')
#get_ipython().magic('reset -sf')

import numpy as np
import math
import operator
from decimal import Decimal

def nCk(n,k):
    if (n - k) < 0 or n < 0 or k < 0:
        return 0;
    return np.float64((Decimal(math.factorial(n)) / Decimal(math.factorial(n - k))) / Decimal(math.factorial(k)))

def probability(type_threshold, no_of_type_in_set, sample_size, set_size) -> np.float64:
    #Type checking - Python does not have built-in type checking
    type_error = False
    ##type_threshold
    try:
        operator.index(type_threshold)
        T = np.int_(type_threshold)
    except TypeError:
        print('Error: \'type_threshold\' - expected int (or int variant), received %s' % (type(type_threshold)))
        type_error = True

    ##no_of_type_in_set
    try:
        operator.index(no_of_type_in_set)
        m = np.int_(no_of_type_in_set)
    except TypeError:
        print('Error: \'no_of_type_in_set\' - expected int (or int variant), received %s' % (type(no_of_type_in_set)))
        type_error = True

    ##sample_size
    try:
        operator.index(sample_size)
        n = np.int_(sample_size)
    except TypeError:
        print('Error: \'sample_size\' - expected int (or int variant), received %s' % (type(sample_size)))
        type_error = True

    ##set_size
    try:
        operator.index(set_size)
        N = np.int_(set_size)
    except TypeError:
        print('Error: \'set_size\' - expected int (or int variant), received %s' % (type(set_size)))
        type_error = True

    if type_error:
        return 0

    #Calculation
    P_tot = 0
    #Calculate probability
    for i in range(T, n + 1):
        P_tot = P_tot + nCk(m, i) * nCk(N - m, n - i) / nCk(N, n)
    return P_tot


#!/usr/bin/python3.9
# Licence MIT copyright (c) 2022 Stéphane Lassalvy
import numpy as np
def my_CribleEratosthene(n):
    # Algorithme du crible d'Eratostène pour calculer les nombres premiers inférieur ou égaux à un entier naturel n
    # Il est basé sur le principe qu'un nombre est premier s'il n'est divisible par aucun des nombres premiers qui le précèdent.
    print("On peut considérer par convention que 1 est un nombre premier.")
    nombrePremiers = [2]
    # i est un candidat à être un nombre premier
    i = 3
    while i <= n:
        restes = []
        # Le nombre i est premier s'il n'est pas divisible par les nombre premiers qui le précédent.
        # Pour le savoir, on établit alors la liste des restes des divisions euclidiennes de i par les nombres premiers qui le précèdent.
        for j in nombrePremiers:
            restes.append(i % j)
        # Une fois qu'on a calculé tous les restes, on regarde le reste le plus petit, et s'il est différent de 0,
        # c'est que tous les restes sont différent de 0, et donc que i n'est divisible par aucun des nombres premiers qui le précèdent.
        # On rajoute dans ce cas i à la liste des nombres premiers connus
        if min(restes) > 0:
            nombrePremiers.append(i)
        # et on passe à l'entier i suivant, on ajoute 2 à i car il est inutile de tester les multiples de 2
        i = i + 2
    print(f"Liste des nombres premiers compris dans l'intervalle [2; {n}]:")
    print(nombrePremiers)
# Exécution du programme
n = input("Indiquez le nombre n pour lequels vous voulez calculez les nombres premiers <= n :\n")
n = int(n)
my_CribleEratosthene(n)


# Chapter 9

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt

## Review

[In the previous notebook](https://colab.research.google.com/github/AllenDowney/BiteSizeBayes/blob/master/08_soccer.ipynb) I introduced the World Cup problem:

>In the 2018 FIFA World Cup final, France defeated Croatia 4 goals to 2.  Based on this outcome:
>
>1. How confident should we be that France is the better team?
>
>2. If the same teams played again, what is the chance Croatia would win?

I started with the assumption that for any team against any other team there is some unknown goal-scoring rate, λ.

And I showed that if we know λ, we can compute the probability of scoring $k$ goals in a game:

$f(k; λ) = λ^k \exp(-λ) ~/~ k!$

This function is the [Poisson distribution](https://en.wikipedia.org/wiki/Poisson_distribution), and we can use SciPy to compute it.

For example, if we know that λ is 1.4, we can compute the distribution of $k$ like this:

from scipy.stats import poisson

λ = 1.4
xs = np.arange(11)
ys = poisson.pmf(xs, λ)
pmf_goals = pd.Series(ys, index=xs)

pmf_goals

In theory, the values of $k$ go to infinity, but I cut it off at 10 because higher values have very low probability.

Here's what the distribution of $k$ looks like:

pmf_goals.plot.bar(color='C0', alpha=0.5, label='Poisson distribution')

plt.xlabel('Number of goals')
plt.ylabel('Probability')
plt.title('Distribution of goals scored')
plt.legend();

Because the values of $k$ are discrete integers, I use a bar chart to plot the distribution.

Next I used a gamma distributon to represent the prior distribution of λ.  I set the parameter of the gamma distribution, α, to 1.4, which is the average number of goals each team scores in World Cup play.

I broke the range of values for λ into 101 discrete possibilities and used SciPy to compute the prior probabilities:

from scipy.stats import gamma

α = 1.4
qs = np.linspace(0, 10, 101)
ps = gamma.pdf(qs, α)

prior = pd.Series(ps, index=qs)
prior /= prior.sum()

Here's what the prior distribution looks like:

prior.plot(label='prior', color='gray')

plt.xlabel('Goal scoring rate')
plt.ylabel('Probability')
plt.title('Prior distribution')
plt.legend();

Because the values of $λ$ are continuous, I use a line to plot the distribution.

Next we did a Bayesian update, using the Poisson distribution to compute the likelihood of the data, $k$, for each possible goal scoring rate, $λ$

$ f(k; λ) = λ^k \exp(-λ) ~/~ k! $

Since France scored 4 goals, the likelihood of the data is:

λs = prior.index
k = 4
likelihood = poisson.pmf(k, λs)

And we can use the following function to do the update:

def bayes_update(pmf, likelihood):
    """Do a Bayesian update.

    pmf: Series that represents the prior
    likelihood: sequence of likelihoods

    returns: float probability of the data
    """
    pmf *= likelihood
    prob_data = pmf.sum()
    pmf /= prob_data
    return prob_data

france = prior.copy()
bayes_update(france, likelihood)

And we can do the same thing for Croatia, given that they scored 2 goals:

k = 2
λs = prior.index
likelihood = poisson.pmf(k, λs)

croatia = prior.copy()
bayes_update(croatia, likelihood)

Here's what the results look like.

prior.plot(label='prior', color='gray')
france.plot(label='France', color='C0')
croatia.plot(label='Croatia', color='C3')

plt.xlabel('Goal scoring rate')
plt.ylabel('Probability')
plt.title('Posterior distributions for France and Croatia')
plt.legend();

In the previous notebook we used the following function to compute the mean of a PMF.

def pmf_mean(pmf):
    """Compute the mean of a PMF.

    pmf: Series representing a PMF

    return: float
    """
    return np.sum(pmf.index * pmf)

We can use it to compute the posterior means for France and Croatia.

pmf_mean(france), pmf_mean(croatia)

Based on the outcome of the game, we have some reason to think France is the better team.

But if we look at the posterior distribution of λ for France and Croatia, there is still a lot of overlap, which means we are still not certain which is the better team.

In the next section, we'll use the posterior distributions to compute the "probability of superiority".

### Probability of superiority

Now that we have a posterior distribution for each team, we can answer the first question: How confident should we be that France is the better team?

In the model, "better" means having a higher goal-scoring rate against the opponent.  We can use the posterior distributions to compute the probability that a random value drawn from France's distribution exceeds a value drawn from Croatia's.

The following function takes a Series that represents a PMF and draws a sample from it.

def pmf_choice(pmf, n):
    """Draw a random sample from a PMF.

    pmf: Series representing a PMF
    n: number of values to draw

    returns: NumPy array
    """
    return np.random.choice(pmf.index,
                            size=n,
                            replace=True,
                            p=pmf)

`pmf_choice` uses `np.random.choice`, which chooses `n` values from the PMF with replacement, so the same value can appear more than once.  It uses the probabilities from the PMF as weights, so the number of times each value appears is proportional to its probability.

Here's a sample from the posterior distribution for France.

sample_france = pmf_choice(france, 1000)
sample_france.mean()

And here's a sample for Croatia.

sample_croatia = pmf_choice(croatia, 1000)
sample_croatia.mean()

To estimate the probability of superiority, we can count the number of times the value from France's distribution exceeds the value from Croatia's distribution.

np.mean(sample_france > sample_croatia)

On the basis of one game, we have moderate confidence that France is actually the better team.

### Predicting the rematch

Now we can take on the second question: If the same teams played again, what is the chance Croatia would win?

To answer this question, we'll generate a sample from the "posterior predictive distribution", which is the number of goals we expect a team to score.

If we knew the goal scoring rate, λ, the distribution of goals would be a Poisson distributon with parameter λ.

Since we don't know λ, we can use the sample we generated in the previous section to generate a sample of goals, like this:

goals_france = np.random.poisson(sample_france)

`np.random.poisson` uses each element of `sample_france` to generate a random value; that is, each value in the result is based on a different value of λ.

To see what the resulting distribution looks like, we'll use this function from a previous notebook:

def pmf_from_seq(seq):
    """Make a PMF from a sequence of values.

    seq: sequence

    returns: Series representing a PMF
    """
    pmf = pd.Series(seq).value_counts(sort=False).sort_index()
    pmf /= pmf.sum()
    return pmf

pmf_france = pmf_from_seq(goals_france)
pmf_france.plot.bar(color='C0', label='France')

plt.xlabel('Goals scored')
plt.ylabel('Probability')
plt.title('Predictive distribution')
plt.legend()

goals_france.mean()

This distribution represents two sources of uncertainty: we don't know the actual value of λ, and even if we did, we would not know the number of goals in the next game.

**Exercise:** Generate and plot the predictive distribution for Croatia.

# Solution goes here

In a sense, these distributions represent the outcomes of 1000 simulated games.

**Exercise:** Compute the fraction of simulated rematches Croatia would win, how many France would win, and how many would end in a tie.

# Solution goes here

# Solution goes here

# Solution goes here

Assuming that Croatia wins half of the ties, their chance of winning the rematch is about 33%.

## Summary

In this notebook, we finished off the World Cup problem:

* We used posterior distributions to generate samples of goal-scoring rates.

* We compared samples to compute a "probability of superiority".

* We used samples and `np.random.poisson` to generate samples of goals score and to estimate their distributions.

* We used those distributions to compute the probabilities of winning, losing, and tying in a rematch.

The goal distributions we computed are called "[posterior predictive distributions](https://en.wikipedia.org/wiki/Posterior_predictive_distribution)" because they use posterior distribution to make predictions.

[In the next notebook](https://colab.research.google.com/github/AllenDowney/BiteSizeBayes/blob/master/10_joint.ipynb) we'll take a break from Bayesian methods and learn about an important idea in probability: joint distributions.

But first, here's an exercise where you can practice what you learned in this notebook.

## Exercise

**Exercise:** Here's a variation on the World Cup Problem:

>In the 2014 FIFA World Cup, Germany played Brazil in a semifinal match. Germany scored after 11 minutes and again at the 23 minute mark. At that point in the match, how many goals would you expect Germany to score after 90 minutes? What was the probability that they would score 5 more goals (as, in fact, they did)?

In this version, notice that the data is not the number of goals in a fixed period of time, but the time between goals.

To compute the likelihood of data like this, we can take advantage of the theory of [Poisson processes](https://en.wikipedia.org/wiki/Poisson_point_process).  In our model of a soccer game, we assume that each team has a goal-scoring rate, λ, in goals per game.  And we assume that λ is constant, so the chance of scoring a goal in the same at any moment of the game.

Under these assumptions, the distribution of goals follows a Poisson distribution, as we've already seen.  Also, the time between goals follows an [exponential distribution](https://en.wikipedia.org/wiki/Exponential_distribution).

If the goal-scoring rate is λ, the probability of seeing an interval between goals of $t$ is proportional to the PDF of the exponential distribution:

$f(t; λ) = λ~\exp(-λ t)$

Because $t$ is a continuous quantity, the value of this expression is not really a probability; technically it is a [probability density](https://en.wikipedia.org/wiki/Probability_density_function).  However, it is proportional to the probability of the data, so we can use it as a likelihood in a Bayesian update.

To see what the exponential distribution looks like, let's assume again that λ is 1.4; we can compute the distribution of $t$ like this:

def expo_pdf(t, λ):
    """Compute the PDF of the exponential distribution.

    t: time
    λ: rate

    returns: probability density
    """
    return λ * np.exp(-λ * t)

λ = 1.4
ts = np.linspace(0, 4, 101)
ys = expo_pdf(ts, λ)
pmf_time = pd.Series(ys, index=ts)
pmf_time /= pmf_time.sum()

pmf_time.plot(label='λ = 1.4')

plt.xlabel('Time between goals (games)')
plt.ylabel('Probability')
plt.title('Distribution of time between goals')
plt.legend();

It is counterintuitive, but true, that the most likely time to score a goal is immediately.  After that, the probability of each possible interval is a little lower.

With a goal-scoring rate of 1.4, it is possible that a team will take more than one game to score a goal, but it is unlikely that they will take more than two games.

Now we're ready to solve the problem.  Here are the steps I recommend:

1. Starting with the same gamma prior we used in the previous problem, compute the likelihood of scoring a goal after 11 minutes for each possible value of λ.  Don't forget to convert all times into units of games.

2. Use `bayes_update` to compute the posterior distribution of λ for Germany after the first goal.  If the total probability of the data is greater than 1, don't panic; because the likelihoods are not probabilities, the return value from `bayes_update` is not a probability either.  But the posterior distribution is still valid.

3. Compute the likelihood of scoring another goal after 12 more minutes and do another update.  Plot the prior, posterior after one goal, and posterior after two goals.

4. Use `pmf_choice` to generate a sample of 1000 values from the posterior distribution of goal scoring rate.

5. Use `np.random.poisson` to compute a sample of goals Germany might score during the remaining time in the game, `90-23` minutes.  Note: you will have to think about how to generate predicted goals for a fraction of a game.

6. Compute and plot the PMF of possible goals scored and select from it the probability of scoring 5 more goals.

# Solution goes here

# Solution goes here

# Solution goes here

# Solution goes here

# Solution goes here

# Solution goes here

# Solution goes here

# Solution goes here

# Solution goes here

# Solution goes here

# Solution goes here


import numpy as np

def griewank(X):
    """ minimum @ 0
    """

    return (
        1
        + (X ** 2).sum(axis=1) / 4000
        - np.prod(
            np.cos(
                X / np.sqrt(np.arange(1, X.shape[-1] + 1))
            ),
            axis=1
        )
    )

# def rastrigin(v):  # min @ 0*
#     return 10*v.size + (v**2 - 10*np.cos(2*np.pi*v)).sum(1)

# def schwefel(v):  # min @ 0*
#     return 418.9829*v.size - (v*np.sin(np.sqrt(np.abs(v)))).sum(1)

def rastrigin(P):
    n = P.shape[-1]
    return (
        P**2 - 10 * np.cos(2*np.pi * P)
    ).sum(axis=1) + 10*n

def schwefel(P):
    n = P.shape[-1]
    return np.sum(
        P * np.sin(
            np.sqrt(
                np.abs(P)
            )
        ),
        axis=1
    ) + 418.9829*n

def sphere(P):
    return np.sum(P**2, axis=1)

def dp(P):
    return (P[:, 0]-1) ** 2 + np.sum(
        (
            2 * P[:, 1:]**2 - P[:, :-1]**2
        ) ** 2 *
        np.arange(2, P.shape[-1] + 1), axis=1
    )

tests = [
    {
        'population_evaluation': griewank,
        'individual_size': 80,
        'domain': (-500, 500)
    },
    {
        'population_evaluation': rastrigin,
        'individual_size': 60,
        'domain': (-5.12, 5.12)
    },
    {
        'population_evaluation': schwefel,
        'individual_size': 60,
        'domain': (-500, 500)
    },
    {
        'population_evaluation': dp,
        'individual_size': 500,
        'domain': (-10, 10)
    },
    {
        'population_evaluation': sphere,
        'individual_size': 60,
        'domain': (-5.12, 5.12)
    }
]

if __name__ == "__main__":
    # pop = np.array([
    #     [1, 2, 3, 0, 1, 0],
    #     [5, 6, 6, 1, 0, 0],
    #     [4, 4, 4, 0, 0, 1]
    # ], dtype=np.float64)
    # x = pop - np.arange(1, pop.shape[-1] + 1)  # [np.newaxis, :]
    # print(x)
    a = np.array([1, -2, -4, 6])
    print(a / a.sum())
    pop = np.vstack((
        np.zeros((1, 15)),
        np.ones((1, 15)) * 600,
        np.arange(15)[np.newaxis, :]
    ))
    print(Griewank().function(pop))


#!/usr/bin/env python3

"""
    Title:  ML Workshop: Simple Linear Regression (SLR)
    Date:   May 15, 2021
    Author: Mario Raciti
"""

import matplotlib.pyplot as plt
import numpy as np
from sklearn.linear_model import LinearRegression
from sklearn.model_selection import cross_validate

np.random.seed(0)

# Create a dataset
# Vars
obs_n = 200
a, b = 3.5, 8
fluctuations = 1
# Feature
X = 1 + 2 * np.random.random(size=(obs_n, 1))
# Target
Y = b + a * X.squeeze() + fluctuations * np.random.randn(obs_n)
# Plot dataset to chart
plt.plot(X, Y, "+")
plt.show()

# Create an SLR model
slr_model = LinearRegression()
# Training
slr_model.fit(X, Y)
print(slr_model.coef_)
print(slr_model.intercept_)

# Predictions
# Single-point prediction
X_pred = np.array([2, ])[:, np.newaxis]
# Multiple-point prediction
X_pred = np.array([2, 3])[:, np.newaxis]
# Multiple-point prediction (dynamic)
first, last, new_obs = X.min(), X.max(), 100
X_pred = np.linspace(first, last, new_obs)[:, np.newaxis]

# Predict
Y_pred = slr_model.predict(X=X_pred)

# Plot predictions to chart
plt.plot(X, Y, '+')
plt.plot(X_pred, Y_pred, 'r-')
plt.show()

# Model Evaluation
# K-fold Cross Validation
K = 5
print(f'Number of folds = {K}\n')
error = 'neg_mean_squared_error'    # NOTE: MSE = RSS/N
results = cross_validate(slr_model, X, Y, cv=K, scoring=error)

# Error for each fold and average error over the folds:
errors = -results['test_score']
print(f'Error for each fold: {errors}')
print(f'Error averaged on all folds: {errors.mean()}')


import numpy as np
import scipy.io as sio
import tensorflow as tf

def one_hot_vector(i):
    one_hot = # Construct a 1D matrix whose cells are filled by zero
    one_hot[i] = # Reassign the position i of one_hot to one
    return one_hot

'''
Step 0: Load the input data, which is stored in digits.mat file.
Then, initialize some variables namely number_of_samples, number_of_features, and number_of_labels
'''

data = sio.loadmat("data/digits.mat")

feature_matrix = data['X']  # the feature matrix is labeled with 'X' inside the file
target_vector = np.squeeze(data['y'])  # the target variable vector is labeled with 'y' inside the file

number_of_samples = feature_matrix.shape[0]  # i.e. 5000 samples
number_of_features = feature_matrix.shape[1]  # i.e. 400 features (20 * 20)
number_of_labels = np.max(target_vector)  # i.e. 1, 2, 3, ..., 10

target_vector_one_hot = np.zeros([number_of_samples, number_of_labels])
for i in range(number_of_samples):
    target_vector_one_hot[i, :] = one_hot_vector(target_vector[i] % 10)

'''
Step 1: Construct TF graph
'''

X = tf.placeholder(tf.float32, [None, number_of_features], name='X')
Y = tf.placeholder(tf.float32, [None, 10], name='Y')  # 0-9 digits recognition. Hence, 10 classes !

thetas = tf.Variable(tf.zeros([number_of_features, 10]), name='Thetas')
theta0 = tf.Variable(tf.zeros([10]), name='theta0')

'''
Step 2: Define the hypothesis function i.e. Softmax function (see Slide# )
'''

hypothesis_function = # Put your code here

'''
Step 3: Define the cost function i.e. the cross-entropy loss
'''

cost_function = # Put your code here

'''
Step 4: Use gradient descent with learning rate of 0.0003 to minimize the cost function
'''

optimizer = # Put your code here

'''
Step 5: Now, we prepare examples for visualizing our test results
'''

# We sample 10 images from the training dataset
random_sample_index = np.random.choice(feature_matrix.shape[0], 10)
feature_random_sample = feature_matrix[random_sample_index, :]
actual_random_sample = target_vector[random_sample_index].T

# Display each randomized image
plt.imshow(feature_random_sample.reshape(-1, 20).T)
plt.axis('off')

class_probability_sample = np.zeros(shape=[10, number_of_labels])

'''
Step 6: Train the model
'''

with tf.Session() as session:
    # Put your code here to:
    # 1) train the model
    # 2) compute the cost for each training step
    # 3) report the number of correct prediction and its accuracy
    # Noted that there may be more than one line of code for this step !

    # Class probability of 10 sampled images
    class_probability_sample = session.run(hypothesis_function, feed_dict={X: feature_random_sample})

    # Compare the actual digits against predicted digits
    print("Actual digit:\n {}".format(actual_random_sample % 10))
    print("Predicted digit:\n {}".format(class_probability_sample.argmax(axis=1)))


import numpy as np
import numpy.linalg as lin
from typing import List, Dict, Any

def multivariate_gaussian(x: np.ndarray, m: np.ndarray, P: np.ndarray) -> float:
    """
    Multivatiate Gaussian Distribution

    :param x: vector
    :param m: distribution mean vector
    :param P: Covariance matrix
    :return: probability density function at x
    """
    first_part = 1 / (((2 * np.pi) ** (x.size / 2.0)) * (lin.det(P) ** 0.5))
    second_part = -0.5 * (x - m) @ lin.inv(P) @ (x - m)
    return first_part * np.exp(second_part)

def multivariate_gaussian_predefined_det_and_inv(x: np.ndarray, m: np.ndarray, detP: np.float64,
                                                 invP: np.ndarray) -> float:
    """
    Multivariate Gaussian Distribution with provided determinant and inverse of the Gaussian mixture.
    Useful in case when we already have precalculted determinant and inverse of the covariance matrix.
    :param x: vector
    :param m: distribution mean
    :param detP: determinant of the covariance matrix
    :param invP: inverse of the covariance matrix
    :return: probability density function at x
    """
    first_part = 1 / (((2 * np.pi) ** (x.size / 2.0)) * (detP ** 0.5))
    second_part = -0.5 * (x - m) @ invP @ (x - m)
    return first_part * np.exp(second_part)

def clutter_intensity_function(z: np.ndarray, lc: int, surveillance_region: np.ndarray):
    """
    Clutter intensity function, with the uniform distribution through the surveillance region, pg. 8
    in "Bayesian Multiple Target Filtering Using Random Finite Sets" by Vo, Vo, Clark.
    :param z:
    :param lc: average number of false detections per time step
    :param surveillance_region: np.ndarray of shape (number_dimensions, 2) giving the range(min and max) for each
                                dimension
    """
    if surveillance_region[0][0] <= z[0] <= surveillance_region[0][1] and surveillance_region[1][0] <= z[1] <= \
            surveillance_region[1][1]:
        # example in two dimensions: lc/((xmax - xmin)*(ymax-ymin))
        return lc / ((surveillance_region[0][1] - surveillance_region[0][0]) * (
                surveillance_region[1][1] - surveillance_region[1][0]))
    else:
        return 0.0

class GaussianMixture:
    def __init__(self, w: List[np.float64], m: List[np.ndarray], P: List[np.ndarray]):
        """
        The Gaussian mixture class

        :param w: list of scalar weights
        :param m: list of np.ndarray means
        :param P: list of np.ndarray covariance matrices

        Note that constructor creates detP and invP variables which can be used instead of P list, for covariance matrix
        determinant and inverse. These lists cen be initialized with assign_determinant_and_inverse function, and
        it is useful in case we already have precalculated determinant and inverse earlier.
        """
        self.w = w
        self.m = m
        self.P = P
        self.detP = None
        self.invP = None

    def set_covariance_determinant_and_inverse_list(self, detP: List[np.float64], invP: List[np.ndarray]):
        """
        For each Gaussian component, provide the determinant and the covariance inverse
        :param detP: list of determinants for each Gaussian component in the mixture
        :param invP: list of covariance inverses for each Gaussian component in the mixture
        """
        self.detP = detP
        self.invP = invP

    def mixture_value(self, x: np.ndarray):
        """
        Gaussian Mixture function for the given vector x
        """
        sum = 0
        if self.detP is None:
            for i in range(len(self.w)):
                sum += self.w[i] * multivariate_gaussian(x, self.m[i], self.P[i])
        else:
            for i in range(len(self.w)):
                sum += self.w[i] * multivariate_gaussian_predefined_det_and_inv(x, self.m[i], self.detP[i],
                                                                                self.invP[i])
        return sum

    def mixture_single_component_value(self, x: np.ndarray, i: int) -> float:
        """
        Single Gaussian Mixture component value for the given vector
        :param x: vector
        :param i: index of the component
        :returns: probability density function at x, multiplied with the component weght at the index i
        """
        if self.detP is None:
            return self.w[i] * multivariate_gaussian(x, self.m[i], self.P[i])
        else:
            return self.w[i] * multivariate_gaussian_predefined_det_and_inv(x, self.m[i], self.detP[i], self.invP[i])

    def mixture_component_values_list(self, x: np.ndarray) -> List[float]:
        """
        Sometimes it is useful to have value of each component multiplied with its weight
        :param x: vector
        :return: List[np.float64]:
        List of components values at x, multiplied with their weight.
        """
        val = []
        if self.detP is None:
            for i in range(len(self.w)):
                val.append(self.w[i] * multivariate_gaussian(x, self.m[i], self.P[i]))
        else:
            for i in range(len(self.w)):
                val.append(
                    self.w[i] * multivariate_gaussian_predefined_det_and_inv(x, self.m[i], self.detP[i], self.invP[i]))
        return val

    def copy(self):
        w = self.w.copy()
        m = []
        P = []
        for m1 in self.m:
            m.append(m1.copy())
        for P1 in self.P:
            P.append(P1.copy())
        return GaussianMixture(w, m, P)

def get_matrices_inverses(P_list: List[np.ndarray]) -> List[np.ndarray]:
    inverse_P_list = []
    for P in P_list:
        inverse_P_list.append(lin.inv(P))
    return inverse_P_list

def get_matrices_determinants(P_list: List[np.ndarray]) -> List[float]:
    """
    :param P_list: list of covariance matrices
    :return:
    """
    detP = []
    for P in P_list:
        detP.append(lin.det(P))
    return detP

def thinning_and_displacement(v: GaussianMixture, p, F: np.ndarray, Q: np.ndarray):
    """
    For the given Gaussian mixture v, perform thinning with probability P and displacement with N(x; F @ x_prev, Q)
    See https://ieeexplore.ieee.org/document/7202905 for details
    """
    w = []
    m = []
    P = []
    for weight in v.w:
        w.append(weight * p)
    for mean in v.m:
        m.append(F @ mean)
    for cov_matrix in v.P:
        P.append(Q + F @ cov_matrix @ F.T)
    return GaussianMixture(w, m, P)

class GmphdFilter:
    def __init__(self, model: Dict[str, Any]):
        """
        The Gaussian Mixture Probability Hypothesis Density filter implementation.
        "The Gaussian mixture probability hypothesis density filter" by Vo and Ma.

        https://ieeexplore.ieee.org/document/1710358

        We assume linear transition and measurement model in the
        following form
            x[k] = Fx[k-1] + w[k-1]
            z[k] = Hx[k] + v[k]
        Inputs:

        - model: dictionary which contains the following elements(keys are strings):

               F: state transition matrix

               H: measurement matrix

               Q: process noise covariance matrix(of variable w[k]).

               R: measurement noise covariance matrix(of variable v[k]).

             p_d: probability of target detection

             p_s: probability of target survival

            Spawning model, see pg. 5. of the paper. It's a Gaussian Mixture conditioned on state

             F_spawn:  d_spawn: Q_spawn: w_spawn: lists of ndarray objects with the same length, see pg. 5

            clutt_int_fun: reference to clutter intensity function, gets only one argument, which is the current measure

               T: U: Jmax: Pruning parameters, see pg. 7.

            birth_GM: The Gaussian Mixture of the birth intensity
        """
        # to do: dtype, copy, improve performance
        self.p_s = model['p_s']
        self.F = model['F']
        self.Q = model['Q']
        self.w_spawn = model['w_spawn']
        self.F_spawn = model['F_spawn']
        self.d_spawn = model['d_spawn']
        self.Q_spawn = model['Q_spawn']
        self.birth_GM = model['birth_GM']
        self.p_d = model['p_d']
        self.H = model['H']
        self.R = model['R']
        self.clutter_density_func = model['clutt_int_fun']
        self.T = model['T']
        self.U = model['U']
        self.Jmax = model['Jmax']

    def spawn_mixture(self, v: GaussianMixture) -> GaussianMixture:
        """
        Spawning targets in prediction step
        """
        w = []
        m = []
        P = []
        for i, w_v in enumerate(v.w):
            for j, w_spawn in enumerate(self.w_spawn):
                w.append(w_v * w_spawn)
                m.append(self.F_spawn[j] @ v.m[i] + self.d_spawn[j])
                P.append(self.Q_spawn[j] + self.F_spawn[j] @ v.P[i] @ self.F_spawn[j].T)
        return GaussianMixture(w, m, P)

    def prediction(self, v: GaussianMixture) -> GaussianMixture:
        """
        Prediction step of the GMPHD filter
        Inputs:
        - v: Gaussian mixture of the previous step
        """
        # v_pred = v_s + v_spawn +  v_new_born
        birth_copy = self.birth_GM.copy()
        # targets that survived v_s:
        v_s = thinning_and_displacement(v, self.p_s, self.F, self.Q)
        # spawning targets
        v_spawn = self.spawn_mixture(v)
        # final phd of prediction
        return GaussianMixture(v_s.w + v_spawn.w + birth_copy.w, v_s.m + v_spawn.m + birth_copy.m,
                               v_s.P + v_spawn.P + birth_copy.P)

    def correction(self, v: GaussianMixture, Z: List[np.ndarray]) -> GaussianMixture:
        """
        Correction step of the GMPHD filter
        Inputs:
        - v: Gaussian mixture obtained from the prediction step
        - Z: Measurement set, containing set of observations
        """
        v_residual = thinning_and_displacement(v, self.p_d, self.H, self.R)
        detP = get_matrices_determinants(v_residual.P)
        invP = get_matrices_inverses(v_residual.P)
        v_residual.set_covariance_determinant_and_inverse_list(detP, invP)

        K = []
        P_kk = []
        for i in range(len(v_residual.w)):
            k = v.P[i] @ self.H.T @ invP[i]
            K.append(k)
            P_kk.append(v.P[i] - k @ self.H @ v.P[i])

        v_copy = v.copy()
        w = (np.array(v_copy.w) * (1 - self.p_d)).tolist()
        m = v_copy.m
        P = v_copy.P

        for z in Z:
            values = v_residual.mixture_component_values_list(z)
            normalization_factor = np.sum(values) + self.clutter_density_func(z)
            for i in range(len(v_residual.w)):
                w.append(values[i] / normalization_factor)
                m.append(v.m[i] + K[i] @ (z - v_residual.m[i]))
                P.append(P_kk[i].copy())

        return GaussianMixture(w, m, P)

    def pruning(self, v: GaussianMixture) -> GaussianMixture:
        """
        See https://ieeexplore.ieee.org/document/7202905 for details
        """
        I = (np.array(v.w) > self.T).nonzero()[0]
        w = [v.w[i] for i in I]
        m = [v.m[i] for i in I]
        P = [v.P[i] for i in I]
        v = GaussianMixture(w, m, P)
        I = (np.array(v.w) > self.T).nonzero()[0].tolist()
        invP = get_matrices_inverses(v.P)
        vw = np.array(v.w)
        vm = np.array(v.m)
        w = []
        m = []
        P = []
        while len(I) > 0:
            j = I[0]
            for i in I:
                if vw[i] > vw[j]:
                    j = i
            L = []
            for i in I:
                if (vm[i] - vm[j]) @ invP[i] @ (vm[i] - vm[j]) <= self.U:
                    L.append(i)
            w_new = np.sum(vw[L])
            m_new = np.sum((vw[L] * vm[L].T).T, axis=0) / w_new
            P_new = np.zeros((m_new.shape[0], m_new.shape[0]))
            for i in L:
                P_new += vw[i] * (v.P[i] + np.outer(m_new - vm[i], m_new - vm[i]))
            P_new /= w_new
            w.append(w_new)
            m.append(m_new)
            P.append(P_new)
            I = [i for i in I if i not in L]

        if len(w) > self.Jmax:
            L = np.array(w).argsort()[-self.Jmax:]
            w = [w[i] for i in L]
            m = [m[i] for i in L]
            P = [P[i] for i in L]

        return GaussianMixture(w, m, P)

    def state_estimation(self, v: GaussianMixture) -> List[np.ndarray]:
        X = []
        for i in range(len(v.w)):
            if v.w[i] >= 0.5:
                for j in range(int(np.round(v.w[i]))):
                    X.append(v.m[i])
        return X

    def filter_data(self, Z: List[List[np.ndarray]]) -> List[List[np.ndarray]]:
        """
        Given the list of collections of measurements for each time step, perform filtering and return the
        estimated sets of tracks for each step.

        :param Z: list of observations(measurements) for each time step
        :return X:
        list of estimated track sets for each time step
        """
        X = []
        v = GaussianMixture([], [], [])
        for z in Z:
            v = self.prediction(v)
            v = self.correction(v, z)
            v = self.pruning(v)
            x = self.state_estimation(v)
            X.append(x)
        return X


import numpy as np
import matplotlib.pyplot as plt

# Make a prediction with weights
def predict(row, weights):
    activation = weights[0]
    for i in range(len(row)-1):
        activation += weights[i + 1] * row[i]
    return 1.0 if activation >=0.0 else -1

# Estimate Perceptron weights using stochastic gradient descent
# n_epoch is number of times to iterate over dataset
# l_rate is the amount by which we will modify the weights in each iteration
def train_weights(train, l_rate, n_epoch):
    weights = [1.0 for i in range(len(train[0]))]
    for epoch in range(n_epoch):
        sum_error = 0.0
        for row in train:
            prediction = predict(row, weights)
            error = row[-1] - prediction
            sum_error += error**2
            # Threshold updation
            weights[0] = weights[0] + l_rate * error
            for i in range(len(row)-1):
                weights[i + 1] = weights[i + 1] + l_rate * error * row[i]
            # For plotting
            # Generating points for showing the line
            if weights[2]!=0:
                y = []
                for i in dataset :
                    y.append(-(weights[0] + weights[1]*i[0])/weights[2])
                plt.cla()
                plt.hold(True)
                plt.scatter(data_array[:,0],data_array[:,1],marker="o",c=data_array[:,2])
                plt.plot(data_array[:,0], y)
                plt.hold(False)
                plt.pause(0.3)
        print('epoch=%d, lrate=%.3f, error=%.3f' % (epoch, l_rate, sum_error))
    return weights

# Calculate weights
dataset = [[1, 43, 1], [2, -43, -1], [3, 58, 1], [4, -37, -1], [5, 73, 1], [6, -34, -1], [7, 74, 1], [8, -30, -1], [9, 85, 1], [10, -15, -1], [11, 73, 1], [12, -11, -1], [13, 83, 1], [14, -14, -1], [15, 95, 1], [16, 5, -1], [17, 110, 1], [18, 1, -1], [19, 115, 1], [20, 16, -1], [21, 111, 1], [22, 25, -1], [23, 126, 1], [24, 23, -1], [25, 133, 1]]

data_array = np.array(dataset)
plt.ion()
plt.figure()
plt.scatter(data_array[:,0],data_array[:,1],marker="o",c=data_array[:,2])
l_rate = 0.01
n_epoch = 2
weights = train_weights(dataset, l_rate, n_epoch)

plt.pause(1000)

print("---------Results---------")
print("Final weights:", weights)


import numpy as np

def random_to_polar(x: np.array):
    """
    Converts an array of 2 random numbers between 0 and 1 into a polar coordinate with distance between 0 and 1 and angle
    between 0 and 2 * pi radians
    :param x: an array [x0, x1] of 2 random numbers
    :return: the corresponding polar coordinate [distance, angle]
    """
    magnitude = x[0]
    angle = x[1] * 2 * np.pi
    return np.array([magnitude, angle])

def polar_to_cartesian(x: np.array):
    """
    Converts a polar coordinate into the corresponding 2D cartesian coordinate
    :param x: an array [distance, angle] representing a polar coordinate
    :return: the corresponding 2D cartesian coordinate [x0, x1]
    """
    cartesian = np.array([x[0] * np.cos(x[1]), x[0] * np.sin(x[1])])
    return cartesian


# firstorder.py - First order polynomial fitting

import sys
import numpy as np

def algsol(x, y):
    """
    Function to fit a straight line to data and estimate slope and
    intercept of the line and corresponding errors using algebraic solution.

    Parameters
    ----------
    x : ndarray
        X-axis data

    y : ndarray
        Y-axis data

    Returns
    -------
    ndarray
        slope, intercept, SDslope, SDintercept

    Reference
    ---------

    """
    # Number of input points
    N = x.shape[0]

    # Estimate intercept and slope of the straight line fitting
    Sxx = np.sum((x - np.mean(x))**2)
    Syy = np.sum((y - np.mean(y))**2)
    Sxy = np.sum((x - np.mean(x)) * (y - np.mean(y)))
    calcSlope = Sxy / Sxx
    calcIntercept = np.mean(y) - calcSlope * np.mean(x)
    Sy = np.sqrt((Syy - Sxx * calcSlope**2) / (N - 2))
    SDSlope = Sy / np.sqrt(Sxx)
    SDIntercept = Sy * np.sqrt(1 / (N - (np.sum(x)**2 / np.sum(x**2))))

    return np.array([[calcSlope, calcIntercept], [SDSlope, SDIntercept]])

def polyFit(x, y):
    """
    Function to fit a straight line to data and estimate slope and
    intercept of the line and corresponding errors using first order
    polynomial fitting.

    Parameters
    ----------
    x : ndarray
        X-axis data

    y : ndarray
        Y-axis data

    Returns
    -------
    ndarray
        slope, intercept, SDslope, SDintercept

    Reference
    ---------

    """
    # Number of input points
    N = x.shape[0]

    # Estimate slope and intercept of fitted line
    slope, intercept = np.polyfit(x, y, 1)

    # Calculate standard deviation of slope and intercept
    yhat = intercept + slope * x
    residual = y - yhat

    Sx2 = np.sum(x**2)
    Sxx = np.sum((x - np.mean(x))**2)
    Sy_x = np.sqrt(np.sum(residual**2) / (N -2))
    SDslope = Sy_x / np.sqrt(Sxx)
    SDintercept = Sy_x * np.sqrt(Sx2 / (N * Sxx))

    return np.array([[slope, intercept], [SDslope, SDintercept]])

def linalgFit(x, y):
    """
    Function to fit a straight line to data and estimate slope and
    intercept of the line and corresponding errors using linear algebra.

    Parameters
    ----------
    x : ndarray
        X-axis data

    y : ndarray
        Y-axis data

    Returns
    -------
    ndarray
        slope, intercept, SDslope, SDintercept

    Reference
    ---------

    """
    # Number of input points
    N = x.shape[0]

    Sx = np.sum(x)
    Sx2 = np.sum(x**2)
    Sy = np.sum(y)
    Sxy = np.sum(x * y)

    X = np.array([[N,Sx],[Sx,Sx2]])
    Y = np.array([[Sy], [Sxy]])

    intercept, slope = np.linalg.lstsq( X, Y )[0]

    # Calculate standard deviation of slope and intercept
    yhat = intercept + slope * x
    residual = y - yhat

    Sxx = np.sum((x - np.mean(x))**2)
    Sy_x = np.sqrt(np.sum(residual**2) / (N -2))
    SDslope = Sy_x / np.sqrt(Sxx)
    SDintercept = Sy_x * np.sqrt(Sx2 / (N * Sxx))

    return np.array([[slope, intercept], [SDslope, SDintercept]])

def MonteCarloWorker(indata):
    """
    Monte Carlo worker function for single processor and multicore processing

    Parameters
    ----------
    indata : ndarray
        input parameters for worker function

    Returns
    -------
    mcoeffs : tuple
        Fitted parameters
    """
    # Unpack input data
    ind, x, coeffs, SDnoise = indata

    print >> sys.stdout, '\n Monte Carlo iteration - ', ind + 1

    # Form polynomical equation based on number of coefficients
    lx = len(x)
    n_coeffs = len(coeffs)
    i, ymc = 0, np.zeros(lx)
    while i < n_coeffs:
        ymc += coeffs[i] * x**i
        i += 1

    # Add normal random noise to y value
    ymc += SDnoise * np.random.randn(lx)

    # Fit the noisy data using numpy polyfit
    mcCoeffs = np.polyfit(x, ymc, n_coeffs-1)

    # return a tuple of slope and intercept
    return mcCoeffs

def MonteCarloSim(x, coeffs, SDnoise = 1.0, niter = 1000, parallel = False):
    """
    Monte Carlo Simulation routine for linear fitting using normal random
    noise. Standard deviation of noise should be closer to the one calculated
    by algebraic method to get relatistic results.

    Parameters
    ----------
    x : ndarray
        X-axis data

    coeffs : ndarray
        Fitted coefficients

    SDnoise : float
        Standard deviation of random noise

    niter : int
        Number of Monte Carlo iterations. Default is 1000.

    parallel : bool
        Do parallel processing? Default is False.

    Returns
    -------
    MCResults : ndarray
        Monte Carlo result
    """
    # Populate input data list
    idata = []
    ind = range(niter)
    idata = map(lambda i: [i, x, coeffs, SDnoise], ind)

    # Import multiprocessing module
    try:
        import multiprocessing as mp
    except:
        n_cpus = 1
    else:
        n_cpus = mp.cpu_count()

    # Conditional loop for single and multicore processing
    if n_cpus == 1 or parallel == False:
        results = map(MonteCarloWorker, idata)
    else:
        pool = mp.Pool( n_cpus )
        results = pool.map(MonteCarloWorker, idata)

    # Convert result array to numpy num_coeffxniter array. The first
    # row is slope values and second row is intercept values
    MCResults = np.array(results).T

    # Output mean value of slope and intercept as well as
    # standard deviation values
    return MCResults

def MonteCarlo(x, y, SDnoise = 1.0, niter = 1000, parallel = False):
    """
    Function to run Monte Carlo to determine mean slope and intercept.
    Standard deviation of slope and intercept is also calculated.

    Parameters
    ----------
    x : ndarray
        X-axis data

    y : ndarray
        Y-axis data

    SDnoise : float
        standard deviation of noise

    niter : int
        number of Monte Carlo iterations

    parallel : bool
        run in parallel mode? Default if False

    Returns
    -------
    ndarray
        Mean slope and intercept and SD of slope and intercept

    References
    ----------

    """
    # Estimate slope and intercept of fitted line
    coeffs = np.polyfit(x, y, 1)

    # Input coefficient array is inverted for ease of processing
    results = MonteCarloSim(x, coeffs[-1::-1], SDnoise, niter, parallel)

    return np.array([[np.mean(results[0,:]), np.mean(results[1,:])], [np.std(results[0,:]), np.std(results[1,:])]])

def BootStrapWorker(indata):
    """
    Bootstrap worker function for single processor and multicore processing

    Parameters
    ----------
    indata : ndarray
        input parameters for worker function

    Returns
    -------
    mcCoeffs : tuple
        Fitted parameters
    """
    # Unpack input values
    i, x, y, coeffs, resample = indata

    print >> sys.stdout, '\n Bootstrap iteration - ', str(i+1)

    lx = len(x)
    n_coeffs = len(coeffs)
    resamplex, resampley = np.copy(x), np.copy(y)
    if resample == 'case':
        # Case resampling - replacement of elements (duplicates allowed)
        j = 0
        while j < lx - 2:
            if np.random.rand() > 0.5:
                resamplex[j] = x[j+1]
                resampley[j] = y[j+1]
                j += 1

    elif resample == 'residual':
        # Calculate residuals
        i, yhat = 0, np.zeros(lx)
        while i < n_coeffs:
            yhat += coeffs[i] * x**i
            i += 1
        residual = y - yhat

        # Residual resampling by using residual of model and actual values.
        ind = np.arange(lx)
        resampley = map(lambda i: yhat[i] + np.random.choice(residual), ind)

    elif resample == 'wild':
        # Calculate residuals
        i, yhat = 0, np.zeros(lx)
        while i < n_coeffs:
            yhat += coeffs[i] * x**i
            i += 1
        residual = y - yhat

        # Wild resampling by using residual of model and actual values
        # and multiplied by normal distributed noise
        ind = np.arange(lx)
        resampley = map(lambda i: yhat[i] + np.random.choice(residual) * np.random.randn(), ind)

    else:
        print >> sys.stdout, 'Error: Only Case, Residual and Wild bootstrap resampling methods allowed. Exiting.'
        sys.exit(-1)

    # Fit the noisy data using numpy polyfit
    mcCoeffs = np.polyfit(resamplex, resampley, n_coeffs-1)

    # return a tuple of slope and intercept
    return mcCoeffs

def BootStrapSim(x, y, coeffs, resample = 'residual', numtrials = 1000, parallel = False):
    """
    Bootstrap Simulation routine for linear fitting.

    Parameters
    ----------
    x : ndarray
        X-axis data

    y: ndarray
        Y-axis data

    coeffs : ndarray
        Fitted coefficients

    resample : string
        Resampling method. Default is residual.

    numtrials : int
        Number of bootstrap trials. Default is 1000.

    parallel : bool
        Do parallel processing? Default is False.

    Returns
    -------
    BSResults : ndarray
        Monte Carlo result
    """
    # Populate input data list
    idata = []
    ind = range(numtrials)
    idata = map(lambda i: [i, x, y, coeffs, resample], ind)

    # Import multiprocessing module
    try:
        import multiprocessing as mp
    except:
        n_cpus = 1
    else:
        n_cpus = mp.cpu_count()

    # Conditional loop for single and multicore processing
    if n_cpus == 1 or parallel == False:
        results = map(BootStrapWorker, idata)
    else:
        pool = mp.Pool( n_cpus )
        results = pool.map(BootStrapWorker, idata)

    # Create a numpy num_coeffs x numtrials array, with each row
    # correspond to coeff. values in all the trials
    BSResults = np.array(results).T

    return BSResults

def BootStrap(x, y, resample = 'residual', numtrials = 1000, parallel = False):
    """
    Function to run Bootstrapping to determine mean slope and intercept.
    Standard deviation of slope and intercept is also calculated.

    Parameters
    ----------
    x : ndarray
        X-axis data

    y : ndarray
        Y-axis data

    resample : string
        Resampling method. Default is residual.

    nimtrials : int
        Number of bootstrapping trials

    parallel : bool
        run in parallel mode? Default if False

    Returns
    -------
    ndarray
        Mean slope and intercept and SD of slope and intercept

    References
    ----------

    """
    # Estimate slope and intercept of fitted line
    coeffs = np.polyfit(x, y, 1)

    # Input coefficient array is inverted for ease of processing
    results = BootStrapSim(x, y, coeffs[-1::-1], resample, numtrials, parallel)

    return np.array([[np.mean(results[0,:]), np.mean(results[1,:]), np.std(results[0,:]), np.std(results[1,:])]])


from sympy import isprime

def primorial_number_series(n):
    primorial_numbers = [1]  # by definition, primorial(0) = 1

    i = 2  # starting the check for prime numbers from 2
    while len(primorial_numbers) != n:  # the loop runs until we have length of the sequence equal to our requirement
        if isprime(i):  # checks whether a number is prime or not
            primorial_numbers.append(i * primorial_numbers[len(primorial_numbers) - 1])  # the new element in the sequence is obtained by multiplying the current prime number by the product of all previous prime numbers

        i += 1

    return primorial_numbers

if __name__ == "__main__":
    n = int(input("Enter the number of terms of the sequence you want to see: "))

    for i, j in enumerate(primorial_number_series(n)):  # iterating over the sequence of primorial numbers
        print("primorial({}) = {}".format(i, j))


'''
Mloc_TUK computes Tukey's M-estimate of
location, i.e.,

mu_hat = arg min_mu SUM_i rho_TUK(y_i - mu)

   INPUTS:
           y: real valued data vector of size N x 1
           c: tuning constant c>=0 . default = 4.685
           max_iters: Number of iterations. default = 1000
           tol_err: convergence error tolerance. default = 1e-5

   OUTPUT:
           mu_hat: Tukey's M-estimate of location
'''
import numpy as np

from robustsp.AuxiliaryFunctions.madn import madn
from robustsp.AuxiliaryFunctions.wtuk import wtuk

def MLocTUK(y,c=4.685, max_iters = 1000, tol_err = 1e-5):
    y = np.asarray(y) # ensure that y is a ndarray

    # previously computed scale estimate
    sigma_0 = madn(y)

    # initial robust location estimate
    mu_n = np.median(y);
    # computes tukey weights
    wtuk = lambda absx,cl: np.square(1-np.square(absx/cl)) * (absx<=cl)

    for n in range(max_iters+1):
        w_n = wtuk(np.absolute(y-mu_n)/sigma_0,c) # compute weights
        mu_n_plus1 = np.sum(w_n*y)/(np.sum(w_n)) # compute weighted average
        if np.absolute(mu_n_plus1-mu_n)/sigma_0 > tol_err: # breaking condition
            mu_n = mu_n_plus1 # update estimate of mean
            n = n+1 # increment iteration counter
        else:
            break

    return mu_n # final estimate


# Working on 2D Tensors basics -types, shape, indexing, slicing and operations

#Import Libraries
import numpy as np
import torch
import pandas as pd
import matplotlib.pyplot as plt

## Types and Shapes

twoD_list=[[11,12,13],[21,22,23],[31,32,33]]
twoD_tensor=torch.tensor(twoD_list)
with open('Lab1_Tensors_2D_Results.txt','a') as f:
    print('2D List <--> 2D Tensor: \n',file=f)
    print(twoD_tensor,file=f)
    print('\n',file=f)

with open('Lab1_Tensors_2D_Results.txt','a') as f:
    print('Basic elements: \n',file=f)
    print('The dimension of 2D tensor: ',twoD_tensor.ndimension(),file=f)
    print('The shape of 2D tensor: ',twoD_tensor.shape,file=f)
    print('The size of 2D tensor: ',twoD_tensor.size(),file=f)
    print('The no. of elements of 2D tensor: ',twoD_tensor.numel(),file=f)
    print('\n',file=f)

## Numpy <--> Tensor, Pandas DF --> Tensor

twoD_numpy=twoD_tensor.numpy()
new_twoD_tensor=torch.from_numpy(twoD_numpy)

with open('Lab1_Tensors_2D_Results.txt','a') as f:
    print('Numpy --> Tensor: \n',file=f)
    print('The numpy after converting:\n',twoD_numpy,file=f)
    print('The type of numpy: ',twoD_numpy.dtype,file=f)
    print('-------------------------------------------',file=f)
    print('The tensor from numpy:\n',new_twoD_tensor,file=f)
    print('The type of new tensor from numpy: ',new_twoD_tensor.dtype,file=f)
    print('\n',file=f)

df=pd.DataFrame({'a':[11,21,31],'b':[12,22,32]})
new_df_tensor=torch.from_numpy(df.values)
with open('Lab1_Tensors_2D_Results.txt','a') as f:
    print('Pandas DF --> Tensor: \n',file=f)
    print('Pandas to numpy:\n',df.values,file=f)
    print('Type before conversion: ',df.values.dtype,file=f)
    print('-------------------------------------------',file=f)
    print('The tensor after converting:\n',new_df_tensor,file=f)
    print('The type of new tensor after converting: ',new_df_tensor.dtype,file=f)
    print('\n',file=f)

## Indexing and Slicing
tensor_example=torch.tensor(twoD_list)
with open('Lab1_Tensors_2D_Results.txt','a') as f:
    print('Indexing: \n',file=f)
    print('Value on 2nd row and 3rd column: ',tensor_example[1,2],file=f)
    print('Value on 2nd row and 3rd column: ',tensor_example[1][2],file=f)
    print('-------------------------------------------',file=f)
    print('Indexing and Slicing: \n',file=f)
    print('Value on 1st row and first two columns: ',tensor_example[0,0:2],file=f)
    print('Value on 1st row and first two columns: ',tensor_example[0][0:2],file=f)
    print('\n',file=f)

## Operations - Add, Scalar Multiplication, Element-wise Product, Matrix Mulitplication

X=torch.tensor([[1,0],[0,1]])
Y=torch.tensor([[2,1],[1,2]])

XYadd=X+Y # Addition
Z=2*Y #Multilpcation
XY_times=X*Y #Element wise product

#If X and Y are not same dimension matrix then X*Y is not necessarily same as Y*X
X_new=torch.tensor([[0,1,1],[1,0,1]]) # 2 x 3 matrix
Y_new=torch.tensor([[1,1],[1,1],[-1,1]]) # 3 x 2 matrix
XY_mm=torch.mm(X_new,Y_new) # 2 x 2 result of product

with open('Lab1_Tensors_2D_Results.txt','a') as f:
    print('Operations: \n',file=f)
    print('Adding X and Y gives:\n',XYadd,file=f)
    print('Scalar Mutiplication:\n',Z,file=f)
    print('Element Wise Product:\n',XY_times,file=f)
    print('Matrix Multiplication Xnew,Ynew:\n',XY_mm,file=f)
    print('\n',file=f)

Deep Neural Networks with PyTorch/Module1/Lab1_Tensors_2D.py


# Implement Bayes Classifier here!
# region Packages
#This Region contains all imported packages that is used inside the code.
import numpy as np
import sklearn
from sklearn.metrics import f1_score
import math
# endregion

def classPartition(trainDataSet,trainLabels):
    '''
    :param trainDataSet:
    :param trainLabels:
    :return: ClassPartitionSet-A Dict having data-elements of a particular class grouped together.
    Author: KD
    Details: This Code Partitions the Train Data as per Class Labels
    '''
    ClassPartitionSet={}
    for i in range(len(trainDataSet)):
        dataElement=list(trainDataSet[i])
        dataLabel=trainLabels[i]
        #if(dataLabel!=3):
        if(dataLabel not in ClassPartitionSet):
            ClassPartitionSet[dataLabel]=[]
        ClassPartitionSet[dataLabel].append(dataElement)
    return ClassPartitionSet

def meanOfClass(featureValues):
    '''
    :param featureValues:
    :return:Mean of Feature Values
    Author: KD
    Details: This Code Calculates the mean of Feature Values.
    '''
    return np.mean(featureValues)

def stdevOfClass(featureValues):
    '''
    :param featureValues:
    :return:Standard Deviation of Feature Values
    Author: KD
    Details: This Code Calculates the Standard Deviation of Feature Values.
    '''
    return math.sqrt(np.var(featureValues))

def calMeanVar(featureSet):
    '''
    :param  featureSet:
    :return: meanVarSet
    Author: KD
    Details: This Code Calculates Mean and Variance for each feature of the feature set.
    '''
    meanVarSet = [(meanOfClass(attribute), stdevOfClass(attribute)) for attribute in zip(*featureSet)]
    return meanVarSet

def calculateGaussianDensity(x, mean, stdev):
    '''
    :param x:
    :param mean:
    :param stdev:
    :return:This code returns probability density f(x)~N(mean,stdev) for each x from test data set.
    Author: KD
    '''
    a=math.pow(x - mean, 2)
    b=math.pow(stdev, 2)
    b=2*b
    c=-(a/b)
    d=np.exp(c)
    exponent = np.exp(-(math.pow(x - mean, 2) / (2 * math.pow(stdev, 2))))
    prob=(1 / (math.sqrt(2 * math.pi) * stdev)) * exponent
    return prob

def calculateClassConditionalDensities(meanStdSet, testInputVector):
    '''
    :param meanStdSet:
    :param testInputVector:
    :return:classConditionalProb
    Author: KD
    Details: This code calculates the class conditional densities of input vector for each class.
    '''
    classConditionalProb = {}
    for classlabel, classmeanStd in meanStdSet.items():
        #classConditionalProb[classlabel] = 1
        for i in range(len(classmeanStd)):
            mean, stdev = classmeanStd[i]
            if (stdev != 0):
                x = testInputVector[i]
                if (classlabel not in classConditionalProb):
                    classConditionalProb[classlabel]=calculateGaussianDensity(x, mean, stdev)
                else:
                    classConditionalProb[classlabel] *= calculateGaussianDensity(x, mean, stdev)
            else:
                continue
    return classConditionalProb

def bayesClassifier(trainData, trainLabels, testData,testLabels):
    '''
        :param dataMatrix:
        :param labelMatrix:
        :return: Accuracy of the classifier
        Author: KD
        Details: This is the main BayesClassifier function.
        '''
    # region handle older versions of sklearn
    if int((sklearn.__version__).split(".")[1]) < 18:
        from sklearn.cross_validation import train_test_split
    # otherwise we're using at lease version 0.18
    else:
        from sklearn.model_selection import train_test_split
    # endregion

    # region Sklearn Package  Bayes
    '''
    from sklearn.naive_bayes import GaussianNB
    print("Executing Sklearn Bayes Classifier")
    gnb = GaussianNB()
    y_pred = gnb.fit(trainData, trainLabels).predict(testData)
    Total = len(testData)
    predicted = Total - (testLabels != y_pred).sum()
    accuracyOfMyCode = (predicted / Total) * 100.0
    f1_score_macro = f1_score(testLabels, y_pred, average='macro')
    f1_score_micro = f1_score(testLabels, y_pred, average='micro')
    '''
    # endregion

    # region Custom  Bayes Classifier
    print("Executing Custom Bayes Classifier")
    classPartitionSet = classPartition(trainData, trainLabels)
    priorProbSet = {}
    for classValue, features in classPartitionSet.items():
        priorProbSet[classValue] = len(features) / len(trainData)

    # region Other Two Data Sets
    meanStdSet = {}
    for classValue, features in classPartitionSet.items():
        meanStdSet[classValue] = calMeanVar(features)

    predictions = []
    accuracy = 0
    for i in range(len(testData)):
        testVector = testData[i]
        probabilities = calculateClassConditionalDensities(meanStdSet, testVector)
        posteriorProb={}
        for key in probabilities:
            posteriorProb[key]=priorProbSet[key]*probabilities[key]
        predictedLabel = None
        PredictionProb = -1
        for classValue, probability in posteriorProb.items():
            if predictedLabel is None or probability > PredictionProb:
                PredictionProb = probability
                predictedLabel = classValue

        predictions.append(predictedLabel)
    # endregion

    for x in range(len(testData)):
        if testLabels[x] == predictions[x]:
            accuracy += 1
    accuracyOfMyCode = (accuracy / len(testData)) * 100.0
    f1_score_macro = f1_score(testLabels, predictions, average='macro')
    f1_score_micro = f1_score(testLabels, predictions, average='micro')
    # endregion

    return accuracyOfMyCode, f1_score_macro, f1_score_micro


"""
    Simple Implementation of Logistic Regression (LR) for binary classification.
    Label (class): y \in {+1, -1}.
    Model: p(y = +1 | x) = g(w'x),
    where g is a logistic function, w' means w transpose.
"""

import numpy as np
import scipy.optimize

def logistic(z):
    """ Logistic (sigmoid) function."""

    return 1.0/(1.0 + np.exp(-z))

def predict(w,x):
    """ If p(y = +1 | x) > 0.5, then class y = +1. Otherwise y = -1."""

    return logistic(np.dot(w,x)) > 0.5 or -1

"""
    Training phase: MLE + L2-regularized log-likelihood.
    To maximize the log-likelihood, we use gradient ascent for simplicity.
"""
def log_likelihood(X, Y, w, C=0.1):
    """
        Log likelihood function with regularization parameter C >= 0 (default: C = 0.1)
        See README for more details.
    """

    return np.sum(np.log(logistic(Y*np.dot(X,w))) - C/2*np.dot(w,w))

def log_likelihood_grad(X, Y, w, C=0.1):
    """ Compute the gradient of log-likelihood function above."""

    # d is dimensionality of a sample.
    d = len(w)
    # N is #training samples.
    N = len(X)
    s = np.zeros(d)

    for i in range(N):
        s += Y[i] * X[i] * logistic(-Y[i] * np.dot(X[i], w))

    s -= C*w
    return s

def grad_num(X, Y, w, f, eps=0.00001):
    """ Compute gradient numerically. Just for testing."""

    d = len(w)
    ident = np.identity(d)
    g = np.zeros(d)

    for i in range(d):
        g[i] += f(X,Y,w+eps*ident[i])
        g[i] -= f(X,Y,w-eps*ident[i])
        g[i] /= 2*eps
    return g

def train(X, Y, C=0.1):
    """
        Training phase by using BFGS algorithm to minimize -log-likelihood function.
    """
    def f(w):
        return -log_likelihood(X, Y, w, C)
    def fprime(w):
        return -log_likelihood_grad(X, Y, w, C)

    d = X.shape[1]
    initial_guess = np.zeros(d)

    return scipy.optimize.fmin_bfgs(f, initial_guess, fprime, disp=False)

def accuracy(X,Y,w):
    """
        First, evaluate the classifier on training data.
    """
    n_correct = 0
    for i in range(len(X)):
        if predict(w, X[i]) == Y[i]:
            n_correct += 1
    return n_correct * 1.0 / len(X)

# Splitting data in ith fold.
def fold(arr, K, i):
    """
        Split origin data to training data + validation data.
        Return: Tuple (hold-out samples, training samples).
    """
    N = len(arr)
    size = np.ceil(1.0 * N / K)
    arange = np.arange(N)
    holdout = np.logical_and(i*size <= arange, arange < (i+1)*size)
    rest = np.logical_not(holdout)
    return arr[holdout], arr[rest]

def kfold(arr, K):
    """ K-fold training data splitting."""
    return [fold(arr, K, i) for i in range(K)]

def avg_accuracy(all_x, all_y, C):
    """
        Calculate the CV score for each setup (C).
    """
    s = 0
    # nfolds is number of folds.
    nfolds = len(all_x)
    for i in range(nfolds):
        x_holdout, x_rest = all_x[i]
        y_holdout, y_rest = all_y[i]

        # Training phase to get w
        w = train(x_rest, y_rest, C)

        # Evaluate with hold-out samples
        s += accuracy(x_holdout, y_holdout, w)
    return s * 1.0 / nfolds

def train_C(X, Y, K=10):
    """ Using k-fold CV to optimize C."""
    # C candidates
    all_C = np.arange(0, 1, 0.1)
    all_X = kfold(X,K)
    all_Y = kfold(Y,K)
    all_acc = np.array([avg_accuracy(all_X, all_Y, C) for C in all_C])
    return all_C[all_acc.argmax()]

def read_data(filename, sep=",", aFilter=int):
    """
        Data format(ith line): label_i,x_i1,x_i2,...x_id
    """
    def split_line(line):
        return line.split(sep)
    def process_line(line):
        return map(aFilter, split_line(line))

    with open(filename) as f:
        lines = map(process_line, f.readlines())
        # adding x[0] = 1 for all rows
        X = np.array([ [1]+line[1:] for line in lines ])
        # the first column is class label of training data
        Y = np.array([line[0] or -1 for line in lines])
    return X, Y

def main(training_file='SPECT.train.txt', test_file='SPECT.test.txt'):
    # Read data
    X_train, Y_train = read_data(training_file)

    # Training phase
    optC = train_C(X_train, Y_train)
    print "Optimized C was", optC
    w = train(X_train, Y_train, optC)
    print 'w was', w

    # Testing phase
    X_test, Y_test = read_data(test_file)
    print 'Accuracy was', accuracy(X_test, Y_test, w)

if __name__ == '__main__':
    main()


# -*- coding: utf-8 -*-
"""
Created on Fri Oct 19 21:44:03 2018

@author: Tim

This solves the 1D Poisson equation by finite differences, assuming a constant
dielectric constant. A tridiagonal matrix is set up and solved.Con

Inputs: number of mesh points, left and right voltage boundary conditions,
dielectric constant.
"""

import numpy as np, time
import matplotlib
import matplotlib.pyplot as plt  # allows to make matlab style plots!

def thomasSolve(matrix, rhs):

    x = np.zeros(num_elements+2)  # array for the solution

   # Forward substitution
    for i in range(2, matrix.num_elements + 1): # recall range uses [ ) ]
        cdiag_ratio = matrix.lower_diag[i-1]/matrix.main_diag[i-1]
        matrix.main_diag[i] -= cdiag_ratio * matrix.upper_diag[i-1]
        rhs[i] -= cdiag_ratio * rhs[i-1]

    # Backward substitution
    x[num_elements] = rhs[num_elements]/matrix.main_diag[num_elements] #lin eqn. corresponding to last row
    for i in range(num_elements, 1, -1):  # 3rd argument is the step (iterate down)
        x[i-1] = (rhs[i-1] - x[i]*matrix.upper_diag[i-1])/matrix.main_diag[i-1]

    return x

class Poisson_matrix():

    def __init__(self, epsilon, num_elements):
        self.epsilon = epsilon  # save epsilon as attribute of the class
        self.num_elements = num_elements
        self.main_diag = -2*epsilon*np.ones(num_elements+1)
        self.upper_diag = epsilon*np.ones(num_elements)
        self.lower_diag = self.upper_diag  #poisson matrix is symmetric

# System setup

#num_elements =  int(input("Enter number of mesh points "))
#V_left_BC = float(input("Voltage at the left boundary "))
#V_right_BC = float(input("Voltage at the right boundary "))
#epsilon = float(input("Relative dielectric constant "))

num_elements = 99
V_left_BC = 0
V_right_BC = 0.9
epsilon = 3.8

start = time.time()

poisson_mat = Poisson_matrix(epsilon, num_elements)  # initialize object

# setup right hand side--> for now just hard code in a dipole, later will
# read the charges from file
rhs =  np.zeros(num_elements+1)  # make it +1 b/c I want to index from 1...
rhs[24] = 0.681
rhs[25] = -0.681

# apply boundary conditions
rhs[1] -= epsilon*V_left_BC
rhs[num_elements] -= epsilon*V_right_BC

V = thomasSolve(poisson_mat, rhs)

# add the boundary values (which didn't need to be solved for)
V[0] = V_left_BC
V[num_elements+1] = V_right_BC

end = time.time()
print(V)
print(f"CPU time: {end - start}")
# C++ version of this with 9999 elements runs in ~0.0032 sec

# make plot
fig, ax = plt.subplots()  # make a figure
ax.plot(V)
ax.set(xlabel = 'Position', ylabel='Voltage (V)', title = 'Poisson solver result')
ax.grid()  # add a grid
fig.savefig("poisson_result.jpg")
plt.show()


"""

Student Name: Hannah Richstein
Professor Name: Dr. Frinchaboy
Class: PHYS 50733
HW 3, Prob 2a, 2b
Last edited: 12 February 2017

Overview:
---------
This program implements two functions using different equation forms to solve
for the roots of the quadratic equation

Input:
------
The coefficients of the quadratic equation

Output:
-------
The two roots, as found by each function

Program Limitations:
--------------------
Machine precision

"""
from numpy import sqrt, array, sort

a = float(input("Please enter the three coefficients. \na: "))
b = float(input("b: "))
c = float(input("c: "))

###############################################################################
# Function implementing the standard formula for finding the roots of the
# quadratic equation

def quad_func(a, b, c):
	discriminant = sqrt(b**2 - (4 * a * c))

	x_pos_d = discriminant
	x_neg_d = -discriminant

	results = array([x_pos_d, x_neg_d])

	results += (-b)
	results /= (2 * a)

	return sort(results)

###############################################################################
# Function implementing the a different, but equivalent formula for finding the
# roots of the quadratic equation

def sec_q_func(a, b, c):
	numerator = 2 * c
	discriminant = sqrt(b**2 - (4 * a * c))

	plus_denom = -b + discriminant
	neg_denom = -b - discriminant

	results = array([(numerator / plus_denom), (numerator / neg_denom)])

	return sort(results)

###############################################################################
# Calculating the roots and printing them
# Printing all of the digits in order to best see where the roots differ

func_1_res = quad_func(a, b, c)

func_2_res = sec_q_func(a, b, c)

print("\nSolutions, Method 1: ({0}, {1})".format(func_1_res[0],
      func_1_res[1]))

print("\nSolutions, Method 2: ({0}, {1})".format(func_2_res[0], func_2_res[1]))

# END PROGRAM


# Exercise 5.15
# Author: Noah Waterfield Price

import numpy as np
import matplotlib.pyplot as plt

def L_k(x, k, xp, yp):
    ans = 1
    for i in range(len(xp)):
        if i != k:
            ans *= (x - xp[i]) / (xp[k] - xp[i])
    return ans

def p_L(x, xp, yp):
    ans = 0
    for k in range(len(yp)):
        ans += yp[k] * L_k(x, k, xp, yp)
    return ans

def verify(xp, yp):
    status = 'Verified!'
    for k in range(len(xp)):
        if abs(p_L(xp[k], xp, yp) - yp[k]) > 1e-15:
            status = 'Error!'
            break
    print status

def verbose_verify(xp, yp):
    print '%10s %10s %10s %12s' % ('x', 'Exact', 'Approx', 'Difference')
    for k in range(len(xp)):
        x = xp[k]
        exact = yp[k]
        approx = p_L(xp[k], xp, yp)
        diff = abs(p_L(xp[k], xp, yp) - yp[k])
        print '%10f %10f %10f %12f' % (x, exact, approx, diff)

def graph(f, n, xmin, xmax, resolution=1001):
    xlist = np.linspace(xmin, xmax, n)
    ylist = f(xlist)
    xlist_fine = np.linspace(xmin, xmax, resolution)
    ylist_fine = p_L(xlist_fine, xlist, ylist)
    plt.plot(xlist, ylist, 'ro')
    plt.plot(xlist_fine, ylist_fine)

def annotate_graph():
    ax = plt.gca()
    ax.set_xlabel('x')
    ax.set_ylabel('f(x)')
    ax.legend(['f(x)', 'Interpolated'])

if __name__ == '__main__':
    xlist = np.linspace(0, np.pi, 5)
    ylist = np.sin(xlist)
    verbose_verify(xlist, ylist)
    graph(np.sin, 5, 0, np.pi)
    annotate_graph()
    plt.show()

"""
Sample run:
python Lagrange_poly2.py
         x      Exact     Approx   Difference
  0.000000   0.000000   0.000000     0.000000
  0.785398   0.707107   0.707107     0.000000
  1.570796   1.000000   1.000000     0.000000
  2.356194   0.707107   0.707107     0.000000
  3.141593   0.000000   0.000000     0.000000
"""


import numpy as np
from scipy.stats import kstest, norm

class LinearRegression(object):
    '''Ordinary Least Squares...'''
    def __init__(self, X, y, fit_intercept=True):
        self.X = X
        self.y = y
        self.fit_intercept = fit_intercept
        self._coeff = None

    def fit(self, check_residuals=True, threshold=0.05):
        if self.fit_intercept:
            self.X = self._add_intercept(self.X)
        self._solve_ols()

        if check_residuals:
            print 'checking residuals...'
            if self._check_residuals(threshold):
                print '...residuals are gaussian distributed at %3.2f...' % threshold
            else:
                print '...residuals are Not gaussian distributed...'

    def _add_intercept(self, X):
        '''add a column of 1s in the X matrix...'''
        return np.insert(X, 0, np.ones_like(X[:, 0]), axis=1)

    def _solve_ols(self):
        '''matrix solution for OLS...'''
        XT = self.X.transpose()
        XTX = np.dot(XT, self.X)
        XTX_i = np.linalg.inv(XTX)
        self._coeff = np.dot(np.dot(XTX_i, XT), self.y)

    def _calculate_residuals(self):
        return self.y - np.dot(self.X, self._coeff)

    def _check_residuals(self, threshold=0.05):
        '''check residuals using ks_test for normality...'''
        residuals = self._calculate_residuals()
        mu, std = np.mean(residuals), np.std(residuals)

        def g_cdf(x):
            return norm.cdf(x, mu, std)

        # standard 2-sided ks test...
        t_stat, p_value = kstest(residuals, g_cdf)
        # returns True for gaussian noise
        return p_value > threshold

    def calc_r_squared(self, adjusted=True):
        '''returns the standard R2 value...'''
        n_obs, n_var = self.X.shape
        y_ = np.mean(self.y)
        p = np.dot(self.X, self._coeff)

        ss_t = np.sum(np.square(self.y - y_))
        ss_e = np.sum(np.square(self.y - p))

        r2 = 1.0 - ss_e/ss_t
        if adjusted:
            return 1.0 - (1.0 - r2) * ((n_obs - 1) / (n_obs - n_var))
        return r2


import numpy as np

def l2(W, alpha=1.0):
    '''
    calculate the l2 penalty (used in ridge regression) for the input weights W

    this is for educational purposes to compare l1 and l2 penalty on different weights

    we expect to see l2 to produce a lower penalty when all values are minimized

    the l2 penalty formula is:

    alpha * sum(W**2)
    '''
    return alpha * (W**2).sum()

def l1(W, alpha=1.0):
    '''
    calculate the l1 penalty (used in lasso regression) for the input weights W

    this is for educational purposes to compare l1 and l2 penalty on different weights

    we expect to see l1 to produce a lower penalty when some values are zero

    the l1 penalty formula is:

    alpha * sum(abs(W))
    '''
    return alpha * abs(W).sum()