# Chapter 7 Combining Different Models for Ensemble Learning
# Learning with ensembles
from scipy.misc import comb
import math
def ensemble_error(n_classifier, error):
    k_start = math.ceil(n_classifier / 2.0)
    probs = [comb(n_classifier, k) * error**k * (1- error) ** (n_classifier - k)
             for k in range(int(k_start), n_classifier + 1)]
    return sum(probs)

ensemble_error(n_classifier=11, error=0.25)
'''
After we've implemented the ensemble_error function, we can compute
the ensemble error rates for a range of different base errors from 0.0 to 1.0
to visualize the relationship between ensemble and base errors in a line graph:
'''
import numpy as np
error_range = np.arange(0.0, 1.01, 0.01)
ens_errors = [ensemble_error(n_classifier=11, error=error) for error in error_range]

import matplotlib.pyplot as plt
plt.plot(error_range, ens_errors, label='Ensemble error', linewidth=2)
plt.plot(error_range, error_range, linestyle='--', label='Base error', linewidth=2)
plt.xlabel('Base error')
plt.ylabel('Base/Ensemble error')
plt.legend(loc='upper left')
plt.grid()
plt.show()

'''
As we can see in the resulting plot, the error probability of an ensemble is always
better than the error of an individual base classifier as long as the base classifiers
perform better than random guessing ( ε < 0.5 ). Note that the y-axis depicts the
base error (dotted line) as well as the ensemble error (continuous line):
'''

# Implementing a simple majority vote classifier
'''
To translate the concept of the weighted majority vote into Python code, we can use
NumPy's convenient argmax and bincount functions:'''
import numpy as np
np.argmax(np.bincount([0, 0, 1], weights=[0.2, 0.2, 0.6]))

'''
To implement the weighted majority vote based on class probabilities, we can again
make use of NumPy using numpy.average and np.argmax'''
ex = np.array([[0.9, 0.1], [0.8, 0.2], [0.4, 0.6]])
p = np.average(ex, axis=0, weights=[0.2, 0.2, 0.6])
p
np.argmax(p)

'''Putting everythin together, let's now implement a MajorityVoteClassifier in Python'''
from sklearn.base import BaseEstimator
from sklearn.base import ClassifierMixin
from sklearn.preprocessing import LabelEncoder
from sklearn.externals import six
from sklearn.base import clone
from sklearn.pipeline import _name_estimators
import numpy as np
import operator

class MajorityVoteClassifier(BaseEstimator, ClassifierMixin):
    '''A majority vote ensemble classifier
    Parameters
    ----------
    classifiers : array-like, shape = [n_classifiers]
      Different classifiers for the ensemble
    vote : str, {'classlabel', 'probability'}
      Default: 'classlabel'
      If 'classlabel' the prediction is based on
      the argmax of class labels. Else if
      'probability', the argmax of the sum of
      probabilities is used to predict the class label
      (recommended for calibrated classifiers).
    weights : array-like, shape = [n_classifiers]
      Optional, default: None
      If a list of `int` or `float` values are
      provided, the classifiers are weighted by
      importance; Uses uniform weights if `weights=None`. '''
    def __init__(self, classifiers, vote='classlabel', weights=None):

        self.classifiers = classifiers
        self.named_classifiers = {key: value for key, value in _name_estimators(classifiers)}

        self.vote = vote
        self.weights = weights

    def fit(self, X, y):
        '''Fit classifiers.

        Parameters
        ----------
        X : {array-like, sparse matrix},
            shape = [n_samples, n_features]
            Matrix of training samples.
        y : array-like, shape = [n_samples]
            Vector of target class labels.
        Returns
        -------
        self : object'''
        # Use LabelEncoder to ensure class labels start
        # with 0, which is important for np.argmax
        # call in self.predict
        self.lablenc_ = LabelEncoder()
        self.lablenc_.fit(y)
        self.classes_ = self.lablenc_.classes_
        self.classifiers_ = [ ]
        for clf in self.classifiers:
            fitted_clf = clone(clf).fit(X, self.lablenc_.transform(y))
            self.classifiers_.append(fitted_clf)
        return self

    def predict(self, X):
        '''Predict class labels for X.
        Parameters
       ----------
       X : {array-like, sparse matrix},
           Shape = [n_samples, n_features]
           Matrix of training samples.
       Returns
       ----------
       maj_vote : array-like, shape = [n_samples]
           Predicted class labels.'''
        if self.vote == 'probability':
            maj_vote = np.argmax(self.predict_proba(X), axis=1)

        else: # 'classlabel' vote
            # Collect results from clf.predict calls
            predictions = np.asarray([clf.predict(X) for clf in self.classifiers_]).T

            maj_vote = np.apply_along_axis(lambda x: np.argmax(np.bincount(x, weights=self.weights)), axis=1, arr=predictions)
        maj_vote = self.lablenc_.inverse_transform(maj_vote)
        return maj_vote

    def predict_proba(self, X):
        '''Predict class probabilities for X.
        Parameters
        ----------
        X : {array-like, sparse matrix},
            shape = [n_samples, n_features]
            Training vectors, where n_samples is
            the number of samples and
            n_features is the number of features.
        Returns
        ----------
        avg_proba : array-like,
            shape = [n_samples, n_classes]
            Weighted average probability for
            each class per sample.'''
        probas = np.asarray([clf.predict_proba(X) for clf in self.classifiers_])
        avg_proba = np.average(probas, axis=0, weights=self.weights)
        return avg_proba

    def get_params(self, deep=True):
        '''Get classifier parameter names for GridSearch'''
        if not deep:
            return super(MajorityVoteClassifier, self).get_params(deep=False)
        else:
            out = self.named_classifiers.copy()
            for name, step in\
                six.iteritems(self.named_classifiers):
                for key, value in six.iteritems(step.get_params(deep=True)):
                    out['%s__%s' % (name, key)] = value
            return out
        '''
        Also, note that we defined our own modified version of the get_params methods
        to use the _name_estimators function in order to access the parameters of individual
        classifiers in the ensemble. This may look a little bit complicated at first, but it will
        make perfect sense when we use grid search for hyperparameter-tuning in later sections.'''

# Combining different algorithms for classification with majority vote

'''We will take a shortcut and load the Iris dataset from scikit-learn's dataset module.
Furthermore, we will only select two features, sepal width and petal length, to make
the classification task more challenging. Although our MajorityVoteClassifier generalizes
to multiclass problems, we will only classify flower samples from the two classes,
Iris-Versicolor and Iris-Virginica, to compute the ROC AUC. The code is as follows:'''

from sklearn import datasets
from sklearn.cross_validation import train_test_split
from sklearn.preprocessing import StandardScaler
from sklearn.preprocessing import LabelEncoder
iris = datasets.load_iris()
X, y = iris.data[50:, [1, 2]], iris.target[50:]
le = LabelEncoder()
y = le.fit_transform(y)
'''
Next we split the Iris samples into 50 percent training and 50 percent test data:'''
X_train, X_test, y_train, y_test = \
train_test_split(X, y, test_size=0.5, random_state=1)
'''
Using the training dataset, we now will train three different classifiers—a
logistic regression classifier, a decision tree classifier, and a k-nearest neighbors
classifier—and look at their individual performances via a 10-fold cross-validation
on the training dataset before we combine them into an ensemble classifier:'''
from sklearn.cross_validation import cross_val_score
from sklearn.linear_model import LogisticRegression
from sklearn.tree import DecisionTreeClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.pipeline import Pipeline
import numpy as np
clf1 = LogisticRegression(penalty='l2', C=0.001, random_state=0)
clf2 = DecisionTreeClassifier(max_depth=1, criterion='entropy', random_state=0)
clf3 = KNeighborsClassifier(n_neighbors=1, p=2, metric='minkowski')

pipe1 = Pipeline([['sc', StandardScaler()], ['clf', clf1]])
pipe3 = Pipeline([['sc', StandardScaler()], ['clf', clf3]])
clf_labels = ['Logistic Regression', 'Decision Tree', 'KNN']
print '10- fold cross validation:\n'

for clf, label in zip([pipe1, clf2, pipe3], clf_labels):
    scores = cross_val_score(estimator=clf, X=X_train, y=y_train, cv=10, scoring='roc_auc')
    print 'ROC AUC: %0.2f (+/- %0.2f) [%s]' % (scores.mean(), scores.std(), label)

'''
Now let's move on to the more exciting part and combine the individual classifiers
for majority rule voting in our MajorityVoteClassifier:'''
mv_clf = MajorityVoteClassifier(classifiers=[pipe1, clf2, pipe3])
clf_labels += ['Majority Voting']
all_clf = [pipe1, clf2, pipe3, mv_clf]
for clf, label in zip(all_clf, clf_labels):
    scores = cross_val_score(estimator=clf, X=X_train, y=y_train, cv=10, scoring='roc_auc')
    print 'Accuracy: %.2f (+/- %.2f) [%s]' % (scores.mean(), scores.std(), label)
'''
As we can see, the performance of the MajorityVotingClassifier has substantially improved
over the individual classifiers in the 10-fold cross-validation evaluation.'''

# Evaluating and tuning the ensemble classifier
'''
the MajorityVoteClassifier generalizes well to unseen data. We should remember that
the test set is not to be used for model selection; its only purpose is to report an unbiased
estimate of the generalization performance of a classifier system. The code is as follows:'''
from sklearn.metrics import roc_curve
from sklearn.metrics import auc
import matplotlib.pyplot as plt
colors = ['black', 'orange', 'blue', 'green']
linestyles = [':', '--', '-.', '-']
for clf, label, clr, ls \
    in zip(all_clf, clf_labels, colors, linestyles):
    # assuming the label of the positive class is 1
    y_pred = clf.fit(X_train, y_train).predict_proba(X_test)[:, 1]
    fpr, tpr, thresholds = roc_curve(y_true=y_test, y_score=y_pred)
    roc_auc = auc(x=fpr, y=tpr)
    plt.plot(fpr, tpr, color=clr, linestyle=ls, label='%s (auc = %0.2f)' % (label, roc_auc))

plt.legend(loc='lower right')
plt.plot([0, 1], [0, 1], linestyle='--', color='gray', linewidth=2)
plt.xlim([-0.1, 1.1])
plt.ylim([-0.1, 1.1])
plt.grid()
plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate')
plt.show()
'''
As we can see in the resulting ROC, the ensemble classifier also performs well on the
test set (ROC AUC = 0.95), whereas the k-nearest neighbors classifier seems to be
overfitting the training data (training ROC AUC = 0.93, test ROC AUC = 0.86):'''
'''
Since we only selected two features for the classification examples, it would be interesting
to see what the decision region of the ensemble classifier actually looks like. Although it is
not necessary to standardize the training features prior to model fitting because our logistic
regression and k-nearest neighbors pipelines will automatically take care of this, we will
standardize the training set so that the decision regions of the decision tree will be on the
same scale for visual purposes. The code is as follows:'''
sc = StandardScaler()
X_train_std = sc.fit_transform(X_train)
from itertools import product
x_min = X_train_std[:, 0].min() - 1
x_max = X_train_std[:, 0].max() + 1
y_min = X_train_std[:, 1].min() - 1
y_max = X_train_std[:, 1].max() + 1

xx, yy = np.meshgrid(np.arange(x_min, x_max, 0.1),
                      np.arange(y_min, y_max, 0.1))
f, axarr = plt.subplots(nrows=2, ncols=2,
                         sharex='col',
                         sharey='row',
                         figsize=(7, 5))

for idx, clf, tt in zip(product([0, 1], [0, 1]), all_clf, clf_labels):
    clf.fit(X_train_std, y_train)
    Z = clf.predict(np.c_[xx.ravel(), yy.ravel()])
    Z = Z.reshape(xx.shape)
    axarr[idx[0], idx[1]].contour(xx, yy, Z, alpha=0.3)
    axarr[idx[0], idx[1]].scatter(X_train_std[y_train==0, 0],
                                                  X_train_std[y_train==0, 1],
                                                  c='red',
                                                  marker='o',
                                                  s=50)
    axarr[idx[0], idx[1]].scatter(X_train_std[y_train == 1, 0],
                                                  X_train_std[y_train == 1, 1],
                                                  c='red',
                                                  marker='o',
                                                  s=50)
    axarr[idx[0], idx[1]].set_title(tt)

plt.text(-3.5, -4.5, s='Sepal width [standardized]', ha='center', va='center', fontsize=12)
plt.text(-10.5, 4.5,
          s='Petal length [standardized]',
          ha='center', va='center',
          fontsize=12, rotation=90)
plt.show()

'''
Interestingly but also as expected, the decision regions of the ensemble classifier seem
to be a hybrid of the decision regions from the individual classifiers. At first glance, the
majority vote decision boundary looks a lot like the decision boundary of the k-nearest
neighbor classifier. However, we can see that it is orthogonal to the y axis for sepal width
 ≥1, just like the decision tree stump:'''
'''
Before you learn how to tune the individual classifier parameters for ensemble classification,
let's call the get_params method to get a basic idea of how we can access the individual
parameters inside a GridSearch object:'''
mv_clf.get_params()

'''
Based on the values returned by the get_params method, we now know how to access the
individual classifier's attributes. Let's now tune the inverse regularization parameter C of the
logistic regression classifier and the decision tree depth via a grid search for demonstration
purposes. The code is as follows:
'''
from sklearn.grid_search import GridSearchCV
params = {'decisiontreeclassifier__max_depth': [1, 2], 'pipeline-1__clf__C': [0.001, 0.1, 100.0]}
grid = GridSearchCV(estimator=mv_clf, param_grid=params, cv=10, scoring='roc_auc')
grid.fit(X_train, y_train)

'''

After the grid search has completed, we can print the different hyperparameter value combinations
and the average ROC AUC scores computed via 10-fold cross-validation. The code is as follows:'''
for params, mean_score, scores in grid.grid_scores_:
    print("%0.3f+/-%0.2f %r"
        % (mean_score, scores.std() / 2, params))

print 'Best parameters: %s' % grid.best_params_

print 'Accuracy: %.2f' % grid.best_score_

'''
As we can see, we get the best cross-validation results when we choose a lower regularization strength
(C = 100.0) whereas the tree depth does not seem to affect the performance at all, suggesting that a decision
stump is sufficient to separate the data. To remind ourselves that it is a bad practice to use the test dataset
more than once for model evaluation, we are not going to estimate the generalization performance of the tuned
hyperparameters in this section. We will move on swiftly to an alternative approach for ensemble learning: bagging.'''

self_practice/Chapter 7 Combining Different Models for Ensemble Learning part I.py


import numpy as np

def _average_path_length(n):
    """
    Average path length formula.
    """
    # Thank you Matwey.
    return 2.0 * (np.log(n) + np.euler_gamma + 1 / (2 * n) - 1 / (12 * n ** 2) - 1.0)

def average_path_length(n):
    """
    Average path length computation.

    Parameters
    ----------
    n
        Either array of tree depths to computer average path length of
        or one tree depth scalar.

    Returns
    -------
    Average path length.
    """
    if np.isscalar(n):
        if n <= 1:
            apl = 0
        else:
            apl = _average_path_length(n)
    else:
        n = np.asarray(n)
        apl = np.zeros_like(n)
        apl[n > 1] = _average_path_length(n[n > 1])

    return apl


from scipy.integrate import quad
from scipy.optimize import root_scalar
import numpy as np

def get_mp_pdf(beta):
    """
    Gets the MP PDF.

    Parameters
    ----------
    beta: float
        TODO: document

    Output
    ------
    pdf, beta_minus, beta_plus

    pdf: callable
        The pdf function.

    beta_minus, beta_plus: float
        The lower and upper bound of the support.
    """

    beta_minus = (1 - np.sqrt(beta)) ** 2
    beta_plus = (1 + np.sqrt(beta)) ** 2

    def pdf(x):
        # assert (beta_minus <= t) and (t <= beta_plus)
        return np.sqrt((beta_plus - x) * (x - beta_minus)) / (2 * np.pi * x * beta)

    return pdf, beta_minus, beta_plus

def get_mp_cdf(beta):
    """
    Gets the MP CDF

    Parameters
    ----------
    beta: float
        TODO

    Output
    ------
    cdf, beta_minus, beta_plus

    cdf: callable
        The cdf function.

    beta_minus, beta_plus: float
        The lower and upper bound of the support.
    """
    pdf, beta_minus, beta_plus = get_mp_pdf(beta)

    def cdf(x):
        return quad(func=pdf, a=beta_minus, b=x)[0]

    return cdf, beta_minus, beta_plus

def compute_mp_quantile(beta, q=0.5):
    """
    Computes the qualtile for the MP distribution.

    Parameters
    ----------
    beta: float
        TODO

    q: float
        The desired quantile.

    """
    assert 0 < q and q < 1
    cdf, beta_minus, beta_plus = get_mp_cdf(beta)

    def root_func(x):
        return cdf(x) - q

    return root_scalar(f=root_func, method=None,
                       bracket=(beta_minus, beta_plus)).root


import sympy

# Used this to search for all sequences below 1e9.
# There are 8 sequences of length 13 or more with differences of
# 8 or less between successive primes.  The first prime in each
# sequence along with the sequence lengths are given here:
#
# In [616]: find_sequence(1,1_000_000_000,8,13)
#
#    First prime   Length
#    --------------------
#    2             30
#    2657          14
#    14713         13
#    86966771      13
#    172039573     13
#    296931731     13
#    369008657     13
#    875753231     13
#
# Some more details
#
#    FirstPrime  SeqLength  LastPrime  DeltaPrime  MeanDiff
# 0           2         30        113         111  3.827586
# 1        2657         14       2719          62  4.769231
# 2       14713         13      14783          70  5.833333
# 3    86966771         13   86966833          62  5.166667
# 4   172039573         13  172039633          60  5.000000
# 5   296931731         13  296931797          66  5.500000
# 6   369008657         13  369008729          72  6.000000
# 7   875753231         13  875753299          68  5.666667
#
#
# The runtime is substantial (about an hour) with the first
# three found almost instantly and the rest taking proportionally
# longer.
#
# FYI: the last value is sympy.prime(44833217) where sympy.prime(1)=2
#

def find_sequence(nstart=2,nstop=1000,maxdiff=8,minlength=13):
    """
    Search for sequences of prime numbers that meet the following
    criteria:

    difference between successive primes <= maxdiff
    length of sequence >= minlength

    Output the first prime in each sequence and the sequence length
    """
    zzz=[]
    lastprime=-1000
    firstprime=0
    seqlen=1
    for i in sympy.primerange(nstart,nstop):
        if (i-lastprime)<=maxdiff:
            seqlen=seqlen+1
        else:
            # Found the end of a sequence
            # If the last sequence was long enough then output the specs
            if seqlen>=minlength:
                print(firstprime,seqlen)
                zzz.append([firstprime,seqlen])
            seqlen=1
            firstprime=i
        lastprime=i
    # Process any sequence that is in progress when the loop ends
    if seqlen>=minlength:
        print(firstprime,seqlen)
        zzz.append([firstprime,seqlen])
    return(zzz)


import numpy as np
import scipy as sp
from scipy.integrate import odeint
import matplotlib.pyplot as plt

class Hodgkin_Huxley:

    def __init__(self):
        # Membrane capacitance and potential
        self.C_m = 1.0
        self.V_m = -70.0

        # Max sodium conductance and membrane potential
        self.g_Na = 120.0
        self.V_Na = 56.0

        # Max potassium conductance and membrane potential
        self.g_K = 36.0
        self.V_K = -77.0

        # Max leak conductance and membrane potential
        self.g_l = 0.3
        self.V_l = -60

        # Time to integrate over
        self.time = np.arange(0.0, 1000.0, 0.1)

        # ion variables
        self.m = 0.05
        self.h = 0.6
        self.n = 0.32

    def I_inj(self, t):
        return np.piecewise(t, [(t < 500) * (t >= 200), (t < 950) * (t >= 700)], [10, 50])
        #return np.piecewise(self.time, [(self.time< 500)*(self.time>=200), (self.time< 950)*(self.time>=700)], [10,50])

    # potassium rate functions
    def a_n(self):
        return 0.01 * (self.V_m + 55) / (1 - np.exp(-(self.V_m + 55.0) / 10)) # + 1.E-7)

    def B_n(self):
        return 0.125 * np.exp(-(self.V_m + 65.0) / 80)

    # sodium rate functions
    def a_m(self):
        return 0.1 * (40 + self.V_m) / (1.0 - np.exp(-(self.V_m + 40.0) / 10.0)) # + 1.E-7)

    def a_h(self):
        return 0.07 * np.exp(-(self.V_m + 65.0) / 20.0)

    def B_m(self):
        return 4 * np.exp(-(self.V_m + 65.0) / 18.0)

    def B_h(self):
        return 1 / (np.exp(-(35 + self.V_m) / 10) + 1)

    # current functions
    def K_current(self):
        return self.g_K * self.n**4 * (self.V_m - self.V_K)

    def Na_current(self):
        return self.g_Na * self.m**3 * self.h * (self.V_m - self.V_Na)

    def leak_current(self):
        return self.g_l * (self.V_m - self.V_l)

    def hh_model(self, y, t, parameters):
        '''Takes in y dynamics variables, V_m, n, m and h. Array of time steps to integrate over.
        List of constant parameters, reversal potentials and conductances

        Returns values for derivatives w.r.t time for Voltage and ion rate coefficience n, m and h.
        '''
        self.V_m, self.n, self.m, self.h = y
        self.g_K, self.V_K, self.g_Na, self.V_Na, self.g_l, self.C_m = parameters

        # Total current through the membrane
        dVdt = self.I_inj(t) - self.K_current() - self.Na_current() - self.leak_current() / self.C_m

        # Derivative of n, potassium channel activation, w.r.t. time
        dndt = self.a_n() * (1 - self.n) - self.B_n() * self.n

        # Derivative of m, sodium channel activion, w.r.t. time
        dmdt = self.a_m() * (1 - self.m) - self.B_m() * self.m

        # Derivative of h, sodium channel in-activion, w.r.t. time

        dhdt = self.a_h() * (1 - self.h) - self.B_h() * self.h

        return [dVdt, dndt, dmdt, dhdt]

    def simulate(self):

        y0 = [self.V_m, self.n, self.m, self.h]
        parameters = [self.g_K, self.V_K, self.g_Na, self.V_Na, self.g_l, self.C_m]

        sol = odeint(self.hh_model, y0, self.time, args=(parameters,))
        return sol

model = Hodgkin_Huxley()
model.simulate()
#print(model.sol)


import numpy as np

def wmean(x, weights):
    """Returns Weighted Mean. Ignores NaNs and handles infinite weights.

    Parameters
    ----------
    x: array-like [n_samples]
        An array-like object that contains the data.

    weights: array-like [n_samples]
        An array-like object that contains the corresponding weights.

    Returns
    ----------------------------------
    m: number
        The weighted mean.
    """

    # Flatten x and weights
    x = x.flatten()
    weights = weights.flatten()

    # Find NaNs
    nans = np.isnan(x)
    infs = np.isinf(weights)

    # If all x are nans, return np.nan
    if nans.all() == True:
        m = np.nan
        return m

    # If there are infinite weights, use the corresponding x
    if infs.any() == True:
        m = np.nanmean(x[infs])
        return m

    # Set NaNs to zero
    x[nans] = 0
    weights[nans] = 0

    # Normalize the weights + calculate Weighted Mean
    weights = weights / np.sum(weights)
    m = np.matmul(weights, x)
    return m


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Wed Jan 12 22:16:41 2022

@author: m102324
"""
import math
import logging
import numpy as np
from cobindability import version

__author__ = "Liguo Wang"
__copyright__ = "Copyleft"
__credits__ = []
__license__ = "MIT"
__version__ = version.version
__maintainer__ = "Liguo Wang"
__email__ = "wang.liguo@mayo.edu"
__status__ = "Development"

def ov_coef(x, y, xy, g):
	"""
	Calculate the overlap coefficient.

	Parameters
	----------
	x : int
		The cardinality of A.
	y : int
		The cardinality of B.
	xy : int
		The cardinality of A AND B.
	g : int
		The cardinality of background.

	Returns
	-------
	float
		Overlap coefficient.
	"""
	if xy > x or xy > y:
		logging.error("Invalid parameters.")
	if min(x,y,xy) < 0:
		logging.error("Invalid parameters.")
	if x == 0 or y == 0 or xy == 0:
		return 0
	else:
		return xy/(math.sqrt(x * y))

def ov_jaccard(x, y, xy, g):
	"""
	Calculate the Jaccard's coefficient (https://en.wikipedia.org/wiki/Jaccard_index).

	Parameters
	----------
	x : int
		The cardinality of A.
	y : int
		The cardinality of B.
	xy : int
		The cardinality of A AND B.
	g : int
		The cardinality of background.

	Returns
	-------
	float
		Overlap coefficient.
	"""
	if xy > x or xy > y:
		logging.error("Invalid parameters.")
	if min(x,y,xy) < 0:
		logging.error("Invalid parameters.")
	if x == 0 or y == 0:
		return 0
	else:
		return xy/(x + y - xy)

def ov_ss(x, y, xy, g):
	"""
	Calculate the Szymkiewicz–Simpson coefficient, (https://en.wikipedia.org/wiki/Overlap_coefficient).

	Parameters
	----------
	x : int
		The cardinality of A.
	y : int
		The cardinality of B.
	xy : int
		The cardinality of A AND B.
	g : int
		The cardinality of background.

	Returns
	-------
	float
		Szymkiewicz–Simpson coefficient.
	"""
	if xy > x or xy > y:
		logging.error("Invalid parameters.")
	if min(x,y,xy) < 0:
		logging.error("Invalid parameters.")
	if x == 0 or y == 0:
		return 0
	else:
		return xy/min(x,y)

def ov_sd(x, y, xy, g):
	"""
	Calculate the Sørensen–Dice coefficient (https://en.wikipedia.org/wiki/Sørensen–Dice_coefficient).
	Sørensen–Dice coefficient also called "Sørensen–Dice index", "Sørensen index" or "Dice's coefficient".

	Parameters
	----------
	x : int
		The cardinality of A.
	y : int
		The cardinality of B.
	xy : int
		The cardinality of A AND B.
	g : int
		The cardinality of background.

	Returns
	-------
	float
		Overlap coefficient.
	"""
	if xy > x or xy > y:
		logging.error("Invalid parameters.")
	if min(x,y,xy) < 0:
		logging.error("Invalid parameters.")
	if x == 0 or y == 0:
		return 0
	else:
		return 2*xy/(x + y)

def pmi_value(x, y, xy, g):
	"""
	Calculate the pointwise mutual information (PMI).

	Parameters
	----------
	x : int
		The cardinality of A.
	y : int
		The cardinality of B.
	xy : int
		The cardinality of A AND B.
	g : int
		The cardinality of background.

	Returns
	-------
	float
		Pointwise mututal information.
	"""
	if g <= 0:
		logging.error("The cardinality of background must be a postive integer.")
	if xy > x or xy > y:
		logging.error("Invalid parameters.")
	if x > g or y > g:
		logging.error("Invalid parameters.")
	if min(x,y,xy) < 0:
		logging.error("Invalid parameters.")
	px = x/g
	py = y/g
	pxy = xy/g
	if pxy == 0:
		return -np.inf
	else:
		return np.log(pxy) - np.log(px) - np.log(py)

def npmi_value(x, y, xy, g):
	"""
	Calculate the normalized pointwise mutual information (NPMI).

	Parameters
	----------
	x : int
		The cardinality of A.
	y : int
		The cardinality of B.
	xy : int
		The cardinality of A AND B.
	g : int
		The cardinality of background.

	Returns
	-------
	float
		Normalized pointwise mutual information.
	"""
	if g <= 0:
		logging.error("The cardinality of background must be a postive integer.")
	if xy > x or xy > y:
		logging.error("Invalid parameters.")
	if x > g or y > g:
		logging.error("Invalid parameters.")
	if min(x,y,xy) < 0:
		logging.error("Invalid parameters.")
	px = x/g
	py = y/g
	pxy = xy/g
	if pxy == 0:
		return -1
	else:
		return np.log(px*py)/np.log(pxy) - 1


import math
import numpy as np

def interpolateDirections(dir1_xyz, dir2_xyz, N):
#
# interpolate two direction vectors by performing spherical (rotational)
# linear interpolation, generating direction vectors in between the two,
# across the great circle passing from them. The angle between the two is
# divided in N intervals, hence N+1 vectors are returned, including the two
# original ones.
#
# dir1_xyz: 	numpy array, star vector
# dir2_xyz: 	numpy array, end vector
# N: 		integer, number of intervals to interpolate

	angle12 = math.acos(dir1_xyz.dot(dir2_xyz))
	cross12 = np.cross(dir1_xyz, dir2_xyz)
	purpvec12 = cross12/math.sqrt(cross12.dot(cross12))

	dtheta = angle12/N
	psi = np.linspace(0,angle12,N+1)
	interp_dirs_xyz = []
	for x in psi:
		dirn_xyz = dir1_xyz * math.cos(x) + np.cross(purpvec12,dir1_xyz)*math.sin(x) + purpvec12.dot(np.cross(purpvec12,dir1_xyz))*(1-math.cos(x))
		interp_dirs_xyz.append(dirn_xyz)

        return np.array(interp_dirs_xyz)


from typing import Tuple, Union, Sequence, Optional
from collections.abc import Iterable

import numpy as np

from PDE.typing import FloatArray, VecApply

class Derivator:
    FDM_ORDER: set[int] = {2, 4}
    MAX_DERIVATIV_ORDER: int = 2
    MAX_SYSTEM_DIM: int = 2
    # (order, derivative) -> Coefs
    C_COEF: dict[tuple[int, int], list[float]] = {
        (2, 1): [-0.5, 0, 0.5],
        (2, 2): [1, 2, 1],
        (4, 1): [1/12, -2/3, 0, 2/3, -1/12],
        (4, 2): [-1/12, 4/3, -5/2, 4/3, -1/12]
    }
    A_COEF: dict[tuple[int, int], list[float]] = {
        (2, 1): [-1.5, 2, -0.5],
        (2, 2): [2, -5, 4, -1],
        (4, 1): [-25/12, 4, -3, 4/3, -1/4],
        (4, 2): [15/14, -77/6, 107/6, -13, 61/12]
    }

    def __init__(
        self,
        system_shape: Tuple[int],
        derive_order: int,
        FDM_order: int,
        space_interval: Union[Sequence[float], float],
        boundary_condition: Optional[VecApply]
        ):
        if derive_order > type(self).MAX_DERIVATIV_ORDER:
            raise ValueError(f"Derive order should not exces {MAX_DERIVATIV_ORDER}")
        if FDM_order not in type(self).FDM_ORDER:
            raise ValueError(f"Derivation methode order not supported")
        self._dim = system_shape
        self.FDM_order = FDM_order
        if isinstance(space_interval, Iterable):
            self._inverth = [1 / h for h in list(space_interval)]
        else:
            self._inverth = [1 / space_interval] * len(self._dim)
        self._deriv = {i: list() for i in range(1, derive_order + 1)}
        self._bound = boundary_condition
        for key in self._deriv:
            for i in range(1, len(self._dim) + 1):
                self._deriv[key].append(self.create_derive(key, i))

    def create_derive(self, dorder: int, axis: int) -> VecApply:
        c = np.array(type(self).C_COEF[(self.FDM_order, dorder)])
        a_f = np.array(type(self).A_COEF[(self.FDM_order, dorder)])
        a_b = -a_f[::-1]
        clim = len(c) // 2
        alim = len(a_f)
        doc= f"""
        Centered finite differences methode
        {dorder} order derivative vectorized over {axis + 1}th dimension

        centered coeficient: {c}
        forward coeficient: {a_f}
        backward coreficient: -forward_coeficient\
        """
        if len(self._dim) == 1:
            M = np.zeros((self._dim[0], self._dim[0]))
            for row in range(len(M)):
                if row < clim:
                    M[row, row:(alim + row)] =a_f
                elif row >= self._dim[0] - clim:
                    M[row, (row - alim + 1):(row + 1)] = a_b
                else:
                    M[row, (row - clim):(row + clim + 1)] = c
            M = M * self._inverth[axis - 1] ** dorder
            def derive(u: FloatArray) -> FloatArray:
                bounded = self._bound(u)
                return M.dot(bounded)
            derive.__doc__ = doc
            return derive

    def D(self, order: int, axis: int, u: FloatArray) -> FloatArray:
        return self._deriv[order][axis](u)


# Chapter 1: AutoRegressive Integrated Moving Average (ARIMA)

Prepared by: Benjur Emmanuel L. Borja and Maria Eloisa M. Ventura

In this notebook, we will introduce our first approach to time-series forecasting which is **ARIMA** or AutoRegressive Integrated Moving Average. This notebook will discuss:
1. Definition and Formulation of ARIMA models
2. Model Parameters (p, d, and q) and Special Cases of ARIMA models
3. Model Statistics and How to Interpret
4. Implementation and Forecasting using ARIMA

#Import required Libraries
import numpy as np, pandas as pd
from statsmodels.graphics.tsaplots import plot_acf, plot_pacf
import matplotlib.pyplot as plt
from statsmodels.tsa.stattools import adfuller
from numpy import log
from statsmodels.tsa.arima_model import ARIMA
from sklearn.metrics import mean_squared_error, mean_absolute_error
from math import sqrt
from pandas import read_csv
import multiprocessing as mp

### Just to remove warnings to prettify the notebook.
import warnings
warnings.filterwarnings("ignore")

## Introduction to ARIMA

ARIMA, or AutoRegressive Integrated Moving Average, is a set of models that explains a time series using its own previous values given by the lags (**A**uto**R**egressive) and lagged errors (**M**oving **A**verage) while considering stationarity corrected by differencing (oppossite of **I**ntegration.) In other words, ARIMA assumes that the time series is described by autocorrelations in the data rather than trends and seasonality. In these context, we define trends and seasonality as the following:
- **Trend**: A time series has a trend if there is a overlying long term increase or decrease in the data, which is not necessarily linear.
- **Seasonality**: A time series data has seasonality when it is affected by seasonal factors such as the time of the year or the day of the week. The seasonality of data is apparent as there is a fixed frequency of pattern occurence.

### Example 1: Stationary and Non-Stationary Univartiate Time Series

To illustrate, lets look at the following examples of time series (__[source](https://otexts.com/fpp2/fpp_files/figure-html/stationary-1.png)__):

(a) Google stock price for 200 consecutive days

(b) Daily change in the Google stock price for 200 consecutive days

(c) Annual number of strikes in the US

(d) Monthly sales of new one-family houses sold in the US; (e) Annual price of a dozen eggs in the US (constant dollars)

(f) Monthly total of pigs slaughtered in Victoria, Australia

(g) Annual total of lynx trapped in the McKenzie River district of north-west Canada; (h) Monthly Australian beer production

(i) Monthly Australian electricity production.

![stationary-1.png](stationary-1.png)

**Which of these series are stationary?**

Obvious seasonality rules out series (d), (h) and (i). Trends and changing levels rules out series (a), (c), (e), (f) and (i). Increasing variance also rules out (i). That leaves only (b) and (g) as stationary series. At a first glance, (g) might seem non-stationary but these cycles are aperiodic, and is dependedent on the complex interactions between lynx population and the availability of the food source. The timing of these cycles is unpredictable hence, this time series is non-stationary.

Now notice the Google Stock Price (a) and the daily change in the Google Stock Price (b). These time-series came from the same system, but exhibits different dynamics. This illustrates that we can make non-stationary time-series stationary by taking the difference between consecutive observations - or **Differencing**. Differencing stabilizes our time-series by removing changes in the level of our time series, therefore reducing trends and seasonality.

### Model Components

As previously mentioned, ARIMA models are built given the following key aspects:

**AR**: Autoregression. A model that uses the dependent relationship between an observation and some number of lagged observations.
**I**: Integrated. The use of differencing of raw observations (e.g. subtracting an observation from an observation at the previous time step) in order to make the time series stationary.
**MA**: Moving Average. A model that uses the dependency between an observation and a residual error from a moving average model applied to lagged observations.

Each of these components are explicitly specified in the model as a parameter. A standard notation is used of ARIMA(p,d,q) where the parameters are substituted with integer values to quickly indicate the specific ARIMA model being used:
**p**: The number of lag observations included in the model, also called the lag order (*deals with **window** of $X_t$*)
**d**: The number of times that the raw observations are differenced, also called the degree of differencing (*deals with order of **differencing** of $X_t$*)
**q**: The size of the moving average window, also called the order of moving average (*deals with **residuals***)

Given this, the general case of ***ARIMA(p,d,q)*** can be written as:

\begin{equation}
X_{t} = \alpha _{1}X_{t-1} + \dots + \alpha _{p}X_{t-p} + \varepsilon _{t}+\theta_{1}\varepsilon _{t-1}+\cdots +\theta _{q}\varepsilon _{t-q}
\end{equation}

Or in words :

**Predicted $X_t$** = Constant + Linear combination of **Lags of $X$ (up to $p$ lags)** + Linear Combination of **Lagged forecast errors (up to q lags)**. Provided that the time-series is already **differenced (up to d terms)** to ensure stationarity.

## Model parameters *p, d, and q* and Special Cases

Before we discuss how we determine p, d, and q that are best to represent a time series, let's first take a look at special cases of ARIMA models that should help us illustrate the formulation of the ARIMA equation.

**Case 1: ARIMA(p,0,0) = autoregressive model**: if the series is stationary and autocorrelated, perhaps it can be predicted as a multiple of its own previous value, plus a constant.

The forecasting equation for ARIMA(1,0,0) is:

\begin{equation}
X_{t}  =  \mu  + \alpha _{1}X_{t-1}
\end{equation}

while ARIMA(2,0,0)

\begin{equation}
X_{t}  =  \mu  + \alpha _{1}X_{t-1} +  \alpha _{2}X_{t-2}
\end{equation}

or in general ARIMA(p,0,0)

\begin{equation}
X_{t}  =  \mu  + \alpha _{1}X_{t-1} +  \alpha _{2}X_{t-2} \dots + \alpha _{p}X_{t-p}
\end{equation}

**Case 2: ARIMA(0,0,q) = moving average model**: if the series is stationary but is correlated to the errors of previous values, we can regress using the past forecast errors.

The forecasting equation for this is ARIMA(0,0,1) given by:

\begin{equation}
X_{t} = \varepsilon _{t}+\theta_{1}\varepsilon _{t-1}
\end{equation}

or in simillar fashion to p, this can be generalized to ARIMA(0,0,q):

\begin{equation}
X_{t} = \varepsilon _{t}+\theta_{1}\varepsilon _{t-1}+\cdots +\theta _{q}\varepsilon _{t-q}
\end{equation}

where $\theta_{q}$ is the coefficient at time $t-q$ of the residual $\varepsilon _{t-q}$.

**Case 3: ARIMA(0,1,0) = Random Walk**: if the series is non-stationary then the simplest model that we can use is a random walk model, which is given by:

\begin{equation}
X_{t}  =  \mu  + X_{t-1}
\end{equation}

*This can then further be generalized to ARIMA(0,d,0) simillar to the first two cases.*

As shown in our cases, we can use value of 0 for a parameter, which indicates to not use that element of the model. This way, the ARIMA model can be configured to perform the function of an ARMA model, and even a simple AR, I, or MA model. **It is good to note that the case ARIMA(0,1,1) is a Simple Exponential Smoothing model** but we'll leave that in another discussion.

Implementing an ARIMA model for a time series assumes that the observations is an ARIMA process. To implement ARIMA, a linear regression model is constructed including the specified number and type of terms, and the data is prepared by a degree of differencing in order to make it stationary, i.e. to remove trend and seasonal structures that negatively affect the regression model.

### Determining *p*, *d*, and *q*

We need to explicitly indicate p, d, and q when implementing ARIMA models. Selecting appropriate p, d, and q can be difficult but there are several methods of automating this process. For example, we can use grid search in python to scan through different values and check which model would be optimal (will be discussed later.) R uses the auto.arima() function to do this automatically. For our case, we will look at the Augmented-Dickey Fuller Test (ADF), AutoCorrelation Function (ACF), and Partial Autocorrelation Function (PACF) to determine our model parameters.

#### Finding the order differencing *d*

As stated before, ARIMA models are assumed to be stationary. Implementing differencing may induce stationarity for various time series. The quickest way to determine *d* for our models is to difference and simply run ADF to check for stationarity. We can also look at the PACF and ACF to see if our time series is stationary after *d* differencing.

To illustrate, let's take a look at the following example:

# We're using a sample data from https://raw.githubusercontent.com/selva86/datasets/master/wwwusage.csv
# Import data
df = pd.read_csv('../data/wwwusage.csv', names=['value'], header=0)
plt.figure(figsize=(15, 2))
plt.plot(df)
plt.title('Original Series')
plt.xlabel('Time')
plt.ylabel('Value')
plt.show()

Initial eyeballing shows that there is a trend for this time series and is non-stationary. Checking using ADF:

result = adfuller(df.value.dropna())
print('ADF Statistic: %f' % result[0])
print('p-value: %f' % result[1])

The null hypothesis of the ADF test is that the time series is non-stationary. So, if the p-value of the test is less than the significance level (0.05) then you reject the null hypothesis and infer that the time series is indeed stationary. For our example, we fail to reject the null hypothesis.

Next we difference our time series and check the results of the ADF test. We will also look at the ACF.

plt.rcParams.update({'figure.figsize':(15,8), 'figure.dpi':120})

# Original Series
fig, axes = plt.subplots(3, 2, sharex=True)
axes[0, 0].plot(df.value); axes[0, 0].set_title('Original Series')
plot_acf(df.value, ax=axes[0, 1])

# 1st Differencing
axes[1, 0].plot(df.value.diff()); axes[1, 0].set_title('1st Order Differencing')
plot_acf(df.value.diff().dropna(), ax=axes[1, 1])

# 2nd Differencing
axes[2, 0].plot(df.value.diff().diff()); axes[2, 0].set_title('2nd Order Differencing')
plot_acf(df.value.diff().diff().dropna(), ax=axes[2, 1])

plt.show()

print('ADF Statistic for 1st Order Differencing')
result = adfuller(df.value.diff().dropna())
print('ADF Statistic: %f' % result[0])
print('p-value: %f' % result[1])
print('Critical Values:')
for key, value in result[4].items():
    print('\t%s: %.3f' % (key, value))

print('\n ADF Statistic for 2nd Order Differencing')
result = adfuller(df.value.diff().diff().dropna())
print('ADF Statistic: %f' % result[0])
print('p-value: %f' % result[1])
print('Critical Values:')
for key, value in result[4].items():
    print('\t%s: %.3f' % (key, value))

Given the results of our ACF and ADF, we can see that our time series reachees stationarity after two orders of differencing. However, the ACF of the 2nd order differencing goes into the negative zone fairly quick. This can indicates that the series might have been over differenced. It is now up to us if we want consider the first or second order differencing for our ARIMA models.

#### Finding the order of the AutoRegressive term *p*

As we have discussed previously, we can look at the PACF plot to determine the lag for our AR terms. Partial autocorrelation can be imagined as the correlation between the series and its lag, after excluding the contributions from the intermediate lags. So, PACF sort of conveys the pure correlation between a lag and the series. That way, we will know if that lag is needed in the AR term or not.

# PACF plot of 1st differenced series
plt.rcParams.update({'figure.figsize':(15,2.5), 'figure.dpi':120})

fig, axes = plt.subplots(1, 2, sharex=True)
axes[0].plot(df.value.diff()); axes[0].set_title('1st Differencing')
axes[1].set(ylim=(0,5))
plot_pacf(df.value.diff().dropna(), ax=axes[1])

plt.show()

Immediately, we can observe that our PACF returns sigificance at Lag 1 and Lag 2, meaning it crosses the significance limit (blue region). We can also observe significance at higher order terms but note that given the amount of lag that we are testing, it is statistically probable to see random spikes in our PACF and ACF plots. *Although this can also be attributed to Seasonality which will be tackled separately.*

With this, we can now decide to use $p = 2$ for our ARIMA model.

#### Finding the order of the Moving Average term *q*

Simillar to how we determined $p$, we will now look at the ACF to determine the $q$ terms to be considered for our MA. The ACF tells how many MA terms are required to remove any autocorrelation in the stationary series.

# ACF plot of 1st differenced series
plt.rcParams.update({'figure.figsize':(15,4), 'figure.dpi':120})

fig, axes = plt.subplots(1, 2, sharex=True)
axes[0].plot(df.value.diff()); axes[0].set_title('1st Differencing')
axes[1].set(ylim=(0,1))
plot_acf(df.value.diff().dropna(), ax=axes[1])

plt.show()

Our results for the ACF is not as apparent compared to our PCF. We can observed several ACF terms that is above our significance level. This may be  attritbuted to the fact that our model has a weak stationarity. This may also be caused by the fact that our time series is not perfectly MA and is an ARIMA model. For now, let's consider $q = 3$.

## Building the ARIMA model
Now that we’ve determined the values of p, d and q, we have everything needed to fit the ARIMA model. Let's implement using this dataset first before we move on to a deeper look at the implementation of ARIMA in the next section. Let’s use the ARIMA() implementation in statsmodels package. As computed, we will use ARIMA(2,1,3):

model = ARIMA(df.value, order=(2,1,3))
model_fit = model.fit(disp=0)
print(model_fit.summary())

There's quite a bit of information to unpack from the summary. Generally, we are interested in the Akaike’s Information Criterion (AIC), coefficients of our AR and MA terms (coef_), and the p-values of the terms (P>|z|). We need the p-values to be less than 0.05 to be significant, which means that our model failed to reach significance for the AR and MA terms. Let's try to be conservative and use small values for p and d, i.e. ARIMA(1,1,1), as given by the p-values of AR1 and MA1 in our results.

model = ARIMA(df.value, order=(1,1,1))
model_fit = model.fit(disp=0)
print(model_fit.summary())

Immediately, we can see that the p-values is now  <<0.05 for both the AR and MA terms and is highly significant. We will now check the residuals of our time-series to ensure that we will not see patterns and will have a constant mean and variance.

# Plot residual errors
residuals = pd.DataFrame(model_fit.resid)
fig, ax = plt.subplots(1,2, figsize=(15,2.5))
residuals.plot(title="Residuals", ax=ax[0])
residuals.plot(kind='kde', title='Density', ax=ax[1])
plt.show()

The residuals is a good final check for our ARIMA models. Ideally, the residual errors should be a Gaussian with a zero mean and uniform variance. With this, we can now proceed with fitting our initial time series with our model.

# Actual vs Fitted
fig, ax = plt.subplots(figsize=(15,2))
ax = df.plot(ax=ax)
fig = model_fit.plot_predict(85, 100, dynamic=False, ax=ax, plot_insample=False)
plt.show()

When we set dynamic=False the in-sample lagged values are used for prediction. That is, the model gets trained up until the previous value to make the next prediction. We can also call this as a **walk-forward** or **one-step ahead** prediction.

However, we should note two things:
1. We used the entire time series to train our model
2. Some of the use-cases that we will be tackling will require us to forecast t-steps ahead

We will be solving the first point in the next section by tackling real-world datasets. The second point can be solved immediately using dynamic=True argument shown below.

# Actual vs Fitted
fig, ax = plt.subplots(figsize=(15,2))
ax = df.plot(ax=ax)
fig = model_fit.plot_predict(90, 110, dynamic=True, ax=ax, plot_insample=False)
plt.show()

Notice that our confidence interval is now increasing as we go farther our given data set since we will be compounding errors given by out forecasted values. Also note that generally, our ARIMA model captured the direction of the trend of our time series but is consistently lower than the actual values. We can correct this by varying the parameters of our ARIMA models and validating using performance metrics.

## Implementation and Forecasting using ARIMA

Given the steps discussed in the first three sections of this notebook, we can now create a framework of how we can approach ARIMA models in general. We can use a modified version of the Hyndman-Khandakar algorithm (Hyndman & Khandakar, 2008), which combines unit root tests, minimisation of the AIC and MLE to obtain an ARIMA model. The steps are outlined below:

1. The differencing term d is determined using repeated ADF tests.
2. The values of p and q are chosen based on the AIC, ACF, and PACF of our differenced time series.
3. Use a step-wise traversal of our parameter space (+-1) for p, d, and q to find a lower AIC. *We can use insights and heuristics to observe our ACF and PACF to determine if we need to add or reduce our p, d, and q.*
4. Check the residuals for Gaussian Distribution to establish stationarity.

Or in our case, we will instead use a grid-search algorithm to find automatically configure our ARIMA and find our hyperparameters. We will search values of p, d, and q for combinations (skipping those that fail to converge), and find the combination that results in the best performance on the test set. We use grid search to explore all combinations in a subset of integer values.

Specifically, we will search all combinations of the following parameters:

p: 0 to 4. d: 0 to 3. q: 0 to 4. This is (4 * 3 * 4), or 48, potential runs of the test harness and will take some time to execute.

The complete worked example with the grid search version of the test harness is listed below.

# create a differenced series
def difference(dataset, interval=1):
    diff = list()
    for i in range(interval, len(dataset)):
        value = dataset[i] - dataset[i - interval]
        diff.append(value)
    return np.array(diff)

# invert differenced value
def inverse_difference(history, yhat, interval=1):
    return yhat + history[-interval]

# evaluate an ARIMA model for a given order (p,d,q) and return RMSE
def evaluate_arima_model(X, arima_order, train_size=None):
    # prepare training dataset
    X = X.astype('float32')
    if train_size is None:
        train_size = int(len(X) * 0.50)
    else:
        train_size = int(train_size)
    train, test = X[0:train_size], X[train_size:]
    history = [x for x in train]
    # make predictions
    predictions = list()
    for t in range(len(test)):
        # difference data
        diff = difference(history, 1)
        model = ARIMA(diff, order=arima_order)
        model_fit = model.fit(trend='nc', disp=0)
        yhat = model_fit.forecast()[0]
        yhat = inverse_difference(history, yhat, 1)
        predictions.append(yhat)
        history.append(test[t])
    # calculate out of sample error
    mae = mean_absolute_error(test, predictions)
    return mae

# evaluate combinations of p, d and q values for an ARIMA model
def evaluate_models(dataset, p_values, d_values, q_values):
    dataset = dataset.astype('float32')
    best_score, best_cfg = float("inf"), None
    for p in p_values:
        for d in d_values:
            for q in q_values:
                order = (p,d,q)
                try:
                    mse = evaluate_arima_model(dataset, order)
                    if mse < best_score:
                        best_score, best_cfg = mse, order
                    print('ARIMA%s RMSE=%.3f' % (order,mse))
                except:
                    continue
    print('Best ARIMA%s RMSE=%.3f' % (best_cfg, best_score))

### Example 2: Forecasting Temperature in Jena Climate Data

We now use the method above to perform **multi-step forecasting** (24-step) in the Jena temperature data.

train_data = pd.read_csv('../data/train_series.csv',index_col=0).loc[:, ['T (degC)']]
val_data = pd.read_csv('../data/val_series.csv',index_col=0).loc[:, ['T (degC)']]
temp = pd.concat([train_data, val_data]).values  #temperature (in degrees Celsius)
plt.figure(figsize=(15,2))
plt.plot(range(len(temp)), temp)
plt.ylabel('Temperature \n(degree Celcius)')
plt.xlabel('Time (every hour)')
plt.show()

#### Model order selection (hyperparameter tuning)

Before we proceed with the grid-search algorithm, we perform some preliminary tests on the time series to narrow down our parameter search. We can start by checking the stationarity of the signal to estimate $d$. Then, we plot the PACF and ACF of the time series to estimate $p$ and $q$, respectively.

result = adfuller(temp)
print('ADF Statistic: %f' % result[0])
print('p-value: %f' % result[1])
print('Critical Values:')
for key, value in result[4].items():
    print('\t%s: %.3f' % (key, value))

**Observation**: Based on the results of the ADF test, the signal is stationary. This implies that $d=0$ so we no longer need to perform differencing.

plt.rcParams.update({'figure.figsize':(15,2.5), 'figure.dpi':120})

fig, axes = plt.subplots(1, 2)
axes[0].plot(temp.flatten())
axes[0].set_title('Temperature Signal')
axes[1].set(xlim=(0,20))
plot_pacf(temp.flatten(), ax=axes[1])
plt.show()

**Observation**: Based on the plot of the PACF, we can see that the function drops to a value of almost zero for lags > 2. So, we can set $p$ to be from 0 to 2.

plt.rcParams.update({'figure.figsize':(15,2.5), 'figure.dpi':120})

fig, axes = plt.subplots(1, 2)
axes[0].plot(temp.flatten())
axes[0].set_title('Temperature Signal')
plot_acf(pd.Series(temp.flatten()), ax=axes[1])
plt.show()

**Observation**: Unlike the PACF which displayed a sharp cutoff, the ACF decays more slowly. We can say that the  series has an *autoregressive signature* which means that the autocorrelation pattern can be explained more easily by adding AR terms than by adding MA terms. Let's test for values $q$ from 0 to $5$.

Due to constraints in computing resources, we will limit the length of the training data (from 35045 to 17524). We will also reduce the number of folds to be used (from 730 to 20).

# evaluate an ARIMA model for a given order (p,d,q) and return RMSE
def evaluate_arima_jena_24hrstep(X, arima_order, train_size=35045):
    # prepare training dataset
    X = X.astype('float32')
    train, test = X[:train_size], X[train_size:]
    test = test[:len(X[train_size:]) - len(X[train_size:])%24]
    test_24 = test.reshape(-1, 24)
    history = train.flatten()

    mae_cv = []
    # There are 730 folds (24 hr chunks), for faster computation, we limit the number of folds to 20 only
    for t in range(len(test_24))[::37]:
        x_cv = np.hstack([history, test_24[:t, :].flatten()])
        y_cv = test_24[t]
        model = ARIMA(x_cv, order=arima_order)
        model_fit = model.fit(disp=0)
        y_hat = model_fit.forecast(steps=24)[0]
        mae_cv.append(mean_absolute_error(y_cv, y_hat))
    mean_mae = np.mean(mae_cv)
    return mean_mae

## Uncomment the lines to perform model order selection
# arima_orders = [(0, 0, 0),
#                 (1, 0, 0),
#                 (0, 0, 1),
#                 (2, 0, 0),
#                 (1, 0, 1),
#                 (1, 0, 2),
#                 (2, 0, 1),
#                 (2, 0, 2)]
# mae_ao_list = []
# for ao in arima_orders:
#     mae_ao = evaluate_arima_jena_24hrstep(temp[-2*len(val_data):],
#                                           arima_order=ao,
#                                           train_size=len(val_data))
#     mae_ao_list.append((ao, mae_ao))
#     print(ao, mae_ao)

# selected_order, best_mae = np.array(mae_ao_list)[np.argmin(np.array(mae_ao_list)[:,-1])]
# print(f'Selected order: {selected_order}, MAE: {best_mae}')

## Expected results from code above:
## (p, d, q) MAE
## ---------------------------
## (0, 0, 0) 6.035602297003233
## (1, 0, 0) 3.025640357465662
## (0, 0, 1) 5.922866295058727
## (2, 0, 0) 3.124825200033295
## (1, 0, 1) 2.967933864811228
## (1, 0, 2) 2.9481444828018253
## (2, 0, 1) 3.5091372748649037
## (2, 0, 2) 3.4143813707023325
## Selected order: (1, 0, 2), MAE: 2.9481444828018253

selected_order = (1, 0, 2)

#### Evaluate model performance on test set

We use the multiprocessing package which supports spawning processes using an API similar to the threading module.

def wrapper_fit_arima(x_vals, order=(1, 0, 2)):
    model = ARIMA(x_vals, order=order)
    model_fit = model.fit(disp=0)
    y_hat = model_fit.forecast(steps=24)[0]
#     print(len(x_vals))
    return y_hat

def evaluate_arima_jena_mp(X, arima_order, train_size=35045):
    # prepare training dataset
    X = X.astype('float32')
    train, test = X[:train_size], X[train_size:]
    test = test[:len(X[train_size:]) - len(X[train_size:])%24]
    test_24 = test.reshape(-1, 24)
    history = train.flatten()

    X_cv = []
    Y_cv = []
    for t in range(len(test_24)):
        x_cv = np.hstack([history, test_24[:t, :].flatten()])
        y_cv = test_24[t]
        X_cv.append(x_cv)
        Y_cv.append(y_cv)

    pool = mp.Pool(processes=mp.cpu_count()-4)
    y_hats = pool.map(wrapper_fit_arima, X_cv)

    mae_cv = []
    for t in range(len(test_24)):
        mae_cv.append(mean_absolute_error(Y_cv[t], y_hats[t]))
    mean_mae = np.mean(mae_cv)
    return mean_mae

# Load test data
test_data= pd.read_csv('../data/test_series.csv',index_col=0).loc[:, ['T (degC)']]
temp2 = pd.concat([val_data, test_data]).values  #temperature (in degrees Celsius)

We fit the ARIMA(1,0,2) model using the validation set to predict the 24-hour chunks of temperature measurements in the test set.

## Uncomment code below to fit and predict temperature values in the test set
## By using CPU count of 28, the code ran for about 70 mins.
# MAE = evaluate_arima_jena_mp(temp2, arima_order=selected_order, train_size=len(val_data))
# print(f'ARIMA MAE: {np.mean(MAE)}')

## Expected result from running the code above:
## ARIMA MAE: 3.191548582794122
print('ARIMA MAE for Jena test data: 3.191548582794122')

**Observation/s**: The ARIMA(1,0,2) model had a poorer performance (ARIMA MAE: 3.19) on the test set compared to our [baseline models](../08_WinningestMethods/lightgbm_jena_forecasting.ipynb) (naive MAE: 3.18, seasonal naive MAE: 2.61).

##### Forecast 24 hours beyond test set

%%time
future_data = wrapper_fit_arima(temp2)
future_data

plt.figure(figsize=(15,2))
plt.plot(temp2[-500:])
plt.plot(np.arange(24)+500, future_data, label='forecast')
plt.legend()
plt.ylabel('Temperature (degree Celsius)')
plt.show()

### Example 3: Using ARIMA for Multivariate Time Series

In this final example, we use ARIMA to forecast different time series signals from an Air Quality Dataset measured in a polluted area at an Italian City. To illustrate, we forecast the data for the last 24 hours for CO, and NO2. The following figure shows the training data for each of these parameters.

train_aq = pd.read_csv('../data/AirQualityUCI/train_data.csv',index_col=0).set_index('Date_Time')
test_aq = pd.read_csv('../data/AirQualityUCI/test_data.csv',index_col=0).set_index('Date_Time')
train_aq.head()

fig,ax = plt.subplots(3, figsize=(15, 5), sharex=True)
train_aq.plot(ax=ax, subplots=True)
plt.xlabel('')
plt.show()

def evaluate_arima_multistep(X, arima_order, train_size=None):
    # prepare training dataset
    X = X.astype('float32')
    if train_size is None:
        train_size = int(len(X) * 0.50)
    else:
        train_size = int(train_size)
    train, test = X[0:train_size], X[train_size:]
    history = [x for x in train]
    # make predictions
    model = ARIMA(history, order=arima_order)
    model_fit = model.fit(disp=0)
    yhat = model_fit.forecast(steps=len(test))
    predictions = yhat[0]
    return mean_absolute_error(test, predictions)

def evaluate_models_mae(dataset, p_values, d_values, q_values, train_size=None):
    dataset = dataset.astype('float32')
    best_score, best_cfg = float("inf"), None
    for p in p_values:
        for d in d_values:
            for q in q_values:
                order = (p,d,q)
                try:
                    mae = evaluate_arima_model_mae(dataset, order, train_size=train_size)
                    if mae < best_score:
                        best_score, best_cfg = mae, order
                    print('ARIMA%s MAE=%.3f' % (order,mae))
                except:
                    continue
    print('Best ARIMA%s AME=%.3f' % (best_cfg, best_score))

To make predictions, we train 3 ARIMA models—one for CO, one for NO2, and one for RH. We perform grid search for the following parameters:

p_values = range(0, 5)
d_values = range(0, 2)
q_values = range(0, 2)

arima_orders = []
for p in p_values:
    for d in d_values:
        for q in q_values:
            arima_orders.append((p, d, q))

var_col = train_aq.columns
print(var_col)

for c in var_col:
    df_var = train_aq[c].values.reshape(-1,1)

    mae_order_list_var = []
    for ao in arima_orders:
        metric = evaluate_arima_multistep(df_var, arima_order=ao, train_size=len(train_aq)-len(test_aq))
        mae_order_list_var.append((ao, metric))

    print(f'{c}: {np.array(mae_order_list_var)[np.argmin(np.array(mae_order_list_var)[:,-1])]}')

# For model order selection, refer to Chapter 1
selected_order = {'CO(GT)': [(0, 1, 0)],
                  'NO2(GT)': [(0, 1, 0)],
                  'RH': [(3, 1, 1)]}

%%time
forecast_arima = {}
for c in var_col:
    model = ARIMA(train_aq[c].values, order=selected_order[c][0])
    model_fit = model.fit(disp=0)
    y_hat = model_fit.forecast(steps=24)[0]
    plt.figure(figsize=(15,2))
    plt.plot(train_aq[c].values, label='train')
    plt.plot(np.arange(len(test_aq))+len(train_aq), test_aq[c].values, label='train')
    plt.plot(np.arange(len(test_aq))+len(train_aq), y_hat, label='train')
    plt.ylabel(c)
    plt.xlim(xmin=3500, xmax=4460)
    plt.show()

## Preview to the next Chapter

ARIMA model is just one of the many algorithms that we will discuss. We can see the inherent limitations of ARIMA. In particular, we are not considering:
1. Trends and seasonality
1. Long-range dependence
1. Multi-variate time series

These factors are generally more frequent and will be important in several applications. For example, trend is important in technical analysis in stock trading. This can be solved using Momentum Forecasting which will be discussed in Chapter 2, among other methods.

In Chapter 3, we will revisit the Air Quality example and use a multivariate approach.  We’ll introduce vector autoregressive (VAR) methods—a family of models that expresses each variable as a linear function of past lags and past lags of the other variables—and show that we can improve the forecasts by extending this to a multivariate time series problem.

## References

The contents of this notebook is compiled from the following sources:
- https://www.machinelearningplus.com/time-series/arima-model-time-series-forecasting-python/
- https://otexts.com/fpp2/arima.html
- https://people.duke.edu/~rnau/411arim.htm
- https://www.analyticsvidhya.com/blog/2018/02/time-series-forecasting-methods/
- https://machinelearningmastery.com/time-series-forecast-study-python-monthly-sales-french-champagne/
- https://medium.com/@josemarcialportilla/using-python-and-auto-arima-to-forecast-seasonal-time-series-90877adff03c
- https://en.wikipedia.org/wiki/Autoregressive_integrated_moving_average
- C.Monterola, *"Notebook 10 Time Series Forecasting Method --- ARIMA MSDS2021"*
- Hyndman, R. J., & Khandakar, Y. (2008). Automatic time series forecasting: The forecast package for R. Journal of Statistical Software, 27(1), 1–22. https://doi.org/10.18637/jss.v027.i03

import numpy,from numpy,from statsmodels


import numpy as np
from .model import Model
from abc import ABC, abstractmethod
from si.util.metrics import mse, mse_prime

__all__ = ['Dense', 'Activation', 'NN']

class Layer(ABC):
    def __init__(self):
        self.input = None
        self.output = None

    @abstractmethod
    def forward(self, input):
        raise NotImplementedError

    @abstractmethod
    def backward(self, output_error, lr):
        #lr is the learning rate (alpha)
        raise NotImplementedError

class Dense(Layer):
    #Fully Connected layer
    '''
    The dense layer’s neuron receives output from every neuron of its preceding layer,
    where neurons of the dense layer perform matrix-vector multiplication,
    where the row vector of the output from the preceding layers is equal to the column vector of the dense layer.
    Values under the matrix are the trained parameters of the preceding layers and also can be updated by the backpropagation.
    '''
    def __init__(self, input_size, output_size):
        self.weights= np.random.rand(input_size, output_size) -0.5 #matriz com shape (no linhas=no elementos matriz input, no colunas=no elementos matriz output)
        self.bias= np.zeros((1, output_size)) #array com shape(1 linha, no colunas=no elementos output)

    def setWeights(self, weights, bias):
        #Sets the weight and the bias for the NN
        if (weights.shape != self.weights.shape):
            raise ValueError(f'shapes mismatch {weights.shape} and {self.weights.shape}") ')
        if (bias.shape != self.bias.shape):
            raise ValueError(f"Shapes missmatch{bias.shape}and {self.bias.shape}")
        self.weights = weights
        self.bias = bias

    def forward(self, input):
        '''
        predictions are made based on the values in the input nodes and the weights.
        '''
        self.input = input
        self.output = np.dot(self.input, self.weights) + self.bias #produto escalar entre inputs and weights e no final soma-se o bias
        return self.output

    def backward(self, output_error, lr):
        '''
        compare the predicted output with the actual output.
        Next, fine-tune weights and the bias
        in such a manner that our predicted output becomes closer to the actual output,
        known as "training the neural network".
        'computes weights error and bias error for a given output_error'
        '''
        # compute the weights error dE/dW = X.T*dE/dY
        weights_error = np.dot(self.input.T, output_error)
        # compute the bias error dE/dB
        bias_error = np.sum(output_error, axis=0)
        input_error = np.dot(output_error, self.weights.T)
        # update parameters
        self.weights -= lr * weights_error
        self.bias -= lr * bias_error
        return input_error

class Activation(Layer):
    '''
    activation function that operates upon the value received
    '''
    def __init__(self, activation):
        self.activation = activation

    def forward(self, input):
        self.input = input
        self.output = self.activation(self.input)
        return self.output

    def backward(self, output_error, lr):
        # learning rate is not used because there is no "learnable" parameters
        # only passes the error do to the previous layer
        return np.multiply(self.activation.prime(self.input), output_error)

class NN(Model):
    def __init__(self, epochs=1000, lr=0.001, verbose=True):
        #neural network model
        self.epochs = epochs #cycle of forward and backward
        self.lr = lr #learning rate (alpha)
        self.verbose = verbose
        self.layers = []
        self.loss = mse #loss function -> Mean squared error
        self.loss_prime = mse_prime

    def add(self, layer):
        self.layers.append(layer)

    def fit(self, dataset):
        X, y = dataset.getXy()
        self.dataset = dataset
        self.history = dict()
        for epoch in range(self.epochs):
            output = X
            # forward propagation
            for layer in self.layers:
                output = layer.forward(output)

            # backward propagation
            error = self.loss_prime(y, output)
            for layer in reversed(self.layers):
                error = layer.backward(error, self.lr)

            # calculate average error on all samples
            err = self.loss(y, output)
            self.history[epoch] = err
            if self.verbose:
                print(f'epoch {epoch+1}/{self.epochs} error = {err}')
        if not self.verbose:
            print(f"error={err}")
        self.is_fitted = True

    def predict(self, input):
        assert self.is_fitted, 'Model must be fit before'
        output = input
        for layer in self.layers:
            output = layer.forward(output)
        return output

    def cost(self, X=None, y=None):
        assert self.is_fitted, 'Model must be fit'
        X = X if X is not None else self.dataset.X
        y = y if y is not None else self.dataset.y
        output = self.predict(X)
        return self.loss(y, output)


from scipy.special import cbrt, exp10, comb, perm, logsumexp
from scipy import linalg, integrate, special
import numpy as np

print('Cube Root is: ', cbrt(125))
print('Exponential is: ', exp10([1,10]))
print('combination is: ', comb(5, 2, exact = False, repetition=True))
print('Permutation is: ', perm(5, 2, exact = True))

#Square Matrix
matA = np.array([[4, 5], [3, 2]])
matB = np.array([[4, 7], [2, 6]])

#Determinant  function det()
print('\nDet of Mat(A) = ', linalg.det(matA), '\n')

#Inverse function inv()
print('Inv of Mat(B):\n', linalg.inv(matB), '\n')

f = lambda x : x**2 #f(x) function
#single integration with a=0 & b=1
result = integrate.quad(f, 0 , 1)
print(result)


import numpy as np
import math
import operator

from functools import reduce

def griewank(d):
    sumatory = 1.0 / 4000 * np.sum([(x**2) for x in d])
    product = np.prod([np.cos(x / np.sqrt(i+1.0)) for i, x in enumerate(d)])

    return 1 + sumatory - product

def sphere(d):
    return np.sum([x**2 for x in d])

def ackley(d, *, a=20, b=0.2, c=2*np.pi):
    sum_part1 = np.sum([x**2 for x in d])
    part1 = -1.0 * a * np.exp(-1.0 * b * np.sqrt((1.0/len(d)) * sum_part1))

    sum_part2 = np.sum([np.cos(c * x) for x in d])
    part2 = -1.0 * np.exp((1.0 / len(d)) * sum_part2)

    return a + np.exp(1) + part1 + part2

def rastrigin(d):
    sum_i = np.sum([x**2 - 10*np.cos(2 * np.pi * x) for x in d])
    return 10 * len(d) + sum_i


import arviz as az
import matplotlib.pyplot as plt
import numpy as np
from scipy.special import comb, beta
from scipy.stats import binom
from scipy import stats

np.random.seed(0)

a_prior, b_prior = 1, 1

Y = stats.bernoulli(0.7).rvs(20)

N1, N0 = Y.sum(), len(Y) - Y.sum()

a_post = a_prior + N1
b_post = b_prior + N0

prior_pred_dist, post_pred_dist = [], []
N = 20

for k in range(N + 1):
    post_pred_dist.append(comb(N, k) * beta(k + a_post, N - k + b_post) / beta(a_post, b_post))
    prior_pred_dist.append(comb(N, k) * beta(k + a_prior, N - k + b_prior) / beta(a_prior, b_prior))

fig, ax = plt.subplots()
ax.bar(np.arange(N + 1), prior_pred_dist, align='center', color='grey')
ax.set_title(f"Prior predictive distribution", fontweight='bold')
ax.set_xlim(-1, 21)
ax.set_xticks(list(range(N + 1)))
ax.set_xticklabels(list(range(N + 1)))
ax.set_ylim(0, 0.15)
ax.set_xlabel("number of success")

fig, ax = plt.subplots()
ax.bar(np.arange(N + 1), post_pred_dist, align='center', color='grey')
ax.set_title(f"Posterior predictive distribution", fontweight='bold')
ax.set_xlim(-1, 21)
ax.set_xticks(list(range(N + 1)))
ax.set_xticklabels(list(range(N + 1)))
ax.set_ylim(0, 0.15)
ax.set_xlabel("number of success")

fig, ax = plt.subplots()
az.plot_dist(np.random.beta(a_prior, b_prior, 10000), plot_kwargs={"color": "0.5"},
             fill_kwargs={'alpha': 1})
ax.set_title("Prior distribution", fontweight='bold')
ax.set_xlim(0, 1)
ax.set_ylim(0, 4)
ax.tick_params(axis='both', pad=7)
ax.set_xlabel("θ")

fig, ax = plt.subplots()
az.plot_dist(np.random.beta(a_post, b_post, 10000), plot_kwargs={"color": "0.5"},
             fill_kwargs={'alpha': 1})
ax.set_title("Posterior distribution", fontweight='bold')
#ax.set_xlim(0, 1)
#ax.set_ylim(0, 4)
ax.tick_params(axis='both', pad=7)
ax.set_xlabel("θ")

plt.show()


import numpy as np
import pandas as pd

def central_standard(df):
    """
    Get the centalised and standardised numpy array

    Parameters:
     - df: A pandas DataFrame
    """
    x = df.values
    # Centralisation
    y = x - np.mean(x, axis=0)
    # Calculate the covariance matrix
    s = np.dot(np.transpose(y), y) / (x.shape[0] - 1)
    # Standardisation
    d = np.diag(s.diagonal()**(-1 / 2))
    z = np.dot(y, d)
    return z

def corr_mtx(df):
    """
    Get the correlation matrix

    Parameters:
     - df: A pandas DataFrame
    """
    n, p = df.shape
    z = central_standard(df)
    cor_matrix = np.dot(z.T, z) / (n - 1)
    return cor_matrix


import matplotlib.pyplot as plt
import numpy as np
from scipy.special import binom

corners = np.array(((0, 1), (1, 1), (0, 0), (1, 0)))

def smoothstep(order, sample_density):
    """ Defines a smoothstep curve of a specified order.
        See https://en.wikipedia.org/wiki/Smoothstep
    """
    n = order
    x = np.linspace(0, 1, sample_density)
    sum_array = np.zeros(x.shape)

    for k in range(n + 1):
        sum_array += binom(n + k, k) * binom(2 * n + 1, n - k) * (-x) ** k

    S = x ** (n + 1) * sum_array
    return S

def gradient_grid(grid_shape):
    """ Initializes a grid with random gradients. """
    grid_rows, grid_cols = grid_shape
    full_rot = np.radians(360)
    grad_rots = np.random.uniform(0, full_rot, (grid_rows+1, grid_cols+1))
    xs = np.cos(grad_rots)
    ys = np.sin(grad_rots)
    grad_grid = np.stack((xs, ys), axis=-1)
    return grad_grid

def sample_grid(sample_density):
    """ Initializes a sample grid where each pixel
    corresponds with image pixels.
    """
    xs = np.linspace(0.01, 0.99, sample_density)
    ys = xs
    samp_grid = np.stack(np.meshgrid(xs, ys), axis=-1)
    return samp_grid

def distance_grid(samp_grid):
    """ Initializes a grid with vectors pointing from
        corners to sample pixel locations.
    """

    dist_grids = list()
    for corner in corners:
        dist_vecs = samp_grid - corner
        dist_grids.append(dist_vecs)

    return dist_grids

def perlin_2d(grid_shape=(10, 10), sample_density=40):
    """ Returns a 2D Perlin noise image. """

    grad_grid = gradient_grid(grid_shape)
    samp_grid = sample_grid(sample_density)
    dist_grid = distance_grid(samp_grid)

    smooth_weight = smoothstep(2, sample_density)
    x_weights, y_weights = np.meshgrid(smooth_weight, np.flip(smooth_weight))

    pixels = np.zeros((sample_density * grid_shape[0],
                       sample_density * grid_shape[1]))

    for row in range(grid_shape[0]):
        for col in range(grid_shape[1]):

            dot_prods = list()
            for idx, corner in enumerate(corners):
                x = corner[0]
                y = corner[1]
                grad = grad_grid[y + row, x + col]
                dist_vectors = dist_grid[idx]
                dot_prod = np.tensordot(dist_vectors, grad, axes=1)
                dot_prods.append(dot_prod)

            a, b, c, d = dot_prods
            ab = a + x_weights * (b - a)
            cd = c + x_weights * (d - c)
            values = ab + y_weights * (cd - ab)
            pixels[row * sample_density: row * sample_density + sample_density,
                   col * sample_density: col * sample_density + sample_density] = values

    return pixels

pixels = perlin_2d()
plt.imshow(pixels, origin='lower', cmap='Greys')
plt.savefig('perlin_noise.png')


from data_getter import Data
from method import thomas_algorithm
from numpy import arange
import matplotlib.pyplot as plt

def left_boundary_conditions():
    X_half = Data.Xn_plus_half(Data.x0)
    p1 = Data.p(Data.x0 + Data.h)
    f1 = Data.f(Data.x0 + Data.h)

    p0 = Data.p(Data.x0)
    f0 = Data.f(Data.x0)

    p_half = (p0 + p1) / 2

    K0 = X_half + Data.h * Data.h * p_half / 8 + Data.h * Data.h * p0 / 4
    M0 = Data.h * Data.h * p_half / 8 - X_half
    P0 = Data.h * Data.F0 + Data.h * Data.h * (3 * f0 + f1) / 4

    return K0, M0, P0

def right__boundary_conditions():
    X_half = Data.Xn_minus_half(Data.l)

    pN = Data.p(Data.l)
    pN1 = Data.p(Data.l - Data.h)
    fN = Data.f(Data.l)
    fN1 = (2 * Data.alpha(Data.l - Data.h)) / Data.R * Data.Tenv

    KN = - (X_half + Data.alphaN * Data.h) / Data.h - Data.h * (5 * pN + pN1) / 16
    MN = X_half / Data.h - Data.h * (pN + pN1) / 16
    PN = - Data.alphaN * Data.Tenv - Data.h * (3 * fN + fN1) / 8

    return KN, MN, PN

def calc_coefficients():
    A = []
    B = []
    C = []
    D = []

    for i in arange(Data.x0, Data.l, Data.h):
        An = Data.Xn_minus_half(i) / Data.h
        Cn = Data.Xn_plus_half(i) / Data.h
        Bn = An + Cn + Data.p(i) * Data.h
        Dn = Data.f(i) * Data.h

        A.append(An)
        B.append(Bn)
        C.append(Cn)
        D.append(Dn)

    return A, B, C, D

if __name__ == "__main__":
    a, b, c, d = calc_coefficients()
    # print(a)
    # print(b)
    # print(c)
    # print(d)

    k0, m0, p0 = left_boundary_conditions()
    # print(k0)
    # print(m0)
    # print(p0)

    kN, mN, pN = right__boundary_conditions()
    # print(kN)
    # print(mN)
    # print(pN)

    T = thomas_algorithm(a, b, c, d, k0, m0, p0, kN, mN, pN)
    print(T)
    x = arange(Data.x0, Data.l, Data.h)

    plt.title('Heating the rod')
    plt.grid(True)
    plt.plot(x, T, 'r', linewidth=0.5)
    plt.xlabel("Length (cm)")
    plt.ylabel("Temperature (K)")

    plt.savefig("plot.png")

    plt.show()


# -*- coding: utf-8 -*-

# Implemented:
# 1 closed-form formulas for geometric Asian call/put options
# 2 closed-from formulas for geometric basket call/put options

import numpy as np
from scipy.stats import norm

def geom_asian_call_option(s_t, sigma, r, T, K, n, t=0):
    """Calculate the value of geometric Asian call option. By default
    calculate the value at time 0.

    Args:
        s_t: The value of asset S at time t
        sigma: The volatility
        r: The risk-free rate
        T: The time to maturity(in years)
        K: Strike
        n: The number of observation times for geometric average n
        t: Time t for option value, default 0

    Returns:
        The geometric Asian call option value at time t
    """
    _T = T-t

    _sigma_hat_sq_T = sigma*sigma*(n+1)*(2*n+1)/(6*n*n)*_T # (sigma^2)*T, convience for latter calculation
    _mu_hat_T = (r-0.5*sigma*sigma)*(n+1)/(2*n)*_T + 0.5*_sigma_hat_sq_T # mu*T, convience for latter calculation

    _d1 = (np.log(s_t/K) + _mu_hat_T + 0.5*_sigma_hat_sq_T) / (np.sqrt(_sigma_hat_sq_T))
    _d2 = _d1 - np.sqrt(_sigma_hat_sq_T)

    _N1 = norm.cdf(_d1)
    _N2 = norm.cdf(_d2)

    return np.exp(-r*_T)*(s_t*np.exp(_mu_hat_T)*_N1 - K*_N2)

def geom_asian_put_option(s_t, sigma, r, T, K, n, t=0):
    """Calculate the value of geometric Asian put option. By default
    calculate the value at time 0.

    Args:
        s_t: The value of asset S at time t
        sigma: The volatility
        r: The risk-free rate
        T: The time to maturity(in years)
        K: Strike
        n: The number of observation times for geometric average n
        t: Time t for option value, default 0

    Returns:
        The geometric Asian put option value at time t
    """
    _T = T-t

    _sigma_hat_sq_T = sigma*sigma*(n+1)*(2*n+1)/(6*n*n)*_T # (sigma^2)*T, convience for latter calculation
    _mu_hat_T = (r-0.5*sigma*sigma)*(n+1)/(2*n)*_T + 0.5*_sigma_hat_sq_T # mu*T, convience for latter calculation

    _d1 = (np.log(s_t/K) + _mu_hat_T + 0.5*_sigma_hat_sq_T) / (np.sqrt(_sigma_hat_sq_T))
    _d2 = _d1 - np.sqrt(_sigma_hat_sq_T)

    _N1 = norm.cdf(-_d1)
    _N2 = norm.cdf(-_d2)

    return np.exp(-r*_T)*(K*_N2 - s_t*np.exp(_mu_hat_T)*_N1)

def geom_basket_call_option(s_t_1, s_t_2, sigma_1, sigma_2, r, T, K, rho, t=0):
    """Calculate the value of geometric basket call option. By default
    calculate the value at time 0.

    Args:
        s_t_1: The value of asset S1 at time t
        s_t_2: The value of asset S2 at time t
        sigma_1: The volatility of S1
        sigma_2: The volatility of S2
        r: The risk-free rate
        T: The time to maturity(in years)
        K: Strike
        rho: The correlation rho
        t: Time t for option value, default 0

    Returns:
        The geometric basket call option value at time t
    """
    _T = T-t

    _sigma_sq_T = (2*sigma_1*sigma_2*rho + sigma_1*sigma_1 + \
        sigma_2*sigma_2) / (2*2) *_T # (sigma^2)*T, convience for latter calculation
    _mu_T = r*_T - 0.5*(sigma_1*sigma_1+sigma_2*sigma_2)/2*_T + 0.5*_sigma_sq_T

    _B_t = np.sqrt(s_t_1*s_t_2)

    _d1 = (np.log(_B_t/K) + _mu_T + 0.5*_sigma_sq_T) / (np.sqrt(_sigma_sq_T))
    _d2 = _d1 - np.sqrt(_sigma_sq_T)

    _N1 = norm.cdf(_d1)
    _N2 = norm.cdf(_d2)

    return np.exp(-r*_T)*(_B_t*np.exp(_mu_T)*_N1 - K*_N2)

def geom_basket_put_option(s_t_1, s_t_2, sigma_1, sigma_2, r, T, K, rho, t=0):
    """Calculate the value of geometric basket call option. By default
    calculate the value at time 0.

    Args:
        s_t_1: The original value of asset S1
        s_t_2: The original value of asset S2
        sigma_1: The volatility of S1
        sigma_2: The volatility of S2
        r: The risk-free rate
        T: The time to maturity(in years)
        K: Strike
        rho: The correlation rho
        t: Time t for option value, default 0

    Returns:
        The geometric basket call option value at time t
    """
    _T = T-t

    _sigma_sq_T = (2*sigma_1*sigma_2*rho + sigma_1*sigma_1 + \
        sigma_2*sigma_2) / (2*2) *_T # (sigma^2)*T, convience for latter calculation
    _mu_T = r*_T - 0.5*(sigma_1*sigma_1+sigma_2*sigma_2)/2*_T + 0.5*_sigma_sq_T

    _B_t = np.sqrt(s_t_1*s_t_2)

    _d1 = (np.log(_B_t/K) + _mu_T + 0.5*_sigma_sq_T) / (np.sqrt(_sigma_sq_T))
    _d2 = _d1 - np.sqrt(_sigma_sq_T)

    _N1 = norm.cdf(-_d1)
    _N2 = norm.cdf(-_d2)

    return np.exp(-r*_T)*(K*_N2 - _B_t*np.exp(_mu_T)*_N1)

def main():
    s_t = 100
    sigma = 0.3
    r = 0.05
    T = 3
    K = 100
    n = 50

    print('Asian call: ' + str(geom_asian_call_option(s_t, sigma, r, T, K, n)))
    print('Asian put: ' + str(geom_asian_put_option(s_t, sigma, r, T, K, n)))

    s_t_1 = 100
    s_t_2 = 100
    sigma_1 = 0.3
    sigma_2 = 0.3
    K = 100
    rho = 0.5

    print('Basket call: ' + str(geom_basket_call_option(s_t_1, s_t_2, sigma_1, sigma_2, r, T, K, rho)))
    print('Basket put: ' + str(geom_basket_put_option(s_t_1, s_t_2, sigma_1, sigma_2, r, T, K, rho)))

if __name__ == '__main__':
    main()


import numpy as np
import pandas as pd
import math

def norm(x):
    return math.sqrt(np.sum(x ** 2))

def dist(x, y):
    return norm(x - y)

class GradientDescent:
    def __init__(self, path):
        data = np.array(pd.read_csv(path, sep=' ', header=None))
        # Первая строка - вектор d в уравнении Ax = d.
        self.d = data[:1][0]

        # Остальные строки - матрица A в уравнении Ax = d.
        self.A = data[1:]

        # Фи, вычисляющиеся итеративно.
        self.all_phi = []

    def compute_tau_k(self, phi_k):
        r_k = self.compute_r_k(phi_k)
        return np.dot(r_k, r_k) / np.dot(np.dot(self.A, r_k), r_k)

    def compute_r_k(self, phi_k):
        return np.dot(self.A, phi_k) - self.d

    def next_phi(self, phi_k):
        tau_k = self.compute_tau_k(phi_k)
        r_k = self.compute_r_k(phi_k)
        return phi_k - tau_k*r_k

    def run(self, eps=0.001, phi_0=None):
        if phi_0 is None:
            phi_0 = [0, 0, 0]

        phi_1 = self.next_phi(phi_0)

        self.all_phi = [phi_0, phi_1]
        iters = 0
        max_iters = 10000
        while iters < max_iters and dist(self.all_phi[-2], self.all_phi[-1]) > eps:
            phi = self.all_phi[-1]
            next_phi = self.next_phi(phi)
            self.all_phi.append(next_phi)
            print('k=', iters, phi, 'tau=', self.compute_tau_k(phi), 'rk=', self.compute_r_k(phi),
                  'dist=', dist(self.all_phi[-2], self.all_phi[-1]))
            iters += 1

        print('Iter number = ', iters)
        print('Result: ', self.all_phi[-1])
        self.all_phi = list()

task = GradientDescent('res/matrix.txt')
task.run()


"""
surrogate model using generalized polynomial chaos expansion
"""
import numpy as np
from scipy.stats import uniform
import os,sys
my_path = os.path.dirname(os.path.realpath(__file__))
sys.path.append(my_path+'/../library')
from gpce import onedgpce

var = uniform(loc = -4., scale =  8.)
dist_type = 'norm'
order = 15
import os,sys
my_path = os.path.dirname(os.path.realpath(__file__))
sys.path.append(my_path+'/../library')

pceModel = onedgpce(dim = 1, var = var, order = order, dist_type = dist_type)
print ("check orthonality: oder4, oder 4", pceModel.check_orthogonal(4, 4))
print ("check orthonality via Quad: oder4, oder 4", pceModel.check_orthogonal_viaQuad(4, 4))
print ("check orthonality: oder2, oder 4", pceModel.check_orthogonal(2, 4))
print ("check orthonality via Quad: oder2, oder 4", pceModel.check_orthogonal_viaQuad(2, 4))

def forward_model(x):
    return  np.sin(x)
def forward_model_logx(logx):
    return  forward_model(np.exp(logx))
#pceModel.get_modelcoef(forward_model, method = "quad", nbint = order +1)
pceModel.get_modelcoef(forward_model)
xssamples = np.sort(var.rvs(size = 1000))
import matplotlib.pyplot as plt
plt.plot(xssamples, forward_model(xssamples), label = 'forward function' )
plt.plot(xssamples, pceModel.pceModeleval(xssamples),'--', label = 'pce approximation' )
plt.legend()
plt.show()


import numpy as np
from   prime_utils import lognorm_pdf, lognorm_cdf

def _incubation(time,incubation_median,incubation_sigma):
    """
    Compute the probability density function of the incubation rate,
    currently modeled as log-normal distribution
    """
    time = np.atleast_1d(time)
    vals = np.zeros_like(time)
    I = np.where(time>=0)
    vals[I] = lognorm_pdf(time[I],incubation_sigma,scale=incubation_median)
    return vals

def _incubation_cdf(time,incubation_median,incubation_sigma):
    """
    Compute the cumulative density function of the incubation rate,
    currently modeled as log-normal distribution
    """
    time = np.atleast_1d(time)
    vals = np.zeros_like(time)
    I = np.where(time>=0)
    vals[I] = lognorm_cdf(time[I],incubation_sigma,scale=incubation_median)
    return vals

def incubation_fcn(time,incubation_median,incubation_sigma,is_cdf=False):
    """
    Computes the incubation rate

    Parameters
    ----------
    time: float, list, or numpy array
        instances in time for the evaluation of the incubation rate model
    incubation_median: float
        median of the incubation rate model
    incubation_sigma: float
        standard deviation of the incubation rate model
    is_cdf: boolean (optional, default False)
        select either the CDF of the incubation rate model (True) or
        its PDF (False)

    Returns
    -------
    vals: numpy array
        incubation rates corresponding to the time values provided as input parameters
    """
    if is_cdf:
        vals = _incubation_cdf(time,incubation_median,incubation_sigma)
    else:
        vals = _incubation(time,incubation_median,incubation_sigma)
    return vals


from scipy.stats import norm
from math import sqrt
import sys

def get_z_score(alpha):
    return -norm.ppf(alpha/2)

def get_beta(z_score, s, d_min, N):
    SE = s / sqrt(N)
    return norm.cdf(z_score * SE, loc=d_min, scale=SE)

def calculate_required_size(s, d_min, Ns=20000, alpha=0.05, beta=0.2):
    """
     :param s: The standard error of the metric with N=1 in each group
     :param d_min: The practical significance level
     :param Ns: The sample sizes to try
     :param alpha:  The desired alpha level of the test
     :param beta: The desired beta level of the test
     :return: The smallest N out of the given Ns that will achieve the desired
              beta. There should be at least N samples in each group of the experiment.
              If none of the given Ns will work, returns -1. N is the number of
              samples in each group.
    """
    if get_beta(get_z_score(alpha), s, d_min, Ns) > beta:
        return -1
    left = 1
    right = Ns
    while left <= right:
        mid = (left + right) // 2
        ans = get_beta(get_z_score(alpha), s, d_min, mid)
        if ans <= beta:
            right = mid - 1
        elif ans > beta:
            left = mid+1
    return mid

if __name__=='__main__':
    s, d_min = float(sys.argv[1]), float(sys.argv[2])
    print(calculate_required_size(s, d_min))


import numpy as np

def LinearFitCasero(x, y):
    # https://en.wikipedia.org/wiki/Simple_linear_regression
    x_avg = _average(x)
    y_avg = _average(y)

    m_sum_0_x = [(x_i - x_avg) for x_i in x]
    m_sum_0_y = [(y_i - y_avg) for y_i in y]
    m_sum_0 = 0
    m_sum_1 = 0
    for i in range(len(x)):
        m_sum_0 += m_sum_0_x[i] * m_sum_0_y[i]
        m_sum_1 += m_sum_0_x[i]**2

    m = m_sum_0/m_sum_1
    b = y_avg-m*x_avg

    return [m*x_i + b for x_i in x], [m, b]

def LinearFit(x, y):
    poly = np.polyfit(x, y, 1)
    return [poly[0]*x_i + poly[1] for x_i in x], poly

def LinearFitOrigin(x, y):
    m = _dot_product(y, x) / _dot_product(x, x)
    return [m*x_i for x_i in x], [m, 0]

def QuadraticFit(x, y):
    poly = np.polyfit(x, y, 2)
    return [poly[0]*x_i**2 + poly[1]*x_i + poly[2] for x_i in x], [poly[0], poly[1], poly[2]]

def NRankFit(x, y, n):
    poly = np.polyfit(x, y, n)
    fit = []
    for x_i in x:
        res = 0
        for i in range(n+1):
            res += poly[i] * x_i**(n-i)
        fit.append(res)
    return fit, poly

# helper internal functions
def _average(a):
    return sum(a)/len(a)

def _dot_product(v, w):
    dot = 0
    for i in range(len(v)):
        dot += v[i] * w[i]
    return dot


import numpy as np

def sum_squares_error(data,x):
    '''
    Calculated sum squared difference between data and x
    data, x: numpy arrays
    '''
    return np.sum((data-x)*(data-x))

def sine_wave(time, period, amplitude, phase, offset):
    '''
    For generating a sine wave flow profile.

    Parameters
    ----------
    period: float
        The period of the sine wave in seconds.

    amplitude: float
        The amplitude of the sine wave (any unit: think of output).

    phase:
        Phase shift

    offset: float
        The centre of the sine wave.

    Returns
    -------
    wave: 1Darray
        An array of flow rate values.
    '''

    wave = amplitude*np.sin(2*np.pi*time/period + phase) + offset

    return wave

def polar_to_xy(magnitude, angle):
    '''
    Convert polar coordinates to cartesian
    magnitude: float
    angle: float (in degrees)

    returns: 2-tuple of floats
        x, y
    '''

    x = magnitude*np.cos(np.deg2rad(angle))
    y = magnitude*np.sin(np.deg2rad(angle))

    return x,y

def circle(r, origin):
    '''
    Calculate a circle of radius r and centre origin

    r: float
    origin: 2-tuple of floats

    returns numpy arrays for x and y coordinates
    '''

    theta = np.linspace(0, 2*np.pi, 100)

    x1 = r*np.cos(theta)
    x2 = r*np.sin(theta)

    return x1+origin[0],x2+origin[1]


import numpy as np
from numpy import ndarray

import solution

def mixture_moments(weights: ndarray,
                    means: ndarray,
                    covs: ndarray,
                    ) -> tuple[ndarray, ndarray]:
    """Calculate the first two moments of a Gaussian mixture.

    Args:
        weights: shape = (N,)
        means: shape = (N, n)
        covs: shape = (N, n, n)

    Returns:
        mean: shape = (n,)
        cov: shape = (n, n)
    """

    N = means.shape[0]
    n = means.shape[1]

    mean = 0
    for i in range(N):
        mean += weights[i]*means[i]

    cov = np.empty([n, n])
    for i in range(N):
        diff = means[i] - mean
        cov += weights[i]*covs[i] + weights[i]*(diff@diff.T)

    return mean, cov


import scipy
from scipy.linalg import eigh
import numpy as np
import matplotlib.pyplot as plt

def load_and_center_dataset(filename):
    data = np.load(filename)
    data = np.reshape(data, (2000,784))
    data -= np.mean(data, axis=0)
    return data

def get_covariance(dataset):
    S = np.dot(dataset.T, dataset)
    S /= (dataset.shape[0] - 1)
    return S

def get_eig(S, m):
    eigh_vals, eigh_vecs = eigh(S, subset_by_index=[S.shape[0]-m, S.shape[0]-1])
    return np.diag(np.flip(eigh_vals)), np.flip(eigh_vecs, axis=1)

def get_eig_perc(S, perc):
    perc_var = eigh(S, eigvals_only=True)
    perc_var /= np.sum(perc_var)

    num_gt = S.shape[0] - np.argmax(perc_var > perc)

    return get_eig(S, num_gt)

def project_image(image, U):
    return U @ U.T.dot(image)

def display_image(orig, proj):
    orig = np.reshape(orig, (28,28))
    proj = np.reshape(proj, (28,28))

    fig, (ax1, ax2) = plt.subplots(1, 2)

    ax1.set_title('Original')
    ax2.set_title('Projection')

    img1 = ax1.imshow(orig, aspect='equal', cmap='gray')
    img2 = ax2.imshow(proj, aspect='equal', cmap='gray')

    fig.colorbar(img1, ax=ax1)
    fig.colorbar(img2, ax=ax2)

    plt.show()


import numpy as np

from post_fractional_estimate import debias_reciprocal

def _sens_sum(a, b):
    return max(abs(a), abs(b))

def release_sxy(data, bounds, epsilon):
    """Releases an `epsilon`-DP estimate of the dot product of the two columns in `data`, x and y."""
    x, y = data.T

    x_bounds, y_bounds = bounds
    sxy_sensitivity = 2 * _sens_sum(*x_bounds) * _sens_sum(*y_bounds)

    return np.random.laplace(
        loc=x @ y,
        scale=sxy_sensitivity / epsilon)

def release_sxx(x, bounds, epsilon):
    """Releases an `epsilon`-DP estimate of the sum of squares of x."""
    sxx_sensitivity = _sens_sum(*bounds) ** 2
    return np.random.laplace(
        loc=x @ x,
        scale=sxx_sensitivity / epsilon)

def release_beta(data, bounds, epsilon, alpha=0.66, debias_recip=False):
    """Release an `epsilon`-DP estimate of the slope between the two columns in `data`.

    :param data: array of shape [n, 2]
    :param bounds: Upper and lower bounds for x and y. ((x_l, x_u), (y_l, y_u))
    :param epsilon: privacy parameter
    :param alpha: proportion of budget to allocate to numerator
    :param debias_recip: apply a transformation to make 1 / sxx unbiased
    :returns slope regression parameter
    """
    sxy = release_sxy(data, bounds, epsilon * alpha)
    sxx = release_sxx(data[:, 0], bounds[0], epsilon * (1 - alpha))

    if debias_recip:
        sxx = debias_reciprocal(
            sxx, _sens_sum(*bounds[0]) ** 2 / (epsilon * (1 - alpha)),
            np.random.laplace(size=10_000)
        )

    return sxy / sxx


import json
import math
import numpy
import scipy.integrate

def ballistics(t, x, cD_sphere, Area, Density, m, g):
    dfdt = [x[1], (1 / m) * (-1 * cD_sphere * Area * Density * (x[1] ** 2)), x[3], (1 / m) * (-1 * cD_sphere * Area * Density * (x[3] ** 2) * numpy.sign(x[3]) - (m * g))]
    return dfdt

def solve_initial_value_problem(initial_velocity, angle, barrel_offset, cD_sphere, area, density_of_air, m, g, tspan, timestep):
    x0 = 0
    x1 = initial_velocity * math.cos(math.radians(angle))
    y0 = barrel_offset
    y1 = initial_velocity * math.sin(math.radians(angle))
    p0 = [x0, x1, y0, y1]
    return scipy.integrate.solve_ivp(ballistics, tspan, p0, t_eval=timestep, args=(cD_sphere, area, density_of_air, m, g))

def solve_angle(initial_velocity, angle, barrel_offset, cD_sphere, area, density_of_air, m, g, tspan, timestep):
    results = solve_initial_value_problem(initial_velocity, angle, barrel_offset, cD_sphere, area, density_of_air, m, g, tspan, timestep)
    p = results.y
    target_val = 0
    while p[2][target_val] > 0:
        target_val = target_val + 1
    return p, target_val

def main():
    steps = 200000
    initial_velocity = 150  # m/s
    bb_diameter = 6e-3  # m
    density_of_air = 1.092  # kg/m^3
    cD_sphere = 0.5
    area = math.pi * 0.25 * (bb_diameter ** 2)  # m^2
    m = 0.2e-3  # kg
    gun_height = 1  # m
    g = 9.81  # m/s
    barrel_offset = 0.03 + gun_height
    tmax = 1000
    angletable = numpy.linspace(-45, 45, 10)
    tspan = [0, tmax]
    timestep = numpy.linspace(0, tmax, steps)
    firing_table = {}
    for angle in angletable:
        p, target_val = solve_angle(initial_velocity, angle, barrel_offset, cD_sphere, area, density_of_air, m, g, tspan, timestep)
        polynomial = numpy.polynomial.polynomial.Polynomial.fit(p[0][0:(target_val + 1)], p[2][0:(target_val + 1)], 5)
        firing_table[angle] = {}
        for coefficient in range(len(polynomial.convert().coef)):
            firing_table[angle][coefficient] = '{:.{prec}f}'.format(polynomial.convert().coef[coefficient], prec=2)
    with open('test.json', 'w') as file:
        json.dump(firing_table, file)

main()


import numpy as np
# X          - single array/vector
# y          - single array/vector
# theta      - single array/vector
# alpha      - scalar
# iterations - scalar

def gradientDescent(X, y, theta, alpha, numIterations):
    '''
    # This function returns a tuple (theta, Cost array)
    '''
    m = len(y)
    arrCost =[];
    transposedX = np.transpose(X) # transpose X into a vector  -> XColCount X m matrix

    for iteration in range(0, numIterations):
        ################PLACEHOLDER3 #start##########################
        # Write your codes to update theta, i.e., the parameters to estimate.
        # Replace the following variables if needed
        hyp = np.dot(theta, transposedX)
        residualError = np.subtract(y, hyp)
        gradient = (1 / m) * np.dot(transposedX, np.subtract(hyp, y))
        change = [alpha * x for x in gradient]
        theta = np.subtract(theta, change)  # or theta = theta - alpha * gradient
        ################PLACEHOLDER3 #end##########################

        ################PLACEHOLDER4 #start##########################
        # Calculate the current cost with the new theta;
        atmp = (1 / m) * np.sum(residualError ** 2)
        arrCost.append(atmp)
        ################PLACEHOLDER4 #start##########################

    return theta, arrCost