\documentclass{beamer}

\mode<presentation>
{
  \setbeamertemplate{background canvas}[square]
  \pgfdeclareimage[width=6em,interpolate=true]{dsailogo}{../dsai-logo}
  \pgfdeclareimage[width=6em,interpolate=true]{erasmuslogo}{../erasmus-logo}
  \titlegraphic{\pgfuseimage{dsailogo} \hspace{0.2in} \pgfuseimage{erasmuslogo}}
  %\usetheme{default}
  \usetheme{Madrid}
  \usecolortheme{rose}
  \usefonttheme[onlysmall]{structurebold}
}

\usepackage{pgf,pgfarrows,pgfnodes,pgfautomata,pgfheaps,pgfshade}
\usepackage{amsmath,amssymb}
\usepackage{graphics}
\usepackage{ragged2e}
\usepackage{array}
\usepackage[latin1]{inputenc}
\usepackage{colortbl}
\usepackage[absolute,overlay]{textpos}
\setlength{\TPHorizModule}{30mm}
\setlength{\TPVertModule}{\TPHorizModule}
\textblockorigin{10mm}{10mm}
\usepackage[english]{babel}
\usepackage{listings}
\setbeamercovered{dynamic}

\AtBeginSection[]{
  \begin{frame}<beamer>
  \frametitle{Outline}
  \tableofcontents[currentsection]
  \end{frame}
}

\title[Machine Learning]{Machine Learning\\Learning Theory}
\author{dsai.asia}
\institute[]{
  Asian Data Science and Artificial Intelligence Master's Program}
\date{}

% My math definitions

\renewcommand{\vec}[1]{\boldsymbol{#1}}
\newcommand{\mat}[1]{\mathtt{#1}}
\newcommand{\ten}[1]{\mathcal{#1}}
\newcommand{\crossmat}[1]{\begin{bmatrix} #1 \end{bmatrix}_{\times}}
\renewcommand{\null}[1]{{\cal N}(#1)}
\newcommand{\class}[1]{{\cal C}_{#1}}
\def\Rset{\mathbb{R}}
\def\Expec{\mathbb{E}}
\def\Pset{\mathbb{P}}
\DeclareMathOperator*{\argmax}{argmax}
\DeclareMathOperator*{\argmin}{argmin}
\DeclareMathOperator*{\sign}{sign}
\DeclareMathOperator*{\cov}{Cov}
\DeclareMathOperator*{\diag}{diag}
\DeclareMathOperator*{\var}{Var}
\def\norm{\mbox{$\cal{N}$}}

\newcommand{\stereotype}[1]{\guillemotleft{{#1}}\guillemotright}

\newcommand{\myfig}[3]{\centerline{\includegraphics[width={#1}]{{#2}}}
    \centerline{\scriptsize #3}}

\begin{document}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%             CONTENTS START HERE

%\setbeamertemplate{navigation symbols}{}

\frame{\titlepage}

%--------------------------------------------------------------------
%\part<presentation>{Part name}
%
%\frame{\partpage}

\begin{frame}
\frametitle{Readings}

Readings for these lecture notes:
\begin{itemize}
\item[-] Bishop, C. (2006), \textit{Pattern Recognition and Machine Learning},
  Springer, Chapter 3.
\item[-] Hastie, T., Tibshirani, R., and Friedman, J. (2016),
  \textit{Elements of Statistical Learning: Data Mining, Inference, and
    Prediction}, Springer, Chapter 7.
\item[-] Ng, A. (2017), \textit{Learning Theory}.
  Lecture note sets 4, 5 for CS229, Stanford University.
\end{itemize}

These notes contain material $\copyright$ Bishop (2006), Hastie et
al.\ (2016), and Ng (2017).

\end{frame}

%======================================================================
\section{Introduction}
%======================================================================

\begin{frame}{Introduction}

  Machine learning is one of those areas where practical rules of
  thumb tend to emerge before the theoretical principles that might
  lead to those rules of thumb.

  \medskip

  In this series, we cover some of the most important theoretical
  principles that underly or seek to explain our practical heuristics:
  \begin{itemize}
  \item The bias-variance tradeoff
  \item How are training set error and test set error related?
  \item When we use a high variance model that is prone to
    overfitting, how can we control it?
  \end{itemize}

\end{frame}

%======================================================================
\section{Bias-variance tradeoff}
%======================================================================

\begin{frame}{Bias-Variance tradeoff}{Test set error}

  In machine learning, now you see that the general approach is to
  \alert{fit} a model $h()$ to a \alert{training dataset} then
  \alert{test} the model on a \alert{test dataset}.

  \medskip

  Consider the linear regression case. After estimating $h_{\vec{\theta}}$,
  we compute the mean squared error over a test set:
  \[ \Expec_{(\vec{x},y) \sim \text{test set}} (h_{\vec{\theta}}(\vec{x})-y)^2. \]

  \medskip

  What are the possible cause of the test set error being \alert{too high}?
  \begin{itemize}
  \item \alert{Overfitting}: the model is too closely tailored to the examples
    in the training set and can't generalize to the test set.
  \item \alert{Underfitting}: the model doesn't capture the link between the
    features $\vec{x}$ and the target $y$.
  \item \alert{Noise}: the data are inherently noisy, and no learning method
    is going to give us lower error on a test set.
  \end{itemize}

\end{frame}

\begin{frame}{Bias-Variance tradeoff}{Randomness of $h_{\vec{\theta}}()$}

  To formalize this intuition, we first suppose that the training and test
  data are all sampled from a distribution
  \[ y = f(\vec{x}) + \varepsilon, \]
  where $\Expec(\varepsilon) = 0$ and $\var(\varepsilon) = \sigma^2$.

  \medskip

  We use a training set to estimate $h_{\vec{\theta}}()$ then apply to each pattern
  $j$ in the test set.

  \medskip

  Our \alert{prediction} of the $y^{(j)}$ for $\vec{x}^{(j)}$ (note that $y^{(j)}$ is equal to
  $f(\vec{x}^{(j)})+\varepsilon^{(j)}$) is $h_{\vec{\theta}}(\vec{x}^{(j)})$.

  \medskip

  Clearly, $\vec{x}^{(j)}$ is fixed when we compute
  $h_{\vec{\theta}}(\vec{x}^{(j)})$, but
  $h_{\vec{\theta}}(\vec{x}^{(j)})$ itself is \alert{random}, since it
  depends on the values $\varepsilon^{(i)}$ in the training set.

\end{frame}

\begin{frame}{Bias-Variance tradeoff}{Bias and variance}

  For linear regression, we define the \alert{bias} of
  $h_{\vec{\theta}}()$ as
  \[ \Expec(h_{\vec{\theta}}(\vec{x})-f(\vec{x})) \]
  and the \alert{variance} of $h_{\vec{\theta}}()$ as
  \[ \Expec \left( (h_{\vec{\theta}}(\vec{x}) - f(\vec{x}))^2 \right). \]

  \medskip

  There are various formal notions of bias and variance for
  classifiers, but there is no general agreement on which one is best.

\end{frame}

\begin{frame}{Bias-Variance tradeoff}{Bias and variance}

  Mean squared error on the test data set can be decomposed as
  \begin{eqnarray}
    \text {Test MSE} & = & \Expec\left( (y - h_{\vec{\theta}}(\vec{x}))^2 \right) \nonumber \\
    & = & \Expec\left( (
    f(\vec{x}) + \varepsilon - h_{\vec{\theta}}(\vec{x})
    )^2 \right) \nonumber \\
    & = & \Expec (\varepsilon^2) + \Expec\left( (f(\vec{x}) - h_{\vec{\theta}}(\vec{x}))^2 \right) \label{squared-sum-eq} \\
    & = & \sigma^2 + \left(\Expec(f(\vec{x})-h_{\vec{\theta}}(\vec{x}))\right)^2
    + \var\left(f(\vec{x})-h_{\vec{\theta}}(\vec{x})\right) \label{squared-eq} \\
    & = & \sigma^2 + \left(\text{Bias } h_{\vec{\theta}}(\vec{x})\right)^2 +
    \var\left( h_{\vec{\theta}}(\vec{x}) \right) \nonumber
  \end{eqnarray}

\end{frame}

\begin{frame}{Bias-Variance tradeoff}{Bias and variance}

  How to get Equation 1?

  \medskip

  We have $\Expec((X_1+X_2)^2)$ where $X_1 = \varepsilon$ and $X_2 =
  f(\vec{x})-h_{\vec{\theta}}(\vec{x})$.

  \medskip

  $\Expec((X_1+X_2)^2) = \Expec(X_1^2+X_2^2+2X_1X_2) =
  \Expec(X_1^2)+\Expec(X_2^2)+2\Expec(X_1X_2)$.

  \medskip

  Since $X_1$ and $X_2$ are independent we have $\Expec(X_1X_2) =
  \Expec(X_1)\Expec(X_2)$. Since $\Expec(\varepsilon) = 0$, in this
  special case, we get $\Expec((X_1+X_2)^2) =
  \Expec(X_1^2)+\Expec(X_2^2)$.

  \medskip

  How to get Equation 2? Start with the definition of variance:
  \begin{eqnarray*}
    \var(X) & = & \Expec((X-\Expec(X))^2) \\
            & = & \Expec(X^2-2X\Expec(X)+(\Expec(X))^2) \\
            & = & \Expec(X^2)-2\Expec(X)\Expec(X)+\Expec(X)\Expec(X) \\
            & = & \Expec(X^2) - (\Expec(X))^2 \\
    \Expec(X^2) & = & (\Expec(X))^2 + \var(X)
  \end{eqnarray*}

\end{frame}

\begin{frame}{Bias-Variance tradeoff}{Bias and variance}

  We cannot predict the \alert{noise} $\varepsilon$.

  \medskip

  The \alert{bias} term in the MSE is due to \alert{underfitting}.

  \medskip

  The \alert{variance} term of the MSE is related to \alert{overfitting}.

  \medskip

  We could say that our grand goal is to find the \alert{sweet spot} of no
  underfitting and no overfitting, in which bias and variance are both
  negligible.

  \medskip

  Unfortunately, in practice, when we reduce bias we usually increase
  variance, and vice versa.

\end{frame}

\begin{frame}{Bias-Variance tradeoff}{Bias and variance}

  As an example, consider the case of linear regression with a single
  input variable.

  \medskip

  We might try models with degree 1, degree 2, and degree 5 in the input variable:

  \medskip

  \myfig{3.5in}{over-under}{Ng (2017), CS229 lecture notes}

  \medskip

  The degree 1 model has high bias, and the degree 5 model has high
  variance.

\end{frame}

%======================================================================
\section{Probabilistic preliminaries}
%======================================================================

\begin{frame}{Probabilistic preliminaries}{Important bounds}

  This section is directly from Ng (2017), \textit{Learning Theory}.

  \medskip

  To get started in understanding the generalization ability of a
  learning model, we need some preliminary facts useful for the
  analysis: the \alert{union bound} and the \alert{Hoeffding
    inequality}.

  \medskip

  \begin{block}{The \alert{union bound}}
    Let $A_1, A_2, \ldots, A_k$ be $k$ different events resulting from
    a probabilistic experiment that are not necessarily independent.
    \[ P(A_1 \cup \cdots \cup A_k) \le P(A_1)+\cdots+P(A_k). \]
  \end{block}

  \medskip

  The union bound follows from Kolmomgorov's axioms of probability
  theory and is sometimes considered an axiom itself.

\end{frame}

\begin{frame}{Probabilistic preliminaries}{Important bounds}

  \begin{block}{The \alert{Hoeffding inequality} or the \alert{Chernoff bound}}

    \begin{enumerate}
    \item Let $Z_1, \ldots, Z_m$ be $m$ independent and identically
      distributed random variables drawn from a Bernoulli($\phi$)
      distribution (i.e., $P(Z_i=1)=\phi, P(Z_i=0)=1-\phi$).
    \item Let $\hat{\phi} = \frac{1}{m}\sum_{i=1}^m Z_i$ be the mean
      of the $Z_i$.
    \item Let $\gamma > 0$ be fixed.
    \end{enumerate}

    Then
    \[ P(|\phi-\hat{\phi}|>\gamma) \le 2 e^{-2\gamma^2m}. \]
  \end{block}

  \medskip

  Essentially, this says that the average of $m$ Bernoulli($\phi$) variables
  will be very close to $\phi$ as long as $m$ is large.

\end{frame}

\begin{frame}{Probabilistic preliminaries}{Empirical risk and generalization error}

  Now, let's restrict ourselves to binary classification with labels
  $y \in \{0,1\}$.

  \medskip

  We are given a training set $S = \{(\vec{x}^{(i)},y^{(i)})\}_{i \in 1..m }$ of
  size $m$ drawn i.i.d.\ from some distribution $\cal D$.

  \medskip

  The \alert{training error} or \alert{empirical risk} or
  \alert{empirical error} of a hypothesis $h$ is defined as
  \[ \hat{\varepsilon}(h) = \frac{1}{m}\sum_{i=1}^m \delta(h(\vec{x}^{(i)}) \ne y^{(i)}). \]

  \medskip

  The \alert{generalization error} is defined as
  \[ \varepsilon(h) = P_{(\vec{x},y)\sim \cal D}(h(\vec{x})\ne y). \]

  \medskip

  The assumption that the \alert{training and test distributions are
    the same} is one of the \alert{PAC} (probably approximately
  correct) assumptions.

\end{frame}

\begin{frame}{Probabilistic preliminaries}{ERM}

  How to determine the best $h$?

  \medskip

  One way, called \alert{empirical risk minimization} (ERM) says to
  pick the $h$ minimizing $\hat{\varepsilon}(h)$.

  \medskip

  For example, for the linear classifier
  \[ h_{\vec{\theta}}(\vec{x}) = \delta(\vec{\theta}^\top \vec{x} \ge 0),\]
  we pick
  \[ \hat{\vec{\theta}} = \argmin_{\vec{\theta}}\hat{\varepsilon}(h_{\vec{\theta}}) . \]
  and output $\hat{h}=h_{\hat{\vec{\theta}}}$.

  \medskip

  Note: algorithms such as logistic regression are not exactly ERM but
  can be thought of as approximations to ERM.

\end{frame}

\begin{frame}{Probabilistic preliminaries}{ERM}

  Now let's generalize beyond linear classifiers.

  \medskip

  Any learning algorithm will consider some set of
  hypotheses $\cal H$.

  \medskip

  Example: linear classifiers are defined by
  \[ {\cal H} = \{ h_{\vec{\theta}} : h_{\vec{\theta}}(\vec{x}) =
  \delta(\vec{\theta}^\top \vec{x} \ge 0), \vec{\theta} \in \Rset^{n+1}\}.
  \]

  Alternatively, ${\cal H}$ might be the set of all classifiers
  represented by a particular neural network architecture.

  \medskip

  Once we have $\cal H$, we can define ERM as determining
  \[ \hat{h} = \argmin_{h\in\cal H}\hat{\varepsilon}(h). \]

\end{frame}

%======================================================================
\section{Bounds on generalization error for finite $\cal H$}
%======================================================================

\begin{frame}{Bounds on generalization error for finite $\cal H$}

  We begin by assuming ${\cal H} = \{ h_1, \ldots, h_k \}$ is finite with $k$
  possible hypotheses.

  \medskip

  We have $k$ functions mapping $\cal X$ to $\{0,1\}$.

  \medskip

  ERM selects the $h_i$ minimzing $\hat{\varepsilon}(h_i)$.

  \medskip

  What we can do:
  \begin{itemize}
  \item Show that $\hat{\varepsilon}(h)$ is a reliable estimate
    of $\varepsilon(h)$ for all $h$.
  \item Bound the generalization error of $\hat{h}$.
  \end{itemize}

\end{frame}

\begin{frame}{Bounds on generalization error for finite $\cal H$}{Uniform convergence}

  Take a particular hypothesis $h_i \in \cal H$.

  \medskip

  Consider a Bernoulli random variable $Z$ whose distribution is
  defined as follows:
  \begin{itemize}
  \item Sample $(\vec{x},y) \sim \cal D$.
  \item Set $Z = \delta(h_i(\vec{x}) \ne y)$.
  \end{itemize}

  \medskip

  $Z$ indicates whether $h_i$ misclassifies $(\vec{x},y)$.

  \medskip

  Next, define $Z_j = \delta(h_i(\vec{x}^{(j)}) \ne y^{(j)})$.

  \medskip

  Since the training set $S$ was drawn i.i.d.\ from $\cal D$ and so
  was $(\vec{x},y)$, we know that $Z$ and Z$_j$ have the same
  distribution.

\end{frame}

\begin{frame}{Bounds on generalization error for finite $\cal H$}{Uniform convergence}

  The expected value of $Z$ is the probability of misclassification of
  a randomly drawn example, which is $\varepsilon(h_i)$.

  \medskip

  As $Z$ and $Z_j$ have the same distribution, the expected value of $Z_j$
  is also $\varepsilon(h_i)$.

  \medskip

  We can write the training error as
  \[ \hat{\varepsilon}(h_i) = \frac{1}{m} \sum_{j=1}^m Z_j .\]
  We have the mean $\hat{\epsilon}(h_i)$ of $m$ random variables $Z_j$,
  all drawn i.i.d. from a Bernoulli distribution with mean
  $\varepsilon(h_i)$.

  \medskip

  Now we can apply the Hoeffding inequality! We obtain
  \[ P(|\varepsilon(h_i)-\hat{\varepsilon}(h_i)| > \gamma) \le 2 e^{-2\gamma^2m} .\]

\end{frame}

\begin{frame}{Bounds on generalization error for finite $\cal H$}{Uniform convergence}

  What have we shown?

  \medskip

  For our $h_i$, \alert{training error will be close to generalization
    error} with probability that increases with $m$.

  \medskip

  What about \alert{all possible $h_i$}?

  \medskip

  Well, let $A_i$ denote the event that $|\varepsilon(h_i)-\hat{\varepsilon}(h_i)| > \gamma$.

  \medskip

  For any $A_i$, we have $P(A_i) \le 2 e^{-2\gamma^2m}$.

\end{frame}

\begin{frame}{Bounds on generalization error for finite $\cal H$}{Uniform convergence}

  Now, we can use the union bound to bound generalization error for
  \alert{any} $h_i$:
  \begin{eqnarray*}
    P(\exists h \in {\cal H}.|\varepsilon(h_i)-\hat{\varepsilon}(h_i)|>\gamma)
    & = & P(A_1 \cup \cdots \cup A_k) \\
    & \le & \sum_{i=1}^k P(A_i) \\
    & \le & \sum_{i=1}^k 2 e^{-2\gamma^2m} \\
    & = & 2 k e^{-2\gamma^2 m} .
  \end{eqnarray*}
  Subtracting both sides from 1, we obtain
  \begin{eqnarray*}
    P(\neg \exists h \in {\cal H}.|\varepsilon(h_i)-\hat{\varepsilon}(h_i)|
    > \gamma) & = & P(\forall h\in {\cal H}.|\varepsilon(h_i) - \hat{\varepsilon}(h_i)| \le \gamma) \\
    & \le & 1 - 2 k e^{-2\gamma^2 m} .
  \end{eqnarray*}

\end{frame}

\begin{frame}{Bounds on generalization error for finite $\cal H$}{Using uniform convergence}

  The previous result is called the \alert{uniform convergence} result.

  \medskip

  The bound holds simultaneously for all $h \in {\cal H}$.

  \medskip

  We can use this, for example, to answer the question, ``Given
  $\gamma$ and some $\delta>0$, find $m$ large enough to guarantee
  that with probability at least $1-\delta$, training error will be
  within $\gamma$ of generalization error.''

  \medskip

  To do it, let $\delta = 2k e^{-2\gamma^2 m}$ and solve for $m$ to find that
  we should use
  \[ m \ge \frac{1}{2\gamma^2}\log\frac{2k}{\delta} .\]

  Now we know \alert{how many training examples are needed to make a
    guarantee about generalization error}. Note that $m$ increases in
  the \alert{logarithm of $k$}!!

  \medskip

  The training set size $m$ needed to achieve a level of performance
  is called the algorithm's \alert{sample complexity}.

\end{frame}

\begin{frame}{Bounds on generalization error for finite $\cal H$}{Using uniform convergence}

  We can also hold $m$ and $\delta$ fixed and solve for $\gamma$:
  \[ |\hat{\varepsilon}(h) - \varepsilon(h)| \le \sqrt{\frac{1}{2m}\log\frac{2k}{\delta}} .\]

  \medskip

  [Think about how you could use this fact...]

\end{frame}

\begin{frame}{Bounds on generalization error for finite $\cal H$}{The bound}

  Now to prove something about the generalization error
  obtained by picking
  \[ \hat{h} = \argmin_{h\in \cal H} \hat{\varepsilon}(h) .\]
  We have $|\varepsilon(h)-\hat{\varepsilon}(h)| \le \gamma$ for all
  $h \in \cal H$.

  \medskip

  Let $h^* = \argmin_{h \in \cal H} \varepsilon(h)$ be the best possible
  hypothesis in $\cal H$.

  \medskip

  Applying uniform convergence (twice) and the fact that
  $\hat{h} = \argmin_{h \in H}\hat{\varepsilon}(h)$, we can obtain
  \begin{eqnarray*}
    \varepsilon(\hat{h}) & \le & \hat{\varepsilon}(\hat{h}) + \gamma \\
    & \le & \hat{\varepsilon}(h^*) + \gamma \\
    & \le & \varepsilon(h^*)+2\gamma
  \end{eqnarray*}

  \medskip

  \alert{The test error of the minimum risk classifier is at most
  $2\gamma$ worse than the best possible classifier!}

\end{frame}

\begin{frame}{Bounds on generalization error for finite $\cal H$}{Bounded generalization error}

  \textbf{Theorem}. Let $|{\cal H}| = k$, and let any $m, \delta$ be
  fixed. Then with probability at least $1-\delta$, we have that
  \[ \varepsilon(\hat{h}) \le \left( \min_{h\in \cal H} \varepsilon(h) \right)
  + 2 \sqrt{\frac{1}{2m}\log\frac{2k}{\delta}} .\]

  \medskip

  Prove this by letting $\gamma = \sqrt{\cdot}$ and apply the previous
  argument.

\end{frame}

\begin{frame}{Bounds on generalization error for finite $\cal H$}{The bound's implications for bias and variance}

  We have not formally defined bias and variance for a classifier, but
  the first term in the bound is a rough bias for $\hat{h}$, and the
  second term is a rough variance.

  \medskip

  Suppose we have a hypothesis class $\cal H$.

  \medskip

  If we consider a larger hypothesis class
  ${\cal H}' \supseteq \cal H$:
  \begin{itemize}
    \item We will improve our bias
      ($\min_{h \in \cal H}\varepsilon(h)$ can only get smaller),
    \item We will increase $k$, thus increasing the variance.
  \end{itemize}

\end{frame}

\begin{frame}{Bounds on generalization error for finite $\cal H$}{Bounded sample complexity}

  If we hold $\gamma$ and $\delta$ fixed, obtain a bound on sample
  complexity.

  \medskip

  \textbf{Corollary}. Let $|{\cal H}| = k$, and let any
  $\delta,\gamma$ be fixed. Then for $\varepsilon(\hat{h}) \le \min_{h
    \in \cal H} \varepsilon(h) + 2 \gamma$ to hold with probability
  at least $1-\delta$, it suffices that
  \begin{eqnarray*}
    m & \ge & \frac{1}{2\gamma^2}\log\frac{2k}{\delta} \\
    & = & O\left(\frac{1}{\gamma^2}\log\frac{k}{\delta}\right) .
  \end{eqnarray*}

\end{frame}

%======================================================================
\section{Bounds on generalization error for infinite $\cal H$}
%======================================================================

\begin{frame}{Bounds on generalization error for infinite $\cal H$}{Motivation}

  We've seen some interesting results for $|{\cal H}| = k$.

  \medskip

  But most of the models we've looked at in this class have
  $|{\cal H}| = \infty$.

  \medskip

  What can we do??

\end{frame}

\begin{frame}{Bounds on generalization error for infinite $\cal H$}{A not-so-good approach}

  To begin with, imagine our parameter vector $\vec{\theta}$ consists
  of $d$ real numbers represented as double-precision (64 bit) IEEE
  floating point numbers.

  \medskip

  Our hypothesis class thus consists of at most $k = 2^{64d}$ hypotheses.

  \medskip

  To guarantee $\varepsilon(\hat{h}) \le \varepsilon(h^*)+2\gamma$ to
  hold with probability at least $1-\delta$ we need
  \[ m \ge O\left(\frac{1}{\gamma^2}\log\frac{2^{64d}}{\delta}\right) =
  O\left(\frac{d}{\gamma^2}\log\frac{1}{\delta}\right) =
  O_{\gamma,\delta}(d) .\]

  \medskip

  So the number of training examples is \alert{linear in the number of
    parameters in the model} if we use finite floating point
  representations of real numbers and manage to implement ERM over
  this space.

\end{frame}

\begin{frame}{Bounds on generalization error for infinite $\cal H$}{A better approach}

  The approach based on finite floating point approximations of
  $\vec{\theta} \in \Rset^d$ depends on the \alert{parameterization of
    the hypothesis}.

  \medskip

  There is a better approach based on the \alert{characteristics
    feature space}.

  \medskip

  \textbf{Definition}: Given a set $S = \{
  \vec{x}^{(1)},\ldots,\vec{x}^{(d)} \}$ of points $\vec{x}^{(i)} \in
  \cal X$, we say that $\cal H$ \alert{shatters} $S$ if $\cal H$ can
  realize \alert{any labeling on $S$}.

  \medskip

  \textbf{Definition}: Given a hypothesis class $\cal H$, we define
  the \alert{Vapnick-Chervonenkis dimension} VC($\cal H$) as the largest
  set that is shattered by $\cal H$.

\end{frame}

\begin{frame}{Bounds on generalization error for infinite $\cal H$}{VC dimension}

  What is the VC dimension of a linear classifier in $\Rset^2$?

  \medskip

  \myfig{1.5in}{shatter-points}{Ng (2017), CS229 lecture notes.}

  \medskip

  Can these three points be shattered?

\end{frame}

\begin{frame}{Bounds on generalization error for infinite $\cal H$}{VC dimension}

  The answer is yes:

  \medskip

  \myfig{4.5in}{shattering}{Ng (2017), CS229 lecture notes.}

\end{frame}

\begin{frame}{Bounds on generalization error for infinite $\cal H$}{VC dimension}

  Note that for VC dimension to be $d$, we have to be able to shatter
  \alert{some} point set of size $d$, \alert{not all} point sets:

  \medskip

  \myfig{3in}{unshatterable-points}{Ng (2017), CS229 lecture notes.}

\end{frame}

\begin{frame}{Bounds on generalization error for infinite $\cal H$}{Bounded generalization error}

  \textbf{Theorem} (Vapnik). Let $\cal H$ be given, and let $d =
  \text{VC}({\cal H})$. Then with probability at least $1-\delta$, we
  have that for all $h \in \cal H$,
  \[ |\varepsilon(h)-\hat{\varepsilon}(h)| \le O\left( \sqrt{\frac{d}{m}\log\frac{m}{d}+\frac{1}{m}\log\frac{1}{\delta}} \right) .\]
  This also means
  \[ \varepsilon(\hat{h}) \le \varepsilon(h^*) + O\left( \sqrt{\frac{d}{m}\log\frac{m}{d}+\frac{1}{m}\log\frac{1}{\delta}} \right) .\]

  We have uniform convergence in $m$ so long as $d$ is finite.

  \medskip

  This is considered by many \alert{the most important theorem} in
  learning theory!

\end{frame}

\begin{frame}{Bounds on generalization error for infinite $\cal H$}{Bounded sample complexity}

  \textbf{Corollary}. For $|\varepsilon(h)-\hat{\varepsilon}(h) \le
  \gamma$ to hold for all $h \in \cal H$, so that
  $\varepsilon(\hat{h}) \le \varepsilon(h^*) + 2\gamma)$ with
  probability at least $1-\delta$, it suffices that $m =
  O_{\gamma,\delta}(d)$.

  \medskip

  The number of training examples needed to learn ``well'' using $\cal
  H$ is linear in the VC dimension of $\cal H$.

  \medskip

  For most hypothesis classes, VC dimension is roughly linear in the
  number of parameters.

  \medskip

  This means that for most classifiers, the number of training examples
  needed is usually roughly linear in the number of parameters of $\cal H$.

\end{frame}

%======================================================================
\section{Conclusion on generalization bounds}
%======================================================================

\begin{frame}{Conclusion on generalization bounds}

  Now we've seen the most important results in learning theory.

  \medskip

  Generally, the story is encouraging: \alert{we can achieve near
    perfection} for most models just by \alert{gathering sufficient
    data}.

  \medskip

  But the devil is in the details. For example:
  \begin{itemize}
  \item SVMs with finite $\phi(\vec{x})$ have finite VC dimension, but
    if $\phi(\vec{x})$ maps $\vec{x}$ to an infinite dimensional space (like
    the RBF kernel does) then the VC dimension is \alert{infinite}.
  \item We find that the $m$ needed to achieve a particular level of
    generalization error is linear in $\text{VC}({\cal H})$. But what
    is the constant of proportionality? (Getting data has a cost, and
    a factor of 100 would be a lot bigger than 10!)
  \item CNNs with millions of parameters can be trained effectively
    with thousands of examples. How is it possible? Are deep learning
    models cheating Vapnik somehow?
  \end{itemize}

\end{frame}

\begin{frame}{Conclusion on generalization bounds}

  Deep learning is particularly interesting here.

  \medskip

  Neural network architectures' VC dimension is $O(|\mat{W}|)$ in the best
  case.

  \medskip

  So a model with millions of parameters needs millions of training
  examples to \alert{guarantee} good generalization according to
  Vapnik.

  \medskip

  However, note that Vapnik's theorem is a theorem of
  \alert{sufficiency}, not necessity. Techniques we have studied help
  us cheat Vapnik:
  \begin{itemize}
  \item Stopping training when validation set cost is minimal
  \item Weight decay and dropout (regularization)
  \end{itemize}

  \medskip

  There is a really interesting take from astrophysicists' perspective
  by Lin, Tegmark, and Roland (2016) at
  \url{https://arxiv.org/abs/1608.08225} that provides a interesting
  theory of how deeper models may better due to the hierarchical
  structure of real world generative processes.

  \medskip

  These are topics needing more research!

\end{frame}

%======================================================================
\section{Model selection}
%======================================================================

\begin{frame}{Model selection}{Finding the best model}

  OK, so now we understand that for a hard problem, the ``right''
  model, the one that is guaranteed to generalize to the test set,
  might be \alert{too biased}.

  \medskip

  We also understand that a less biased model may have VC dimension
  too high for the number of examples we have and will thus be
  \alert{prone to overfitting} (too much variance).

  \medskip

  How can we find the ``best'' model? For example,
  \begin{itemize}
  \item In polynomial regression $h_{\vec{\theta}}(x) = g(\theta_0 + \theta_1 x
    + \theta_2 x^2 + \cdots + \theta_k x^k)$, how to decide the right $k$?
  \item In locally-weighted regression, how to choose the bandwidth
    parameter $\tau$?
  \item With SVMs, how to choose $C$ (and $\gamma$ for the RBF
    kernel)?
  \end{itemize}

\end{frame}

\begin{frame}{Model selection}{Cross validation}

  In each of these cases, we have a set of models ${\cal M} = \{ M_1,
  \ldots, M_d\}$ we would like to select from.

  \medskip

  From ERM we might train each $M_i$ and pick the $M_i$ with the lowest
  training error.

  \medskip

  But this will give us a high-variance model that doesn't test well.

  \medskip

  Our main alternative (using a validation set) is called \alert{cross
    validation}:
  \begin{itemize}
  \item Split the training set $S$ into $S_{train}$ and $S_{validation}$ using
    a ratio such as 70\% to 30\% or 80\% to 20\%.
  \item Train each model $M_i$ on $S_{train}$.
  \item Select the $M_i$ with lowest error on $S_{validation}$.
  \item (Optional) Re-train $M_i$ on all of $S$.
  \item Use $M_i$.
  \end{itemize}

\end{frame}

\begin{frame}{Model selection}{$k$-fold cross validation}

  When data is scarse, we'd prefer to select the model that is best on
  \alert{all} the data, not just 30\% of the data!

  \medskip

  Then \alert{$k$-fold cross validation} makes more sense:
  \begin{enumerate}
  \item Split $S$ into $k$ disjoint subsets $S_1,\ldots,S_k$.
  \item For each $M_i$
    \begin{itemize}
    \item For $j = 1, \ldots, k$
      \begin{enumerate}
      \item Train $M_i$ on $S_{train} = S_1,\ldots,S_{j-1},S_{j+1},\ldots,S_k$
      \item Test the trained $M_i$ on $S_{validation} = S_j$
      \end{enumerate}
    \end{itemize}
    \item Select the $M_i$ with the lowest average error on
      $S_{validation}$ over the $k$ folds.
    \item Retrain $M_i$ on all of $S$.
  \end{enumerate}

  \medskip

  Typical values of $k$ are 5, 10, and $m$ (\alert{leave-one-out cross
    validation}).

\end{frame}

\begin{frame}{Model selection}{$k$-fold cross validation}

  Be careful about \alert{how you split}!
  \begin{itemize}
  \item Usually, the partitioning of examples into $k$ folds is random
    uniform.
  \item However, sometimes training items are related to each other
    (e.g.\ multiple crops of the same object in the same image).
  \item Randomized partitioning will put some related examples in
    different folds, leading to overestimated performance.
  \item In this case, it's necessary to ensure that the related
    examples are \alert{in the same fold}.
  \end{itemize}

\end{frame}

\begin{frame}{Model selection}{Feature selection}

  A special case of model selection is \alert{feature selection}.

  \medskip

  Suppose you have many features, possibly even $n \gg
  m$.\footnote{This is possible in some domains such as biological
    sequence analysis, where we might want to find segments of DNA
    sequences in a particular genome that code for a particular
    biological function, or to a lesser extent in text
    classification.}

  \medskip

  We know that the VC dimension of a classifier with very large $n$
  will be too high.

  \medskip

  So we treat the problem as model selection: among the $2^n$ possible
  subsets of features, \alert{find the subset that gives the best cross
  validation performance}.

  \medskip

  Trying all $2^n$ subsets is impossible unless $n$ is very small. So
  we need a more efficient approach.

\end{frame}

\begin{frame}{Model selection}{Forward feature selection}

  The \alert{forward search} method of feature selection:
  \begin{enumerate}
  \item Initialize ${\cal F} = \emptyset$.
  \item For $j \in \{ 1, \dots, n \}$
    \begin{enumerate}
    \item For $i \in \{ 1, \ldots, n \} - {\cal F}$
      \begin{enumerate}
      \item Let ${\cal F}_i = {\cal F} \cup \{ i \}$
      \item Train on ${\cal F}_i$ with cross validation and obtain the
        cross validation error
      \end{enumerate}
    \item Let ${\cal F} = {\cal F} \cup \{ i \}$, where $i$ is the
      feature that gave the lowest cross validation error.
    \end{enumerate}
    \item Select the feature set that gave the lowest cross validation error
    over all tests.
  \end{enumerate}

\end{frame}

\begin{frame}{Model selection}{Other wrapper methods}

  When we have a method that \alert{wraps} our learning algorithm,
  supplying a different feature set on each iteration, this is called
  \alert{wrapper model selection}.

  \medskip

  Another wrapper method is \alert{backward search}, in which we start
  with ${\cal F} = \{ 1, \ldots, n \}$ and \alert{iteratively remove
    the least informative feature}.

  \medskip

  Wrapper methods work well but take a lot of time, for example,
  $O(kn^2)$ calls to the optimization algorithm for forward selection
  with $k$-fold cross validation.

\end{frame}

\begin{frame}{Model selection}{Filter feature selection}

  \alert{Filter feature selection} methods use a heuristic to decide
  which subsets of features to try.

  \medskip

  Example: we may \alert{rank} features according to some measure of
  relatedness to the desired output, then incrementally add features
  in that order if they improve generalization.

  \medskip

  Most common measure of relatedness for discrete features:
  \alert{mutual information} $MI(X_i,Y)$ between feature $X_i$ and
  target $Y$:
  \[ MI(X_i,Y) = \sum_{x_i\in X_i} \sum_{y\in Y} p(x_i,y)\log\frac{p(x_i,y)}{p(x_i)p(y)} \]
  [Think about the value of $MI(X_i,Y)$ if $X_i$ and $Y$ are
    independent or identical and why that would be a good indication
    of relatedness.]

\end{frame}

%======================================================================
\section{Regularization}
%======================================================================

\begin{frame}{Regularization}{Frequentist vs.\ Bayesian statistics}

  We've seen several specific techniques for \alert{regularization}
  with different models so far.

  \medskip

  Here we'll discuss the general framework of Bayesian statistics and
  how the Bayesian perspective on parameter estimation can be used for
  regularization.

  \medskip

  The \alert{frequentist} perspective in statistics is that our
  parameter vector $\vec{\theta}$ is \alert{constant but unknown}.

  \medskip

  Maximum likelihood is a general technique to identify such a
  constant but unknown parameter.

  \medskip

  The \alert{Bayesian} perspective in statistics is that
  $\vec{\theta}$ is itself a \alert{random variable} whose value is
  unknown.

\end{frame}

\begin{frame}{Regularization}{Bayesian estimation}

  The frequentist approach of maximum likelihood says that given $S =
  \{ (\vec{x}^{(i)},y^{(i)})\}_{i=1..m}$, we should find
  \[ \vec{\theta}_{ML} = \argmax_{\vec{\theta}} \prod_{i=1}^m p(y^{(i)} \mid \vec{x}^{(i)}; \vec{\theta}) .\]
  The Bayesian approach says we should consider the \alert{posterior distribution over the parameters}
  \begin{eqnarray*}
    p(\vec{\theta} \mid S) & = & \frac{p(S \mid \vec{\theta}) p(\vec{\theta})}{p(S)} \\
    & = & \frac{\left(\prod_{i=1}^m p(y^{(i)}\mid \vec{x}^{(i)}, \vec{\theta})\right) p(\vec{\theta})}{\int_{\vec{\theta}}\left( \prod_{i=1}^m p(y^{(i)}\mid \vec{x}^{(i)},\vec{\theta}) p(\vec{\theta})\right) d\vec{\theta}}
  \end{eqnarray*}
  when making a decision.

\end{frame}

\begin{frame}{Regularization}{Fully Bayesian prediction}

  Both probabilities $p(y^{(i)}\mid \vec{x}^{(i)}; \vec{\theta})$
  (frequentist) and $p(y^{(i)}\mid \vec{x}^{(i)}, \vec{\theta})$
  (Bayesian) are given by the model.

  \medskip

  For logistic regression, for example, we would use
  $h_{\vec{\theta}}(\vec{x}^{(i)})^{y^{(i)}}(1-h_{\vec{\theta}}(\vec{x}^{(i)}))^{1-y^{(i)}}$ with
  $h_{\vec{\theta}}(\vec{x}^{(i)}) = \frac{1}{1+e^{\vec{\theta}^{\top} \vec{x}^{(i)}}}$.

  \medskip

  Decision making would treat $\vec{\theta}$ as random:
  \[ p(y \mid \vec{x}, S) = \int_{\vec{\theta}} p(y \mid \vec{x},\vec{\theta})
  p(\vec{\theta} \mid S) d\vec{\theta} \]
  [How did we get that??]

  \medskip

  Then our answer might be the expected value of $y$ given $\vec{x}$:
  \[ \Expec[y \mid \vec{x},S] = \int_y y p(y \mid \vec{x},S) dy \]

\end{frame}

\begin{frame}{Regularization}{MAP prediction}

  Usually, the full Bayesian predictor on the previous slide is too
  hard to compute in practice.

  \medskip

  Instead, we can go with a single point estimate for $\vec{\theta}$,
  leading to the \alert{maximum a posteriori} (MAP) estimate of
  $\vec{\theta}$:
  \[ \vec{\theta}_{MAP} = \argmax_{\vec{\theta}} \prod_{i=1}^m p(y^{(i)} \mid \vec{x}^{(i)},\vec{\theta}) p(\vec{\theta}) . \]
  Note that this is the same as the ML estimate except for the
  \alert{prior term} $p(\vec{\theta})$.  One possible prior is
  $\vec{\theta} \sim {\cal N}(\vec{0},\tau^2 \mat{I})$.

  \medskip

  This will assign higher scores to models with smaller parameter
  vectors, reducing the model's variance.

  \medskip

  \alert{Bayesian logistic regression} works well for many problems
  such as text classification where $n \gg m$.

\end{frame}

\end{document}


\documentclass[11pt]{article}
\usepackage{pgfplots}
\usepackage{amssymb}
\usepackage{url}
\pgfplotsset{compat=newest}
\setlength\topmargin{-0.6cm}
\setlength\textheight{23.4cm}
\setlength\textwidth{17.0cm}
\setlength\oddsidemargin{0cm}
\begin{document}
\title{Ling 572 HW10}
\author{Daniel Campos  \tt {dacampos@uw.edu}}
\date{03/20/2019}
\maketitle
\section{Q1}
\begin{table}[h]
\centering
\caption{Classification accuracy with {\bf sigmoid} activation function}
\label{table1}
  \begin{tabular}{|c|r|l|l|l|l|l|} \hline
    Expt & \# of          & \# of neurons in & \# of    & mini-batch & test  & CPU time   \\
    id   &  hidden layer  & hidden layers    & epoches & size       & accuracy (\%) & minutes,seconds \\ \hline
    1   &  1              &  30              &  30     &  10       &  73.24   & 38s \\ \hline
    2   &  1              &  30              &  30     &  50       &  60.86   & 50s \\ \hline
    3   &  1              &  30              &  100    &  10       &  78.76   & 2m 3s  \\ \hline
    4   &  1              &  60              &  30     &  10       &  73.1429   & 53s \\ \hline
    5   &  2              &  30, 30          &  30     &  10       &  73.81  & 46s \\ \hline
    6   &  2              &  40, 20          &  30     &  10       &  72.29   & 50s\\ \hline
    7   &  3              &  20, 20, 20      &  30     &  10       &  70.76  & 53s \\ \hline
  \end{tabular}
\end{table}
\section{Q2}
For Q2 I changed two functions:$tanh$ and $tanh\_prime$ on line $160$ and $165$ in network.py. First, I implemented the code with the unsimplified methods(full equations).\\ Next, using the identities learned in class and HW9($tanh(z) = 2*sigmoid(2*z) -1$ and $tanh' (z)= (1-tanh(z)*tanh(z))$), I modified the functions to use the sigmoid and sigmoid prime methods for ease of use. \\ In general, we see the sigmoid function as a better fit for this data and the addition of more training epochs proving the highest accuracy at the expense of training time.
\begin{table}[h]
\centering
\caption{Classification accuracy with {\bf tanh} activation function}
\label{table1}
  \begin{tabular}{|c|r|l|l|l|l|l|} \hline
    Expt & \# of          & \# of neurons in & \# of    & mini-batch & test  & CPU time   \\
    id   &  hidden layer  & hidden layers    & epoches & size       & accuracy (\%) & minutes,seconds \\ \hline
    1   &  1              &  30              &  30     &  10       &  72.19   & 35s\\ \hline
    2   &  1              &  30              &  30     &  50       &  59.90   & 48s\\ \hline
    3   &  1              &  30              &  100    &  10       &  75.90   & 3m 16 s \\ \hline
    4   &  1              &  60              &  30     &  10       &  63.14   & 56s \\ \hline
    5   &  2              &  30, 30          &  30     &  10       &  73.52   & 38s\\ \hline
    6   &  2              &  40, 20          &  30     &  10       &  62.67   & 41s \\ \hline
    7   &  3              &  20, 20, 20      &  30     &  10       &  62.57   & 44s\\ \hline
  \end{tabular}
\end{table}
\end{document}


\documentclass[12pt, answers]{exam}
%\documentclass[12pt]{exam}
\usepackage[top=1in, bottom=1in, left=1in, right=1in]{geometry}
\usepackage{setspace}
\onehalfspacing
\setlength{\parindent}{0mm} \setlength{\parskip}{1em}

% packages
\RequirePackage{amssymb, amsfonts, amsmath, latexsym, verbatim, xspace, setspace}
\RequirePackage{tikz}
% The float package HAS to load before hyperref
\usepackage{float} % for psuedocode formatting
\usepackage{amsthm}
\usepackage{epsfig}
\usepackage{times}
\renewcommand{\ttdefault}{cmtt}
\usepackage{amsmath}
\usepackage{graphicx} % for graphics files

% for creating indented blocks
\usepackage{scrextend}
\usepackage{paralist, tabularx}

% from Denovo Methods Manual
\usepackage{mathrsfs}
\usepackage[mathcal]{euscript}
\usepackage{color}
\usepackage{array}

\usepackage[pdftex]{hyperref}
\usepackage[parfill]{parskip}
\usepackage{cancel}

\newcommand{\nth}{n\ensuremath{^{\text{th}}} }
\newcommand{\ve}[1]{\ensuremath{\mathbf{#1}}}
\newcommand{\Macro}{\ensuremath{\Sigma}}
\newcommand{\vOmega}{\ensuremath{\hat{\Omega}}}
\newcommand{\cc}[1]{\ensuremath{\overline{#1}}}
\newcommand{\ccm}[1]{\ensuremath{\overline{\mathbf{#1}}}}

%--------------------------------------------------------------------
%--------------------------------------------------------------------
\begin{document}
\begin{center}
{\bf NE 155, Topics 8 \& 9, S21 \\
Differentiation \& Integration \\ February 23, 2021}
\end{center}

\setlength{\unitlength}{1in}
\begin{picture}(6,.1)
\put(0,0) {\line(1,0){6.25}}
\end{picture}

%--------------------------------------------------------------------
\section*{Differentiation}
We're building the skills required to solve
\begin{equation}
\frac{1}{v}\frac{\partial}{\partial t}\phi(\vec{r}, t)
-\nabla \cdot D\nabla \phi(\vec{r}, t) +
\Sigma_a \phi(\vec{r}, t) =
\nu \Sigma_f \phi(\vec{r}, t) +
S(\vec{r}, t) \:. \nonumber
\end{equation}
In practice, we can rarely get an analytical solution for $\phi$ so we have to solve it some other way.
If we don't know what $\phi$ looks like, how do we take its derivative?
We need \textbf{numerical differentiation} so that we can come up with a way to express $\nabla^2 \phi$ or $\frac{\partial}{\partial t}\phi$ (or anything similar in any other equation).

\underline{Problem}: Given a function $f(x)$ at $x_0, x_1, \dots, x_n$, approximate $f'(x)$ or $f''(x)$, etc.

Numerical differentiation is useful when a function is defined by
\vspace*{-0.5em}
\begin{compactitem}
\item data
\item differential equations (ODEs, PDEs)
\end{compactitem}

\textbf{Example} Given $f$ is $ C^2 \in [a,b]$ and $x_0 \in [a,b]$, find an approximation to $f'(x_0)$.

We're going to come at this from \textbf{Taylor's theorem}, which gives an approximation of a k-times differentiable function around a given point by a k-th order Taylor polynomial.
% https://en.wikipedia.org/wiki/Taylor%27s_theorem

We'll start with a first order expansion for the point $x_0$:
\[
f(x) = f(x_0) + f'(x_0)(x - x_0) + \frac{f''(c)}{2}(x-x_0)^2 \]
\ifprintanswers
\[
\therefore f'(x_0) = \frac{f(x) - f(x_0)}{(x - x_0)} - \frac{f''(c)}{2}(x-x_0) \]
\else
\\\vspace*{2em}
\fi
%
\begin{align*}
\text{Let } h &= x-x_0 \rightarrow x = x_0 + h \\
%
\therefore f'(x_0) &= \underbrace{\frac{f(x_0 + h) - f(x_0)}{h}}_{\text{computable approx.}} - \underbrace{\frac{f''(c)}{2}h}_{\text{error term}}
\end{align*}
%
We can extend this principle to get higher levels of accuracy. To do that we do a Taylor expansion for each point in our collection and choose how many points to combine and in what ways.
%
\begin{center}
\begin{tikzpicture}
\draw (-.25,0)--(1.25,0);
\draw[dotted] (1.25,0)--(2.75,0);
\draw (2.75,0)--(5.25,0);
\draw[dotted] (5.25,0)--(6.75,0);
\draw (6.75,0)--(8.25,0);
%\draw (4,0)--(5.25,0);
\draw (0,-.25)--(0,.25);
\draw (1,-.25)--(1,.25);
%\draw (2,-.25)--(2,.25);
\draw (3,-.25)--(3,.25);
\draw (4,-.25)--(4,.25);
\draw (5,-.25)--(5,.25);
\draw (7,-.25)--(7,.25);
\draw (8,-.25)--(8,.25);
\node[below] at (0,-.25) {$x_0$};
\node[below] at (1,-.25) {$x_1$};
\node[below] at (3,-.25) {$x_{i-1}$};
\node[below] at (4,-.25) {$x_i$};
\node[below] at (5,-.25) {$x_{i+1}$};
\node[below] at (7,-.25) {$x_{n-1}$};
\node[below] at (8,-.25) {$x_n$};
\end{tikzpicture}
\end{center}
%

\[
f(x_i) = f(x_i)\]
%
\ifprintanswers
\[
f(x_i \pm h) = f(x_i) \pm hf'(x_i) + h^2\frac{f''(x_i)}{2} \pm h^3\frac{f'''(x_i)}{6} + f^{(4)}(c_1)\frac{h^4}{24} \]
\else
\\\vspace*{2em}
\fi
%
\[
f(x_i \pm 2h) = f(x_i) \pm 2h f'(x_i) + 2 h^2 f''(x_i) \pm \frac{4}{3} h^3 f'''(x_i) + \frac{2}{3}h^4 f^{(4)}(c_2)\]

% ---------------------------------------------------------
\subsection*{Forward difference:}
\underline{$O(h)$}: linearly combine the point and the next point forward
\[f'(x_i) = \frac{f(x_i + h) - f(x_i)}{h} - \frac{1}{2}hf''(c)\]

\underline{$O(h^2)$}: linearly combine the point and the next two points forward; we're going to need to figure out the coefficients though:

\[
a f(x_i) + b f(x_i + h) + c f(x_i + 2h) = f'(x_i) \]
%
\ifprintanswers
\begin{align*}
a f(x_i) &+ b[f(x_i) + hf'(x_i) + h^2\frac{f''(x_i)}{2} + h^3\frac{f'''(c_1)}{6}] \nonumber \\
&+ c[f(x_i) + 2h f'(x_i) + 2 h^2 f''(x_i) + \frac{4}{3} h^3 f'''(c_2)] =f'(x_i)
\end{align*}
\else
\\\vspace*{3em}
\fi
\begin{align*}
(a + b &+ c)f(x_i) + h(b + 2c)f'(x_i) + h^2(\frac{b}{2} + 2c) f''(x_i) \\
&+ h^3(\frac{b}{6} +  \frac{4c}{3}) f'''(\mu) = f'(x_i)
\end{align*}
%
Use mean value theorem to get $\mu \in [c_1, c_2]$ to be able to combine the error terms to get only one term.

*KEY* set the coefficients to get what we want:
%
\begin{align*}
(a + b + c) &= 0\\
h(b + 2c) &= 1\\
h^2(\frac{b}{2} + 2c) &= 0
\end{align*}
%
We don't have more degrees of freedom, so we're left with $f'''$ for the error term.

Solving all of that gives
%
\begin{align*}
a &= -\frac{3}{2h} \\
b &= \frac{2}{h} \\
c &= -\frac{1}{2h} \\
&\text{error term } = -\frac{1}{3}h^2 f'''(\mu)
\end{align*}
\ifprintanswers
\[
\therefore \quad \boxed{f'(x_i) = \frac{-3 f(x_i) + 4f(x_i + h) - f(x_i + 2h)}{2h} -\frac{1}{3}h^2 f'''(\mu)}\]
\else
\vspace*{2em}
\fi

%--------------------------------------------------------------------
\subsection*{Backward difference:}
Essentially, the signs all flip because we use points in the other direction.

\underline{$O(h)$}: combine the point and the next point backward
\[f'(x_i) = \frac{f(x_i) - f(x_i - h)}{h} + \frac{1}{2}hf''(\mu)\]

\underline{$O(h^2)$}: combine the point and the next two points backward
%
\[f'(x_i) = \frac{3 f(x_i) - 4f(x_i - h) + f(x_i - 2h)}{2h} + \frac{1}{3}h^2 f'''(\mu)\]

%--------------------------------------------------------------------
\subsection*{Central difference:}
Use points on either side (the average of forward and backward difference) $\rightarrow$ more accuracy for the same number of evaluation  points.

\underline{$O(h^2)$}: combine one point on either side
\[f'(x_i) = \frac{f(x_i + h) - f(x_i - h)}{2h} - \frac{1}{6}h^2 f'''(\mu)\]

\underline{$O(h^4)$}: combine two points on either side
\ifprintanswers
\[f'(x_i) = \frac{f(x_i - 2h) - 8f(x_i - h) + 8f(x_i + h) - f(x_i + 2h)}{2h} - \frac{1}{30}h^4 f^{(5)}(\mu)\]
\else
\\\vspace*{2em}
\fi

\subsubsection*{Higher Order Derivatives}
By solving for different terms in the Taylor expansion, combining points, and solving for coefficients we can get other derivative terms as well.

E.g. $a f(x_i + h) + b f(x_i) + c f(x_i - h) = f''(x_i)$

These are all $O(h^2)$:
\begin{align*}
f''(x_i) &= \frac{f(x_i - h) - 2f(x_i) + f(x_i + h)}{h^2} + \frac{h^2}{12}f^{(4)}(\mu) \\
%
f^{(3)}(x_i) &= \frac{-f(x_i - 2h) + 2f(x_i - h) - 2f(x_i + h) + f(x_i + 2h)}{2h^3}\\
%
f^{(4)}(x_i) &= \frac{f(x_i - 2h) - 4f(x_i - h) +6f(x_i) - 4f(x_i + h) + f(x_i + 2h)}{h^4}
\end{align*}

\subsubsection*{Two Variables}
\begin{align*}
\frac{\partial^2 f_{i,j}}{\partial x^2} = \frac{\partial}{\partial x}\bigl(\frac{\partial f_{i,j}}{\partial x}\bigr) =
\frac{f_{i-1,j} - 2f_{i,j} + f_{i+1,j}}{\Delta x^2} \\
%
\frac{\partial^2 f_{i,j}}{\partial y^2} = \frac{\partial}{\partial y}\bigl(\frac{\partial f_{i,j}}{\partial y}\bigr) =
\frac{f_{i,j-1} - 2f_{i,j} + f_{i,j+1}}{\Delta y^2}
\end{align*}
\begin{center}
\begin{tikzpicture}
\draw (-.25,0)--(7.25,0);
\draw (1,-.25)--(1,.25);
\draw (3,-.25)--(3,5.25);
\draw (4,-.25)--(4,5.25);
\draw (5,-.25)--(5,5.25);
\draw (7,-.25)--(7,.25);
\node[below] at (0,-.25) {$x_i$};
\node[below] at (1,-.25) {$x_1$};
\node[below] at (3,-.25) {$x_{i-1}$};
\node[below] at (4,-.25) {$x_i$};
\node[below] at (5,-.25) {$x_{i+1}$};
\node[below] at (7,-.25) {$x_n$};
\node at (3.5, 2) {$\Delta x$};
% begin y
\draw (0,-.25)--(0,7.25);
\draw (-.25,1)--(.25,1);
\draw (-.25,3)--(5.25,3);
\draw (-.25,4)--(5.25,4);
\draw (-.25,5)--(5.25,5);
\draw (-.25,7)--(.25,7);
\node[left] at (-.25, 0) {$y_0$};
\node[left] at (-.25,1) {$y_1$};
\node[left] at (-.25,3) {$y_{j-1}$};
\node[left] at (-.25,4) {$y_j$};
\node[left] at (-.25,5) {$y_{j+1}$};
\node[left] at (-.25,7) {$y_n$};
\node at (2,3.5) {$\Delta y$};
\end{tikzpicture}
\end{center}

%--------------------------------------------------------------------
%-------------------------------------------------------------------
\vspace*{-2em}
\section*{Integration}
Why might we need to numerically integrate?
\begin{itemize}
\ifprintanswers
\item The integrand, $f(x)$, is complicated enough that its
indefinite or even definite integral over a particular range cannot
be found.
\else
\item
\fi
\item The indefinite or definite integral can be found, but may take
long time to calculate analytically.
\item The integrand is given in terms of a table. That is, we are
provided with discrete values of $f(x)$ at $n+1$ data points.
\end{itemize}

For us, we use this particularly for the scattering and fission in the Transport Equation:
\[\int_{4\pi} d\vOmega' \int_0^{\infty} dE' \Sigma_s(E', \vOmega' \rightarrow E, \vOmega) \psi(\vec{r}, E', \vOmega', t)  +
 \frac{\chi(E)}{4\pi} \int_0^{\infty} dE' \nu(E') \Sigma_f(E') \int_{4\pi} d\vOmega' \psi(\vec{r}, E', \vOmega', t) \]

The general strategy is:
\begin{itemize}
\item Approximate the integrand $f(x)$ with either a global interpolating
polynomial defined over the whole domain of integration, or a
collection of local interpolating polynomials defined over intervals
that are subtended by a small group of data points.
\ifprintanswers
\item Integrate the interpolating polynomial(s) over their individual
domains of definition.
\else
\item
\fi
\end{itemize}

Thus, given $f \in[a,b]$, compute $I(f) = \int_a^b f(x) dx$.

The way we do this is called \textbf{numerical integration} or \textbf{numerical quadrature}:
\ifprintanswers
\[I(f) \approx I_n(f) = \sum_{i=0}^n w_i f(x_i)\]
\vspace*{-3em}

$w_i \equiv$ quadrature weights\\
$x_i \equiv$ quadrature points
\else
\vspace*{4em}
\fi

\textbf{Approach \#1}\\
Fix quadrature points ($x_i$), then choose quadrature weights ($w_i$). Usually fit a polynomial to $f(x_i)$ and integrate.
%
\begin{enumerate}%[a.]
\item \underline{Newton-Cotes}: use a single poly over the entire interval
\item \underline{composite Newton-Cotes}: split the interval
\item \underline{Romberg integration}: extrapolation
\end{enumerate}

\textbf{Approach \#2}\\
For a given $n$, choose the ``best" quadrature points ($x_i$) and quadrature weights ($w_i$):\\
\underline{Gaussian Quadrature}

\vspace*{1em}
the \textbf{degree of precision} of a quadrature formula, $I_n(f)$, is the positive integer $m$ s.t.:
\begin{enumerate}
\item $I(p) = I_n(p)$ for every poly of degree $\leq m$.
\item $I(p) \neq I_n(p)$ for some poly of degree $m+1$.
\end{enumerate}

%--------------------------------------------------------------------
\subsection*{Newton-Cotes formula}

\underline{idea}: fix $x_0, x_1, \dots, x_n$ then interpolate $f(x)$ by a poly of degree $\leq n$.
%
\ifprintanswers
\begin{align*}
I(f) &\approx I_n(f) = I(p_n)\\
\text{true integral } &\approx \text{ N-C formula } = \text{ true integral of interpolating poly}
\end{align*}
\else
\\\vspace*{2em}\\
\fi
%
The only error is in the polynomial interpolation.

%--------------------------------------------------------------------
\subsection*{Lagrange form of NC}
Choosing Lagrange polynomials prevents us from needing to recalculate weights for every $f(x)$ (since we've fixed the points). Note, we're assuming equally spaced points: $h = (b-a)/n$. This is not a necessary assumption; for unequally spaced points we would index $h$ in the substitution step.
%
\begin{align*}
P_n(x) &= \sum_{i=0}^{n}f(x_i)L_i(x) \\
%
I(P_n(x)) &= \sum_{i=0}^{n} \bigl[ \int_a^b L_i(x)dx \bigr] f(x_i) \\
%
&= \sum_{i=0}^{n} w_i f(x_i)
\end{align*}
\ifprintanswers
\begin{align*}
\text{where } w_i &=  \int_a^b L_i(x)dx = \int_a^b \prod_{\substack{i=0\\ i \neq k}}^n \frac{(x-x_i)}{(x_k-x_i)}dx \\
&\text{sub in } x = a + sh\\
&=  (b-a)\frac{1}{n}\int_0^n \prod_{\substack{i=0\\ i \neq k}}^n \frac{(s-i)}{(k-i)}ds
\end{align*}
\else
\\\vspace*{4em}\\
\fi
We can see $w_i$ are not dependent on $f(x)$.

%--------------------------------------------------------------------
\subsubsection*{Trapezoid Rule}
This is the Lagrange form of Newton-Cotes integration with $n=1$.

$x_0 = a$, $x_1 = b$, $h = x_1 - x_0$
%
\begin{align*}
&L_0(x) = \frac{x-x_1}{x_0-x_1}\:, \qquad L_1(x) = \frac{x-x_0}{x_1-x_0} \\
%
w_0 &= \int_{x_0}^{x_1} \bigl(\frac{x-x_1}{x_0-x_1}\bigr) dx\:;
\\ &\text{let } x = x_0 + th  \:, \quad dx = h\:dt\:, \quad t = \frac{x-x_0}{h}\:,\\
&\text{change bounds and note that the denominator is }-h
\end{align*}
\begin{align*}
w_0 &= h \int_0^1 \bigl(\frac{x_1 - x_0 - ht}{h}\bigr)dt = \int_0^1 h(1-t)dt \\
%
w_0 &= h\bigl(t - \frac{1}{2}t^2 \bigr) |_0^1 = \frac{1}{2}h\\
%
w_1 &= \int_{x_0}^{x_1} \bigl(\frac{x-x_0}{x_1-x_0}\bigr) dx = \frac{1}{2}h
%
\end{align*}
\ifprintanswers
\[\therefore I_1(f) = w_0f(x_0) + w_1f(x_1)  = \boxed{\frac{h}{2}\bigl(f(x_0) + f(x_1)\bigr)} \rightarrow \text{area of a trapezoid}
\]
\else
\\\vspace*{2em}\\
\fi
%
The trapezoid rule has degree of precision 1.

\begin{figure}
\begin{center}
  \includegraphics[height=4in,clip]{QuadratureComparison}
\end{center}
\caption{examples of how different integration rules capture a function}
\end{figure}

%--------------------------------------------------------------------
\subsubsection*{Error Analysis}
\begin{enumerate}
\item interpolate:
\[f(x) = P_n(x) + R_n(x)\]
\item integrate:
\[\underbrace{\int_a^b f(x) dx}_{I(f)} = \underbrace{\int_a^b P_n(x)dx}_{I_n(f)} + \int_a^b R_n(x) dx\]
\end{enumerate}

\textbf{Example:} Trapezoid Rule\\
Recall from Lagrange polynomials that we know how to express $R_n$. We can then use that to compute the error of the integral.
\[
R_1(x) = \frac{f''(c)}{2!}(x-x_0)(x-x_1) \]
\ifprintanswers
\[
I(f) - I_1(f) = \int_{x_0}^{x_1} \frac{f''(c)}{2!}\underbrace{(x-x_0)(x-x_1)}_{\text{doesn't change sign over interval}} dx\]
\else
\vspace*{2em}
\fi

%
Recall that $c$ depends on $x$. To get an expression, we're going to use the mean value theorem.

\underline{Mean Value Theorem for Integrals}\\
\[\int_a^b f(x)g(x)dx = f(\xi) \int_a^b g(x)dx\:, \qquad a\leq \xi \leq b\]
Where $g(x)$ must always be positive or always negative (cannot change sign over the interval).

Thus, we can use MVT for our integral to see that
\[
I(f) - I_1(f) = \frac{1}{2}f''(\eta) \int_{x_0}^{x_1} (x-x_0)(x-x_1) dx \]
\ifprintanswers
\[= \frac{1}{2}f''(\eta) \bigl(\frac{-h^3}{6}\bigr) \rightarrow \boxed{\frac{-h^3}{12}f''(\eta)}\]
\else
\vspace*{2em}\\
\fi
is the error term for trapezoid rule. You can see that this has \underline{degree of precision 1 b/c the second derivative} \underline{drives the error}. In addition, \textit{the error changes as $O(h^3)$ with mesh spacing}.

%--------------------------------------------------------------------
\vspace*{-1em}
\subsubsection*{Simpson's Rule}
This is the Lagrange form of Newton-Cotes integration with $n=2$.

\begin{align*}
x_0 &= a \qquad x_1 = \frac{a+b}{2} \qquad x_2=b \qquad h=\frac{b-a}{2}\\
%
L_0 &= \frac{(x-x_1)(x-x_2)}{(x_0-x_1)(x_0-x_2)}\\
L_1 &= \frac{(x-x_0)(x-x_2)}{(x_1-x_0)(x_1-x_2)}\\
L_2 &= \frac{(x-x_0)(x-x_1)}{(x_2-x_0)(x_2-x_1)}\\
%
w_0 &= \int_{x_0}^{x_2} L_0(x)dx = \frac{h}{3}\\
w_1 &= \int_{x_0}^{x_2} L_1(x)dx = \frac{4h}{3}\\
w_2 &= \int_{x_0}^{x_2} L_2(x)dx = \frac{h}{3}\\
&\text{Note: weights sum to }2h\text{, the size of the interval}\nonumber
\end{align*}
\ifprintanswers
\[\therefore I_2(f) = \frac{h}{3}\bigl(f(x_0) + 4f(x_1) + f(x_2)\bigr) \]
\else
\vspace*{2em}
\fi

The error term comes from integrating $R_2 = \frac{1}{3!}f'''(c)(x-x_0)(x-x_1)(x-x_2)$. However, the middle term now changes sign over the interval and we can't just use MVT.

We're going to skip the details, but what we get out is that the error for Simpson's Rule looks like:
\[\frac{-f^{(4)}(\eta)}{4!}\frac{h^5}{\frac{4}{15}} \rightarrow \boxed{\frac{-f^{(4)}(\eta)}{90}h^5}\]

Thus, Simpson's rule has degree of precision 3 and is $O(h^5)$.

\textbf{In General:}
\vspace*{-1em}
\begin{center}
\begin{tabular}{c c c}
$n$  & error        & precision \\ \hline
odd  & $O(h^{n+2})$ & $n$ \\
even & $O(h^{n+3})$ & $n+1$ \\
\end{tabular}
\end{center}

\underline{Simpson's 3/8 rule}:\\
If $n=3$:
\[\int_{x_0}^{x_3} f(x)dx = \frac{3h}{8}\bigl(f(x_0) + 3f(x_1) + 3f(x_2) + f(x_3)\bigr) - \frac{3h^5}{80}f^{(4)}(\xi)\]
where $x_0 < \xi < x_3$ and $h=(x_3 - x_0)/3$.

%--------------------------------------------------------------------
\subsubsection*{Composite Newton-Cotes}
Some difficulties with Newton-Cotes interpolation:
\begin{enumerate}
\ifprintanswers
\item To add points we must recompute the weights, $w_i$, which can be a lot of work.
\else
\item\fi
\item high order polynomial interpolation on equally spaced points is BAD. (using optimally-spaced points is Gaussian quadrature; we'll get to that later).
\end{enumerate}

\underline{Idea:} subdivide $[a,b]$ into subintervals. On each of these apply a low-order Newton-Cotes formula.

\textbf{Composite Trapezoid}:\\
Just apply the trapezoid rule on each subinterval. Given $x_0, x_1, \dots, x_n$ and $f(x_0), f(x_1), \dots f(x_n)$, the grid space is $h = \frac{x_n-x_0}{n}$.

\textit{DRAW PICTURE}
\vspace*{-1em}
\[
\int_{a}^{b} f(x)dx = \sum_{i=1}^n \int_{x_{i-1}}^{x_i} f(x)dx\]
\ifprintanswers
\begin{align*}
&= \sum_{i=1}^n \bigl[ \frac{h}{2}\bigl(f(x_{i-1}) + f(x_i)\bigr) - \frac{h^3}{12}f''(c_i) \bigr] \\
&= \frac{h}{2}\bigl(f(a) + 2\sum_{i=1}^{n-1}f(x_i) + f(b)\bigr) - \frac{h^3}{12} \underbrace{\sum_{i=1}^n f''(c_i)}_{\substack{n f''(\mu),\\ n = (b-a)/h}}\\
\int_{a}^{b} f(x)dx &= \boxed{\frac{h}{2}\bigl(f(a) + 2\sum_{i=1}^{n-1}f(x_i) + f(b)\bigr) - \frac{h^2}{12}(b-a)f''(\mu) }
\end{align*}
\else
\vspace*{9em}
\fi

\textbf{Composite Simpson}:\\
Similarly, apply the Simpson rule on each subinterval. Note that for the functionally-equivalent sense of $n$ intervals we really need to use $2n$ points since each Simpson is applied using 3 points, and the end points are re-used. Therefore $\int_{a}^{b} f(x)dx = $

\begin{align*}
\sum_{i=1}^{n/2}\int_{x_{2i-1}}^{x_{2i}} f(x)dx &= \sum_{i=1}^{n/2} \bigl[ \frac{h}{3}\bigl(f(x_{2i-2}) + 4f(x_{2i-1}) + f(x_{2i})\bigr) - \frac{f^{(4)}(c_i)}{90}h^5 \bigr]\\
&\quad \text{MVT: } \sum_{i=1}^{n/2}f^{(4)}(c_i) = \frac{n}{2}f^{(4)}(\mu) = \frac{b-a}{2h}f^{(4)}(\mu)\\
&= \frac{h}{3}\bigl(f(a) + \underbrace{4\sum_{i=1}^{n/2} f(x_{2i-1})}_{\text{odd points}} + \underbrace{2\sum_{i=1}^{n/2-1} f(x_{2i})}_{\text{even points}} + f(b)\bigr) - \frac{h^4}{180}(b-a)f^{(4)}(\mu)
\end{align*}

%--------------------------------------------------------------------
\subsubsection*{Open Newton-Cotes}
The previous examples were called \textbf{closed} Newton-Cotes because $f(x)$ is evaluated at the first and last points.

\ifprintanswers
\textbf{Open} Newton-Cotes does not evaluate $f(x)$ at the endpoints. The node points are still defined as $x_{j} = x_0 + jh, j = 0, \dots, n$,  but now
\vspace*{-1em}
\else
\vspace*{2em}
\fi
\begin{center}
\begin{tabular}{c c c}
Item & Open & Closed \\ \hline
$h$  & $\frac{b-a}{n+2}$ & $\frac{b-a}{n}$ \\
$x_0$ & $a+h$ & $a$ \\
$x_n$ & $b-h$ & $b$ \\
\end{tabular}
\end{center}
%
Now our real endpoints are $x_{-1}$ and $x_{n+1}$ (so we're doing $\int_{x_{-1}}^{x_{n+1}} f(x)dx$), which gives
\[
 w_i =  \int_a^b L_i(x)dx = \int_a^b \prod_{\substack{i=0\\ i \neq k}}^n \frac{(x-x_i)}{(x_k-x_i)}dx =  (b-a)\frac{1}{n+2}\int_0^n \prod_{\substack{i=0\\ i \neq k}}^n \frac{(s-i)}{(k-i)}ds
 \]

\begin{figure}
\begin{center}
  \includegraphics[height=2in,clip]{ClosedNewtonCotes}
  \includegraphics[height=1.75in,clip]{OpenNewtonCotes}
\end{center}
\caption{[left] example of \textbf{closed} Newton Cotes (where the interpolation \textit{matches} at the end points); [right] example of \textbf{open} Newton Cotes (where the interpolation \textit{does not match} at the end points)}
\end{figure}

%A quick explanation of difference I've found on slides 17 and 18:\\ \href{http://www.math.uconn.edu/~leykekhman/courses/MATH_5520/Notes/quadrature_1.pdf}{http://www.math.uconn.edu/$\sim$leykekhman/courses/MATH\_5520/Notes/quadrature\_1.pdf}

\vspace*{1 em}
\ifprintanswers
The \underline{midpoint rule} is what we get with \underline{$n=0$}; it only uses one point:
\[\int_a^b f(x)dx = \int_{x_{-1}}^{x_{1}} f(x)dx = 2hf(x_0) + \frac{h^3}{3}f''(\xi)\]
where $x_{-1} < \xi < x_{1}$ and $h=(b-a)/2$.
\else
\vspace*{5em}
\fi

\underline{$n=1$}:
\[ \int_{x_{-1}}^{x_{2}} f(x)dx = \frac{3h}{2}[f(x_0) + f(x_1)] + \frac{3h^3}{4}f''(\xi)\]
where $x_{-1} < \xi < x_{2}$ and $h=(b-a)/3$.

\underline{$n=2$}:
\[ \int_{x_{-1}}^{x_{3}} f(x)dx = \frac{4h}{3}[2f(x_0) - f(x_1) + 2f(x_2)] + \frac{14h^5}{45}f^{(4)}(\xi)\]
where $x_{-1} < \xi < x_{3}$ and $h=(b-a)/4$.

%--------------------------------------------------------------------
%--------------------------------------------------------------------
%\bibliographystyle{plain}
%\bibliography{LinearSolns}

\end{document}


\documentclass[]{article}

\usepackage[english]{babel}
\usepackage{graphicx}
\usepackage{booktabs}
\usepackage{float}
\usepackage{hyperref}

\title{Recurrent Neural Networks}
\author{Juan Lao Tebar}

\begin{document}

	\maketitle

	\section{Introduction}

	In this practice we explore the effects of the different elements of RNNs using a specific dataset\footnote{\url{https://upc-mai-dl.github.io/rnn-lab-autonomous/}} by extending the Wind prediction example of the previous guided session.

	In the original example we predicted one step of the future wind---next 15 minutes---using a window of previous measurements. Here we extend the problem in different ways:

	\begin{description}
		\item [Complementary variables.] In the original experiment we used only the wind speed as input data. Now we include also the air density, the temperature and the pressure.

		\item [Complementary sites.] In the original experiment we predicted the wind speed using the data from one site, but the dataset includes three additional sites that are geographically close---they are in the vertices of a 2 km square---. Here we leverage the information of the wind speed from the other sites.

		\item [Multi step prediction.] Here we obtain a multi step prediction using the original model simply by adding the value predicted to the current set of measurements and discarding the oldest one.

		\item [Sequence to sequence prediction.] An alternative to shifting the input and adding the predictions to have a multi step prediction is a network that links a window of measurements to a window of predictions.
	\end{description}

	In this practice we change the stopping criteria of the training process, as we remove the limit of epochs and we implement a stopper based on the validation loss: when we detect a minimum value, we stop the training after 30 epochs if it does not decrease again. If the model does not show any sign of convergence, we consider that we should not stop the training process just because it reached an arbitrary number of epochs.

	In the guided laboratory we trained the model with 50,000 samples, validated with 10,000 and tested with 10,000, while in this practice we use the whole dataset, training our model with 122,736 samples, validating with 61,368 and testing with 61,368.

	The code related to this work is public and available at github\footnote{\url{https://github.com/juanlao7/DL-lab2}}.

	\section{Baseline}

	Before performing any experiment, we define a baseline model in order to compare the results and determine how each change affects the accuracy of the system.

	Our baseline model is similar to the one proposed in the guided laboratory: an input window size of 6 steps, 1 output, 512 LSTM units in one single hidden layer and no dropout, trained with RMSprop---with a learning rate of 0.0001, $ \rho = 0.9 $, $ \varepsilon = 1 \cdot 10^{-8} $ and no learning rate decay.

	Table \ref{t:baseline} shows the results obtained with this model.

	\begin{table}[H]
		\centering
		\begin{tabular}{@{}ccc@{}}
			\toprule
			Training loss & Validation loss & Test loss \\ \midrule
			0.02729       & 0.02924         & 0.03339   \\ \bottomrule
		\end{tabular}
		\caption{Results obtained with the baseline model.}
		\label{t:baseline}
	\end{table}

	\section{Complementary Variables}

	\subsection{Plan}

	In this session we modify the architecture of the system adequately for taking all the variables as input---but still giving only one variable as output---. Then we study how the MSE is affected by the size of the hidden layer, the dropout and the size of the input window.

	\begin{itemize}
		\item In the 1st experiment we test different layer sizes: 32, 128, 512 and 1024 units.

		\item In the 2nd experiment we test different dropout factors applied in the LSTM layer, including its gates: 0\%, 25\%, 50\%, 75\%.

		\item In the 3rd experiment we test different dropout factors only applied on inter-layer connections, between the hidden and the output layer: 0\%, 25\%, 50\%, 75\%.

		\item In the 4th experiment we test different input window sizes: 1, 3, 6, 12, 24, 48. In this experiment, in the case of the window size of 48, we had to clip all parameter gradients to a maximum norm of 1 in order to prevent exploding gradients.
	\end{itemize}

	\subsection{Expectations}

	Regarding to the learning techniques:

	\begin{itemize}
		\item In theory, adding more units to the hidden layer leads to better training accuracies. However, the model may lose power of generalization, overfitting the training data and offering worse validation and test accuracies.

		\item If the network overfits the training data, we expect that applying a dropout factor will successfully regularize the model. However, the dropout implementation we use---``dropout'' argument in Keras implementation of LSTM layers---affects also to the LSTM gates, thus it is possible that the model will end up performing even worse due to the vanishing gradient problem. For this reason we also test an inter-layer dropout, that does not affect to LSTM gates. On the other hand, if the model does not overfit, dropout will slightly reduce the gap between training and validation losses, but not in a considerable way.

		\item We suspect that in this dataset there are no data relations over long intervals of time, hence we will not obtain a better result by increasing the window size. However, decreasing it may lead to a worse accuracy.
	\end{itemize}

	Regarding to this specific problem:

	\begin{itemize}
		\item More information---variables---are always helpful and the model may find new inter-variable relations that lead to a better accuracy.
	\end{itemize}

	\subsection{Results}

	\begin{itemize}
		\item Table \ref{t:variables_1} shows the results obtained after testing different layer sizes, while figure \ref{f:variables_1} shows the evolution of the training and validation loss in each case.

		\item Table \ref{t:variables_2} shows the results obtained after testing different dropout factors, while figure \ref{f:variables_2} shows the evolution of the training and validation loss in each case.

		\item Table \ref{t:variables_3} shows the results obtained after testing different inter-layer dropout factors, while figure \ref{f:variables_3} shows the evolution of the training and validation loss in each case.

		\item Table \ref{t:variables_4} shows the results obtained after testing different input window sizes, while figures \ref{f:variables_4_1} and \ref{f:variables_4_2} show the evolution of the training and validation loss in each case.
	\end{itemize}

	\begin{table}[H]
		\centering
		\begin{tabular}{@{}cccc@{}}
			\toprule
			& Training loss & Validation loss & Test loss \\ \midrule
			32 LSTM units   & 0.02770       & 0.02939         & 0.03327   \\
			128 LSTM units  & 0.02621       & 0.02835         & 0.03218   \\
			512 LSTM units  & 0.02562       & 0.02911         & 0.03266   \\
			1024 LSTM units & 0.02531       & 0.02883         & 0.03247   \\ \bottomrule
		\end{tabular}
		\caption{Results obtained after testing different layer sizes.}
		\label{t:variables_1}
	\end{table}

	\begin{figure}[H]
		\centering
		\includegraphics[width=0.5\textwidth]{variables_1}
		\caption{Validation and training loss with each layer size.}
		\label{f:variables_1}
	\end{figure}

	\begin{table}[H]
		\centering
		\begin{tabular}{@{}cccc@{}}
			\toprule
			& Training loss & Validation loss & Test loss \\ \midrule
			0\% dropout  & 0.02562       & 0.02911         & 0.03266   \\
			25\% dropout & 0.24172       & 0.08519         & 0.08502   \\
			50\% dropout & 0.47773       & 0.27423         & 0.26216   \\
			75\% dropout & 0.73191       & 0.57920         & 0.54833   \\ \bottomrule
		\end{tabular}
		\caption{Results obtained after testing different dropout factors.}
		\label{t:variables_2}
	\end{table}

	\begin{figure}[H]
		\centering
		\includegraphics[width=0.5\textwidth]{variables_2}
		\caption{Validation and training loss with each dropout factor.}
		\label{f:variables_2}
	\end{figure}

	\begin{table}[H]
		\centering
		\begin{tabular}{@{}cccc@{}}
			\toprule
			& Training loss & Validation loss & Test loss \\ \midrule
			0\% inter-layer dropout  & 0.02562       & 0.02911         & 0.03266   \\
			25\% inter-layer dropout & 0.02830       & 0.02886         & 0.03258   \\
			50\% inter-layer dropout & 0.03110       & 0.02884         & 0.03262   \\
			75\% inter-layer dropout & 0.04012       & 0.02975         & 0.03337   \\ \bottomrule
		\end{tabular}
		\caption{Results obtained after testing different inter-layer dropout factors.}
		\label{t:variables_3}
	\end{table}

	\begin{figure}[H]
		\centering
		\includegraphics[width=0.5\textwidth]{variables_3}
		\caption{Validation and training loss with each inter-layer dropout factor.}
		\label{f:variables_3}
	\end{figure}

	\begin{table}[H]
		\centering
		\begin{tabular}{@{}cccc@{}}
			\toprule
			& Training loss & Validation loss & Test loss \\ \midrule
			Window size of 1  & 0.03146       & 0.03305         & 0.03599   \\
			Window size of 3  & 0.02690       & 0.02880         & 0.03254   \\
			Window size of 6  & 0.02562       & 0.02911         & 0.03266   \\
			Window size of 12 & 0.02586       & 0.02884         & 0.03260   \\
			Window size of 24 & 0.02465       & 0.02941         & 0.03375   \\
			Window size of 48 & 0.02543       & 0.02885         & 0.03316   \\ \bottomrule
		\end{tabular}
		\caption{Results obtained after testing different input window sizes.}
		\label{t:variables_4}
	\end{table}

	\begin{figure}[H]
		\centering
		\includegraphics[width=0.5\textwidth]{variables_4_1}
		\caption{Validation and training loss with window sizes 1, 3 and 6.}
		\label{f:variables_4_1}
	\end{figure}

	\begin{figure}[H]
		\centering
		\includegraphics[width=0.5\textwidth]{variables_4_2}
		\caption{Validation and training loss with window sizes 6, 12, 24 and 48.}
		\label{f:variables_4_2}
	\end{figure}

	\subsection{Conclusions}

	Regarding to the learning techniques:

	\begin{itemize}
		\item \emph{As expected}, adding more units to the hidden layer leads to better training accuracies and, for the case of 1024 units, overfits the training data and offers worse validation and test accuracies.

		\item \emph{As expected}, dropout factors applied to LSTM gates lead to far worse results than the original approach.

		\item \emph{As expected}, since our model does not overfit the training data, certain dropout factors---25\% in this case---reduce the gap between the training and validation loss, but does not increase the accuracy of the system.

		\item \emph{As expected}, smaller window sizes lead to worse accuracies while bigger window sizes do not offer much improvement. However, it seems that the model converges in a lesser number of epochs.
	\end{itemize}

	Regarding to this specific problem:

	\begin{itemize}
		\item \emph{As expected}, more information---variables---are helpful and the model finds new inter-variable relations that lead to a \emph{slightly} better accuracy. The baseline model got a test accuracy of 0.03339 while our best model here got 0.03218---and several configurations got a mark under 0.03290---. We conclude that air density, pressure and temperature help to predict wind speed at 100m in site ``90-45142''.
	\end{itemize}

	\section{Complementary Sites}

	\subsection{Plan}

	In this session we leverage the information of the wind speed from other sites and we study how the MSE is affected by the size of the hidden layer, the dropout and the size of the input window.

	\begin{itemize}
		\item In the 1st experiment we test different layer sizes: 32, 128, 512 and 1024 units.

		\item In the 2nd experiment we test different dropout factors applied in the LSTM layer, including its gates: 0\%, 25\%, 50\%, 75\%.

		\item In the 3rd experiment we test different dropout factors only applied on inter-layer connections, between the hidden and the output layer: 0\%, 25\%, 50\%, 75\%.

		\item In the 4th experiment we test different input window sizes: 1, 3, 6, 12, 24, 48. In this experiment, in the case of the window size of 48, we had to clip all parameter gradients to a maximum norm of 1 in order to prevent exploding gradients.
	\end{itemize}

	\subsection{Expectations}

	Exactly the same as in Complementary Variables.

	\subsection{Results}

	\begin{itemize}
		\item Table \ref{t:sites_1} shows the results obtained after testing different layer sizes, while figure \ref{f:sites_1} shows the evolution of the training and validation loss in each case.

		\item Table \ref{t:sites_2} shows the results obtained after testing different dropout factors, while figure \ref{f:sites_2} shows the evolution of the training and validation loss in each case.

		\item Table \ref{t:sites_3} shows the results obtained after testing different inter-layer dropout factors, while figure \ref{f:sites_3} shows the evolution of the training and validation loss in each case.

		\item Table \ref{t:sites_4} shows the results obtained after testing different input window sizes, while figures \ref{f:sites_4_1} and \ref{f:sites_4_2} show the evolution of the training and validation loss in each case.
	\end{itemize}

	\begin{table}[H]
		\centering
		\begin{tabular}{@{}cccc@{}}
			\toprule
			& Training loss & Validation loss & Test loss \\ \midrule
			32 LSTM units   & 0.02666       & 0.02821         & 0.03231   \\
			128 LSTM units  & 0.02694       & 0.02855         & 0.03246   \\
			512 LSTM units  & 0.02631       & 0.02847         & 0.03225   \\
			1024 LSTM units & 0.02591       & 0.02882         & 0.03242   \\ \bottomrule
		\end{tabular}
		\caption{Results obtained after testing different layer sizes.}
		\label{t:sites_1}
	\end{table}

	\begin{figure}[H]
		\centering
		\includegraphics[width=\textwidth]{sites_1}
		\caption{Validation and training loss with each layer size.}
		\label{f:sites_1}
	\end{figure}

	\begin{table}[H]
		\centering
		\begin{tabular}{@{}cccc@{}}
			\toprule
			& Training loss & Validation loss & Test loss \\ \midrule
			0\% dropout  & 0.02631       & 0.02847         & 0.03225   \\
			25\% dropout & 0.03554       & 0.08211         & 0.08097   \\
			50\% dropout & 0.09668       & 0.26654         & 0.25337   \\
			75\% dropout & 0.34805       & 0.54228         & 0.51081   \\ \bottomrule
		\end{tabular}
		\caption{Results obtained after testing different dropout factors.}
		\label{t:sites_2}
	\end{table}

	\begin{figure}[H]
		\centering
		\includegraphics[width=0.5\textwidth]{sites_2}
		\caption{Validation and training loss with each dropout factor.}
		\label{f:sites_2}
	\end{figure}

	\begin{table}[H]
		\centering
		\begin{tabular}{@{}cccc@{}}
			\toprule
			& Training loss & Validation loss & Test loss \\ \midrule
			0\% inter-layer dropout  & 0.02631       & 0.02847         & 0.03225   \\
			25\% inter-layer dropout & 0.02836       & 0.02887         & 0.03240   \\
			50\% inter-layer dropout & 0.03084       & 0.02876         & 0.03277   \\
			75\% inter-layer dropout & 0.03898       & 0.02872         & 0.03251   \\ \bottomrule
		\end{tabular}
		\caption{Results obtained after testing different inter-layer dropout factors.}
		\label{t:sites_3}
	\end{table}

	\begin{figure}[H]
		\centering
		\includegraphics[width=\textwidth]{sites_3}
		\caption{Validation and training loss with each inter-layer dropout factor.}
		\label{f:sites_3}
	\end{figure}

	\begin{table}[H]
		\centering
		\begin{tabular}{@{}cccc@{}}
			\toprule
			& Training loss & Validation loss & Test loss \\ \midrule
			Window size of 1  & 0.03136       & 0.03277         & 0.03559   \\
			Window size of 3  & 0.02669       & 0.02843         & 0.03226   \\
			Window size of 6  & 0.02631       & 0.02847         & 0.03225   \\
			Window size of 12 & 0.02558       & 0.02935         & 0.03302   \\
			Window size of 24 & 0.02571       & 0.02879         & 0.03264   \\ \bottomrule
		\end{tabular}
		\caption{Results obtained after testing different input window sizes.}
		\label{t:sites_4}
	\end{table}

	\begin{figure}[H]
		\centering
		\includegraphics[width=0.5\textwidth]{sites_4_1}
		\caption{Validation and training loss with window sizes 1, 3 and 6.}
		\label{f:sites_4_1}
	\end{figure}

	\begin{figure}[H]
		\centering
		\includegraphics[width=\textwidth]{sites_4_2}
		\caption{Validation and training loss with window sizes 6, 12, 24 and 48.}
		\label{f:sites_4_2}
	\end{figure}

	\subsection{Conclusions}

	Regarding to the learning techniques: exactly the same as in Complementary Variables.

	Regarding to this specific problem:

	\begin{itemize}
		\item \emph{As expected}, more information---variables---are helpful and the model finds new inter-variable relations that lead to a \emph{slightly} better accuracy. The baseline model got a test accuracy of 0.03339 while our best model here got 0.03225---and several configurations got a mark under 0.03290---. We can conclude that wind speeds at 100m of sites ``90-45143', ``90-45229'' and ``90-45230'' help to predict wind speed at 100m in site ``90-45142''.
	\end{itemize}

	\section{Multi Step Prediction}

	\subsection{Plan}

	In this session we analyze the behavior of the model when it is fed with its own outputs as new inputs; we study how the MSE degrades the more steps in the future we predict and how the MSE changes if we extend the input window.

	In this experiment we test different input window sizes---1, 3, 6, 12, 24---and we observe the evolution of MSE when trying to predict the next 55 steps.

	\subsection{Expectations}

	\begin{itemize}
		\item In previous experiments we found that small window sizes lead to a higher error, and big window sizes do not show any improvement. We expect that the error of the model accumulates as more steps in the future we predict, leading to a higher error on small window sizes and more stable accuracies in big window sizes.
	\end{itemize}

	\subsection{Results}

	Figure \ref{f:multistep} shows the results obtained after testing different input window sizes.

	\begin{figure}[H]
		\centering
		\includegraphics[width=0.5\textwidth]{multistep}
		\caption{MSE evolution for each window size.}
		\label{f:multistep}
	\end{figure}

	\subsection{Conclusions}

	\begin{itemize}
		\item \emph{Surprisingly}, as we can see in figure \ref{f:multistep}, it turns out that with small window sizes the error seems to converge while in big window sizes the error grows exponentially. Sincerely, we do not have a logical explanation for this interesting phenomena.
	\end{itemize}

	\section{Sequence to Sequence Prediction}

	\subsection{Plan}

	In this session we modify the architecture of the system adequately for linking a window of measurements to a window of predictions, and we study how the MSE of the predictions changes with the length of the predicted sequence.

	In this experiment we test different prediction lengths---1, 3, 6, 12, 24---and we observe the evolution of the MSE.

	\subsection{Expectations}

	\begin{itemize}
		\item Models that try to predict more steps forward will lead to lower accuracies.
		\item This sequence-to-sequence model will have a lower error than the previous multi step approach, since this model does not accumulates the error.
	\end{itemize}

	\subsection{Results}

	Figure \ref{f:seq2seq_1} shows the evolution of the training and validation loss in each case, while figure \ref{f:seq2seq_2} shows the MSE evolution.

	\begin{figure}[H]
		\centering
		\includegraphics[width=\textwidth]{seq2seq_1}
		\caption{Validation and training loss with each number of predictions.}
		\label{f:seq2seq_1}
	\end{figure}

	\begin{figure}[H]
		\centering
		\includegraphics[width=\textwidth]{seq2seq_2}
		\caption{MSE evolution relation with the number of predictions.}
		\label{f:seq2seq_2}
	\end{figure}

	\subsection{Conclusions}

	\begin{itemize}
		\item \emph{As expected}, models that try to predict more steps forward lead to lower accuracies.
		\item \emph{As expected}, this sequence-to-sequence model has a lower error than the previous multi step approach. The previous model, with a window size of 6 and 24 unknown inputs, has a MSE of 1.06895 while this model has a MSE of 0.368767 when predicting 24 steps.
	\end{itemize}

\end{document}


\section{Introduction}

This paper proposes an out-of-sample (\oos) test statistic that is
asymptotically normal and correctly centered even when the models
studied are nested. The test is based on one proposed by
\citet{ClW:06,ClW:07}, but we propose estimating the benchmark model
with a recursive window and the alternative model with a fixed length
rolling window. The rolling window ensures asymptotic normality, as in
\citet{GiW:06}, and the recursive window allows the null hypothesis to
be a statement about the specification of the Data Generating Process,
which is the focus of the vast majority of the \oos\ testing
literature.%
\footnote{In particular, this is the focus of \citet{Wes:96},
  \citet{ClM:01}, \citet{Mcc:07}, \citet{ClW:06}, \citet{ClW:07}, and
  many others but not \citet{GiW:06}. See \citet{Wes:06} and
  \citet{ClM:13} for a thorough overview of this literature.} %
This combination of estimation windows also gives our test statistic
high power against alternatives that cause the benchmark model
to be unstable --- structural breaks, time-varying coefficients, or
forms of nonlinearity, for example --- which is a common motivation
for using these tests.

\oos\ tests are common in International Macroeconomics,
Macroeconomics, and Finance (see, for example, \citealt{MeR:83};
\citealt{StW:03}; and \citealt{GoW:08}) and there is a substantial
literature developing the theoretical properties of these statistics,
beginning primarily with \citet{DiM:95} and
\citet{Wes:96}.
In a pair of papers,
\citet{ClW:06,ClW:07} develop an \oos\ test of the null hypothesis
that a small benchmark model is correctly specified.  Their test
compares the forecasting performance of a pair of nested models, and
the null hypothesis is that the innovations in the smaller model form
a Martingale Difference Sequence (\mds).  This test procedure is popular, and
one assumes that this is due in part to the statistic's convenience,
the statistic is approximately normal after adjusting for the
estimation error of the larger model.  Normality comes from a
fixed-length rolling window, as in \citet{GiW:06}, and the adjustment
centers the statistic to have mean-zero under the null.  This statistic is especially
convenient because other \oos\ tests for similar hypotheses
(\citealt{CCS:01}; \citealt{ClM:01,ClM:05}; \citealt{CoS:02,CoS:04};
and \citealt{Mcc:07}; among others) have a nonstandard limit
distribution and place restrictions on the models under consideration,
while other asymptotically normal statistics test a different null
hypothesis \citep{GiW:06} or place assumptions on the models and \dgp\
that are often violated in empirical work (\citealt{DiM:95};
\citealt{Wes:96}; \citealt{WeM:98};
\citealt{Mcc:00}).%
\footnote{\citet{DiM:95} assume that the models are
  not estimated. \citet{Wes:96}, \citet{WeM:98}, and \citet{Mcc:00}
  assume that the models do not converge to the same limit,
  which rules out nesting.} %
However, Clark and West's statistic is only ``approximately normal''
in an informal sense.  Clark and West present Monte Carlo evidence of
the statistic's distribution, but only prove that the statistic is
asymptotically normal with mean zero when the benchmark model is
not estimated \citep{ClW:06}. Estimating the parameters of the smaller
model invalidates their proof.

This paper proposes a modified version of Clark and West's (2006,
2007) statistic and shows that it is
asymptotically normal even when the smaller model is estimated.  To
achieve normality, the pseudotrue benchmark model must be estimated
consistently, but the larger alternative model must continue to be
estimated inconsistently so that the test statistic is not degenerate
when the models are nested. We can meet both needs by
using different window strategies for each model: the benchmark model
is estimated using a recursive window and the alternative with a
fixed-length rolling window.
This approach has the further advantage over existing \oos\ tests for
nested models that the alternative can be essentially arbitrary as
long as high level moment conditions hold. In particular, researchers
can use model selection techniques like the \aic\ or \bic\ to
determine the number of lags to include, the particular exogenous
variables to include, etc. Moreover, although we focus on nested
models in this paper, the approach can be used with non-nested models
as well. As \citet{ClM:11b} have recently argued,
\citepos{Wes:96} results do not hold when the true \dgp\ is nested by
the benchmark and alternative models, which is allowable under the
null hypothesis of interest. (\citealp{ClM:11b}, call this scenario ``overlapping models.'')

The next section presents the intuition and theory for our new
statistic.  Section~\ref{sec:2} presents simulations that compare our
pairwise \oos\ test to \poscw\ original statistics.
Section~\ref{sec:3} demonstrates the use of our statistic by
reanalyzing \citepos{GoW:08} study of excess return predictability and
demonstrates how our results can be used in settings with many
alternative models. Section~\ref{sec:4} concludes. Our results follow
from arguments similar to \citepos{Wes:96} and have been put in a
separate appendix along with some supporting lemmas \citep{Cal:15b}.

%%% Local Variables:
%%% mode: latex
%%% TeX-master: "mixedwindow"
%%% TeX-command-extra-options: "-shell-escape"
%%% End:


\subsection{Path-connect spaces}


%\vspace{-5pt}
\section{Window Selection}
\label{sec:window}

\begin{figure}
  \centering
  \includegraphics[width=\linewidth]{figs/correlation.pdf}
  \caption{Absolute value of correlation between input tokens and target token,
  scaled by 0.02, as a function of position of a token from the target,
  for Linux source.}
  \label{fig:correlation}
\end{figure}

\noindent
To study the influence of tokens in an incomplete piece of code on the next
(target) token, we computed the correlation between the input tokens and the
target token.
Figure~\ref{fig:correlation} shows the result (scaled absolute correlation) for
codes that we train on and codes that we test on, for Linux
source code.
We compute the correlation between every token and preceding tokens, whenever
there are at least 100 tokens before the target.
The value of a key token is between 1 and $K$, and the value of a positional
token between $K+1$ and $K+W$, with $W=100$ here.
As we expected, the absolute value of correlation between input tokens and
target token gradually decreases as the distance between them increases.
However, interestingly, when an input token is 3 tokens or more away from the
target token, tokens that are even number of tokens away (tokens 4, 6, 8, ...
from the end) have a larger magnitude of correlation with the target token.
We think this is because special characters (operators) and words such as variable and
function names typically alternate. If other projects and languages
also exhibit a similar trend, we would like to see if we can improve
predictions by considering larger windows of preceding tokens, but after
removing tokens that are odd number of tokens away from the target token.


%% SECTION HEADER /////////////////////////////////////////////////////////////////////////////////////
\section{Time Integration}
\label{sec:time}

%% SECTION CONTENT ////////////////////////////////////////////////////////////////////////////////////
Assuming \(\textbf{b}\) and \(\textbf{f}\) represent order lists of the electrode nodes and free nodes of the \ac{pzt}, respectively, the electrical potential vector is rewritten:
\begin{eqnarray}
	\widehat{\boldsymbol{\phi}} = \left \{\begin{array}{cc}
		\widehat{\boldsymbol{\phi}}(\textbf{b}) &
		\widehat{\boldsymbol{\phi}}(\textbf{f})
	\end{array}\right \}^T.
	\label{eq:potential}
\end{eqnarray}

Then, Equation~(\ref{eq:piezocoupling}) is expressed as:
\begin{eqnarray}
	\left [\begin{array}{c}
		\textbf{K}_{\phi d}(\textbf{b},:) \\
		\textbf{K}_{\phi d}(\textbf{f},:)
	\end{array}\right]
	\left \{\widehat{\textbf{d}}\right\} +
	\left [\begin{array}{cc}
		\textbf{K}_{\phi \phi}(\textbf{b},\textbf{b}) & \textbf{K}_{\phi \phi}(\textbf{b},\textbf{f})\\
		\textbf{K}_{\phi \phi}(\textbf{f},\textbf{b}) & \textbf{K}_{\phi \phi}(\textbf{f},\textbf{f})
	\end{array}\right]
	\left \{\begin{array}{c}
		\widehat{\boldsymbol{\phi}}(\textbf{b}) \\
		\widehat{\boldsymbol{\phi}}(\textbf{f})
	\end{array}\right \} =
	\left \{\begin{array}{c}
		\textbf{Q} \\
		\textbf{0}
	\end{array}\right \},
	\label{eq:piezoboundary}
\end{eqnarray}
where the notation \(\textbf{K}(\textbf{r},\textbf{c})\) uses vectors \(\textbf{r}\) and \(\textbf{c}\) to extract rows and columns from the matrix \(\textbf{K}\), respectively, and \((:)\) means all rows or columns of \(\textbf{K}\).
The electrical potential of the free nodes can be extracted from Equation~(\ref{eq:piezoboundary}):
\begin{eqnarray}
	\widehat{\boldsymbol{\phi}}(\textbf{f}) = -\textbf{K}_{\phi\phi}^{-1}(\textbf{f},\textbf{f})\left[\textbf{K}_{\phi d}(\textbf{f},:) \widehat{\textbf{d}} + \textbf{K}_{\phi\phi}(\textbf{f},\textbf{b})\widehat{\boldsymbol{\phi}}(\textbf{b}) \right].
	\label{eq:freePotetial}
\end{eqnarray}

Substituting Equations~(\ref{eq:potential}) and (\ref{eq:freePotetial}) into Equation~(\ref{eq:motion}), the equation of motion can be rearranged into the form:
\begin{eqnarray}
	\textbf{M}_{dd} \widehat{\ddot{\textbf{d}}} + \textbf{C}_{dd} \widehat{\dot{\textbf{d}}} + (\textbf{K}_{dd}-\textbf{K}_{s}) \widehat{\textbf{d}}  = \textbf{F} + \textbf{K}_{a} \widehat{\boldsymbol{\phi}}(b) - \textbf{G}^T \boldsymbol{\lambda},
	\label{eq:motionD}
\end{eqnarray}
where  \(\textbf{K}_a=\textbf{K}_{d\phi}(:,f)\textbf{K}_{\phi \phi}^{-1}(f,f)\textbf{K}_{\phi \phi}(\textbf{f},\textbf{b})-\textbf{K}_{d\phi}(:,\textbf{b})\), \(\textbf{K}_s=\textbf{K}_{d \phi}(:,\textbf{f})\textbf{K}_{\phi \phi}^{-1}(\textbf{f},\textbf{f})\textbf{K}_{\phi d}(\textbf{f},:)\).
The unknown displacement vector \(\widehat{\textbf{d}}_t\) is found using a central difference algorithm \cite{kudela20093d}.
Thus, Equation~(\ref{eq:motionD}) is rewritten as:
\begin{equation}
	\begin{array}{c}
		\left(\frac{1}{\Delta t^2}\textbf{M}_{dd}+\frac{1}{2\Delta t}\textbf{C}_{dd} \right)\widehat{\textbf{d}}_{t+\Delta t}=
		\textbf{F}_t+\textbf{K}_a\widehat{\boldsymbol{\phi}}_t(b)-\left( \textbf{K}_{dd}-\textbf{K}_s\right)\widehat{\textbf{d}}_t+\\
		+\frac{2}{\Delta t^2}\textbf{M}_{dd}\widehat{\textbf{d}}_t-\left(\frac{1}{\Delta t^2}\textbf{M}_{dd}-\frac{1}{2\Delta t}\textbf{C}_{dd}\right)\widehat{\textbf{d}}_{t-\Delta t}-\textbf{G}^T\boldsymbol{\lambda}_t,
	\end{array}
	\label{eq:CDE}
\end{equation}
where \(\Delta t\) is the time increment.

Imposing the constrain Equation~(\ref{eq:cond_disp}), the vector of Lagrange multipliers \(\boldsymbol{\lambda}_t\) can be extracted from Equation~(\ref{eq:CDE}):
\begin{eqnarray}
	\boldsymbol{\lambda}_t = {\left(\textbf{G}\textbf{L}_+^{-1}\textbf{G}^T \right)}^{-1}\textbf{G}\textbf{L}_+^{-1} \Bigg[ \textbf{F}_t+\textbf{K}_a\widehat{\boldsymbol{\phi}}_t(b)+\left.\left(\frac{2}{\Delta t^2}\textbf{M}_{dd}-\textbf{K}_{dd}+\textbf{K}_s\right)\widehat{\textbf{d}}_t -\textbf{L}_-\widehat{\textbf{d}}_{t-\Delta t} \right],
	\label{eq:lambda}
\end{eqnarray}
where \(\textbf{L}_{\pm}=\frac{1}{\Delta t^2}\textbf{M}_{dd}\pm\frac{1}{2\Delta t}\textbf{C}_{dd}\).